diff --git a/Apps/2DCavity/CMakeLists.txt b/Apps/2DCavity/CMakeLists.txt new file mode 100644 index 0000000..e35ee9c --- /dev/null +++ b/Apps/2DCavity/CMakeLists.txt @@ -0,0 +1,29 @@ + +cmake_minimum_required(VERSION 3.18) +set(AppName Cavity2D) +set(AppSrc lbm2d_cavity.cpp) +set(LibSrc evolution.cpp scheme.cpp scheme_wrapper.cpp configuration.cpp model.cpp model_wrapper.cpp block.cpp flowfield.cpp flowfield_wrapper.cpp boundary.cpp boundary_wrapper.cpp) +set(LibHeadList type.h flowfield_host_device.h boundary_host_device.h model_host_device.h) +set(SpaceDim 2) +if (NOT OPTIMISE) + # Set the path for LibSrc + set(LibSrcPath "") + foreach(Src IN LISTS LibSrc) + list(APPEND LibSrcPath ${LibDir}/${Src}) + endforeach(Src IN LISTS LibSrc) + SeqDevTarget("${SpaceDim}" 0) + MpiDevTarget("${SpaceDim}" 0) +else() + #set the files needed to be translated by ops.py from the app side + set(AppSrcGenList lbm2d_cavity.cpp) + set(AppKernelGenList cavity2d_kernel.inc) + set(AppHeadList "") + CreateTempDir() + set(TMP_SOURCE_DIR ${CMAKE_CURRENT_BINARY_DIR}/tmp) + set(HeadList ${LibHeadList} ${AppHeadList}) + WriteJsonConfig(${TMP_SOURCE_DIR} ${AppName} "${LibSrc}" "${AppSrcGenList}" "${AppKernelGenList}" "${HeadList}" ${SpaceDim}) + TranslateSourceCodes(${LibDir} "${LibSrcGenList}" "${AppSrcGenList}" ${TMP_SOURCE_DIR}) + SeqTarget("${SpaceDim}") + MpiTarget("${SpaceDim}") + CudaTarget("${SpaceDim}") +endif () diff --git a/Apps/2DCavity/Cavity.json b/Apps/2DCavity/Cavity.json new file mode 100644 index 0000000..2344f06 --- /dev/null +++ b/Apps/2DCavity/Cavity.json @@ -0,0 +1,95 @@ +{ + + "CaseName": "Cavity", + "SpaceDim": 3, + "CompoNames": ["Fluid"], + "CompoIds": [0], + "LatticeName":["d3q19"], + "MacroVarNames":["rho","u","v","w"], + "MacroVarIds":[0,1,2,3], + "MacroCompoIds":[0,0,0,0], + "MacroVarTypes":["Variable_Rho","Variable_U","Variable_V","Variable_W"], + "CollisionType":["Collision_BGKIsothermal2nd"], + "CollisionCompoIds":[0], + "InitialType":["Initial_BGKFeq2nd"], + "InitialCompoIds":[0], + "BodyForceType":["BodyForce_None"], + "BodyForceCompoId":[0], + "SchemeType":"Scheme_StreamCollision", + + "BoundaryCondition0":{ + "BlockIndex": 0, + "ComponentId": 0, + "GivenVars":[0.01,0,0], + "BoundarySurface":"Top", + "BoundaryScheme": "EQMDiffuseREfl", + "BoundaryType": "Wall", + "MacroVarTypesatBoundary": ["Variable_U","Variable_V","Variable_W"] + + }, + + "BoundaryCondition1":{ + "BlockIndex": 0, + "ComponentId": 0, + "GivenVars":[0,0,0], + "BoundarySurface":"Bottom", + "BoundaryScheme": "EQMDiffuseREfl", + "BoundaryType": "Wall", + "MacroVarTypesatBoundary": ["Variable_U","Variable_V","Variable_W"] + + }, + + "BoundaryCondition2":{ + "BlockIndex": 0, + "ComponentId": 0, + "GivenVars":[0.0,0,0], + "BoundarySurface":"Left", + "BoundaryScheme": "EQMDiffuseREfl", + "BoundaryType": "Wall", + "MacroVarTypesatBoundary": ["Variable_U","Variable_V","Variable_W"] + + }, + + "BoundaryCondition3":{ + "BlockIndex": 0, + "ComponentId": 0, + "GivenVars":[0,0,0], + "BoundarySurface":"Right", + "BoundaryScheme": "EQMDiffuseREfl", + "BoundaryType": "Wall", + "MacroVarTypesatBoundary": ["Variable_U","Variable_V","Variable_W"] + + }, + + "BoundaryCondition4":{ + "BlockIndex": 0, + "ComponentId": 0, + "GivenVars":[0,0,0], + "BoundarySurface":"Front", + "BoundaryScheme": "EQMDiffuseREfl", + "BoundaryType": "Wall", + "MacroVarTypesatBoundary": ["Variable_U","Variable_V","Variable_W"] + + }, + + "BoundaryCondition5":{ + "BlockIndex": 0, + "ComponentId": 0, + "GivenVars":[0,0,0], + "BoundarySurface":"Back", + "BoundaryScheme": "EQMDiffuseREfl", + "BoundaryType": "Wall", + "MacroVarTypesatBoundary": ["Variable_U","Variable_V","Variable_W"] + + }, + + "BlockNum": 1, + "BlockSize":[33,33,33], + "MeshSize":0.03125, + "StartPos":[0,0,0], + "TauRef": [0.01], + "Transient": false, + "TimeSteps": 3, + "ConvergenceCriteria":1e-8, + "CheckPeriod":1000 +} diff --git a/Apps/2DCavity/Makefile b/Apps/2DCavity/Makefile new file mode 100644 index 0000000..9539922 --- /dev/null +++ b/Apps/2DCavity/Makefile @@ -0,0 +1,316 @@ +#OPS_INSTALL_PATH := $(OPS_INSTALL_PATH)/c +OPS_INC = -I$(OPS_INSTALL_PATH)/include +OPS_LIB = -L$(OPS_INSTALL_PATH)/lib +CUDA_LIB = -L$(CUDA_INSTALL_PATH)/lib +CUDA_INC = -I$(CUDA_INSTALL_PATH)/include + +OPENCL_INC = -I$(OPENCL_INSTALL_PATH)/include +OPENCL_LIB = -L$(OPENCL_INSTALL_PATH)/lib64 -lOpenCL + +ifdef MPLB_DIR + MPLB_INC = -I$(MPLB_DIR) + MPLB_LIB = -L$(MPLB_DIR) +endif + + +ifdef HDF5_INSTALL_PATH + HDF5_INC := -I$(HDF5_INSTALL_PATH)/include + HDF5_LIB := -L$(HDF5_INSTALL_PATH)/lib +endif +ifdef CHECKPOINTING + HDF5_INC := $(HDF5_INC) -DCHECKPOINTING +endif + +ifeq ($(OPS_COMPILER),clang) + CPP = clang++ +ifdef DEBUG + CPPFLAGS = -O0 -g -DUNIX -Wall -std=c++11 #-Wall +else + CPPFLAGS = -O3 -fPIC -DUNIX -Wall -std=c++11 #-Wall +endif + OMPFLAGS = ""#-fopenmp + MPICPP = mpic++ + MPIFLAGS = $(CPPFLAGS) +else +ifeq ($(OPS_COMPILER),gnu) + CPP = g++ +ifdef DEBUG + CPPFLAGS = -ggdb -DUNIX -Wall +else + CPPFLAGS = -O3 -std=c++11 -fPIC -DUNIX +endif + OMPFLAGS = -fopenmp + MPICPP = mpicxx + MPIFLAGS = $(CPPFLAGS) +else +ifeq ($(OPS_COMPILER),intel) + CPP = icpc +ifdef DEBUG + CCFLAGS = -O0 -g -no-prec-div -openmp -fp-model strict -fp-model source -prec-div -prec-sqrt -DMPICH_IGNORE_CXX_SEEK #-DOPS_DEBUG +else + #CCFLAGS = -O3 -ipo -no-prec-div -fp-model strict -fp-model source -prec-div -prec-sqrt -vec-report2 -xSSE4.2 -parallel #-DCOMM_PERF #-DDEBUG + CCFLAGS = -O3 -ipo -no-prec-div -restrict -fno-alias -fp-model strict -fp-model source -prec-div -prec-sqrt -DMPICH_IGNORE_CXX_SEEK#-vec-report +endif + CPPFLAGS = $(CCFLAGS) + OMPFLAGS = -openmp -openmp-report2 + MPICPP = $(MPI_INSTALL_PATH)/bin/mpicxx + MPIFLAGS = $(CCFLAGS) +else +ifeq ($(OPS_COMPILER),mic) + CPP = icpc +ifdef DEBUG + CCFLAGS = -O0 -g -mmic -DMPICH_IGNORE_CXX_SEEK#-vec-report +else + #CCFLAGS = -O3 -ipo -no-prec-div -restrict -fno-alias -fp-model strict -fp-model source -prec-div -prec-sqrt -DMPICH_IGNORE_CXX_SEEK#-vec-report + #CCFLAGS = -O3 -ipo -no-prec-div -restrict -fno-alias -mmic -vec-report -finline -inline-forceinline -DSIMD_VEC=32 -DMPICH_IGNORE_CXX_SEEK#-vec-report + CCFLAGS = -O3 -ipo -no-prec-div -restrict -fno-alias -mmic -vec-report -DSIMD_VEC=32 -DMPICH_IGNORE_CXX_SEEK -finline -vec-report +endif + CPPFLAGS = $(CCFLAGS) + OMPFLAGS = -openmp -openmp-report2 + MPICPP = $(MPI_INSTALL_PATH)/bin/mpicxx + MPIFLAGS = $(CCFLAGS) + OPENCL_LIB = -L$(OPENCL_INSTALL_PATH)/lib64 -lintelocl -lOpenCL #for offload opencl execution on the phi +else +ifeq ($(OPS_COMPILER),pgi) + CPP = pgc++ + CC = pgcc + CUDA_ALIGN_FLAG := -D__x86_64 -D__align__\(n\)=__attribute__\(\(aligned\(n\)\)\) -D__location__\(a\)=__annotate__\(a\) -DCUDARTAPI= +ifdef DEBUG + CCFLAGS = -O2 -Minline -Kieee -Minform=inform -Minfo=all +else + CCFLAGS = -O3 -fast -Kieee -Minline -Minform=severe -Minfo=all -std=c++11 +endif + CPPFLAGS = $(CCFLAGS) + OMPFLAGS = -mp + MPICPP = $(MPI_INSTALL_PATH)/bin/mpicxx + MPICC = $(MPI_INSTALL_PATH)/bin/mpicc + MPIFLAGS = $(CPPFLAGS) +else +ifeq ($(OPS_COMPILER),xl) + CPP = xlc++ +ifdef DEBUG + CPPFLAGS = -O0 -g -qfullpath -Q -qlistopt -qattr=full -qlist -qreport -qxref=full -qsource -qsuppress=1506-224:1500-036 -qsrcmsg -qfloat=nomaf +else + CPPFLAGS = -O5 -q64 -qnoeh -qipa=partition=large -qfullpath -Q -qlistopt -qattr=full -qlist -qreport -qxref=full -qsource -qsuppress=1506-224:1500-036 -qsrcmsg -qfloat=nomaf +endif + OMPFLAGS = -fopenmp + MPICPP = $(MPI_INSTALL_PATH)/bin/mpicxx + MPIFLAGS = $(CPPFLAGS) +else +ifeq ($(OPS_COMPILER),cray) + CPP = CC + CUDA_ALIGN_FLAG := -D__x86_64 -D__align__\(n\)=__attribute__\(\(aligned\(n\)\)\) -D__location__\(a\)=__annotate__\(a\) -DCUDARTAPI= +ifdef DEBUG + CCFLAGS = -O2 +else + CCFLAGS = -O3 +endif + CPPFLAGS = $(CCFLAGS) + OMPFLAGS = + MPICPP = CC + MPICC = cc + MPIFLAGS = $(CPPFLAGS) +else +print: + @echo "unrecognised value for OPS_COMPILER" +endif +endif +endif +endif +endif +endif +endif + +NVCC := $(CUDA_INSTALL_PATH)/bin/nvcc +# flags for nvcc +# set NV_ARCH to select the correct one +ifndef NV_ARCH + MESSAGE=select an NVIDA device to compile in CUDA, e.g. make NV_ARCH=KEPLER + NV_ARCH=Kepler +endif +ifeq ($(NV_ARCH),Fermi) + CODE_GEN_CUDA=-gencode arch=compute_20,code=sm_21 +else +ifeq ($(NV_ARCH),Kepler) + CODE_GEN_CUDA=-gencode arch=compute_35,code=sm_35 +endif +endif + + +ifdef DEBUG + NVCC_FLAGS = -O0 -g -G +else + NVCC_FLAGS = -O3 -std=c++11 +endif + + +# +# master to make all versions +# + +TARGETS = lbm2d_dev_seq lbm2d_seq lbm2d_dev_mpi lbm2d_mpi lbm2d_openmp lbm2d_mpi_openmp lbm2d_cuda lbm2d_mpi_cuda lbm2d_opencl lbm2d_mpi_opencl + +ifeq ($(OPS_COMPILER),pgi) +#TARGETS += lbm2d_openacc lbm2d_mpi_openacc +else +#TARGETS += openacc_error +endif + +all: clean $(TARGETS) + +env: + @echo "MPLBDIR = $(MPLB_DIR)" + @echo "HDF5_INCLUDE_PATH = $(HDF5_INCLUDE_PATH)" + @echo "HDF5_LIB_PATH = $(HDF5_LIB_PATH)" + @echo "OPS_INSTALL_PATH = $(OPS_INSTALL_PATH)" + @echo "Compiler = $(OPS_COMPILER)" + +lbm3d_dev_seq : + $(MPICPP) $(CPPFLAGS) $(MAINCPP) ${MPLB_DIR}/evolution3d.cpp ${MPLB_DIR}/scheme.cpp ${MPLB_DIR}/scheme_wrapper.cpp ${MPLB_DIR}/configuration.cpp ${MPLB_DIR}/model.cpp ${MPLB_DIR}/model_wrapper.cpp ${MPLB_DIR}/block.cpp ${MPLB_DIR}/flowfield.cpp ${MPLB_DIR}/flowfield_wrapper.cpp ${MPLB_DIR}/boundary.cpp ${MPLB_DIR}/boundary_wrapper.cpp $(OPS_INC) $(HDF5_INC) $(MPLB_INC) $(OPS_LIB) $(HDF5_LIB) -DOPS_3D -DCPU -D$(LEVEL) -lops_hdf5_seq -lops_seq -lhdf5_hl -lhdf5 -lz -o lbm3d_dev_seq + +lbm3d_dev_seq_lib : + $(CPP) $(CPPFLAGS) $(MAINCPP) $(OPS_INC) $(HDF5_INC) $(MPLB_INC) $(OPS_LIB) $(MPLB_LIB) -DOPS_3D -DCPU -D$(LEVEL) -lmplb3d_seq -lops_seq -lops_hdf5_seq $(HDF5_LIB) -o lbm3d_dev_seq + +lbm3d_dev_mpi : + $(MPICPP) $(CPPFLAGS) $(MAINCPP) ${MPLB_DIR}/evolution3d.cpp ${MPLB_DIR}/scheme.cpp ${MPLB_DIR}/scheme_wrapper.cpp ${MPLB_DIR}/configuration.cpp ${MPLB_DIR}/model.cpp ${MPLB_DIR}/model_wrapper.cpp ${MPLB_DIR}/block.cpp ${MPLB_DIR}/flowfield.cpp ${MPLB_DIR}/flowfield_wrapper.cpp ${MPLB_DIR}/boundary.cpp ${MPLB_DIR}/boundary_wrapper.cpp $(OPS_INC) $(HDF5_INC) $(MPLB_INC) $(OPS_LIB) $(MPLB_LIB) $(HDF5_LIB) -DOPS_3D -DOPS_MPI -DCPU -D$(LEVEL) -lops_hdf5_mpi -lops_mpi -lhdf5_hl -lhdf5 -lz -o lbm3d_dev_mpi + +lbm3d_dev_mpi_lib : + $(MPICPP) $(MPIFLAGS) $(MAINCPP) $(OPS_INC) $(HDF5_INC) $(MPLB_INC) $(OPS_LIB) $(MPLB_LIB) -DOPS_MPI -DOPS_3D -DCPU -D$(LEVEL) -lmplb3d_mpi -lops_mpi -lops_hdf5_mpi $(HDF5_LIB) -o lbm3d_dev_mpi + +lbm2d_dev_seq : + $(CPP) $(CPPFLAGS) $(MAINCPP) $(shell find $(MPLB_DIR) -type f \( -iname "*.cpp" ! -iname "setup*" ! -iname "evolution3d.cpp" \)) $(OPS_INC) $(HDF5_INC) $(MPLB_INC) $(OPS_LIB) $(MPLB_LIB) -DOPS_2D -DCPU -D$(LEVEL) -lops_seq -lops_hdf5_seq $(HDF5_LIB) -o lbm2d_dev_seq + +lbm2d_dev_seq_lib : + $(CPP) $(CPPFLAGS) $(MAINCPP) $(OPS_INC) $(HDF5_INC) $(MPLB_INC) $(OPS_LIB) $(MPLB_LIB) -DOPS_2D -DCPU -D$(LEVEL) -lmplb2d_seq -lops_seq -lops_hdf5_seq $(HDF5_LIB) -o lbm2d_dev_seq + +lbm2d_dev_mpi : + $(MPICPP) $(MPIFLAGS) $(MAINCPP) $(shell find $(MPLB_DIR) -type f \( -iname "*.cpp" ! -iname "setup*" ! -iname "evolution3d.cpp" \)) $(OPS_INC) $(HDF5_INC) $(MPLB_INC) $(OPS_LIB) $(MPLB_LIB) -DOPS_MPI -DOPS_2D -DCPU -D$(LEVEL) -lops_mpi -lops_hdf5_mpi $(HDF5_LIB) -o lbm2d_dev_mpi + +lbm2d_dev_mpi_lib : + $(MPICPP) $(MPIFLAGS) $(MAINCPP) $(OPS_INC) $(HDF5_INC) $(MPLB_INC) $(OPS_LIB) $(MPLB_LIB) -DOPS_MPI -DOPS_2D -DCPU -D$(LEVEL) -lmplb2d_mpi -lops_mpi -lops_hdf5_mpi $(HDF5_LIB) -o lbm2d_dev_mpi + +# Old pre-processor 2D +setupdomain: Makefile setup_comput_domain.cpp scheme.cpp model.cpp boundary.cpp Case_Setup.cpp $(OPS_INSTALL_PATH)/lib/libops_seq.a + $(CPP) $(CPPFLAGS) $(OPS_INC) $(HDF5_INC) $(OPS_LIB) -DOPS_2D type.cpp boundary.cpp setup_comput_domain.cpp scheme.cpp model.cpp -lops_seq -lops_hdf5_seq $(HDF5_LIB) -o setup_comput_domain + +# Old pre-processor 3D +setupdomain3D: Makefile setup_comput_domain.cpp scheme.cpp model.cpp boundary.cpp $(OPS_INSTALL_PATH)/lib/libops_seq.a + $(CPP) $(CPPFLAGS) $(OPS_INC) $(HDF5_INC) $(OPS_LIB) -DOPS_3D type.cpp boundary.cpp setup_comput_domain.cpp scheme.cpp model.cpp -lops_seq -lops_hdf5_seq $(HDF5_LIB) -o setup_comput_domain + +# +# mpi version +# + +SEQ_KERNELS = lbm2d_seq_kernels.cpp + +lbm2d_mpi: Makefile type_ops.cpp boundary_ops.cpp scheme_ops.cpp flowfield_ops.cpp model_ops.cpp evolution_ops.cpp lbm2d_ops.cpp $(OPS_INSTALL_PATH)/lib/libops_mpi.a + $(MPICPP) $(MPIFLAGS) -DOPS_MPI $(OPS_INC) $(HDF5_INC) $(OPS_LIB) type_ops.cpp boundary_ops.cpp scheme_ops.cpp flowfield_ops.cpp model_ops.cpp evolution_ops.cpp lbm2d_ops.cpp -I. ./MPI/$(SEQ_KERNELS) -lops_mpi $(HDF5_LIB) -o lbm2d_mpi + +# +# OpenMP version +# +OMP_KERNELS = lbm2d_omp_kernels.cpp + +lbm2d_mpi_openmp: Makefile type_ops.cpp boundary_ops.cpp scheme_ops.cpp flowfield_ops.cpp model_ops.cpp lbm2d_ops.cpp $(OPS_INSTALL_PATH)/lib/libops_mpi.a + $(MPICPP) $(OMPFLAGS) $(MPIFLAGS) -DOPS_MPI $(OPS_INC) $(HDF5_INC) $(OPS_LIB) type_ops.cpp boundary_ops.cpp scheme_ops.cpp flowfield_ops.cpp model_ops.cpp lbm2d_ops.cpp -I. ./MPI_OpenMP/$(OMP_KERNELS) -lops_mpi $(HDF5_LIB) -o lbm2d_mpi_openmp + + +lbm2d_openmp: Makefile type_ops.cpp boundary_ops.cpp scheme_ops.cpp flowfield_ops.cpp model_ops.cpp evolution_ops.cpp lbm2d_ops.cpp $(OPS_INSTALL_PATH)/lib/libops_seq.a + $(MPICPP) $(OMPFLAGS) $(HDF5) $(MPIFLAGS) $(OPS_INC) $(HDF5_INC) $(OPS_LIB) type_ops.cpp boundary_ops.cpp scheme_ops.cpp flowfield_ops.cpp model_ops.cpp evolution_ops.cpp lbm2d_ops.cpp -I. ./MPI_OpenMP/$(OMP_KERNELS) -lops_seq $(HDF5_LIB) -o lbm2d_openmp + +# +# Sequential version +# +lbm3d_seq : + $(CPP) $(CPPFLAGS) ./MPI_OpenMP/lbm3d_cavity_cpu_kernels.cpp boundary.cpp evolution3d_ops.cpp scheme.cpp configuration.cpp model.cpp flowfield_ops.cpp lbm3d_cavity_ops.cpp type.cpp $(OPS_INC) $(HDF5_INC) $(MPLB_INC) $(OPS_LIB) $(MPLB_LIB) -DOPS_3D -DCPU -D$(LEVEL) -lops_seq -lops_hdf5_seq $(HDF5_LIB) -o lbm3d_seq + +lbm2d_seq: Makefile type_ops.cpp boundary_ops.cpp scheme_ops.cpp flowfield_ops.cpp model_ops.cpp evolution_ops.cpp lbm2d_ops.cpp $(OPS_INSTALL_PATH)/lib/libops_seq.a + $(MPICPP) $(MPIFLAGS) $(OPS_INC) $(HDF5_INC) $(OPS_LIB) -I. ./MPI/$(SEQ_KERNELS) type_ops.cpp boundary_ops.cpp scheme_ops.cpp flowfield_ops.cpp model_ops.cpp evolution_ops.cpp lbm2d_ops.cpp -lops_seq $(HDF5_LIB) -o lbm2d_seq + +# +# CUDA version +# $(OMPFLAGS) + +lbm2d_cuda: Makefile ./CUDA/lbm2d_kernels_cu.o type_ops.cpp boundary_ops.cpp scheme_ops.cpp flowfield_ops.cpp model_ops.cpp evolution_ops.cpp lbm2d_ops.cpp $(OPS_INSTALL_PATH)/lib/libops_cuda.a + $(MPICPP) $(OMPFLAGS) $(MPIFLAGS) $(CUDA_INC) $(HDF5_INC) $(OPS_INC) $(OPS_LIB) $(CUDA_LIB) type_ops.cpp boundary_ops.cpp scheme_ops.cpp flowfield_ops.cpp model_ops.cpp evolution_ops.cpp lbm2d_ops.cpp ./CUDA/lbm2d_kernels_cu.o -lcudart -lops_cuda $(HDF5_LIB) -o lbm2d_cuda + +lbm2d_mpi_cuda: Makefile ./CUDA/lbm2d_kernels_mpi_cu.o type_ops.cpp boundary_ops.cpp scheme_ops.cpp flowfield_ops.cpp model_ops.cpp lbm2d_ops.cpp $(OPS_INSTALL_PATH)/lib/libops_mpi_cuda.a + $(MPICPP) $(OMPFLAGS) $(CPPFLAGS) -DOPS_MPI $(CUDA_INC) $(OPS_INC) $(OPS_LIB) $(CUDA_LIB) type_ops.cpp boundary_ops.cpp scheme_ops.cpp flowfield_ops.cpp model_ops.cpp lbm2d_ops.cpp ./CUDA/lbm2d_kernels_mpi_cu.o -lcudart -lops_mpi_cuda $(HDF5_LIB) -o lbm2d_mpi_cuda + +./CUDA/lbm2d_kernels_cu.o: ./CUDA/lbm2d_kernels.cu Makefile + $(NVCC) $(VAR) $(INC) $(NVCC_FLAGS) $(CODE_GEN_CUDA) $(OPS_INC) -I. -c -o ./CUDA/lbm2d_kernels_cu.o ./CUDA/lbm2d_kernels.cu + +./CUDA/lbm2d_kernels_mpi_cu.o: ./CUDA/lbm2d_kernels.cu Makefile + $(NVCC) $(VAR) $(INC) $(NVCC_FLAGS) -DOPS_MPI $(CODE_GEN_CUDA) $(OPS_INC) -I. -I$(MPI_INSTALL_PATH)/include \ + -DMPICH_IGNORE_CXX_SEEK -c -o ./CUDA/lbm2d_kernels_mpi_cu.o ./CUDA/lbm2d_kernels.cu + +openacc_obj_list = $(shell find OpenACC/ -name "*.cpp" | sed s/\\.cpp/\\.o/g | tr "\n" " ") +openacc_c_obj_list = $(shell find OpenACC/ -name "*_c.c" | sed s/\\.c/\\.o/g | tr "\n" " ") + +openacc_mpi_obj_list = $(shell find OpenACC/ -name "*.cpp" | sed s/\\.cpp/\\_mpi.o/g | tr "\n" " ") +openacc_mpi_c_obj_list = $(shell find OpenACC/ -name "*_c.c" | sed s/\\.c/\\_mpi.o/g | tr "\n" " ") + + +./OpenACC/lbm2d_kernels.o: ./OpenACC/lbm2d_kernels.cpp Makefile + $(CPP) -acc -ta=tesla:cc35 -ta=tesla:maxregcount:64 $(CPPFLAGS) $(CUDA_ALIGN_FLAG) $(OPS_INC) $(CUDA_INC) \ + -I. OpenACC/lbm2d_kernels.cpp -c -o OpenACC/lbm2d_kernels.o + + +./OpenACC/lbm2d_kernels_mpi.o: ./OpenACC/lbm2d_kernels.cpp Makefile + $(MPICPP) -acc -ta=tesla:cc35 -ta=tesla:maxregcount:64 -DOPS_MPI $(MPIFLAGS) $(CUDA_ALIGN_FLAG) $(OPS_INC) $(CUDA_INC) \ + -I. OpenACC/lbm2d_kernels.cpp -c -o OpenACC/lbm2d_kernels_mpi.o + +./OpenACC/%_mpi.o: ./OpenACC/%.cpp ./OpenACC/%_c.c Makefile + @echo "Compiling kernel" $* + $(MPICC) -acc -ta=tesla:cc35 -ta=tesla:maxregcount:64 $(MPIFLAGS) -DOPS_MPI $(CUDA_ALIGN_FLAG) $(OPS_INC) $(CUDA_INC) \ + -I. OpenACC/$*_c.c -c -o OpenACC/$*_c_mpi.o + $(MPICPP) -acc -ta=tesla:cc35 -ta=tesla:maxregcount:64 $(MPIFLAGS) -DOPS_MPI $(CUDA_ALIGN_FLAG) $(OPS_INC) $(CUDA_INC) \ + -I. OpenACC/$*.cpp -c -o OpenACC/$*_mpi.o + +./OpenACC/%.o: ./OpenACC/%.cpp ./OpenACC/%_c.c Makefile + @echo "Compiling kernel" $* + $(CC) -acc -ta=tesla:cc35 -ta=tesla:maxregcount:64 $(CCFLAGS) $(CUDA_ALIGN_FLAG) $(OPS_INC) $(CUDA_INC) \ + -I. OpenACC/$*_c.c -c -o OpenACC/$*_c.o + $(CPP) -acc -ta=tesla:cc35 -ta=tesla:maxregcount:64 $(CPPFLAGS) $(CUDA_ALIGN_FLAG) $(OPS_INC) $(CUDA_INC) \ + -I. OpenACC/$*.cpp -c -o OpenACC/$*.o + +lbm2d_openacc: $(openacc_obj_list) ./OpenACC/lbm2d_kernels.o type_ops.cpp boundary_ops.cpp scheme_ops.cpp flowfield_ops.cpp model_ops.cpp lbm2d_ops.cpp Makefile $(OPS_INSTALL_PATH)/lib/libops_cuda.a + $(MPICPP) -acc -ta=tesla:cc35 $(MPIFLAGS) $(OPS_INC) $(OPS_LIB) -DOPS_MPI $(CUDA_INC) $(CUDA_LIB) \ + type_ops.cpp boundary_ops.cpp scheme_ops.cpp flowfield_ops.cpp model_ops.cpp lbm2d_ops.cpp -I. $(openacc_obj_list) $(openacc_c_obj_list) -lcudart -lops_cuda $(HDF5_LIB -o lbm2d_openacc + + +lbm2d_mpi_openacc: $(openacc_mpi_obj_list) ./OpenACC/lbm2d_kernels_mpi.o type_ops.cpp boundary_ops.cpp scheme_ops.cpp flowfield_ops.cpp model_ops.cpp lbm2d_ops.cpp Makefile $(OPS_INSTALL_PATH)/lib/libops_mpi_cuda.a + $(MPICPP) -acc -ta=tesla:cc35 $(MPIFLAGS) $(OPS_INC) $(OPS_LIB) $(CUDA_INC) $(CUDA_LIB) -DOPS_MPI \ + type_ops.cpp boundary_ops.cpp scheme_ops.cpp flowfield_ops.cpp model_ops.cpp lbm2d_ops.cpp -I. $(openacc_mpi_obj_list) $(openacc_mpi_c_obj_list) -lcudart -lops_mpi_cuda $(HDF5_LIB -o lbm2d_mpi_openacc + + +# +#OpenCL version +# +lbm2d_opencl: ./OpenCL/lbm2d_opencl_kernels.o lbm2d_ops.cpp lbm2d_kernel.h lbm2d_print_kernel.h lbm2d_copy_kernel.h Makefile $(OPS_INSTALL_PATH)/lib/libops_opencl.a + $(CPP) $(MPIFLAGS) $(OPS_INC) $(OPS_LIB) $(OPENCL_LIB) lbm2d_ops.cpp ./OpenCL/lbm2d_opencl_kernels.o -lops_opencl -o lbm2d_opencl + +lbm2d_mpi_opencl: ./OpenCL/lbm2d_mpi_opencl_kernels.o lbm2d_ops.cpp lbm2d_kernel.h lbm2d_print_kernel.h lbm2d_copy_kernel.h Makefile $(OPS_INSTALL_PATH)/lib/libops_mpi_opencl.a + $(MPICPP) $(MPIFLAGS) -DOPS_MPI $(OPS_INC) $(OPS_LIB) $(OPENCL_LIB) lbm2d_ops.cpp ./OpenCL/lbm2d_opencl_kernels.o -lops_mpi_opencl -o lbm2d_mpi_opencl + +./OpenCL/lbm2d_opencl_kernels.o: ./OpenCL/lbm2d_opencl_kernels.cpp Makefile + $(CPP) $(CPPFLAGS) $(VAR) $(OPENCL_INC) $(INC) $(OPS_INC) -I. \ + -c -o ./OpenCL/lbm2d_opencl_kernels.o ./OpenCL/lbm2d_opencl_kernels.cpp + +./OpenCL/lbm2d_mpi_opencl_kernels.o: ./OpenCL/lbm2d_opencl_kernels.cpp Makefile + $(MPICPP) $(CPPFLAGS) -DOPS_MPI $(VAR) $(OPENCL_INC) $(INC) $(OPS_INC) -I. \ + -c -o ./OpenCL/lbm2d_opencl_kernels.o ./OpenCL/lbm2d_opencl_kernels.cpp + +# +#error +# +openacc_error: + @echo "lbm2d_mpi_openacc cannot be built with $(OPS_COMPILER) compilers .. requires an OpenACC compiler (e.g. pgi)" + + +# +# cleanup +# + +clean: + rm -f lbm2d_dev_seq lbm2d_seq lbm2d_dev_seq lbm2d_mpi lbm2d_dev_mpi lbm2d_openmp lbm2d_mpi_openmp lbm2d_cuda lbm2d_mpi_cuda lbm2d_openacc lbm2d_mpi_openacc lbm2d_opencl lbm2d_mpi_opencl ./CUDA/*.o ./OpenACC/*.o *.o lbm2d_opencl lbm3d_dev_seq lbm3d_seq lbm3d_dev_seq lbm3d_mpi lbm3d_dev_mpi./OpenCL/*.o *.o diff --git a/Apps/2DCavity/cavity2d_kernel.inc b/Apps/2DCavity/cavity2d_kernel.inc new file mode 100644 index 0000000..bc8fb36 --- /dev/null +++ b/Apps/2DCavity/cavity2d_kernel.inc @@ -0,0 +1,42 @@ +/** + * Copyright 2019 United Kingdom Research and Innovation + * + * Authors: See AUTHORS + * + * Contact: [jianping.meng@stfc.ac.uk and/or jpmeng@gmail.com] + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice + * this list of conditions and the following disclaimer in the documentation + * and or other materials provided with the distribution. + * 3. Neither the name of the copyright holder nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * ANDANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. +*/ + +#ifndef CAVITY2D_KERNEL_INC +#define CAVITY2D_KERNEL_INC + +void KerSetInitialMacroVars(ACC& rho, ACC& u, ACC& v, + const ACC& coordinates, const int* idx) { + rho(0, 0) = 1; + u(0, 0) = 0; + v(0, 0) = 0; +} +#endif // CAVITY2D_KERNEL_INC diff --git a/Apps/2DCavity/lbm2d_cavity.cpp b/Apps/2DCavity/lbm2d_cavity.cpp new file mode 100644 index 0000000..b835c90 --- /dev/null +++ b/Apps/2DCavity/lbm2d_cavity.cpp @@ -0,0 +1,219 @@ +/** + * Copyright 2019 United Kingdom Research and Innovation + * + * Authors: See AUTHORS + * + * Contact: [jianping.meng@stfc.ac.uk and/or jpmeng@gmail.com] + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice + * this list of conditions and the following disclaimer in the documentation + * and or other materials provided with the distribution. + * 3. Neither the name of the copyright holder nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * ANDANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. +*/ + +/** @brief An example main source code of stimulating 3D lid-driven cavity flow + * @author Jianping Meng + **/ +#include +#include +#include +#include +#include "mplb.h" +#include "ops_seq_v2.h" +#include "cavity2d_kernel.inc" +//Provide macroscopic initial conditions +void SetInitialMacrosVars() { + for (auto idBlock : g_Block()) { + Block& block{idBlock.second}; + std::vector iterRng; + iterRng.assign(block.WholeRange().begin(), block.WholeRange().end()); + const int blockIdx{block.ID()}; + for (auto& idCompo : g_Components()) { + const Component& compo{idCompo.second}; + const int rhoId{compo.macroVars.at(Variable_Rho).id}; + ops_par_loop(KerSetInitialMacroVars, "KerSetInitialMacroVars", + block.Get(), SpaceDim(), iterRng.data(), + ops_arg_dat(g_MacroVars().at(rhoId).at(blockIdx), 1, + LOCALSTENCIL, "Real", OPS_RW), + ops_arg_dat(g_MacroVars().at(compo.uId).at(blockIdx), + 1, LOCALSTENCIL, "Real", OPS_RW), + ops_arg_dat(g_MacroVars().at(compo.vId).at(blockIdx), + 1, LOCALSTENCIL, "Real", OPS_RW), + ops_arg_dat(g_CoordinateXYZ()[blockIdx], SpaceDim(), + LOCALSTENCIL, "Real", OPS_READ), + ops_arg_idx()); + } + } +} +//Provide macroscopic body-force term +void UpdateMacroscopicBodyForce(const Real time) {} + +void simulate() { + + std::string caseName{"2D_lid_Driven_cavity"}; + SizeType spaceDim{2}; + DefineCase(caseName, spaceDim); + std::vector blockIds{0}; + std::vector blockNames{"Cavity"}; + std::vector blockSize{101, 101}; + Real meshSize{1. / 100}; + std::map> startPos{{0, {0.0, 0.0}}}; + DefineBlocks(blockIds, blockNames, blockSize, meshSize, startPos); + + std::vector compoNames{"Fluid"}; + std::vector compoid{0}; + std::vector lattNames{"d2q9"}; + std::vector tauRef{0.01}; + DefineComponents(compoNames, compoid, lattNames, tauRef); + + std::vector marcoVarTypes{Variable_Rho, Variable_U, + Variable_V}; + std::vector macroVarNames{"rho", "u", "v"}; + std::vector macroVarId{0, 1, 2}; + std::vector macroCompoId{0, 0, 0}; + DefineMacroVars(marcoVarTypes, macroVarNames, macroVarId, macroCompoId); + + std::vector collisionTypes{Collision_BGKIsothermal2nd}; + std::vector collisionCompoId{0}; + DefineCollision(collisionTypes, collisionCompoId); + + std::vector bodyForceTypes{BodyForce_None}; + std::vector bodyForceCompoId{0}; + DefineBodyForce(bodyForceTypes, bodyForceCompoId); + + SchemeType scheme{Scheme_StreamCollision}; + DefineScheme(scheme); + + // Setting boundary conditions + SizeType blockIndex{0}; + SizeType componentId{0}; + std::vector macroVarTypesatBoundary{ + Variable_U, + Variable_V, + }; + std::vector noSlipStationaryWall{0, 0}; + // Left noSlipStationaryWall + DefineBlockBoundary(blockIndex, componentId, BoundarySurface::Left, + BoundaryScheme::EQMDiffuseRefl, macroVarTypesatBoundary, + noSlipStationaryWall); + // Right noSlipStationaryWall + DefineBlockBoundary(blockIndex, componentId, BoundarySurface::Right, + BoundaryScheme::EQMDiffuseRefl, macroVarTypesatBoundary, + noSlipStationaryWall); + // Top noslipMovingWall + std::vector noSlipMovingWall{0.01, 0}; + DefineBlockBoundary(blockIndex, componentId, BoundarySurface::Top, + BoundaryScheme::EQMDiffuseRefl, macroVarTypesatBoundary, + noSlipMovingWall); + // bottom noSlipStationaryWall + DefineBlockBoundary(blockIndex, componentId, BoundarySurface::Bottom, + BoundaryScheme::EQMDiffuseRefl, macroVarTypesatBoundary, + noSlipStationaryWall); + + std::vector initType{Initial_BGKFeq2nd}; + std::vector initalCompoId{0}; + DefineInitialCondition(initType,initalCompoId); + Partition(); + ops_diagnostic_output(); + SetInitialMacrosVars(); + PreDefinedInitialCondition(); + SetTimeStep(meshSize / SoundSpeed()); + + const Real convergenceCriteria{1E-7}; + const SizeType checkPeriod{1000}; + Iterate(StreamCollision,convergenceCriteria, checkPeriod); +} + +void simulate(const Configuration & config, const SizeType timeStep=0) { + // DefineCase(config.caseName, config.spaceDim); + // DefineBlocks(config.blockNum, config.blockSize, config.meshSize, + // config.startPos); + // if (timeStep == 0) { + // DefineComponents(config.compoNames, config.compoIds, config.lattNames); + // DefineMacroVars(config.macroVarTypes, config.macroVarNames, + // config.macroVarIds, config.macroCompoIds); + // } else { + // // restart from a time step + // DefineComponents(config.compoNames, config.compoIds, config.lattNames, + // timeStep); + // DefineMacroVars(config.macroVarTypes, config.macroVarNames, + // config.macroVarIds, config.macroCompoIds,timeStep); + // } + + // DefineCollision(config.CollisionTypes, config.CollisionCompoIds); + // DefineBodyForce(config.bodyForceTypes, config.bodyForceCompoIds); + // DefineScheme(config.schemeType); + // DefineInitialCondition(config.initialTypes,config.initialConditionCompoId); + // for (auto& bcConfig : config.blockBoundaryConfig) { + // DefineBlockBoundary(bcConfig.blockIndex, bcConfig.componentID, + // bcConfig.boundarySurface, bcConfig.boundaryScheme, + // bcConfig.macroVarTypesatBoundary, + // bcConfig.givenVars, bcConfig.boundaryType); + // } + // Partition(); + // if (timeStep == 0) { + // SetInitialMacrosVars(); + // PreDefinedInitialCondition3D(); + // } else{ + // //Help function for restart a steady simulation + // //Mainly make the residual calculation correct at first iteration. + // RestartMacroVars4SteadySim(); + // } + + // SetTauRef(config.tauRef); + // SetTimeStep(config.meshSize / SoundSpeed()); + // if (config.transient){ + // Iterate(config.timeSteps, config.checkPeriod,timeStep); + // } else{ + // Iterate(config.convergenceCriteria, config.checkPeriod,timeStep); + // } +} + +int main(int argc, const char** argv) { + // OPS initialisation + ops_init(argc, argv, 4); + double ct0, ct1, et0, et1; + ops_timers(&ct0, &et0); + // start a simulation by hard-coding + if (argc <= 1) { + simulate(); + } + // start a new simulaton from a configuration file + if (argc>1 && argc <=2){ + std::string configFileName(argv[1]); + ReadConfiguration(configFileName); + simulate(Config()); + } + // restart from the time step specified by argv[2] + if (argc>2 && argc <=3){ + std::string configFileName(argv[1]); + ReadConfiguration(configFileName); + const SizeType timeStep{static_cast(std::stoi(argv[2]))}; + simulate(Config(),timeStep); + } + + ops_timers(&ct1, &et1); + ops_printf("\nTotal Wall time %lf\n", et1 - et0); + //Print OPS performance details to output stream + ops_timing_output(std::cout); + ops_exit(); +} \ No newline at end of file diff --git a/Apps/2DChannel/lbm2d_hilemms_solid_body.cpp b/Apps/2DChannel/lbm2d_hilemms_solid_body.cpp index 7fb43ab..7ab81a0 100644 --- a/Apps/2DChannel/lbm2d_hilemms_solid_body.cpp +++ b/Apps/2DChannel/lbm2d_hilemms_solid_body.cpp @@ -39,7 +39,7 @@ #include #include "boundary.h" #include "evolution.h" -#include "evolution3d.h" +#include "evolution.h" #include "flowfield.h" #include "hilemms.h" #include "model.h" diff --git a/Apps/3DCavity/CMakeLists.txt b/Apps/3DCavity/CMakeLists.txt new file mode 100644 index 0000000..5f134cf --- /dev/null +++ b/Apps/3DCavity/CMakeLists.txt @@ -0,0 +1,30 @@ + +cmake_minimum_required(VERSION 3.18) +set(AppName Cavity3D) +set(AppSrc lbm3d_cavity.cpp) +set(LibSrc evolution.cpp scheme.cpp scheme_wrapper.cpp configuration.cpp model.cpp model_wrapper.cpp block.cpp flowfield.cpp flowfield_wrapper.cpp boundary.cpp boundary_wrapper.cpp) +set(LibHeadList type.h flowfield_host_device.h boundary_host_device.h model_host_device.h) +set(SpaceDim 3) +if (NOT OPTIMISE) + # Set the path for LibSrc + set(LibSrcPath "") + foreach(Src IN LISTS LibSrc) + list(APPEND LibSrcPath ${LibDir}/${Src}) + endforeach(Src IN LISTS LibSrc) + SeqDevTarget("${SpaceDim}" 0) + MpiDevTarget("${SpaceDim}" 0) +else() + + #set the files needed to be translated by ops.py from the app side + set(AppSrcGenList lbm3d_cavity.cpp) + set(AppKernelGenList cavity3d_kernel.inc) + set(AppHeadList "") + CreateTempDir() + set(TMP_SOURCE_DIR ${CMAKE_CURRENT_BINARY_DIR}/tmp) + set(HeadList ${LibHeadList} ${AppHeadList}) + WriteJsonConfig(${TMP_SOURCE_DIR} ${AppName} "${LibSrc}" "${AppSrcGenList}" "${AppKernelGenList}" "${HeadList}" ${SpaceDim}) + TranslateSourceCodes(${LibDir} "${LibSrcGenList}" "${AppSrcGenList}" ${TMP_SOURCE_DIR}) + SeqTarget("${SpaceDim}") + MpiTarget("${SpaceDim}") + CudaTarget("${SpaceDim}") +endif () diff --git a/Apps/3DCavity/lbm3d_cavity.cpp b/Apps/3DCavity/lbm3d_cavity.cpp index 46acd75..da85c97 100644 --- a/Apps/3DCavity/lbm3d_cavity.cpp +++ b/Apps/3DCavity/lbm3d_cavity.cpp @@ -141,13 +141,14 @@ void simulate() { std::vector initalCompoId{0}; DefineInitialCondition(initType,initalCompoId); Partition(); + ops_diagnostic_output(); SetInitialMacrosVars(); PreDefinedInitialCondition3D(); SetTimeStep(meshSize / SoundSpeed()); const Real convergenceCriteria{1E-7}; const SizeType checkPeriod{1000}; - Iterate(StreamCollision3D,convergenceCriteria, checkPeriod); + Iterate(StreamCollision,convergenceCriteria, checkPeriod); } void simulate(const Configuration & config, const SizeType timeStep=0) { diff --git a/Apps/3DLChannel/CMakeLists.txt b/Apps/3DLChannel/CMakeLists.txt new file mode 100644 index 0000000..a2502fe --- /dev/null +++ b/Apps/3DLChannel/CMakeLists.txt @@ -0,0 +1,29 @@ + +cmake_minimum_required(VERSION 3.18) +set(AppName "LChannel3D") +set(AppSrc "lbm3d_L.cpp") +set(LibSrc evolution.cpp scheme.cpp scheme_wrapper.cpp configuration.cpp model.cpp model_wrapper.cpp block.cpp flowfield.cpp flowfield_wrapper.cpp boundary.cpp boundary_wrapper.cpp) +set(LibHeadList type.h flowfield_host_device.h boundary_host_device.h model_host_device.h) +set(SpaceDim 3) +if (NOT OPTIMISE) + # Set the path for LibSrc + set(LibSrcPath "") + foreach(Src IN LISTS LibSrc) + list(APPEND LibSrcPath ${LibDir}/${Src}) + endforeach(Src IN LISTS LibSrc) + SeqDevTarget("${SpaceDim}" 0) + MpiDevTarget("${SpaceDim}" 0) +else() + #set the files needed to be translated by ops.py from the app side + set(AppSrcGenList lbm3d_L.cpp) + set(AppKernelGenList LChannel_kernel.inc) + set(AppHeadGenList "") + CreateTempDir() + set(TMP_SOURCE_DIR ${CMAKE_CURRENT_BINARY_DIR}/tmp) + set(HeadList ${LibHeadList} ${AppHeadList}) + WriteJsonConfig(${TMP_SOURCE_DIR} ${AppName} "${LibSrc}" "${AppSrcGenList}" "${AppKernelGenList}" "${HeadList}" ${SpaceDim}) + TranslateSourceCodes(${LibDir} "${LibSrcGenList}" "${AppSrcGenList}" ${TMP_SOURCE_DIR}) + SeqTarget("${SpaceDim}") + MpiTarget("${SpaceDim}") + CudaTarget("${SpaceDim}") +endif () \ No newline at end of file diff --git a/Apps/3DLChannel/lbm3d_L.cpp b/Apps/3DLChannel/lbm3d_L.cpp index 078525e..3213991 100644 --- a/Apps/3DLChannel/lbm3d_L.cpp +++ b/Apps/3DLChannel/lbm3d_L.cpp @@ -209,7 +209,7 @@ void simulate() { const Real convergenceCriteria{1E-5}; const SizeType checkPeriod{200}; - Iterate(StreamCollision3D,convergenceCriteria, checkPeriod); + Iterate(StreamCollision,convergenceCriteria, checkPeriod); } void simulate(const Configuration & config, const SizeType timeStep=0) { diff --git a/Apps/LBM-DEM/LAMMPS-fixes/fix_lbm3d_mui.h b/Apps/LBM-DEM/LAMMPS-fixes/fix_lbm3d_mui.h new file mode 100644 index 0000000..7cf2bad --- /dev/null +++ b/Apps/LBM-DEM/LAMMPS-fixes/fix_lbm3d_mui.h @@ -0,0 +1,77 @@ +/* ---------------------------------------------------------------------- + LIGGGHTS - LAMMPS Improved for General Granular and Granular Heat + Transfer Simulations + + LIGGGHTS is part of the CFDEMproject + www.liggghts.com | www.cfdem.com + + C. Tsigginos STFC + Copyright 2019- STFC Daresbury Laboratory, Daresbury, UK + + LIGGGHTS is based on LAMMPS + LAMMPS - Large-scale Atomic/Molecular Massively Parallel Simulator + http://lammps.sandia.gov, Sandia National Laboratories + Steve Plimpton, sjplimp@sandia.gov + + This software is distributed under the GNU General Public License. + + See the README file in the top-level directory. +------------------------------------------------------------------------- */ +#ifdef FIX_CLASS + +FixStyle(lbm3d/mui,FixLbm3DMui); + +#else + +#ifndef FIX_LBM3D_MUI_H_ +#define FIX_LBM3D_MUI_H_ + +#include "fix.h" +#include "../mui/mui.h" //Define location + +namespace LAMMPS_NS { + +class FixLbm3DMui : public Fix { +public: + + FixLbm3DMui(class LAMMPS*, int, char **); + ~FixLbm3DMui(); + + virtual int setmask(); + virtual double memory_usage(); + virtual void grow_arrays(int ); + virtual void copy_arrays(int, int); + virtual void set_arrays(int); + virtual int pack_exchange(int, double *); + virtual int unpack_exchange(int, double *); + + virtual void end_of_step(); + virtual void post_force(int); + virtual void pre_force(int); + + virtual void init(); + virtual void setup(int); + +private: + void send_MUI_data(); + void recv_MUI_data(); + + + + + double maxdt(double, double); + double dtLBM; //Timestep of LBM, actually it is fixed. + double **Fd; //Drag force + double **Md; //Drag moment + + mui::uniface3d *interface; + int timeInit; //Initial time step that the fix is constructed. + bool flagSetup; + double alpha1; //Timestep ratio. + int boxExpand; +}; +} + + +#endif +#endif /* FIX_LBM3D_MUI_H_ */ diff --git a/Apps/LBM-DEM/LAMMPS-fixes/fix_lbm_mui.cpp b/Apps/LBM-DEM/LAMMPS-fixes/fix_lbm_mui.cpp new file mode 100644 index 0000000..3794c9f --- /dev/null +++ b/Apps/LBM-DEM/LAMMPS-fixes/fix_lbm_mui.cpp @@ -0,0 +1,586 @@ +/* ---------------------------------------------------------------------- + LIGGGHTS - LAMMPS Improved for General Granular and Granular Heat + Transfer Simulations + + LIGGGHTS is part of the CFDEMproject + www.liggghts.com | www.cfdem.com + + C. Tsigginos STFC + Copyright 2019- STFC Daresbury Laboratory, Daresbury, UK + + LIGGGHTS is based on LAMMPS + LAMMPS - Large-scale Atomic/Molecular Massively Parallel Simulator + http://lammps.sandia.gov, Sandia National Laboratories + Steve Plimpton, sjplimp@sandia.gov + + This software is distributed under the GNU General Public License. + + See the README file in the top-level directory. +------------------------------------------------------------------------- */ + +#include "math.h" +#include "stdlib.h" +#include "string.h" +#include "fix_lbm_mui.h" +#include "../mui/mui.h" //to be verified +#include "atom.h" +#include "memory.h" +#include "force.h" +#include "update.h" +#include "domain.h" +#include "domain.h" +#include "error.h" +#include + + + +using namespace LAMMPS_NS; +using namespace FixConst; + +FixLbmMui::FixLbmMui(LAMMPS *lmp, int narg, char **arg) : + Fix(lmp, narg, arg) { + + int periodicFlag = 0; + double dx; + //create interface + + //In the future modified accordingly. Based on keys, But now + //want to debug. + + if (domain->dimension == 2) { + interface2d = new mui::uniface2d("mpi://DEM/ifs"); + + } + else if (domain->dimension == 3) { + interface3d = new mui::uniface3d("mpi://DEM/ifs"); + fprintf(screen, "3D interface is created\n"); + } + + //Pass the only argument + if (narg < 5) error->all(FLERR,"Illegal fix lbm_mui command"); + + int iarg = 3; + + while (iarg < narg) { + if (strcmp(arg[iarg],"dtLBM") == 0) { + dtLBM = atof(arg[iarg+1]); + iarg += 2; + } else error->all(FLERR,"Illegal fix nve/sphere command"); + } + + if (dtLBM <= 0.0) + error->all(FLERR,"Fix lbm/mui/iter negative timestep"); + + timeInit = update->ntimestep; + + alpha1 = update ->dt / dtLBM; + + //setting up nevery + if (alpha1 < 0.0000001) + alpha1 = 1; + + + if (alpha1 <= 1) { + nevery = static_cast(1.0/alpha1); + update->dt = dtLBM/ static_cast(nevery); + update->niter = 1; + } + else if (alpha1 > 1.0) { + nevery = 1; + update->niter = static_cast(alpha1); //DEM subcycling scheme-OPS will handle the + //transformation to integer + update->dt = dtLBM * static_cast(update->niter); + } + alpha1 = update ->dt / dtLBM; + flagSetup = false; + + Fd = NULL; //To be verified + Md = NULL; + //setting up calling back + grow_arrays(atom->nmax); + atom->add_callback(0); + atom->add_callback(1); + + // size_peratom_cols = 6; + // array_flag = 1; + // peratom_flag = 1; + // peratom_freq = 1; + + //modify the DEM domain + if (domain->dimension==3) + if ((domain->periodicity[0]==1) || (domain->periodicity[1]==1)|| (domain->periodicity[2]==1)) { + periodicFlag = 1; + } + else if (domain->dimension==2) + if ((domain->periodicity[0]==1) || (domain->periodicity[1]==1)) + periodicFlag = 1; + + if (periodicFlag == 1) { + dx = dtLBM * sqrt(3); + if (domain->periodicity[0]==1) { + domain->boxlo[0] -=0.5 * dx; + domain->boxhi[0] +=0.5 * dx; + } + + if (domain->periodicity[1]==1) { + domain->boxlo[1] -=0.5 * dx; + domain->boxhi[1] +=0.5 * dx; + } + + if (domain->dimension == 3) { + if (domain->periodicity[2] == 1) { + domain->boxlo[2] -= 0.5 * dx; + domain->boxhi[2] += 0.5 * dx; + } + } + + domain->set_initial_box(); + domain->set_global_box(); + domain->set_local_box(); + domain->print_box(" "); + } +} + +FixLbmMui::~FixLbmMui() { + + atom->delete_callback(id,0); + atom->delete_callback(id,1); + + memory->destroy(Fd); + memory->destroy(Md); + + if (domain->dimension==2) + delete interface2d; + else if (domain->dimension==3) + delete interface3d; + //Check if I need to destroy the MUI interface +} + +int FixLbmMui::setmask() { + + int mask = 0; + mask |= PRE_FORCE; + mask |= POST_FORCE; + mask |= END_OF_STEP; + + return mask; +} +void FixLbmMui::init() { + + if (!atom->sphere_flag) + error->all(FLERR,"Fix lbm/mui requires atom style sphere"); +} +void FixLbmMui::setup(int vflag) { + + + + flagSetup = false; + if (domain->dimension==2) + send_MUI_data(); + else if (domain->dimension ==3) + send_MUI_data3D(); + + + + if (domain->dimension==2) + recv_MUI_data(); + else if (domain->dimension == 3) { + recv_MUI_data3D(); + } + + flagSetup = true; + + if (strstr(update->integrate_style,"verlet")) + post_force(vflag); + + +} +void FixLbmMui::pre_force(int vflag) { + + //printf("Timestep %d\n",update->ntimestep); + if (update->ntimestep % nevery == 0) { + if (domain->dimension == 2) + send_MUI_data(); + else if (domain->dimension == 3) + send_MUI_data3D(); + } + + +} +void FixLbmMui::post_force(int vflag) { + + + //fprintf(screen,"Timestep %d\n",update->ntimestep); + double **f = atom->f; + double **torque = atom-> torque; + int nlocal = atom->nlocal; + int *mask = atom->mask; + + + + + + for (int i = 0; i < nlocal; i++) { + if (mask[i] & groupbit) { + f[i][0] += Fd[i][0]; + f[i][1] += Fd[i][1]; + f[i][2] += Fd[i][2]; + + torque[i][0] += Md[i][0]; + torque[i][1] += Md[i][1]; + torque[i][2] += Md[i][2]; + } + + } +} + +void FixLbmMui::send_MUI_data3D() { + + double sublox, subhix, subloy, subhiy, subloz, subhiz; + int timeCur, timeNext, data1; + int nlocal = atom->nlocal; + double **x = atom->x; + double **v = atom->v; + double **omega = atom->omega; + double *radius = atom->radius; + int *mask = atom->mask; + + sublox = domain->sublo[0]; + subhix = domain->subhi[0]; + + subloy = domain->sublo[1]; + subhiy = domain->subhi[1]; + + + subloz = domain->sublo[2]; + subhiz = domain->subhi[2]; + + mui::geometry::box3d send_recv_region({sublox, subloy, subloz}, {subhix, subhiy, subhiz}); + + if (flagSetup == false) { + fprintf(screen, "FlagSetup is false\n"); + if (domain->box_change == 1) { + timeCur = 0; + timeNext = update->nsteps + 1; + } + else { + timeCur = 0; + timeNext = update->nsteps + 10000; + } + + interface3d->push("alpha",alpha1); + interface3d->push("Nsteps",update->nsteps); + + interface3d->push("xPerFlag", domain->periodicity[0]); + interface3d->push("yPerFlag", domain->periodicity[1]); + interface3d->push("zPerFlag", domain->periodicity[2]); + + if ((domain->periodicity[0] == 1) || (domain->periodicity[1]==1) || (domain->periodicity[2]==1)) { + interface3d->push("xper", domain->prd[0]); + interface3d->push("yper", domain->prd[1]); + interface3d->push("zper", domain->prd[2]); + + double cutoffx = 0.5 * comm->cutghost[0] + 0.5 * neighbor->skin; + double cutoffy = 0.5 * comm->cutghost[1] + 0.5 * neighbor->skin; + double cutoffz = 0.5 * comm->cutghost[2] + 0.5 * neighbor->skin; + interface3d->push("xCutOff", cutoffx); + interface3d->push("yCutOff", cutoffy); + interface3d->push("zCutOff", cutoffz); + + } + + interface3d->announce_send_span(timeCur, timeNext, send_recv_region); + interface3d->announce_recv_span(timeCur, timeNext, send_recv_region); + + + } + else { + timeCur = update->ntimestep - timeInit; + + + if (domain->box_change==1) { + timeNext = (update-> ntimestep + 1) - timeInit; + interface3d->announce_send_span(timeCur, timeNext, send_recv_region); + interface3d->announce_recv_span(timeCur, timeNext, send_recv_region); + } + + } + + //sending data to LBM solver + //fprintf(screen, "LIGGGHTS: Number of particles of %d\n",nlocal); + for (int i = 0; i < nlocal; i++) { + if (mask[i] & groupbit) { + + interface3d->push("radius",{x[i][0],x[i][1],x[i][2]},radius[i]); + interface3d->push("u",{x[i][0],x[i][1],x[i][2]},v[i][0]); + interface3d->push("v",{x[i][0],x[i][1],x[i][2]},v[i][1]); + interface3d->push("w",{x[i][0],x[i][1],x[i][2]},v[i][2]); + interface3d->push("ox",{x[i][0],x[i][1],x[i][2]},omega[i][0]); + interface3d->push("oy",{x[i][0],x[i][1],x[i][2]},omega[i][1]); + interface3d->push("oz",{x[i][0],x[i][1],x[i][2]},omega[i][2]); + + + + } + } + //fprintf(screen,"LIGGGHTS: Data are send at time %d\n",timeCur); + data1 = interface3d->commit(timeCur); + // fprintf(screen,"Data that I send: %d\n",data1); +} + +void FixLbmMui::send_MUI_data() { + + double sublox, subhix, subloy, subhiy, subloz, subhiz; + int timeCur, timeNext, data1; + int nlocal = atom->nlocal; + double **x = atom->x; + double **v = atom->v; + double **omega = atom->omega; + double *radius = atom->radius; + int *mask = atom->mask; + + //setting_up communication domain + + + mui::geometry::box2d send_recv_region({sublox, subloy},{subhix,subhiy}); + + if (flagSetup == false) { + if (domain->box_change==1) { + timeCur = 0; + timeNext = update->nsteps + 1; + } + else { + timeCur = 0; + timeNext = update->nsteps + 10000; + } + interface2d->push("alpha",alpha1); + interface2d->push("Nsteps",update->nsteps); //Flag Setup-Not general + + interface2d->push("xPerFlag", domain->periodicity[0]); + interface2d->push("yPerFlag", domain->periodicity[1]); + + if ((domain->periodicity[0] == 1) || (domain->periodicity[1]==1)) { + interface2d->push("xper", domain->prd[0]); + interface2d->push("yper", domain->prd[1]); + + double cutoffx = 0.5 * comm->cutghost[0] + 0.5 * neighbor->skin; + double cutoffy = 0.5 * comm->cutghost[1] + 0.5 * neighbor->skin; + interface2d->push("xCutOff", cutoffx); + interface2d->push("yCutOff", cutoffy); + + } + + interface2d->announce_send_span(timeCur, timeNext, send_recv_region); + interface2d->announce_recv_span(timeCur, timeNext, send_recv_region); + fprintf(screen, "LIGGGHTS: I send alpha and Nsteps\n"); + } + else { + timeCur = update->ntimestep - timeInit; + + + if (domain->box_change==1) { + timeNext = (update-> ntimestep + 1) - timeInit; + interface2d->announce_send_span(timeCur, timeNext, send_recv_region); + interface2d->announce_recv_span(timeCur, timeNext, send_recv_region); + } + + } + + + + //Sending data to LBM solver + for (int i = 0; i < nlocal; i++) { + if (mask[i] & groupbit) { + if (domain->dimension==2) { + interface2d->push("radius",{x[i][0],x[i][1]},radius[i]); + interface2d->push("u",{x[i][0],x[i][1]},v[i][0]); + interface2d->push("v",{x[i][0],x[i][1]},v[i][1]); + interface2d->push("oz",{x[i][0],x[i][1]},omega[i][2]); + } + } + } + + data1 = interface2d->commit(timeCur); //replace with integer timestep for safety + + +} +void FixLbmMui::recv_MUI_data3D() { + int timeCur; + int nlocal = atom->nlocal; + double **x = atom->x; + int *mask = atom->mask; + + double dt = maxdt(update->dt, dtLBM); + if (flagSetup == false) + timeCur = 0; + else + timeCur = update->ntimestep - timeInit; + + mui::chrono_sampler_exact3d timeS; + mui::sampler_dem3d spatial; + + for (int i = 0; i < nlocal; i++) { + if (mask[i] & groupbit) { + Fd[i][0] = 0.0; + Fd[i][1] = 0.0; + Fd[i][2] = 0.0; + Md[i][0] = 0.0; + Md[i][1] = 0.0; + Md[i][2] = 0.0; + + auto FxTemp = interface3d->fetch("Fdx", {x[i][0],x[i][1],x[i][2]}, timeCur, spatial, timeS); + auto FyTemp = interface3d->fetch("Fdy", {x[i][0],x[i][1],x[i][2]}, timeCur, spatial, timeS); + auto FzTemp = interface3d->fetch("Fdz", {x[i][0],x[i][1],x[i][2]}, timeCur, spatial, timeS); + auto MxTemp = interface3d->fetch("Mdx", {x[i][0],x[i][1],x[i][2]}, timeCur, spatial, timeS); + auto MyTemp = interface3d->fetch("Mdy", {x[i][0],x[i][1],x[i][2]}, timeCur, spatial, timeS); + auto MzTemp = interface3d->fetch("Mdz", {x[i][0],x[i][1],x[i][2]}, timeCur, spatial, timeS); + + Fd[i][0] += FxTemp; + Fd[i][1] += FyTemp; + Fd[i][2] += FzTemp; + Md[i][0] += MxTemp; + Md[i][1] += MyTemp; + Md[i][2] += MzTemp; + + } + } + //printf("LIGGGHTS: I received particle data at timestep %d",timeCur); + interface3d->forget(timeCur); +} +void FixLbmMui::recv_MUI_data() { + + int timeCur; + //setting up the names of the receiving lists + std::string Fx1 = "Fdx"; + std::string Fy1 = "Fdy"; + std::string Mz1 = "Mdz"; + + int nlocal = atom->nlocal; + double **x = atom->x; + int *mask = atom->mask; + + double dt = maxdt(update->dt, dtLBM); + //setting up time span for data Receiving + if (flagSetup == false) { + timeCur = 0; + } + else { + timeCur = update->ntimestep - timeInit; + } + //Receiving timestep + mui::chrono_sampler_exact2d timeS; + mui::sampler_dem2d spatial; + + + for (int i = 0; i < nlocal; i++) { + if (mask[i] & groupbit) { + Fd[i][0] = 0.0; + Fd[i][1] = 0.0; + Fd[i][2] = 0.0; + Md[i][0] = 0.0; + Md[i][1] = 0.0; + Md[i][2] = 0.0; + + + auto FxTemp = interface2d->fetch("Fdx",{x[i][0],x[i][1]}, timeCur, spatial, timeS); + auto FyTemp = interface2d->fetch("Fdy",{x[i][0],x[i][1]}, timeCur, spatial, timeS); + auto MzTemp = interface2d->fetch("Mdz",{x[i][0],x[i][1]}, timeCur, spatial, timeS); + + Fd[i][0] += FxTemp; + Fd[i][1] += FyTemp; + Md[i][2] += MzTemp; + + + } + + } + + interface2d->forget(timeCur); + printf("LIGGGHTS: I received my drag forces for t=%d\n",timeCur); + + +} + +void FixLbmMui::end_of_step() { + + //send_MUI_data(); + if (domain->dimension == 2) + recv_MUI_data(); + else if(domain->dimension == 3) + recv_MUI_data3D(); +} + + +double FixLbmMui::memory_usage() { + + double bytes; + bytes = 6.0 * atom->nmax * sizeof(double); + + return bytes; + +} + +void FixLbmMui::grow_arrays(int nmax) { + + memory->grow(Fd, nmax, 3, "lbm_mui:Fd"); + memory->grow(Md, nmax, 3, "lbm_mui:Md"); +} + +int FixLbmMui::pack_exchange(int i, double *buf) { + + int m = 0; + + buf[m++] = Fd[i][0]; + buf[m++] = Fd[i][1]; + buf[m++] = Fd[i][2]; + buf[m++] = Md[i][0]; + buf[m++] = Md[i][1]; + buf[m++] = Md[i][2]; + + return m; +} + +int FixLbmMui::unpack_exchange(int nlocal, double *buf) { + + int m = 0; + Fd[nlocal][0] = buf[m++]; + Fd[nlocal][1] = buf[m++]; + Fd[nlocal][2] = buf[m++]; + + Md[nlocal][0] = buf[m++]; + Md[nlocal][1] = buf[m++]; + Md[nlocal][2] = buf[m++]; + + return m; +} + +void FixLbmMui::set_arrays(int i) { + + Fd[i][0] = 0.0; + Fd[i][1] = 0.0; + Fd[i][2] = 0.0; + + Md[i][0] = 0.0; + Md[i][1] = 0.0; + Md[i][2] = 0.0; +} + +void FixLbmMui::copy_arrays(int i, int j) { + + Fd[j][0] = Fd[i][0]; + Fd[j][1] = Fd[i][1]; + Fd[j][2] = Fd[i][2]; + + Md[j][0] = Md[i][0]; + Md[j][1] = Md[i][1]; + Md[j][2] = Md[i][2]; + +} + +double FixLbmMui::maxdt(double a, double b) { + + if (a>=b) + return a; + + return b; +} diff --git a/Apps/LBM-DEM/Makefile b/Apps/LBM-DEM/Makefile new file mode 100644 index 0000000..2494793 --- /dev/null +++ b/Apps/LBM-DEM/Makefile @@ -0,0 +1,336 @@ +OPS_INSTALL_PATH := $(OPS_INSTALL_PATH)/c +OPS_INC = -I$(OPS_INSTALL_PATH)/include +OPS_LIB = -L$(OPS_INSTALL_PATH)/lib/gnu + +CUDA_LIB = -L$(CUDA_INSTALL_PATH)/lib +CUDA_INC = -I$(CUDA_INSTALL_PATH)/include + +OPENCL_INC = -I$(OPENCL_INSTALL_PATH)/include +OPENCL_LIB = -L$(OPENCL_INSTALL_PATH)/lib64 -lOpenCL + +ifdef MPLB_DIR + MPLB_INC = -I$(MPLB_DIR) + MPLB_LIB = -L$(MPLB_DIR) +endif + +ifdef HDF5_INCLUDE_PATH + HDF5_INC := -I$(HDF5_INCLUDE_PATH) +ifdef CHECKPOINTING + HDF5_INC := $(HDF5_INC) -DCHECKPOINTING +endif +endif +ifdef HDF5_LIB_PATH + HDF5_LIB := -L$(HDF5_LIB_PATH) +endif + +ifeq ($(OPS_COMPILER),clang) + CPP = clang++ +ifdef DEBUG + CPPFLAGS = -O0 -g -DUNIX -Wall -std=c++11 #-Wall +else + CPPFLAGS = -O3 -fPIC -DUNIX -Wall -std=c++11 #-Wall +endif + OMPFLAGS = ""#-fopenmp + MPICPP = mpic++ + MPIFLAGS = $(CPPFLAGS) +else +ifeq ($(OPS_COMPILER),gnu) + CPP = g++ +ifdef DEBUG + CPPFLAGS = -ggdb -DUNIX -Wall +else + CPPFLAGS = -O3 -std=c++11 -fPIC -DUNIX +endif + OMPFLAGS = -fopenmp + MPICPP = mpicxx + MPIFLAGS = $(CPPFLAGS) +else +ifeq ($(OPS_COMPILER),archerGnu) + CPP = CC +ifdef DEBUG + CPPFLAGS = -ggdb -DUNIX -Wall -std=c++11 +else + CPPFLAGS = -O3 -std=c++11 -fPIC -DUNIX +endif + OMPFLAGS = -fopenmp + MPICPP = CC + MPIFLAGS = $(CPPFLAGS) +else +ifeq ($(OPS_COMPILER),intel) + CPP = icpc +ifdef DEBUG + CCFLAGS = -O0 -g -no-prec-div -openmp -fp-model strict -fp-model source -prec-div -prec-sqrt -DMPICH_IGNORE_CXX_SEEK #-DOPS_DEBUG +else + #CCFLAGS = -O3 -ipo -no-prec-div -fp-model strict -fp-model source -prec-div -prec-sqrt -vec-report2 -xSSE4.2 -parallel #-DCOMM_PERF #-DDEBUG + CCFLAGS = -O3 -ipo -no-prec-div -restrict -fno-alias -fp-model strict -fp-model source -prec-div -prec-sqrt -DMPICH_IGNORE_CXX_SEEK#-vec-report +endif + CPPFLAGS = $(CCFLAGS) + OMPFLAGS = -openmp -openmp-report2 + MPICPP = $(MPI_INSTALL_PATH)/bin/mpicxx + MPIFLAGS = $(CCFLAGS) +else +ifeq ($(OPS_COMPILER),mic) + CPP = icpc +ifdef DEBUG + CCFLAGS = -O0 -g -mmic -DMPICH_IGNORE_CXX_SEEK#-vec-report +else + #CCFLAGS = -O3 -ipo -no-prec-div -restrict -fno-alias -fp-model strict -fp-model source -prec-div -prec-sqrt -DMPICH_IGNORE_CXX_SEEK#-vec-report + #CCFLAGS = -O3 -ipo -no-prec-div -restrict -fno-alias -mmic -vec-report -finline -inline-forceinline -DSIMD_VEC=32 -DMPICH_IGNORE_CXX_SEEK#-vec-report + CCFLAGS = -O3 -ipo -no-prec-div -restrict -fno-alias -mmic -vec-report -DSIMD_VEC=32 -DMPICH_IGNORE_CXX_SEEK -finline -vec-report +endif + CPPFLAGS = $(CCFLAGS) + OMPFLAGS = -openmp -openmp-report2 + MPICPP = $(MPI_INSTALL_PATH)/bin/mpicxx + MPIFLAGS = $(CCFLAGS) + OPENCL_LIB = -L$(OPENCL_INSTALL_PATH)/lib64 -lintelocl -lOpenCL #for offload opencl execution on the phi +else +ifeq ($(OPS_COMPILER),pgi) + CPP = pgc++ + CC = pgcc + CUDA_ALIGN_FLAG := -D__x86_64 -D__align__\(n\)=__attribute__\(\(aligned\(n\)\)\) -D__location__\(a\)=__annotate__\(a\) -DCUDARTAPI= +ifdef DEBUG + CCFLAGS = -O2 -Minline -Kieee -Minform=inform -Minfo=all +else + CCFLAGS = -O3 -fast -Kieee -Minline -Minform=severe -Minfo=all -std=c++11 +endif + CPPFLAGS = $(CCFLAGS) + OMPFLAGS = -mp + MPICPP = $(MPI_INSTALL_PATH)/bin/mpicxx + MPICC = $(MPI_INSTALL_PATH)/bin/mpicc + MPIFLAGS = $(CPPFLAGS) +else +ifeq ($(OPS_COMPILER),xl) + CPP = xlc++ +ifdef DEBUG + CPPFLAGS = -O0 -g -qfullpath -Q -qlistopt -qattr=full -qlist -qreport -qxref=full -qsource -qsuppress=1506-224:1500-036 -qsrcmsg -qfloat=nomaf +else + CPPFLAGS = -O5 -q64 -qnoeh -qipa=partition=large -qfullpath -Q -qlistopt -qattr=full -qlist -qreport -qxref=full -qsource -qsuppress=1506-224:1500-036 -qsrcmsg -qfloat=nomaf +endif + OMPFLAGS = -fopenmp + MPICPP = $(MPI_INSTALL_PATH)/bin/mpicxx + MPIFLAGS = $(CPPFLAGS) +else +ifeq ($(OPS_COMPILER),cray) + CPP = CC + CUDA_ALIGN_FLAG := -D__x86_64 -D__align__\(n\)=__attribute__\(\(aligned\(n\)\)\) -D__location__\(a\)=__annotate__\(a\) -DCUDARTAPI= +ifdef DEBUG + CCFLAGS = -O2 +else + CCFLAGS = -O3 +endif + CPPFLAGS = $(CCFLAGS) + OMPFLAGS = + MPICPP = CC + MPICC = cc + MPIFLAGS = $(CPPFLAGS) +else +print: + @echo "unrecognised value for OPS_COMPILER" +endif +endif +endif +endif +endif +endif +endif + +endif + +NVCC := $(CUDA_INSTALL_PATH)/bin/nvcc +# flags for nvcc +# set NV_ARCH to select the correct one +ifndef NV_ARCH + MESSAGE=select an NVIDA device to compile in CUDA, e.g. make NV_ARCH=KEPLER + NV_ARCH=Kepler +endif +ifeq ($(NV_ARCH),Fermi) + CODE_GEN_CUDA=-gencode arch=compute_20,code=sm_21 +else +ifeq ($(NV_ARCH),Kepler) + CODE_GEN_CUDA=-gencode arch=compute_35,code=sm_35 +endif +endif + + +ifdef DEBUG + NVCC_FLAGS = -O0 -g -G +else + NVCC_FLAGS = -O3 -std=c++11 +endif + + +# +# master to make all versions +# + + +TARGETS = lbm2d_dev_seq lbm2d_seq lbm2d_dev_mpi lbm2d_mpi lbm2d_openmp lbm2d_mpi_openmp lbm2d_cuda lbm2d_mpi_cuda lbm2d_opencl lbm2d_mpi_opencl lbm3d_psm_dev_mpi + +ifeq ($(OPS_COMPILER),pgi) +#TARGETS += lbm2d_openacc lbm2d_mpi_openacc +else +#TARGETS += openacc_error +endif + +all: clean $(TARGETS) + +env: + @echo "MPLBDIR = $(MPLB_DIR)" + @echo "HDF5_INCLUDE_PATH = $(HDF5_INCLUDE_PATH)" + @echo "HDF5_LIB_PATH = $(HDF5_LIB_PATH)" + @echo "OPS_INSTALL_PATH = $(OPS_INSTALL_PATH)" + @echo "Compiler = $(OPS_COMPILER)" + +lbm3d_dev_seq : + $(MPICPP) $(CPPFLAGS) $(MAINCPP) ${MPLB_DIR}/evolution3d.cpp ${MPLB_DIR}/scheme.cpp ${MPLB_DIR}/scheme_wrapper.cpp ${MPLB_DIR}/configuration.cpp ${MPLB_DIR}/model.cpp ${MPLB_DIR}/model_wrapper.cpp ${MPLB_DIR}/block.cpp ${MPLB_DIR}/flowfield.cpp ${MPLB_DIR}/flowfield_wrapper.cpp ${MPLB_DIR}/boundary.cpp ${MPLB_DIR}/boundary_wrapper.cpp $(OPS_INC) $(HDF5_INC) $(MPLB_INC) $(OPS_LIB) $(HDF5_LIB) -DOPS_3D -DCPU -D$(LEVEL) -lops_hdf5_seq -lops_seq -lhdf5_hl -lhdf5 -lz -o lbm3d_dev_seq + +lbm3d_dev_seq_lib : + $(CPP) $(CPPFLAGS) $(MAINCPP) $(OPS_INC) $(HDF5_INC) $(MPLB_INC) $(OPS_LIB) $(MPLB_LIB) -DOPS_3D -DCPU -D$(LEVEL) -lmplb3d_seq -lops_seq -lops_hdf5_seq $(HDF5_LIB) -o lbm3d_dev_seq + +lbm3d_dev_mpi : + $(MPICPP) $(CPPFLAGS) $(MAINCPP) ${MPLB_DIR}/evolution3d.cpp ${MPLB_DIR}/scheme.cpp ${MPLB_DIR}/scheme_wrapper.cpp ${MPLB_DIR}/configuration.cpp ${MPLB_DIR}/model.cpp ${MPLB_DIR}/model_wrapper.cpp ${MPLB_DIR}/block.cpp ${MPLB_DIR}/flowfield.cpp ${MPLB_DIR}/flowfield_wrapper.cpp ${MPLB_DIR}/boundary.cpp ${MPLB_DIR}/boundary_wrapper.cpp $(OPS_INC) $(HDF5_INC) $(MPLB_INC) $(OPS_LIB) $(MPLB_LIB) $(HDF5_LIB) -DOPS_3D -DOPS_MPI -DCPU -D$(LEVEL) -lops_hdf5_mpi -lops_mpi -lhdf5_hl -lhdf5 -lz -o lbm3d_dev_mpi + +lbm3d_dem_mpi : + $(MPICPP) $(CPPFLAGS) $(MAINCPP) ${MPLB_DIR}/evolution3d.cpp ${MPLB_DIR}/scheme.cpp ${MPLB_DIR}/scheme_wrapper.cpp ${MPLB_DIR}/configuration.cpp ${MPLB_DIR}/model.cpp ${MPLB_DIR}/model_wrapper.cpp ${MPLB_DIR}/block.cpp ${MPLB_DIR}/flowfield.cpp ${MPLB_DIR}/flowfield_wrapper.cpp ${MPLB_DIR}/boundary.cpp ${MPLB_DIR}/boundary_wrapper.cpp block_particle_helper.cpp block_particles.cpp dem_handle.cpp fsi_base.cpp mui_interface.cpp mui_wrapper.cpp particle.cpp particle_shape.cpp particle_to_grid_base.cpp poros_grid.cpp poros_spherical.cpp psm.cpp -Imui $(OPS_INC) $(HDF5_INC) $(MPLB_INC) $(OPS_LIB) $(MPLB_LIB) $(HDF5_LIB) -DOPS_3D -DOPS_MPI -DCPU -D$(LEVEL) -lops_hdf5_mpi -lops_mpi -lhdf5_hl -lhdf5 -lz -o lbm3d_dem_mpi + +lbm3d_dev_mpi_lib : + $(MPICPP) $(MPIFLAGS) $(MAINCPP) $(OPS_INC) $(HDF5_INC) $(MPLB_INC) $(OPS_LIB) $(MPLB_LIB) -DOPS_MPI -DOPS_3D -DCPU -D$(LEVEL) -lmplb3d_mpi -lops_mpi -lops_hdf5_mpi $(HDF5_LIB) -o lbm3d_dev_mpi + +lbm2d_dev_seq : + $(CPP) $(CPPFLAGS) $(MAINCPP) $(shell find $(MPLB_DIR) -type f \( -iname "*.cpp" ! -iname "setup*" ! -iname "evolution3d.cpp" \)) $(OPS_INC) $(HDF5_INC) $(MPLB_INC) $(OPS_LIB) $(MPLB_LIB) -DOPS_2D -DCPU -D$(LEVEL) -lops_seq -lops_hdf5_seq $(HDF5_LIB) -o lbm2d_dev_seq + +lbm2d_dev_seq_lib : + $(CPP) $(CPPFLAGS) $(MAINCPP) $(OPS_INC) $(HDF5_INC) $(MPLB_INC) $(OPS_LIB) $(MPLB_LIB) -DOPS_2D -DCPU -D$(LEVEL) -lmplb2d_seq -lops_seq -lops_hdf5_seq $(HDF5_LIB) -o lbm2d_dev_seq + +lbm2d_dev_mpi : + $(MPICPP) $(MPIFLAGS) $(MAINCPP) $(shell find $(MPLB_DIR) -type f \( -iname "*.cpp" ! -iname "setup*" ! -iname "evolution3d.cpp" \)) $(OPS_INC) $(HDF5_INC) $(MPLB_INC) $(OPS_LIB) $(MPLB_LIB) -DOPS_MPI -DOPS_2D -DCPU -D$(LEVEL) -lops_mpi -lops_hdf5_mpi $(HDF5_LIB) -o lbm2d_dev_mpi + +lbm2d_dev_mpi_lib : + $(MPICPP) $(MPIFLAGS) $(MAINCPP) $(OPS_INC) $(HDF5_INC) $(MPLB_INC) $(OPS_LIB) $(MPLB_LIB) -DOPS_MPI -DOPS_2D -DCPU -D$(LEVEL) -lmplb2d_mpi -lops_mpi -lops_hdf5_mpi $(HDF5_LIB) -o lbm2d_dev_mpi + +# Old pre-processor 2D +setupdomain: Makefile setup_comput_domain.cpp scheme.cpp model.cpp boundary.cpp Case_Setup.cpp $(OPS_INSTALL_PATH)/lib/libops_seq.a + $(CPP) $(CPPFLAGS) $(OPS_INC) $(HDF5_INC) $(OPS_LIB) -DOPS_2D type.cpp boundary.cpp setup_comput_domain.cpp scheme.cpp model.cpp -lops_seq -lops_hdf5_seq $(HDF5_LIB) -o setup_comput_domain + +# Old pre-processor 3D +setupdomain3D: Makefile setup_comput_domain.cpp scheme.cpp model.cpp boundary.cpp $(OPS_INSTALL_PATH)/lib/libops_seq.a + $(CPP) $(CPPFLAGS) $(OPS_INC) $(HDF5_INC) $(OPS_LIB) -DOPS_3D type.cpp boundary.cpp setup_comput_domain.cpp scheme.cpp model.cpp -lops_seq -lops_hdf5_seq $(HDF5_LIB) -o setup_comput_domain + +# +# mpi version +# + +SEQ_KERNELS = lbm2d_seq_kernels.cpp + +lbm2d_mpi: Makefile type_ops.cpp boundary_ops.cpp scheme_ops.cpp flowfield_ops.cpp model_ops.cpp evolution_ops.cpp lbm2d_ops.cpp $(OPS_INSTALL_PATH)/lib/libops_mpi.a + $(MPICPP) $(MPIFLAGS) -DOPS_MPI $(OPS_INC) $(HDF5_INC) $(OPS_LIB) type_ops.cpp boundary_ops.cpp scheme_ops.cpp flowfield_ops.cpp model_ops.cpp evolution_ops.cpp lbm2d_ops.cpp -I. ./MPI/$(SEQ_KERNELS) -lops_mpi $(HDF5_LIB) -o lbm2d_mpi + +# +# OpenMP version +# +OMP_KERNELS = lbm2d_omp_kernels.cpp + +lbm2d_mpi_openmp: Makefile type_ops.cpp boundary_ops.cpp scheme_ops.cpp flowfield_ops.cpp model_ops.cpp lbm2d_ops.cpp $(OPS_INSTALL_PATH)/lib/libops_mpi.a + $(MPICPP) $(OMPFLAGS) $(MPIFLAGS) -DOPS_MPI $(OPS_INC) $(HDF5_INC) $(OPS_LIB) type_ops.cpp boundary_ops.cpp scheme_ops.cpp flowfield_ops.cpp model_ops.cpp lbm2d_ops.cpp -I. ./MPI_OpenMP/$(OMP_KERNELS) -lops_mpi $(HDF5_LIB) -o lbm2d_mpi_openmp + + +lbm2d_openmp: Makefile type_ops.cpp boundary_ops.cpp scheme_ops.cpp flowfield_ops.cpp model_ops.cpp evolution_ops.cpp lbm2d_ops.cpp $(OPS_INSTALL_PATH)/lib/libops_seq.a + $(MPICPP) $(OMPFLAGS) $(HDF5) $(MPIFLAGS) $(OPS_INC) $(HDF5_INC) $(OPS_LIB) type_ops.cpp boundary_ops.cpp scheme_ops.cpp flowfield_ops.cpp model_ops.cpp evolution_ops.cpp lbm2d_ops.cpp -I. ./MPI_OpenMP/$(OMP_KERNELS) -lops_seq $(HDF5_LIB) -o lbm2d_openmp + +# +# Sequential version +# +lbm3d_seq : + $(CPP) $(CPPFLAGS) ./MPI_OpenMP/lbm3d_cavity_cpu_kernels.cpp boundary.cpp evolution3d_ops.cpp scheme.cpp configuration.cpp model.cpp flowfield_ops.cpp lbm3d_cavity_ops.cpp type.cpp $(OPS_INC) $(HDF5_INC) $(MPLB_INC) $(OPS_LIB) $(MPLB_LIB) -DOPS_3D -DCPU -D$(LEVEL) -lops_seq -lops_hdf5_seq $(HDF5_LIB) -o lbm3d_seq + +lbm2d_seq: Makefile type_ops.cpp boundary_ops.cpp scheme_ops.cpp flowfield_ops.cpp model_ops.cpp evolution_ops.cpp lbm2d_ops.cpp $(OPS_INSTALL_PATH)/lib/libops_seq.a + $(MPICPP) $(MPIFLAGS) $(OPS_INC) $(HDF5_INC) $(OPS_LIB) -I. ./MPI/$(SEQ_KERNELS) type_ops.cpp boundary_ops.cpp scheme_ops.cpp flowfield_ops.cpp model_ops.cpp evolution_ops.cpp lbm2d_ops.cpp -lops_seq $(HDF5_LIB) -o lbm2d_seq + +# +# CUDA version +# $(OMPFLAGS) + +lbm2d_cuda: Makefile ./CUDA/lbm2d_kernels_cu.o type_ops.cpp boundary_ops.cpp scheme_ops.cpp flowfield_ops.cpp model_ops.cpp evolution_ops.cpp lbm2d_ops.cpp $(OPS_INSTALL_PATH)/lib/libops_cuda.a + $(MPICPP) $(OMPFLAGS) $(MPIFLAGS) $(CUDA_INC) $(HDF5_INC) $(OPS_INC) $(OPS_LIB) $(CUDA_LIB) type_ops.cpp boundary_ops.cpp scheme_ops.cpp flowfield_ops.cpp model_ops.cpp evolution_ops.cpp lbm2d_ops.cpp ./CUDA/lbm2d_kernels_cu.o -lcudart -lops_cuda $(HDF5_LIB) -o lbm2d_cuda + +lbm2d_mpi_cuda: Makefile ./CUDA/lbm2d_kernels_mpi_cu.o type_ops.cpp boundary_ops.cpp scheme_ops.cpp flowfield_ops.cpp model_ops.cpp lbm2d_ops.cpp $(OPS_INSTALL_PATH)/lib/libops_mpi_cuda.a + $(MPICPP) $(OMPFLAGS) $(CPPFLAGS) -DOPS_MPI $(CUDA_INC) $(OPS_INC) $(OPS_LIB) $(CUDA_LIB) type_ops.cpp boundary_ops.cpp scheme_ops.cpp flowfield_ops.cpp model_ops.cpp lbm2d_ops.cpp ./CUDA/lbm2d_kernels_mpi_cu.o -lcudart -lops_mpi_cuda $(HDF5_LIB) -o lbm2d_mpi_cuda + +./CUDA/lbm2d_kernels_cu.o: ./CUDA/lbm2d_kernels.cu Makefile + $(NVCC) $(VAR) $(INC) $(NVCC_FLAGS) $(CODE_GEN_CUDA) $(OPS_INC) -I. -c -o ./CUDA/lbm2d_kernels_cu.o ./CUDA/lbm2d_kernels.cu + +./CUDA/lbm2d_kernels_mpi_cu.o: ./CUDA/lbm2d_kernels.cu Makefile + $(NVCC) $(VAR) $(INC) $(NVCC_FLAGS) -DOPS_MPI $(CODE_GEN_CUDA) $(OPS_INC) -I. -I$(MPI_INSTALL_PATH)/include \ + -DMPICH_IGNORE_CXX_SEEK -c -o ./CUDA/lbm2d_kernels_mpi_cu.o ./CUDA/lbm2d_kernels.cu + +openacc_obj_list = $(shell find OpenACC/ -name "*.cpp" | sed s/\\.cpp/\\.o/g | tr "\n" " ") +openacc_c_obj_list = $(shell find OpenACC/ -name "*_c.c" | sed s/\\.c/\\.o/g | tr "\n" " ") + +openacc_mpi_obj_list = $(shell find OpenACC/ -name "*.cpp" | sed s/\\.cpp/\\_mpi.o/g | tr "\n" " ") +openacc_mpi_c_obj_list = $(shell find OpenACC/ -name "*_c.c" | sed s/\\.c/\\_mpi.o/g | tr "\n" " ") + + +./OpenACC/lbm2d_kernels.o: ./OpenACC/lbm2d_kernels.cpp Makefile + $(CPP) -acc -ta=tesla:cc35 -ta=tesla:maxregcount:64 $(CPPFLAGS) $(CUDA_ALIGN_FLAG) $(OPS_INC) $(CUDA_INC) \ + -I. OpenACC/lbm2d_kernels.cpp -c -o OpenACC/lbm2d_kernels.o + + +./OpenACC/lbm2d_kernels_mpi.o: ./OpenACC/lbm2d_kernels.cpp Makefile + $(MPICPP) -acc -ta=tesla:cc35 -ta=tesla:maxregcount:64 -DOPS_MPI $(MPIFLAGS) $(CUDA_ALIGN_FLAG) $(OPS_INC) $(CUDA_INC) \ + -I. OpenACC/lbm2d_kernels.cpp -c -o OpenACC/lbm2d_kernels_mpi.o + +./OpenACC/%_mpi.o: ./OpenACC/%.cpp ./OpenACC/%_c.c Makefile + @echo "Compiling kernel" $* + $(MPICC) -acc -ta=tesla:cc35 -ta=tesla:maxregcount:64 $(MPIFLAGS) -DOPS_MPI $(CUDA_ALIGN_FLAG) $(OPS_INC) $(CUDA_INC) \ + -I. OpenACC/$*_c.c -c -o OpenACC/$*_c_mpi.o + $(MPICPP) -acc -ta=tesla:cc35 -ta=tesla:maxregcount:64 $(MPIFLAGS) -DOPS_MPI $(CUDA_ALIGN_FLAG) $(OPS_INC) $(CUDA_INC) \ + -I. OpenACC/$*.cpp -c -o OpenACC/$*_mpi.o + +./OpenACC/%.o: ./OpenACC/%.cpp ./OpenACC/%_c.c Makefile + @echo "Compiling kernel" $* + $(CC) -acc -ta=tesla:cc35 -ta=tesla:maxregcount:64 $(CCFLAGS) $(CUDA_ALIGN_FLAG) $(OPS_INC) $(CUDA_INC) \ + -I. OpenACC/$*_c.c -c -o OpenACC/$*_c.o + $(CPP) -acc -ta=tesla:cc35 -ta=tesla:maxregcount:64 $(CPPFLAGS) $(CUDA_ALIGN_FLAG) $(OPS_INC) $(CUDA_INC) \ + -I. OpenACC/$*.cpp -c -o OpenACC/$*.o + +lbm2d_openacc: $(openacc_obj_list) ./OpenACC/lbm2d_kernels.o type_ops.cpp boundary_ops.cpp scheme_ops.cpp flowfield_ops.cpp model_ops.cpp lbm2d_ops.cpp Makefile $(OPS_INSTALL_PATH)/lib/libops_cuda.a + $(MPICPP) -acc -ta=tesla:cc35 $(MPIFLAGS) $(OPS_INC) $(OPS_LIB) -DOPS_MPI $(CUDA_INC) $(CUDA_LIB) \ + type_ops.cpp boundary_ops.cpp scheme_ops.cpp flowfield_ops.cpp model_ops.cpp lbm2d_ops.cpp -I. $(openacc_obj_list) $(openacc_c_obj_list) -lcudart -lops_cuda $(HDF5_LIB -o lbm2d_openacc + + +lbm2d_mpi_openacc: $(openacc_mpi_obj_list) ./OpenACC/lbm2d_kernels_mpi.o type_ops.cpp boundary_ops.cpp scheme_ops.cpp flowfield_ops.cpp model_ops.cpp lbm2d_ops.cpp Makefile $(OPS_INSTALL_PATH)/lib/libops_mpi_cuda.a + $(MPICPP) -acc -ta=tesla:cc35 $(MPIFLAGS) $(OPS_INC) $(OPS_LIB) $(CUDA_INC) $(CUDA_LIB) -DOPS_MPI \ + type_ops.cpp boundary_ops.cpp scheme_ops.cpp flowfield_ops.cpp model_ops.cpp lbm2d_ops.cpp -I. $(openacc_mpi_obj_list) $(openacc_mpi_c_obj_list) -lcudart -lops_mpi_cuda $(HDF5_LIB -o lbm2d_mpi_openacc + + +# +#OpenCL version +# +lbm2d_opencl: ./OpenCL/lbm2d_opencl_kernels.o lbm2d_ops.cpp lbm2d_kernel.h lbm2d_print_kernel.h lbm2d_copy_kernel.h Makefile $(OPS_INSTALL_PATH)/lib/libops_opencl.a + $(CPP) $(MPIFLAGS) $(OPS_INC) $(OPS_LIB) $(OPENCL_LIB) lbm2d_ops.cpp ./OpenCL/lbm2d_opencl_kernels.o -lops_opencl -o lbm2d_opencl + +lbm2d_mpi_opencl: ./OpenCL/lbm2d_mpi_opencl_kernels.o lbm2d_ops.cpp lbm2d_kernel.h lbm2d_print_kernel.h lbm2d_copy_kernel.h Makefile $(OPS_INSTALL_PATH)/lib/libops_mpi_opencl.a + $(MPICPP) $(MPIFLAGS) -DOPS_MPI $(OPS_INC) $(OPS_LIB) $(OPENCL_LIB) lbm2d_ops.cpp ./OpenCL/lbm2d_opencl_kernels.o -lops_mpi_opencl -o lbm2d_mpi_opencl + +./OpenCL/lbm2d_opencl_kernels.o: ./OpenCL/lbm2d_opencl_kernels.cpp Makefile + $(CPP) $(CPPFLAGS) $(VAR) $(OPENCL_INC) $(INC) $(OPS_INC) -I. \ + -c -o ./OpenCL/lbm2d_opencl_kernels.o ./OpenCL/lbm2d_opencl_kernels.cpp + +./OpenCL/lbm2d_mpi_opencl_kernels.o: ./OpenCL/lbm2d_opencl_kernels.cpp Makefile + $(MPICPP) $(CPPFLAGS) -DOPS_MPI $(VAR) $(OPENCL_INC) $(INC) $(OPS_INC) -I. \ + -c -o ./OpenCL/lbm2d_opencl_kernels.o ./OpenCL/lbm2d_opencl_kernels.cpp + +# +#error +# +openacc_error: + @echo "lbm2d_mpi_openacc cannot be built with $(OPS_COMPILER) compilers .. requires an OpenACC compiler (e.g. pgi)" + + +# +# cleanup +# + +clean: + rm -f lbm2d_dev_seq lbm3d_dem_mpi lbm2d_seq lbm2d_dev_seq lbm2d_mpi lbm2d_dev_mpi lbm2d_openmp lbm2d_mpi_openmp lbm2d_cuda lbm2d_mpi_cuda lbm2d_openacc lbm2d_mpi_openacc lbm2d_opencl lbm2d_mpi_opencl ./CUDA/*.o ./OpenACC/*.o *.o lbm2d_opencl lbm3d_dev_seq lbm3d_seq lbm3d_dev_seq lbm3d_mpi lbm3d_dev_mpi./OpenCL/*.o *.o + diff --git a/Apps/LBM-DEM/block_particle_helper.cpp b/Apps/LBM-DEM/block_particle_helper.cpp new file mode 100644 index 0000000..8329f1c --- /dev/null +++ b/Apps/LBM-DEM/block_particle_helper.cpp @@ -0,0 +1,445 @@ +/** + * Copyright 2019 United Kingdom Research and Innovation + * + * Authors: See AUTHORS + * + * Contact: [jianping.meng@stfc.ac.uk and/or jpmeng@gmail.com] + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice + * this list of conditions and the following disclaimer in the documentation + * and or other materials provided with the distribution. + * 3. Neither the name of the copyright holder nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * ANDANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +/*! + * @brief Wrapper functions for fluid-particle interaction and particle data + * @author C. Tsigginos + */ + + + +#include "block_particle_helper.h" +#include +//#include "model_kernel.inc" +#include "dem_data.h" +#include "block_particles.h" +#include + +//#include "particle.h" +//#include +std::map> fluidPartInteractionModels; + +void DefineBlockParticles( int spacedim, Real cutoff, Real dx1, std::string particleType) { + + + ParticleShapeDiscriptor particleTypeNum; + + //Compare strings to find particletype + std::string s1{"spherical"}; + std::string s2{"quadratic"}; + std::string s3{"mesh"}; + if (particleType.compare(s1) == 0) + particleTypeNum = spherical; + else if (particleType.compare(s2) == 0) + particleTypeNum = quadratic; + else if (particleType.compare(s3) == 0) + particleTypeNum = mesh; + else { + ops_printf("MPLB: This type of particle is not supported\n"); + ops_printf("MPLB: Reversing to spherical particles\n"); + particleTypeNum = spherical; + } + + for (const auto& idBlock : g_Block()) { + const Block& myBlock{idBlock.second}; + + BlockParticles ParticleNew(spacedim, dx1, cutoff, myBlock, particleTypeNum,0 ,0); + BlockParticleList.emplace(idBlock.first, ParticleNew); + + } + + +} + +void DefineInteractionModel(std::vector FluidParticleInteractionType, + std::vector fsiCompoId, Real* forceUser, + double gamma, SizeType timeStep) { + + int noComp = FluidParticleInteractionType.size(); + int noFSICompo = fsiCompoId.size(); + FSIType fsiModel; + int idCompo; + std::vector porosModel; + int nelem; + + std::map components = g_Components(); + int numComponents = ComponentNum(); + + + ops_printf("gamma = %f\n",gamma); + + if (noComp != numComponents) { + ops_printf("Error: Number of FSI components inconsistent with actual components\n"); + assert(noComp != numComponents); + } + + if (noFSICompo != numComponents) { + ops_printf("Error: Number of fsiCompoId differs from actual number of components\n"); + assert(noFSICompo != numComponents); + } + + ParticleShapeDiscriptor particleType = (ParticleShapeDiscriptor) BlockParticleList + .at(0).GetParticleShape(); + //find particle mapping schemes + int iter = 0; + for (int iComp = 0; iComp < noComp; iComp++) { + fsiModel = FluidParticleInteractionType.at(iComp); + if ( fsiModel == Model_PSM) { + if (iter == 0) { + if (particleType == spherical) + porosModel.push_back(Mode_Spherical); + else + porosModel.push_back(Mode_Grid); + iter = 1; + } + else + porosModel.push_back(Mode_Copy); + } + else if (fsiModel == Model_None) + porosModel.push_back(Mode_None); + else { + ops_printf("This type of model is not supported\n"); + exit(EXIT_FAILURE); + } + + + } + + if (particleType == spherical) + nelem = 2; + else + nelem = 8; + + + for (int iComp = 0; iComp < noComp; iComp++) { + fsiModel = (FSIType) FluidParticleInteractionType.at(iComp); + + idCompo = components.at(iComp).id; + + switch (fsiModel) { + case Model_None: { +#ifdef CPU + #if DebugLevel >= 2 + ops_printf("Base scheme (no fluid-particle interactions assigned to component %d\n", idCompo); + #endif +#endif + std::shared_ptr model(new FsiBase( components.at(iComp), + SpaceDim(), forceUser, false, porosModel.at(iComp), gamma)); + fluidPartInteractionModels.insert(std::make_pair(idCompo, model)); + } + break; + case Model_PSM: { + +//#ifdef CPU +// #if DebugLevel >= 2 + ops_printf("PSM scheme used for component %d\n", idCompo); +// #endif +//#endif + std::shared_ptr model(new Psm(components.at(iComp), SpaceDim(), + forceUser, true , porosModel.at(iComp), gamma, nelem, + particleType)); + fluidPartInteractionModels.insert(std::make_pair(idCompo, model)); + } + break; + default: + ops_printf("The chosen model is not supported\n"); + exit(EXIT_FAILURE); + } + + + + } + + for (auto& interactionModel : fluidPartInteractionModels) { + interactionModel.second->DefineVariables(timeStep); + } + +} +void UpdateParticleMappingDragForceRestart(SizeType currentStep) { + + Real dt = TimeStep(); + InitializeDragForce(); + + ParticleEnvelopes(); + + CalculateParticleMomentum(); + + + CalculateDragForce(dt, currentStep); + +} + + +void ParticleMapping(int flag) { + + int checkDistance; + + checkDistance = CheckParticleDistance(); + + + if (flag == 1) + checkDistance = 1; + + + if (checkDistance == 1) { + UpdateOldParticleLocation(); + + ParticleEnvelopes(); + + InitializeFSILists(); //FSI schemes + + MappingParticlesToLBMGrid(); //FSI Wrappers + + } + else + UpdateParticlesToLBMGrid(); +} + + +int CheckParticleDistance() { + int distanceGlobal, distanceTmp; + distanceGlobal = 0; + for ( auto& blockParticle : BlockParticleList) { + if (blockParticle.second.OwnedStatus()) { + distanceTmp = blockParticle.second.CheckDistanceBlock(); + if (distanceTmp == 1) { + distanceGlobal = 1; + break; + } + } + } + +#ifdef OPS_MPI + distanceTmp = distanceGlobal; + if (ops_num_procs()>1) + MPI_Allreduce(&distanceTmp, &distanceGlobal, 1, MPI_INT, MPI_MAX, OPS_MPI_GLOBAL); +#endif + +#ifdef CPU +#if DebugLevel >= 2 + printf("Rank %d: distanceGlobal = %d\n",ops_get_proc(), distanceGlobal); +#endif +#endif + + return distanceGlobal; + +} + +void UpdateOldParticleLocation() { + + for (auto & blockParticle : BlockParticleList) { + if (blockParticle.second.OwnedStatus()) + blockParticle.second.UpdateOldParticlePosition(); + } + +} + +void ParticleEnvelopes() { + + for (auto & blockParticle : BlockParticleList) { + if (blockParticle.second.OwnedStatus()) + blockParticle.second.FindStencil(); + } +} + +void MappingParticlesToLBMGrid() { + + for (auto& fsi : fluidPartInteractionModels) { + fsi.second->MappingFunction(true); + } + + + +} + +void UpdateParticlesToLBMGrid() { + for (auto& fsi : fluidPartInteractionModels) { + fsi.second->MappingFunction(false); + } +} + +void PostVelocityFSIFunctions() { + + for (auto &fsi : fluidPartInteractionModels) { + fsi.second->PostVelocityCalculation(); + } +} + +void PreCollisionFSIFunctions() { + + for (auto &fsi : fluidPartInteractionModels) { + fsi.second->PreCollision(); + } + +} + +void PostStreamingFSIFunctions() { + + for (auto &fsi : fluidPartInteractionModels) { + fsi.second->PostStreaming(); + } + +} + +void CalculateParticleMomentum() { + + for (auto &fsi : fluidPartInteractionModels) { + fsi.second->CalculateDragForce(); + } +} + +void InitializeFSILists() { + + for (auto &fsi : fluidPartInteractionModels) { + fsi.second->InitializeVariables(); + } +} + +void FluidParticleCollisions() { + + int loop[2]; + +#if OPS_3D + int velId[3]; +#endif + +#if OPS_2D + int velId[2]; +#endif + + CollisionType collisionModel; + Real tauRef; + int componentId, rhoId, Tid; + for (auto &fsi : fluidPartInteractionModels) { + + if (fsi.second->collisionOwned) { + fsi.second->ModelCollision(); + } + else { + //TODO need to obtain the id of the + fsi.second->ObtainID(velId, loop, tauRef, collisionModel,componentId, rhoId, Tid); + PreDefinedCollision3D(velId, loop, tauRef, collisionModel, componentId, rhoId, Tid); + } + + } +} + +void CalculateDragForce(Real dt, SizeType currentStep) { + + for (auto& blockParticle : BlockParticleList) { + blockParticle.second.EvaluateDragForce(dt); + } +} + +void InitializeDragForce() { + + for (auto& blockParticle : BlockParticleList) { + if (blockParticle.second.OwnedStatus()) + blockParticle.second.InitializeDragForce(); + + } + + +} + +void UpdateFPIVelocities3D() { + + UpdateMacroVars3D(); + PostVelocityFSIFunctions(); + + +} + +void WriteFPIDataToHdf5(SizeType currentStep) { + + for (auto &fsi : fluidPartInteractionModels) { + fsi.second->WriteToHdf5(CaseName(), currentStep); + } + +} + +void AssignParticlesToBlocksSpheres(int Nparticles,Real* xTmp, Real* yTmp,Real* zTmp, + Real* radTmp, Real* uTmp,Real* vTmp, Real* wTmp, Real* oxTmp, + Real* oyTmp, Real* ozTmp) { + + int spaceDim; +#ifdef OPS_3D + spaceDim = 3; +#endif + +#ifdef OPS_2D + spaceDim = 2; +#endif + int idx; + std::vector shape(1.0); + std::vector extra; + shape.reserve(1); + int ipx = 0; + Real xParticle[spaceDim], uParticle[spaceDim], omParticle[spaceDim]; + for (int iPar = 0; iPar < Nparticles; iPar++) { + ipx += 1; + xParticle[0] = xTmp[iPar]; + xParticle[1] = yTmp[iPar]; + xParticle[2] = zTmp[iPar]; + uParticle[0] = uTmp[iPar]; + uParticle[1] = vTmp[iPar]; + uParticle[2] = wTmp[iPar]; + omParticle[0] = oxTmp[iPar]; + omParticle[1] = oyTmp[iPar]; + omParticle[2] = ozTmp[iPar]; + shape.at(0) = radTmp[iPar]; + int ix = 0; + for (auto &blockParticle : BlockParticleList) { + ix = ix + 1; + idx = blockParticle.second.InsertParticle(xParticle, radTmp[iPar], shape, + uParticle, omParticle, extra); + printf("Rank %d: Idx = %d for particle %d\n", ops_get_proc(), idx, iPar ); + } + } + + +} + +void DefineBlockOwnership() { + + for (auto& blockParticle : BlockParticleList) + blockParticle.second.GetOwnership(); + +} + +void DefineLocalBoxBound() { + + for (auto& blockParticle : BlockParticleList) { + if (blockParticle.second.owned) + blockParticle.second.FindBoxLocalBound(); + } +} diff --git a/Apps/LBM-DEM/block_particle_helper.h b/Apps/LBM-DEM/block_particle_helper.h new file mode 100644 index 0000000..cf7295e --- /dev/null +++ b/Apps/LBM-DEM/block_particle_helper.h @@ -0,0 +1,96 @@ +/** + * Copyright 2019 United Kingdom Research and Innovation + * + * Authors: See AUTHORS + * + * Contact: [jianping.meng@stfc.ac.uk and/or jpmeng@gmail.com] + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice + * this list of conditions and the following disclaimer in the documentation + * and or other materials provided with the distribution. + * 3. Neither the name of the copyright holder nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * ANDANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +/*! + * @brief Wrapper functions for fluid-particle interaction and particle data + * @author C. Tsigginos + */ + + +#ifndef BLOCK_PARTICLES_HELPER_H_ +#define BLOCK_PARTICLES_HELPER_H_ + +#include "block.h" +#include "scheme.h" +#include "block_particles.h" +#include "flowfield.h" +#include "type.h" +#include "ops_lib_core.h" +#ifdef OPS_MPI +#include "ops_mpi_core.h" +#endif +#include "fpi.h" +#include +#include + +void ParticleMapping(int flag); + +int CheckParticleDistance(); +void UpdateOldParticleLocation(); +void ParticleEnvelopes(); +void MappingParticlesToLBMGrid(); +void UpdateParticlesToLBMGrid(); + + +//Wrapper for FSI object functions +void PostVelocityFSIFunctions(); +void PreCollisionFSIFunctions(); +void PostStreamingFSIFunctions(); +void CalculateParticleMomentum(); + +void FluidParticleCollisions(); +void PreDefineCollision3DFSI(int* velID, int* loop, Real tauRef, CollisionType collisionType); + +void CalculateDragForce(Real dt, SizeType currentStep); +void InitializeDragForce(); +void InitializeFSILists(); +void UpdateFPIVelocities3D(); +void WriteFPIDataToHdf5(SizeType currentStep); + +void UpdateParticleMappingDragForceRestart(SizeType currentStep); + +void DefineBlockParticles(int spacedim, Real cutoff, Real dx1, + std::string particleType); + +void DefineInteractionModel(std::vector FluidParticleInteractionType, + std::vector fsiCompoId, Real* forceUser, + double gamma, SizeType timeStep); + +//PArticle assigning wrappers +void AssignParticlesToBlocksSpheres(int Nparticles,Real* xTmp, Real* yTmp,Real* zTmp, + Real* radTmp, Real* uTmp,Real* vTmp, Real* wTmp, Real* oxTmp, + Real* oyTmp, Real* ozTmp); + +void DefineBlockOwnership(); +void DefineLocalBoxBound(); + +#endif /* APPS_LBM_DEM_BLOCK_PARTICLES_WRAPPER_H_ */ diff --git a/Apps/LBM-DEM/block_particles.cpp b/Apps/LBM-DEM/block_particles.cpp new file mode 100644 index 0000000..994ab9a --- /dev/null +++ b/Apps/LBM-DEM/block_particles.cpp @@ -0,0 +1,409 @@ +/** + * Copyright 2019 United Kingdom Research and Innovation + * + * Authors: See AUTHORS + * + * Contact: [jianping.meng@stfc.ac.uk and/or jpmeng@gmail.com] + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice + * this list of conditions and the following disclaimer in the documentation + * and or other materials provided with the distribution. + * 3. Neither the name of the copyright holder nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * ANDANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +/*! + * @brief Class for handling particles at various blocks + * @author C. Tsigginos + * @details: Handling of particles that owned by a given block + */ + +#include "block_particles.h" + +#include +#include +#include "ops_seq_v2.h" +#include "block_particles.inc" + +ListBlockParticles BlockParticleList; + +BlockParticles::BlockParticles(int spacedim, Real dx1, Real cutoff,const Block& myBlock, + ParticleShapeDiscriptor particleType, int nInputExtraVariables, + int nOutputExtraVariables) : currentBlock(myBlock){ + + spaceDim = spacedim; + int size = 2 * spaceDim; +// xBoundGlobal = new Real [size]; +// xBoundLocal = new Real [size]; + + owned = true; + dx = dx1; + + if ((spaceDim < 1) && (spaceDim > 3)) { + + ops_printf("ERROR: Spacedim is not consistent"); + exit(EXIT_FAILURE); + } + + + cutOff = cutoff; + if (cutOff <= 0.0) { + ops_printf("Error Particle object: Cut-off %12.9e is non-positive number\n", cutOff); + exit(EXIT_FAILURE); + } + + + if (dx <= 0.0) { + ops_printf("Error BlockParticles: Grid size is not positive\n"); + exit(EXIT_FAILURE); + } + + + + + + NParticles = 0; + NPeriodic = 0; + particleShape = particleType; + //sanity checks + switch (particleShape) { + case spherical: + ops_printf("Spherical particles will be generated for Block %d\n", myBlock.ID()); + break; + case quadratic: + ops_printf("Quadratic particles will be generated for Block %d\n", myBlock.ID()); + break; + case mesh: + ops_printf("Polygonal particles will be generated for Block %d\n", myBlock.ID()); + break; + default: + ops_printf("This types of particles is not supported.\n Exiting\n"); + exit(EXIT_FAILURE); + } + + Nmax = 0; + NReserve = 100; + particleList.reserve(NReserve); + + if (nInputExtraVariables > 0) { + hasExtraInputVariables = true; + extraInputSize = nInputExtraVariables; + } + else { + hasExtraInputVariables = false; + extraInputSize = 0; + } + + if (nOutputExtraVariables > 0) { + hasExtraOutputVariables = true; + extraOutputSize = nOutputExtraVariables; + } + else { + hasExtraOutputVariables = false; + extraOutputSize = 0; + } + +} + +BlockParticles::~BlockParticles() { + +// delete[] xBoundGlobal; +// delete[] xBoundLocal; +// delete[] Nf; + +} + +int BlockParticles::CheckDistanceBlock() { + + int nTotal = NParticles + NPeriodic; + Real distance; + int cut = 0; + for (int iPart = 0; iPart < nTotal; iPart++) { + distance = 0.0; + for (int iDim = 0; iDim < spaceDim; iDim++) + distance += (particleList[iPart].xParticle[iDim] - particleList[iPart].xOld[iDim]) * + (particleList[iPart].xParticle[iDim] - particleList[iPart].xOld[iDim]); + + if (distance > cutOff * cutOff) { + cut = 1; + break; + } + } + return cut; +} + +void BlockParticles::UpdateOldParticlePosition() { + + int nTotal = NParticles + NPeriodic; + for (int iPart = 0; iPart < nTotal; iPart++) { + for (int iDim = 0; iDim < spaceDim; iDim++) + particleList.at(iPart).UpdateOldParticlePositions(); + } +} + +void BlockParticles::InitializeDragForce() { + + for (int iPart = 0; iPart < NParticles; iPart++) + particleList[iPart].InitializeDrag(); +} + +void BlockParticles::EvaluateDragForce(Real dt) { + + for (int iPart = 0; iPart < NParticles; iPart++) + particleList[iPart].EvaluateDrag(dt); +} + +int BlockParticles::UpdateParticle(Real* xpos, Real radius, std::vector shape) { + + + NParticles += 1; + if (NParticles > Nmax) { + Nmax += 1; + particleList.push_back(Particle(spaceDim, radius, xpos, shape, particleShape, + extraInputSize, extraOutputSize)); + if (Nmax > NReserve) { + NReserve = Nmax + 100; + +#ifdef CPU +#if DebugLevel > 0 + printf("Rank %d at Block %d reserves totally %d particles\n", ops_get_proc(), currentBlock.ID(), Nmax+100); +#endif +#endif + particleList.reserve(NReserve); + } + } + else { + particleList[NParticles-1].UpdateParticleLocation(xpos); + particleList[NParticles-1].UpdateParticleShape(radius, shape); + } + + return NParticles-1; + +} +void BlockParticles::UpdateParticleVelocities(Real* uPart, Real* omegaT) { + + particleList.at(NParticles - 1).UpdateParticleVelocities(uPart, omegaT); +} + + +int BlockParticles::InsertParticle(Real* xpos, Real radius, std::vector shape, + Real* uPart, Real* omegaT, std::vector& inputData) { + + int idParticle; + bool flag = SphereParallepipedIntersection(xpos, radius); + + if (flag) { +#ifdef CPU +#if DebugLevel >=2 + printf("Rank %d: Particle [%f %f %f] will be inserted in the particleList\n", + ops_get_proc(), xpos[0], xpos[1], xpos[2]); +#endif +#endif + idParticle = UpdateParticle(xpos, radius, shape); + UpdateParticleVelocities(uPart, omegaT); + if (hasExtraInputVariables) + GetAdditionalInputVariables(inputData, NParticles-1); + return idParticle; + } + + + return -1; +} + + +void BlockParticles::FindStencil() { + + int nTotal = NPeriodic + NParticles; + + for (int iPart = 0; iPart < nTotal; iPart++) + particleList[iPart].UpdateStencil(xBoundGlobal, Nf, dx); + +} + +bool BlockParticles:: SphereParallepipedIntersection(Real* xpos,Real radius) { + + Real testX[spaceDim]; + Real distance; + + + + + for (int iDim = 0; iDim < spaceDim; iDim++) { + + testX[iDim] = xpos[iDim]; + if (xpos[iDim] < xBoundLocal[2 * iDim]) + testX[iDim] = xBoundLocal[2 * iDim]; + else if (xpos[iDim] > xBoundLocal[2 * iDim + 1]) + testX[iDim] = xBoundLocal[2 * iDim + 1]; + } + + distance = 0.0; + for (int iDim = 0; iDim < spaceDim; iDim++) + distance += (xpos[iDim] - testX[iDim]) * (xpos[iDim] - testX[iDim]); + + if (distance <= radius * radius) + return true; + + return false; + +} + +void BlockParticles::FindBoxLocalBound() { + + int start[spaceDim], end[spaceDim], range[2 * spaceDim], disp[spaceDim]; + int size = 2 * spaceDim; + int iterRng[size]; + Real xb[spaceDim]; + for (int nDim = 0; nDim < spaceDim; nDim++) { + xBoundLocal[2 * nDim] = std::numeric_limits::max(); + xBoundLocal[2 * nDim + 1] = -1.0 * std::numeric_limits::max(); + } + int blockIndex = currentBlock.ID(); + + if (owned) { + //Find block ranges + std::vector iterRng1; + iterRng1.assign(currentBlock.WholeRange().begin(), currentBlock.WholeRange().end()); + + for (int iDim = 0; iDim < 2 * spaceDim; iDim++) + range[iDim] = iterRng1.at(iDim); + + ops_get_abs_owned_range(currentBlock.Get(), range, start, end, disp); + + + + for (int iDir = 0; iDir < spaceDim; iDir++) { + Nf[2 * iDir] = start[iDir]; + Nf[2 * iDir + 1] = end[iDir]; + } + + + + for (int iDim = 0; iDim < spaceDim; iDim++) { + iterRng[2 * iDim] = Nf[2 *iDim]; + iterRng[2 * iDim + 1] = Nf[2 * iDim] + 1; + } + + + int spacedim = spaceDim; + ops_par_loop(KerCarBound,"KerCarBound", currentBlock.Get(), spaceDim, iterRng, + ops_arg_dat(g_CoordinateXYZ()[blockIndex], spaceDim, + LOCALSTENCIL, "double", OPS_READ), + ops_arg_gbl(xb, spaceDim, "double", OPS_READ), + ops_arg_gbl(&spacedim, 1, "int", OPS_READ)); + + for (int iDim = 0; iDim < spaceDim; iDim++) { + xBoundLocal[2 * iDim] = xb[iDim] - 0.5 * dx; + } + + + //2nd point + for (int iDir = 0; iDir < spaceDim; iDir++) { + iterRng[2 * iDir] = Nf[2 * iDir + 1] - 1; + iterRng[2 * iDir + 1] = Nf[2 * iDir + 1]; + } + + + + ops_par_loop(KerCarBound, "KerCarBound", currentBlock.Get(), spaceDim, iterRng, + ops_arg_dat(g_CoordinateXYZ()[blockIndex], spaceDim, + LOCALSTENCIL, "double", OPS_READ), + ops_arg_gbl(xb, spaceDim, "double", OPS_READ), + ops_arg_gbl(&spacedim, 1, "int", OPS_READ)); + + for (int iDim = 0; iDim < spaceDim; iDim++) + xBoundLocal[2 * iDim + 1] = xb[iDim] + 0.5 * dx; +#ifdef CPU +#if DebugLevel >= 2 + for (int iDim = 0; iDim < spaceDim; iDim++) + printf("Block %d: Current local Bound [%f %f]\n",currentBlock.ID(), xBoundLocal[2*iDim], xBoundLocal[2 * iDim+1]); +#endif +#endif + } + +} + + +void BlockParticles::ExtractBound(Real* xMin, Real* xMax) { + + for (int iDim = 0; iDim < spaceDim; iDim++) { + xMin[iDim] = xBoundLocal[2 * iDim]; + xMax[iDim] = xBoundLocal[2 * iDim + 1]; + } + + +} + +void BlockParticles::GetLocalBound(Real* xBound) { + + for (int iDim = 0; iDim <2 * spaceDim; iDim++) + xBound[iDim] = xBoundLocal[iDim]; + + +} + +void BlockParticles::SetGlobalBound(Real* xBound) { + + for (int iDim = 0; iDim < 2* spaceDim; iDim++) + xBoundGlobal[iDim] = xBound[iDim]; +#ifdef CPU +#if DebugLevel >= 2 + printf("Block %d at rank %d Global box ",currentBlock.ID(), ops_get_proc()); + for (int iDim = 0; iDim < spaceDim; iDim++) + printf("[%f %f] ", xBoundGlobal[2 * iDim], xBoundGlobal[2* iDim + 1]); +#endif +#endif +} + +void BlockParticles::GetOwnership() { + + owned = true; +#ifdef OPS_MPI + int blockIndex = currentBlock.ID(); + + sub_block_list sb = OPS_sub_block_list[blockIndex]; + if (sb->owned) + owned = true; + else + owned = false; +#endif +} + +void BlockParticles::ExtractDragForce(Real* Fd, Real* Td,int iParticle) { + + particleList.at(iParticle).PushDrag(Fd, Td); + +} + +void BlockParticles::ExtractPositions(Real* xPos,int iParticle) { + + particleList.at(iParticle).GetParticlePositions(xPos); +} + +void BlockParticles::GetAdditionalOutputVariables(std::vector& output,int iParticle) { + + particleList.at(iParticle).GetOutputVariables(output); +} + +void BlockParticles::GetAdditionalInputVariables(std::vector& input,int idParticle) { + + particleList.at(idParticle).SetInputVariables(input); +} diff --git a/Apps/LBM-DEM/block_particles.h b/Apps/LBM-DEM/block_particles.h new file mode 100644 index 0000000..29ce972 --- /dev/null +++ b/Apps/LBM-DEM/block_particles.h @@ -0,0 +1,117 @@ +/** + * Copyright 2019 United Kingdom Research and Innovation + * + * Authors: See AUTHORS + * + * Contact: [jianping.meng@stfc.ac.uk and/or jpmeng@gmail.com] + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice + * this list of conditions and the following disclaimer in the documentation + * and or other materials provided with the distribution. + * 3. Neither the name of the copyright holder nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * ANDANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +/*! + * @brief Class for handling particles at various blocks + * @author C. Tsigginos + */ +#ifndef BLOCK_PARTICLES_H_ +#define BLOCK_PARTICLES_H_ + +#include +#include "particle.h" +#include "type.h" +#include "block.h" +#include "scheme.h" +#include "flowfield.h" +#include "ops_lib_core.h" +#include "dem_data.h" +#include +#include +#ifdef OPS_MPI +#include "ops_mpi_core.h" +#endif +//#include "block_particles_wrapper.h" + +class BlockParticles { + + public: + int NParticles; //number of particles owned by the block + int NPeriodic; //Periodic particle inserted in the block + int Nmax; //Maximum particles inserted in the given block + bool owned; //Block is owned or not + std::vector particleList; //Particle data + BlockParticles(int spacedim, Real dx1, Real cutoff,const Block& myBlock, + ParticleShapeDiscriptor particleType, + int nInputExtraVariables = 0, int nOutputExtraVariables = 0); + ~BlockParticles(); + int CheckDistanceBlock(); + void InitializeDragForce(); + void EvaluateDragForce(Real dt); + void ExtractDragForce(Real* Fd, Real* Td,int iParticle); + void ExtractPositions(Real* xPos,int iParticle); + int UpdateParticle(Real* xpos, Real radius, std::vector shape); + int InsertParticle(Real* xpos, Real radius, std::vector shape, + Real* uPart, Real* omegaT, std::vector& inputData); + void UpdateParticleVelocities(Real* uPart,Real* omegaT); + void FindStencil(); + void FindBoxLocalBound(); + static void KerCarBound(const ACC& xf, Real* xb,const int* spacedim); + void GetLocalBound(Real* xBound); + void SetGlobalBound(Real* xBound); + void GetOwnership(); + ParticleShapeDiscriptor GetParticleShape() { return particleShape;}; + void UpdateOldParticlePosition(); + void ExtractBound(Real* xMin, Real* xMax); + void GetAdditionalOutputVariables(std::vector& output,int iParticle); + void GetAdditionalInputVariables(std::vector& inputVariables,int idParticle); + bool OwnedStatus() { return owned;}; + bool HasExtraInput() { return hasExtraInputVariables;}; + bool HasExtraOutput() {return hasExtraOutputVariables;}; + + + + inline void ClearParticles() {NParticles = 0; + NPeriodic = 0;}; + const Block& GetBlock() { return currentBlock; }; + private: + int spaceDim; //Spatial size + Real cutOff; //Distance for not updating the particle mapping + Real xBoundGlobal[6]; //Region of block + Real xBoundLocal[6]; //Region of block owned by this rank + Real dx; //Grid size + int Nf[6]; //Part of the block in grid nodes + int NReserve; //Maximum size that resrved for particleList + const Block& currentBlock; //The block to which the particle list is mapped + ParticleShapeDiscriptor particleShape; //Type of particles + bool SphereParallepipedIntersection(Real* xpos,Real radius); + bool hasExtraInputVariables; //Additional input parameters (i.e. T) + bool hasExtraOutputVariables; //Output parameters other than (Fd, Td) + int extraInputSize; + int extraOutputSize; +}; + + +using ListBlockParticles = std::map; +extern ListBlockParticles BlockParticleList; +#endif + diff --git a/Apps/LBM-DEM/block_particles.inc b/Apps/LBM-DEM/block_particles.inc new file mode 100644 index 0000000..6a5d576 --- /dev/null +++ b/Apps/LBM-DEM/block_particles.inc @@ -0,0 +1,71 @@ +/** + * Copyright 2019 United Kingdom Research and Innovation + * + * Authors: See AUTHORS + * + * Contact: [jianping.meng@stfc.ac.uk and/or jpmeng@gmail.com] + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice + * this list of conditions and the following disclaimer in the documentation + * and or other materials provided with the distribution. + * 3. Neither the name of the copyright holder nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * ANDANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +/*! + * @brief Class for handling particles at various blocks + * @author C. Tsigginos + * @Details Kernel function for finding box Bounds + */ + +#ifndef BLOCK_PARTICLES_INC_ +#define BLOCK_PARTICLES_INC_ + +#include "ops_lib_core.h" +#ifdef OPS_MPI +#include "ops_mpi_core.h" +#endif +#include "type.h" +#include "block_particles.h" + +void BlockParticles::KerCarBound(const ACC& xf, Real* xb, const int* spacedim) { + + int size = (* spacedim); + + + for (int idx = 0; idx < size; idx++) { +#ifdef OPS_2D + xb[idx] = xf(idx, 0, 0); +#endif + +#ifdef OPS_3D + xb[idx] = xf(idx, 0 , 0, 0); +#endif + + } + + + +} + + + +#endif /* APPS_LBM_DEM_BLOCK_PARTICLES_INC_ */ diff --git a/Apps/LBM-DEM/box_handling.cpp b/Apps/LBM-DEM/box_handling.cpp new file mode 100644 index 0000000..b6ace68 --- /dev/null +++ b/Apps/LBM-DEM/box_handling.cpp @@ -0,0 +1,176 @@ +/* + * box_handling.cpp + * + * Created on: Sep 18, 2020 + * Author: jpd38567 + */ +#include "box_handling.h" +#include +#include +#include + + +Real* x_bounds; +Real* xBoundLocal; +int* Nf; + + +void allocateBlockVariables(int blockNum) { + + x_bounds= new Real[2 * SPACEDIM * blockNum ]; + xBoundLocal = new Real [2 * SPACEDIM * blockNum]; + Nf = new int[2 * SPACEDIM * blockNum]; +} + +void deleteBlockVariables() { + + delete[] x_bounds; + delete[] xBoundLocal; + delete[] Nf; + +} + +void FindingBoxBound() { + + int Nstep_cr; + int start[SPACEDIM], end[SPACEDIM], range[SPACEDIM], disp[SPACEDIM]; + int* iterRng = (int *) malloc(2 * SPACEDIM *sizeof(int)); + int* BlockIterRngWhole = IterRngWhole(); + int* BlockIterLocal = (int *) malloc (2* SPACEDIM *sizeof(int)); + Real dx = dxLBM(); + Real xb[SPACEDIM]; + + + + std::numeric_limits::max(); + for (int blockIndex = 0; blockIndex < BlockNum(); blockIndex++) { + xBoundLocal[2 * SPACEDIM * blockIndex] = std::numeric_limits::max(); + xBoundLocal[2 * SPACEDIM * blockIndex + 1] = -1.0 * std::numeric_limits::max(); + xBoundLocal[2 * SPACEDIM * blockIndex + 2] = std::numeric_limits::max(); + xBoundLocal[2 * SPACEDIM * blockIndex + 3] = -1.0 * std::numeric_limits::max(); + if (SPACEDIM == 3) { + xBoundLocal[2 * SPACEDIM * blockIndex + 4] = std::numeric_limits::max(); //10000000.0; + xBoundLocal[2 * SPACEDIM * blockIndex + 5] = -std::numeric_limits::min(); + } +#ifdef OPS_MPI + sub_block_list sb = OPS_sub_block_list[blockIndex]; + if (!sb->owned) continue; +#endif + + BlockIterLocal[0] = BlockIterRngWhole[2 * SPACEDIM * blockIndex]; + BlockIterLocal[1] = BlockIterRngWhole[2 * SPACEDIM * blockIndex + 1]; + BlockIterLocal[2] = BlockIterRngWhole[2 * SPACEDIM * blockIndex + 2]; + BlockIterLocal[3] = BlockIterRngWhole[2 * SPACEDIM * blockIndex + 3]; + if (SPACEDIM == 3) { + BlockIterLocal[4] = BlockIterRngWhole[2 * SPACEDIM * blockIndex + 4]; + BlockIterLocal[5] = BlockIterRngWhole[2 * SPACEDIM * blockIndex + 5]; + } + + ops_get_abs_owned_range(g_Block[blockIndex], BlockIterLocal, start, end, disp); + + for (int idir = 0; idir < SPACEDIM; idir++) { + Nf[2 * SPACEDIM * blockIndex + 2 * idir] = start[idir]; + Nf[2 * SPACEDIM * blockIndex + 2 * idir + 1] = end[idir]; + } + +// ops_printf("Nf [%d %d %d %d %d]\n",Nf[0], Nf[1], Nf[2], Nf[3], Nf[4], Nf[5]); +// printf("Start %d %d %d\n", start[0], start[1], start[2]); +// printf("End %d %d %d\n", end[0], end[1], end[2]); + + //1st point finding (xmin, ymin, zmin) + + + iterRng[0] = Nf[0]; + iterRng[1] = Nf[0] + 1; + iterRng[2] = Nf[2]; + iterRng[3] = Nf[2] + 1; + if (SPACEDIM==3) { + iterRng[4] = Nf[4]; + iterRng[5] = Nf[4] + 1; + } + for (int idir = 0; idir < SPACEDIM; idir++) + xb[idir] = -10000.0; + + ops_par_loop(KerCartBounds,"KerCartBounds", g_Block[blockIndex], SPACEDIM, iterRng, + ops_arg_gbl(xb,SPACEDIM, "double", OPS_READ), + ops_arg_dat(g_CoordinateXYZ[blockIndex], SPACEDIM, LOCALSTENCIL, "double", OPS_READ)); + + + xBoundLocal[2 * SPACEDIM * blockIndex] = xb[0] - 0.5 * dx; + xBoundLocal[2 * SPACEDIM * blockIndex + 2] = xb[1] - 0.5 * dx; + + if (SPACEDIM==3) + xBoundLocal[2 * SPACEDIM * blockIndex + 4] = xb[2] - 0.5 * dx; + + //2nd point (xmax,ymax, zmax) + iterRng[0] = Nf[1] - 1; + iterRng[1] = Nf[1]; + iterRng[2] = Nf[3] - 1; + iterRng[3] = Nf[3]; + if (SPACEDIM==3) { + iterRng[4] = Nf[5] - 1; + iterRng[5] = Nf[5]; + } + + for (int idir = 0; idir < SPACEDIM; idir++) + xb[idir] = -10000.0; + + ops_par_loop(KerCartBounds,"KerCartBounds", g_Block[blockIndex], SPACEDIM, iterRng, + ops_arg_gbl(xb,SPACEDIM, "double", OPS_READ), + ops_arg_dat(g_CoordinateXYZ[blockIndex], SPACEDIM, LOCALSTENCIL, "double", OPS_READ)); + + xBoundLocal[2 * SPACEDIM * blockIndex + 1] = xb[0] + 0.5 * dx; + xBoundLocal[2 * SPACEDIM * blockIndex + 3] = xb[1] + 0.5 * dx; + if (SPACEDIM == 3) + xBoundLocal[2 * SPACEDIM * blockIndex + 5] = xb[2] + 0.5 * dx; + +// printf("proc %d Nf [%d %d] [%d %d] [%d %d]\n", ops_get_proc(), Nf[0], Nf[1], Nf[2], Nf[3], Nf[4], Nf[5]); +// printf("proc %d xbound_local x=[%f %f] y=[%f %f] z=[%f %f]\n", ops_get_proc(), xBoundLocal[2 * SPACEDIM * blockIndex], +// xBoundLocal[2 * SPACEDIM * blockIndex + 1], xBoundLocal[2 * SPACEDIM * blockIndex + 2], xBoundLocal[2 * SPACEDIM * blockIndex + 3], +// xBoundLocal[2 * SPACEDIM * blockIndex + 4], xBoundLocal[2 * SPACEDIM * blockIndex + 5]); + +#ifdef OPS_MPI + Real minMaxTmp[SPACEDIM]; + Real minMaxLoc[SPACEDIM]; + + //x- + minMaxLoc[0] = xBoundLocal[2 * LATTDIM * blockIndex + 0]; + minMaxLoc[1] = xBoundLocal[2 * LATTDIM * blockIndex + 2]; + if (SPACEDIM == 3) + minMaxLoc[2] = xBoundLocal[2 * LATTDIM * blockIndex + 4]; + + MPI_Allreduce(minMaxLoc, minMaxTmp, SPACEDIM, MPI_DOUBLE, MPI_MIN, OPS_MPI_GLOBAL); + + x_bounds[2 * LATTDIM * blockIndex] = minMaxTmp[0]; + x_bounds[2 * LATTDIM * blockIndex + 2] = minMaxTmp[1]; + x_bounds[2 * LATTDIM * blockIndex + 4] = minMaxTmp[2]; + + //Finding x_bounds maximum values + minMaxLoc[0] = xBoundLocal[2 * LATTDIM * blockIndex + 1]; + minMaxLoc[1] = xBoundLocal[2 * LATTDIM * blockIndex + 3]; + if (SPACEDIM == 3) + minMaxLoc[2] = xBoundLocal[2 * LATTDIM * blockIndex + 5]; + + MPI_Allreduce(minMaxLoc, minMaxTmp, SPACEDIM, MPI_DOUBLE, MPI_MAX, OPS_MPI_GLOBAL); + + x_bounds[2 * LATTDIM * blockIndex + 1] = minMaxTmp[0]; + x_bounds[2 * LATTDIM * blockIndex + 3] = minMaxTmp[1]; + if (SPACEDIM == 3) + x_bounds[2 * LATTDIM * blockIndex + 5] = minMaxTmp[2]; +#else + for (int iIndex = 0; iIndex < 2 * SPACEDIM; iIndex++) { + x_bounds[2 * LATTDIM * blockIndex + iIndex] = xBoundLocal[2 * LATTDIM * blockIndex + iIndex]; + } +#endif + +// printf("Block: %d Rank %d x_bounds[%f %f] [%f %f] [%f %f]\n",blockIndex, ops_get_proc(), x_bounds[2 * LATTDIM * blockIndex], +// x_bounds[2 * LATTDIM * blockIndex + 1], x_bounds[2 * LATTDIM * blockIndex + 2], x_bounds[2 * LATTDIM * blockIndex +3], +// x_bounds[2 * LATTDIM * blockIndex +4], x_bounds[2 * LATTDIM * blockIndex +5]); + } + + free(iterRng); + free(BlockIterLocal); + + +} + diff --git a/Apps/LBM-DEM/box_handling.h b/Apps/LBM-DEM/box_handling.h new file mode 100644 index 0000000..ee308c7 --- /dev/null +++ b/Apps/LBM-DEM/box_handling.h @@ -0,0 +1,57 @@ +/** + * Copyright 2019 United Kingdom Research and Innovation + * + * Authors: See AUTHORS + * + * Contact: [jianping.meng@stfc.ac.uk and/or jpmeng@gmail.com] + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice + * this list of conditions and the following disclaimer in the documentation + * and or other materials provided with the distribution. + * 3. Neither the name of the copyright holder nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * ANDANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +/*! + * @brief Functions for LBM box handling + * @author Chrysovalantis Tsigginos + * @details Default functions for handling blocks + */ + + +#ifndef BOX_HANDLING_H_ +#define BOX_HANDLING_H_ + +#include "type.h" +#include "flowfield.h" + +extern Real* x_bounds; +extern Real* xBoundLocal; +extern int* Nf; //Bounding cells for each process and block + + +void allocateBlockVariables(int blockNum); +void deleteBlockVariables(); +void FindingBoxBound(); + + + +#endif /* APPS_LBM_DEM_CLASS_INTERFACE_BOX_HANDLING_H_ */ diff --git a/Apps/LBM-DEM/cavity3d_kernel.inc b/Apps/LBM-DEM/cavity3d_kernel.inc new file mode 100644 index 0000000..8cfb033 --- /dev/null +++ b/Apps/LBM-DEM/cavity3d_kernel.inc @@ -0,0 +1,44 @@ +/** + * Copyright 2019 United Kingdom Research and Innovation + * + * Authors: See AUTHORS + * + * Contact: [jianping.meng@stfc.ac.uk and/or jpmeng@gmail.com] + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice + * this list of conditions and the following disclaimer in the documentation + * and or other materials provided with the distribution. + * 3. Neither the name of the copyright holder nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * ANDANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. +*/ + +#ifndef CAVITY3D_KERNEL_INC +#define CAVITY3D_KERNEL_INC + +void KerSetInitialMacroVars(ACC& rho, ACC& u, ACC& v, + ACC& w, const ACC& coordinates, + const int* idx) { + rho(0, 0, 0) = 1; + u(0, 0, 0) = 0; + v(0, 0, 0) = 0; + w(0, 0, 0) = 0; +} +#endif // CAVITY3D_KERNEL_INC diff --git a/Apps/LBM-DEM/dem_data.h b/Apps/LBM-DEM/dem_data.h new file mode 100644 index 0000000..bb8440f --- /dev/null +++ b/Apps/LBM-DEM/dem_data.h @@ -0,0 +1,64 @@ +/** + * Copyright 2019 United Kingdom Research and Innovation + * + * Authors: See AUTHORS + * + * Contact: [jianping.meng@stfc.ac.uk and/or jpmeng@gmail.com] + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice + * this list of conditions and the following disclaimer in the documentation + * and or other materials provided with the distribution. + * 3. Neither the name of the copyright holder nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * ANDANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +/*! + * @brief Types used for DEM-LBM simulations + * @author C. Tsigginos + */ + +#ifndef DEM_DATA_H_ +#define DEM_DATA_H_ + +#include "type.h" + + +enum ParticleShapeDiscriptor{spherical, quadratic, mesh}; + +struct InteractionData { + SizeType nSteps; + SizeType nStart; + bool restartFlag; + bool muiFlag; + Real dtDEM; + Real convergenceRate; + SizeType Nf; //number of DEM iterations + SizeType Npl; //Need to give definition + SizeType checkPeriod; + SizeType maxIters; //maximum steps for Steady state simulations + SizeType checkPeriodStS; + ParticleShapeDiscriptor particleShape; + +}; + + + +#endif /* APPS_LBM_DEM_INTERACTION_DATA_H_ */ diff --git a/Apps/LBM-DEM/dem_handle.cpp b/Apps/LBM-DEM/dem_handle.cpp new file mode 100644 index 0000000..f7385cf --- /dev/null +++ b/Apps/LBM-DEM/dem_handle.cpp @@ -0,0 +1,595 @@ + +/** + * Copyright 2019 United Kingdom Research and Innovation + * + * Authors: See AUTHORS + * + * Contact: [jianping.meng@stfc.ac.uk and/or jpmeng@gmail.com]s + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice + * this list of conditions and the following disclaimer in the documentation + * and or other materials provided with the distribution. + * 3. Neither the name of the copyright holder nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * ANDANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. +*/ + +/*! @brief Define functions and variables for handling particles + * @author C. Tsigginos + * + * @details: Functions for handling particle data, function for iterating functions + * in DEM-LBM coupled simulations + */ + +#include "dem_handle.h" +#include "flowfield_host_device.h" +#include "boundary.h" +#include "mui_wrapper.h" +#include +#include +#include "dem_data.h" +#include + +void SetDemLbMParams(InteractionData* data, bool flag, bool muiOn, Real convergeRate, + SizeType checkperiod, SizeType timeStep, SizeType checkPeriodSteady, + SizeType maximumIterations, std::string particleType) { + + data->restartFlag = flag; + if (data->restartFlag) + data->nStart = timeStep; + else + data->nStart = 0; + + data->muiFlag = muiOn; + data->convergenceRate = convergeRate; + data->checkPeriod = checkperiod; + data->checkPeriodStS = checkPeriodSteady; + data->maxIters = maximumIterations; + //Finding Particle type + std::string s1{"spherical"}; + std::string s2{"quadratic"}; + std::string s3{"mesh"}; + if (particleType.compare(s1) == 0) + data->particleShape = spherical; + else if (particleType.compare(s2) == 0) + data->particleShape = quadratic; + else if (particleType.compare(s3) == 0) + data->particleShape = mesh; + else { + ops_printf("MPLB: This type of particle is not supported\n"); + ops_printf("MPLB: Reversing to spherical particles\n"); + data->particleShape = spherical; + } +} + + + +/*void DefineInteractionModel(std::vector FluidParticleInteractionType, + int* porosModel, std::vector fsiCompoId, Real* forceUser, + double gamma, SizeType timeStep) { + + int noComp = FluidParticleInteractionType.size(); + int noFSICompo = fsiCompoId.size(); + FsiBase* model; + int idCompo; + + std::map components = g_Components(); + int numComponents = ComponentNum(); + + if (noComp != numComponents) { + ops_printf("Error: Number of FSI components inconsistent with actual components\n"); + assert(noComp != numComponents); + } + + if (noFSICompo != numComponents) { + ops_printf("Error: Number of fsiCompoId differs from actual number of components\n"); + assert(noFSICompo != numComponents); + } + + for (int iComp = 0; iComp < noComp; iComp++) { + + switch (FluidParticleInteractionType[iComp]) { + case Model_None: + model = new FsiBase( c.at(iComp), SpaceDim(), forceUser, false, porosModel[0], gamma); + break; + case Model_PSM: + model = new Psm(c.at(iComp), SpaceDim(),true , porosModel[0], gamma, porosModel[1], porosModel[2], + porosModel[3]); + break; + case Model_Prati: + model = new Prati(c.at(iComp), SpaceDim(), true, porosModel[0], gamma, porosModel[1], porosModel[2], + porosModel[3]); + break; + default: + ops_printf("The chosen model is not supported\n"); + exit(EXIT_FAILURE); + } + idCompo = components.at(iComp).id; + fluidPartInteractionModels.emplace(idCompo, model); //TODO DEfine it + + } + +} +*/ + +void SetupBlockParticles(InteractionData data, SizeType maxStep) { + + //Defined if block is owned + DefineBlockOwnership(); + + //SetupGlobalBox: + DefineLocalBoxBound(); + DefineGlobalBlockBox(); + + if (data.muiFlag) + SetMuiDomains( maxStep); + +} +void UserDefineInputOutputParams(std::vector& inputDemParams, + std::vector& outPutDemParams) { + +} +void CreateMuiInterface(InteractionData data, std::vector models, Real skin) { + + + std::vector inputDemParams; + std::vector outputDemParams; + std::vector particleParams; + + if (data.particleShape== quadratic) { + particleParams.push_back("A"); + particleParams.push_back("B"); + particleParams.push_back("C"); + particleParams.push_back("a1"); + particleParams.push_back("b1"); + particleParams.push_back("c1"); + + //quaternions: Particle rotations + particleParams.push_back("q1"); + particleParams.push_back("q2"); + particleParams.push_back("q3"); + } + + UserDefineInputOutputParams(inputDemParams, outputDemParams); + + DefineMuiInterface(&BlockParticleList, + inputDemParams, particleParams, outputDemParams,skin); + + + +} + +void SetupParticleBoxes(InteractionData data) { + + + DefineBlockOwnership(); + + + + DefineLocalBoxBound(); + + DefineGlobalBlockBox(); + + if (data.muiFlag) + SetMuiDomains(100000000); + + +} + +void DefineGlobalBlockBox() { + int spaceDim1 = SpaceDim(); + Real xMinMaxTmp[spaceDim1]; + Real xb[2 * spaceDim1], xMin[spaceDim1], xMax[spaceDim1], xTemp[2 * spaceDim1]; + for (auto& idBlock : BlockParticleList) { + BlockParticles& particles = idBlock.second; + const int blockIdx{idBlock.first}; + bool owned = particles.OwnedStatus(); + + if (owned) { + particles.GetLocalBound(xb); + + for (int iDim = 0; iDim < spaceDim1; iDim++) { + xMin[iDim] = xb[2 * iDim]; + xMax[iDim] = xb[2 * iDim + 1]; + } + } + else { + for (int iDim = 0; iDim < spaceDim1; iDim++) { + xMin[iDim] = std::numeric_limits::max(); + xMax[iDim] = -1.0 * std::numeric_limits::max(); + } + } + + +#ifdef OPS_MPI + + if (ops_num_procs()>1) { + + MPI_Allreduce(xMin, xMinMaxTmp, spaceDim1, MPI_DOUBLE, MPI_MIN, OPS_MPI_GLOBAL); + + for (int iDim = 0; iDim < spaceDim1; iDim++) { + xTemp[2 * iDim] = xMinMaxTmp[iDim]; + } + + MPI_Allreduce(xMax, xMinMaxTmp, spaceDim1, MPI_DOUBLE, MPI_MAX, OPS_MPI_GLOBAL); + for (int iDim = 0; iDim < spaceDim1; iDim++) { + xTemp[2 * iDim + 1] = xMinMaxTmp[iDim]; + } + } + else { + for (int iDim = 0; iDim < 2 * spaceDim1; iDim++) + xTemp[iDim] = xb[iDim]; + } + +#else + for (int iDim = 0; iDim < 2 * spaceDim1; iDim++) + xTemp[iDim] = xb[iDim]; +#endif + +#ifdef CPU +#if DebugLevel >= 2 + ops_printf("----------------------------------------------------------------------\n"); + printf(" Block %d: ", particles.GetBlock().ID()); + for (int iDir = 0; iDir < spaceDim1; iDir++) + printf(" [%f %f] ", xTemp[2*iDir], xTemp[2 * iDir + 1]); + printf("\n"); + +#endif +#endif + particles.SetGlobalBound(xTemp); + } +} + + +void StreamCollisionFSI3D(int flag) { + +#if DebugLevel>1 + ops_printf("Particle mapping\n"); +#endif + + ParticleMapping(flag); + +#if DebugLevel >= 1 + ops_printf("Calculating the macroscopic variables...\n"); +#endif + + UpdateMacroVars3D(); + + + CopyBlockEnvelopDistribution3D(g_fStage(), g_f()); + + PostVelocityFSIFunctions(); + +/* + * TODO Shift to bodyForce term global calculation + */ +/*#if DebugLevel >= 1 + ops_printf("Calculating the mesoscopic body force term...\n"); +#endif + UpdateMacroscopicBodyForce(time); + PreDefinedBodyForce3D(); + #if DebugLevel >= 1 + ops_printf("Calculating the collision term...\n"); + #endif +*/ + +#if DebugLevel >=1 + ops_printf("PreCollision FSI functions\n"); +#endif + PreCollisionFSIFunctions(); + + +#if DebugLevel >=1 + ops_printf("Collisions in FSI model\n"); +#endif + FluidParticleCollisions(); + + + //PreDefinedCollision3D(); + +#if DebugLevel >= 1 + ops_printf("Updating the halos...\n"); +#endif + + TransferHalos(); + +#if DebugLevel >= 1 + ops_printf("Streaming...\n"); +#endif + + Stream3D(); + +#if DebugLevel >= 1 + ops_printf("PostStreaming FSI functions\n"); +#endif + PostStreamingFSIFunctions(); + + +#if DebugLevel >= 1 + ops_printf("Implementing the boundary conditions...\n"); +#endif + // TODO This function shall be inside evolution3D.cpp + // TODO The data structure for BCs shall be inside boundary module + ImplementBoundary3D(); + +#if DebugLevel >=1 + ops_printf("Calculate Drag Force\n"); +#endif + CalculateParticleMomentum(); + + +} + +void IterateFSI(InteractionData data, int savingFlag) { + + Real dtD; + int flag{1}; + if (data.Nf <= 1) + dtD = TimeStep(); + else + dtD = data.dtDEM; + + SizeType initStep = data.nStart; + SizeType currentStep = initStep; + for (SizeType step = 0; step < data.nSteps; step++) { + currentStep++; + + if (step == 0) { + if (data.restartFlag) + flag = 0; + } + ops_printf("Current step %d\n", currentStep); + if ((currentStep % data.Npl)==0) { + if (data.muiFlag) { + ExtractParticleData(currentStep); + } + InitializeDragForce(); + + for (SizeType jfl = 0; jfl < data.Nf; jfl++) { + StreamCollisionFSI3D(flag); + } + } + flag = 0; + CalculateDragForce(dtD, currentStep); + + if (data.muiFlag) { + SendParticleData(currentStep); + } + + if (savingFlag==1) { + if ((currentStep % data.checkPeriod) == 0) { + + UpdateFPIVelocities3D(); + + CalcResidualError3D(); + DispResidualError3D(currentStep, data.checkPeriod); + + WriteFlowfieldToHdf5(currentStep); + WriteDistributionsToHdf5(currentStep); + WriteNodePropertyToHdf5(currentStep); + + //TODO ADD FSI distributions + WriteFPIDataToHdf5(currentStep); + } + } + } + + if (savingFlag == 1) { + UpdateFPIVelocities3D(); + WriteFlowfieldToHdf5(currentStep); + WriteDistributionsToHdf5(currentStep); + WriteNodePropertyToHdf5(currentStep); + + //TODO ADD FSI distributions + WriteFPIDataToHdf5(currentStep); + } + + + +} + +void IterateFSI(Real convergenceRate,const SizeType checkPointPeriod,const SizeType maxIters) { + + Real dtD = TimeStep(); + + + InitializeDragForce(); + + ParticleMapping(1); + + Real residualError{1}; + SizeType iter = 0; + do { + InitializeDragForce(); + + StreamCollisionFSI3D(0); + iter +=1; + if ((iter % checkPointPeriod) == 0) { + UpdateFPIVelocities3D(); + CalcResidualError3D(); + residualError = GetMaximumResidual(checkPointPeriod); + DispResidualError3D(iter, checkPointPeriod); + printf("Iter %d: Error = %12.9e Target = %12.9e\n", iter,residualError, convergenceRate); + } + + CalculateDragForce(TimeStep(), 0); + if (iter > maxIters) + break; + + + + } while (residualError >= convergenceRate); + + UpdateFPIVelocities3D(); + WriteFlowfieldToHdf5(0); + WriteDistributionsToHdf5(0); + WriteNodePropertyToHdf5(0); + WriteFPIDataToHdf5(0); + +} + +void SetupRestartSimulation(SizeType timestep, SizeType endStep) { + + UpdateRegions(timestep, endStep); + + UpdateOldParticleLocation(); + + UpdateParticleMappingDragForceRestart(timestep); + //TODO Update FSI Schemes to be ready for the simulations +} + +void SetupDEMLBM(InteractionData& data) { + + int myRank, timeSetup; + Real alpha; + int flags[2]; + +#ifdef OPS_MPI + myRank = ops_get_proc(); +#endif + + + if (data.muiFlag) { + //TODO Pass mui data to upgrade data + ExtractParticleData(data.nStart); + ExtractSimulationData(data.nStart, data.nStart, data.nSteps, alpha, flags, + data.particleShape); + UpdateRegions(data.nStart, data.nSteps + data.nStart); + } + else { + switch(data.particleShape) { + case spherical: + ReadParticleDataSpherical(); + break; + default: + ops_printf("Only spherical particles are currently supported\n"); + } + alpha = 1.0; + data.nSteps = 100; + + } + + + if (alpha < 1.0) { + data.Npl = static_cast(1.0 / alpha); + data.Nf = 1; + } + else { + data.Npl = 1; + data.Nf = static_cast(alpha); + } + + data.dtDEM = alpha * TimeStep(); + +#ifdef CPU +#if DebugLevel >= 2 + ops_printf("Rank %d: Nf = %d Np = %d, Starting point: %d Number of steps: %d dtDEM = %f\n", + data.Nf, data.Npl, data.nStart, data.nSteps, data.dtDEM); +#endif +#endif + + + + ops_printf("Initializing LBM-DEM coupled simulation\n"); + if (!data.restartFlag) + IterateFSI(data.convergenceRate, data.checkPeriodStS, data.maxIters); + else + SetupRestartSimulation(data.nStart, data.nSteps + data.nStart); + + if (data.muiFlag) + SendParticleData(data.nStart); + + + + +} + +//Read particle Data for coupled simulations +void ReadParticleDataSpherical() { + + int myRank; + int Nparticles, Nsize; + + myRank = 0; +#ifdef OPS_MPI + myRank = ops_get_proc(); +#endif + + FILE *cfilex; + cfilex = fopen("input_particles.txt","r"); + if (cfilex == NULL) { + ops_printf("ERROR: File cannot be read\n"); + exit(EXIT_FAILURE); + } + bool flag; + + + if (myRank == 0) { + fscanf(cfilex, "%d\n", &Nparticles); + } + +#ifdef OPS_MPI + MPI_Bcast(&Nparticles, 1, MPI_INT, 0, OPS_MPI_GLOBAL); +#endif + + if (Nparticles < 1) + Nsize = 1; + else + Nsize = Nparticles; + + Real xTmp[Nsize], yTmp[Nsize], zTmp[Nsize], radTmp[Nsize]; + Real uTmp[Nsize], vTmp[Nsize], wTmp[Nsize]; + Real oxTmp[Nsize], oyTmp[Nsize], ozTmp[Nsize]; + + if (Nparticles > 0) { + + if (myRank == 0) + for (int iPar = 0; iPar < Nparticles; iPar++) { + fscanf(cfilex,"%lf %lf %lf %lf %lf %lf %lf %lf %lf %lf\n", + &xTmp[iPar], &yTmp[iPar], &zTmp[iPar], &uTmp[iPar], + &vTmp[iPar], &wTmp[iPar], &oxTmp[iPar], &oyTmp[iPar], + &ozTmp[iPar], &radTmp[iPar]); + + } + +#ifdef OPS_MPI + MPI_Bcast(xTmp, Nparticles, MPI_DOUBLE, 0, OPS_MPI_GLOBAL); + MPI_Bcast(yTmp, Nparticles, MPI_DOUBLE, 0, OPS_MPI_GLOBAL); + MPI_Bcast(zTmp, Nparticles, MPI_DOUBLE, 0, OPS_MPI_GLOBAL); + MPI_Bcast(uTmp, Nparticles, MPI_DOUBLE, 0, OPS_MPI_GLOBAL); + MPI_Bcast(vTmp, Nparticles, MPI_DOUBLE, 0, OPS_MPI_GLOBAL); + MPI_Bcast(wTmp, Nparticles, MPI_DOUBLE, 0, OPS_MPI_GLOBAL); + MPI_Bcast(oxTmp, Nparticles, MPI_DOUBLE, 0, OPS_MPI_GLOBAL); + MPI_Bcast(oyTmp, Nparticles, MPI_DOUBLE, 0, OPS_MPI_GLOBAL); + MPI_Bcast(ozTmp, Nparticles, MPI_DOUBLE, 0, OPS_MPI_GLOBAL); + MPI_Bcast(radTmp, Nparticles, MPI_DOUBLE, 0, OPS_MPI_GLOBAL); +#endif + + } + + for (int iDir = 0; iDir < Nparticles; iDir++) + printf("Rank %d: Particle %d of %d: [%f %f %f] Radius = %f\n", ops_get_proc(), iDir, Nparticles, + xTmp[iDir], xTmp[iDir], zTmp[iDir], radTmp[iDir]); + + //Assign particles to blocks + AssignParticlesToBlocksSpheres(Nparticles, xTmp, yTmp, + zTmp, radTmp, uTmp,vTmp, wTmp, oxTmp,oyTmp, ozTmp); + fclose(cfilex); +} diff --git a/Apps/LBM-DEM/dem_handle.h b/Apps/LBM-DEM/dem_handle.h new file mode 100644 index 0000000..dd4d284 --- /dev/null +++ b/Apps/LBM-DEM/dem_handle.h @@ -0,0 +1,71 @@ + +/** + * Copyright 2019 United Kingdom Research and Innovation + * + * Authors: See AUTHORS + * + * Contact: [jianping.meng@stfc.ac.uk and/or jpmeng@gmail.com]s + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice + * this list of conditions and the following disclaimer in the documentation + * and or other materials provided with the distribution. + * 3. Neither the name of the copyright holder nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * ANDANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. +*/ + +/*! @brief Define functions and variables for DEM-LBM simulations + * @author C. Tsigginos + */ + +#ifndef DEM_HANDLE_H_ +#define DEM_HANDLE_H_ + +#include "block_particles.h" +#include "flowfield.h" +#include +#include "block.h" +#include "fpi.h" +#include +#include "model.h" +#include "block_particle_helper.h" +#include "dem_data.h" +#include + + + + +void SetupBlockParticles(InteractionData data, SizeType maxStep); +void DefineGlobalBlockBox(); +void CreateMuiInterface(InteractionData data, std::vector models, Real skin); +void StreamCollisionFSI3D(int flag); +void IterateFSI(InteractionData data, int savingFlag); +void IterateFSI(Real convergenceRate,const SizeType checkPointPeriod,const SizeType maxIters); + +void SetupDEMLBM(InteractionData& data); +void ReadParticleDataSpherical(); + +void SetDemLbMParams(InteractionData* data, bool flag, bool muiOn, Real convergeRate, + SizeType checkperiod, SizeType timeStep, SizeType checkPeriodSteady, + SizeType maximumIterations, std::string particleType); + +void SetupParticleBoxes(InteractionData data); +void SetupRestartSimulation(SizeType timestep, SizeType endStep); +#endif /* APPS_LBM_DEM_DEM_HANDLE_H_ */ diff --git a/Apps/LBM-DEM/dem_handle3d.cpp b/Apps/LBM-DEM/dem_handle3d.cpp new file mode 100644 index 0000000..e47bfbd --- /dev/null +++ b/Apps/LBM-DEM/dem_handle3d.cpp @@ -0,0 +1,916 @@ +/** + * Copyright 2019 United Kingdom Research and Innovation + * + * Authors: See AUTHORS + * + * Contact: [jianping.meng@stfc.ac.uk and/or jpmeng@gmail.com] + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice + * this list of conditions and the following disclaimer in the documentation + * and or other materials provided with the distribution. + * 3. Neither the name of the copyright holder nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * ANDANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +/*! + * @brief Functions for particle handling in the LBM code + * @author Chrysovalantis Tsigginos + * @details General functions for direct DEM-LBM simulations + * User must define functions for fluid-particle interaction scheme + */ +#include "dem_handle3d.h" + + +#include "box_handling.h" +#include "mui_interface.h" + +using namespace std; +class muiInterface; + + + +Real dtDEM; +Real cutoff_distance; +int flag_vel; +int Nfl, Npl; +Real alpha; + +int muiInterfaceFlag; + +muiInterface* interface; + +int restartFlag = 0; +int restartTimeStep; + +void SetupDEMLBMcoupling() { + + Real skin = 0.001; + cutoff_distance = skin; + flag_vel = 0; +} + +Real DemTimeStep() { + + return dtDEM; +} + +void SetDemTimestep(Real timestep) { + + dtDEM = timestep; +} + +void AllocateDomainVariables() { + + int blockNum = BlockNum(); + StenList = new int[2 * SPACEDIM]; + + muiInterfaceFlag = 0; + + allocateBlockVariables(blockNum); + +} + +void AllocateParticles() { + int blockNum1 = BlockNum(); + + particleAllocation(blockNum1); + + + AllocateFluidParticleInteractionVeriables(); +} + +void SetupParticles() { + + AllocateDomainVariables(); + AllocateParticles(); + SetupDEMLBMcoupling(); + FindingBoxBound(); + +} + + + + +//Auxiliary general functions for particle mapping into fluid domain + +int CheckDistance() { + + int flag; + int localFlag = 0; + Real distance; + Real cutOffSq = cutoff_distance * cutoff_distance; + Real dx, dy, dz, dx1; + dx1 = dxLBM(); + + //printf("Rank %d: CutOffSq = %f and number of particles %d\n",ops_get_proc(), cutOffSq, Nparticles[0]); + localFlag = 1; + for (int blockIndex = 0; blockIndex < BlockNum(); blockIndex++) { +#ifdef OPS_MPI + sub_block_list sb = OPS_sub_block_list[blockIndex]; + if (!sb->owned) continue; +#endif + + if (Nparticles[blockIndex]==0) + localFlag = 0; + + for (int particleIndex = 0; particleIndex < Nparticles[blockIndex]; particleIndex++) { + dx = xp[blockIndex][particleIndex] - xp_old[blockIndex][particleIndex]; + dy = yp[blockIndex][particleIndex] - yp_old[blockIndex][particleIndex]; + dz = zp[blockIndex][particleIndex] - zp_old[blockIndex][particleIndex]; + + distance = dx * dx + dy * dy + dz * dz; + if (distance > cutOffSq * dx1 * dx1) { + localFlag = 1; + break; + } + else + localFlag = 0; + // printf("Rank %d: Particle %d Old [%f %f %f] new [%f %f %f]\n", ops_get_proc(), particleIndex, xp_old[blockIndex][particleIndex],yp_old[blockIndex][particleIndex], zp_old[blockIndex][particleIndex], + // xp[blockIndex][particleIndex], yp[blockIndex][particleIndex], zp[blockIndex][particleIndex]); + } + + } + flag = localFlag; +#ifdef OPS_MPI + MPI_Allreduce(&localFlag, &flag, 1, MPI_INT, MPI_MAX, OPS_MPI_GLOBAL); +#endif + + return flag; +} + + +void UpdateOldParticleLocation() { + + for (int blockIndex = 0; blockIndex < BlockNum(); blockIndex++) { +#ifdef OPS_MPI + sub_block_list sb = OPS_sub_block_list[blockIndex]; + if (!sb->owned) continue; +#endif + for (int iPar = 0; iPar < Nparticles[blockIndex]; iPar++) { + xp_old[blockIndex][iPar] = xp[blockIndex][iPar]; + yp_old[blockIndex][iPar] = yp[blockIndex][iPar]; + zp_old[blockIndex][iPar] = zp[blockIndex][iPar]; + } + + } +} + +void InitializeParticleOldLocation() { + + for (int blockIndex = 0; blockIndex < BlockNum(); blockIndex++) { +#ifdef OPS_MPI + sub_block_list sb = OPS_sub_block_list[blockIndex]; + if (!sb->owned) continue; +#endif + for (int iPar = 0; iPar < Nparticles[blockIndex]; iPar++) { + xp_old[blockIndex][iPar] = 0.0; + yp_old[blockIndex][iPar] = 0.0; + + zp_old[blockIndex][iPar] = 0.0; + + } + } +} + +//Particle stencil +void ParticleMapping() { + + int ix, iy, iz; + int iminl, imaxl, jminl, jmaxl, kminl, kmaxl; + int ap; + double dx = dxLBM(); + double x_m[BlockNum()], y_m[BlockNum()], z_m[BlockNum()]; + + for (int blockIndex = 0; blockIndex < BlockNum(); blockIndex++) { + x_m[blockIndex] = x_bounds[2 * SPACEDIM * blockIndex]; + y_m[blockIndex] = x_bounds[2 * SPACEDIM * blockIndex + 2]; + z_m[blockIndex] = x_bounds[2 * SPACEDIM * blockIndex + 4]; + + + } + + for (int blockIndex = 0; blockIndex < BlockNum(); blockIndex++) { +#ifdef OPS_MPI + sub_block_list sb = OPS_sub_block_list[blockIndex]; + if (!sb->owned) continue; +#endif + + //Finding particle stencil + int nlocal = Nparticles[blockIndex] + Nperiodic[blockIndex]; + for (int iPar = 0; iPar < nlocal; iPar++) { + ap = (int) ceil(Radius[blockIndex][iPar] / dx); + + ix = (int) floor((xp[blockIndex][iPar] - x_m[blockIndex]) / dx); + iy = (int) floor((yp[blockIndex][iPar] - y_m[blockIndex]) / dx); + + iz = (int) floor((zp[blockIndex][iPar] - z_m[blockIndex]) / dx); + + iminl = - ap; + imaxl = ap + 1; + jminl = - ap; + jmaxl = ap + 1; + + + kminl = -ap; + kmaxl = ap + 1; + + //Shift to global system + iminl += ix; + imaxl += ix; + jminl += iy; + jmaxl += iy; + + kminl += iz; + kmaxl += iz; + + + //Verify that particle stencil do not exceed the sub-domain. + + if (iminl < Nf[2 * SPACEDIM * blockIndex]) + iminl = Nf[2 * SPACEDIM * blockIndex]; + + if (jminl < Nf[2 * SPACEDIM * blockIndex +2]) + jminl = Nf[2 * SPACEDIM * blockIndex +2]; + + if (imaxl >= Nf[2 * SPACEDIM * blockIndex + 1]) + imaxl = Nf[2 * SPACEDIM * blockIndex + 1]; + + if (jmaxl >= Nf[2 * SPACEDIM * blockIndex + 3]) + jmaxl = Nf[2 * SPACEDIM * blockIndex + 3]; + + + + if (kminl < Nf[2 * SPACEDIM * blockIndex + 4]) + kminl = Nf[2 * SPACEDIM * blockIndex + 4]; + + if (kmaxl >= Nf[2 * SPACEDIM * blockIndex + 5]) + kmaxl = Nf[2 * SPACEDIM * blockIndex + 5]; + + ParticleList[blockIndex][2 * SPACEDIM * iPar] = iminl; + ParticleList[blockIndex][2 * SPACEDIM * iPar + 1] = imaxl; + ParticleList[blockIndex][2 * SPACEDIM * iPar + 2] = jminl; + ParticleList[blockIndex][2 * SPACEDIM * iPar + 3] = jmaxl; + + ParticleList[blockIndex][2 * SPACEDIM * iPar + 4] = kminl; + ParticleList[blockIndex][2 * SPACEDIM * iPar + 5] = kmaxl; + +/* ops_printf("Particle %d stencil: [%d %d %d %d %d %d]\n",iPar, ParticleList[blockIndex][2 * SPACEDIM * iPar + 0], + ParticleList[blockIndex][2 * SPACEDIM * iPar + 1], ParticleList[blockIndex][2 * SPACEDIM * iPar + 2], + ParticleList[blockIndex][2 * SPACEDIM * iPar + 3], ParticleList[blockIndex][2 * SPACEDIM * iPar + 4], + ParticleList[blockIndex][2 * SPACEDIM * iPar + 5]); +*/ + } + } +} + + +void CalculateDragForce(Real dt, SizeType iter) { + + int nMax; + nMax = 6; + + + for (int blockIndex = 0; blockIndex < BlockNum(); blockIndex++) { +#ifdef OPS_MPI + sub_block_list sb = OPS_sub_block_list[blockIndex]; + if (!sb->owned) continue; +#endif + for (int iPar = 0; iPar < Nparticles[blockIndex]; iPar++) { + for (int index = 0 ; index < nMax; index++) + FDrag[blockIndex][nMax * iPar + index] /= dt; + } + + } + +// if (iter % 10 == 0) +// for (int iPar = 0 ; iPar < Nparticles[0]; iPar++) { +// printf("Rank %d at iter %d: Particle %d [%f %f %f]: Fd = [%14.9e %14.9e %14.9e] and Md = [%14.9e %14.9e %14.9e]\n", ops_get_proc(),iter, iPar, xp[0][iPar], yp[0][iPar], +//zp[0][iPar], +// FDrag[0][nMax * iPar], FDrag[0][nMax * iPar + 1], FDrag[0][nMax * iPar + +//2], FDrag[0][nMax * iPar + 3], FDrag[0][nMax * iPar + 4], FDrag[0][nMax * iPar + 5]); + +// } + +} + +void InitializeDragForce() { + int nMax; + if (SPACEDIM == 2) + nMax = 3; + else if (SPACEDIM == 3) + nMax = 6; + + for (int blockIndex = 0; blockIndex < BlockNum(); blockIndex++) { +#ifdef OPS_MPI + sub_block_list sb = OPS_sub_block_list[blockIndex]; + if (!sb->owned) continue; +#endif + for (int iPar = 0; iPar < Nparticles[blockIndex]; iPar++) + for (int index =0; index < nMax; index++) + FDrag[blockIndex][nMax * iPar + index] = 0; + + } +} + +void ReadParticleDetails() { + + + FILE *cfilex; + cfilex = fopen("input_particles.txt","r"); + if (cfilex == NULL) { + ops_printf("ERROR: File cannot be read\n"); + exit(EXIT_FAILURE); + } + bool flag; + int Np; + + if (ops_get_proc()==0) + fscanf(cfilex,"%d\n",&Np); + + + MPI_Bcast(&Np, 1, MPI_INT, 0, OPS_MPI_GLOBAL); + int Np1; + if (Np < 1) { + Np1 = 1; + } + else + Np1 = Np; + + Real xTmp[Np1], yTmp[Np1], zTmp[Np1], upTmp[Np1], vpTmp[Np1], wpTmp[Np1], omegaXTmp[Np1], omegaYTmp[Np1], omegaZTmp[Np1], radTmp[Np1]; + + if (ops_get_proc()==0) { + for (int iPar = 0; iPar < Np; iPar++) + fscanf(cfilex,"%lf %lf %lf %lf %lf %lf %lf %lf %lf %lf\n", &xTmp[iPar], &yTmp[iPar], &zTmp[iPar], &upTmp[iPar], &vpTmp[iPar], &wpTmp[iPar], + &omegaXTmp[iPar], &omegaYTmp[iPar], &omegaZTmp[iPar], &radTmp[iPar]); + } + + if (Np > 0) { + MPI_Bcast(xTmp, Np, MPI_DOUBLE, 0, OPS_MPI_GLOBAL); + MPI_Bcast(yTmp, Np, MPI_DOUBLE, 0, OPS_MPI_GLOBAL); + MPI_Bcast(zTmp, Np, MPI_DOUBLE, 0, OPS_MPI_GLOBAL); + MPI_Bcast(upTmp, Np, MPI_DOUBLE, 0, OPS_MPI_GLOBAL); + MPI_Bcast(vpTmp, Np, MPI_DOUBLE, 0, OPS_MPI_GLOBAL); + MPI_Bcast(wpTmp, Np, MPI_DOUBLE, 0, OPS_MPI_GLOBAL); + MPI_Bcast(omegaXTmp, Np, MPI_DOUBLE, 0, OPS_MPI_GLOBAL); + MPI_Bcast(omegaYTmp, Np, MPI_DOUBLE, 0, OPS_MPI_GLOBAL); + MPI_Bcast(omegaZTmp, Np, MPI_DOUBLE, 0, OPS_MPI_GLOBAL); + MPI_Bcast(radTmp, Np, MPI_DOUBLE, 0, OPS_MPI_GLOBAL); + } + + + + for (int iPar = 0; iPar < Np; iPar++) { + + + for (int blockIndex = 0; blockIndex < BlockNum(); blockIndex++) { + + + flag = SphereParallepidedIntersection(xTmp[iPar], yTmp[iPar], zTmp[iPar], radTmp[iPar], xBoundLocal[2 * SPACEDIM * blockIndex], xBoundLocal[2 * SPACEDIM * blockIndex + 1], + xBoundLocal[2 * SPACEDIM * blockIndex+2], xBoundLocal[2 * SPACEDIM * blockIndex + 3], xBoundLocal[2 * SPACEDIM * blockIndex + 4], + xBoundLocal[2 * SPACEDIM * blockIndex + 5]); + + + + if (flag == true) { + + Nparticles[blockIndex] += 1; + if (Nmax[blockIndex] <= Nparticles[blockIndex]) { + ReAllocateMemory(1000, blockIndex); + } + int Np1 = Nparticles[blockIndex] - 1; + xp[blockIndex][Np1] = xTmp[iPar]; + yp[blockIndex][Np1] = yTmp[iPar]; + Radius[blockIndex][Np1] = radTmp[iPar]; + + up[blockIndex][Np1] = upTmp[iPar]; + vp[blockIndex][Np1] = vpTmp[iPar]; + omegaZ[blockIndex][Np1] = omegaZTmp[iPar]; + + zp[blockIndex][Np1] = zTmp[iPar]; + + wp[blockIndex][Np1] = wpTmp[iPar]; + omegaX[blockIndex][Np1] = omegaXTmp[iPar]; + omegaY[blockIndex][Np1] = omegaYTmp[iPar]; + +// printf("Rank %d: Particle of radius %f inserted at location [%f %f %f] with initial vel [%f %f %f] and om=[%f %f %f] \n", +// ops_get_proc(), Radius[blockIndex][Np1], xp[blockIndex][Np1], yp[blockIndex][Np1], zp[blockIndex][Np1], +// up[blockIndex][Np1], vp[blockIndex][Np1], wp[blockIndex][Np1], +// omegaX[blockIndex][Np1], omegaY[blockIndex][Np1],omegaZ[blockIndex][Np1]); +// + + } + } + + + + } + +} + +void UpdateParticlMappingDragForce(SizeType timeSetup) { + + double dt = TimeStep(); + + //Initialize the Drag force + + InitializeDragForce(); + + //map the particles + if (periodicFlag == 1) { + PeriodicExchange(); + InitializePeriodicDragForce(); + } + + ParticleMapping(); + + EstimateDragForce2ndOrder3D(); + if (periodicFlag == 1) + ReverseComm(); + + CalculateDragForce(dt, timeSetup); + + + + +} + +void InitializeDEMLBMRestart(SizeType &maxStep, SizeType timeSetup,SizeType RunStep, int savingFlag) { + + int myRank;; + int tempPerFlags[SPACEDIM]; + Real xper[SPACEDIM], xcutOff[SPACEDIM]; + long int maxStep1; + +#ifdef OPS_MPI + myRank = ops_get_proc(); +#endif + + if (muiInterfaceFlag == 1) { + long int timeSetup1 = timeSetup; + interface->extractData(timeSetup1, maxStep1, alpha, tempPerFlags); + + if (tempPerFlags[0] != periodic[0]) { + ops_printf("ERROR MP-LBM: Inconsistent B.C in x-direction between in DEM-LBM coupled simulations\n"); + exit(EXIT_FAILURE); + } + + if (tempPerFlags[1] != periodic[1]) { + ops_printf("ERROR MP-LBM: Inconsistent B.C in y-direction between in DEM-LBM coupled simulations\n"); + exit(EXIT_FAILURE); + } + + if (tempPerFlags[2] != periodic[2]) { + ops_printf("ERROR MP-LBM: Inconsistent B.C in z-direction between in DEM-LBM coupled simulations\n"); + exit(EXIT_FAILURE); + } + + if (periodicFlag==1) { + interface->extractDataPeriodic(xper, xcutOff); + SetPeriodicSize3D(xper[0], xper[1], xper[2]); + SetCutOff3D(xcutOff[0], xcutOff[1], xcutOff[2]); + } + + ExtractParticleData(timeSetup); + + maxStep = maxStep1; + } + else { + alpha = 1; + maxStep = RunStep + timeSetup; + ops_printf("Maxstep %d\n", maxStep); + ReadParticleDetails(); + + if (periodicFlag == 1) { + Real xper1, yper1, zper1; + + if (periodic[0]==1) + xper1 = BlockSize(0)[0] + dxLBM(); + else + xper1 = BlockSize(0)[0]; + + if (periodic[1]==1) + yper1 = BlockSize(0)[1] + dxLBM(); + else + yper1 = BlockSize(0)[1]; + + if (periodic[2]==1) + zper1 = BlockSize(0)[2] + dxLBM(); + else + zper1 = BlockSize(0)[3]; + + SetPeriodicSize3D(xper1, yper1, zper1); + + Real xCutOff = dxLBM(); + Real yCutOFf = dxLBM(); + Real zCutOff = dxLBM(); + + SetCutOff3D(xCutOff, yCutOFf, zCutOff); + + ops_printf("I initialize distancres\n"); + } + + + } + + UpdateOldParticleLocation(); + + if (alpha < 0.000001) { + ops_printf("alpha is equal to %f\n", alpha); + exit(EXIT_FAILURE); + } + + + if (alpha <= 1) { + Npl = static_cast(1.0/alpha); + Nfl = 1; + } + else { + Npl = 1; + Nfl = static_cast(alpha); + } + + SetDemTimestep(alpha * TimeStep()); + + + if (periodicFlag == 1) + PeriodicPartition(); + + + if (timeSetup == 0 ) { + Real convergenceRate = 1e-7; + int maxIters = 454; + IterateDEMLBMSS(convergenceRate, 10000, maxIters,savingFlag); //fix the values + + if (muiInterfaceFlag == 1) + SendParticleData(timeSetup); + } + else { + UpdateParticlMappingDragForce(timeSetup); + + if (muiInterfaceFlag ==1) + SendParticleData(timeSetup); + } + + + + +} + +void InitializeDEMLBM(Real convergenceRate, int maxIters, int checkPeriod,SizeType &maxStep,int savingFlag) { + + int myRank, timeSetup; + int tempPerFlags[SPACEDIM]; + Real xper[SPACEDIM], xcutOff[SPACEDIM]; +#ifdef OPS_MPI + myRank = ops_get_proc(); +#endif + + if (muiInterfaceFlag == 1) { + + long int timeSetup = 0; + long int maxStep1; + interface->extractData(timeSetup, maxStep1, alpha, tempPerFlags); + + maxStep = maxStep1; + + if (tempPerFlags[0] != periodic[0]) { + ops_printf("ERROR MP-LBM: Inconsistent B.C in x-direction between in DEM-LBM coupled simulations\n"); + exit(EXIT_FAILURE); + } + + if (tempPerFlags[1] != periodic[1]) { + ops_printf("ERROR MP-LBM: Inconsistent B.C in y-direction between in DEM-LBM coupled simulations\n"); + exit(EXIT_FAILURE); + } + + if (tempPerFlags[2] != periodic[2]) { + ops_printf("ERROR MP-LBM: Inconsistent B.C in z-direction between in DEM-LBM coupled simulations\n"); + exit(EXIT_FAILURE); + } + + if (periodicFlag==1) { + interface->extractDataPeriodic(xper, xcutOff); + SetPeriodicSize3D(xper[0], xper[1], xper[2]); + SetCutOff3D(xcutOff[0], xcutOff[1], xcutOff[2]); + } + + + ExtractParticleData(0); + + +// ops_printf("Rank %d: %d particles extracted\n",ops_get_proc(), Nparticles[0]); +// for (int iPar = 0; iPar < Nparticles[0]; iPar++) { +// printf("Rank %d: %d [%f %f %f] of radius %f\n", ops_get_proc(), iPar, xp[0][iPar], yp[0][iPar], zp[0][iPar],Radius[0][iPar]); +// } + + if (restartFlag == 1) { + UpdateOldParticleLocation(); + } + else + InitializeParticleOldLocation(); + + ops_printf("I Initialze MUI particles\n"); + + } + else { + alpha = 1; + maxStep = 10000; + ReadParticleDetails(); + InitializeParticleOldLocation(); + + + + if (periodicFlag == 1) { + + Real xper1 = BlockSize(0)[0] + dxLBM(); + Real yper1 = BlockSize(0)[1] + dxLBM(); + Real zper1 = BlockSize(0)[2] + dxLBM(); + + SetPeriodicSize3D(xper1, yper1, zper1); + + Real xCutOff = dxLBM(); + Real yCutOFf = dxLBM(); + Real zCutOff = dxLBM(); + + SetCutOff3D(xCutOff, yCutOFf, zCutOff); + + ops_printf("I initialize distancres\n"); + } + } + + if (alpha < 0.000001) { + ops_printf("alpha is equal to %f\n", alpha); + exit(EXIT_FAILURE); + } + + + if (alpha <= 1) { + Npl = static_cast(1.0/alpha); + Nfl = 1; + } + else { + Npl = 1; + Nfl = static_cast(alpha); + } + + SetDemTimestep(alpha * TimeStep()); + + + if (periodicFlag == 1) + PeriodicPartition(); + + + + + ops_printf("Particle set. Ready to iterate\n"); + ops_printf("restartFlag = %d\n", restartFlag); + + ops_printf("convergence Rate=% f checkPeriod = %d, maxIters = %d\n", convergenceRate, checkPeriod, maxIters); + IterateDEMLBMSS(convergenceRate, checkPeriod, maxIters, savingFlag); //fix the values + + if (muiInterfaceFlag == 1) + SendParticleData(0); + + +} +//MUI FUNCTIONS + +void DestroyParticleParams() { + + free2d(xp); + free2d(yp); + free2d(up); + free2d(vp); + free2d(omegaZ); + free2d(Radius); + + + + free2d(xp_old); + free2d(yp_old); + + + free2d(zp); + free2d(wp); + free2d(omegaX); + free2d(omegaY); + free2d(zp_old); + + + free2d(FDrag); + + free(Nparticles); + free(Nmax); + free(Nperiodic); + + free2d(ParticleList); + delete[] StenList; + + + + deleteBlockVariables(); + + +} + +void DefineMUIInterface(Real Rmax) { + + ops_printf("MP_LBM: Ready to define uniface\n"); + + //new mui::uniface2d("mpi://LBM/ifs"); + + interface = new muiInterface(Rmax); + + + //interface = new mui::uniface3d("mpi://LBM/ifs"); + ops_printf("MP-LBM: Defined uniface3d since %dD\n", SPACEDIM); + + + + ops_printf("MP-LBM: Defined uniface %d\n", SPACEDIM); +} + +void SetupMUICommunication(int maxIter) { + + interface->setDomains(maxIter); +} + +void DestroyMUIInterface() { + + + delete interface; + + + +} + +//#ifdef OPS_3D +void ExtractParticles3D(int time) { + + vector xpTemp; + vector ypTemp; + vector zpTemp; + + vector radius; + vector upTemp; + vector vpTemp; + vector wpTemp; + vector oxTemp; + vector oyTemp; + vector ozTemp; + + + //ops_printf("Time to extract: %d\n",time); + bool flag; + int nsd = 2 * SPACEDIM; + + //for (int blockIndex = 0; blockIndex < BlockNum(); blockIndex++) { +//#ifdef OPS_MPI +// sub_block_list sb = OPS_sub_block_list[blockIndex]; +// if (!sb->owned) continue; +//#endif + + + interface->updateParticles(time, xpTemp, ypTemp, zpTemp, radius, upTemp, vpTemp, wpTemp, oxTemp, oyTemp, ozTemp); + + for (int blockIndex = 0; blockIndex < BlockNum(); blockIndex++) { + Nparticles[blockIndex] = 0; + } + + for (int iVec = 0; iVec < xpTemp.size(); ++iVec) { + + Real xtemp = xpTemp[iVec]; + Real ytemp = ypTemp[iVec]; + Real ztemp = zpTemp[iVec]; + Real radTmp = radius[iVec]; + + + + for (int blockIndex = 0; blockIndex < BlockNum(); blockIndex++) { +#ifdef OPS_MPI + sub_block_list sb = OPS_sub_block_list[blockIndex]; + if (!sb->owned) continue; +#endif + flag = SphereParallepidedIntersection(xtemp,ytemp,ztemp, radTmp, + xBoundLocal[nsd * blockIndex], xBoundLocal[nsd * blockIndex + 1], + xBoundLocal[nsd * blockIndex + 2], xBoundLocal[nsd * blockIndex + 3], + xBoundLocal[nsd * blockIndex + 4], xBoundLocal[nsd * blockIndex + 5]); + + if (flag == true) { + Nparticles[blockIndex] += 1; + if (Nmax[blockIndex] <= Nparticles[blockIndex]) + ReAllocateMemory(1000, blockIndex); + + + int Np = Nparticles[blockIndex] - 1; + xp[blockIndex][Np] = xtemp; + yp[blockIndex][Np] = ytemp; + zp[blockIndex][Np] = ztemp; + + Radius[blockIndex][Np] = radTmp; + up[blockIndex][Np] = upTemp[iVec]; + vp[blockIndex][Np] = vpTemp[iVec]; + wp[blockIndex][Np] = wpTemp[iVec]; + + omegaX[blockIndex][Np] = oxTemp[iVec]; + omegaY[blockIndex][Np] = oyTemp[iVec]; + omegaZ[blockIndex][Np] = ozTemp[iVec]; + } + } + + } + + //ops_printf("I extracted %d particles\n", Nparticles[0]); + +} + + + +void ExtractParticleData(SizeType time) { + + + int time1 = time; + + //ops_printf("Entering Extract Particles 3D\n"); + ExtractParticles3D( time1); + + +} + +void SendParticleData3D(int t) { + + interface->sendParticles(t); + +} + + +void SendParticleData(SizeType timestep) { + + + int t = timestep; + + SendParticleData3D(t); + +} + +void ForgetData(int t) { + + + +} + +//Auxiliary functions for particle mapping + + +bool SphereParallepidedIntersection(Real cx, Real cy, Real cz, Real radius, Real xbMin, + Real xbMax, Real ybMin, Real ybMax, Real zbMin, Real zbMax) { + + Real testX = cx; + Real testY = cy; + Real testZ = cz; + + if (cx < xbMin) + testX = xbMin; + else if (cx > xbMax) + testX = xbMax; + + if (cy < ybMin) + testY = ybMin; + else if (cy > ybMax) + testY = ybMax; + + if (cz < zbMin) + testZ = zbMin; + else if (cz > zbMax) + testZ = zbMax; + + Real distX = cx - testX; + Real distY = cy - testY; + Real distZ = cz - testZ; + + Real distance = distX * distX + distY * distY + distZ * distZ; + + if (distance <= radius * radius) + return true; + + return false; + +} diff --git a/Apps/LBM-DEM/dem_handle3d.h b/Apps/LBM-DEM/dem_handle3d.h new file mode 100644 index 0000000..ed1e4c8 --- /dev/null +++ b/Apps/LBM-DEM/dem_handle3d.h @@ -0,0 +1,111 @@ +/** + * Copyright 2019 United Kingdom Research and Innovation + * + * Authors: See AUTHORS + * + * Contact: [jianping.meng@stfc.ac.uk and/or jpmeng@gmail.com] + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice + * this list of conditions and the following disclaimer in the documentation + * and or other materials provided with the distribution. + * 3. Neither the name of the copyright holder nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * ANDANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +/*! + * @brief Functions for particle handling in the LBM code + * @author Chrysovalantis Tsigginos + * @details General functions for direct DEM-LBM simulations + * User must define functions for fluid-particle interaction scheme + */ + +#ifndef DEM_HANDLE3D_H_ +#define DEM_HANDLE3D_H_ + +#include +#include + +#include "dem_particles.h" +#include "type.h" +#include "flowfield.h" +#include "mui.h" +#include "memory_handle.h" +#include "iterate_dem.h" +#include "dem_particles.h" +#include "periodic.h" +#include "mui_interface.h" + +extern Real cutoff_distance; //cut-off distance for building solid fraction list +extern int flag_vel; //For violent build of solid fraction list + + +extern Real dtDEM; //DEM timestep +extern int restartFlag, restartTimeStep; //RestartFlag and TimeStep for +extern int Nfl, Npl; //Number of iterations for LBM and DEM solver before data exchange. + +extern Real alpha; // timestep ratio of DEM to LBM + +extern int muiInterfaceFlag; //Flag for activation of mui interface + + +//DEM timestep functions +void SetupDEMLBMcoupling(); +Real DemTimeStep(); + + +//Particle and FSI allocation functions +void AllocateParticles(); //Allocate memory for particle lists +void AllocateDomainVariables(); //Allocate domain variables + + +void SetupDEMLBMcoupling(); +void SetupParticles(); //Function for particle initialization +void ReadParticleDetails(); +void InitializeDEMLBM(Real convergenceRate, int maxIters, int checkPeriod,SizeType &maxStep, int SavingFlag = 1); +void UpdateParticlMappingDragForce(SizeType timeSetup); +void InitializeDEMLBMRestart(SizeType &maxStep, SizeType timeSetup,SizeType RunStep = 1000, int SavingFlag = 1); + +//Particle handling auxiliary functions +int CheckDistance(); //Decide for particle re-mapping based on distances +void UpdateOldParticleLocation(); //Particle locations during particle mapping +void InitializeParticleOldLocation(); //Set initial particle locatios +void ParticleMapping(); //Create particle stencil +void CalculateDragForce(Real dt,SizeType iter); +void InitializeDragForce(); +void DestroyParticleParams(); +//MUI Functions +void DefineMUIInterface(Real Rmax); //Initialization of MUI library +void SetupMUICommunication(int maxIter); //Setup MUI communication +void DestroyMUIInterface(); + + +void ExtractParticles3D(int time); // Particle extraction for three dimensional simulations +void ExtractParticleData(SizeType time); //Extract particle Data through the MUI library + +void SendParticleData3D(int t); //Send particle for three dimensional simulations +void SendParticleData(SizeType timestep); //Send data to DEM code + +void ForgetData(int t); //Forget data at time t +//Auxiliary MUI functions + +bool SphereParallepidedIntersection(Real cx, Real cy, Real cz, Real radius, Real xbMin, + Real xbMax, Real ybMin, Real ybMax, Real zbMin, Real zbMax); +#endif diff --git a/Apps/LBM-DEM/dem_particles.cpp b/Apps/LBM-DEM/dem_particles.cpp new file mode 100644 index 0000000..817cb9d --- /dev/null +++ b/Apps/LBM-DEM/dem_particles.cpp @@ -0,0 +1,145 @@ +/* + * dem_particles.cpp + * + * Created on: Sep 18, 2020 + * Author: jpd38567 + */ +#include "dem_particles.h" +#include +#include + +int* StenList; //Cells containing a particles +int* Nparticles; //Number of particles per block +int* Nmax; // Maximum particles that can be inserted in a block without memory re-allocation +int* Nperiodic; //Periodic particles list + +Real** xp; //Particle location in x +Real** yp; //Particle location in y +Real** zp; //Particle location iz z +Real** up; //Particle velocity in u +Real** vp; //Particle velocity in v +Real** wp; //Particle velocity in w +Real** omegaX; //Rotational velocity in x +Real** omegaY; //Rotational velocity in y +Real** omegaZ; //Rotational velocity in w +Real** Radius; //Particle radius + +Real** FDrag; //Drag Force +int** ParticleList; //Particle stencil + +Real** xp_old; //Old particle location in x +Real** yp_old; //Old particle location in y +Real** zp_old; //Old particle location in z + + +void particleAllocation(int blockNo) { + + Nparticles = (int *) malloc(blockNo * sizeof(int)); + Nmax = (int *) malloc(blockNo * sizeof(int)); + + Nperiodic = (int *) malloc(blockNo * sizeof(int)); + + xp = (Real **) malloc(blockNo * sizeof(Real *)); + yp = (Real **) malloc(blockNo * sizeof(Real *)); + + up = (Real **) malloc(blockNo * sizeof(Real *)); + vp = (Real **) malloc(blockNo * sizeof(Real *)); + + omegaZ = (Real **) malloc(blockNo * sizeof(Real *)); + Radius = (Real **) malloc(blockNo * sizeof(Real *)); + + + zp = (Real **) malloc(blockNo * sizeof(Real *)); + wp = (Real **) malloc(blockNo * sizeof(Real *)); + + omegaX = (Real **) malloc(blockNo * sizeof(Real *)); + omegaY = (Real **) malloc(blockNo * sizeof(Real *)); + + + + for (int blockIndex = 0; blockIndex < blockNo; blockIndex++) { + Nmax[blockIndex] = 10; + Nparticles[blockIndex] = 0; + Nperiodic[blockIndex] = 0; + allocateMemory(xp[blockIndex], Nmax[blockIndex], 1); + allocateMemory(yp[blockIndex], Nmax[blockIndex], 1); + + allocateMemory(up[blockIndex], Nmax[blockIndex], 1); + allocateMemory(vp[blockIndex], Nmax[blockIndex], 1); + allocateMemory(Radius[blockIndex], Nmax[blockIndex], 1); + allocateMemory(omegaZ[blockIndex], Nmax[blockIndex], 1); + + allocateMemory(zp[blockIndex], Nmax[blockIndex], 1); + allocateMemory(wp[blockIndex], Nmax[blockIndex], 1); + allocateMemory(omegaX[blockIndex], Nmax[blockIndex], 1); + allocateMemory(omegaY[blockIndex], Nmax[blockIndex], 1); + + } + +} + +void AllocateFluidParticleInteractionVeriables() { + + int blockNo = BlockNum(); + ParticleList = (int **) malloc(blockNo * sizeof(int *)); + FDrag = (Real **) malloc(blockNo * sizeof (Real *)); + xp_old = (Real **) malloc(blockNo * sizeof(Real *)); + yp_old = (Real **) malloc(blockNo * sizeof(Real *)); + + + zp_old = (Real **) malloc(blockNo * sizeof(Real *)); + + + for (int blockIndex = 0; blockIndex < blockNo; blockIndex++) { + + allocateMemory(xp_old[blockIndex], Nmax[blockIndex], 1); + allocateMemory(yp_old[blockIndex], Nmax[blockIndex], 1); + allocateMemory(zp_old[blockIndex], Nmax[blockIndex], 1); + + allocateMemory(ParticleList[blockIndex], Nmax[blockIndex], 6); + allocateMemory(FDrag[blockIndex], Nmax[blockIndex], 6); + + } + +} + + +void ReallocateMemoryFSI(int blockIndex) { + Reallocate(xp_old[blockIndex], Nmax[blockIndex], 1); + Reallocate(yp_old[blockIndex], Nmax[blockIndex], 1); + + + Reallocate(zp_old[blockIndex], Nmax[blockIndex], 1); + Reallocate(ParticleList[blockIndex], Nmax[blockIndex], 6); + Reallocate(FDrag[blockIndex], Nmax[blockIndex], 6); + +} + +void ReAllocateMemory(int Np, int blockIndex) { + Nmax[blockIndex] += Np; + + Reallocate(xp[blockIndex], Nmax[blockIndex], 1); + Reallocate(up[blockIndex], Nmax[blockIndex], 1); + + Reallocate(yp[blockIndex], Nmax[blockIndex], 1); + Reallocate(vp[blockIndex], Nmax[blockIndex], 1); + + Reallocate(omegaZ[blockIndex], Nmax[blockIndex], 1); + + + Reallocate(zp[blockIndex], Nmax[blockIndex], 1); + Reallocate(wp[blockIndex], Nmax[blockIndex], 1); + + Reallocate(omegaX[blockIndex], Nmax[blockIndex], 1); + Reallocate(omegaY[blockIndex], Nmax[blockIndex], 1); + + + Reallocate(Radius[blockIndex], Nmax[blockIndex], 1); + + ReallocateMemoryFSI(blockIndex); + +} + + + + diff --git a/Apps/LBM-DEM/dem_particles.h b/Apps/LBM-DEM/dem_particles.h new file mode 100644 index 0000000..c68ce55 --- /dev/null +++ b/Apps/LBM-DEM/dem_particles.h @@ -0,0 +1,45 @@ +/* + * DEM_particles.h + * + * Created on: Sep 18, 2020 + * Author: jpd38567 + */ + +#ifndef DEM_PARTICLES_H_ +#define DEM_PARTICLES_H_ + +#include "box_handling.h" +#include "memory_handle.h" + + +extern int* StenList; //Cells containing a particles + +extern int* Nparticles; //Number of particles per block +extern int* Nmax; // Maximum particles that can be inserted in a block without memory re-allocation +extern int* Nperiodic; //Periodic particles list + +extern Real** xp; //Particle location in x +extern Real** yp; //Particle location in y +extern Real** zp; //Particle location iz z +extern Real** up; //Particle velocity in u +extern Real** vp; //Particle velocity in v +extern Real** wp; //Particle velocity in w +extern Real** omegaX; //Rotational velocity in x +extern Real** omegaY; //Rotational velocity in y +extern Real** omegaZ; //Rotational velocity in w +extern Real** Radius; //Particle radius + +extern Real** FDrag; //Drag Force +extern int** ParticleList; //Not remembering + +extern Real** xp_old; //Old particle location in x +extern Real** yp_old; //Old particle location in y +extern Real** zp_old; //Old particle location in z + +void particleAllocation(int blockNo); +void AllocateFluidParticleInteractionVeriables(); + +void ReallocateMemoryFSI(int blockIndex); +void ReAllocateMemory(int Np, int blockIndex); + +#endif /* DEM_PARTICLES_H_ */ diff --git a/Apps/LBM-DEM/fpi.h b/Apps/LBM-DEM/fpi.h new file mode 100644 index 0000000..58d0bc6 --- /dev/null +++ b/Apps/LBM-DEM/fpi.h @@ -0,0 +1,46 @@ +/** + * Copyright 2019 United Kingdom Research and Innovation + * + * Authors: See AUTHORS + * + * Contact: [jianping.meng@stfc.ac.uk and/or jpmeng@gmail.com] + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice + * this list of conditions and the following disclaimer in the documentation + * and or other materials provided with the distribution. + * 3. Neither the name of the copyright holder nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * ANDANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +/*! @brief Head files for configuring the fluid-particle interaction model + * @author C. Tsigginos + * @details And new fluid-particle (structure) interaction model must be + * inserted here. + */ +#ifndef FPI_H_ +#define FPI_H_ + +#include "fsi_base.h" +#include "psm.h" + + + +#endif /* APPS_LBM_DEM_FPI_H_ */ diff --git a/Apps/LBM-DEM/fsi_base.cpp b/Apps/LBM-DEM/fsi_base.cpp new file mode 100644 index 0000000..7261d23 --- /dev/null +++ b/Apps/LBM-DEM/fsi_base.cpp @@ -0,0 +1,89 @@ +/** + * Copyright 2019 United Kingdom Research and Innovation + * + * Authors: See AUTHORS + * + * Contact: [jianping.meng@stfc.ac.uk and/or jpmeng@gmail.com] + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice + * this list of conditions and the following disclaimer in the documentation + * and or other materials provided with the distribution. + * 3. Neither the name of the copyright holder nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * ANDANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. +*/ + +/*! @brief Base class for fluid-particle interaction modeling + * @author C. Tsigginos + * Discription: Base class for handling the fluid-particle interactions + **/ + +#include "fsi_base.h" +#include +FsiBase::FsiBase(Component& compoUser, int spacedim, Real* forceUser, bool owned, + SolFracType porosUser, Real gammaUser) : compo{compoUser}{ + + gamma = gammaUser; + spaceDim = spacedim; + porosModel = porosUser; + + force = new Real[spaceDim]; + Real sumF = 0.0; + for (int iDim = 0; iDim < spaceDim; iDim++) { + force[iDim] = forceUser[iDim]; + sumF += abs(force[iDim]); + } + + forceFlag = 0; + if (sumF != 0.0) + forceFlag = 1; + collisionOwned = owned; + +} + +FsiBase::~FsiBase() { + + delete[] force; +} + +void FsiBase::ObtainID(int* idVel, int* loop, Real& tauCompo, CollisionType& collisModel, + int& idComponent,int& rhoId,int &thId) { + + idVel[0] = compo.uId; + idVel[1] = compo.vId; + +#if OPS_3D + idVel[2] = compo.wId; +#endif + + loop[0] = compo.index[0]; + loop[1] = compo.index[1]; + tauCompo = compo.tauRef; + collisModel = compo.collisionType; + idComponent = compo.id; + + rhoId = compo.macroVars.at(Variable_Rho).id; + + if (isThermalModel == 1) + thId = compo.macroVars.at(Variable_T).id; + else + thId = -1; + +} diff --git a/Apps/LBM-DEM/fsi_base.h b/Apps/LBM-DEM/fsi_base.h new file mode 100644 index 0000000..c310cdd --- /dev/null +++ b/Apps/LBM-DEM/fsi_base.h @@ -0,0 +1,88 @@ +/** + * Copyright 2019 United Kingdom Research and Innovation + * + * Authors: See AUTHORS + * + * Contact: [jianping.meng@stfc.ac.uk and/or jpmeng@gmail.com] + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice + * this list of conditions and the following disclaimer in the documentation + * and or other materials provided with the distribution. + * 3. Neither the name of the copyright holder nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * ANDANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. +*/ + +/*! @brief Base class for fluid-particle interaction modeling + * @author C. Tsigginos + **/ + +#ifndef FSI_BASE_H_ +#define FSI_BASE_H_ + +#include "flowfield.h" +#include +#include "block.h" +#include "type.h" +#include "model.h" + + +enum FSIType {Model_PSM = 1, Model_None = 0}; +enum SolFracType {Mode_None = 0, Mode_Spherical = 1, Mode_Grid = 2, Mode_Copy = 3}; + + +class FsiBase { + + protected: + Component compo; //Component associated with this FSI model + Real gamma; //User-defined parameter + SolFracType porosModel; //Model for mapping particles to the grid + int spaceDim; + Real* force; //Base force + int forceFlag; //Flag for force update + int isThermalModel = 0; //Thermal flag + public: + bool collisionOwned; //(true) model invoked own collision + + FsiBase(Component& compoUser, int spacedim, Real* forceUser, bool owned = false, + SolFracType porosModel = Mode_None, Real gammaUser = 0.0); + virtual ~FsiBase(); + virtual void ModelCollision() {} //inputs to be determined + virtual void PostVelocityCalculation() { } //inputs to be determined + virtual void PreCollision() { } //inputs to be de determined + virtual void PostStreaming() {} //inputs to be determined + virtual void MappingFunction(bool flag) {} + virtual void InitializeVariables() {} + virtual void CalculateDragForce() { } // inputs to be determined + virtual void SetupSimulation() { } //inputs to be determined + virtual void RestartSimulation() { } // inputs to be determined + virtual void WriteToHdf5(const std::string& caseName, const SizeType timeStep) {} + virtual void DefineConstants() {} //To be added + virtual void DefineVariables(SizeType timestep = 0) {} //Base functions for defining model required variables + virtual void ObtainID(int* velId, int* loop, Real& tauCompo, + CollisionType& collisModel, int& idComponent, int& rhoId,int &thId); + virtual int GetThermalFlag() {return isThermalModel;} + + +}; + + + +#endif /* APPS_LBM_DEM_FLPART_INTERACTION_BASE_H_ */ diff --git a/Apps/LBM-DEM/iterate_dem.cpp b/Apps/LBM-DEM/iterate_dem.cpp new file mode 100644 index 0000000..a3bc635 --- /dev/null +++ b/Apps/LBM-DEM/iterate_dem.cpp @@ -0,0 +1,238 @@ +/** + * Copyright 2019 United Kingdom Research and Innovation + * + * Authors: See AUTHORS + * + * Contact: [jianping.meng@stfc.ac.uk and/or jpmeng@gmail.com] + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice + * this list of conditions and the following disclaimer in the documentation + * and or other materials provided with the distribution. + * 3. Neither the name of the copyright holder nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * ANDANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +/*! + * @brief Wrap functions for DEM-LBM iteration scheme + * @author Chrysovalantis Tsigginos + * @details Define functions for DEM-LBM iteration scheme + * */ + +#include "iterate_dem.h" + +SizeType MAXITER; +int CHECKPERIOD; + +void IterateDEMLBM(const SizeType maxIter, const SizeType checkPeriod, const SizeType timeStart, int savingFlag) { + + Real dtD; //TimeStep for force Calculation + CHECKPERIOD = checkPeriod; + MAXITER = maxIter; + SizeType initPoint; + if (Nfl <= 1) + dtD = TimeStep(); + else + dtD = DemTimeStep(); + int flag; + ops_printf("MP-LBM: Number of iterations: %d\n", maxIter); + ops_printf("Npl = %d Nfl= %d\n", Npl, Nfl); + + + double ct0I, ct1I, et0I, et1I; + double et0MUI, et1MUI; + double timeMUI = 0.0; + + ops_timers(&ct0I, &et0I); + + + //exit(EXIT_FAILURE); + initPoint = timeStart; + SizeType iter = timeStart; + for (SizeType iter1 = timeStart; iter1 < MAXITER; iter1++) { + + ++iter; + if (iter1 == initPoint) + flag = 1; + else + flag = 0; + //ops_printf("Iteration %d\n", iter); + + if ( (iter% static_cast(Npl)) == 0) { + if (muiInterfaceFlag == 1) { + ops_timers(&ct0I, &et0MUI); + + ExtractParticleData(iter); + + ops_timers(&ct1I, &et1MUI); + timeMUI += et1MUI - et0MUI; + } + InitializeDragForce(); + for (int jfl = 1; jfl < Nfl + 1; jfl++) { + //Here we need to introduce DEM-LBM type scheme-since there are many +#ifdef OPS_3D + StreamCollisionPSM3D(flag); +#endif + +#ifdef OPS_2D + StreamCollisionPSM2D(); +#endif + } + CalculateDragForce(dtD, iter); + if (muiInterfaceFlag == 1) { + + ops_timers(&ct0I, &et0MUI); + + SendParticleData(iter); + //ForgetData(iter); + + ops_timers(&ct1I, &et1MUI); + timeMUI += et1MUI - et0MUI; + } + } + + if (savingFlag == 1) { + if (iter % checkPeriod == 0) { +#ifdef OPS_3D + UpdateMacroVars3D(); + if (forceFlag == 1) + UpdateMacroVarsForceFluidSolid(); + + CorrectFluidVelocities(); + + + CalcResidualError3D(); + DispResidualError3D(iter, CHECKPERIOD * TimeStep()); + WriteFlowfieldToHdf5(iter); + WriteDistributionsToHdf5(iter); + WriteNodePropertyToHdf5(iter); + WritePSMVariablesToHdf5(iter); +#endif + +#ifdef OPS_2D + UpdateMacroVars(); + CalcResidualError(); + DispResidualError(iter, CHECKPERIOD * TimeStep()); + WriteFlowfieldToHdf5(iter); + WriteDistributionsToHdf5(iter); + WriteNodePropertyToHdf5(iter); + WritePSMVariablesToHdf5(iter); +#endif + } + } + + } + + ops_timers(&ct1I, &et1I); + ops_printf("--------------------------------------------------------------------------\n"); + ops_printf("Total running time: %12.9f\n", et1I-et0I); + ops_printf("MUI Running time: %12.9f\n", timeMUI); + ops_printf("--------------------------------------------------------------------------\n"); + + + + +#ifdef OPS_3D + UpdateMacroVars3D(); + if (forceFlag == 1) + UpdateMacroVarsForceFluidSolid(); + + CorrectFluidVelocities(); + WriteFlowfieldToHdf5(MAXITER); + WriteDistributionsToHdf5(MAXITER); + WriteNodePropertyToHdf5(MAXITER); + WritePSMVariablesToHdf5(MAXITER); + +#endif + + if (periodicFlag==1) + DestroyPeriodic(); + + DestroyModel(); + DestroyFlowfield(); + DestroyParticleParams(); +} + + +void IterateDEMLBMSS(const Real convergenceCriteria, const int checkPeriod, const int maxIter, int savingFlag) { + + CHECKPERIOD = checkPeriod; + int iter =0; + Real residualError = 10000; + Real dt = TimeStep(); + int flag; + ops_printf("maxIter = %d iter1 = %d CHECKPERIOD = %d\n", maxIter, iter, CHECKPERIOD); + + while (residualError >= convergenceCriteria) { + if (iter == 0) + flag = 1; + else + flag = 0; + InitializeDragForce(); +#ifdef OPS_3D + StreamCollisionPSM3D(flag); + if ((iter % CHECKPERIOD) == 0) { + UpdateMacroVars3D(); //Add correction term + + if (forceFlag == 1) + UpdateMacroVarsForceFluidSolid(); + + + CorrectFluidVelocities(); + + CalcResidualError3D(); + residualError = GetMaximumResidual(checkPeriod); + + DispResidualError3D(iter,static_cast( CHECKPERIOD * TimeStep())); + } +#endif + +#ifdef OPS_2D + StreamCollisionPSM2D(); + if ((iter1 % CHECKPERIOD) == 0) { + UpdateMacroVars(); + + if (forceFlag == 1) + UpdateMacroVarsForceFluidSolid(); + + + CorrectFluidVelocities(); + + CalcResidualError(); + residualError = GetMaximumResidual(checkPeriod); + DispResidualError(iter, CHECKPERIOD * TimeStep()); + } +#endif + CalculateDragForce(dt, iter); + ops_printf("Residual = %e, Convergence target = %e\n", residualError, convergenceCriteria); + + iter += 1; + + + if (iter > maxIter) + break; + } + if (savingFlag == 1) { + WriteFlowfieldToHdf5(0); + WriteDistributionsToHdf5(0); + WriteNodePropertyToHdf5(0); + WritePSMVariablesToHdf5(0); + } +} diff --git a/Apps/LBM-DEM/iterate_dem.h b/Apps/LBM-DEM/iterate_dem.h new file mode 100644 index 0000000..424c5a3 --- /dev/null +++ b/Apps/LBM-DEM/iterate_dem.h @@ -0,0 +1,57 @@ +/** + * Copyright 2019 United Kingdom Research and Innovation + * + * Authors: See AUTHORS + * + * Contact: [jianping.meng@stfc.ac.uk and/or jpmeng@gmail.com] + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice + * this list of conditions and the following disclaimer in the documentation + * and or other materials provided with the distribution. + * 3. Neither the name of the copyright holder nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * ANDANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +/*! + * @brief Wrap functions for DEM-LBM iteration scheme + * @author Chrysovalantis Tsigginos + * @details Define functions for DEM-LBM iteration scheme + * */ + +#ifndef ITERATE_DEM_H_ +#define ITERATE_DEM_H_ + +#include "boundary.h" +#include "evolution.h" +#include "evolution3d.h" +#include "psm.h" +#include "flowfield.h" +#include "dem_handle3d.h" +#include "periodic.h" +#include "type.h" + +void IterateDEMLBM(const SizeType maxIter, const SizeType checkPeriod, const SizeType startPeriod = 0, int savingFlag = 1); +void IterateDEMLBMSS(const Real convergenceCriteria, const int checkPeriodic,const int maxIter,int savingFlag = 1); + + + + +#endif /* ITERATE_DEM_H_ */ diff --git a/Apps/LBM-DEM/lbm3d_dem_psm.cpp b/Apps/LBM-DEM/lbm3d_dem_psm.cpp new file mode 100644 index 0000000..f3592ca --- /dev/null +++ b/Apps/LBM-DEM/lbm3d_dem_psm.cpp @@ -0,0 +1,301 @@ +/** + * Copyright 2019 United Kingdom Research and Innovation + * + * Authors: See AUTHORS + * + * Contact: [jianping.meng@stfc.ac.uk and/or jpmeng@gmail.com] + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice + * this list of conditions and the following disclaimer in the documentation + * and or other materials provided with the distribution. + * 3. Neither the name of the copyright holder nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * ANDANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. +*/ + +/** @brief An example main source code for coupling the DEM with the LBM with the PRATI scheme + * @author Chrysovalantis Tsigginos + **/ +#include +#include +#include +#include +#include "mplb.h" +#include "ops_seq_v2.h" +#include "mplb_dem.h" +#include "fpi.h" +#include "cavity3d_kernel.inc" + +void SetInitialMacrosVars() { + for (auto idBlock : g_Block()) { + Block& block{idBlock.second}; + std::vector iterRng; + iterRng.assign(block.WholeRange().begin(), block.WholeRange().end()); + const int blockIdx{block.ID()}; + for (auto& idCompo : g_Components()) { + const Component& compo{idCompo.second}; + const int rhoId{compo.macroVars.at(Variable_Rho).id}; + ops_par_loop(KerSetInitialMacroVars, "KerSetInitialMacroVars", + block.Get(), SpaceDim(), iterRng.data(), + ops_arg_dat(g_MacroVars().at(rhoId).at(blockIdx), 1, + LOCALSTENCIL, "Real", OPS_RW), + ops_arg_dat(g_MacroVars().at(compo.uId).at(blockIdx), + 1, LOCALSTENCIL, "Real", OPS_RW), + ops_arg_dat(g_MacroVars().at(compo.vId).at(blockIdx), + 1, LOCALSTENCIL, "Real", OPS_RW), + ops_arg_dat(g_MacroVars().at(compo.wId).at(blockIdx), + 1, LOCALSTENCIL, "Real", OPS_RW), + ops_arg_dat(g_CoordinateXYZ()[blockIdx], SpaceDim(), + LOCALSTENCIL, "Real", OPS_READ), + ops_arg_idx()); + } + } +} + + +void UpdateMacroscopicBodyForce(const Real time) {} + + +void simulate() { + + + //Define simulations and blocks + ops_printf("Define simulation and blocks\n"); + std::string caseName{"LBM-DEM3D\n"}; + SizeType spaceDim{3}; + std::vector blockIds{0}; + std::vector blockNames{"Cavity"}; + std::vector blockSize{81, 81, 81}; + int Nx=blockSize.at(0) - 1; + Real meshSize(1./Nx); + std::map> startPos{{0, {0.0, 0.0, 0.0}}}; + DefineBlocks(blockIds, blockNames, blockSize, meshSize, startPos); + + + //Define fluid components + std::vector componentNames{"Fluid"}; + std::vector compoId{0}; + std::vector lattNames {"d3q19"}; + std::vector tauRef{0.01}; + DefineComponents(componentNames, compoId, lattNames, tauRef); + + //Define macro-variables + std::vector marcoVarTypes{Variable_Rho, Variable_U, + Variable_V, Variable_W}; + std::vector macroVarNames{"rho", "u", "v", "w"}; + std::vector macroVarId{0, 1, 2, 3}; + std::vector macroCompoId{0, 0, 0, 0}; + DefineMacroVars(marcoVarTypes, macroVarNames, macroVarId, macroCompoId); + + std::vector collisionTypes{Collision_BGKIsothermal2nd}; + std::vector collisionCompoId{0}; + DefineCollision(collisionTypes, collisionCompoId); + + std::vector bodyForceTypes{BodyForce_None}; + std::vector bodyForceCompoId{0}; + DefineBodyForce(bodyForceTypes, bodyForceCompoId); + + SchemeType scheme{Scheme_StreamCollision}; + DefineScheme(scheme); + + ops_printf("Defining fluid-particle interaction\n"); + Real cutoff = 0.01 * meshSize; + std::string particleType{"spherical"}; + DefineBlockParticles(spaceDim, cutoff, meshSize, particleType); + + //Define FSI-model + std::vector fluidModel{Model_PSM}; //Model_Prati + ops_printf("FSI model is %d\n", static_cast(fluidModel.at(0))); + std::vector porosModelParams{2};//, 2, 2}; + std::vector fsiComponent{0}; + Real force[spaceDim]={0.0, 0.0, 0.0}; + DefineInteractionModel(fluidModel, fsiComponent, force, 0.5, 0); + + + // Setting boundary conditions + SizeType blockIndex{0}; + SizeType componentId{0}; + std::vector macroVarTypesatBoundary{Variable_U, Variable_V, + Variable_W}; + std::vector noSlipStationaryWall{0, 0, 0}; + std::vector noSlipMovingWall{0.01, 0.00, 0.0}; + + + // Left noSlipStationaryWall + DefineBlockBoundary(blockIndex, componentId, BoundarySurface::Left, + BoundaryScheme::EQMDiffuseRefl, macroVarTypesatBoundary, + noSlipMovingWall); + // Right noSlipStationaryWall + DefineBlockBoundary(blockIndex, componentId, BoundarySurface::Right, + BoundaryScheme::EQMDiffuseRefl, macroVarTypesatBoundary, + noSlipStationaryWall); + // Top noslipMovingWall + + DefineBlockBoundary(blockIndex, componentId, BoundarySurface::Top, + BoundaryScheme::EQMDiffuseRefl, macroVarTypesatBoundary, + noSlipStationaryWall); + // bottom noSlipStationaryWall + DefineBlockBoundary(blockIndex, componentId, BoundarySurface::Bottom, + BoundaryScheme::EQMDiffuseRefl, macroVarTypesatBoundary, + noSlipStationaryWall); + // front noSlipStationaryWall + DefineBlockBoundary(blockIndex, componentId, BoundarySurface::Front, + BoundaryScheme::EQMDiffuseRefl, macroVarTypesatBoundary, + noSlipStationaryWall); + // back noSlipStationaryWall + DefineBlockBoundary(blockIndex, componentId, BoundarySurface::Back, + BoundaryScheme::EQMDiffuseRefl, macroVarTypesatBoundary, + noSlipStationaryWall); + + std::vector initType{Initial_BGKFeq2nd}; + std::vector initalCompoId{0}; + DefineInitialCondition(initType,initalCompoId); + + + Partition(); + + + SetInitialMacrosVars(); + SetTimeStep(meshSize / SoundSpeed()); + SetGridSize(meshSize); + PreDefinedInitialCondition3D(); + + //Initialize Particle Information + InteractionData lbmDemData; + bool muiflag = true; + SizeType maximumIterations{40}; + SetDemLbMParams(&lbmDemData, false, muiflag, 1e-7, 500, 0, + 10, maximumIterations, particleType); + + +#ifdef CPU +#if DebugLevel >= 2 + + ops_printf("-------------------------------------------------------------------------------------\n"); + ops_printf("Input LBM-DEM parameters\n"); + ops_printf("Number of steps before saving %d\n", lbmDemData.checkPeriod); + ops_printf("Number of steps before saving in SS %d\n", lbmDemData.checkPeriodStS); + ops_printf("Targeted convergence error in steady state simulations %e\n", + lbmDemData.convergenceRate); + ops_printf("Dem timestep %f\n", lbmDemData.dtDEM); + ops_printf("Max iterations for steady state simulations %d\n", lbmDemData.maxIters); + ops_printf("First step at the DEM-LBM coupled simulations %d\n", lbmDemData.nStart); + ops_printf("Maximum number of steps in LBM-DEM simulations %d\n", lbmDemData.nSteps); + + if (lbmDemData.muiFlag) + ops_printf("Mui interface is activated\n"); + else + ops_printf("Mui interface is not operating\n"); + + if (lbmDemData.restartFlag) + ops_printf("Simulation will restarted\n"); + else + ops_printf("Simulation starts at t=0\n"); + + ops_printf("-------------------------------------------------------------------------------------\n"); + +#endif +#endif + //SetMuiInterface + if (lbmDemData.muiFlag) { + CreateMuiInterface(lbmDemData, fluidModel, cutoff); + } + + + SetupParticleBoxes(lbmDemData); + + + //Setting up DEM-LBM + SetupDEMLBM(lbmDemData); + + IterateFSI(lbmDemData, 1); + ops_printf("I run the simulation\n"); + exit(EXIT_FAILURE); +} + +void simulate(const Configuration & config, const SizeType timeStep=0) { + + +} + +int main(int argc, const char** argv) { + // OPS initialisation + ops_init(argc, argv, 4); + double ct0, ct1, et0, et1; + ops_timers(&ct0, &et0); + // start a simulation by hard-coding + std::string s1("readFile"); + std::string s2("restartFile"); + int iLoc, output = 0; + + if (argc <= 1) { + simulate(); + } + else { + + for (int iArgs = 0; iArgs < argc; iArgs++) { + + std::string argName (argv[iArgs]); + output = argName.compare(s1); + if ( argName.compare(s1) == 0) { + iLoc = iArgs; + output = 1; + break; + } + else if (argName.compare(s2) == 0) { + iLoc = iArgs; + output = 2; + break; + } + } + + if (output == 1) { + std::string configFileName(argv[iLoc+1]); + //ReadConfigurationDEM(configFileName); /* TODO ADD THE JSON FILE */ + simulate(Config()); /*TODO ADD IT WITH JSON */ + } + else if (output==2) { + std::string configFileName(argv[iLoc+1]); + //ReadConfigurationDEM(configFileName); /* TODO ADD THE JSON FILE */ + const SizeType timeStep{static_cast(std::stoi(argv[iLoc+2]))}; + simulate(Config(),timeStep); + } + else { + ops_printf("WARNING: Option %d not supported\n", output); + } + + } + + + ops_timers(&ct1, &et1); + ops_printf("\nTotal Wall time %lf\n", et1 - et0); + //Print OPS performance details to output stream + ops_timing_output(std::cout); + ops_exit(); + +} + + + + + + + + diff --git a/Apps/LBM-DEM/lbm3d_psm.cpp b/Apps/LBM-DEM/lbm3d_psm.cpp new file mode 100644 index 0000000..944c901 --- /dev/null +++ b/Apps/LBM-DEM/lbm3d_psm.cpp @@ -0,0 +1,580 @@ +/** + * Copyright 2019 United Kingdom Research and Innovation + * + * Authors: See AUTHORS + * + * Contact: [jianping.meng@stfc.ac.uk and/or jpmeng@gmail.com] + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice + * this list of conditions and the following disclaimer in the documentation + * and or other materials provided with the distribution. + * 3. Neither the name of the copyright holder nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * ANDANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. +*/ + +/** @brief The main source file for direct DEM-LBM sims with the PSM + * @author Chrysovalantis Tsigginos + */ + +#include +#include +#include +#include +#include +#include "boundary.h" +#include "evolution3d.h" +#include "flowfield.h" +#include "model.h" +#include "scheme.h" +#include "type.h" +#include "configuration.h" +#include "psm.h" +#include "dem_handle3d.h" +#include "iterate_dem.h" +#include "periodic.h" +#include "read_input.h" + + + + +void SetInitialMacrosVars() { + Real initialVars[SPACEDIM + 1]; + + initialVars[0] = 1.0; + initialVars[1] = 0.0; + initialVars[2] = 0.0; + initialVars[3] = 0.0; + for (int blockIdx = 0; blockIdx < BlockNum(); blockIdx++) { + int* iterRng = BlockIterRng(blockIdx, IterRngWhole()); + ops_par_loop(KerSetInitialMacroVars, "KerSetInitialMacroVars", + g_Block[blockIdx], SPACEDIM, iterRng, + ops_arg_dat(g_MacroVars[blockIdx], NUMMACROVAR, + LOCALSTENCIL, "double", OPS_RW), + ops_arg_dat(g_CoordinateXYZ[blockIdx], SPACEDIM, + LOCALSTENCIL, "double", OPS_READ), + ops_arg_gbl(initialVars, SPACEDIM+1, "double", OPS_READ)); + } +} + +void UpdateMacroscopicBodyForce(const Real time) {} + + +void simulate(configData input, SizeType restartStep = 0) { + + //ops_printf("I entered in the simulate\n"); + DefineCase(input.casename, input.spacedim); //set transient to true + + //Define geometry + DefineBlocks(input.blockNum, input.gridBlock, input.dx, input.gridStart); //Define coordinates, nodeType and Geometry property. + + //Define lattice model + std::vector compoNames{"Fluid"}; + std::vector compoid{0}; + std::vector lattNames{"d3q19"}; + DefineComponents(compoNames, compoid, lattNames, input.restartStep); //Define g_f and g_fStage + + ops_printf("Distribution functions defined at timestep = %d\n", restartStep); + + + //Define macroscopic variables + std::vector marcoVarTypes{Variable_Rho, Variable_U, + Variable_V, Variable_W}; + std::vector macroVarNames{"rho", "u", "v", "w"}; + std::vector macroVarId{0, 1, 2, 3}; + std::vector macroCompoId{0, 0, 0, 0}; + DefineMacroVars(marcoVarTypes, macroVarNames, macroVarId, macroCompoId, restartStep); //Define macroscopic variables and error handling + + //Define PSM variables + DefinePSMVariables(restartStep, input.gamma); + + //Define bodyForces + std::vector bodyForceTypes{BodyForce_None}; + std::vector bodyForceCompoId{0}; + DefineBodyForce(bodyForceTypes, bodyForceCompoId); + + //Define collision model + std::vector collisionTypes{Collision_BGKIsothermal2nd}; + std::vector collisionCompoId{0}; + DefineCollision(collisionTypes, collisionCompoId); + + //Define bodyForces + SetupBodyForces(); + + ops_printf("MP-LBM: BodyForces have been set\n"); + //Define collision scheme + SchemeType scheme{Scheme_StreamCollision}; + DefineScheme(scheme); //Defines halodepth + + // Setting boundary conditions + SizeType blockIndex{0}; + SizeType componentId{0}; + +//Boundary conditions + BoundaryScheme boundaryType[6] = {BoundaryScheme::EQMDiffuseRefPois, BoundaryScheme::FreeFlux, BoundaryScheme::EQMDiffuseRefl, BoundaryScheme::EQMDiffuseRefl, + BoundaryScheme::Periodic,BoundaryScheme::Periodic,}; + + + + DefinePeriodicBoundariesRestart(input.boundaries.BoundaryType, input.boundaries.DirBc); + + + //Define VertexType for different boundaries + + VertexType vertexType[6]; + + if (periodicFlag == 0) { + for (int iDim = 0; iDim < 6; iDim++) { + vertexType[iDim] = VertexType::Wall; + } + } + else if (periodicFlag == 1) { + for (int iDim = 0; iDim < 3; iDim++) { + int direction = input.boundaries.DirBc[iDim]; + if (periodic[direction]==1) { + vertexType[2*iDim] = VertexType::Periodic; + vertexType[2*iDim + 1] = VertexType::Periodic; + } + else { + vertexType[2*iDim] = VertexType::Wall; + vertexType[2*iDim + 1] = VertexType::Wall; + } + } + } + + //Define walls + + //Input Velocities-Please note that periodic boundaries should come first + + std::vector macroVarTypesatBoundary{Variable_U, Variable_V, + Variable_W}; + + DefineBlockBoundary(blockIndex, componentId, input.boundaries.BoundarySurfaces[0], + input.boundaries.BoundaryType[0], macroVarTypesatBoundary, + input.boundaries.velocities[0], vertexType[0]); + + DefineBlockBoundary(blockIndex, componentId, input.boundaries.BoundarySurfaces[1], + input.boundaries.BoundaryType[1], macroVarTypesatBoundary, + input.boundaries.velocities[1], vertexType[1]); + + DefineBlockBoundary(blockIndex, componentId, input.boundaries.BoundarySurfaces[2], + input.boundaries.BoundaryType[2], macroVarTypesatBoundary, + input.boundaries.velocities[2], vertexType[2]); + + DefineBlockBoundary(blockIndex, componentId, input.boundaries.BoundarySurfaces[3], + input.boundaries.BoundaryType[3], macroVarTypesatBoundary, + input.boundaries.velocities[3], vertexType[3]); + + DefineBlockBoundary(blockIndex, componentId, input.boundaries.BoundarySurfaces[4], + input.boundaries.BoundaryType[4], macroVarTypesatBoundary, + input.boundaries.velocities[4], vertexType[4]); + + DefineBlockBoundary(blockIndex, componentId, input.boundaries.BoundarySurfaces[5], + input.boundaries.BoundaryType[5], macroVarTypesatBoundary, + input.boundaries.velocities[5], vertexType[5]); + + + //setup initial condition + + std::vector initType{Initial_BGKFeq2nd}; + std::vector initalCompoId{0}; + DefineInitialCondition(initType,initalCompoId); + + ops_printf("Ready to partition the domain\n"); + Partition(); + + + + if (restartStep == 0) { + SetInitialMacrosVars(); + PreDefinedInitialCondition3D(); + } + else + RestartMacroVars4SteadySim(); + + + std::vector tauRef{input.tauRef}; + + SetTauRef(tauRef); + + SetTimeStep(input.dx / SoundSpeed()); + + SetDxLBM(input.dx); + SetupParticles(); + + + + if (periodicFlag == 1) { + InitializePeriodic(); + ops_printf("MP-LBM: Initialize periodic variables\n"); + } + + + //SetupParticles work correct !!! + if (input.muiFlag==0) + muiInterfaceFlag = 0; + else + muiInterfaceFlag = 1; + + + if (muiInterfaceFlag == 1) { + DefineMUIInterface(input.Rmax); + ops_printf("Setting up communication\n"); + SetupMUICommunication(10000000); //To be corrected in the next stage + } + + ops_printf("I setup the communication\n"); + //To add here additional fumctions for running the code + + int savingFlag; + if (input.storageOption.compare(0,input.storageOption.size(), "noSaving")==0) { + ops_printf("No data are saved during the simulation\n"); + savingFlag = 0; + } + else { + ops_printf("Data are saved during the simulation\n"); + savingFlag = 1; + } + + + SizeType maxStep; + + if (restartStep == 0) { + + const Real convergenceRate{10000}; + const int checkPeriod{1}; + int maxIters = 0; + + ops_printf("Before entering to InitializeDEM-LBM: ConvergenceRate: %e maxIters: %d, checkPeriod: %d\n", + convergenceRate, maxIters, checkPeriod); + + InitializeDEMLBM(convergenceRate, maxIters, checkPeriod, maxStep, savingFlag); + + if (muiInterfaceFlag==0) + maxStep = restartStep + input.iters; + + } + else { + //To be added + InitializeDEMLBMRestart(maxStep, restartStep, input.iters, savingFlag); + } + + ops_printf("Restart timeStep = %d\n", restartStep); + + IterateDEMLBM(maxStep, input.exportSteps, restartStep, savingFlag); + + if (muiInterfaceFlag == 1) + DestroyMUIInterface(); + + +} + +void simulate() { + + //Define simulation space and simulation name + std::string caseName{"Classic_case"}; + SizeType spaceDim{3}; + DefineCase(caseName, spaceDim); //set transient to true + + //Define geometry + SizeType blockNum{1}; + //std::vector blockSize{75, 75, 75}; + //Real meshSize{20. / 74}; + std::vector blockSize{641, 41, 41}; + Real Lx = 16.0; + + Real meshSize{Lx / (static_cast(blockSize[0])-1.0)}; + std::vector startPos{0.0, 0.0, 0.0}; + DefineBlocks(blockNum, blockSize, meshSize, startPos); //Define coordinates, nodeType and Geometry property. + + //Define lattice model + std::vector compoNames{"Fluid"}; + std::vector compoid{0}; + std::vector lattNames{"d3q19"}; + DefineComponents(compoNames, compoid, lattNames); //Define g_f and g_fStage + + //Define macroscopic variables + std::vector marcoVarTypes{Variable_Rho, Variable_U, + Variable_V, Variable_W}; + std::vector macroVarNames{"rho", "u", "v", "w"}; + std::vector macroVarId{0, 1, 2, 3}; + std::vector macroCompoId{0, 0, 0, 0}; + DefineMacroVars(marcoVarTypes, macroVarNames, macroVarId, macroCompoId); //Define macroscopic variables and error handling + + //Define PSM variables + DefinePSMVariables(); + + //Define bodyForces + std::vector bodyForceTypes{BodyForce_None}; + std::vector bodyForceCompoId{0}; + DefineBodyForce(bodyForceTypes, bodyForceCompoId); + + //Define collision model + std::vector collisionTypes{Collision_BGKIsothermal2nd}; + std::vector collisionCompoId{0}; + DefineCollision(collisionTypes, collisionCompoId); + + //Define bodyForces + SetupBodyForces(); + + ops_printf("MP-LBM: BodyForces have been set\n"); + //Define collision scheme + SchemeType scheme{Scheme_StreamCollision}; + DefineScheme(scheme); //Defines halodepth + + // Setting boundary conditions + SizeType blockIndex{0}; + SizeType componentId{0}; + + + + + +// Original versopm +// BoundaryScheme boundaryType[6] = {BoundaryScheme::EQMDiffuseRefl,BoundaryScheme::EQMDiffuseRefl, BoundaryScheme::EQMDiffuseRefl, +// BoundaryScheme::EQMDiffuseRefl, BoundaryScheme::Periodic,BoundaryScheme::Periodic}; + + + BoundaryScheme boundaryType[6] = {BoundaryScheme::EQMDiffuseRefPois, BoundaryScheme::FreeFlux, BoundaryScheme::EQMDiffuseRefl, BoundaryScheme::EQMDiffuseRefl, + BoundaryScheme::Periodic,BoundaryScheme::Periodic,}; + + DefinePeriodicBoundaries(boundaryType); + + + + //Define VertexType for different boundaries + + VertexType vertexType[6]; + + SetupVertexType(vertexType); + if (periodicFlag == 0) { + for (int iDim = 0; iDim < 6; iDim++) { + vertexType[iDim] = VertexType::Wall; + } + } + else if (periodicFlag == 1) { + for (int iDim = 0; iDim < 3; iDim++) { + if (periodic[iDim]==1) { + vertexType[2*iDim] = VertexType::Periodic; + vertexType[2*iDim + 1] = VertexType::Periodic; + } + else { + vertexType[2*iDim] = VertexType::Wall; + vertexType[2*iDim + 1] = VertexType::Wall; + } + } + } + + //Define walls + + //Input Velocities-Please note that periodic boundaries should come first + + std::vector macroVarTypesatBoundary{Variable_U, Variable_V, + Variable_W}; + std::vector noSlipStationaryWall{0, 0, 0}; + std::vector noSlipMovingWall{0.005, 0, 0}; + std::vector noSlipMovingWall1{-0.0, 0, 0}; + + + // Back Wall + DefineBlockBoundary(blockIndex, componentId, BoundarySurface_Back, + boundaryType[4], macroVarTypesatBoundary, + noSlipMovingWall, vertexType[4]); + + //Front Wall + DefineBlockBoundary(blockIndex, componentId, BoundarySurface_Front, + boundaryType[5], macroVarTypesatBoundary, + noSlipMovingWall1, vertexType[5]); + + + + + + // Left Wall + DefineBlockBoundary(blockIndex, componentId, BoundarySurface_Left, + boundaryType[0], macroVarTypesatBoundary, + noSlipMovingWall, vertexType[0]); + // Right Wall + DefineBlockBoundary(blockIndex, componentId, BoundarySurface_Right, + boundaryType[1], macroVarTypesatBoundary, + noSlipMovingWall1, vertexType[1]); + + + + // Bottom Wall + DefineBlockBoundary(blockIndex, componentId, BoundarySurface_Bottom, + boundaryType[2], macroVarTypesatBoundary, + noSlipStationaryWall, vertexType[2]); + + //Top Wall + DefineBlockBoundary(blockIndex, componentId, BoundarySurface_Top, + boundaryType[3], macroVarTypesatBoundary, + noSlipStationaryWall, vertexType[3]); + + + + + + + + + + + + + + + //setup initial condition + + std::vector initType{Initial_BGKFeq2nd}; + std::vector initalCompoId{0}; + DefineInitialCondition(initType,initalCompoId); + + ops_printf("Ready to partition the domain\n"); + Partition(); + SetInitialMacrosVars(); + PreDefinedInitialCondition3D(); + std::vector tauRef{0.001}; + SetTauRef(tauRef); + SetTimeStep(meshSize / SoundSpeed()); + + SetDxLBM(meshSize); + SetupParticles(); + + + + if (periodicFlag == 1) { + InitializePeriodic(); + ops_printf("MP-LBM: Initialize periodic variables\n"); + } + + + //SetupParticles work correct !!! + muiInterfaceFlag = 1; + double Rmax = 0.3; + if (muiInterfaceFlag == 1) { + DefineMUIInterface(Rmax); + ops_printf("Setting up communication\n"); + SetupMUICommunication(10000000); + } + + ops_printf("I setup the communication\n"); + //To add here additional fumctions for running the code + + const Real convergenceRate{10000}; + const int checkPeriod{1}; + SizeType maxStep; + int maxIters = 0; + + ops_printf("Before entering to InitializeDEM-LBM: ConvergenceRate: %e maxIters: %d, checkPeriod: %d\n", + convergenceRate, maxIters, checkPeriod); + + InitializeDEMLBM(convergenceRate, maxIters, checkPeriod, maxStep); + + IterateDEMLBM(maxStep, 10000); + + if (muiInterfaceFlag == 1) + DestroyMUIInterface(); + + + +} + + +int main(const int argc,const char** argv) { + + + //OPS initialization + ops_init(argc, argv, 1); + + + double ct0, ct1, et0, et1; + + ops_timers(&ct0, &et0); + + + if (argc <= 1) { //hard-code simulation + simulate(); + } + else { + //This part will later be modified to JSON file input. At the moment will be a txt reading +// if (argc > 3) { +// ops_printf("ERROR: Number of arguments less than two\n"); +// exit(EXIT_FAILURE); +// } + + + + //Compare strings + + if (strcmp(argv[1],"restart")==0) { + configData readData; + ops_printf("I entered here\n"); + if (argc < 3) { + ops_printf("ERROR: Restart Timestep not defined\n"); + exit(EXIT_FAILURE); + } + + long int timeStep = atoi(argv[2]); + ReadInputFile(readData, timeStep); //Later to pass to Json + simulate(readData, readData.restartStep); + + } + else if (strcmp(argv[1],"readFile")==0) { + configData readData; + ReadInputFile(readData); //Later to pass to Json + simulate(readData); + + } + else { + ops_printf("ERROR: This option is not supported. Supported options: restart and readFile\n"); + } + + + + + } + + +/* if (argc>1 && argc <=2) { //start a simulation from input file + std::string configFileName(argv[1]); + ReadConfiguration(configFileName); + simulate(Config()); + } + + if (argc> 2 && argc <=3) { + std::string configFileName(argv[1]); + ReadConfiguration(configFileName); //NEED TO DISCUSS THIS + const SizeType timeStep{std::stoi(argv[2])}; + simulate(Config(),timeStep); + restartFlag = 1; + } + +*/ + ops_timers(&ct1, &et1); + ops_printf("\nTotal Wall time %lf\n", et1 - et0); + //Print OPS performance details to output stream + ops_timing_output(stdout); + ops_exit(); + + + +} + diff --git a/Apps/LBM-DEM/mapping_models.h b/Apps/LBM-DEM/mapping_models.h new file mode 100644 index 0000000..2055ece --- /dev/null +++ b/Apps/LBM-DEM/mapping_models.h @@ -0,0 +1,46 @@ +/** + * Copyright 2019 United Kingdom Research and Innovation + * + * Authors: See AUTHORS + * + * Contact: [jianping.meng@stfc.ac.uk and/or jpmeng@gmail.com] + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice + * this list of conditions and the following disclaimer in the documentation + * and or other materials provided with the distribution. + * 3. Neither the name of the copyright holder nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * ANDANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +/*! @brief Head files for configuring the fluid-particle interaction model + * @author C. Tsigginos + * @details The header of any new mapping model must be inserted here in. + */ +#ifndef MAPPING_MODELS_H_ +#define MAPPING_MODELS_H_ + +#include "poros_spherical.h" +#include "particle_to_grid_base.h" +#include "poros_grid.h" + + + +#endif /* APPS_LBM_DEM_MAPPING_MODELS_H_ */ diff --git a/Apps/LBM-DEM/memory_handle.h b/Apps/LBM-DEM/memory_handle.h new file mode 100644 index 0000000..c5910ea --- /dev/null +++ b/Apps/LBM-DEM/memory_handle.h @@ -0,0 +1,69 @@ +/** + * Copyright 2019 United Kingdom Research and Innovation + * + * Authors: See AUTHORS + * + * Contact: [jianping.meng@stfc.ac.uk and/or jpmeng@gmail.com] + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice + * this list of conditions and the following disclaimer in the documentation + * and or other materials provided with the distribution. + * 3. Neither the name of the copyright holder nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * ANDANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +/*! + * @brief Function for dynamic allocation and re-allocation of particle related lists + * @author Chrysovalantis Tsigginos + * @details Functions for dynamic allocation of particle related lists + */ + +#ifndef MEMORY_HANDLE_H_ +#define MEMORY_HANDLE_H_ + +#include +#include +#include "type.h" +#include "flowfield.h" +#include +#include + +template +inline void free2d(T **&a) { + + for (int blockIndex = 0; blockIndex < BlockNum(); blockIndex++) { + free(a[blockIndex]); + } + + free(a); +} + +template +inline void Reallocate(T *&a, int size, int bm) { + + a= (T *) realloc(a, size * bm * sizeof(T)); + +} +template +inline void allocateMemory(T *&a, int size, int bm) { + a = (T *) malloc(size * bm * sizeof(T)); +} +#endif /* APPS_LBM_DEM_MEMORY_HANDLE_H_ */ diff --git a/Apps/LBM-DEM/mplb_dem.h b/Apps/LBM-DEM/mplb_dem.h new file mode 100644 index 0000000..ce3e763 --- /dev/null +++ b/Apps/LBM-DEM/mplb_dem.h @@ -0,0 +1,52 @@ +/** + * Copyright 2019 United Kingdom Research and Innovation + * + * Authors: See AUTHORS + * + * Contact: [jianping.meng@stfc.ac.uk and/or jpmeng@gmail.com] + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice + * this list of conditions and the following disclaimer in the documentation + * and or other materials provided with the distribution. + * 3. Neither the name of the copyright holder nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * ANDANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + * /*! @brief Head files for configuring the fluid-particle interaction model + * @author C. Tsigginos + */ + +#ifndef APPS_LBM_DEM_MPLB_DEM_H_ +#define APPS_LBM_DEM_MPLB_DEM_H_ + +#include "dem_data.h" +#include "dem_handle.h" +#include "block_particle_helper.h" +#include "fpi.h" +//#include "psm.h" +//#include "prati.h" +//#include "solidFraction.h" +//#include "mui_interface.h" + + + + + +#endif /* APPS_LBM_DEM_MPLB_DEM_H_ */ diff --git a/Apps/LBM-DEM/mui_interface.cpp b/Apps/LBM-DEM/mui_interface.cpp new file mode 100644 index 0000000..d80eb39 --- /dev/null +++ b/Apps/LBM-DEM/mui_interface.cpp @@ -0,0 +1,291 @@ +/** + * Copyright 2019 United Kingdom Research and Innovation + * + * Authors: See AUTHORS + * + * Contact: [jianping.meng@stfc.ac.uk and/or jpmeng@gmail.com] + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice + * this list of conditions and the following disclaimer in the documentation + * and or other materials provided with the distribution. + * 3. Neither the name of the copyright holder nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * ANDANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +/*! + * @brief Class for data exchange between MPLB and DEM solver + * @author C. Tsigginos + * @details: Functions for intersolver communications via MUI + */ + + +#include "mui_interface.h" +#include +#include + +MuiInterface::MuiInterface( ListBlockParticles* blockParticles, + std::vector input, std::vector particleShape, + std::vector output, Real skin, SizeType timeStep) +{ + + + interface = new mui::uniface3d("mpi://LBM/ifs"); + maxStep = 0; + Rmax = skin; + + startStep = timeStep; + blockPointer = blockParticles; + spaceDim = 2; +#ifdef OPS_3D + spaceDim = 3; +#endif + + inputData = input; + outputData = output; + particleShapeData = particleShape; + + inputDataSize = inputData.size(); + outputDataSize = outputData.size(); + inputParticleSize = particleShapeData.size(); + +} + +MuiInterface::~MuiInterface() { + + delete interface; +} + +void MuiInterface::SetDomains(SizeType maxIteration) { + + maxStep = 2 * maxIteration + startStep; + Real xmin[spaceDim], xmax[spaceDim]; + long int firstStep = static_cast(startStep); + long int lastStep = static_cast(maxStep); + + DefineProcBox(xmin, xmax); + + mui::geometry::box3d send_recv_region({xmin[0], xmin[1], xmin[2]}, + {xmax[0], xmax[1], xmax[2]}); + + interface->announce_send_span(firstStep, lastStep, send_recv_region); + interface->announce_recv_span(firstStep, lastStep, send_recv_region); + +} + +void MuiInterface::DefineProcBox(Real* xMin, Real* xMax) { + + Real xMinTmp[spaceDim], xMaxTmp[spaceDim]; + xMin[0] = std::numeric_limits::max(); + xMin[1] = xMin[0]; + xMin[2] = xMin[0]; + + xMax[0] = -1.0 * xMin[0]; + xMax[1] = xMax[0]; + xMax[2] = xMax[0]; + + + for (auto particleBlock = blockPointer->begin(); particleBlock != blockPointer->end(); + ++particleBlock) { + particleBlock->second.ExtractBound(xMinTmp, xMaxTmp); + + for (int iDir = 0; iDir < spaceDim; iDir++) { + if (xMinTmp[iDir] < xMin[iDir]) + xMin[iDir] = xMinTmp[iDir]; + if (xMaxTmp[iDir] > xMax[iDir]) + xMax[iDir] = xMaxTmp[iDir]; + } + } + +#ifdef CPU +#if DebugLevel >= 2 + printf("Send MUI region of rank %d: [(%e %e %e), (%e %e %e])]\n", ops_get_proc(), + xMin[0], xMin[1], xMin[2], xMax[0], xMax[1], xMax[2]); +#endif +#endif +} + +void MuiInterface::ExtractData(SizeType currentStep, SizeType &firstStep, SizeType& maxIter, + Real& alpha, int* flags, ParticleShapeDiscriptor& particleShape) { + + std::vector posMUI; + mui::chrono_sampler_exact3d time_sampler; + posMUI = interface->fetch_points("radius", currentStep, time_sampler); + + auto maxIter1 = interface->fetch("Nsteps"); + auto alpha1 = interface->fetch("alpha1"); + auto particleCase = interface->fetch("particleShape"); + auto firstStep1 = interface->fetch("firstStep"); + firstStep = static_cast(firstStep1); + maxIter = static_cast(maxIter1) - firstStep; + alpha = alpha1; + particleShape = (ParticleShapeDiscriptor) particleCase; +} + +void MuiInterface::ExtractParticles(SizeType timestep) { + + + Real uTmp[spaceDim], oTmp[spaceDim]; + Real xTemp[spaceDim], radTmp; + Real partTemp; + std::vector particleShape; + particleShape.reserve(inputParticleSize); + std::vector inputVariables; + inputVariables.reserve(inputDataSize); + long int timeframe = static_cast(timestep); + std::vector posMUI; + int iDir; + + mui::chrono_sampler_exact3d timeSampler; + mui::sampler_exact3d s1; + + posMUI = interface->fetch_points("radius", + timeframe, timeSampler); + + for (auto particleBlock = blockPointer->begin(); particleBlock != blockPointer->end(); ++particleBlock) + particleBlock->second.ClearParticles(); + + //Copy particles to correct location + for (auto iVec = 0; iVec < posMUI.size(); ++iVec) { + auto pointTmp = posMUI[iVec];//check access + + xTemp[0] = pointTmp[0]; + xTemp[1] = pointTmp[1]; + xTemp[2] = pointTmp[2]; + + //fetch other parameters + radTmp = interface->fetch("radius", {xTemp[0], xTemp[1], xTemp[2]}, timeframe, + s1, timeSampler); + uTmp[0] = interface->fetch("u", {xTemp[0], xTemp[1], xTemp[2]}, timeframe, + s1, timeSampler); + uTmp[1] = interface->fetch("v", {xTemp[0], xTemp[1], xTemp[2]}, timeframe, + s1, timeSampler); + uTmp[2] = interface->fetch("w", {xTemp[0], xTemp[1], xTemp[2]}, timeframe, + s1, timeSampler); + + oTmp[0] = interface->fetch("ox", {xTemp[0], xTemp[1], xTemp[2]}, timeframe, + s1, timeSampler); + oTmp[1] = interface->fetch("oy", {xTemp[0], xTemp[1], xTemp[2]}, timeframe, + s1, timeSampler); + oTmp[2] = interface->fetch("oz", {xTemp[0], xTemp[1], xTemp[2]}, timeframe, + s1, timeSampler); +#ifdef CPU +#if DebugLevel >= 2 + printf("Rank %d: [%f %f %f] R= %f u =[%f %f %f] omega=[%f %f %f]\n", + ops_get_proc(), xTemp[0], xTemp[1], xTemp[2], radTmp, uTmp[0], + uTmp[1], uTmp[2], oTmp[0], oTmp[1], oTmp[2]); +#endif +#endif + if (inputParticleSize > 0) { + ops_printf("Additional particle data are passed to MPLB via MUI\n"); + + iDir = 0; + for (auto& input : particleShapeData) { + iDir += 1; + auto dataTmp = interface->fetch(input, {xTemp[0], xTemp[1], xTemp[2]}, timeframe, s1, timeSampler); + particleShape.at(iDir) = dataTmp; + } + } + + if (inputDataSize > 0) { + ops_printf("Additional input are passed to MPLB via MUI\n"); + iDir = 0; + for (auto& input: inputData) { + iDir += 1; + auto dataTmp = interface->fetch(input, {xTemp[0], xTemp[1], xTemp[2]}, timeframe, s1, timeSampler); + inputVariables.at(iDir) = dataTmp; + } + } + + + for (auto particleBlock = blockPointer->begin(); particleBlock != blockPointer->end(); ++particleBlock) { + + int idParticle = particleBlock->second.InsertParticle(xTemp, radTmp, particleShape, + uTmp, oTmp, inputVariables); + //TODO ADD the additional parameters. + if (inputParticleSize > 0 && idParticle > -1) + particleBlock->second.GetAdditionalInputVariables(inputVariables, idParticle); + } + + + } + + interface->forget(timeframe); + +} + +void MuiInterface::SendParticles(SizeType timeStep) { + + if (timeStep > maxStep) + SetDomains(10 * maxStep ); + long int timeframe = static_cast(timeStep); + Real Fd[spaceDim], Td[spaceDim], xPos[spaceDim]; + std::vector output; + output.reserve(outputDataSize); + int Nparticles; + for (auto particleBlock = blockPointer->begin(); particleBlock != blockPointer->end(); ++particleBlock) { + Nparticles = particleBlock->second.NParticles; + for (int iParticle = 0; iParticle < Nparticles; iParticle++) { + particleBlock->second.ExtractDragForce(Fd, Td, iParticle); + particleBlock->second.ExtractPositions(xPos, iParticle); + + printf("Rank %d at timestep %d: Sends for particle [%f %f %f], Fd=[%12.9e %12.9e %12.9e] Td=[%12.9e %12.9e %12.9e\n", + ops_get_proc(), timeStep, xPos[0], xPos[1], xPos[2], Fd[0], Fd[1], Fd[2], Td[0], Td[1], Td[2]); + interface->push("Fdx", {xPos[0], xPos[1], xPos[2]}, Fd[0]); + interface->push("Fdy", {xPos[0], xPos[1], xPos[2]}, Fd[1]); + interface->push("Fdz", {xPos[0], xPos[1], xPos[2]}, Fd[2]); + interface->push("Mdx", {xPos[0], xPos[1], xPos[2]}, Td[0]); + interface->push("Mdy", {xPos[0], xPos[1], xPos[2]}, Td[1]); + interface->push("Mdz", {xPos[0], xPos[1], xPos[2]}, Td[2]); + + if (outputDataSize > 0) { + particleBlock->second.GetAdditionalOutputVariables(output, iParticle); + for (int iDir = 0; iDir < outputDataSize; iDir++) + interface->push(outputData[iDir], {xPos[0], xPos[1], xPos[2]}, output[iDir]); + } + } + + } + + interface->commit(timeframe); +} + +void MuiInterface::UpdateRegionFirstLast(SizeType firstStep, SizeType endStep) { + + int a1 = 0; + int a2 = 0; + if (firstStep > startStep) { + a1 = 1; + startStep = firstStep; + } + if (endStep > maxStep) { + a2 = 1; + maxStep = endStep; + + } + + if ((a1 == 1) || (a2 == 1)) { + SizeType iters= maxStep - firstStep; + SetDomains(iters); + } + +} + + \ No newline at end of file diff --git a/Apps/LBM-DEM/mui_interface.h b/Apps/LBM-DEM/mui_interface.h new file mode 100644 index 0000000..631c82f --- /dev/null +++ b/Apps/LBM-DEM/mui_interface.h @@ -0,0 +1,85 @@ +/** + * Copyright 2019 United Kingdom Research and Innovation + * + * Authors: See AUTHORS + * + * Contact: [jianping.meng@stfc.ac.uk and/or jpmeng@gmail.com] + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice + * this list of conditions and the following disclaimer in the documentation + * and or other materials provided with the distribution. + * 3. Neither the name of the copyright holder nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * ANDANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +/*! @brief Head files for the base Mui class + * @author C. Tsigginos + */ + +#ifndef MUI_INTERFACE_BASE_H_ +#define MUI_INTERFACE_BASE_H_ + +#include +#include +#include +#include +#include "block_particles.h" +#include "mui.h" +#include "type.h" +#include "dem_data.h" +class MuiInterface { + + public: + MuiInterface(ListBlockParticles* blockParticles, + std::vector input, std::vector particleShape, + std::vector output, Real skin = 1.0, SizeType timeStep = 0); + ~MuiInterface(); + void SetDomains(SizeType maxIteration); + void UpdateDomains(SizeType steps); + void ExtractData(SizeType currentStep, SizeType &firstStep, SizeType& maxStep, + Real& alpha, int* flags, ParticleShapeDiscriptor& particleShape); + void ExtractParticles(SizeType timeStep); + void SendParticles(SizeType timesStep); + void ForgetData(SizeType timeStep); + void UpdateRegionFirstLast(SizeType firstStep, SizeType endStep); + + + private: + mui::uniface3d* interface; //mui object + SizeType maxStep; //maximum step that the assinged region is ok + Real Rmax; //maximum particle size for defining region + void DefineProcBox(Real* xmin, Real* xmax); + ListBlockParticles* blockPointer; //point to list block particles + int spaceDim; //size of spatial space + SizeType startStep; //Start point of simulation + std::vector inputData; + std::vector outputData; + std::vector particleShapeData; + int inputDataSize; + int outputDataSize; + int inputParticleSize; + void SendExtraParameters(); + void ReceiveExtraParameters(); + void ReceiveParticleDetails(); + +}; + +#endif /* APPS_LBM_DEM_MUI_INTERFACE_BASE_H_ */ \ No newline at end of file diff --git a/Apps/LBM-DEM/mui_wrapper.cpp b/Apps/LBM-DEM/mui_wrapper.cpp new file mode 100644 index 0000000..3465033 --- /dev/null +++ b/Apps/LBM-DEM/mui_wrapper.cpp @@ -0,0 +1,78 @@ +/** + * Copyright 2019 United Kingdom Research and Innovation + * + * Authors: See AUTHORS + * + * Contact: [jianping.meng@stfc.ac.uk and/or jpmeng@gmail.com] + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice + * this list of conditions and the following disclaimer in the documentation + * and or other materials provided with the distribution. + * 3. Neither the name of the copyright holder nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * ANDANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +/*! + * @brief Wrapper functions for the data exchange operations + * @author C. Tsigginos + * @details: Wrapper functions for intersolver communication + */ + + +#include "mui_wrapper.h" + +MuiInterface* interface; + +void DefineMuiInterface(ListBlockParticles* blockParticles, + std::vector input, std::vector particleShape, + std::vector output, Real skin) { + + interface = new MuiInterface(blockParticles, input, particleShape, output, skin); + +} + +void SetMuiDomains(SizeType maxStep) { + + interface->SetDomains(maxStep); + +} + +void UpdateRegions(SizeType startStep, SizeType lastStep) { + + interface->UpdateRegionFirstLast(startStep, lastStep); +} + +void ExtractSimulationData(SizeType timeStep, SizeType& start, SizeType& maxIter, Real& alpha, int* flags, + ParticleShapeDiscriptor& particleShape) { + + interface->ExtractData(timeStep, start, maxIter, alpha, flags, particleShape); + +} + +void ExtractParticleData(SizeType timeStep) { + + interface->ExtractParticles(timeStep); +} + +void SendParticleData(SizeType timeStep) { + + interface->SendParticles(timeStep); +} diff --git a/Apps/LBM-DEM/mui_wrapper.h b/Apps/LBM-DEM/mui_wrapper.h new file mode 100644 index 0000000..dc63b59 --- /dev/null +++ b/Apps/LBM-DEM/mui_wrapper.h @@ -0,0 +1,27 @@ +/* + * mui_wrapper.h + * + * Created on: Jun 7, 2021 + * Author: jpd38567 + */ + +#ifndef MUI_WRAPPER_H_ +#define MUI_WRAPPER_H_ + +#include "mui_interface.h" +#include +#include +#include "block_particles.h" +#include "type.h" + +void DefineMuiInterface(ListBlockParticles* blockParticles, + std::vector input, std::vector particleShape, + std::vector output, Real skin = 1.0); + +void SetMuiDomains(SizeType maxStep); +void ExtractSimulationData(SizeType timeStep,SizeType& firstStep, SizeType& maxIter, + Real& alpha, int* flags, ParticleShapeDiscriptor& particleShape); +void ExtractParticleData(SizeType timeStep); +void SendParticleData(SizeType timeStep); +void UpdateRegions(SizeType startStep, SizeType lastStep); +#endif /* APPS_LBM_DEM_MUI_WRAPPER_H_ */ diff --git a/Apps/LBM-DEM/particle.cpp b/Apps/LBM-DEM/particle.cpp new file mode 100644 index 0000000..c191f4b --- /dev/null +++ b/Apps/LBM-DEM/particle.cpp @@ -0,0 +1,253 @@ +/** + * Copyright 2019 United Kingdom Research and Innovation + * + * Authors: See AUTHORS + * + * Contact: [jianping.meng@stfc.ac.uk and/or jpmeng@gmail.com] + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice + * this list of conditions and the following disclaimer in the documentation + * and or other materials provided with the distribution. + * 3. Neither the name of the copyright holder nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * ANDANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +/*! @brief Head files for handling a single particle + * @author C. Tsigginos + * @details Class for handling the data stored per given particle + */ + + +#include "particle.h" +#include +#include +#include "ops_seq_v2.h" +#include + +Particle::Particle(int Dimension, Real radius, Real* xp, std::vector Shape, + ParticleShapeDiscriptor particleImport, int inputVariablesSize, + int outputVariableSize) { + + spaceDim = Dimension; + if (spaceDim < 2 && spaceDim > 3) { + ops_printf("Error: Space dimensions not consistent\n"); + exit(EXIT_FAILURE); + } + + for (int iDim = 0; iDim < spaceDim; iDim++) { + xParticle[iDim] = xp[iDim]; + xOld[iDim] = 0.0; + } + + //TODO Assign the Particle Shape + particleShapeType = particleImport; + + switch(particleShapeType) { + case spherical: + particleShape.reset(new ParticleShape(radius, spherical)); + break; + case quadratic: + particleShape.reset(new ParticleShapeQuadratic(radius, quadratic, Shape)); + break; + case mesh: + particleShape.reset(new ParticleShape(radius, mesh)); //TODO ADD ACTUAL MESH PARTICLE + break; + default: + ops_printf("ERROR: This type of particle type is not supported\n"); + exit(EXIT_FAILURE); + } + + //Assing extra variables + if (inputVariablesSize > 0) { + nInputExtraVariables = inputVariablesSize; + inputVariables.reserve(nInputExtraVariables); + } + else + nInputExtraVariables = 0; + + if (outputVariableSize > 0) { + nOutputExtraVariables = outputVariableSize; + outputVariables.reserve(nOutputExtraVariables); + } + else + nOutputExtraVariables = 0; + + +} + +Particle::~Particle() { + + +} + +void Particle::InitializeDrag() { + + for (int iDim = 0; iDim < spaceDim; iDim++) { + FDrag[iDim] = 0.0; + TDrag[iDim] = 0.0; + } + +} + +std::string Particle::GetParticleShape() { + + switch (particleShapeType) { + case spherical: + return "sphere"; + case quadratic: + return "quadratic"; + case mesh: + return "mesh"; + + } + + return "none"; +} + +void Particle::UpdateParticleLocation(Real* xp) { + + for (int iDim = 0; iDim < spaceDim; iDim++) + xParticle[iDim] = xp[iDim]; +} + +void Particle::UpdateParticleShape(Real radius, std::vector shape) { + + particleShape->UpdateShape(radius, shape); + particleShape->Rotate(); +} + +void Particle::AddDrag(Real* Fp, Real* Tp) { + + for (int iDim = 0; iDim < spaceDim; iDim++) { + FDrag[iDim] += Fp[iDim]; + TDrag[iDim] += Tp[iDim]; + } + +} + +void Particle::EvaluateDrag(Real dt) { + + for (int iDim = 0; iDim < spaceDim;iDim++) { + FDrag[iDim] /= dt; + TDrag[iDim] /= dt; + } + + +} + +void Particle::PushDrag(Real* Fd, Real* Td) { + + for (int iDim = 0; iDim < spaceDim ; iDim++) { + Fd[iDim] = FDrag[iDim]; + Td[iDim] = TDrag[iDim]; + } +} + +void Particle::UpdateOldParticlePositions() { + + for (int iDim =0; iDim < spaceDim; iDim++) + xOld[iDim] = xParticle[iDim]; +} + +void Particle::UpdateParticleVelocities(Real* uP, Real* omP) { + + for (int iDim = 0; iDim < spaceDim; iDim++) { + uParticle[iDim] = uP[iDim]; + omegaParticle[iDim] = omP[iDim]; + } + +} + +void Particle::UpdateStencil(Real* xBounds, int *Nf, Real dx) { + + Real xMin[spaceDim]; + int iLocal[spaceDim]; + int iRadius; + for (int iDim = 0; iDim < spaceDim; iDim++) + xMin[iDim] = xBounds[2 * iDim]; + + + iRadius = (int) ceil(particleShape->Rparticle / dx); + for (int iDim = 0; iDim < spaceDim; iDim++) + iLocal[iDim] = (int) floor((xParticle[iDim] - xMin[iDim]) / dx); + + //Identify the particle stencil with no corrections + for (int iDim = 0; iDim < spaceDim; iDim++) { + stenList[2 * iDim] = iLocal[iDim] - iRadius; + stenList[2 * iDim + 1] = iLocal[iDim] + iRadius + 1; + } + + for (int iDim = 0; iDim < spaceDim; iDim++) { + if (stenList[2 * iDim] < Nf[2 * iDim]) + stenList[2 * iDim] = Nf[2 * iDim]; + + if (stenList[2 * iDim + 1] > Nf[2 * iDim + 1]) + stenList[2 * iDim + 1] = Nf[2 * iDim + 1]; + } + + + +} + + +void Particle::GetParticlePositions(Real* xPos) { + + for (int iDim = 0; iDim < spaceDim; iDim++) + xPos[iDim] = xParticle[iDim]; +} + +void Particle::SetInputVariables(std::vector& inputData) { + + if (inputData.size() > nInputExtraVariables) { + nInputExtraVariables = inputData.size(); + inputVariables.reserve(nInputExtraVariables); + } + + for (int iDir = 0; iDir < nInputExtraVariables; iDir++) + inputVariables.at(iDir) = inputData.at(iDir); +} + +void Particle::GetInputVariables(std::vector& inputData) { + + for (int iDir = 0; iDir < nInputExtraVariables; iDir++) + inputData.at(iDir) = inputVariables.at(iDir); +} + + +void Particle::GetOutputVariables(std::vector& outputData) { + + for (int iDir = 0; iDir < nOutputExtraVariables; iDir++) { + outputData.at(iDir) = outputVariables.at(iDir); + } + +} + +void Particle::SetOutputVariables(std::vector& outputData) { + + if (outputData.size() > nOutputExtraVariables) { + nOutputExtraVariables = outputData.size(); + outputVariables.reserve(nOutputExtraVariables); + } + + for (int iDir = 0; iDir < nOutputExtraVariables; iDir++) + outputVariables.at(iDir) = outputData.at(iDir); + +} diff --git a/Apps/LBM-DEM/particle.h b/Apps/LBM-DEM/particle.h new file mode 100644 index 0000000..95f2988 --- /dev/null +++ b/Apps/LBM-DEM/particle.h @@ -0,0 +1,94 @@ +/** + * Copyright 2019 United Kingdom Research and Innovation + * + * Authors: See AUTHORS + * + * Contact: [jianping.meng@stfc.ac.uk and/or jpmeng@gmail.com] + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice + * this list of conditions and the following disclaimer in the documentation + * and or other materials provided with the distribution. + * 3. Neither the name of the copyright holder nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * ANDANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +/*! @brief Head files for handling a single particle + * @author C. Tsigginos + */ +#ifndef PARTICLE_H_ +#define PARTICLE_H_ + +#include "type.h" +#include "particle_shape.h" +#include "dem_data.h" +#include +#include +#include +class Particle { + + public: + Real xParticle[3]; //Particle Center of Mass + std::shared_ptr particleShape; + Real FDrag[3]; + Real TDrag[3]; + Real xOld[3]; + Real uParticle[3]; + Real omegaParticle[3]; + int stenList[6]; + ParticleShapeDiscriptor particleShapeType; + int spaceDim; + + + Particle(int Dimension, Real radius, Real* xp, std::vector Shape, + ParticleShapeDiscriptor particleImport, int inputVariablesSize, + int outputVariableSize); + ~Particle(); + void InitializeDrag(); + void AddDrag(Real* Fp, Real* Tp); + void EvaluateDrag(Real dt); + void GetParticlePositions(Real* xPos); + std::string GetParticleShape(); + void UpdateParticleLocation(Real* xp); + void UpdateParticleShape(Real Rmax, std::vector Shape); + void UpdateParticleVelocities(Real* uPart, Real* omegaPart); + void UpdateExtraVariable(std::vector inputdata) { }; + void UpdateOutputVariable(std::vector& outputData) { }; + void PushDrag(Real* Fd, Real* Td); + void UpdateStencil(Real* xBounds, int *Nf, Real dx); + void UpdateOldParticlePositions(); + void SetInputVariables(std::vector& inputData); + void GetInputVariables(std::vector& inputData); + void GetOutputVariables(std::vector& outputData); + void SetOutputVariables(std::vector& outputData); + + private: + int nInputExtraVariables; + int nOutputExtraVariables; + std::vector inputVariables; //User defined input variables + std::vector outputVariables; //User defined output variables +}; + + + + + + +#endif /* APPS_LBM_DEM_PARTICLE_H_ */ diff --git a/Apps/LBM-DEM/particle_shape.cpp b/Apps/LBM-DEM/particle_shape.cpp new file mode 100644 index 0000000..fee7450 --- /dev/null +++ b/Apps/LBM-DEM/particle_shape.cpp @@ -0,0 +1,54 @@ +/** + * Copyright 2019 United Kingdom Research and Innovation + * + * Authors: See AUTHORS + * + * Contact: [jianping.meng@stfc.ac.uk and/or jpmeng@gmail.com] + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice + * this list of conditions and the following disclaimer in the documentation + * and or other materials provided with the distribution. + * 3. Neither the name of the copyright holder nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * ANDANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +/*! @brief Head files for handling a single particle + * @author C. Tsigginos + * @Details: Class for particle shape containers + */ + + +#include "particle_shape.h" + + +ParticleShape::ParticleShape(Real Rp, ParticleShapeDiscriptor particleShape) { + + Rparticle = Rp; + typeParticle = particleShape; + +} + +ParticleShapeQuadratic::ParticleShapeQuadratic(Real Rp, ParticleShapeDiscriptor particleShape, + std::vector shape) : ParticleShape(Rp, particleShape) { + + particleParameters = shape; + +} diff --git a/Apps/LBM-DEM/particle_shape.h b/Apps/LBM-DEM/particle_shape.h new file mode 100644 index 0000000..ae710e3 --- /dev/null +++ b/Apps/LBM-DEM/particle_shape.h @@ -0,0 +1,91 @@ +/** + * Copyright 2019 United Kingdom Research and Innovation + * + * Authors: See AUTHORS + * + * Contact: [jianping.meng@stfc.ac.uk and/or jpmeng@gmail.com] + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice + * this list of conditions and the following disclaimer in the documentation + * and or other materials provided with the distribution. + * 3. Neither the name of the copyright holder nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * ANDANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +/*! @brief Classes for handling and storing different particle shapes. + * @author C. Tsigginos + */ + +#ifndef PARTICLE_SHAPE_H_ +#define PARTICLE_SHAPE_H_ + +#include "type.h" +#include +#include +#include "dem_data.h" +class ParticleShape{ + + public: + Real Rparticle; + ParticleShapeDiscriptor typeParticle; + ParticleShape(Real Rp, ParticleShapeDiscriptor particleShape); + virtual ~ParticleShape() { }; + virtual void Rotate() { }; + virtual void UpdateShape(Real radius, std::vector shape) { Rparticle = radius; }; + virtual Real GetArea() { return 4.0/3.0 * PI * Rparticle * Rparticle * Rparticle;}; + virtual Real GetSurface() {return 4.0 * PI * Rparticle * Rparticle;} + virtual Real GetEquivalentRadius() {return Rparticle; }; + + +}; + +class ParticleShapeQuadratic : public ParticleShape { + + public: + std::vector particleParameters; + ParticleShapeQuadratic(Real Rp, ParticleShapeDiscriptor particleShape, + std::vector shape); + virtual void Rotate() { }; + virtual void UpdateShape(Real radius, std::vector shape) { }; + virtual Real GetArea() {return 0;}; + virtual Real GetSurface() {return 0;}; + virtual Real GetEquivalentRadius() {return 0;}; + +}; + +class ParticleShapeMesh : public ParticleShape { + + public: + std::vector gridPoints; + ParticleShapeMesh(Real Rp, ParticleShapeDiscriptor particleShape, std::vector shape); + virtual void UpdateShape(Real radius, std::vector shape) { }; + virtual void Rotate() { }; + virtual Real GetArea() {return 0;} + virtual Real GetSurface() {return 0;}; + virtual Real GetEquivalentRadius() {return 0;}; + + +}; + + + + +#endif /* APPS_LBM_DEM_PARTICLE_SHAPE_H_ */ diff --git a/Apps/LBM-DEM/particle_to_grid_base.cpp b/Apps/LBM-DEM/particle_to_grid_base.cpp new file mode 100644 index 0000000..5d2dba8 --- /dev/null +++ b/Apps/LBM-DEM/particle_to_grid_base.cpp @@ -0,0 +1,73 @@ +/** + * Copyright 2019 United Kingdom Research and Innovation + * + * Authors: See AUTHORS + * + * Contact: [jianping.meng@stfc.ac.uk and/or jpmeng@gmail.com] + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice + * this list of conditions and the following disclaimer in the documentation + * and or other materials provided with the distribution. + * 3. Neither the name of the copyright holder nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * ANDANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. +*/ + +/*! @brief Base calls for mapping particles in the LBM grid + * @author C. Tsigginos + * @details Base consturctor and output files + */ + +#include "particle_to_grid_base.h" + +ParticleToGridBase::ParticleToGridBase(int particleshape, int spacedim) { + + spaceDim = spacedim; + + particleDiscriptor = (ParticleType) particleshape; + ops_printf("Particle type passed is %d\n", particleshape); + ops_printf("Particle type assigned is %d\n", (ParticleType) particleDiscriptor); + noElem = 1; + + requiresCopy = false; + +} + +void ParticleToGridBase::WriteToHdf5(const std::string& caseName, const SizeType timeStep) { + + for (auto& element : mappingRealVariableList ) + element.WriteToHDF5(caseName, timeStep); + + for (auto& element : mappingIntVariableList) + element.WriteToHDF5(caseName, timeStep); + +} + +RealField& ParticleToGridBase::GetRealFieldVariable(int index) { + + return mappingRealVariableList.at(index); +} + +IntField& ParticleToGridBase::GetIntFieldVariable(int index) { + + return mappingIntVariableList.at(index); +} + + diff --git a/Apps/LBM-DEM/particle_to_grid_base.h b/Apps/LBM-DEM/particle_to_grid_base.h new file mode 100644 index 0000000..4fd902f --- /dev/null +++ b/Apps/LBM-DEM/particle_to_grid_base.h @@ -0,0 +1,78 @@ +/** + * Copyright 2019 United Kingdom Research and Innovation + * + * Authors: See AUTHORS + * + * Contact: [jianping.meng@stfc.ac.uk and/or jpmeng@gmail.com] + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice + * this list of conditions and the following disclaimer in the documentation + * and or other materials provided with the distribution. + * 3. Neither the name of the copyright holder nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * ANDANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. +*/ + +/*! @brief Base class for mapping particles into the LBM grid, + * @author C. Tsigginos + **/ + + + +#ifndef PARTICLE_TO_GRID_BASE_H_ +#define PARTICLE_TO_GRID_BASE_H_ + +#include "field.h" +#include "type.h" +#include +#include + +class ParticleToGridBase { + + protected: + enum ParticleType {ParticleSpherical = 0, + ParticleSuperQuadratic= 1, ParticleMesh = 2, ParticleNone = 3}; + + ParticleType particleDiscriptor; + int spaceDim; + int requiresCopy; + int noElem; + std::vector mappingRealVariableList; + std::vector mappingIntVariableList; + + public: + + ParticleToGridBase(int particleshape, int spacedim); + virtual ~ParticleToGridBase() { }; + virtual int particleShape() {return 0;}; + virtual void DefineVariables(int noElem, SizeType timestep = 0) { }; + virtual void ParticleProjection() { }; + virtual void UpdateProjection() { }; + virtual void WriteToHdf5(const std::string& caseName, const SizeType timeStep); + virtual void InitializeVariables() { }; + virtual void PrintMappingVariables() { }; + virtual RealField& GetRealFieldVariable(int index); + virtual IntField& GetIntFieldVariable(int index); + virtual void ReturnParticleShape() { ops_printf("Base class do not contain particle shape\n"); }; +}; + + + +#endif diff --git a/Apps/LBM-DEM/periodic.cpp b/Apps/LBM-DEM/periodic.cpp new file mode 100644 index 0000000..16a8802 --- /dev/null +++ b/Apps/LBM-DEM/periodic.cpp @@ -0,0 +1,1254 @@ +/** + * Copyright 2019 United Kingdom Research and Innovation + * + * Authors: See AUTHORS + * + * Contact: [jianping.meng@stfc.ac.uk and/or jpmeng@gmail.com] + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice + * this list of conditions and the following disclaimer in the documentation + * and or other materials provided with the distribution. + * 3. Neither the name of the copyright holder nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * ANDANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +/*! + * @brief Functions for the implementation of periodic conditions at DEM-LBM simulations + * @author Chrysovalantis Tsigginos + * @details Wrap and Kernel functions for the implementation of periodic B.C + */ + + +#include "periodic.h" +#include +#include + +#define BIG 1.0e8 + + +int periodicFlag = 0; +int periodic[3]; + +Real* cutDEM; +Real* xper; //Size of periodic domain in all directions + +int** procSend; //Proc that each process sends to +int** procRecv;//Proce that each process receives form +int** pbcFlag;// Flag for sending data through the periodic BC + +Real** slablo; //lower bound of slab that will be send in each swap +Real** slabhi; //upper bound of slab that will be send in each swap + +int*** sendList; //List of particles that will be transmitted to the periodic rank +int** maxSend; //maximum number of particles that each rank send at each swap +int** sendNum; //Number of particles send at each swap +int** recvNum; //Number of particles received at each swap +int** firstRecv; //First particle that received +int** reverseSendNum; +int** reverseRecvNum; + +Real sizeForward; //size of forward data send per particle +Real sizeReverse; //size of reverse data send per particle + +int maxSending; +int maxReceiving; + +Real* buffSend; //buffer for sending +Real* buffRecv; //buffer for receiving + +ops_halo* g_HalosX; //Halos in the x-directions +ops_halo_group g_HaloGroupsX; + +ops_halo* g_HalosY; //Halos in the y-direction +ops_halo_group g_HaloGroupsY; + +ops_halo* g_HalosZ; +ops_halo_group g_HaloGroupsZ; + + +void DefinePeriodicHaloTransfer() { + + int haloDepth, dp[SPACEDIM], dm[SPACEDIM]; + int baseTo[SPACEDIM], baseFrom[SPACEDIM], haloSize[SPACEDIM], dir[SPACEDIM]; + int nx, ny, nz; + int haloCrt; + int nHalosX = 0; + int nHalosY = 0; + int nHalosZ = 0; + + if (periodic[0] == 1) { + nHalosX = 2; + g_HalosX = new ops_halo[2]; + } + + if (periodic[1] == 1) { + nHalosY = 2; + g_HalosY = new ops_halo[2]; + } + +#ifdef OPS_3D + if (periodic[2] == 1) { + nHalosZ = 2; + g_HalosZ = new ops_halo[2]; + } +#endif + + if (BlockNum()>1) { + ops_printf("MP-LBM: ERROR: Periodic scheme requires 1 block \nEXITING\n"); + exit(EXIT_FAILURE); + } + + haloDepth = HaloPtNum(); + ops_printf("HaloDepth = %d\n", haloDepth); + nx = BlockSize(0)[0]; + dir[0] = 1; + ny = BlockSize(0)[1]; + dir[1] = 2; + + for (int idim = 0; idim < SPACEDIM; idim++) { + dp[idim] = haloDepth; + dm[idim] = - haloDepth; + } + +#ifdef OPS_3D + nz = BlockSize(0)[2]; + dir[2] = 3; +#endif + + for (int idim = 0; idim < SPACEDIM; idim++) { + if (periodic[idim] == 1) { + + if(idim == 0) { + baseFrom[0] = 0; + baseFrom[1] = 0; + + baseTo[0] = nx; + baseTo[1] = 0; + + haloSize[0] = 1; + haloSize[1] = ny; + +#ifdef OPS_3D + baseTo[2] = 0; + baseFrom[2] = 0; + haloSize[2] = nz; +#endif + + g_HalosX[0] = ops_decl_halo(g_fStage[0], g_fStage[0], haloSize, baseFrom, baseTo, dir, dir); + + //Setup the 2nd halo + baseFrom[0] = nx-1; + baseTo[0] = dm[0]; + + g_HalosX[1] = ops_decl_halo(g_fStage[0], g_fStage[0], haloSize, baseFrom, baseTo, dir, dir); + + g_HaloGroupsX = ops_decl_halo_group(2, g_HalosX); +#if DebugLevel >= 1 + ops_printf("Defined x-periodic halos\n"); +#endif + } + else if (idim == 1) { + + baseFrom[0] = dm[0]; + baseFrom[1] = 0; + + baseTo[0] = dm[0]; + baseTo[1] = ny; + + haloSize[0] = nx+ dp[0] -dm[0]; + haloSize[1] = 1; + +#ifdef OPS_3D + baseTo[2] = 0; + baseFrom[2] = 0; + haloSize[2] = nz; +#endif + + g_HalosY[0] = ops_decl_halo(g_fStage[0], g_fStage[0], haloSize, baseFrom, baseTo, dir, dir); + baseFrom[1] = ny+dm[1]; + baseTo[1] = dm[1]; + + g_HalosY[1] = ops_decl_halo(g_fStage[0], g_fStage[0], haloSize, baseFrom, baseTo, dir, dir); + + g_HaloGroupsY = ops_decl_halo_group(2, g_HalosY); + } +#ifdef OPS_3D + else if (idim ==2) { + baseFrom[0] = dm[0]; + baseFrom[1] = dm[1]; + baseFrom[2] = 0; + + baseTo[0] = dm[0]; + baseTo[1] = dm[1]; + baseTo[2] = nz; + + haloSize[0] = nx + dp[0] - dm[0]; + haloSize[1] = ny + dp[1] - dm[1]; + haloSize[2] = 1; + + g_HalosZ[0] = ops_decl_halo(g_fStage[0], g_fStage[0], haloSize, baseFrom, baseTo, dir, dir); + + baseFrom[2] = nz+dm[1]; + baseTo[2] = dm[1]; + + g_HalosZ[1] = ops_decl_halo(g_fStage[0], g_fStage[0], haloSize, baseFrom, baseTo, dir, dir); + g_HaloGroupsZ = ops_decl_halo_group(2, g_HalosZ); + } +#endif + + } + + } + +} + +void DestroyPeriodic() { + + free(cutDEM); + + free2d(procSend); + free2d(procRecv); + free2d(pbcFlag); + + free2d(maxSend); + free2d(slabhi); + free2d(slablo); + + free2d(sendNum); + free2d(recvNum); + free2d(firstRecv); + free2d(reverseSendNum); + free2d(reverseRecvNum); + + free(buffRecv); + free(buffSend); + free(xper); +} + +void PeriodicHaloTransfer() { + + if (periodic[0]==1) + ops_halo_transfer(g_HaloGroupsX); + + if (periodic[1]==1) + ops_halo_transfer(g_HaloGroupsY); + +#ifdef OPS_3D + if (periodic[2]==1) + ops_halo_transfer(g_HaloGroupsZ); +#endif + +} + + + +void ImplementBoundary3DPeriodic() { + + for (auto boundary : BlockBoundaries()) { + + int *range{BoundarySurfaceRange(boundary.blockIndex, + boundary.boundarySurface)}; + + if (boundary.boundaryScheme != BoundaryScheme::Periodic) { + TreatBlockBoundary3D(boundary.blockIndex, boundary.componentID, + boundary.givenVars.data(), range, + boundary.boundaryScheme, boundary.boundarySurface); + + } + } +} + +void InitializePeriodic() { + + int blockNum = BlockNum(); + + cutDEM = (Real *) malloc(SPACEDIM * sizeof(Real)); + int Nswap; + + procSend = (int **) malloc(blockNum * sizeof(int *)); + procRecv = (int **) malloc(blockNum * sizeof(int *)); + pbcFlag = (int **) malloc(blockNum * sizeof(int *)); + + slablo = (Real **) malloc(blockNum * sizeof(Real *)); + slabhi = (Real **) malloc(blockNum * sizeof(Real *)); + + sendList = (int ***) malloc(blockNum * sizeof(int **)); + maxSend = (int **) malloc(blockNum * sizeof(int *)); + + sendNum = (int **) malloc(blockNum * sizeof(int *)); + recvNum = (int **) malloc(blockNum * sizeof(int *)); + firstRecv = (int **) malloc(blockNum * sizeof(int *)); + + reverseSendNum = (int **) malloc(blockNum * sizeof(int *)); + reverseRecvNum = (int **) malloc(blockNum * sizeof(int *)); + + + for (int blockIndex = 0; blockIndex < blockNum; blockIndex++) { + + allocateMemory(procSend[blockIndex], 2 * SPACEDIM, 1); + allocateMemory(procRecv[blockIndex], 2 * SPACEDIM, 1); + + Nswap = 2 * SPACEDIM; + + allocateMemory(maxSend[blockIndex], Nswap, 1); + allocateMemory(pbcFlag[blockIndex], Nswap, SPACEDIM); + allocateMemory(slabhi[blockIndex], Nswap , SPACEDIM); + allocateMemory(slablo[blockIndex], Nswap, SPACEDIM); + + allocateMemory(sendNum[blockIndex], Nswap, 1); + allocateMemory(recvNum[blockIndex], Nswap, 1); + allocateMemory(firstRecv[blockIndex], Nswap, 1); + allocateMemory(reverseSendNum[blockIndex], Nswap, 1); + allocateMemory(reverseRecvNum[blockIndex], Nswap, 1); + + sendList[blockIndex] = (int **) malloc (Nswap * sizeof(int *)); + for (int iswap = 0; iswap < Nswap; iswap++) { + allocateMemory(sendList[blockIndex][iswap],100, 1); + maxSend[blockIndex][iswap] = 100; + } + } + + if (SPACEDIM == 2) { + sizeForward = 6; + sizeReverse = 3; + } + else if (SPACEDIM == 3) { + sizeForward = 10; + sizeReverse = 6; + } + else { + ops_printf("MP-LBM ERROR: Inconsistent dimension\n"); + exit(EXIT_FAILURE); + } + + maxSending = sizeForward * maxSend[0][0]; + allocateMemory(buffSend, maxSending, 1); + maxReceiving = sizeForward * maxSend[0][0]; + allocateMemory(buffRecv, maxReceiving, 1); + + allocateMemory(xper, SPACEDIM, 1); + +} + +void DefinePeriodicBoundariesRestart(std::vector& boundaryType,std::vector &dir) { + + int direction; + + for (int iDir = 0; iDir < SPACEDIM; iDir++) { + + direction = dir[iDir]; + if ((boundaryType[2*iDir] == BoundaryScheme::Periodic) && (boundaryType[2 * iDir + 1]== BoundaryScheme::Periodic)) + periodic[direction] = 1; + else if ((boundaryType[2 * iDir] == BoundaryScheme::Periodic) && (boundaryType[2 * iDir + 1] != BoundaryScheme::Periodic)) { + ops_printf("MP-LBM3D: Inconsistent %d periodic boundary condition\n", direction); + exit(EXIT_FAILURE); + } + else if ((boundaryType[2 * iDir] != BoundaryScheme::Periodic) &&(boundaryType[2 * iDir + 1] == BoundaryScheme::Periodic)) { + ops_printf("MP-LBM3D: Inconsistent %d periodic boundary condition\n", direction); + exit(EXIT_FAILURE); + } + else + periodic[direction] = 0; + + } + + periodicFlag = 0; + + if ((periodic[0]==1) || (periodic[1] == 1) || (periodic[2] == 1)) + periodicFlag = 1; + + + ops_printf("Periodic Flag is %d\n", periodicFlag); + ops_printf("x-Per: %d y-Per: %d z-Per: %d\n", periodic[0], periodic[1], periodic[2]); + + if (periodicFlag==1) + ops_printf("x-Per: %d y-Per: %d z-Per: %d\n", periodic[0], periodic[1], periodic[2]); + + //Define periodic Partition + if (periodicFlag == 1) + DefinePeriodicHaloTransfer(); + + ops_printf("PeriodicFlag = %d\n",periodicFlag); + +} + +void DefinePeriodicBoundaries(BoundaryScheme* boundaryType) { + + //Check for periodic Boundary conditions + if ((boundaryType[0] == BoundaryScheme::Periodic) && (boundaryType[1] == BoundaryScheme::Periodic)) + periodic[0] = 1; + else if ((boundaryType[0] == BoundaryScheme::Periodic) && (boundaryType[1] != BoundaryScheme::Periodic)) { + ops_printf("MP-LBM3D: Inconsistent x-periodic boundary condition\n"); + exit(EXIT_FAILURE); + } + else if ((boundaryType[0] != BoundaryScheme::Periodic) &&(boundaryType[1] == BoundaryScheme::Periodic)) { + ops_printf("MP-LBM3D: Inconsistent x-periodic boundary condition\n"); + exit(EXIT_FAILURE); + } + else + periodic[0] = 0; + + + + if ((boundaryType[2] == BoundaryScheme::Periodic) && (boundaryType[3] == BoundaryScheme::Periodic)) + periodic[1] = 1; + else if ((boundaryType[2] != BoundaryScheme::Periodic) && (boundaryType[3] == BoundaryScheme::Periodic)) { + ops_printf("MP-LBM3D: Inconsistent y-periodic boundary condition\n"); + exit(EXIT_FAILURE); + } + else if ((boundaryType[2] == BoundaryScheme::Periodic) && (boundaryType[3] != BoundaryScheme::Periodic)) { + ops_printf("MP-LBM3D: Inconsistent y-periodic boundary condition\n"); + exit(EXIT_FAILURE); + } + else + periodic[1] = 0; + + if ((boundaryType[4] == BoundaryScheme::Periodic) && (boundaryType[5] == BoundaryScheme::Periodic)) + periodic[2] = 1; + else if ((boundaryType[4] != BoundaryScheme::Periodic) && (boundaryType[5] == BoundaryScheme::Periodic)) { + ops_printf("MP-LBM3D: Inconsistent z-periodic boundary condition\n"); + exit(EXIT_FAILURE); + } + else if ((boundaryType[4] == BoundaryScheme::Periodic) && (boundaryType[5] != BoundaryScheme::Periodic)) { + ops_printf("MP-LBM3D: Inconsistent z-periodic boundary condition\n"); + exit(EXIT_FAILURE); + } + else + periodic[2] = 0; + + periodicFlag = 0; + + if ((periodic[0]==1) || (periodic[1] == 1) || (periodic[2] == 1)) + periodicFlag = 1; + + + ops_printf("Periodic Flag is %d\n", periodicFlag); + ops_printf("x-Per: %d y-Per: %d z-Per: %d\n", periodic[0], periodic[1], periodic[2]); + + if (periodicFlag==1) + ops_printf("x-Per: %d y-Per: %d z-Per: %d\n", periodic[0], periodic[1], periodic[2]); + + //Define periodic Partition + if (periodicFlag == 1) + DefinePeriodicHaloTransfer(); + + ops_printf("PeriodicFlag = %d\n",periodicFlag); +} + + +void InitializePeriodicDragForce() { + + int nlocal; + int sizeFd; + if (SPACEDIM == 2) + sizeFd = SPACEDIM + 1; + if (SPACEDIM == 3) + sizeFd = 2 * SPACEDIM + 1; + + for (int blockIndex = 0; blockIndex < BlockNum(); blockIndex++) { +#ifdef OPS_MPI + sub_block_list sb = OPS_sub_block_list[blockIndex]; + if (!sb->owned) continue; +#endif + + nlocal = Nparticles[blockIndex] + Nperiodic[blockIndex]; + + for (int iPar = Nparticles[blockIndex]; iPar < nlocal; iPar++) { + for (int iDir = 0; iDir < sizeFd; iDir++) + FDrag[blockIndex][sizeFd * iPar + iDir] = 0.0; + + } + } + +} + + +#ifdef OPS_2D +void SetPeriodicSize2D(Real x1,Real y1) { + xper[0] = x1; + xper[1] = y1; +} + +void SetCutOff2D(Real xcut, Real ycut) { + + cutDEM[0] = xcut; + cutDEM[1] = ycut; +} +#endif + + +#ifdef OPS_3D +void SetPeriodicSize3D(Real x1, Real y1, Real z1) { + + xper[0] = x1; + xper[1] = y1; + xper[2] = z1; +} + +void SetCutOff3D(Real xcut, Real ycut, Real zcut) { + + cutDEM[0] = xcut; + cutDEM[1] = ycut; + cutDEM[2] = zcut; +} +#endif + +void PeriodicPartition() { + + int periods[SPACEDIM], coords[SPACEDIM]; + int coordPeriodicLeft[SPACEDIM], coordPeriodicRight[SPACEDIM]; + int size1[SPACEDIM]; + int rankPeriodic; + Real perNeigh[SPACEDIM]; + int periodicNeighborRight[BlockNum()][SPACEDIM]; + int periodicNeighborLeft[BlockNum()][SPACEDIM]; + Real dx = dxLBM(); + + for (int blockIndex = 0; blockIndex < BlockNum(); blockIndex++) { + +#ifdef OPS_MPI + sub_block_list sb = OPS_sub_block_list[blockIndex]; + if (!sb->owned) continue; +#else + ops_printf("OPS-LBM-MUI: Error Require MPI version.\n"); + exit(EXIT_FAILURE); +#endif + +#if DebugLevel >= 2 + if (SPACEDIM==2) + ops_printf("Periodic x: %d Periodic y: %d\n",periodic[0], periodic[1]); + else if (SPACEDIM == 3) + ops_printf("Periodic x: %d Periodic y: %d Periodic z: %d\n", periodic[0], periodic[1], periodic[2]); +#endif + MPI_Cart_get(sb->comm, SPACEDIM, size1, periods, coords); + + if (periodic[0] == 1) { //x-periodic + if (size1[0] == 1) { //return same process + MPI_Cart_rank(sb->comm, coords, &rankPeriodic); + periodicNeighborRight[blockIndex][0] = rankPeriodic; + periodicNeighborLeft[blockIndex][0] = rankPeriodic; + } + else { + coordPeriodicLeft[1] = coords[1]; + coordPeriodicRight[1] = coords[1]; +#ifdef OPS_3D + coordPeriodicLeft[2] = coords[2]; + coordPeriodicRight[2] = coords[2]; +#endif + if (coords[0] == 0) { //Rank on the left boundary + coordPeriodicLeft[0] = size1[0]-1; + MPI_Cart_rank(sb->comm, coordPeriodicLeft, &rankPeriodic); + periodicNeighborLeft[blockIndex][0] = rankPeriodic; + periodicNeighborRight[blockIndex][0] = -1; + } + else if (coords[0] == size1[0] - 1) { + coordPeriodicRight[0] = 0; + MPI_Cart_rank(sb->comm, coordPeriodicRight, &rankPeriodic); + periodicNeighborLeft[blockIndex][0] = -1; + periodicNeighborRight[blockIndex][0] = rankPeriodic; + } + else { + periodicNeighborLeft[blockIndex][0] = -1; + periodicNeighborRight[blockIndex][0] = -1; + } + } + + } + else { + periodicNeighborLeft[blockIndex][0] = -1; + periodicNeighborRight[blockIndex][0] =-1; + } + + if (periodic[1] == 1) { //y-periodic B.C + if (size1[1] == 1) {//Return same process on MPI_Comm sb->comm. + MPI_Cart_rank(sb->comm, coords, &rankPeriodic); + periodicNeighborRight[blockIndex][1] = rankPeriodic; + periodicNeighborLeft[blockIndex][1] = rankPeriodic; + } + else { + coordPeriodicLeft[0] = coords[0]; + coordPeriodicRight[0] = coords[0]; +#ifdef OPS_3D + + coordPeriodicLeft[2] = coords[2]; + coordPeriodicRight[2] = coords[2]; +#endif + if (coords[1] == 0) { //Rank on left boundary + coordPeriodicLeft[1] = size1[1] - 1; + MPI_Cart_rank(sb->comm, coordPeriodicLeft, &rankPeriodic); + periodicNeighborLeft[blockIndex][1] = rankPeriodic; + periodicNeighborRight[blockIndex][1] = -1; + } + else if (coords[1] == size1[1] - 1) { + coordPeriodicRight[1] = 0; + MPI_Cart_rank(sb->comm, coordPeriodicRight, &rankPeriodic); + periodicNeighborRight[blockIndex][1] = rankPeriodic; + periodicNeighborLeft[blockIndex][1] = -1; + + } + else { + periodicNeighborLeft[blockIndex][1] = -1; + periodicNeighborRight[blockIndex][1] = -1; + } + + } + + } + else { + periodicNeighborLeft[blockIndex][1] = -1; + periodicNeighborRight[blockIndex][1] = -1; + } +#ifdef OPS_3D + if (periodic[2] == 1) { //z-periodic B.C + if (size1[2] == 1) { + MPI_Cart_rank(sb->comm, coords, &rankPeriodic); + periodicNeighborRight[blockIndex][2] = rankPeriodic; + periodicNeighborLeft[blockIndex][2] = rankPeriodic; + } + else { + coordPeriodicRight[0] = coords[0]; + coordPeriodicLeft[0] = coords[0]; + + coordPeriodicRight[1] = coords[1]; + coordPeriodicLeft[1] = coords[1]; + + if (coords[2] == 0) { //Rank on bottom boundary + coordPeriodicLeft[2] = size1[2] - 1; + MPI_Cart_rank(sb->comm, coordPeriodicLeft, &rankPeriodic); + periodicNeighborLeft[blockIndex][2] = rankPeriodic; + periodicNeighborRight[blockIndex][2] = -1; + } + else if (coords[2] == size1[2]-1) { + coordPeriodicRight[2] = 0; + MPI_Cart_rank(sb->comm, coordPeriodicRight, &rankPeriodic); + periodicNeighborLeft[blockIndex][2] = -1; + periodicNeighborRight[blockIndex][2] = rankPeriodic; + + } + else { //no periodic process + periodicNeighborRight[blockIndex][2] = -1; + periodicNeighborLeft[blockIndex][2] = -1; + } + + } + } + else { + periodicNeighborRight[blockIndex][2] = -1; + periodicNeighborLeft[blockIndex][2] = -1; + } +#endif + + int globalRank = ops_get_proc(); + int localRank; + MPI_Comm_rank(sb->comm, &localRank); + + /* + * Identifying the periodic swaps + * Two swaps per direction + * 1st swap: Receives from left sends to right + * 2nd swap: Receives from right sends to left + * + */ + + int nswap = 0; + for (int idim = 0; idim < SPACEDIM; idim++) { + for (int iswap = 0; iswap < 2; iswap++) { + + //Verify size of slablo and slabhi + pbcFlag[blockIndex][nswap * SPACEDIM] = 0; + pbcFlag[blockIndex][nswap * SPACEDIM + 1] = 0; + + slablo[blockIndex][nswap * SPACEDIM] = xBoundLocal[2 * SPACEDIM * blockIndex];// + 0.5 * dx; + slablo[blockIndex][nswap * SPACEDIM + 1] = xBoundLocal[2 * SPACEDIM * blockIndex + 2];// + 0.5 * dx; + + slabhi[blockIndex][nswap * SPACEDIM] = xBoundLocal[2 * SPACEDIM * blockIndex + 1];// - 0.5 * dx; + slabhi[blockIndex][nswap * SPACEDIM + 1] = xBoundLocal[2 * SPACEDIM * blockIndex + 3];// - 0.5 * dx; + +#ifdef OPS_3D + pbcFlag[blockIndex][nswap * SPACEDIM + 2] = 0; + + slablo[blockIndex][nswap * SPACEDIM + 2] = xBoundLocal[2 * SPACEDIM * blockIndex + 4];// + 0.5 * dx; + slabhi[blockIndex][nswap * SPACEDIM + 2] = xBoundLocal[2 * SPACEDIM * blockIndex + 5];// - 0.5 * dx; +#endif + + procSend[blockIndex][nswap] = -1; + procRecv[blockIndex][nswap] = -1; + + + if (periodic[idim] == 1) { + if ( (iswap % 2) == 0) { + procSend[blockIndex][nswap] = periodicNeighborRight[blockIndex][idim]; + procRecv[blockIndex][nswap] = periodicNeighborLeft[blockIndex][idim]; + if (coords[idim] == size1[idim]-1) {//Only this one will send + pbcFlag[blockIndex][nswap * SPACEDIM + idim] = -1; + slablo[blockIndex][nswap * SPACEDIM + idim] = slabhi[blockIndex][nswap * SPACEDIM + idim] - cutDEM[idim]; + slabhi[blockIndex][nswap * SPACEDIM + idim] = BIG; + } + + } + else { + procSend[blockIndex][nswap] = periodicNeighborLeft[blockIndex][idim]; + procRecv[blockIndex][nswap] = periodicNeighborRight[blockIndex][idim]; + if (coords[idim] == 0) { + pbcFlag[blockIndex][nswap * SPACEDIM + idim] = 1; + slabhi[blockIndex][nswap * SPACEDIM + idim] = slablo[blockIndex][nswap * SPACEDIM + idim] + cutDEM[idim]; + slablo[blockIndex][nswap * SPACEDIM + idim] = -BIG; + } + } + + } + nswap++; + } + + } + +#if DebugLevel >= 2 + printf("\n\n-----------------------------------------------------------------------------------------------------------------\n"); + printf("Local rank %d: Swap 0: Recv %d Send %d, Swap 1: Recv %d Send %d, Swap 2: Recv %d Send %d, Swap 3: Recv %d Send %d Swap 4 Recv %d Send %d, Swap 5 %d %d\n", + localRank,procRecv[blockIndex][0], procSend[blockIndex][0], procRecv[blockIndex][1], procSend[blockIndex][1], + procRecv[blockIndex][2], procSend[blockIndex][2], procRecv[blockIndex][3], procSend[blockIndex][3], + procRecv[blockIndex][4], procSend[blockIndex][4], procRecv[blockIndex][5], procSend[blockIndex][5]); + + printf("\n\n Local rank %d: Swap 0: pbcFlag [%d %d %d], Swap 1: pbcFlag [%d %d %d], Swap 2: pbcFlag [%d %d %d], Swap 3: pbcFlag [%d %d %d], Swap 4: pbcFlag [%d %d %d], Swap 5: pbcFlag[%d %d %d]\n", + localRank, pbcFlag[blockIndex][0 * SPACEDIM], pbcFlag[blockIndex][0 * SPACEDIM + 1], pbcFlag[blockIndex][0 * SPACEDIM + 2], + pbcFlag[blockIndex][1 * SPACEDIM], pbcFlag[blockIndex][1 * SPACEDIM + 1], pbcFlag[blockIndex][1 * SPACEDIM + 2], + pbcFlag[blockIndex][2 * SPACEDIM], pbcFlag[blockIndex][2 * SPACEDIM + 1], pbcFlag[blockIndex][2 * SPACEDIM + 2], + pbcFlag[blockIndex][3 * SPACEDIM], pbcFlag[blockIndex][3 * SPACEDIM + 1], pbcFlag[blockIndex][3 * SPACEDIM + 2], + pbcFlag[blockIndex][4 * SPACEDIM], pbcFlag[blockIndex][4 * SPACEDIM + 1], pbcFlag[blockIndex][4 * SPACEDIM + 2], + pbcFlag[blockIndex][5 * SPACEDIM], pbcFlag[blockIndex][5 * SPACEDIM + 1], pbcFlag[blockIndex][5 * SPACEDIM + 2]); + printf("\n\nLocal Rank %d:", localRank); + for (int iswap = 0; iswap < nswap; iswap++) { + if (procSend[blockIndex][iswap] > -1) { + printf("Swap %d: boxSlab [%f %f]x[%f %f] x[%f %f] ",iswap, slablo[blockIndex][iswap * SPACEDIM], slabhi[blockIndex][iswap * SPACEDIM], + slablo[blockIndex][iswap * SPACEDIM + 1], slabhi[blockIndex][iswap * SPACEDIM + 1], + slablo[blockIndex][iswap * SPACEDIM + 2], slabhi[blockIndex][iswap * SPACEDIM + 2]); + } + } + + printf("\n"); + printf("---------------------------------------------------------------------------------------------------------------------\n"); + + +/* + * NOTES: Function periodicNeighborLeft, periodicNeighborRight + */ +#endif + } + +} + +bool decide(Real xp, Real xlo, Real xhi) { + + if ((xlo < xp)&&(xp <= xhi)) + return true; + + return false; +} + +void PeriodicExchange() { + + int idim; + int nfirst, nlast; + int nsend, nrecv1, nrecv2, nrecv; + int meLocal; + Real lo, hi; + Real xl; + bool flag; + Real *buf; + MPI_Status status; + MPI_Request request; + ops_printf("Entering PEriodic Exchange\n"); + + for (int blockIndex = 0; blockIndex < BlockNum(); blockIndex++) { +#ifdef OPS_MPI + sub_block_list sb = OPS_sub_block_list[blockIndex]; + if (!sb->owned) continue; +#endif + + MPI_Comm_rank(sb->comm, &meLocal); + Nperiodic[blockIndex] = 0; + idim = -1; + nlast = 0; + + for (int iswap = 0; iswap < 2 * SPACEDIM; iswap++) { + //First and second swap transfer only local particles + //The rest of the swaps include transfers and periodic also. + if (iswap % 2 ==0) { + idim++; + nfirst = 0; + nlast = Nparticles[blockIndex] + Nperiodic[blockIndex]; +#if DebugLevel >= 2 + printf("Rank %d: Swap %d: nlast = %d\n", ops_get_proc(), iswap, nlast); +#endif + } + lo = slablo[blockIndex][iswap * SPACEDIM + idim]; + hi = slabhi[blockIndex][iswap * SPACEDIM + idim]; + + if (periodic[idim] == 0) { + sendNum[blockIndex][iswap] = 0; + recvNum[blockIndex][iswap] = 0; + reverseSendNum[blockIndex][iswap] = 0; + reverseRecvNum[blockIndex][iswap] = 0; + firstRecv[blockIndex][iswap] = -1; + continue; + } + + nsend = 0; + nrecv = 0; + + if (procSend[blockIndex][iswap] > -1) { + for (int iParticle = nfirst; iParticle < nlast; iParticle++) { + if (idim == 0) + xl = xp[blockIndex][iParticle]; + else if (idim == 1) + xl = yp[blockIndex][iParticle]; +#ifdef OPS_3D + else if (idim == 2) + xl = zp[blockIndex][iParticle]; +#endif + flag = decide(xl, lo, hi); + + if (flag) { + if (nsend == maxSend[blockIndex][iswap]) { + Reallocate(sendList[blockIndex][iswap], nsend+100, 1); + maxSend[blockIndex][iswap] = nsend + 100; + } + sendList[blockIndex][iswap][nsend++] = iParticle; + } + } + + if (nsend * sizeForward > maxSending) { + maxSending = nsend * sizeForward; + Reallocate(buffSend, sizeForward * nsend, 1); + Reallocate(buffRecv, sizeForward * nsend, 1); + } + + if (nsend > 0) { +#ifdef OPS_2D + packForward2D(blockIndex, nsend, iswap, sendList[blockIndex][iswap], buffSend, periodic, pbcFlag[blockIndex]); +#endif + +#ifdef OPS_3D + packForward3D(blockIndex, nsend, iswap, sendList[blockIndex][iswap], buffSend, periodic, pbcFlag[blockIndex]); +#endif + } + + if (procSend[blockIndex][iswap] != meLocal) + MPI_Send(&nsend, 1, MPI_INT, procSend[blockIndex][iswap],0, sb->comm); + + } + + if (procRecv[blockIndex][iswap] > -1) { + if (meLocal == procRecv[blockIndex][iswap]) + nrecv = nsend; + else + MPI_Recv(&nrecv, 1, MPI_INT, procRecv[blockIndex][iswap], 0, sb->comm, &status); + } + + if ((nrecv > 0) || (nsend > 0)) { + if (procSend[blockIndex][iswap] == meLocal) + buf = buffSend; + else { + if (nrecv * sizeForward > maxReceiving) { + maxReceiving = nrecv * sizeForward; + Reallocate(buffRecv, sizeForward * nrecv, 1); + Reallocate(buffSend, sizeForward * nrecv, 1); + } + + if (nrecv) + MPI_Irecv(buffRecv, nrecv * sizeForward, MPI_DOUBLE, procRecv[blockIndex][iswap], 0,sb->comm, &request); + + if (nsend) + MPI_Send(buffSend,nsend * sizeForward,MPI_DOUBLE,procSend[blockIndex][iswap],0,sb->comm); + + if (nrecv) + MPI_Wait(&request, &status); + + buf = buffRecv; + } + + //Unpack buffer + if (nrecv) { +#ifdef OPS_2D + unpackComm2D(blockIndex, nrecv, Nparticles[blockIndex] + Nperiodic[blockIndex], buf); +#endif + +#ifdef OPS_3D + unpackComm3D(blockIndex, nrecv, Nparticles[blockIndex] + Nperiodic[blockIndex], buf); +#endif + } + + } + + sendNum[blockIndex][iswap] = nsend; + recvNum[blockIndex][iswap] = nrecv; + + reverseSendNum[blockIndex][iswap] = nrecv; + reverseRecvNum[blockIndex][iswap] = nsend; + + if (recvNum[blockIndex][iswap] > 0) { + firstRecv[blockIndex][iswap] = Nparticles[blockIndex] + Nperiodic[blockIndex]; + Nperiodic[blockIndex] += nrecv; + } + else { + firstRecv[blockIndex][iswap] = -1; + } + + } + } + +#if DebugLevel >= 2 + printf("Rank %d: Actual Particles %d: Periodic particles %d\n", meLocal, Nparticles[0], Nperiodic[0]); + for (int iswap = 0; iswap < 2 * SPACEDIM; iswap++) + printf("Rank: %d, Swap: %d, sendNum: %d, recvNum: %d, firstReceive: %d\n",meLocal, iswap, sendNum[0][iswap], + recvNum[0][iswap], firstRecv[0][iswap]); + + ops_printf("Printing Particles\n"); + for (int iPar = 0; iPar < Nparticles[0] + Nperiodic[0]; iPar++) { + if (iPar < Nparticles[0]) + printf("Rank %d, Particle %i (Actual) x = %f y=%f z=%f\n",meLocal, iPar, xp[0][iPar], yp[0][iPar],zp[0][iPar]); + else + printf("Rank %d, Particle %i (Periodic) x= %f y=%f z=%f\n", meLocal, iPar, xp[0][iPar], yp[0][iPar],zp[0][iPar]); + } +#endif + +} + +void ForwardComm() { + + MPI_Request request; + MPI_Status status; + Real* buf; + int idim, meLocal, sizeRcv, sizeSnd; + for (int blockIndex = 0; blockIndex < BlockNum(); blockIndex++) { +#ifdef OPS_MPI + sub_block_list sb = OPS_sub_block_list[blockIndex]; + if (!sb->owned) continue; +#endif + + idim = -1; + MPI_Comm_rank(sb->comm, &meLocal); + for (int iswap = 0; iswap < 2 * SPACEDIM; iswap++) { + + if (iswap % 2 == 0) + idim ++; + if (periodic[idim] == 0) //Non-periodic in direction idim + continue; + + if ((sendNum[blockIndex][iswap] > 0) || (recvNum[blockIndex][iswap] > 0)) { + + if (recvNum[blockIndex][iswap] > 0) { //Data will be received in this swap + if (procRecv[blockIndex][iswap] != meLocal) { + sizeRcv = recvNum[blockIndex][iswap] * sizeForward; + MPI_Irecv(buffRecv, sizeRcv, MPI_DOUBLE, procRecv[blockIndex][iswap], 0, sb->comm, &request); + } + } + //packing data to be send + if (sendNum[blockIndex][iswap] > 0) {//Data will be send in this swap +#ifdef OPS_2D + packForward2D(blockIndex, sendNum[blockIndex][iswap], iswap, sendList[blockIndex][iswap], buffSend, periodic, pbcFlag[blockIndex]); +#endif + +#ifdef OPS_3D + packForward3D(blockIndex, sendNum[blockIndex][iswap], iswap, sendList[blockIndex][iswap], buffSend, periodic, pbcFlag[blockIndex]); +#endif + if (procSend[blockIndex][iswap] != meLocal) { + sizeSnd = sendNum[blockIndex][iswap] * sizeForward; + MPI_Send(buffSend, sizeSnd, MPI_DOUBLE, procSend[blockIndex][iswap], 0, sb->comm); + } + } + + if (recvNum[blockIndex][iswap] > 0) { + if (procRecv[blockIndex][iswap] == meLocal) + buf = buffSend; + else { + MPI_Wait(&request, &status); + buf = buffRecv; + } +#ifdef OPS_2D + unpackComm2D(blockIndex, recvNum[blockIndex][iswap], firstRecv[blockIndex][iswap], buf); +#endif + +#ifdef OPS_3D + unpackComm3D(blockIndex, recvNum[blockIndex][iswap], firstRecv[blockIndex][iswap], buf); +#endif + } + + + } + + } + } +} + +void ReverseComm() { + + int idim, meLocal, size; + Real * buff; + MPI_Status status; + MPI_Request request; + + for (int blockIndex = 0; blockIndex < BlockNum(); blockIndex++) { +#ifdef OPS_MPI + sub_block_list sb = OPS_sub_block_list[blockIndex]; + if (!sb->owned) continue; +#endif + + + idim = SPACEDIM; + MPI_Comm_rank(sb->comm, &meLocal); + for (int iswap = 2 * SPACEDIM - 1; iswap >= 0; iswap--) { + if (iswap %2 != 0) + idim--; + + if (periodic[idim] == 0) + continue; + + //printf("Rank %d Swap %d: Receiving number %d\n",ops_get_proc(), iswap, reverseRecvNum[blockIndex][iswap]); + if (reverseRecvNum[blockIndex][iswap] > 0) { + if (procSend[blockIndex][iswap] != meLocal) { + size = reverseRecvNum[blockIndex][iswap] * sizeReverse; + MPI_Irecv(buffRecv,size,MPI_DOUBLE, procSend[blockIndex][iswap],0,sb->comm,&request); + + } + } + + if (reverseSendNum[blockIndex][iswap] > 0) { +#ifdef OPS_2D + packReverse2D(blockIndex, reverseSendNum[blockIndex][iswap], firstRecv[blockIndex][iswap],buffSend); +#endif + +#ifdef OPS_3D + packReverse3D(blockIndex, reverseSendNum[blockIndex][iswap], firstRecv[blockIndex][iswap],buffSend); +#endif + if (procRecv[blockIndex][iswap] != meLocal) { + size = reverseSendNum[blockIndex][iswap] * sizeReverse; + MPI_Send(buffSend, size ,MPI_DOUBLE, procRecv[blockIndex][iswap],0,sb->comm); + } + } + + if (reverseRecvNum[blockIndex][iswap] > 0) { + if (procSend[blockIndex][iswap] != meLocal) { + MPI_Wait(&request, &status); + buff = buffRecv; + } + else + buff = buffSend; +#ifdef OPS_2D + unPackReverse2D(blockIndex, reverseRecvNum[blockIndex][iswap], sendList[blockIndex][iswap], buff); +#endif + +#ifdef OPS_3D + unPackReverse3D(blockIndex, reverseRecvNum[blockIndex][iswap], sendList[blockIndex][iswap], buff); +#endif + } + + } + } + + +} + +#ifdef OPS_2D +void packForward2D(int blockIndex, int nsend, int iswap, int* sendList, Real* buff, int* periodic, int* pbc) { + + Real dx, dy; + int iPar; + dx = static_cast(pbc[iswap * SPACEDIM + 0]) * xper[0]; + dy = static_cast(pbc[iswap * SPACEDIM + 1]) * xper[1]; + + int m = 0; + for (int ibuff = 0; ibuff < nsend; ibuff++) { + iPar = sendList[ibuff]; + buff[m++] = xp[blockIndex][iPar] + dx; + buff[m++] = yp[blockIndex][iPar] + dy; + buff[m++] = Radius[blockIndex][iPar]; + buff[m++] = up[blockIndex][iPar]; + buff[m++] = vp[blockIndex][iPar]; + buff[m++] = omegaZ[blockIndex][iPar]; + + } + +} + + +void unpackComm2D(int blockIndex, int nrecv, int first, Real *buf) { + + int m = 0; + int last = first + nrecv; + + for (int iPar = first; iPar < last; iPar++) { + if (Nmax[blockIndex] == iPar) + reallocate2dMemory(100, blockIndex); + xp[blockIndex][iPar] = buf[m++]; + yp[blockIndex][iPar] = buf[m++]; + Radius[blockIndex][iPar] = buf[m++]; + up[blockIndex][iPar] = buf[m++]; + vp[blockIndex][iPar] = buf[m++]; + omegaZ[blockIndex][iPar] = buf[m++]; + } + +} + +void packReverse2D(int blockIndex, int nParticles, int nfirst, Real* buff) { + + int m = 0; + int nlast = nfirst + nParticles; + + for (int iPar = nfirst; iPar < nlast; iPar++) { + buff[m++] = FDrag[blockIndex][3*iPar]; + buff[m++] = FDrag[blockIndex][3 * iPar + 1]; + buff[m++] = FDrag[blockIndex][3 * iPar + 2]; + + FDrag[blockIndex][3 * iPar] = 0.0; + FDrag[blockIndex][3 * iPar + 1] = 0.0; + FDrag[blockIndex][3 * iPar + 2] = 0.0; + } + + +} + +#endif + + +#ifdef OPS_3D +void packForward3D(int blockIndex, int nsend, int iswap, int* sendList,Real* buff,int* periodic,int * pbc) { + + Real dx, dy, dz; + int iPar; + dx = static_cast(pbc[iswap * SPACEDIM + 0]) * xper[0]; + dy = static_cast(pbc[iswap * SPACEDIM + 1]) * xper[1]; + dz = static_cast(pbc[iswap * SPACEDIM + 2]) * xper[2]; + + int m = 0; + for (int ibuff = 0; ibuff < nsend; ibuff++) { + iPar = sendList[ibuff]; + buff[m++] = xp[blockIndex][iPar] + dx; + buff[m++] = yp[blockIndex][iPar] + dy; + buff[m++] = zp[blockIndex][iPar] + dz; + buff[m++] = Radius[blockIndex][iPar]; + + buff[m++] = up[blockIndex][iPar]; + buff[m++] = vp[blockIndex][iPar]; + buff[m++] = wp[blockIndex][iPar]; + + buff[m++] = omegaX[blockIndex][iPar]; + buff[m++] = omegaY[blockIndex][iPar]; + buff[m++] = omegaZ[blockIndex][iPar]; + } + + +} + + +void unpackComm3D(int blockIndex, int nrecv, int first, Real *buf) { + + int m = 0; + int last = first + nrecv; + + for (int iPar = first; iPar < last; iPar++) { + if (Nmax[blockIndex] == iPar) + ReAllocateMemory(100, blockIndex); + xp[blockIndex][iPar] = buf[m++]; + yp[blockIndex][iPar] = buf[m++]; + zp[blockIndex][iPar] = buf[m++]; + Radius[blockIndex][iPar] = buf[m++]; + up[blockIndex][iPar] = buf[m++]; + vp[blockIndex][iPar] = buf[m++]; + wp[blockIndex][iPar] = buf[m++]; + + omegaX[blockIndex][iPar] = buf[m++]; + omegaY[blockIndex][iPar] = buf[m++]; + omegaZ[blockIndex][iPar] = buf[m++]; + } + +} + +void packReverse3D(int blockIndex, int nParticles, int nfirst, Real* buff) { + + int m = 0; + int nlast = nfirst + nParticles; + + for (int iPar = nfirst; iPar < nlast; iPar++) { + buff[m++] = FDrag[blockIndex][6 * iPar]; + buff[m++] = FDrag[blockIndex][6 * iPar + 1]; + buff[m++] = FDrag[blockIndex][6 * iPar + 2]; + buff[m++] = FDrag[blockIndex][6 * iPar + 3]; + buff[m++] = FDrag[blockIndex][6 * iPar + 4]; + buff[m++] = FDrag[blockIndex][6 * iPar + 5]; + + FDrag[blockIndex][6 * iPar] = 0.0; + FDrag[blockIndex][6 * iPar + 1] = 0.0; + FDrag[blockIndex][6 * iPar + 2] = 0.0; + FDrag[blockIndex][6 * iPar + 3] = 0.0; + FDrag[blockIndex][6 * iPar + 4] = 0.0; + FDrag[blockIndex][6 * iPar + 5] = 0.0; + } + + +} + +void unPackReverse3D(int blockIndex, int nTot, int* sendList, Real* buff) { + int m = 0; + int jPar; + + + m = 0; + for (int iPar = 0; iPar < nTot; iPar++) { + jPar = sendList[iPar]; + + FDrag[blockIndex][6 * jPar] += buff[m++]; + + FDrag[blockIndex][6 * jPar + 1] += buff[m++]; + + FDrag[blockIndex][6 * jPar + 2] += buff[m++]; + + FDrag[blockIndex][6 * jPar + 3] += buff[m++]; + + FDrag[blockIndex][6 * jPar + 4] += buff[m++]; + + FDrag[blockIndex][6 * jPar + 5] += buff[m++]; + } +} + + +void SetupVertexType(VertexType* vertexType) { + if (periodicFlag == 0) { + for (int iDim = 0; iDim < 6; iDim++) { + vertexType[iDim] = VertexType::Wall; + } + } + else if (periodicFlag == 1) { + for (int iDim = 0; iDim < 3; iDim++) { + if (periodic[iDim]==1) { + vertexType[2*iDim] = VertexType::Periodic; + vertexType[2*iDim + 1] = VertexType::Periodic; + } + else { + vertexType[2*iDim] = VertexType::Wall; + vertexType[2*iDim + 1] = VertexType::Wall; + } + } + } +} + + +#endif diff --git a/Apps/LBM-DEM/periodic.h b/Apps/LBM-DEM/periodic.h new file mode 100644 index 0000000..028a310 --- /dev/null +++ b/Apps/LBM-DEM/periodic.h @@ -0,0 +1,92 @@ +/** + * Copyright 2019 United Kingdom Research and Innovation + * + * Authors: See AUTHORS + * + * Contact: [jianping.meng@stfc.ac.uk and/or jpmeng@gmail.com] + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice + * this list of conditions and the following disclaimer in the documentation + * and or other materials provided with the distribution. + * 3. Neither the name of the copyright holder nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * ANDANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +/*! + * @brief Functions for the implementation of periodic conditions at DEM-LBM simulations + * @author Chrysovalantis Tsigginos + * @details Wrap and Kernel functions for the implementation of periodic B.C + */ + +#ifndef PERIODIC_H_ +#define PERIODIC_H_ + +#include "type.h" +#include "flowfield.h" +#include "psm.h" +#include "memory_handle.h" +#include "evolution3d.h" +#include "evolution.h" + + +extern int periodicFlag; //0: No Periodic B.C in any direction 1: Periodic B.C at least in one direction +extern int periodic[3]; //Periodic Flag for each direction 1: Periodic B.C 0: No Periodic B.C + +void DefinePeriodicHaloTransfer(); +void PeriodicHaloTransfer(); +void InitializePeriodic(); +void DestroyPeriodic(); +void ImplementBoundary3DPeriodic(); + +void InitializePeriodicDragForce(); + +//Two dimension functions +void SetPeriodicSize2D(Real x1,Real y1); +void SetCutOff2D(Real xcut, Real ycut); + +//Three dimension functions +void SetPeriodicSize3D(Real x1,Real y1, Real z1); +void SetupVertexType(VertexType* vertexType); +void SetCutOff3D(Real xcut, Real ycut, Real zcut); + +void DefinePeriodicBoundaries(BoundaryScheme* boundaryType); +void DefinePeriodicBoundariesRestart(std::vector& types,std::vector& dir); + +bool decide(Real xp, Real xlo, Real xhi); + +void PeriodicPartition(); +void PeriodicExchange(); +void ForwardComm(); +void ReverseComm(); +//Packing functions +void packForward2D(int blockIndex, int nsend, int iswap, int* sendList, Real* buff, int* periodic, int* pbc); +void packForward3D(int blockIndex, int nsend, int iswap, int* sendList,Real* buff,int* periodic,int * pbc); + +void unpackComm2D(int blockIndex, int nrecv, int first, Real *buf); +void unpackComm3D(int blockIndex, int nrecv, int first, Real *buf); + +void packReverse2D(int blockIndex, int nParticles, int nfirst, Real* buff); +void packReverse3D(int blockIndex, int nParticles, int nfirst, Real* buff); +void unPackReverse2D(int blockIndex, int nTot, int* sendList, Real* buff); +void unPackReverse3D(int blockIndex, int nTot, int* sendList, Real* buff); + + +#endif /* APPS_LBM_DEM_PERIODIC_H_ */ diff --git a/Apps/LBM-DEM/poros_grid.cpp b/Apps/LBM-DEM/poros_grid.cpp new file mode 100644 index 0000000..1e8d336 --- /dev/null +++ b/Apps/LBM-DEM/poros_grid.cpp @@ -0,0 +1,228 @@ +/* + * poros_grid.cpp + * + * Created on: May 29, 2021 + * Author: jpd38567 + */ + +#include "poros_grid.h" +#include "flowfield.h" +#include "ops_seq_v2.h" +#include "block_particles.h" +#include +#include + +#include "ops_lib_core.h" +#ifdef OPS_MPI +#include "ops_mpi_core.h" +#endif + +#include "poros_grid.inc" +PorosGrid::PorosGrid(int particleshape, int spacedim) : + ParticleToGridBase( particleshape, spacedim) { + + //Assign variables + RealField poros{"SolFracGrid"}; + mappingRealVariableList.push_back(poros); + RealField vParts{"velPartGrid"}; + mappingRealVariableList.push_back(vParts); + IntField idParts{"idParticlesGrid"}; + mappingIntVariableList.push_back(idParts); + RealField xAverage{"xAvgGrid"}; + mappingRealVariableList.push_back(xAverage); + +} + +void PorosGrid::DefineVariables(int noelem, SizeType timeStep) { + + + noElem = noelem; + int size = noElem * spaceDim; + mappingRealVariableList.at(0).SetDataDim(noElem); + mappingRealVariableList.at(1).SetDataDim(size); + mappingRealVariableList.at(2).SetDataDim(size); + + for (auto& mappingVariable : mappingRealVariableList) { + if (timeStep == 0) { + mappingVariable.CreateFieldFromScratch(g_Block()); + } + else { + mappingVariable.CreateFieldFromFile(CaseName(), g_Block(), timeStep); + + } + + } + + mappingIntVariableList.at(0).SetDataDim(noElem); + if (timeStep == 0) + mappingIntVariableList.at(0).CreateFieldFromScratch(g_Block()); + else + mappingIntVariableList.at(0).CreateFieldFromFile(CaseName(), g_Block(), timeStep); + +} + +void PorosGrid::MappingFunction(bool flag) { + + if (flag) + ParticleProjection(); + else + UpdateProjection(); + +} + +void PorosGrid::ParticleProjection() { + + int size = noElem * spaceDim; + int idParticle; + int blockIndex; + const Real Dx {GetDx()}; + Real xPos[spaceDim], Radius, uParticle[spaceDim], omParticle[spaceDim]; + int stenList[2 * spaceDim]; + for (const auto& idBlock : BlockParticleList) { + BlockParticles ParticleCurrentBlock = idBlock.second; + if ( !ParticleCurrentBlock.owned ) continue; + blockIndex = ParticleCurrentBlock.GetBlock().ID(); + int nlocal = ParticleCurrentBlock.NParticles + + ParticleCurrentBlock.NPeriodic; + for (int iPart = 0; iPart < nlocal; iPart++) { + xPos[0] = ParticleCurrentBlock.particleList.at(iPart).xParticle[0]; + xPos[1] = ParticleCurrentBlock.particleList.at(iPart).xParticle[1]; + xPos[2] = ParticleCurrentBlock.particleList.at(iPart).xParticle[2]; + Radius = ParticleCurrentBlock.particleList.at(iPart).particleShape->Rparticle; + + uParticle[0] = ParticleCurrentBlock.particleList.at(iPart).uParticle[0]; + uParticle[1] = ParticleCurrentBlock.particleList.at(iPart).uParticle[1]; + uParticle[2] = ParticleCurrentBlock.particleList.at(iPart).uParticle[2]; + + omParticle[0] = ParticleCurrentBlock.particleList.at(iPart).omegaParticle[0]; + omParticle[1] = ParticleCurrentBlock.particleList.at(iPart).omegaParticle[1]; + omParticle[2] = ParticleCurrentBlock.particleList.at(iPart).omegaParticle[2]; + idParticle = iPart; + for (int iDir = 0; iDir < 2 * spaceDim; iDir++) + stenList[iDir] = ParticleCurrentBlock.particleList.at(iPart).stenList[iDir]; + + switch (particleDiscriptor) { + case (ParticleSpherical): + ops_par_loop(KerSolidFracGridSphere, "KerSolidFracGridSphere", + ParticleCurrentBlock.GetBlock().Get(), spaceDim, stenList, + ops_arg_dat(mappingIntVariableList.at(0).at(blockIndex), noElem, + LOCALSTENCIL,"int", OPS_RW), + ops_arg_dat(mappingRealVariableList.at(0).at(blockIndex), noElem, + LOCALSTENCIL,"double", OPS_RW), + ops_arg_dat(mappingRealVariableList.at(1).at(blockIndex), size, + LOCALSTENCIL,"double", OPS_RW), + ops_arg_dat(mappingRealVariableList.at(2).at(blockIndex), size, + LOCALSTENCIL,"double", OPS_RW), + ops_arg_dat(g_CoordinateXYZ()[blockIndex], spaceDim, + LOCALSTENCIL,"double", OPS_READ), + ops_arg_gbl(xPos, spaceDim, "double", OPS_READ), + ops_arg_gbl(&Radius, 1, "double", OPS_READ), + ops_arg_gbl(&idParticle, 1, "double", OPS_READ), + ops_arg_gbl(uParticle, spaceDim, "double", OPS_READ), + ops_arg_gbl(omParticle, spaceDim, "double", OPS_READ), + ops_arg_gbl(&Dx, 1, "double", OPS_READ), + ops_arg_gbl(&Npoints, 1, "double", OPS_READ), + ops_arg_gbl(&noElem, 1, "double", OPS_READ), + ops_arg_gbl(&spaceDim, 1, "double", OPS_READ)); + break; + default: + break; + } + + } + } + + + +} + +void PorosGrid::UpdateProjection() { + + int size = noElem * spaceDim; + Real velP[spaceDim], omP[spaceDim]; + Real xPar[spaceDim], radius; + int stenList[2 * spaceDim]; + int nlocal; + int idParticle; + Real dx = GetDx(); + for (const auto& idBlock : BlockParticleList) { + BlockParticles ParticleCurrentBlock = idBlock.second; + if ( !ParticleCurrentBlock.owned ) continue; + int blockIndex = ParticleCurrentBlock.GetBlock().ID(); + int nlocal = ParticleCurrentBlock.NParticles + + ParticleCurrentBlock.NPeriodic; + for (int iPart = 0; iPart < nlocal; iPart++) { + xPar[0] = ParticleCurrentBlock.particleList.at(iPart).xParticle[0]; + xPar[1] = ParticleCurrentBlock.particleList.at(iPart).xParticle[1]; + xPar[2] = ParticleCurrentBlock.particleList.at(iPart).xParticle[2]; + radius = ParticleCurrentBlock.particleList.at(iPart).particleShape->Rparticle; + + velP[0] = ParticleCurrentBlock.particleList.at(iPart).uParticle[0]; + velP[1] = ParticleCurrentBlock.particleList.at(iPart).uParticle[1]; + velP[2] = ParticleCurrentBlock.particleList.at(iPart).uParticle[2]; + + omP[0] = ParticleCurrentBlock.particleList.at(iPart).omegaParticle[0]; + omP[1] = ParticleCurrentBlock.particleList.at(iPart).omegaParticle[1]; + omP[2] = ParticleCurrentBlock.particleList.at(iPart).omegaParticle[2]; + idParticle = iPart; + for (int iDir = 0; iDir < 2 * spaceDim; iDir++) + stenList[iDir] = ParticleCurrentBlock.particleList.at(iPart).stenList[iDir]; + + switch (particleDiscriptor) { + case (ParticleSpherical): + ops_par_loop(KerSolidVelocityUpdate,"KerSolidVelocityUpdate", + ParticleCurrentBlock.GetBlock().Get(), spaceDim, stenList, + ops_arg_dat(mappingRealVariableList.at(1).at(blockIndex), size, + LOCALSTENCIL, "double", OPS_RW), + ops_arg_dat(mappingIntVariableList.at(0).at(blockIndex), noElem, + LOCALSTENCIL,"int", OPS_READ), + ops_arg_dat(mappingRealVariableList.at(2).at(blockIndex), size, + LOCALSTENCIL, "double", OPS_READ), + ops_arg_gbl(&idParticle, 1, "int", OPS_READ), + ops_arg_gbl(xPar, spaceDim, "double", OPS_READ), + ops_arg_gbl(&radius, 1, "double", OPS_READ), + ops_arg_gbl(velP, spaceDim, "double", OPS_READ), + ops_arg_gbl(omP, spaceDim, "double", OPS_READ), + ops_arg_gbl(&dx, 1, "double", OPS_READ), + ops_arg_gbl(&spaceDim, 1, "int", OPS_READ), + ops_arg_gbl(&noElem, 1, "int", OPS_READ)); + + break; + default: + ops_printf("Only spherical particles are implemented\n"); + } + } + } +} + +void PorosGrid::InitializeVariables() { + int size = noElem * spaceDim; + for (const auto& idBlock : BlockParticleList) { + BlockParticles ParticleCurrentBlock = idBlock.second; + std::vector iterRng; + iterRng.assign(ParticleCurrentBlock.GetBlock().WholeRange().begin(), + ParticleCurrentBlock.GetBlock().WholeRange().end()); + + const int blockIndex{ParticleCurrentBlock.GetBlock().ID()}; + ops_par_loop(KerInitializePorousGrid,"KerInitializePorousGrid", + ParticleCurrentBlock.GetBlock().Get(),spaceDim, iterRng.data(), + ops_arg_dat(mappingRealVariableList.at(0).at(blockIndex), size, + LOCALSTENCIL, "double", OPS_WRITE), + ops_arg_dat(mappingRealVariableList.at(1).at(blockIndex), noElem, + LOCALSTENCIL, "double", OPS_WRITE), + ops_arg_dat(mappingRealVariableList.at(2).at(blockIndex), size, + LOCALSTENCIL, "double", OPS_WRITE), + ops_arg_dat(mappingIntVariableList.at(0).at(blockIndex), size, + LOCALSTENCIL, "int", OPS_WRITE), + ops_arg_gbl(&spaceDim, 1, "int", OPS_READ), + ops_arg_gbl(&noElem, 1, "int", OPS_READ)); + + + + + } + +} + + + diff --git a/Apps/LBM-DEM/poros_grid.h b/Apps/LBM-DEM/poros_grid.h new file mode 100644 index 0000000..0ac5885 --- /dev/null +++ b/Apps/LBM-DEM/poros_grid.h @@ -0,0 +1,54 @@ +/* + * poros_grid.h + * + * Created on: May 29, 2021 + * Author: jpd38567 + */ + +#ifndef POROS_GRID_H_ +#define POROS_GRID_H_ + +#include "particle_to_grid_base.h" + + +class PorosGrid : public ParticleToGridBase { + public: + PorosGrid(int ParticleType,int spaceDim); + virtual int particleShape() {return 1;} + virtual void DefineVariables(int noElem, SizeType timestep = 0); + virtual void ParticleProjection(); + virtual void UpdateProjection(); + virtual void MappingFunction(bool flag); + virtual void InitializeVariables(); + virtual void ReturnParticleShape() { ops_printf("Class handles abritrary shape particles\n"); } + + protected: + static void KerSolidFracGridSphere(ACC& idp, ACC& sfp, ACC& uPar, + ACC& xAvg,const ACC& xF, const Real* xPart, const Real* Radius, + const int* idParticle, const Real* vParticle, const Real* omParticle, + const Real* dx, const int* Ngrid, const int* noelem, const int* spacedim); + + static void KerSolidVelocityUpdate(ACC& vP, const ACC& id, + const ACC& xAvg, const int* idParticle, + const Real* xPos, const Real* radPart, const Real* velP, const Real* omP, + const Real* dx, const Real* spacedim, const Real* noelem); + + + static void KerInitializePorousGrid(ACC&sfP, ACC& vP, ACC& xAvg, + ACC& id, const int* spacedim, const int* noelem); + + static inline OPS_FUN_PREFIX Real ComputeCircularSolidFractionGrid(const Real* xfl,const Real* xPart, + const Real Rp, Real* xavg, const Real dx, const int noGrid); + + + + private: + int sizeofData = 4; + int Npoints = 9; +}; + + + + + +#endif /* APPS_LBM_DEM_POROS_GRID_H_ */ diff --git a/Apps/LBM-DEM/poros_grid.inc b/Apps/LBM-DEM/poros_grid.inc new file mode 100644 index 0000000..2b99376 --- /dev/null +++ b/Apps/LBM-DEM/poros_grid.inc @@ -0,0 +1,171 @@ +/* + * poros_grid.inc + * + * Created on: May 29, 2021 + * Author: jpd38567 + */ + +#ifndef POROS_GRID_INC_ +#define POROS_GRID_INC_ + +#include "poros_grid.h" +#include "poros_grid_host_device.h" + +#include "ops_lib_core.h" + +#ifdef OPS_MPI +#include "ops_mpi_core.h" +#endif +#include "type.h" + + +void PorosGrid::KerSolidFracGridSphere(ACC& idp, ACC& sfp, ACC& uPar, + ACC& xAvg,const ACC& xF, const Real* xPart, const Real* Radius, + const int* idParticle, const Real* vParticle, const Real* omParticle, + const Real* dx, const int* Ngrid, const int* noelem, const int* spacedim) { + + const int SPACEDIM = (*spacedim); + const int NOELEM = (*noelem); + const int noGrid = (*Ngrid); + Real xfl[SPACEDIM]; + Real xe, ye, ze; + Real dd; + Real dx1, dy1, dz1; + Real xavg[SPACEDIM]; + int ip, jp, kp; + int status, nEdges, noIndex; + int itot = 0; + int sx[26] = { 1, 1, 1, 1, -1, -1, -1, -1, 1, -1, 0, 0, 0, 0, 1, -1, + 1, -1, 0, 0, 0, 0, 1, 1, -1, -1}; + int sy[26] = { 1, 1, -1, -1, 1, 1, -1, -1, 0, 0, 1, -1, + 0, 0, 1, 1,-1, -1, 1, -1, 1, -1, 0, 0, 0, 0}; + int sz[26] = { 1, -1, 1, -1, 1, -1, 1, -1, 0, 0, 0, 0, 1, -1, + 0, 0, 0, 0, 1, 1, -1, -1, 1, -1, 1, -1}; + + nEdges = 26; + dx1 = (*dx); + int no = 0; + + xfl[0] = xF(0, 0, 0, 0); + xfl[1] = xF(1, 0, 0, 0); + xfl[2] = xF(2, 0, 0, 0); + + no = 0; + for (int iPar = 0; iPar < NOELEM; iPar++) + no += (idp(iPar, 0, 0, 0)>-1); + + + for (int edgeIndex = 0; edgeIndex < nEdges; edgeIndex++) { + xe = xfl[0] + 0.5 * (dx1) * static_cast(sx[edgeIndex]); + ye = xfl[1] + 0.5 * (dx1) * static_cast(sy[edgeIndex]); + ze = xfl[2] + 0.5 * (dx1) * static_cast(sz[edgeIndex]); + + dd = (xe - xPart[0]) * (xe - xPart[0]) + (ye - xPart[1]) * (ye - xPart[1]) + (ze - xPart[2]) * (ze - xPart[2]); + + if (dd <= (*Radius) * (*Radius)) + itot += 1; + } + + + if (itot == nEdges) { + no += 1; + noIndex = no - 1; + idp(noIndex, 0, 0, 0) = (*idParticle); + sfp(noIndex, 0, 0, 0) = 1.0; + dx1 = xfl[0] - xPart[0]; + dy1 = xfl[1] - xPart[1]; + dz1 = xfl[2] - xPart[2]; + + uPar(SPACEDIM * noIndex, 0, 0, 0) = vParticle[0] + omParticle[1] * dz1 + - omParticle[2] * dy1; + uPar(SPACEDIM * noIndex +1, 0, 0, 0) = vParticle[1] + omParticle[2] * dx1 + - omParticle[0] * dz1; + uPar(SPACEDIM * noIndex +2, 0, 0, 0) = vParticle[2] + omParticle[0] * dy1 + - omParticle[1] * dx1; + + xAvg(SPACEDIM * noIndex, 0, 0, 0) = xfl[0]; + xAvg(SPACEDIM * noIndex + 1, 0, 0, 0) = xfl[1]; + xAvg(SPACEDIM * noIndex + 2, 0, 0, 0) = xfl[2]; + + } + else if (itot > 0) { + no += 1; + noIndex = no - 1; + idp(noIndex, 0, 0, 0) = (*idParticle); + sfp(noIndex, 0, 0, 0) = ComputeCircularSolidFractionGrid(xfl, xPart, (*Radius), + xavg, (*dx), noGrid); + + xAvg(SPACEDIM * noIndex, 0, 0, 0) = xavg[0]; + xAvg(SPACEDIM * noIndex + 1, 0, 0, 0) = xavg[1]; + xAvg(SPACEDIM * noIndex + 2, 0, 0, 0) = xavg[2]; + + dx1 = xavg[0] - xPart[0]; + dy1 = xavg[1] - xPart[1]; + dz1 = xavg[2] - xPart[2]; + + uPar(SPACEDIM * noIndex, 0, 0, 0) = vParticle[0] + omParticle[1] * dz1 + - omParticle[2] * dy1; + uPar(SPACEDIM * noIndex +1, 0, 0, 0) = vParticle[1] + omParticle[2] * dx1 + - omParticle[0] * dz1; + uPar(SPACEDIM * noIndex +2, 0, 0, 0) = vParticle[2] + omParticle[0] * dy1 + - omParticle[1] * dx1; + + + } + + + +} + +void PorosGrid::KerSolidVelocityUpdate(ACC& vP, const ACC& id, + const ACC& xAvg, const int* idParticle, + const Real* xPos, const Real* radPart, const Real* velP, const Real* omP, + const Real* dx, const Real* spacedim, const Real* noelem) { + + int SPACEDIM = (*spacedim); + int NELEM = (*noelem); + Real sfPart; + Real xf[SPACEDIM], vEst[SPACEDIM]; + Real dx1, dy1, dz1; + for (int iPart = 0; iPart < NELEM; iPart++) { + if (id(iPart, 0, 0, 0) == (*idParticle)) { + xf[0] = xAvg(iPart * SPACEDIM, 0, 0, 0); + xf[1] = xAvg(iPart * SPACEDIM + 1, 0, 0, 0); + xf[2] = xAvg(iPart * SPACEDIM + 2, 0, 0, 0); + + + + dx1 = xf[0] - xPos[0]; + dy1 = xf[1] - xPos[1]; + dz1 = xf[2] - xPos[2]; + + vP(SPACEDIM * iPart, 0, 0, 0) = velP[0] + omP[1] * dz1 - omP[2] * dy1; + vP(SPACEDIM * iPart + 1, 0, 0, 0) = velP[1] + omP[2] * dx1 - omP[0] * dz1; + vP(SPACEDIM * iPart + 2, 0, 0, 0) = velP[2] + omP[0] * dy1 - omP[1] * dx1; + + + break; + } + } +} + +void PorosGrid::KerInitializePorousGrid(ACC&sfP, ACC& vP, ACC& xAvg, + ACC& id, const int* spacedim, const int* noelem) { + + const int SPACEDIM = (*spacedim); + const int NELEM = (*noelem); + + for (int iPart = 0; iPart < NELEM; iPart++) { + sfP(iPart, 0, 0, 0) = 0.0; + for (int iDir = 0; iDir < SPACEDIM; iDir++) { + id(iPart * SPACEDIM + iDir, 0, 0, 0) = -1; + xAvg(iPart * SPACEDIM + iDir, 0, 0, 0) = -1.0; + } + } + +} + + + + +#endif /* POROS_GRID_INC_ */ diff --git a/Apps/LBM-DEM/poros_grid_host_device.h b/Apps/LBM-DEM/poros_grid_host_device.h new file mode 100644 index 0000000..b184435 --- /dev/null +++ b/Apps/LBM-DEM/poros_grid_host_device.h @@ -0,0 +1,65 @@ +#ifndef POROS_GRID_HOST_DEVICE_H_ +#define POROS_GRID_HOST_DEVICE_H_ + +#ifndef OPS_FUN_PREFIX +#define OPS_FUN_PREFIX +#endif + +#include +#include "poros_grid.h" + +inline OPS_FUN_PREFIX Real PorosGrid::ComputeCircularSolidFractionGrid(const Real* xfl,const Real* xPart, + const Real Rp, Real* xavg, const Real dx, const int noGrid) { + + int Nx = noGrid + 2; + int Ny = noGrid + 2; + int Nz = noGrid + 2; + int inter = 0; + + Real dxp, dyp, dzp, dd; + Real xmin, ymin, zmin; + Real xtrial, ytrial, ztrial; + Real dxc = dx/static_cast(Nx-1); + + xmin = xfl[0] - 0.5 * dx; + ymin = xfl[1] - 0.5 * dx; + zmin = xfl[2] - 0.5 * dx; + + for (int iDir = 0; iDir < 3; iDir++) + xavg[iDir] = 0.0; + + for (int ip = 0; ip < Nx; ip++) { + xtrial = xmin + static_cast(ip) * dxc; + dxp = (xtrial - xPart[0]) * (xtrial - xPart[0]); + for (int jp = 0; jp < Ny; jp++) { + ytrial = ymin + static_cast(jp) * dxc; + dyp = (ytrial - xPart[1]) * (ytrial - xPart[1]); + for (int kp = 0; kp < Nz; kp++) { + ztrial = zmin + static_cast(kp) * dxc; + dzp = (ztrial - xPart[2]) * (ztrial - xPart[2]); + dd = dxp + dyp + dzp; + if (dd < Rp * Rp ) { + inter+= 1; + xavg[0] += xtrial; + xavg[1] += ytrial; + xavg[2] += ztrial; + } + } + } + } + + for (int iDir = 0; iDir < 3; iDir++) + xavg[iDir] /= static_cast(inter); + + return static_cast(inter) / static_cast(Nx * Ny * Nz); + + + + +} + + + + + +#endif /* APPS_LBM_DEM_POROS_GRID_HOST_DEVICE_H_ */ diff --git a/Apps/LBM-DEM/poros_spherical.cpp b/Apps/LBM-DEM/poros_spherical.cpp new file mode 100644 index 0000000..634b54a --- /dev/null +++ b/Apps/LBM-DEM/poros_spherical.cpp @@ -0,0 +1,286 @@ +/** + * Copyright 2019 United Kingdom Research and Innovation + * + * Authors: See AUTHORS + * + * Contact: [jianping.meng@stfc.ac.uk and/or jpmeng@gmail.com] + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice + * this list of conditions and the following disclaimer in the documentation + * and or other materials provided with the distribution. + * 3. Neither the name of the copyright holder nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * ANDANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. +*/ + +/*! @brief Definition of mapping spherical particles with solid fraction + * @author C. Tsigginos + * @details Functions for evaluating solid fraction for spherical particles + * + */ + + +#include "poros_spherical.h" +#include "flowfield.h" +#include "ops_seq_v2.h" +#include "block_particles.h" +#include +#include +#include "type.h" + +#include "ops_lib_core.h" +#ifdef OPS_MPI +#include "ops_mpi_core.h" +#endif + + +#include "poros_spherical.inc" +PorosSpherical::PorosSpherical(int particleshape, int spacedim) : + ParticleToGridBase( particleshape, spacedim) { + + //Verifications for particle shape + if (particleDiscriptor != ParticleSpherical) { + ops_printf("ERROR: Class PorosSpherical supports only spherical" + "particles\n"); + assert(particleDiscriptor == ParticleSpherical); + + } + + //Assign variables + RealField poros{"SolFracSph"}; + mappingRealVariableList.push_back(poros); + RealField vParts{"velPartSph"}; + mappingRealVariableList.push_back(vParts); + IntField idParts{"idParticlesSph"}; + mappingIntVariableList.push_back(idParts); + RealField xAverage{"xAvgSph"}; + mappingRealVariableList.push_back(xAverage); + +} + +void PorosSpherical::DefineVariables(int noelem, SizeType timeStep) { + + + noElem = noelem; + int size = noElem * spaceDim; + mappingRealVariableList.at(0).SetDataDim(noElem); + mappingRealVariableList.at(1).SetDataDim(size); + mappingRealVariableList.at(2).SetDataDim(size); + + for (auto& mappingVariable : mappingRealVariableList) { + if (timeStep == 0) { + mappingVariable.CreateFieldFromScratch(g_Block()); + } + else { + mappingVariable.CreateFieldFromFile(CaseName(), g_Block(), timeStep); + + } + + } + + mappingIntVariableList.at(0).SetDataDim(noElem); + for (auto& mappingVariable : mappingIntVariableList) { + if (timeStep == 0) { + mappingVariable.CreateFieldFromScratch(g_Block()); + } + else { + mappingVariable.CreateFieldFromFile(CaseName(), g_Block(), timeStep); + + } + } + +} +void PorosSpherical::MappingFunction(bool flag) { + + if (flag) + ParticleProjection(); + else + UpdateProjection(); + + +} + +void PorosSpherical::ParticleProjection() { + + ops_printf("I entered on particle projection\n"); + + int size = noElem * spaceDim; + int idParticle; + int blockIndex; + const Real Dx {GetDx()}; + Real xPos[spaceDim], Radius, uParticle[spaceDim], omParticle[spaceDim]; + int iterate[2 * spaceDim]; + for (const auto& idBlock : BlockParticleList) { + BlockParticles ParticleCurrentBlock = idBlock.second; + if ( !ParticleCurrentBlock.OwnedStatus() ) continue; + blockIndex = ParticleCurrentBlock.GetBlock().ID(); + int nlocal = ParticleCurrentBlock.NParticles + + ParticleCurrentBlock.NPeriodic; + for (int iPart = 0; iPart < nlocal; iPart++) { + xPos[0] = ParticleCurrentBlock.particleList.at(iPart).xParticle[0]; + xPos[1] = ParticleCurrentBlock.particleList.at(iPart).xParticle[1]; + xPos[2] = ParticleCurrentBlock.particleList.at(iPart).xParticle[2]; + Radius = ParticleCurrentBlock.particleList.at(iPart).particleShape->Rparticle; + + uParticle[0] = ParticleCurrentBlock.particleList.at(iPart).uParticle[0]; + uParticle[1] = ParticleCurrentBlock.particleList.at(iPart).uParticle[1]; + uParticle[2] = ParticleCurrentBlock.particleList.at(iPart).uParticle[2]; + + omParticle[0] = ParticleCurrentBlock.particleList.at(iPart).omegaParticle[0]; + omParticle[1] = ParticleCurrentBlock.particleList.at(iPart).omegaParticle[1]; + omParticle[2] = ParticleCurrentBlock.particleList.at(iPart).omegaParticle[2]; + idParticle = iPart; + for (int iDir = 0; iDir < 2 * spaceDim; iDir++) + iterate[iDir] = ParticleCurrentBlock.particleList.at(iPart).stenList[iDir]; + + ops_par_loop(KerSolidFracSphere, "KerSolidFracSphere", ParticleCurrentBlock.GetBlock().Get(), + spaceDim, iterate, + ops_arg_dat(mappingIntVariableList.at(0).at(blockIndex), noElem, + LOCALSTENCIL,"int", OPS_RW), + ops_arg_dat(mappingRealVariableList.at(0).at(blockIndex), noElem, + LOCALSTENCIL,"double", OPS_RW), + ops_arg_dat(mappingRealVariableList.at(1).at(blockIndex), size, + LOCALSTENCIL,"double", OPS_RW), + ops_arg_dat(mappingRealVariableList.at(2).at(blockIndex), size, + LOCALSTENCIL,"double", OPS_RW), + ops_arg_dat(g_CoordinateXYZ()[blockIndex], spaceDim, + LOCALSTENCIL,"double", OPS_READ), + ops_arg_gbl(xPos, spaceDim, "double", OPS_READ), + ops_arg_gbl(&Radius, 1, "double", OPS_READ), + ops_arg_gbl(&idParticle, 1, "double", OPS_READ), + ops_arg_gbl(uParticle, spaceDim, "double", OPS_READ), + ops_arg_gbl(omParticle, spaceDim, "double", OPS_READ), + ops_arg_gbl(&Dx, 1, "double", OPS_READ), + ops_arg_gbl(&noElem, 1, "double", OPS_READ), + ops_arg_gbl(&spaceDim, 1, "double", OPS_READ)); + + } + } + + + +} + +void PorosSpherical::UpdateProjection() { + + int size = noElem * spaceDim; + Real velP[spaceDim], omP[spaceDim]; + Real xPar[spaceDim], radius; + int stenList[2 * spaceDim]; + int nlocal; + int idParticle; + Real dx = GetDx(); + for (const auto& idBlock : BlockParticleList) { + BlockParticles ParticleCurrentBlock = idBlock.second; + if ( !ParticleCurrentBlock.OwnedStatus() ) continue; + int blockIndex = ParticleCurrentBlock.GetBlock().ID(); + int nlocal = ParticleCurrentBlock.NParticles + + ParticleCurrentBlock.NPeriodic; + for (int iPart = 0; iPart < nlocal; iPart++) { + xPar[0] = ParticleCurrentBlock.particleList.at(iPart).xParticle[0]; + xPar[1] = ParticleCurrentBlock.particleList.at(iPart).xParticle[1]; + xPar[2] = ParticleCurrentBlock.particleList.at(iPart).xParticle[2]; + radius = ParticleCurrentBlock.particleList.at(iPart).particleShape->Rparticle; + + velP[0] = ParticleCurrentBlock.particleList.at(iPart).uParticle[0]; + velP[1] = ParticleCurrentBlock.particleList.at(iPart).uParticle[1]; + velP[2] = ParticleCurrentBlock.particleList.at(iPart).uParticle[2]; + + omP[0] = ParticleCurrentBlock.particleList.at(iPart).omegaParticle[0]; + omP[1] = ParticleCurrentBlock.particleList.at(iPart).omegaParticle[1]; + omP[2] = ParticleCurrentBlock.particleList.at(iPart).omegaParticle[2]; + idParticle = iPart; + for (int iDir = 0; iDir < 2 * spaceDim; iDir++) + stenList[iDir] = ParticleCurrentBlock.particleList.at(iPart).stenList[iDir]; + + ops_par_loop(KerSolidVelocitySphere,"KerSolidVelocitySphere", + ParticleCurrentBlock.GetBlock().Get(), spaceDim, stenList, + ops_arg_dat(mappingRealVariableList.at(1).at(blockIndex), size, + LOCALSTENCIL, "double", OPS_RW), + ops_arg_dat(mappingIntVariableList.at(0).at(blockIndex), noElem, + LOCALSTENCIL,"int", OPS_READ), + ops_arg_dat(mappingRealVariableList.at(2).at(blockIndex), size, + LOCALSTENCIL, "double", OPS_READ), + ops_arg_gbl(&idParticle, 1, "int", OPS_READ), + ops_arg_gbl(xPar, spaceDim, "double", OPS_READ), + ops_arg_gbl(&radius, 1, "double", OPS_READ), + ops_arg_gbl(velP, spaceDim, "double", OPS_READ), + ops_arg_gbl(omP, spaceDim, "double", OPS_READ), + ops_arg_gbl(&dx, 1, "double", OPS_READ), + ops_arg_gbl(&spaceDim, 1, "int", OPS_READ), + ops_arg_gbl(&noElem, 1, "int", OPS_READ)); + } + } + +} + +void PorosSpherical::InitializeVariables() { + int size = noElem * spaceDim; + for (const auto& idBlock : BlockParticleList) { + BlockParticles ParticleCurrentBlock = idBlock.second; + std::vector iterRng; + iterRng.assign(ParticleCurrentBlock.GetBlock().WholeRange().begin(), + ParticleCurrentBlock.GetBlock().WholeRange().end()); + + const int blockIndex{ParticleCurrentBlock.GetBlock().ID()}; + ops_par_loop(KerInitializePorousSpherical,"KerInitializePorousSpherical", + ParticleCurrentBlock.GetBlock().Get(),spaceDim, iterRng.data(), + ops_arg_dat(mappingRealVariableList.at(0).at(blockIndex), size, + LOCALSTENCIL, "double", OPS_WRITE), + ops_arg_dat(mappingRealVariableList.at(1).at(blockIndex), noElem, + LOCALSTENCIL, "double", OPS_WRITE), + ops_arg_dat(mappingRealVariableList.at(2).at(blockIndex), size, + LOCALSTENCIL, "double", OPS_WRITE), + ops_arg_dat(mappingIntVariableList.at(0).at(blockIndex), size, + LOCALSTENCIL, "int", OPS_WRITE), + ops_arg_gbl(&spaceDim, 1, "int", OPS_READ), + ops_arg_gbl(&noElem, 1, "int", OPS_READ)); + + } +} + +void PorosSpherical::PrintMappingVariables() { + + int size = noElem * spaceDim; + + for (const auto& idBlock : BlockParticleList) { + BlockParticles ParticleCurrentBlock = idBlock.second; + std::vector iterRng; + iterRng.assign(ParticleCurrentBlock.GetBlock().WholeRange().begin(), + ParticleCurrentBlock.GetBlock().WholeRange().end()); + + const int blockIndex{ParticleCurrentBlock.GetBlock().ID()}; + ops_par_loop(KerPrintPorousData,"KerPrintPorousData", + ParticleCurrentBlock.GetBlock().Get(),spaceDim, iterRng.data(), + ops_arg_dat(mappingRealVariableList.at(0).at(blockIndex), size, + LOCALSTENCIL, "double", OPS_WRITE), + ops_arg_dat(mappingRealVariableList.at(1).at(blockIndex), noElem, + LOCALSTENCIL, "double", OPS_WRITE), + ops_arg_dat(mappingRealVariableList.at(2).at(blockIndex), size, + LOCALSTENCIL, "double", OPS_WRITE), + ops_arg_dat(mappingIntVariableList.at(0).at(blockIndex), size, + LOCALSTENCIL, "int", OPS_WRITE), + ops_arg_dat(g_CoordinateXYZ()[blockIndex], spaceDim, + LOCALSTENCIL,"double", OPS_READ), + ops_arg_gbl(&spaceDim, 1, "int", OPS_READ), + ops_arg_gbl(&noElem, 1, "int", OPS_READ)); + } +} + diff --git a/Apps/LBM-DEM/poros_spherical.h b/Apps/LBM-DEM/poros_spherical.h new file mode 100644 index 0000000..9c6f184 --- /dev/null +++ b/Apps/LBM-DEM/poros_spherical.h @@ -0,0 +1,83 @@ +/** + * Copyright 2019 United Kingdom Research and Innovation + * + * Authors: See AUTHORS + * + * Contact: [jianping.meng@stfc.ac.uk and/or jpmeng@gmail.com] + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice + * this list of conditions and the following disclaimer in the documentation + * and or other materials provided with the distribution. + * 3. Neither the name of the copyright holder nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * ANDANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. +*/ + +/*! @brief Class for calculating the solid fraction for spherical particles + * @author C. Tsigginos + */ + +#ifndef POROS_SPHERICAL_H_ +#define POROS_SPHERICAL_H_ + +#include "particle_to_grid_base.h" +#include "type.h" +#include "flowfield_host_device.h" +#include + +class PorosSpherical : public ParticleToGridBase { + public: + PorosSpherical(int ParticleType,int spaceDim); + virtual int particleShape() {return 1;} + virtual void DefineVariables(int noElem, SizeType timestep = 0); + virtual void ParticleProjection(); + virtual void UpdateProjection(); + virtual void MappingFunction(bool flag); + virtual void InitializeVariables(); + virtual void ReturnParticleShape() { ops_printf("Handles spherical particles only"); }; + virtual void PrintMappingVariables(); + + protected: + static void KerSolidFracSphere(ACC& id, ACC& sfp, ACC& vp, + ACC& xAvg, const ACC& xf, const Real* xPos, const Real* Radius, + const int* idP, const Real* vPart, const Real* omPart, const Real* dx, + const int* nelem, const int* spacedim); + + static void KerSolidVelocitySphere(ACC& vP, const ACC& id, + const ACC& xAvg, const int* idParticle, + const Real* xPos, const Real* radPart, const Real* velP, const Real* omP, + const Real* dx, const Real* spacedim, const Real* noelem); + + static void KerInitializePorousSpherical(ACC&sfP, ACC& vP, ACC& xAvg, + ACC& id, const int* spacedim, const int* noelem); + + static inline OPS_FUN_PREFIX Real CalculateSolidFractionSpheres(const Real* xfl, + const Real Ravg, const Real* xPos, const Real Rp, Real* xAvg); + + static void KerPrintPorousData(const ACC&sfP,const ACC& vP, + const ACC& xAvg, const ACC& id, const ACC& xf, + const int* spacedim, const int* noelem); + + private: + int sizeofData = 4; +}; + + +#endif /* APPS_LBM_DEM_POROS_SPHERICAL_H_ */ diff --git a/Apps/LBM-DEM/poros_spherical.inc b/Apps/LBM-DEM/poros_spherical.inc new file mode 100644 index 0000000..86c470a --- /dev/null +++ b/Apps/LBM-DEM/poros_spherical.inc @@ -0,0 +1,143 @@ +/* + * poros_spherical.inc + * + * Created on: May 28, 2021 + * Author: jpd38567 + */ + +#ifndef POROS_SPHERICAL_INC_ +#define POROS_SPHERICAL_INC_ + +#ifndef OPS_FUN_PREFIX +#define OPS_FUN_PREFIX +#endif + +#include "poros_spherical.h" +#include "poros_spherical_host_device.h" + +#include "ops_lib_core.h" + +#ifdef OPS_MPI +#include "ops_mpi_core.h" +#endif +#include "type.h" + + +void PorosSpherical::KerSolidFracSphere(ACC& id, ACC& sfp, ACC& vp, + ACC& xAvg, const ACC& xf, const Real* xPos, const Real* Radius, + const int* idP, const Real* vPart, const Real* omPart, const Real* dx, + const int* nelem, const int* spacedim) { + + const int SPACEDIM = (*spacedim); + const int NELEM = (*nelem); + int no = 0; + int noIndex; + Real vf[SPACEDIM], xfl[SPACEDIM], xavg[SPACEDIM]; + Real sfEst; + Real dx1, dy1, dz1; + xfl[0] = xf(0, 0, 0, 0); + xfl[1] = xf(1, 0, 0, 0); + xfl[2] = xf(2, 0, 0, 0); + + Real Ravg = (*dx) * sqrt(1/PI); + + sfEst = CalculateSolidFractionSpheres(xfl, Ravg, xPos, (*Radius), xavg); //TODO + + //find noTot; + for (int iPar = 0; iPar < NELEM; iPar++) + no += (id(iPar, 0, 0, 0)>-1); + + + if (sfEst > 0.0) { + no+=1; + noIndex = no - 1; + id(noIndex, 0, 0, 0) = (*idP); + sfp(noIndex, 0, 0, 0) = sfEst; + xAvg(noIndex * SPACEDIM, 0, 0, 0) = xavg[0]; + xAvg(noIndex * SPACEDIM + 1, 0, 0, 0) = xavg[1]; + xAvg(noIndex * SPACEDIM + 2, 0, 0, 0 ) = xavg[2]; + + dx1 = xavg[0] - xPos[0]; + dy1 = xavg[1] - xPos[1]; + dz1 = xavg[2] - xPos[2]; + vp(SPACEDIM * noIndex, 0, 0, 0) = vPart[0] + omPart[1] * dz1 - omPart[2] * dy1; + vp(SPACEDIM * noIndex + 1, 0, 0, 0) = vPart[1] + omPart[2] * dx1 - omPart[0] * dz1; + vp(SPACEDIM * noIndex + 2, 0, 0, 0) = vPart[2] + omPart[0] * dy1 - omPart[1] * dx1; + + } + + +} + +void PorosSpherical::KerSolidVelocitySphere(ACC& vP, const ACC& id, + const ACC& xAvg, const int* idParticle, + const Real* xPos, const Real* radPart, const Real* velP, const Real* omP, + const Real* dx, const Real* spacedim, const Real* noelem) { + + int SPACEDIM = (*spacedim); + int NELEM = (*noelem); + Real sfPart; + Real xf[SPACEDIM], vEst[SPACEDIM]; + Real dx1, dy1, dz1; + for (int iPart = 0; iPart < NELEM; iPart++) { + if (id(iPart, 0, 0, 0) == (*idParticle)) { + xf[0] = xAvg(iPart * SPACEDIM, 0, 0, 0); + xf[1] = xAvg(iPart * SPACEDIM + 1, 0, 0, 0); + xf[2] = xAvg(iPart * SPACEDIM + 2, 0, 0, 0); + + + + dx1 = xf[0] - xPos[0]; + dy1 = xf[1] - xPos[1]; + dz1 = xf[2] - xPos[2]; + + vP(SPACEDIM * iPart, 0, 0, 0) = velP[0] + omP[1] * dz1 - omP[2] * dy1; + vP(SPACEDIM * iPart + 1, 0, 0, 0) = velP[1] + omP[2] * dx1 - omP[0] * dz1; + vP(SPACEDIM * iPart + 2, 0, 0, 0) = velP[2] + omP[0] * dy1 - omP[1] * dx1; + + + break; + } + } +} + +void PorosSpherical::KerInitializePorousSpherical(ACC&sfP, ACC& vP, + ACC& xAvg, ACC& id, const int* spacedim, const int* noelem) { + + const int SPACEDIM = (*spacedim); + const int NELEM = (*noelem); + + for (int iPart = 0; iPart < NELEM; iPart++) { + sfP(iPart, 0, 0, 0) = 0.0; + for (int iDir = 0; iDir < SPACEDIM; iDir++) { + id(iPart * SPACEDIM + iDir, 0, 0, 0) = -1; + xAvg(iPart * SPACEDIM + iDir, 0, 0, 0) = -1.0; + } + } + +} + + +void PorosSpherical::KerPrintPorousData(const ACC&sfP,const ACC& vP, + const ACC& xAvg, const ACC& id, const ACC& xf, + const int* spacedim, const int* noelem) { + + const int SPACEDIM = (*spacedim); + const int NELEM = (*noelem); + + for (int iPart = 0; iPart < NELEM; iPart++) { + printf("[%f %f %f]: sfP = %f, idP = %d vAvf=[%f %f %f] xAvg =[%f %f %f]\n", + xf(0, 0, 0, 0), xf(1, 0, 0, 0), xf(2, 0, 0, 0), + sfP(iPart, 0, 0, 0), id(iPart, 0, 0, 0), vP(iPart*SPACEDIM, 0, 0, 0), + vP(iPart*SPACEDIM + 1, 0, 0, 0), vP(iPart*SPACEDIM + 2, 0, 0, 0), + xAvg(iPart*SPACEDIM, 0, 0, 0), + xAvg(iPart*SPACEDIM + 1, 0, 0, 0), xAvg(iPart*SPACEDIM + 2, 0, 0, 0)); + + } + + + +} + + +#endif /* APPS_LBM_DEM_POROS_SPHERICAL_INC_ */ diff --git a/Apps/LBM-DEM/poros_spherical_host_device.h b/Apps/LBM-DEM/poros_spherical_host_device.h new file mode 100644 index 0000000..0ce2aaf --- /dev/null +++ b/Apps/LBM-DEM/poros_spherical_host_device.h @@ -0,0 +1,70 @@ +/* + * poros_spherical_device.h + * + * Created on: May 29, 2021 + * Author: jpd38567 + */ + +#ifndef POROS_SPHERICAL_HOST_DEVICE_H_ +#define POROS_SPHERICAL_HOST_DEVICE_H_ + +#ifndef OPS_FUN_PREFIX +#define OPS_FUN_PREFIX +#endif + +#include +#include "poros_spherical.h" + +inline OPS_FUN_PREFIX Real PorosSpherical::CalculateSolidFractionSpheres(const Real* xfl, + const Real Ravg, const Real* xPos, const Real Rp, Real* xAvg) { + + + Real d, x, sf, h1, h2, V1, V2, Vol, norm; + Real z1, z2, zcm, n[3]; + d = (xPos[0] - xfl[0]) * (xPos[0] - xfl[0]) + + (xPos[1] - xfl[1]) * (xPos[1] - xfl[1]) + + (xPos[2] - xfl[2]) * (xPos[2] - xfl[2]); + d=sqrt(d); + if (d < Ravg + Rp) { + if (d<=fabs(Rp-Ravg)) { + for (int iDim = 0; iDim < 3; iDim++) + xAvg[iDim] = xfl[iDim]; + return 1.0; + } + x = (d * d - Rp * Rp + Ravg * Ravg)/(2.0 * d); + h1 = Ravg - x; + h2 = Rp - d + x; + V1 = (1.0/3.0) * PI * h1 * h1 * (3.0 * Ravg - h1); + V2 = (1.0/3.0) * PI * h2 * h2 * (3.0 * Rp - h2); + Vol = (4.0/3.0) * PI * Ravg * Ravg * Ravg; + + z1 = 3.0 * (2.0 * Ravg - h1) * (2.0 * Ravg - h1) / (4.0 * (3.0 * Ravg - h1)); + z2 = 3.0 * (2.0 * Rp - h2) * (2.0 * Rp - h2) / (4.0 * (3.0 * Rp - h2)); + //Transform z2 into z1 + z2 = d - z2; + zcm = (z2 * V2 + z1 * V1) / (V1 + V2); + + //Transofmration of zcm into global coordinate system + n[0] = xPos[0] - xfl[0]; + n[1] = xPos[1] - xfl[1]; + n[2] = xPos[2] - xfl[2]; + norm = n[0] * n[0] + n[1] * n[1] + n[2] * n[2]; + norm = sqrt(norm); + for (int iDir = 0; iDir < 3; iDir++) + n[iDir] /= norm; + + + for (int iDir = 0; iDir < 3; iDir++) + xAvg[iDir] = xfl[iDir] + zcm * n[iDir]; + + return (V1 + V2) / Vol; + } + xAvg[0] = -1.0; + xAvg[1] = -1.0; + xAvg[2] = -1.0; + return 0.0; +} + + + +#endif diff --git a/Apps/LBM-DEM/psm.cpp b/Apps/LBM-DEM/psm.cpp new file mode 100644 index 0000000..cdca7d6 --- /dev/null +++ b/Apps/LBM-DEM/psm.cpp @@ -0,0 +1,248 @@ +/** + * Copyright 2019 United Kingdom Research and Innovation + * + * Authors: See AUTHORS + * + * Contact: [jianping.meng@stfc.ac.uk and/or jpmeng@gmail.com] + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice + * this list of conditions and the following disclaimer in the documentation + * and or other materials provided with the distribution. + * 3. Neither the name of the copyright holder nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * ANDANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. +*/ + +/*! @brief Implement member functions for PRATI scheme + * @author C. Tsigginos + * @details Implementing functions for Partially saturate method. +*/ + +#include "psm.h" +#include "flowfield.h" +#include "flowfield_host_device.h" +#include "scheme.h" +#include +#include +#include "ops_seq_v2.h" +#include "psm.inc" + +Psm::Psm(Component componentUser, int spacedim, Real* forceUser, bool owned, + SolFracType porosModel, Real gammaUser, int nelem, int ParticleType) : + FsiBase(componentUser, spacedim, forceUser, owned, porosModel), + Fd{"FdPSM"} { + + noElem = nelem; + + if ((porosModel!= Mode_Spherical) && (porosModel != Mode_Grid)) { + ops_printf("ERROR: Implemented porosity model %d not consistent with PSM scheme\n", porosModel); + exit(EXIT_FAILURE); + } + + if (porosModel == Mode_Spherical) { + ops_printf("Porosity model for spherical particles for component %d\n", + componentUser.id); + poros = new PorosSpherical(ParticleType, spaceDim);; + } + else if (porosModel == Mode_Grid) { + ops_printf("Grid model for porosity calculation for component %d\n", + componentUser.id); + poros = new PorosGrid(ParticleType, spaceDim); + + } +} + +void Psm::DefineVariables(SizeType timestep) { + + int size = noElem * spaceDim; + + Fd.SetDataDim(size); + if (timestep == 0) { + Fd.CreateFieldFromScratch(g_Block()); + poros->DefineVariables(noElem); + } + else { + Fd.CreateFieldFromFile(CaseName(), g_Block(), timestep); + poros->DefineVariables(noElem, timestep); + } + +} + +void Psm::ModelCollision() { + + int size = noElem * spaceDim; + + + + for (const auto& idBlock : g_Block()) { + const Block& block{idBlock.second}; + std::vector iterRng; + iterRng.assign(block.WholeRange().begin(), block.WholeRange().end()); + + Real tauRef = compo.tauRef; + const Real* pdt {pTimeStep()}; + const int blockIndex{block.ID()}; + ops_par_loop(KerCollisionPSM3D,"KerCollisionPSM3D", block.Get(), + spaceDim, iterRng.data(), + ops_arg_dat(g_fStage()[blockIndex], NUMXI, LOCALSTENCIL, + "double", OPS_WRITE), + ops_arg_dat(Fd[blockIndex], size, LOCALSTENCIL, + "double", OPS_WRITE), + ops_arg_dat(g_f()[blockIndex], NUMXI, LOCALSTENCIL, + "double", OPS_READ), + ops_arg_dat(g_CoordinateXYZ()[blockIndex], spaceDim, + LOCALSTENCIL, "double", OPS_READ), + ops_arg_dat(g_NodeType().at(compo.id).at(blockIndex), 1, + LOCALSTENCIL, "double", OPS_READ), + ops_arg_dat(g_MacroVars().at(compo.macroVars.at(Variable_Rho).id) + .at(blockIndex), 1, LOCALSTENCIL, "double", OPS_READ), + ops_arg_dat(g_MacroVars().at(compo.uId).at(blockIndex), + 1, LOCALSTENCIL, "double", OPS_READ), + ops_arg_dat(g_MacroVars().at(compo.vId).at(blockIndex), + 1, LOCALSTENCIL, "double", OPS_READ), + ops_arg_dat(g_MacroVars().at(compo.wId).at(blockIndex), + 1, LOCALSTENCIL, "double", OPS_READ), + ops_arg_dat(poros->GetIntFieldVariable(0).at(blockIndex), noElem, + LOCALSTENCIL, "int", OPS_READ), + ops_arg_dat(poros->GetRealFieldVariable(0).at(blockIndex), noElem, + LOCALSTENCIL, "double", OPS_READ), + ops_arg_dat(poros->GetRealFieldVariable(1).at(blockIndex), size, + LOCALSTENCIL, "double", OPS_READ), + ops_arg_gbl(pdt, 1, "double", OPS_READ), + ops_arg_gbl(&tauRef, 1, "int", OPS_READ), + ops_arg_gbl(&forceFlag, 1, "int", OPS_READ), + ops_arg_gbl(force, spaceDim, "double", OPS_READ), + ops_arg_gbl(&noElem, 1, "int", OPS_READ), + ops_arg_gbl(&spaceDim, 1, "int", OPS_READ), + ops_arg_gbl(compo.index, 2, "int", OPS_READ)); + } + + + +} + +void Psm::InitializeVariables() { + + int size = noElem * spaceDim; + + poros->InitializeVariables(); + + for (const auto& idBlock : g_Block()) { + const Block& block{idBlock.second}; + std::vector iterRng; + iterRng.assign(block.WholeRange().begin(), block.WholeRange().end()); + + const int blockIndex{block.ID()}; + ops_par_loop(KerInitialize,"KerInitialize", block.Get(), + spaceDim, iterRng.data(), + ops_arg_dat(Fd[blockIndex], noElem, LOCALSTENCIL, + "double", OPS_WRITE), + ops_arg_gbl(&size, 1, "int", OPS_READ)); + + } +} + + +void Psm::CalculateDragForce() { + + + Real tauRef = compo.tauRef; + const Real* pdt {pTimeStep()}; + const Real dx {GetDx()}; //TODO ADD to flowfield + Real xPos[spaceDim], FdLocal[spaceDim], TdLocal[spaceDim]; + int size = noElem * spaceDim; + int StenList[2 * spaceDim]; + int blockIndex; + int idParticle; + for (auto& idBlock : BlockParticleList) { + BlockParticles& ParticlesCurrentBlock = idBlock.second; + if ( !ParticlesCurrentBlock.owned ) continue; + blockIndex = ParticlesCurrentBlock.GetBlock().ID(); + int nLocal = ParticlesCurrentBlock.NParticles + + ParticlesCurrentBlock.NPeriodic; + for (int iPart = 0; iPart < nLocal; iPart++) { + + xPos[0] = ParticlesCurrentBlock.particleList[iPart].xParticle[0]; + xPos[1] = ParticlesCurrentBlock.particleList[iPart].xParticle[1]; + xPos[2] = ParticlesCurrentBlock.particleList[iPart].xParticle[2]; + + idParticle = iPart; + + for (int iDir = 0; iDir < spaceDim; iDir++) { + TdLocal[iDir] = 0.0; + FdLocal[iDir] = 0.0; + } + + for (int iDir = 0; iDir < 2 * spaceDim; iDir++) + StenList[iDir] = ParticlesCurrentBlock.particleList[iPart].stenList[iDir]; + + + ops_par_loop(KerDragPSM,"KerDragPSM", ParticlesCurrentBlock.GetBlock().Get(), + spaceDim, StenList, + ops_arg_dat(poros->GetIntFieldVariable(0).at(blockIndex), + noElem, LOCALSTENCIL, "int", OPS_READ), + ops_arg_dat(poros->GetRealFieldVariable(0).at(blockIndex), + noElem, LOCALSTENCIL,"double", OPS_READ), + ops_arg_dat(poros->GetRealFieldVariable(2).at(blockIndex), + size, LOCALSTENCIL,"double", OPS_READ), + ops_arg_dat(Fd[blockIndex], size, LOCALSTENCIL, + "double", OPS_READ), + ops_arg_gbl(FdLocal, spaceDim, "double", OPS_READ), + ops_arg_gbl(TdLocal, spaceDim, "double", OPS_READ), + ops_arg_gbl(&idParticle, 1, "int", OPS_READ), + ops_arg_gbl(xPos, spaceDim, "double", OPS_READ), + ops_arg_gbl(pdt, 1, "double", OPS_READ), + ops_arg_gbl(&tauRef, 1, "double", OPS_READ), + ops_arg_gbl(&spaceDim, 1, "int", OPS_READ), + ops_arg_gbl(&noElem, 1, "int", OPS_READ)); + + for (int iDir = 0; iDir < spaceDim; iDir++) { + ParticlesCurrentBlock.particleList[iPart].FDrag[iDir] += FdLocal[iDir] * dx * dx * dx; + ParticlesCurrentBlock.particleList[iPart].TDrag[iDir] += TdLocal[iDir] * dx * dx * dx; + } + + printf("Rank %d particle [%f %f %f] Fd = [%e %e %e] Td=[%e %e %e]\n", + ops_get_proc(), xPos[0], xPos[1], xPos[2], + FdLocal[0] * dx * dx * dx, FdLocal[1] * dx * dx *dx, + FdLocal[2] * dx * dx * dx, TdLocal[0] * dx * dx * dx, + TdLocal[1] * dx * dx * dx, TdLocal[2] * dx * dx * dx); + } + } + + +} + +void Psm::MappingFunction(bool flag) { + + if (flag) { + poros->ParticleProjection(); + } + else + poros->UpdateProjection(); + +} + +void Psm::WriteToHdf5(const std::string& caseName, const SizeType timeStep) { + + Fd.WriteToHDF5(caseName, timeStep); + + poros->WriteToHdf5(caseName, timeStep); + +} \ No newline at end of file diff --git a/Apps/LBM-DEM/psm.h b/Apps/LBM-DEM/psm.h new file mode 100644 index 0000000..bdb3fd8 --- /dev/null +++ b/Apps/LBM-DEM/psm.h @@ -0,0 +1,87 @@ +/** + * Copyright 2019 United Kingdom Research and Innovation + * + * Authors: See AUTHORS + * + * Contact: [jianping.meng@stfc.ac.uk and/or jpmeng@gmail.com] + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice + * this list of conditions and the following disclaimer in the documentation + * and or other materials provided with the distribution. + * 3. Neither the name of the copyright holder nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * ANDANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. +*/ + +/*! @brief Partially saturated method + * @author C. Tsigginos + **/ + + +#ifndef PSM_H_ +#define PSM_H_ + +#include "fsi_base.h" +#include "field.h" +#include "block.h" +#include "type.h" +#include +#include "model.h" +#include "block_particles.h" +#include "particle_to_grid_base.h" +#include "poros_spherical.h" +#include "poros_grid.h" + +class Psm : public FsiBase { + + protected: + RealField Fd; + ParticleToGridBase* poros; //TODO: Add the porosity model + int noElem = 1; + + public: + Psm(Component componentUser, int spacedim, Real* forceUser, bool owned = false, + SolFracType porosModel = Mode_None, Real gammaUser = 0.0, int nelem = 2, int ParticleType = 1); + ~Psm() {} + virtual void ModelCollision(); //Dpme + virtual void CalculateDragForce();//Done + virtual void MappingFunction(bool flag); //Done + virtual void DefineVariables(SizeType timestep = 0); //DONE + virtual void InitializeVariables(); //DONE + virtual void WriteToHdf5(const std::string& caseName, const SizeType timeStep); + + static void KerCollisionPSM3D(ACC& fcopy, ACC& Fd, const ACC& f, + const ACC& coordinates, const ACC& nodeType, + const ACC& Rho, const ACC& U, const ACC& V, + const ACC& W,const ACC &id,const ACC& sfp, + const ACC& vp, const Real* dt, const Real* tauRef, + const int* forceFlag, const Real* force, + const int* nElem, const int* spacedim, const int* lattIdx); + + static void KerDragPSM(const ACC& id, const ACC& sfp, const ACC& xf, + const ACC& Fd, Real* FDp, Real* TDp, const int* idp, const Real* xp, + const Real* dt, const Real* tau, + const int* spacedim, const int* nelem); + + static void KerInitialize(ACC& Fd, const int* size); +}; + + +#endif \ No newline at end of file diff --git a/Apps/LBM-DEM/psm.inc b/Apps/LBM-DEM/psm.inc new file mode 100644 index 0000000..1dc6ebd --- /dev/null +++ b/Apps/LBM-DEM/psm.inc @@ -0,0 +1,241 @@ +/** + * Copyright 2019 United Kingdom Research and Innovation + * + * Authors: See AUTHORS + * + * Contact: [jianping.meng@stfc.ac.uk and/or jpmeng@gmail.com] + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice + * this list of conditions and the following disclaimer in the documentation + * and or other materials provided with the distribution. + * 3. Neither the name of the copyright holder nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * ANDANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. +*/ + +/*! @brief Implement kernel function for Partially saturated method + * @author C. Tsigginos +*/ + +#ifndef PSM_INC_ +#define PSM_INC_ + +#include "psm.h" + +void Psm::KerCollisionPSM3D(ACC& fcopy, ACC& Fd, const ACC& f, + const ACC& coordinates, const ACC& nodeType, + const ACC& Rho, const ACC& U, const ACC& V, + const ACC& W,const ACC &id,const ACC& sfp, + const ACC& vp, const Real* dt, const Real* tauRef, + const int* forceFlag, const Real* force, + const int* nElem, const int* spacedim, const int* lattIdx) { + + int NELEM = (*nElem); + + VertexType vt = (VertexType)nodeType(0, 0, 0); + bool collision = (vt != VertexType::ImmersedSolid); + if (collision) { + int spaceDim = (*spacedim); + int NELEM = (*nElem); + Real sumx[NELEM], sumy[NELEM], sumz[NELEM]; + Real dtOverTauPlusDt, dtOverTauF; + Real OmegaS, OmegaS1, OmegaS2; + Real alpha, sf; + Real poros = 1.0; + Real eff, porosEff; + + Real u[spaceDim], rho; + int oppos, oppos1, noTot1; + int noTot = 0; + + rho= Rho(0, 0, 0); + u[0] = U(0, 0, 0); + u[1] = V(0, 0, 0); + u[2] = W(0, 0, 0); + const Real T{1}; + Real tau = (*tauRef); + + porosEff = 1.0; + for (int pIndex = 0; pIndex < NELEM; pIndex++) { + sumx[pIndex] = 0.0; + sumy[pIndex] = 0.0; + sumz[pIndex] = 0.0; + noTot+= (id(pIndex, 0, 0, 0) > -1); + poros -=sfp(pIndex, 0, 0, 0); + } + + + for (int pIndex = 0; pIndex < noTot; pIndex++) + porosEff -= sfp(pIndex, 0, 0) * tau / (tau - (1.0 - sfp(pIndex, 0, 0, 0)) * (*dt)); + + noTot1 = noTot; + if (noTot == 0) + noTot1 = 1; + + + + dtOverTauPlusDt = porosEff * (*dt) / (tau + 0.5 * (*dt)); + dtOverTauF = porosEff * (*dt); + alpha = tau/(tau + 0.5 * porosEff * (*dt)); + int numXi = lattIdx[1] - lattIdx[0] + 1; + Real up[spaceDim * noTot1]; + Real feq[numXi], feqS[numXi * noTot1], bodyForce[numXi]; + + for (int pIndex = 0; pIndex < noTot; pIndex++) { + up[spaceDim * pIndex] = vp(pIndex * spaceDim, 0, 0, 0); + up[spaceDim * pIndex + 1] = vp(pIndex * spaceDim + 1, 0, 0, 0); + up[spaceDim * pIndex + 2] = vp(pIndex * spaceDim + 2, 0, 0, 0); + } + + //Calculate BodyForce and equilibrium functions + for (int xiIndex = lattIdx[0]; xiIndex <= lattIdx[1]; xiIndex++) { + feq[xiIndex] = CalcBGKFeq(xiIndex, rho, u[0], u[1], u[2], T, 2); + if ((*forceFlag) == 1) + bodyForce[xiIndex] = CalcBodyForce2ndOrder(xiIndex, rho, u[0], u[1], u[2], force); //TODO ADD it to the model_kernel.inc + else + bodyForce[xiIndex] = 0.0; + + for (int iPar = 0; iPar < noTot; iPar++) { + if (sfp(iPar, 0, 0, 0) > 0.0) + feqS[iPar * numXi + xiIndex] = CalcBGKFeq(xiIndex, rho, up[spaceDim * iPar], + up[spaceDim * iPar + 1], up[spaceDim * iPar + 2], T, 2); + else + feqS[iPar * numXi + xiIndex] = 0.0; + } + } + + for (int xiIndex1 = lattIdx[0]; xiIndex1 <= lattIdx[1]; xiIndex1++) { + OmegaS = 0.0; + int xiIndex = xiIndex1 - lattIdx[0]; + if (noTot > 0) { + oppos1 = OPP[xiIndex1]; + oppos = oppos1 - lattIdx[0]; + OmegaS1 = (f(oppos1, 0, 0, 0) - feq[oppos]); + OmegaS2 = 0.0; + for (int iPar = 0; iPar < noTot; iPar++) { + eff = tau / (tau + (1.0 - sfp(iPar, 0, 0, 0)) * (*dt)); + OmegaS2 = (feqS[iPar * numXi + xiIndex] - f(xiIndex1, 0, 0, 0)); + OmegaS += sfp(iPar, 0, 0) * eff * (OmegaS1 + OmegaS2); + sumx[iPar] -= (OmegaS1 + OmegaS2) * XI[xiIndex1 * LATTDIM] * CS; + sumy[iPar] -= (OmegaS1 + OmegaS2) * XI[xiIndex1 * LATTDIM + 1] * CS; + sumz[iPar] -= (OmegaS1 + OmegaS2) * XI[xiIndex1 * LATTDIM + 2] * CS; + } + } + fcopy(xiIndex1, 0, 0, 0) = f(xiIndex1, 0, 0, 0) - dtOverTauPlusDt + * (f(xiIndex1, 0, 0, 0)-feq[xiIndex]) + dtOverTauF * bodyForce[xiIndex] + OmegaS; + + + +#ifdef CPU + const Real res{fcopy(xiIndex, 0, 0, 0)}; + if (std::isnan(res) || res < 0 || std::isinf(res)) { + ops_printf( + "Error-PRATI: Distribution function %f becomes " + "invalid for the component %i at the lattice " + "%i\n", + res, 0, xiIndex); + assert(!(std::isnan(res) || res < 0 || std::isinf(res))); + } +#endif // CPU + + + } + + + for (int iPar = 0; iPar < noTot; iPar++) { + Fd(spaceDim * iPar, 0, 0, 0) = sumx[iPar]; + Fd(spaceDim * iPar + 1, 0, 0, 0) = sumy[iPar]; + Fd(spaceDim * iPar + 2, 0, 0, 0) = sumz[iPar]; + + } + } + + + +} + +void Psm::KerInitialize(ACC& Fd, const int* size) { + + int total = (*size); +#ifdef OPS_3D + for (int iPart = 0; iPart < total; iPart++) + Fd(iPart, 0, 0, 0) = 0.0; +#endif + +#ifdef OPS_2D + for (int iPart = 0; iPart < total; iPart++) + Fd(iPart, 0, 0) = 0.0; + +#endif +} + +void Psm::KerDragPSM(const ACC& id, const ACC& sfp, const ACC& xf, + const ACC& Fd, Real* FDp, Real* TDp, const int* idp, const Real* xp, + const Real* dt, const Real* tau, + const int* spacedim, const int* nelem) { + + int SPACEDIM = (*spacedim); + int NELEM = (*nelem); + + Real alpha, xfl[SPACEDIM]; + int loc; + int noTot = 0; + Real poros = 1.0; + Real Beta, sf; + Real porosEff, eff; + + xfl[0] = xf(0, 0, 0, 0); + xfl[1] = xf(1, 0, 0, 0); + xfl[2] = xf(2, 0, 0, 0); + + for (int pIndex = 0; pIndex < NELEM; pIndex++) { + noTot += (id(pIndex, 0, 0, 0) > -1); + poros -= sfp(pIndex, 0, 0, 0); + } + + for (int pIndex = 0; pIndex < noTot; pIndex++) { + if ((*idp) == id(pIndex, 0, 0, 0)) { + eff = (*tau) / ((*tau) + (1.0 - sfp(pIndex, 0, 0, 0)) * (*dt)); + Beta = sfp(pIndex, 0, 0, 0) * eff; + loc = pIndex * SPACEDIM; + + xfl[0] = xf(loc , 0, 0, 0); + xfl[1] = xf(loc + 1, 0, 0, 0); + xfl[2] = xf(loc + 2, 0, 0, 0); + + + FDp[0] += Beta * Fd(loc, 0, 0, 0); + FDp[1] += Beta * Fd(loc + 1, 0, 0, 0); + FDp[2] += Beta * Fd(loc + 2, 0, 0, 0); + TDp[0] += Beta * ((xfl[1] - xp[1]) * Fd(loc+2, 0, 0, 0) - (xfl[2] - xp[2]) * Fd(loc+1,0,0, 0)); + TDp[1] += Beta * ((xfl[2] - xp[2]) * Fd(loc, 0, 0, 0) - (xfl[0] - xp[0]) * Fd(loc+2, 0, 0, 0)); + TDp[2] += Beta * ((xfl[0] - xp[0]) * Fd(loc+1, 0, 0, 0) - (xfl[1] - xp[1]) * Fd(loc, 0, 0, 0)); + + break; + } + } + + +} + + + + +#endif /* APPS_LBM_DEM_PSM_INC_ */ diff --git a/Apps/LBM-DEM/psm_kernel.h b/Apps/LBM-DEM/psm_kernel.h new file mode 100644 index 0000000..6408f50 --- /dev/null +++ b/Apps/LBM-DEM/psm_kernel.h @@ -0,0 +1,1244 @@ +/* + * psm_kernel.h + * + * Created on: 13 Jan 2020 + * Author: valantis + */ + +#ifndef PSM_KERNEL_H_ +#define PSM_KERNEL_H_ + +#include "psm.h" +#include +#include +void KerSetInitialMacroVars(ACC& macros, const ACC& coords, const Real* initialMacros) { + +#ifdef OPS_2D + macros(0, 0, 0) = initialMacros[0]; + macros(1, 0, 0) = initialMacros[1]; + macros(2, 0, 0) = initialMacros[2]; +#endif + + +#ifdef OPS_3D + macros(0, 0, 0, 0) = initialMacros[0]; + macros(1, 0, 0, 0) = initialMacros[1]; + macros(2, 0, 0, 0) = initialMacros[2]; + macros(3, 0, 0, 0) = initialMacros[3]; +#endif + + +} + +void KerInitialize(ACC& sfp, ACC& vp, ACC& id, ACC& Fd) { + int sizet = NELEM * SPACEDIM; + +#ifdef OPS_2D + for (int elem = 0; elem < NELEM; elem++) { + sfp(elem, 0, 0) = 0.0; + id(elem, 0, 0) = -1; + } + + for (int elem = 0; elem < sizet; elem++) { + vp(elem, 0, 0) = 0.0; + Fd(elem, 0, 0) = 0.0; + } +#endif + +#ifdef OPS_3D + for (int elem = 0; elem < NELEM; elem++) { + sfp(elem, 0, 0, 0) = 0.0; + id(elem, 0, 0, 0) = -1; + } + + for (int elem = 0; elem < sizet; elem++) { + vp(elem, 0, 0, 0) = 0.0; + Fd(elem, 0, 0, 0) = 0.0; + } +#endif + +} + + + + +void KerCalcMacroVarsForceFluidSolid(ACC& macroVars, const ACC& nodeType, const ACC& sfp, const Real* dt, const Real* force) { +#ifdef OPS_2D + VertexType vt = (VertexType) nodeType[OPS_ACC1(0,0)]; + if (vt != VertexType::ImmersedSolid) { + Real poros = 1.0; + for (int iPar = 0; iPar < NELEM; iPar++) + poros -= sfp(iPar, 0, 0); + + + for (int m = 0; m < SPAisnanCEDIM; m++) + macroVars(m+1, 0, 0) += 0.5 * poros * force[m] * (*dt); + + } +#endif + +#ifdef OPS_3D + VertexType vt = (VertexType) nodeType(0, 0, 0); + + if (vt != VertexType::ImmersedSolid) { + Real poros = 1.0; + for (int iPar = 0; iPar < NELEM; iPar++) + poros -= sfp(iPar, 0, 0, 0); + + for (int m = 0; m < LATTDIM; m++) + macroVars(m+1, 0, 0, 0) += 0.5 * poros * force[m] * (*dt); + + } +#endif + +} + +//2D functions +#ifdef OPS_2D +void KerSolVelocityUpdateMP2D(ACC& vp, const ACC& xf, const ACC& id, const int* idp, const Real* xp, const Real* up,const Real* tau) { + double xfl[2]; + double vf[2]; + int noTot = 0; + + + xfl[0]= xf(0, 0, 0); + xfl[1] = xf(1, 0, 0); + + for (int pIndex = 0; pIndex < NELEM; pIndex++) + noTot += id(pIndex, 0, 0>-1); + + if (noTot > 0) { + for (int pIndex = 0; pIndex < noTot; pIndex++) { + if ((*idp)==id(pIndex, 0, 0)) { + EstimateVelocity2D(vf, up, xp, xfl); + vp(SPACEDIM * pIndex, 0, 0) = vf[0]; + vp(SPACEDIM * pIndex+1, 0, 0) = vf[1]; + break; + } + } + + + } + + +} + + +void KerEvaluateSolidFractionMP2D(ACC& sfp, ACC& vp, ACC& id, const ACC& xf, const int* idp, const Real* xp, + const Real* velP, const Real* dx, const int* intOrder) { + + Real xfl[SPACEDIM]; + Real xe, ye, dd; + int nEdges, noIndex; + int tot = 0; + Real vf[SPACEDIM]; + int s[16] = {1,0, 0, 1, -1,0, 0, -1, 1, 1, -1, 1,-1,-1,1,-1}; //control points + nEdges = 8; + int no = 0; + + xfl[0] = xf(0, 0, 0); + xfl[1] = xf(1, 0, 0); + + for (int edgeIndex = 0; edgeIndex < nEdges; edgeIndex++) { + xe = xfl[0] + 0.5 * (*dx) * static_cast(s[2*edgeIndex]); + ye = xfl[1] + 0.5 * (*dx) * static_cast(s[2*edgeIndex+1]); + dd = (xp[0] - xe) * (xp[0] - xe) + (xp[1] - ye) * (xp[1] - ye); + if (dd <= xp[2] * xp[2]) + tot+=1; + } + + if (tot == nEdges) { + no += 1; + noIndex = no-1; + id(noIndex,0,0) = *idp; + sfp(noIndex, 0, 0)= 1.0; + vp(SPACEDIM * noIndex, 0, 0) = velP[0] - (xfl[1] - xp[1]) * velP[2]; + vp(SPACEDIM * noIndex + 1, 0, 0) = velP[1] + (xfl[0] - xp[0]) * velP[2]; + } + else if (tot > 0) { //Partially saturated node + no += 1; + noIndex = no-1; + id(noIndex,0,0) = *idp; + + sfp(noIndex, 0, 0) = ComputeCellSF(xfl, xp, (*dx), *intOrder); + + EstimateVelocity2D(vf, velP, xp, xfl); + vp(2 * noIndex, 0, 0) = vf[0]; + vp(2 * noIndex + 1, 0, 0) = vf[1]; + } + +} + + + +void KerPRE2D(ACC& fcopy, ACC& Fd,const ACC& f, const ACC& macros,const ACC& nodeType, + const ACC& id, const ACC& sfp, const Real* tau, const Real* dt, const int* forceFlag, const Real* force) { + + VertexType vt = (VertexType) nodeType(0, 0); + bool collisionRequired = (vt != VertexType::ImmersedSolid); + + if (collisionRequired) { + Real dtOverTauPlusDt, dtOverTauF; + Real sumx[NELEM], sumy[NELEM]; + Real OmegaS, OmegaS1, OmegaS2, OmegaS3; + Real sf; + Real poros = 1.0; + Real u[SPACEDIM], rho; + int oppos; + int noTot = 0; + + for (int pIndex = 0; pIndex < NELEM; pIndex++) { + sumx[pIndex] = 0.0; + sumy[pIndex] = 0.0; + noTot+= (id(pIndex, 0, 0) > -1); + poros-=sf(pIndex, 0, 0); + } + + sf = 1.0 - poros; + + + int noTot1; + if (noTot > 0) + noTot1 = noTot; + else + noTot1 = 1; + + dtOverTauPlusDt = poros * (*dt) / ((*tau) + 0.5 * poros * (*dt)); + dtOverTauF = dtOverTauPlusDt * (*tau); + invDtOverTauF = (*tau ) / ((*tau) + 0.5 * poros * (*dt)) + + + Real up[SPACEDIM * noTot1]; + Real feq[NUMXI], feqS[NUMXI * noTot1], bodyForce[NUMXI]; + + for (int pIndex = 0; pIndex < noTot; pIndex++) { + up[SPACEDIM * pIndex] = vp(pIndex * SPACEDIM, 0, 0); + up[SPACEDIM * pIndex + 1] = vp(pIndex * SPACEDIM +1, 0, 0); + + } + + rho = macros(0, 0, 0); + u[0] = macros(1, 0, 0); + u[1] = macros(2, 0, 0); + + + //Calculate BodyForces and equilibrium functions + for (int xiIndex = 0; xiIndex < NUMXI; xiIndex++) { + feq[xiIndex] = CalcBGKFeq(xiIndex, rho, u[0], u[1], 1, 2); + //printf("xiIndex = %d: feq = %12.9e,u = %12.9e v= %12.9e\n",xiIndex, feq[xiIndex], u[0], u[1]); + if ((*forceFlag) == 1) + bodyForce[xiIndex] = CalcBodyForce2ndOrder(xiIndex, rho, u, force); + else + bodyForce[xiIndex] = 0.0; + + for (int iPar = 0; iPar < noTot; iPar++) + feqS[iPar * NUMXI + xiIndex] = CalcBGKFeq(xiIndex, rho, up[SPACEDIM * iPar], up[SPACEDIM * iPar + 1], 1, 2); + + } + + for (int xiIndex = 0; xiIndex < NUMXI; xiIndex++) { + OmegaS = 0.0; + if (noTot > 0) { + oppos= OPP[xiIndex]; + OmegaS1 = (f(oppos, 0, 0) - feq[oppos]); + + + OmegaS3 = 0.5 * porosEff * (*dt) * (bodyForce[oppos]-bodyForce[xiIndex]); + + + for (int iPar = 0; iPar < noTot; iPar++) { + + + OmegaS2 = feqS[iPar * NUXMI xiIndex] - f(xiIndex, 0, 0); + + + OmegaS += (OmegaS1 + OmegaS2 + OmegaS3) * sfp(iPar, 0, 0); + + sumx[iPar] = sumx[iPar] - (OmegaS1 + OmegaS2 + OmegaS3) * XI[xiIndex * LATTDIM] * CS; + sumy[iPar] = sumy[iPar] - (OmegaS1 + OmegaS2 + OmegaS3) * XI[xiIndex * LATTDIM + 1] * CS; + } + + } + //OmegaS = 0.0; + fcopy(xiIndex, 0, 0) = f(xiIndex, 0, 0) - dtOverTauPlusDt * (f(xiIndex, 0, 0)-feq[xiIndex]) + + dtOverTauF * bodyForce[xiIndex] + OmegaS * invDtOverTauF; + + } + + for (int iPar = 0; iPar < noTot; iPar++) { + Fd(2*iPar, 0, 0) = sumx[iPar]; + Fd(2*iPar+1, 0, 0) = sumy[iPar]; + } + } +} + +void KerDragForcePRE2D(const ACC&id, const ACC& sfp, const ACC&xf, const ACC&Fd, const int* idp, const Real* xp, + Real* FDp, const Real* dt, const Real* tau) { + + Real alpha; + int loc; + int noTot = 0; + Real poros = 1.0; + Real Beta, sf; + + + + + for (int pIndex = 0; pIndex < NELEM; pIndex++) + noTot += (id(pIndex, 0, 0) > -1); + + for (int pIndex = 0; pIndex < noTot; pIndex++) + poros -= sfp(pIndex, 0, 0); + + + + sf = 1.0 - poros; + + for (int pIndex = 0; pIndex < noTot; pIndex++) { + if ((*idp) == id(pIndex, 0, 0)) { + Beta = sfp(pIndex, 0, 0) * (*tau) / ((*tau) + 0.5 * poros * (*dt)); + loc = pIndex * SPACEDIM; + FDp[0] += Beta * Fd(loc, 0, 0); + FDp[1] += Beta * Fd(loc + 1, 0, 0); + FDp[2] +=Beta * ((xf(0,0,0)-xp[0]) * Fd(loc+1,0,0) - + (xf(1,0,0) - xp[1]) * Fd(loc,0,0)); + + break; + } + } + + +} +#endif + + +//3D kernels + + +void KerEvaluateSolidFractionMP3D(ACC& id, ACC& sfp, ACC& vp, const ACC& xf, const int* idp, const Real* xp, + const Real* velP, const Real* dx) { + + Real vf[SPACEDIM] , xfl[SPACEDIM]; + Real xe, ye, ze; + Real dd; + Real dx1, dy1, dz1; + int ip, jp, kp; + int status, nEdges, noIndex; + int itot = 0; + int sx[26] = { 1, 1, 1, 1, -1, -1, -1, -1, 1, -1, 0, 0, 0, 0, 1, -1, 1, -1, 0, 0, 0, 0, 1, 1, -1, -1}; + int sy[26] = { 1, 1, -1, -1, 1, 1, -1, -1, 0, 0, 1, -1, 0, 0, 1, 1,-1, -1, 1, -1, 1, -1, 0, 0, 0, 0}; + int sz[26] = { 1, -1, 1, -1, 1, -1, 1, -1, 0, 0, 0, 0, 1, -1, 0, 0, 0, 0, 1, 1, -1, -1, 1, -1, 1, -1}; + + + int Npoints = 9; + nEdges = 26; + dx1 = (*dx); + int no = 0; + + xfl[0] = xf(0, 0, 0, 0); + xfl[1] = xf(1, 0, 0, 0); + xfl[2] = xf(2, 0, 0, 0); + + for (int edgeIndex = 0; edgeIndex < nEdges; edgeIndex++) { + xe = xfl[0] + 0.5 * (*dx) * static_cast(sx[edgeIndex]); + ye = xfl[1] + 0.5 * (*dx) * static_cast(sy[edgeIndex]); + ze = xfl[2] + 0.5 * (*dx) * static_cast(sz[edgeIndex]); + + dd = (xe - xp[0]) * (xe - xp[0]) + (ye - xp[1]) * (ye - xp[1]) + (ze - xp[2]) * (ze - xp[2]); + + if (dd <= xp[3] * xp[3]) + itot += 1; + } + + if (itot == nEdges) { + no += 1; + noIndex = no - 1; + id(noIndex, 0, 0, 0) = (*idp); + sfp(noIndex, 0, 0, 0) = 1.0; + dx1 = xfl[0] - xp[0]; + dy1 = xfl[1] - xp[1]; + dz1 = xfl[2] - xp[2]; + vp(SPACEDIM * noIndex, 0, 0, 0) = velP[0] + velP[4] * dz1 - velP[5] * dy1; + vp(SPACEDIM * noIndex +1, 0, 0, 0) = velP[1] + velP[5] * dx1 - velP[3] * dz1; + vp(SPACEDIM * noIndex +2, 0, 0, 0) = velP[2] + velP[3] * dy1 - velP[4] * dx1; + } + else if (itot > 0) { + no += 1; + noIndex = no - 1; + id(noIndex, 0, 0, 0) = (*idp); + sfp(noIndex, 0, 0, 0) = ComputeSolidFraction3D(xfl,xp, dx1, Npoints); + EstimateVelocity3D(vf, velP, xp, xfl); //To be replaced by an average concept + vp(SPACEDIM * noIndex, 0, 0, 0) = vf[0]; + vp(SPACEDIM * noIndex +1, 0, 0, 0) = vf[1]; + vp(SPACEDIM * noIndex +2, 0, 0, 0) = vf[2]; + } + + +#ifdef CPU + const Real res{sfp(0, 0, 0, 0)}; + if (std::isnan(res) || res < 0 || res > 1.0) { + ops_printf( + "Error! Solid fraction %f not in [0, 1]\n"); + assert(!(std::isnan(res) || res < 0 || res > 1.0)); + } +#endif // CPU + +} + + +void KerSolVelocityUpdateMP3D(ACC& vp, const ACC& id, const ACC& xf, const int* idp, const Real* xp, const Real* up) { + + Real xfl[3]; + Real vf[3]; + int noTot = 0; + + for (int iPar = 0; iPar < NELEM; iPar++) { + noTot += (id(iPar, 0, 0, 0)>-1); + } + + xfl[0] = xf(0, 0, 0, 0); + xfl[1] = xf(1, 0, 0, 0); + xfl[2] = xf(2, 0, 0, 0); + + if (noTot > 0) { + for (int pIndex = 0; pIndex < noTot; pIndex++) { + if ((*idp)==id(pIndex, 0, 0, 0)) { + EstimateVelocity3D(vf, up, xp, xfl); + vp(SPACEDIM * pIndex, 0, 0, 0) = vf[0]; + vp(SPACEDIM * pIndex + 1, 0, 0, 0) = vf[1]; + vp(SPACEDIM * pIndex + 2, 0, 0, 0) = vf[2]; + break; + } + } + } + +} + + + +void KerPRE3D(ACC& fcopy, ACC& Fd,const ACC& f, const ACC& macros,const ACC& nodeType, + const ACC& id, const ACC& sfp, const ACC& vp, const Real* tau, const Real* dt, + const int* forceFlag, const Real* force) { + + VertexType vt = (VertexType) nodeType(0, 0, 0); + bool collisionRequired = (vt != VertexType::ImmersedSolid); + + if (collisionRequired) { + Real dtOverTauPlusDt, dtOverTauF, invDtOverTau; + Real sumx[NELEM], sumy[NELEM], sumz[NELEM]; + Real OmegaS, OmegaS1, OmegaS2, OmegaS3; + Real alpha, sf; + Real poros = 1.0; + Real eff, porosEff; + + + Real u[SPACEDIM], rho; + int oppos; + int noTot = 0; + + + + for (int pIndex = 0; pIndex < NELEM; pIndex++) { + sumx[pIndex] = 0.0; + sumy[pIndex] = 0.0; + sumz[pIndex] = 0.0; + noTot+= (id(pIndex, 0, 0, 0) > -1); + poros-=sfp(pIndex, 0, 0, 0); + + } + + int noTot1; + if (noTot > 0) + noTot1 = noTot; + else + noTot1 = 1; + + sf = 1.0 - poros; + + + + dtOverTauPlusDt = poros * (*dt) / ((*tau) + 0.5 * poros * (*dt)); + dtOverTauF = poros * (*dt) * (*tau) / ((*tau) + 0.5 * poros * (*dt)); + invDtOverTau = (*tau) / ((*tau) + 0.5 * poros * (*dt)); + + + //if (poros < 1) + // printf("Rank %d: sf = %f alpha = %f eff = %f\n", ops_get_proc(), 1-poros, alpha, eff); + + Real up[SPACEDIM * noTot1]; + Real feq[NUMXI], feqS[NUMXI * noTot1], bodyForce[NUMXI]; + + for (int pIndex = 0; pIndex < noTot; pIndex++) { + up[SPACEDIM * pIndex] = vp(pIndex * SPACEDIM, 0, 0, 0); + up[SPACEDIM * pIndex + 1] = vp(pIndex * SPACEDIM + 1, 0, 0, 0); + up[SPACEDIM * pIndex + 2] = vp(pIndex * SPACEDIM + 2, 0, 0, 0); + } + + rho = macros(0, 0, 0, 0); + u[0] = macros(1, 0, 0, 0); + u[1] = macros(2, 0, 0, 0); + u[2] = macros(3, 0, 0, 0); + + + //Calculate BodyForces and equilibrium functions + for (int xiIndex = 0; xiIndex < NUMXI; xiIndex++) { + feq[xiIndex] = CalcBGKFeq(xiIndex, rho, u[0], u[1], u[2], 1, 2); + //printf("xiIndex = %d: feq = %12.9e,u = %12.9e v= %12.9e\n",xiIndex, feq[xiIndex], u[0], u[1]); + if ((*forceFlag) == 1) + bodyForce[xiIndex] = CalcBodyForce2ndOrder(xiIndex, rho, u, force); + else + bodyForce[xiIndex] = 0.0; + + for (int iPar = 0; iPar < noTot; iPar++) + if (sfp(iPar, 0, 0, 0) > 0) + feqS[iPar * NUMXI + xiIndex] = CalcBGKFeq(xiIndex, rho, up[SPACEDIM * iPar], up[SPACEDIM * iPar + 1],up[SPACEDIM * iPar + 2], 1, 2); + else + feqS[iPar * NUMXI + xiIndex] = 0.0; + } + + for (int xiIndex = 0; xiIndex < NUMXI; xiIndex++) { + OmegaS = 0.0; + if (noTot > 0) { + oppos= OPP[xiIndex]; + OmegaS1 = (f(oppos, 0, 0, 0) - feq[oppos]); + OmegaS2 = 0.5 * poros * (*dt) * (bodyForce[oppos]-bodyForce[xiIndex]); + + + for (int iPar = 0; iPar < noTot; iPar++) { + + OmegaS2 = feqS(iPar * NUMXI + xiIndex, 0, 0, 0) - f(xiIndex, 0, 0, 0); + + OmegaS += (OmegaS1 + OmegaS2 + OmegaS3) * sfp(iPar, 0, 0); + +#if DebugLevel >= 3 + printf("Rank %d: xiIndex %d OmegaS = %f\n", ops_get_proc(), xiIndex, OmegaS); +#endif + sumx[iPar] = sumx[iPar] - (OmegaS1 + OmegaS2 + OmegaS3) * XI[xiIndex * LATTDIM] * CS; + sumy[iPar] = sumy[iPar] - (OmegaS1 + OmegaS2 + OmegaS3) * XI[xiIndex * LATTDIM + 1] * CS; + sumz[iPar] = sumz[iPar] - (OmegaS1 + OmegaS2 + OmegaS3) * XI[xiIndex * LATTDIM + 2] * CS; + } + + } + //OmegaS = 0.0; + fcopy(xiIndex, 0, 0, 0) = f(xiIndex, 0, 0, 0) - dtOverTauPlusDt * (f(xiIndex, 0, 0, 0)-feq[xiIndex]) + + dtOverTauF * bodyForce[xiIndex] + invDtOverTau * OmegaS; + + +#ifdef CPU + const Real res{fcopy(xiIndex, 0, 0, 0)}; + if (std::isnan(res) || res < 0 || std::isinf(res)) { + ops_printf( + "Error-PSM: Distribution function %f becomes " + "invalid for the component %i at the lattice " + "%i\n", + res, 0, xiIndex); + assert(!(std::isnan(res) || res < 0 || std::isinf(res))); + } +#endif // CPU + + } + + for (int iPar = 0; iPar < noTot; iPar++) { + Fd(SPACEDIM * iPar, 0, 0, 0) = sumx[iPar]; + Fd(SPACEDIM * iPar + 1, 0, 0, 0) = sumy[iPar]; + Fd(SPACEDIM * iPar + 2, 0, 0, 0) = sumz[iPar]; + } + } +} + +void KerDragPRE3D(const ACC& id, const ACC& sfp, const ACC& xf, const ACC& Fd, + Real* FDp, const int* idp, const Real* xp, const Real* dt, const Real* tau, const Real* gamma) { + + Real alpha, xfl[SPACEDIM]; + int loc; + int noTot = 0; + Real poros = 1.0; + Real Beta, sf; + Real porosEff, eff; + xfl[0] = xf(0, 0, 0, 0); + xfl[1] = xf(1, 0, 0, 0); + xfl[2] = xf(2, 0, 0, 0); + + for (int pIndex = 0; pIndex < NELEM; pIndex++) + noTot += (id(pIndex, 0, 0, 0) > -1); + + + for (int pIndex = 0; pIndex < noTot; pIndex++) + poros -= sfp(pIndex, 0, 0, 0); + sf = 1.0 - poros; + + + + for (int pIndex = 0; pIndex < noTot; pIndex++) { + if ((*idp) == id(pIndex, 0, 0, 0)) { + Beta = sfp(pIndex, 0, 0, 0) * (*tau) / ((*tau) + 0.5 * poros * (*dt)); + loc = pIndex * SPACEDIM; + + FDp[0] += Beta * Fd(loc, 0, 0, 0); + FDp[1] += Beta * Fd(loc + 1, 0, 0, 0); + FDp[2] += Beta * Fd(loc + 2, 0, 0, 0); + FDp[3] += Beta * ((xfl[1] - xp[1]) * Fd(loc+2, 0, 0, 0) - (xfl[2] - xp[2]) * Fd(loc+1,0,0, 0)); + FDp[4] += Beta * ((xfl[2] - xp[2]) * Fd(loc, 0, 0, 0) - (xfl[0] - xp[0]) * Fd(loc+2, 0, 0, 0)); + FDp[5] += Beta * ((xfl[0] - xp[0]) * Fd(loc+1, 0, 0, 0) - (xfl[1] - xp[1]) * Fd(loc, 0, 0, 0)); + + // ops_printf("rank %d: FDp = [%e %e %e]\n", ops_get_proc(), FDp[0], FDp[1], FDp[2]); + break; + } + } + +} +void KerStreamPeriodic3D(ACC & f, const ACC& fStage, + const ACC& nodeType, const ACC& geometry) { +#ifdef OPS_3D + VertexGeometryType vg = (VertexGeometryType)geometry(0, 0, 0); + for (int compoIndex = 0; compoIndex < NUMCOMPONENTS; compoIndex++) { + VertexType vt = (VertexType)nodeType(compoIndex, 0, 0, 0); + for (int xiIndex = COMPOINDEX[2 * compoIndex]; + xiIndex <= COMPOINDEX[2 * compoIndex + 1]; xiIndex++) { + int cx = (int)XI[xiIndex * LATTDIM]; + int cy = (int)XI[xiIndex * LATTDIM + 1]; + int cz = (int)XI[xiIndex * LATTDIM + 2]; + + if (vt == VertexType::Fluid) { + f(xiIndex, 0, 0, 0) = fStage(xiIndex, -cx, -cy, -cz); + } + + if (vt == VertexType::Periodic) + f(xiIndex, 0, 0, 0) = fStage(xiIndex, -cx, -cy, -cz); + + if (vt != VertexType::ImmersedSolid && vt != VertexType::Fluid && vt!= VertexType::Periodic) { + // TODO te be determined if necessary + bool streamRequired{true}; + if (streamRequired) { + if ((cx == 0) && (cy == 0) && (cz == 0)) { + f(xiIndex, 0, 0, 0) = fStage(xiIndex, 0, 0, 0); + continue; + } + } + + switch (vg) { + // faces six types + case VG_IP: + // (cx=0 means stream is implemented at i=0,so here we + // disable the step at boundary) + if (streamRequired) { + if (cx <= 0) { + f(xiIndex, 0, 0, 0) = + fStage(xiIndex, -cx, -cy, -cz); + } + } else { + if (cx < 0) { + f(xiIndex, 0, 0, 0) = + fStage(xiIndex, -cx, -cy, -cz); + } + } + break; + case VG_IM: + if (streamRequired) { + if (cx >= 0) { + f(xiIndex, 0, 0, 0) = + fStage(xiIndex, -cx, -cy, -cz); + } + } else { + if (cx > 0) { + f(xiIndex, 0, 0, 0) = + fStage(xiIndex, -cx, -cy, -cz); + } + } + break; + case VG_JP: + if (streamRequired) { + if (cy <= 0) { + f(xiIndex, 0, 0, 0) = + fStage(xiIndex, -cx, -cy, -cz); + } + } else { + if (cy < 0) { + f(xiIndex, 0, 0, 0) = + fStage(xiIndex, -cx, -cy, -cz); + } + } + break; + case VG_JM: + if (streamRequired) { + if (cy >= 0) { + f(xiIndex, 0, 0, 0) = + fStage(xiIndex, -cx, -cy, -cz); + } + } else { + if (cy > 0) { + f(xiIndex, 0, 0, 0) = + fStage(xiIndex, -cx, -cy, -cz); + } + } + break; + case VG_KP: + // (cx=0 means stream is implemented at i=0,so here we + // disable the step at boundary) + if (streamRequired) { + if (cz <= 0) { + f(xiIndex, 0, 0, 0) = + fStage(xiIndex, -cx, -cy, -cz); + } + } else { + if (cz < 0) { + f(xiIndex, 0, 0, 0) = + fStage(xiIndex, -cx, -cy, -cz); + } + } + break; + case VG_KM: + if (streamRequired) { + if (cz >= 0) { + f(xiIndex, 0, 0, 0) = + fStage(xiIndex, -cx, -cy, -cz); + } + + } else { + if (cz > 0) { + f(xiIndex, 0, 0, 0) = + fStage(xiIndex, -cx, -cy, -cz); + } + } + break; + // faces six types end + // 12 edges + case VG_IPJP_I: + if (streamRequired) { + if (cy <= 0 && cx <= 0) { + f(xiIndex, 0, 0, 0) = + fStage(xiIndex, -cx, -cy, -cz); + } + } else { + if (cy < 0 && cx < 0) { + f(xiIndex, 0, 0, 0) = + fStage(xiIndex, -cx, -cy, -cz); + } + } + break; + case VG_IPJM_I: + if (streamRequired) { + if (cy >= 0 && cx <= 0) { + f(xiIndex, 0, 0, 0) = + fStage(xiIndex, -cx, -cy, -cz); + } + } else { + if (cy > 0 && cx < 0) { + f(xiIndex, 0, 0, 0) = + fStage(xiIndex, -cx, -cy, -cz); + } + } + break; + case VG_IMJP_I: + if (streamRequired) { + if (cy <= 0 && cx >= 0) { + f(xiIndex, 0, 0, 0) = + fStage(xiIndex, -cx, -cy, -cz); + } + } else { + if (cy < 0 && cx > 0) { + f(xiIndex, 0, 0, 0) = + fStage(xiIndex, -cx, -cy, -cz); + } + } + break; + case VG_IMJM_I: + if (streamRequired) { + if (cy >= 0 && cx >= 0) { + f(xiIndex, 0, 0, 0) = + fStage(xiIndex, -cx, -cy, -cz); + } + } else { + if (cy > 0 && cx > 0) { + f(xiIndex, 0, 0, 0) = + fStage(xiIndex, -cx, -cy, -cz); + } + } + break; + // k + case VG_IPKP_I: + if (streamRequired) { + if (cz <= 0 && cx <= 0) { + f(xiIndex, 0, 0, 0) = + fStage(xiIndex, -cx, -cy, -cz); + } + } else { + if (cz < 0 && cx < 0) { + f(xiIndex, 0, 0, 0) = + fStage(xiIndex, -cx, -cy, -cz); + } + } + break; + case VG_IPKM_I: + if (streamRequired) { + if (cz >= 0 && cx <= 0) { + f(xiIndex, 0, 0, 0) = + fStage(xiIndex, -cx, -cy, -cz); + } + } else { + if (cz > 0 && cx < 0) { + f(xiIndex, 0, 0, 0) = + fStage(xiIndex, -cx, -cy, -cz); + } + } + break; + case VG_IMKP_I: + if (streamRequired) { + if (cz <= 0 && cx >= 0) { + f(xiIndex, 0, 0, 0) = + fStage(xiIndex, -cx, -cy, -cz); + } + } else { + if (cz < 0 && cx > 0) { + f(xiIndex, 0, 0, 0) = + fStage(xiIndex, -cx, -cy, -cz); + } + } + break; + case VG_IMKM_I: + if (streamRequired) { + if (cz >= 0 && cx >= 0) { + f(xiIndex, 0, 0, 0) = + fStage(xiIndex, -cx, -cy, -cz); + } + } else { + if (cz > 0 && cx > 0) { + f(xiIndex, 0, 0, 0) = + fStage(xiIndex, -cx, -cy, -cz); + } + } + break; + case VG_JPKP_I: + if (streamRequired) { + if (cz <= 0 && cy <= 0) { + f(xiIndex, 0, 0, 0) = + fStage(xiIndex, -cx, -cy, -cz); + } + } else { + if (cz < 0 && cy < 0) { + f(xiIndex, 0, 0, 0) = + fStage(xiIndex, -cx, -cy, -cz); + } + } + break; + case VG_JPKM_I: + if (streamRequired) { + if (cz >= 0 && cy <= 0) { + f(xiIndex, 0, 0, 0) = + fStage(xiIndex, -cx, -cy, -cz); + } + } else { + if (cz > 0 && cy < 0) { + f(xiIndex, 0, 0, 0) = + fStage(xiIndex, -cx, -cy, -cz); + } + } + break; + case VG_JMKP_I: + if (streamRequired) { + if (cz <= 0 && cy >= 0) { + f(xiIndex, 0, 0, 0) = + fStage(xiIndex, -cx, -cy, -cz); + } + } else { + if (cz < 0 && cy > 0) { + f(xiIndex, 0, 0, 0) = + fStage(xiIndex, -cx, -cy, -cz); + } + } + break; + case VG_JMKM_I: + if (streamRequired) { + if (cz >= 0 && cy >= 0) { + f(xiIndex, 0, 0, 0) = + fStage(xiIndex, -cx, -cy, -cz); + } + } else { + if (cz > 0 && cy > 0) { + f(xiIndex, 0, 0, 0) = + fStage(xiIndex, -cx, -cy, -cz); + } + } + break; + // K + // k_out + case VG_IPJP_O: + if (streamRequired) { + if (cy <= 0 || cx <= 0) { + f(xiIndex, 0, 0, 0) = + fStage(xiIndex, -cx, -cy, -cz); + } + } else { + if (cy < 0 || cx < 0) { + f(xiIndex, 0, 0, 0) = + fStage(xiIndex, -cx, -cy, -cz); + } + } + break; + case VG_IPJM_O: + if (streamRequired) { + if (cy >= 0 || cx <= 0) { + f(xiIndex, 0, 0, 0) = + fStage(xiIndex, -cx, -cy, -cz); + } + } else { + if (cy > 0 || cx < 0) { + f(xiIndex, 0, 0, 0) = + fStage(xiIndex, -cx, -cy, -cz); + } + } + break; + case VG_IMJP_O: + if (streamRequired) { + if (cy <= 0 || cx >= 0) { + f(xiIndex, 0, 0, 0) = + fStage(xiIndex, -cx, -cy, -cz); + } + } else { + if (cy < 0 || cx > 0) { + f(xiIndex, 0, 0, 0) = + fStage(xiIndex, -cx, -cy, -cz); + } + } + break; + case VG_IMJM_O: + if (streamRequired) { + if (cy >= 0 || cx >= 0) { + f(xiIndex, 0, 0, 0) = + fStage(xiIndex, -cx, -cy, -cz); + } + } else { + if (cy > 0 || cx > 0) { + f(xiIndex, 0, 0, 0) = + fStage(xiIndex, -cx, -cy, -cz); + } + } + break; + // IK + case VG_IPKP_O: + if (streamRequired) { + if (cz <= 0 || cx <= 0) { + f(xiIndex, 0, 0, 0) = + fStage(xiIndex, -cx, -cy, -cz); + } + } else { + if (cz < 0 || cx < 0) { + f(xiIndex, 0, 0, 0) = + fStage(xiIndex, -cx, -cy, -cz); + } + } + break; + case VG_IPKM_O: + if (streamRequired) { + if (cz >= 0 || cx <= 0) { + f(xiIndex, 0, 0, 0) = + fStage(xiIndex, -cx, -cy, -cz); + } + } else { + if (cz > 0 || cx < 0) { + f(xiIndex, 0, 0, 0) = + fStage(xiIndex, -cx, -cy, -cz); + } + } + break; + case VG_IMKP_O: + if (streamRequired) { + if (cz <= 0 || cx >= 0) { + f(xiIndex, 0, 0, 0) = + fStage(xiIndex, -cx, -cy, -cz); + } + } else { + if (cz < 0 || cx > 0) { + f(xiIndex, 0, 0, 0) = + fStage(xiIndex, -cx, -cy, -cz); + } + } + break; + case VG_IMKM_O: + if (streamRequired) { + if (cz >= 0 || cx >= 0) { + f(xiIndex, 0, 0, 0) = + fStage(xiIndex, -cx, -cy, -cz); + } + } else { + if (cz > 0 || cx > 0) { + f(xiIndex, 0, 0, 0) = + fStage(xiIndex, -cx, -cy, -cz); + } + } + break; + // JK + case VG_JPKP_O: + if (streamRequired) { + if (cz <= 0 || cy <= 0) { + f(xiIndex, 0, 0, 0) = + fStage(xiIndex, -cx, -cy, -cz); + } + } else { + if (cz < 0 || cy < 0) { + f(xiIndex, 0, 0, 0) = + fStage(xiIndex, -cx, -cy, -cz); + } + } + break; + case VG_JPKM_O: + if (streamRequired) { + if (cz >= 0 || cy <= 0) { + f(xiIndex, 0, 0, 0) = + fStage(xiIndex, -cx, -cy, -cz); + } + } else { + if (cz > 0 || cy < 0) { + f(xiIndex, 0, 0, 0) = + fStage(xiIndex, -cx, -cy, -cz); + } + } + break; + case VG_JMKP_O: + if (streamRequired) { + if (cz <= 0 || cy >= 0) { + f(xiIndex, 0, 0, 0) = + fStage(xiIndex, -cx, -cy, -cz); + } + } else { + if (cz < 0 || cy > 0) { + f(xiIndex, 0, 0, 0) = + fStage(xiIndex, -cx, -cy, -cz); + } + } + break; + case VG_JMKM_O: + if (streamRequired) { + if (cz >= 0 || cy >= 0) { + f(xiIndex, 0, 0, 0) = + fStage(xiIndex, -cx, -cy, -cz); + } + } else { + if (cz > 0 || cy > 0) { + f(xiIndex, 0, 0, 0) = + fStage(xiIndex, -cx, -cy, -cz); + } + } + break; + // k_out end + // 12 edges end + // 8 corners + // inner corners + case VG_IPJPKP_I: + if (streamRequired) { + if (cx <= 0 && cy <= 0 && cz <= 0) { + f(xiIndex, 0, 0, 0) = + fStage(xiIndex, -cx, -cy, -cz); + } + } else { + if (cx < 0 && cy < 0 && cz < 0) { + f(xiIndex, 0, 0, 0) = + fStage(xiIndex, -cx, -cy, -cz); + } + } + break; + case VG_IPJPKM_I: + if (streamRequired) { + if (cx <= 0 && cy <= 0 && cz >= 0) { + f(xiIndex, 0, 0, 0) = + fStage(xiIndex, -cx, -cy, -cz); + } + } else { + if (cx < 0 && cy < 0 && cz > 0) { + f(xiIndex, 0, 0, 0) = + fStage(xiIndex, -cx, -cy, -cz); + } + } + break; + case VG_IPJMKP_I: + if (streamRequired) { + if (cx <= 0 && cy >= 0 && cz <= 0) { + f(xiIndex, 0, 0, 0) = + fStage(xiIndex, -cx, -cy, -cz); + } + } else { + if (cx < 0 && cy > 0 && cz < 0) { + f(xiIndex, 0, 0, 0) = + fStage(xiIndex, -cx, -cy, -cz); + } + } + break; + case VG_IPJMKM_I: + if (streamRequired) { + if (cx <= 0 && cy >= 0 && cz >= 0) { + f(xiIndex, 0, 0, 0) = + fStage(xiIndex, -cx, -cy, -cz); + } + } else { + if (cx < 0 && cy > 0 && cz > 0) { + f(xiIndex, 0, 0, 0) = + fStage(xiIndex, -cx, -cy, -cz); + } + } + break; + case VG_IMJPKP_I: + if (streamRequired) { + if (cx >= 0 && cy <= 0 && cz <= 0) { + f(xiIndex, 0, 0, 0) = + fStage(xiIndex, -cx, -cy, -cz); + } + } else { + if (cx > 0 && cy < 0 && cz < 0) { + f(xiIndex, 0, 0, 0) = + fStage(xiIndex, -cx, -cy, -cz); + } + } + break; + case VG_IMJPKM_I: + if (streamRequired) { + if (cx >= 0 && cy <= 0 && cz >= 0) { + f(xiIndex, 0, 0, 0) = + fStage(xiIndex, -cx, -cy, -cz); + } + } else { + if (cx > 0 && cy < 0 && cz > 0) { + f(xiIndex, 0, 0, 0) = + fStage(xiIndex, -cx, -cy, -cz); + } + } + break; + case VG_IMJMKP_I: + if (streamRequired) { + if (cx >= 0 && cy >= 0 && cz <= 0) { + f(xiIndex, 0, 0, 0) = + fStage(xiIndex, -cx, -cy, -cz); + } + } else { + if (cx > 0 && cy > 0 && cz < 0) { + f(xiIndex, 0, 0, 0) = + fStage(xiIndex, -cx, -cy, -cz); + } + } + break; + case VG_IMJMKM_I: + if (streamRequired) { + if (cx >= 0 && cy >= 0 && cz >= 0) { + f(xiIndex, 0, 0, 0) = + fStage(xiIndex, -cx, -cy, -cz); + } + } else { + if (cx > 0 && cy > 0 && cz > 0) { + f(xiIndex, 0, 0, 0) = + fStage(xiIndex, -cx, -cy, -cz); + } + } + break; + // out corner + case VG_IPJPKP_O: + if (streamRequired) { + if (cx <= 0 || cy <= 0 || cz <= 0) { + f(xiIndex, 0, 0, 0) = + fStage(xiIndex, -cx, -cy, -cz); + } + } else { + if (cx < 0 || cy < 0 || cz < 0) { + f(xiIndex, 0, 0, 0) = + fStage(xiIndex, -cx, -cy, -cz); + } + } + break; + case VG_IPJPKM_O: + if (streamRequired) { + if (cx <= 0 || cy <= 0 || cz >= 0) { + f(xiIndex, 0, 0, 0) = + fStage(xiIndex, -cx, -cy, -cz); + } + } else { + if (cx < 0 || cy < 0 || cz > 0) { + f(xiIndex, 0, 0, 0) = + fStage(xiIndex, -cx, -cy, -cz); + } + } + break; + case VG_IPJMKP_O: + if (streamRequired) { + if (cx <= 0 || cy >= 0 || cz <= 0) { + f(xiIndex, 0, 0, 0) = + fStage(xiIndex, -cx, -cy, -cz); + } + } else { + if (cx < 0 || cy > 0 || cz < 0) { + f(xiIndex, 0, 0, 0) = + fStage(xiIndex, -cx, -cy, -cz); + } + } + break; + case VG_IPJMKM_O: + if (streamRequired) { + if (cx <= 0 || cy >= 0 || cz >= 0) { + f(xiIndex, 0, 0, 0) = + fStage(xiIndex, -cx, -cy, -cz); + } + } else { + if (cx < 0 || cy > 0 || cz > 0) { + f(xiIndex, 0, 0, 0) = + fStage(xiIndex, -cx, -cy, -cz); + } + } + break; + case VG_IMJPKP_O: + if (streamRequired) { + if (cx >= 0 || cy <= 0 || cz <= 0) { + f(xiIndex, 0, 0, 0) = + fStage(xiIndex, -cx, -cy, -cz); + } + } else { + if (cx > 0 || cy < 0 || cz < 0) { + f(xiIndex, 0, 0, 0) = + fStage(xiIndex, -cx, -cy, -cz); + } + } + break; + case VG_IMJPKM_O: + if (streamRequired) { + if (cx >= 0 || cy <= 0 || cz >= 0) { + f(xiIndex, 0, 0, 0) = + fStage(xiIndex, -cx, -cy, -cz); + } + } else { + if (cx > 0 || cy < 0 || cz > 0) { + f(xiIndex, 0, 0, 0) = + fStage(xiIndex, -cx, -cy, -cz); + } + } + break; + case VG_IMJMKP_O: + if (streamRequired) { + if (cx >= 0 || cy >= 0 || cz <= 0) { + f(xiIndex, 0, 0, 0) = + fStage(xiIndex, -cx, -cy, -cz); + } + } else { + if (cx > 0 || cy > 0 || cz < 0) { + f(xiIndex, 0, 0, 0) = + fStage(xiIndex, -cx, -cy, -cz); + } + } + break; + case VG_IMJMKM_O: + if (streamRequired) { + if (cx >= 0 || cy >= 0 || cz >= 0) { + f(xiIndex, 0, 0, 0) = + fStage(xiIndex, -cx, -cy, -cz); + } + } else { + if (cx > 0 || cy > 0 || cz > 0) { + f(xiIndex, 0, 0, 0) = + fStage(xiIndex, -cx, -cy, -cz); + } + } + break; + default: + break; + } + } + } + } +#endif // OPS_3D +} +#endif /* APPS_LBM_DEM_PSM_KERNEL_H_ */ diff --git a/Apps/LBM-DEM/read_input.cpp b/Apps/LBM-DEM/read_input.cpp new file mode 100644 index 0000000..6618ba3 --- /dev/null +++ b/Apps/LBM-DEM/read_input.cpp @@ -0,0 +1,483 @@ +/* + * read_input.cpp + * + * Created on: Sep 24, 2020 + * Author: jpd38567 + */ + + + +#include "read_input.h" +#include +#include +#include +#include +//#include "flowfield.h" + +void ReadBoundaryConditions(configData ¶m) { + + int spacedim; + int BCDir[param.spacedim]; + int bcsize = 2 * param.spacedim; + int BCType[bcsize]; + int size = param.spacedim * param.spacedim * 2; + double uWall[size]; + + for (int iW = 0; iW < size; iW++) + uWall[iW] = 0; + + + if (ops_get_proc()==0) { + + printf("Reading boundary conditions\n"); + FILE *cfilex; + cfilex = fopen("input_boundaries.txt","r"); + if (cfilex == NULL) { + printf("FILE CANNOT BE OPENED\n"); + exit (EXIT_FAILURE); + } + fscanf(cfilex, "%d", &spacedim); + + if (spacedim != param.spacedim) { + printf("MP-LBM ERROR: Dimensional space in boundary conditions not consistent\n"); + printf("Actual SPACEDIM: %d and Boundary condition spacedim: %d", param.spacedim, spacedim); + exit(EXIT_FAILURE); + } + for (int iDim = 0; iDim < spacedim; iDim++) { + //Read first direction walls + printf("Reading walls in %d direction\n",iDim); + fscanf(cfilex, "%d", &BCDir[iDim]); + fscanf(cfilex, "%d", &BCType[2 * iDim]); +#ifdef OPS_3D + fscanf(cfilex, "%lf %lf %lf", &uWall[2 * spacedim * iDim], &uWall[2 * spacedim * iDim +1], &uWall[2 * spacedim * iDim + 2]); +#endif + +#ifdef OPS_2D + fscanf(cfilex, "%lf %lf", &uWall[2 * spacedim * iDim], &uWall[2 * spacedim * iDim +1] ); +#endif + + //2nd wall + fscanf(cfilex, "%d", &BCType[2 * iDim + 1]); + +#ifdef OPS_3D + fscanf(cfilex, "%lf %lf %lf", &uWall[2 * spacedim * iDim + 3], &uWall[2 * spacedim * iDim + 4], &uWall[2 * spacedim * iDim + 5]); +#endif + +#ifdef OPS_2D + fscanf(cfilex, "%lf %lf", &uWall[2 * spacedim * iDim + 2], &uWall[2 * spacedim * iDim + 3] ); +#endif + } + + fclose(cfilex); + } + + spacedim = param.spacedim; + MPI_Bcast(BCDir, spacedim, MPI_INT, 0, OPS_MPI_GLOBAL); + MPI_Bcast(BCType, bcsize, MPI_INT, 0, OPS_MPI_GLOBAL); + MPI_Bcast(uWall, size, MPI_DOUBLE, 0, OPS_MPI_GLOBAL); + + param.updateBoundaries(BCDir, BCType, uWall, spacedim); + + + + + +} + + +void ReadInputFile(configData ¶m, long int timestep) { + + char tmp[101], tmpStor[101]; + int blockSize, spacedim, size, muiInterfaceFlag, iters, exportStep; + int *grid; + Real *gridStart; + + std::vector gridTmp; + std::vector startTmp; + double Lx, tauRef, RadTmp, gamma; + + if (ops_get_proc()==0) { + printf("Rank %d: Ready to read input parameters\n", ops_get_proc()); + FILE *cfilex; + + + + cfilex = fopen("input_params.txt","r"); + if (cfilex == NULL) { + printf("FILE CANNOT BE OPENED\n"); + exit (EXIT_FAILURE); + } + + fscanf(cfilex,"%100s",tmp); + + printf("I read the filename\n"); + + fscanf(cfilex, "%d", &blockSize); + fscanf(cfilex, "%d", &spacedim); + + int temp; + Real temp1; + for (int iBlock = 0; iBlock < blockSize; iBlock++) { + for (int iDir = 0; iDir < spacedim; iDir++) { + fscanf(cfilex, "%d", &temp); + gridTmp.push_back(temp); + } + for (int iDir = 0; iDir < spacedim; iDir++) { + fscanf(cfilex, "%lf", &temp1); + startTmp.push_back(temp1); + } + } + + fscanf(cfilex, "%lf", &Lx); + fscanf(cfilex, "%d", &muiInterfaceFlag); + fscanf(cfilex, "%lf", &tauRef); + fscanf(cfilex, "%lf", &RadTmp); + fscanf(cfilex, "%d", &iters); + fscanf(cfilex, "%d", &exportStep); + fscanf(cfilex, "%lf", &gamma); + + fscanf(cfilex, "%100s", tmpStor); + + fclose(cfilex); + //Broadcasting the options + + } + + MPI_Bcast(tmp, 100, MPI_CHAR, 0, OPS_MPI_GLOBAL); + MPI_Bcast(&blockSize, 1 , MPI_INT, 0, OPS_MPI_GLOBAL); + MPI_Bcast(&spacedim, 1, MPI_INT, 0, OPS_MPI_GLOBAL); + MPI_Bcast(&Lx, 1, MPI_DOUBLE, 0, OPS_MPI_GLOBAL); + MPI_Bcast(&muiInterfaceFlag, 1, MPI_INT,0, OPS_MPI_GLOBAL); + MPI_Bcast(&tauRef, 1, MPI_DOUBLE, 0, OPS_MPI_GLOBAL); + MPI_Bcast(&RadTmp, 1, MPI_DOUBLE, 0, OPS_MPI_GLOBAL); + MPI_Bcast(&iters, 1, MPI_INT, 0, OPS_MPI_GLOBAL); + MPI_Bcast(&exportStep, 1, MPI_INT, 0, OPS_MPI_GLOBAL); + MPI_Bcast(&gamma, 1, MPI_DOUBLE, 0, OPS_MPI_GLOBAL); + MPI_Bcast(tmpStor, 100, MPI_CHAR, 0, OPS_MPI_GLOBAL); + +// printf("I passed broadcastingStep\n"); + + //Ready to add to the class object + //printf("Rank %d: Spacedim: %d blocks: %d\n", ops_get_proc(), spacedim, blockSize); + size = spacedim * blockSize; + grid = new int[size]; + gridStart = new Real[size]; + + if (ops_get_proc()==0) { + for (int iLoc = 0; iLoc < size; iLoc++) + grid[iLoc] = gridTmp[iLoc]; + + for (int iLoc = 0; iLoc < size; iLoc++) + gridStart[iLoc] = startTmp[iLoc]; + + for (int iLoc = 0; iLoc < size; iLoc++) + printf("%d grid =%d starts at %f\n", iLoc, grid[iLoc], gridStart[iLoc]); + + } + MPI_Bcast(grid, size, MPI_INT, 0, OPS_MPI_GLOBAL); + MPI_Bcast(gridStart, size, MPI_DOUBLE, 0, OPS_MPI_GLOBAL); + + + + + param.casename = tmp; + + //Sanity checks and data extraction + if (blockSize < 1) { + ops_printf("MP-LBM ERROR: Number of blocks smaller than zero\n"); + exit(EXIT_FAILURE); + } + else + param.blockNum = blockSize; + + if (spacedim < 2 || spacedim > 3) { + ops_printf("MP-LBM ERROR: Current dimension %d not supported\n", spacedim); + exit(EXIT_FAILURE); + } + else + param.spacedim = spacedim; + + if (tauRef < 0.0) { + ops_printf("MP-LBM ERROR: Relaxation time smaller than zero\n"); + exit(EXIT_FAILURE); + } + else + param.tauRef = tauRef; + + if (Lx <= 0.0) { + ops_printf("MP-LBM ERROR: The size of the simulation domain is not finite\n"); + exit(EXIT_FAILURE); + } + else + param.dx = Lx; + + if (RadTmp < 0.0) { + ops_printf("MP-LBM ERROR: Non positive radius\n"); + exit(EXIT_FAILURE); + } + else + param.Rmax = RadTmp; + + param.storageOption = tmpStor; + + + + + param.updateGrid(grid, size); + + //printf("Rank %d: UpdatedGrid() passed\n", ops_get_proc()); + param.updateStart(gridStart); + //printf("Rank %d: UpdateStart() passed\n", ops_get_proc()); + + if (muiInterfaceFlag < 0 && muiInterfaceFlag > 1) { + ops_printf("MP-LBM ERROR: The flag for mui should be 0 or 1\n"); + exit(EXIT_FAILURE); + } + else + param.muiFlag = muiInterfaceFlag; + + if (timestep < 0 ) { + ops_printf("MP-LBM ERROR: The restart step is smaller than zero\n"); + exit(EXIT_FAILURE); + } + else + param.restartStep = timestep; + + if (iters < 0) { + ops_printf("MP-LBM ERROR: Non positive number of iterations\n"); + exit(EXIT_FAILURE); + } + else + param.iters = static_cast(iters); + + if (exportStep < 0) { + ops_printf("MP-LBM ERROR: Non positive export steps\n"); + exit(EXIT_FAILURE); + } + else + param.exportSteps = exportStep; + + if (gamma < 0) { + ops_printf("MP-LBM ERROR: Non positive gamma. Gamma set to zero\n"); + param.gamma = 0.0; + } + else + param.gamma = gamma; + + ReadBoundaryConditions(param); + + param.boundaries.PrintBoundaries(); + delete[] grid; + delete[] gridStart; + //We + +} + +configData::configData() { + + spacedim = 2; + tauRef = 0.001; + blockNum = 1; + dx = 0.00001; + muiFlag = 0; + restartStep = 0; + Rmax = 1.0; + gamma = 0.0; +} + +void configData::updateGrid(int* grid, int gridSize) { + + int size = spacedim * blockNum; + if (gridSize != size) { + ops_printf("MP-LBM ERROR: The size of imported not equal to internal size\n"); + exit(EXIT_FAILURE); + } + + + for (int iDir = 0; iDir < size; iDir++) { + if (grid[iDir] > 0) + gridBlock.push_back(grid[iDir]); + else { + ops_printf("MP-LBM ERROR: Non positive value for block %d in %d direction\n", iDir/(spacedim * blockNum), iDir - spacedim * blockNum); + exit(EXIT_FAILURE); + } + } + + dx /= (static_cast(gridBlock[0]) - 1.0); + +} + + +void configData::printData() { + printf("Simulation case: %s\n", casename.c_str()); + printf("Rank %d: The dimensional space is %d and %d blocks employed\n",ops_get_proc(), spacedim, blockNum); + printf("Rank %d: A uniform grid of dx = %f is employed\n",ops_get_proc(), dx); + for (int iDir = 0; iDir < blockNum; iDir++) { + printf("Rank %d [",ops_get_proc()); + for (int ix =0; ix < spacedim; ix++) + printf("%d, ", gridBlock[iDir * spacedim + ix]); + printf("] "); + + printf("which starts at["); + for (int ix = 0; ix uVels; + + for (int iDir = 0; iDir < spacedim; iDir++) { + BoundaryScheme temp = static_cast (BType[2 * iDir]); + BoundaryType.push_back(temp); + + for (int jDir = 0; jDir < spacedim ; jDir++) { + uVels.push_back(vels[2 * iDir * spacedim + jDir]); + } + + velocities.push_back(uVels); + + + //Second wall + uVels.clear(); + temp = static_cast(BType[2 * iDir + 1]); + BoundaryType.push_back(temp); + + for (int jDir = 0; jDir < spacedim; jDir++) + uVels.push_back(vels[(2 * iDir + 1) * spacedim + jDir]); + + velocities.push_back(uVels); + uVels.clear(); + + } + +} + + +void BoundaryData::PrintBoundaries() { + + int rank = ops_num_procs() - 1; + + + if (rank == ops_get_proc()) { + + + if (DirBc.empty()) { + printf("MP-LBM: BoundaryData not defined properly"); + return; + } +#ifdef OPS_3D + printf("--------------------------------------------------------------------------------------------------------------------------------\n"); + printf("Printing boundaries at rank %d\n", ops_get_proc()); + + printf("Boundary 1: Dir %d wall location is %d boundary scheme %d and Uw=[%f %f %f]\n", DirBc[0], BoundarySurfaces[0], BoundaryType[0], + velocities[0][0], velocities[0][1],velocities[0][2]); + + printf("Boundary 2: Dir %d wall location is %d boundary scheme %d and Uw=[%f %f %f]\n", DirBc[0], BoundarySurfaces[1], BoundaryType[1], + velocities[1][0], velocities[1][1],velocities[1][2]); + + printf("Boundary 3: Dir %d wall location is %d boundary scheme %d and Uw=[%f %f %f]\n", DirBc[1], BoundarySurfaces[2], BoundaryType[2], + velocities[2][0], velocities[2][1],velocities[2][2]); + + printf("Boundary 4: Dir %d wall location is %d boundary scheme %d and Uw=[%f %f %f]\n", DirBc[1], BoundarySurfaces[3], BoundaryType[3], + velocities[3][0], velocities[3][1], velocities[3][2]); + + printf("Boundary 5: Dir %d wall location is %d boundary scheme %d and Uw=[%f %f %f]\n", DirBc[2], BoundarySurfaces[4], BoundaryType[4], + velocities[4][0], velocities[4][1], velocities[4][2]); + + printf("Boundary 6: Dir %d wall location is %d boundary scheme %d and Uw=[%f %f %f]\n", DirBc[2], BoundarySurfaces[5], BoundaryType[5], + velocities[5][0], velocities[5][1], velocities[5][2]); +#endif + +#ifdef OPS_2D + printf("Boundary 1: Dir %d wall location is %d boundary scheme %d and Uw=[%f %f]\n", DirBc[0], BoundarySurfaces[0], BoundaryType[0], + velocities[0][0], velocities[0][1]); + + + printf("Boundary 2: Dir %d wall location is %d boundary scheme %d and Uw=[%f %f]\n", DirBc[0], BoundarySurfaces[1], BoundaryType[1], + velocities[1][0], velocities[1][1]); + + printf("Boundary 3: Dir %d wall location is %d boundary scheme %d and Uw=[%f %f]\n", DirBc[1], BoundarySurfaces[2], BoundaryType[2], + velocities[2][0], velocities[2][1]); + + printf("Boundary 3: Dir %d wall location is %d boundary scheme %d and Uw=[%f %f]\n", DirBc[1], BoundarySurfaces[3], BoundaryType[3], + velocities[3][0], velocities[3][1]); +#endif + + printf("--------------------------------------------------------------------------------------------------------------------------------\n"); + } +} diff --git a/Apps/LBM-DEM/read_input.h b/Apps/LBM-DEM/read_input.h new file mode 100644 index 0000000..c4ad02a --- /dev/null +++ b/Apps/LBM-DEM/read_input.h @@ -0,0 +1,54 @@ +/* + * read_data.h + * + * Created on: Sep 24, 2020 + * Author: jpd38567 + */ + +#ifndef READ_INPUT_H_ +#define READ_INPUT_H_ + +#include +#include +#include "type.h" + +class BoundaryData { + public: + std::vector DirBc; //Direction of boundary walls + std::vector BoundaryType; //Type of boundary + std::vector> velocities; + std::vector BoundarySurfaces; + BoundaryData(); + void ImportDirection(int * dir, int spacedim); + void ImportBoundaryData(int *BType, double* vels); + void PrintBoundaries(); +}; + +class configData { + public: + std::string casename; + std::string storageOption; + SizeType blockNum; + int spacedim; + int muiFlag; + Real dx; + std::vector gridBlock; + std::vector gridStart; + SizeType restartStep; + double tauRef; + double Rmax; + double gamma; + SizeType iters; + int exportSteps; + BoundaryData boundaries; + configData(); + void updateGrid(int* grid, int gridSize); + void printData(); + void updateBoundaries(int *dir, int *BType, double* vels, int spacedim); + void updateStart(Real* gridStart); +}; + + + +void ReadInputFile(configData &data,long int timestep = 0); +#endif /* APPS_LBM_DEM_CLASS_INTERFACE_READ_DATA_H_ */ diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 0000000..641a13b --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,305 @@ +cmake_minimum_required(VERSION 3.18) +project(MPLB C CXX) +option(VERBOSE "Turn on verbose warning messages" OFF) +option(OPTIMISE "Turn on optimised mode" OFF) +#option(TEST "Turn on tests for Apps" OFF) +if (NOT VERBOSE) + message("We show concise compiling information by defautl! Use -DVERBOSE=ON to switch on.") +endif() +if (NOT OPTIMISE) + message("We use the development mode by defautl! Use -DOPTIMISE=ON to use the optimised mode.") +endif() +set(CMAKE_VERBOSE_MAKEFILE ${VERBOSE}) +set(LibDir ${CMAKE_SOURCE_DIR}/Src) +# Use the Release mode by default +if ( NOT CMAKE_BUILD_TYPE ) + message("We choose the Release mode! Please uses -DCMAKE_BUILD_TYPE to change.") + set(CMAKE_BUILD_TYPE Release) +endif() +# Configure Compilers +# C +set(CMAKE_C_STANDARD 99) +#TODO:Shall we keep the "-g" in the release mode? It increases file size. +if (${CMAKE_C_COMPILER_ID} STREQUAL GNU) + set(CMAKE_C_FLAGS "-fPIC -Wall") + set(CMAKE_C_FLAGS_RELEASE "-O3") + set(CMAKE_C_FLAGS_DEBUG "-O0 -g") +elseif (${CMAKE_C_COMPILER_ID} STREQUAL Clang) + set(CMAKE_C_FLAGS "-fPIC -DUNIX -Wall") + set(CMAKE_C_FLAGS_RELEASE "-O3") + set(CMAKE_C_FLAGS_DEBUG "-g -O0") +elseif (${CMAKE_C_COMPILER_ID} STREQUAL MSVC) + set(CMAKE_C_FLAGS "-W1") + set(CMAKE_C_FLAGS_RELEASE "/O2 /Ot ") + set(CMAKE_C_FLAGS_DEBUG "/Z7 /DEBUG") +elseif (${CMAKE_C_COMPILER_ID} STREQUAL Intel) + set(CMAKE_C_FLAGS "-xHost") + set(CMAKE_C_FLAGS_RELEASE "-O3") + set(CMAKE_C_FLAGS_DEBUG "-O0 -g") +else () + message(WARNING "There is no default flag for the chosen C compiler! Please use -DCFLAG=XXX to supply!") +endif () + +# C++ +set(CMAKE_CXX_STANDARD 11) +if (${CMAKE_CXX_COMPILER_ID} STREQUAL GNU) + set(CMAKE_CXX_FLAGS "-fPIC -Wall") + set(CMAKE_CXX_FLAGS_RELEASE "-O3") + set(CMAKE_CXX_FLAGS_DEBUG "-g -O0") +elseif (${CMAKE_CXX_COMPILER_ID} STREQUAL Clang) + set(CMAKE_CXX_FLAGS "-fPIC -DUNIX -Wall") + set(CMAKE_CXX_FLAGS_RELEASE "-O3") + set(CMAKE_CXX_FLAGS_DEBUG "-g -O0") +elseif (${CMAKE_CXX_COMPILER_ID} STREQUAL MSVC) + set(CMAKE_CXX_FLAGS "-W1") + set(CMAKE_CXX_FLAGS_RELEASE "/O2 /Ot ") + set(CMAKE_CXX_FLAGS_DEBUG "/Z7 /DEBUG") +elseif (${CMAKE_CXX_COMPILER_ID} STREQUAL Intel) + set(CMAKE_CXX_FLAGS "-xHost") + set(CMAKE_CXX_FLAGS_RELEASE "-O3") + set(CMAKE_CXX_FLAGS_DEBUG " -O0 -g -DOPS_DEBUG ") +else () + message(WARNING "There is no default flag for the chosen C++ compiler! Please use -DCXXFLAG=XXX to supply!") +endif () + +if (CFLAG) + SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${CFLAG}") +else () + message("Extra C compiler options by using CFLAG, e.g., -DCFLAG=\"-ffloat-store\" for GNU!") +endif () + +if (CXXFLAG) + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CXXFLAG}") +else () + message("Extra C++ compiler options by using CXXFLAG, e.g., -DCXXFLAG=\"-ffloat-store\" for GNU!") +endif () +# Try to find the required dependency +set(HDF5_PREFER_PARALLEL true) +find_package(MPI QUIET) +find_package(HDF5 QUIET COMPONENTS C HL) +# Configure the "include" dir for compiling + +if (NOT HDF5_FOUND) + message (WARNING "We cannot find the parallel HDF5 library. The HDF5 IO routines won't work! Please use -DHDF5_ROOT to specify the path!") +endif () +if (NOT MPI_FOUND) + message (WARNING "We cannot find a MPI environment! The MPI codes won't compile!") +endif () +if (OPTIMISE) + find_package(CUDAToolkit QUIET) + find_package(OpenACC QUIET) + find_package(OpenCL QUIET) + find_package(OpenMP QUIET) + find_package(Python2 QUIET) + if (NOT Python2_FOUND) + message (FATAL_ERROR "We cannot find Python2 and the Python translator needs Python2! Please use -DPython2_EXECUTABLE to specify the path.") + else() + FIND_PATH (opsc NAMES "ops.py" PATHS ${OPS_ROOT} PATH_SUFFIXES bin/ops_translator/c) + FIND_PATH (opsfortran NAMES "ops_fortran.py" PATHS ${OPS_ROOT} PATH_SUFFIXES bin/ops_translator/fortran) + if (NOT opsc) + message (FATAL_ERROR "We cannot find the python C/C++ translator!") + endif() + if (NOT opsfortran) + message (FATAL_ERROR "We cannot find the python fortran translator!") + endif() + set(OPS_C_TRANSLATOR "${opsc}/ops.py") + set(OPS_F_TRANSLATOR "${opsfortran}/ops_fortran.py") + endif () # python2 + + if (CUDAToolkit_FOUND) + set(CMAKE_CUDA_COMPILER ${CUDAToolkit_NVCC_EXECUTABLE}) + if (GPU_ARCH) + set(CMAKE_CUDA_ARCHITECTURES 70 CACHE STRING "CUDA architectures") + else() + message(WARNING "Please the GPU architecture using -DGPU_ARCH=XXX!") + endif() + set(CMAKE_CUDA_FLAGS_DEBUG " -G ${CMAKE_CUDA_FLAGS_DEBUG}") + enable_language(CUDA) + else() + message (WARNING "We cannot find a CUDA kit! The CUDA codes won't compile!") + endif () + + if (OpenCL_FOUND) + include_directories(${OpenCL_INCLUDE_DIRS}) + else() + message (WARNING "We cannot find a OpenCL kit! The OpenCL codes won't compile!") + endif () +endif () # optimise +if (OPS_ROOT) + set(OPS_DIR ${OPS_ROOT}) + find_package(OPS PATHS ${OPS_DIR}/lib/cmake) +else() + find_package(OPS) +endif() +if (NOT OPS_FOUND) + message(FATAL_ERROR "We cannot find the OPS library! If it is installed, please specify its directory by using -DOPS_ROOT=XXX.") +endif() + +set(HDF5_SEQ ${HDF5_FOUND}) +if (OPTIMISE) + set(CUDA ${CUDAToolkit_FOUND}) + set(OPENCL ${OpenCL_FOUND}) +endif() + +if (MPI_FOUND) + set(MPI TRUE) + set(HDF5_MPI ${HDF5_FOUND}) + if (OPTIMISE) + set(CUDA_MPI ${CUDAToolkit_FOUND}) + set(OPENCL_MPI ${OpenCL_FOUND}) + endif() +endif() +#prepare ctest +if (TEST) + enable_testing() + include(ProcessorCount) + ProcessorCount(CPU_NUMBER) + if(CPU_NUMBER GREATER 12) + set(CPU_NUMBER 12) + endif() + if (NOT GPU_NUMBER) + message(WARNING "GPU tests are disabled! Please use -DGPU_NUMBER to specify!") + endif() +endif() + +# clean the tmp directory for the code generation +macro(CreateTempDir) + if(EXISTS ${CMAKE_CURRENT_BINARY_DIR}/tmp) + file(REMOVE_RECURSE "${CMAKE_CURRENT_BINARY_DIR}/tmp") + endif(EXISTS ${CMAKE_CURRENT_BINARY_DIR}/tmp ) + file(MAKE_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/tmp" ) +endmacro() + +macro(SeqDevTarget SpaceDim DebugLevel) + add_executable(${AppName}SeqDev ${LibSrcPath} ${AppSrc}) + target_include_directories(${AppName}SeqDev PRIVATE ${LibDir} ${CMAKE_CURRENT_SOURCE_DIR}) + target_link_libraries(${AppName}SeqDev OPS::ops_hdf5_seq OPS::ops_seq hdf5::hdf5 hdf5::hdf5_hl MPI::MPI_CXX) + target_compile_definitions(${AppName}SeqDev PRIVATE -DOPS_${SpaceDim}D -DCPU -DLEVEL=DebugLevel=${DebugLevel}) +endmacro(SeqDevTarget DebugLevel) + +macro(MpiDevTarget SpaceDim DebugLevel) + if (MPI) + add_executable(${AppName}MpiDev ${LibSrcPath} ${AppSrc}) + target_include_directories(${AppName}MpiDev PRIVATE ${LibDir} ${CMAKE_CURRENT_SOURCE_DIR}) + target_link_libraries(${AppName}MpiDev OPS::ops_hdf5_mpi OPS::ops_mpi hdf5::hdf5 hdf5::hdf5_hl MPI::MPI_CXX) + target_compile_definitions(${AppName}MpiDev PRIVATE -DOPS_${SpaceDim}D -DOPS_MPI -DCPU -DLEVEL=DebugLevel=${DebugLevel} ) + endif() +endmacro(MpiDevTarget DebugLevel) + +# The files needed to be translated by ops.py from the library side +set(LibSrcGenList boundary_wrapper.cpp flowfield_wrapper.cpp model_wrapper.cpp scheme_wrapper.cpp model.cpp) +set(LibKernelGenList boundary_kernel.inc flowfield_kernel.inc model_kernel.inc scheme_kernel.inc) + +function (WriteJsonConfig Dir AppName LibSrc AppSrcGenList AppKernelGenList HeadList SpaceDim) + set(SourceKey "\"source\":[" ) + set(LibGenList "${LibSrcGenList}") + foreach(LibGen IN LISTS LibGenList) + if (NOT (LibGen IN_LIST LibSrc)) + list(REMOVE_ITEM LibGenList ${LibGen}) + endif() + endforeach() + + foreach(Source IN LISTS LibGenList AppSrcGenList) + string(APPEND SourceKey "\"${Source}\",") + endforeach() + string(LENGTH ${SourceKey} Len) + math(EXPR Len "${Len}-1") + string(SUBSTRING ${SourceKey} 0 ${Len} SourceKey) + string(APPEND SourceKey "],") + + set(KernelKey "\"kernel\":[" ) + foreach(Kernel IN LISTS LibKernelGenList AppKernelGenList) + string(APPEND KernelKey "\"${Kernel}\",") + endforeach() + string(LENGTH ${KernelKey} Len) + math(EXPR Len "${Len}-1") + string(SUBSTRING ${KernelKey} 0 ${Len} KernelKey) + string(APPEND KernelKey "],") + list(LENGTH HeadList HeadNum) + if (${HeadNum} LESS 1) + set(HeadKey "") + else() + set(HeadKey "\"head\":[" ) + foreach(Head IN LISTS HeadList) + string(APPEND HeadKey "\"${Head}\",") + endforeach() + string(LENGTH ${HeadKey} Len) + math(EXPR Len "${Len}-1") + string(SUBSTRING ${HeadKey} 0 ${Len} HeadKey) + string(APPEND HeadKey "],") + endif() + + set(SpaceDimKey "\"spacedim\": ${SpaceDim},") + set(CaseKey "\"case\": \"${AppName}\",") + set(BackendKey "\"backend\":[\"mpi_lazy\",\"mpi_cuda\"]") + set(TypeDefKey "\"typedef\": { \"Real\": \"double\"},") + + file(WRITE "${Dir}/OPSPYConfig.json" "{ ${CaseKey} ${SourceKey} ${KernelKey} ${TypeDefKey} ${HeadKey} ${SpaceDimKey} ${BackendKey} }") +endfunction(WriteJsonConfig AppName AppSrcGenList AppKernelGenList AppHeadGenList SpaceDim) + +macro (TranslateSourceCodes LibDir LibSrcGenList AppSrcGenList Destination) + file(GLOB LibFiles ${LibDir}/*.cpp ${LibDir}/*.h ${LibDir}/*.inc ${LibDir}/*.hpp) + file(COPY ${LibFiles} DESTINATION ${Destination}) + file(GLOB AppFiles *.cpp *.h *.inc *.hpp) + file(COPY ${AppFiles} DESTINATION ${Destination}) + execute_process ( + COMMAND ${OPS_C_TRANSLATOR} + WORKING_DIRECTORY ${Destination} + ) + set(TranslatedSrc "") + set(LibSrcCopy ${LibSrc}) + foreach(Src IN LISTS LibSrcCopy) + if (Src IN_LIST LibSrcGenList) + list(REMOVE_ITEM LibSrc ${Src}) + get_filename_component(SrcName "${Src}" NAME_WE) + list(APPEND TranslatedSrc ${Destination}/${SrcName}_ops.cpp) + endif() + endforeach() + + set(TmpSrcPath "") + foreach(Src IN LISTS LibSrc LibTranslated) + list(APPEND TmpSrcPath ${Destination}/${Src}) + endforeach(Src IN LISTS LibSrc LibTranslated) + list(REMOVE_ITEM AppSrc ${AppSrcGenList}) + set(TmpAppSrcPath "") + foreach(Src IN LISTS AppSrc) + list(APPEND TmpAppSrcPath ${Destination}/${Src}) + endforeach(Src IN LISTS AppSrc) + foreach(Src IN LISTS AppSrcGenList) + get_filename_component(SrcName ${Src} NAME_WLE) + list(APPEND TranslatedSrc ${Destination}/${SrcName}_ops.cpp) + endforeach(Src IN LISTS AppSrc) +endmacro(TranslateSourceCodes LibDir LibSrcGenList AppSrcGenList Destination) + +macro(SeqTarget SpaceDim) + add_executable(${AppName}Seq ${TMP_SOURCE_DIR}/MPI_OpenMP/${AppName}_cpu_kernels.cpp ${TmpSrcPath} ${TmpAppSrcPath} ${TranslatedSrc}) + target_include_directories(${AppName}Seq PRIVATE ${TMP_SOURCE_DIR}) + target_link_libraries(${AppName}Seq PRIVATE OPS::ops_hdf5_seq OPS::ops_seq hdf5::hdf5 hdf5::hdf5_hl MPI::MPI_CXX) + target_compile_definitions(${AppName}Seq PRIVATE -DOPS_${SpaceDim}D -DLEVEL=DebugLevel=0) +endmacro(SeqTarget) + +macro(MpiTarget SpaceDim) + if (MPI) + add_executable(${AppName}Mpi ${TMP_SOURCE_DIR}/MPI_OpenMP/${AppName}_cpu_kernels.cpp ${TmpSrcPath} ${TmpAppSrcPath} ${TranslatedSrc}) + target_include_directories(${AppName}Mpi PRIVATE ${TMP_SOURCE_DIR}) + target_link_libraries(${AppName}Mpi PRIVATE OPS::ops_hdf5_mpi OPS::ops_mpi hdf5::hdf5 hdf5::hdf5_hl MPI::MPI_CXX) + target_compile_definitions(${AppName}Mpi PRIVATE -DOPS_${SpaceDim}D -DOPS_MPI -DLEVEL=DebugLevel=0 ) + endif() +endmacro(MpiTarget) + +macro(CudaTarget SpaceDim) +if (CUDAToolkit_FOUND) + add_executable(${AppName}Cuda ${TMP_SOURCE_DIR}/CUDA/${AppName}_kernels.cu ${TmpSrcPath} ${TmpAppSrcPath} ${TranslatedSrc}) + set_property(TARGET ${AppName}Cuda PROPERTY CUDA_STANDARD 11) + target_include_directories(${AppName}Cuda PRIVATE ${TMP_SOURCE_DIR}) + target_link_libraries(${AppName}Cuda PRIVATE OPS::ops_hdf5_seq OPS::ops_cuda CUDA::cudart_static hdf5::hdf5 hdf5::hdf5_hl MPI::MPI_CXX) + target_compile_definitions(${AppName}Cuda PRIVATE -DOPS_${SpaceDim}D -DLEVEL=DebugLevel=0) +endif() +endmacro(CudaTarget) +add_subdirectory(Apps/3DCavity) +add_subdirectory(Apps/2DCavity) +add_subdirectory(Apps/3DLChannel) +add_subdirectory(Tests/FieldBlock) + + + diff --git a/Docs/Python_Script_usage_Dist_function.md b/Docs/Python_Script_usage_Dist_function.md index ad4653a..3e703f3 100644 --- a/Docs/Python_Script_usage_Dist_function.md +++ b/Docs/Python_Script_usage_Dist_function.md @@ -64,7 +64,7 @@ For this example, the file `lbm3d_hilemms.cpp` was defined as follows:- #include #include "boundary.h" #include "evolution.h" -#include "evolution3d.h" +#include "evolution.h" #include "flowfield.h" #include "hilemms.h" #include "model.h" diff --git a/Src/PostProcess.py b/Src/PostProcess.py index 3b3ef54..7b4059b 100644 --- a/Src/PostProcess.py +++ b/Src/PostProcess.py @@ -119,8 +119,8 @@ def ReadVariableFromHDF5(fileName,varName,varLen=1,haloNum=1,withHalo=False): else: res = data if spaceDim==2: - nx = int(rawData.shape[2]/varLen)-2*haloNum - ny = rawData.shape[1]-2*haloNum + nx = int(rawData.shape[1]/varLen)-2*haloNum + ny = rawData.shape[0]-2*haloNum if (varLen == 1): if not withHalo: data = rawData[haloNum:-haloNum, haloNum:-haloNum] diff --git a/Src/block.cpp b/Src/block.cpp index 40fef89..f6a3414 100644 --- a/Src/block.cpp +++ b/Src/block.cpp @@ -18,19 +18,28 @@ int Block::RangeStart(const int axis, const BoundarySurface surface) { int start{RangeStart(axis)}; if (axis == xaxis) { if (surface == BoundarySurface::Right || - surface == BoundarySurface::RightBack || - surface == BoundarySurface::RightBottom || - surface == BoundarySurface::RightFront || - surface == BoundarySurface::RightTop) + surface == BoundarySurface::RightTop || + surface == BoundarySurface::RightBottom +#ifdef OPS_3D + || surface == BoundarySurface::RightFront || + surface == BoundarySurface::RightBack +#endif + ) { start = size.at(axis) - 1; + } } if (axis == yaxis) { if (surface == BoundarySurface::Top || - surface == BoundarySurface::TopBack || - surface == BoundarySurface::TopFront || surface == BoundarySurface::LeftTop || - surface == BoundarySurface::RightTop) + surface == BoundarySurface::RightTop +#ifdef OP_3D + || surface == BoundarySurface::TopBack || + surface == BoundarySurface::TopFront +#endif + + ) { start = size.at(axis) - 1; + } } #ifdef OPS_3D if (axis == zaxis) { @@ -38,8 +47,9 @@ int Block::RangeStart(const int axis, const BoundarySurface surface) { surface == BoundarySurface::LeftFront || surface == BoundarySurface::RightFront || surface == BoundarySurface::TopFront || - surface == BoundarySurface::BottomFront) + surface == BoundarySurface::BottomFront) { start = size.at(axis) - 1; + } } #endif return start; @@ -49,19 +59,28 @@ int Block::RangeEnd(const int axis, const BoundarySurface surface) { int end{RangeEnd(axis)}; if (axis == xaxis) { if (surface == BoundarySurface::Left || - surface == BoundarySurface::LeftBack || surface == BoundarySurface::LeftBottom || - surface == BoundarySurface::LeftFront || - surface == BoundarySurface::LeftTop) + surface == BoundarySurface::LeftTop +#ifdef OPS_3D + || surface == BoundarySurface::LeftFront || + surface == BoundarySurface::LeftBack +#endif + ) { end = 1; + } } if (axis == yaxis) { if (surface == BoundarySurface::Bottom || - surface == BoundarySurface::BottomBack || - surface == BoundarySurface::BottomFront || surface == BoundarySurface::LeftBottom || - surface == BoundarySurface::RightBottom) + surface == BoundarySurface::RightBottom +#ifdef OPS_3D + + || surface == BoundarySurface::BottomBack || + surface == BoundarySurface::BottomFront +#endif + ) { end = 1; + } } #ifdef OPS_3D if (axis == zaxis) { @@ -69,8 +88,9 @@ int Block::RangeEnd(const int axis, const BoundarySurface surface) { surface == BoundarySurface::LeftBack || surface == BoundarySurface::RightBack || surface == BoundarySurface::TopBack || - surface == BoundarySurface::BottomBack) + surface == BoundarySurface::BottomBack) { end = 1; + } } #endif return end; @@ -106,14 +126,14 @@ Block::Block(const int blockId, const std::string& blockName, bulkRange.at(5) = size.at(2) - 1; #endif for (const auto surface : AllBoundarySurface) { - boundarySurfaceRange[surface] = {RangeStart(xaxis, surface), - RangeEnd(xaxis, surface), - RangeStart(yaxis, surface), - RangeEnd(yaxis, surface) + boundarySurfaceRange[surface] = {RangeStart(xaxis, surface), + RangeEnd(xaxis, surface), + RangeStart(yaxis, surface), + RangeEnd(yaxis, surface) #ifdef OPS_3D - , - RangeStart(zaxis, surface), - RangeEnd(zaxis, surface) + , + RangeStart(zaxis, surface), + RangeEnd(zaxis, surface) #endif }; } diff --git a/Src/boundary.cpp b/Src/boundary.cpp index 56b5531..a967220 100644 --- a/Src/boundary.cpp +++ b/Src/boundary.cpp @@ -384,4 +384,15 @@ void ImplementBoundary3D() { boundary.boundarySurface); } } -#endif //OPS_3D +#endif + +#ifdef OPS_2D +void ImplementBoundary() { + for (const auto& boundary : BlockBoundaries()) { + const Block& block{g_Block().at(boundary.blockIndex)}; + TreatBlockBoundary(block, boundary.componentID, + boundary.givenVars.data(), boundary.boundaryScheme, + boundary.boundarySurface); + } +} +#endif diff --git a/Src/boundary.h b/Src/boundary.h index 6725ade..9881388 100644 --- a/Src/boundary.h +++ b/Src/boundary.h @@ -114,5 +114,13 @@ void TreatBlockBoundary3D(const Block& block, const int componentID, const BoundaryScheme boundaryScheme, const BoundarySurface boundarySurface); void ImplementBoundary3D(); -#endif // OPS_3D +#endif + +#ifdef OPS_2D +void TreatBlockBoundary(const Block& block, const int componentID, + const Real* givenVars, + const BoundaryScheme boundaryScheme, + const BoundarySurface boundarySurface); +void ImplementBoundary(); +#endif #endif // BOUNDARY_H diff --git a/Src/boundary_kernel.inc b/Src/boundary_kernel.inc index e874b08..24c48cd 100644 --- a/Src/boundary_kernel.inc +++ b/Src/boundary_kernel.inc @@ -51,963 +51,518 @@ #include "boundary_host_device.h" #ifdef OPS_2D// Boundary conditions for two-dimensional problems - -void KerCutCellZeroFlux(const ACC &nodeType, - const ACC &geometryProperty, ACC &f) { -#ifdef OPS_2D - VertexGeometryTypes vg = (VertexGeometryTypes)geometryProperty(0, 0); - switch (vg) { - case VG_IP: - for (int xiIndex = 0; xiIndex < NUMXI; xiIndex++) { - f(xiIndex, 0, 0) = f(xiIndex, 1, 0); - } - break; - case VG_IM: - for (int xiIndex = 0; xiIndex < NUMXI; xiIndex++) { - f(xiIndex, 0, 0) = f(xiIndex, -1, 0); - } - break; - case VG_JP: - for (int xiIndex = 0; xiIndex < NUMXI; xiIndex++) { - f(xiIndex, 0, 0) = f(xiIndex, 0, 1); - } - break; - case VG_JM: - for (int xiIndex = 0; xiIndex < NUMXI; xiIndex++) { - f(xiIndex, 0, 0) = f(xiIndex, 0, -1); - } - break; - - case VG_IPJP_I: - // VG_IP - if (vt == nodeType(0, 1)) { - for (int xiIndex = 0; xiIndex < NUMXI; xiIndex++) { - f(xiIndex, 0, 0) = f(xiIndex, 1, 0); - } - } - // VG_JP - if (vt == nodeType(1, 0)) { - for (int xiIndex = 0; xiIndex < NUMXI; xiIndex++) { - f(xiIndex, 0, 0) = f(xiIndex, 0, 1); - } - } - break; - case VG_IPJM_I: - // VG_IP - if (vt == nodeType(0, -1)) { - for (int xiIndex = 0; xiIndex < NUMXI; xiIndex++) { - f(xiIndex, 0, 0) = f(xiIndex, 1, 0); - } - } - // VG_JM - if (vt == nodeType(1, 0)) { - for (int xiIndex = 0; xiIndex < NUMXI; xiIndex++) { - f(xiIndex, 0, 0) = f(xiIndex, 0, -1); - } - } - break; - case VG_IMJP_I: - // VG_IM - if (vt == nodeType(0, 1)) { - for (int xiIndex = 0; xiIndex < NUMXI; xiIndex++) { - f(xiIndex, 0, 0) = f(xiIndex, -1, 0); - } - } - // VG_JP - if (vt == nodeType(-1, 0)) { - for (int xiIndex = 0; xiIndex < NUMXI; xiIndex++) { - f(xiIndex, 0, 0) = f(xiIndex, 0, 1); - } - } - break; - case VG_IMJM_I: - // VG_IM - if (vt == nodeType(0, -1)) { - for (int xiIndex = 0; xiIndex < NUMXI; xiIndex++) { - f(xiIndex, 0, 0) = f(xiIndex, -1, 0); - } - } - // VG_JM - if (vt == nodeType(-1, 0)) { - for (int xiIndex = 0; xiIndex < NUMXI; xiIndex++) { - f(xiIndex, 0, 0) = f(xiIndex, 0, -1); - } - } - break; - default: - break; - } - -#endif OPS_2D -} - // Need to be modified for ImmersedSolid -void KerCutCellEmbeddedBoundary(const ACC &nodeType, - const ACC &geometryProperty, - ACC &f) { -#ifdef OPS_2D - /*! - For the bounce back scheme,We consider zero velocity boundary first. - To make sure the velocity at boundary is zero, the implementation - is lattice specific. - */ - VertexType vt = (VertexType)nodeType(0, 0); - VertexGeometryTypes vg = (VertexGeometryTypes)geometryProperty(0, 0); - // TODO to be changed for embeded boundary - if (vt == VertexType::ImmersedBoundary) { - switch (vt) { - case Vertex_EQMDiffuseRefl: { - Real u{0}; - Real v{0}; - const Real sqrt3 = sqrt(3); - switch (vg) { - case VG_IP: { - const Real f3 = f(3, 0, 0); - const Real f7 = f(7, 0, 0); - const Real f6 = f(6, 0, 0); - const Real rhow = - 6 * (f3 + f6 + f7) / (u * u - sqrt3 * u + 1); - f(5, 0, 0) = f7 + rhow * (u + v) / (6 * sqrt3); - f(1, 0, 0) = f3 + 2 * rhow * u / (3 * sqrt3); - f(8, 0, 0) = f6 + rhow * (u - v) / (6 * sqrt3); - f(0, 0, 0) = 2 * rhow * (2 - u * u - v * v) / 9; - f(2, 0, 0) = - -(u * u - 2 * (1 + sqrt3 * v + v * v) * rhow) / 18; - f(4, 0, 0) = - -((-2 + u * u + 2 * sqrt3 * v - 2 * v * v) * rhow) / - 18; - } break; - case VG_IM: { - const Real f5 = f(5, 0, 0); - const Real f1 = f(1, 0, 0); - const Real f8 = f(8, 0, 0); - const Real rhow = - 6 * (f1 + f5 + f8) / (u * u + sqrt3 * u + 1); - f(7, 0, 0) = f5 - rhow * (u + v) / (6 * sqrt3); - f(3, 0, 0) = f1 - 2 * rhow * u / (3 * sqrt3); - f(6, 0, 0) = f8 + rhow * (v - u) / (6 * sqrt3); - f(0, 0, 0) = 2 * rhow * (2 - u * u - v * v) / 9; - f(2, 0, 0) = - -(u * u - 2 * (1 + sqrt3 * v + v * v) * rhow) / 18; - f(4, 0, 0) = - -((-2 + u * u + 2 * sqrt3 * v - 2 * v * v) * rhow) / - 18; - } break; - case VG_JP: { - const Real f4 = f(4, 0, 0); - const Real f8 = f(8, 0, 0); - const Real f7 = f(7, 0, 0); - const Real rhow = - 6 * (f4 + f8 + f7) / (v * v - sqrt3 * v + 1); - f(2, 0, 0) = f4 + 2 * rhow * v / (3 * sqrt3); - f(6, 0, 0) = f8 + rhow * (v - u) / (6 * sqrt3); - f(5, 0, 0) = f7 + rhow * (u + v) / (6 * sqrt3); - f(1, 0, 0) = - ((2 + 2 * sqrt3 * u + 2 * u * u - v * v) * rhow) / - 18; - f(3, 0, 0) = - -((-2 + 2 * sqrt3 * u - 2 * u * u + v * v) * rhow) / - 18; - f(0, 0, 0) = 2 * rhow * (2 - u * u - v * v) / 9; - } break; - case VG_JM: { - const Real f2 = f(2, 0, 0); - const Real f5 = f(5, 0, 0); - const Real f6 = f(6, 0, 0); - const Real rhow = - 6 * (f2 + f5 + f6) / (v * v + sqrt3 * v + 1); - f(4, 0, 0) = f2 - 2 * rhow * v / (3 * sqrt3); - f(8, 0, 0) = f6 + rhow * (u - v) / (6 * sqrt3); - f(7, 0, 0) = f5 - rhow * (u + v) / (6 * sqrt3); - f(1, 0, 0) = - ((2 + 2 * sqrt3 * u + 2 * u * u - v * v) * rhow) / - 18; - f(3, 0, 0) = - -((-2 + 2 * sqrt3 * u - 2 * u * u + v * v) * rhow) / - 18; - f(0, 0, 0) = 2 * rhow * (2 - u * u - v * v) / 9; - } break; - case VG_IPJP_I: { - const Real f3 = f(3, 0, 0); - const Real f7 = f(7, 0, 0); - const Real f4 = f(4, 0, 0); - const Real rhow = - (-36 * (f3 + f4 + f7)) / - (-9 + 5 * sqrt3 * u - 3 * u * u + 5 * sqrt3 * v - - 3 * u * v - 3 * v * v); - f(1, 0, 0) = f3 + 2 * rhow * u / (3 * sqrt3); - f(5, 0, 0) = f7 + rhow * (u + v) / (6 * sqrt3); - f(2, 0, 0) = f4 + 2 * rhow * v / (3 * sqrt3); - f(0, 0, 0) = 2 * rhow * (2 - u * u - v * v) / 9; - f(6, 0, 0) = (1 + u * u + sqrt3 * v + v * v - - u * (sqrt3 + 3 * v)) * - rhow / 36; - f(8, 0, 0) = (1 + u * u + u * (sqrt3 - 3 * v) - - sqrt3 * v + v * v) * - rhow / 36; - } break; - case VG_IPJM_I: { - const Real f2 = f(2, 0, 0); - const Real f3 = f(3, 0, 0); - const Real f6 = f(6, 0, 0); - const Real rhow = - (-36 * (f2 + f3 + f6)) / - (-9 + 5 * sqrt3 * u - 3 * u * u - 5 * sqrt3 * v + - 3 * u * v - 3 * v * v); - f(8, 0, 0) = f6 + rhow * (u - v) / (6 * sqrt3); - f(1, 0, 0) = f3 + 2 * rhow * u / (3 * sqrt3); - f(4, 0, 0) = f2 - 2 * rhow * v / (3 * sqrt3); - f(0, 0, 0) = 2 * rhow * (2 - u * u - v * v) / 9; - f(5, 0, 0) = (1 + u * u + sqrt3 * v + v * v + - u * (sqrt3 + 3 * v)) * - rhow / 36; - f(7, 0, 0) = (1 - sqrt3 * u + u * u - sqrt3 * v + - 3 * u * v + v * v) * - rhow / 36; - } break; - case VG_IMJP_I: { - const Real f1 = f(1, 0, 0); - const Real f4 = f(4, 0, 0); - const Real f8 = f(8, 0, 0); - const Real rhow = - (36 * (f1 + f4 + f8)) / - (9 + 5 * sqrt3 * u + 3 * u * u - 5 * sqrt3 * v - - 3 * u * v + 3 * v * v); - f(6, 0, 0) = f8 + rhow * (v - u) / (6 * sqrt3); - f(3, 0, 0) = f1 - 2 * rhow * u / (3 * sqrt3); - f(2, 0, 0) = f4 + 2 * rhow * v / (3 * sqrt3); - f(0, 0, 0) = 2 * rhow * (2 - u * u - v * v) / 9; - f(5, 0, 0) = (1 + u * u + sqrt3 * v + v * v + - u * (sqrt3 + 3 * v)) * - rhow / 36; - f(7, 0, 0) = (1 - sqrt3 * u + u * u - sqrt3 * v + - 3 * u * v + v * v) * - rhow / 36; - } break; - case VG_IMJM_I: { - const Real f1 = f(1, 0, 0); - const Real f5 = f(5, 0, 0); - const Real f2 = f(2, 0, 0); - const Real rhow = - (36 * (f1 + f2 + f5)) / - (9 + 5 * sqrt3 * u + 3 * u * u + 5 * sqrt3 * v + - 3 * u * v + 3 * v * v); - f(3, 0, 0) = f1 - 2 * rhow * u / (3 * sqrt3); - f(4, 0, 0) = f2 - 2 * rhow * v / (3 * sqrt3); - f(7, 0, 0) = f5 - rhow * (u + v) / (6 * sqrt3); - f(0, 0, 0) = 2 * rhow * (2 - u * u - v * v) / 9; - f(6, 0, 0) = (1 + u * u + sqrt3 * v + v * v - - u * (sqrt3 + 3 * v)) * - rhow / 36; - f(8, 0, 0) = (1 + u * u + u * (sqrt3 - 3 * v) - - sqrt3 * v + v * v) * - rhow / 36; - } break; - case VG_IPJP_O: { // outter corner point - const Real f3 = f(3, 0, 0); - const Real f7 = f(7, 0, 0); - const Real f4 = f(4, 0, 0); - const Real rhow = - (-36 * (f3 + f4 + f7)) / - (-9 + 5 * sqrt3 * u - 3 * u * u + 5 * sqrt3 * v - - 3 * u * v - 3 * v * v); - f(1, 0, 0) = f3 + 2 * rhow * u / (3 * sqrt3); - f(5, 0, 0) = f7 + rhow * (u + v) / (6 * sqrt3); - f(2, 0, 0) = f4 + 2 * rhow * v / (3 * sqrt3); - f(0, 0, 0) = 2 * rhow * (2 - u * u - v * v) / 9; - f(6, 0, 0) = (1 + u * u + sqrt3 * v + v * v - - u * (sqrt3 + 3 * v)) * - rhow / 36; - f(8, 0, 0) = (1 + u * u + u * (sqrt3 - 3 * v) - - sqrt3 * v + v * v) * - rhow / 36; - } break; - case VG_IPJM_O: { // outter corner point - const Real f2 = f(2, 0, 0); - const Real f3 = f(3, 0, 0); - const Real f6 = f(6, 0, 0); - const Real rhow = - (-36 * (f2 + f3 + f6)) / - (-9 + 5 * sqrt3 * u - 3 * u * u - 5 * sqrt3 * v + - 3 * u * v - 3 * v * v); - f(8, 0, 0) = f6 + rhow * (u - v) / (6 * sqrt3); - f(1, 0, 0) = f3 + 2 * rhow * u / (3 * sqrt3); - f(4, 0, 0) = f2 - 2 * rhow * v / (3 * sqrt3); - f(0, 0, 0) = 2 * rhow * (2 - u * u - v * v) / 9; - f(5, 0, 0) = (1 + u * u + sqrt3 * v + v * v + - u * (sqrt3 + 3 * v)) * - rhow / 36; - f(7, 0, 0) = (1 - sqrt3 * u + u * u - sqrt3 * v + - 3 * u * v + v * v) * - rhow / 36; - } break; - case VG_IMJP_O: { // outter corner point - const Real f1 = f(1, 0, 0); - const Real f4 = f(4, 0, 0); - const Real f8 = f(8, 0, 0); - const Real rhow = - (36 * (f1 + f4 + f8)) / - (9 + 5 * sqrt3 * u + 3 * u * u - 5 * sqrt3 * v - - 3 * u * v + 3 * v * v); - f(6, 0, 0) = f8 + rhow * (v - u) / (6 * sqrt3); - f(3, 0, 0) = f1 - 2 * rhow * u / (3 * sqrt3); - f(2, 0, 0) = f4 + 2 * rhow * v / (3 * sqrt3); - f(0, 0, 0) = 2 * rhow * (2 - u * u - v * v) / 9; - f(5, 0, 0) = (1 + u * u + sqrt3 * v + v * v + - u * (sqrt3 + 3 * v)) * - rhow / 36; - f(7, 0, 0) = (1 - sqrt3 * u + u * u - sqrt3 * v + - 3 * u * v + v * v) * - rhow / 36; - } break; - case VG_IMJM_O: { // outter corner point - const Real f1 = f(1, 0, 0); - const Real f5 = f(5, 0, 0); - const Real f2 = f(2, 0, 0); - const Real rhow = - (36 * (f1 + f2 + f5)) / - (9 + 5 * sqrt3 * u + 3 * u * u + 5 * sqrt3 * v + - 3 * u * v + 3 * v * v); - f(3, 0, 0) = f1 - 2 * rhow * u / (3 * sqrt3); - f(4, 0, 0) = f2 - 2 * rhow * v / (3 * sqrt3); - f(7, 0, 0) = f5 - rhow * (u + v) / (6 * sqrt3); - f(0, 0, 0) = 2 * rhow * (2 - u * u - v * v) / 9; - f(6, 0, 0) = (1 + u * u + sqrt3 * v + v * v - - u * (sqrt3 + 3 * v)) * - rhow / 36; - f(8, 0, 0) = (1 + u * u + u * (sqrt3 - 3 * v) - - sqrt3 * v + v * v) * - rhow / 36; - } break; - default: - break; - } // vg - } // case Vertex_EQMDiffRefl - break; - case Vertex_KineticDiffuseWall: { - Real u = 0; - Real v = 0; // means non-moving wall - Real wallNormalVector[]{0, 0}; - const Real sqrt2Inverse = 1 / sqrt(2); - switch (vg) { - case VG_IP: { - wallNormalVector[0] = 1; - wallNormalVector[1] = 0; - } break; - case VG_IM: { - wallNormalVector[0] = -1; - wallNormalVector[1] = 0; - } break; - case VG_JP: { - wallNormalVector[0] = 0; - wallNormalVector[1] = 1; - } break; - case VG_JM: { - wallNormalVector[0] = 0; - wallNormalVector[1] = -1; - } break; - case VG_IPJP_I: { - wallNormalVector[0] = sqrt2Inverse; - wallNormalVector[1] = sqrt2Inverse; - } break; - case VG_IPJM_I: { - wallNormalVector[0] = sqrt2Inverse; - wallNormalVector[1] = -sqrt2Inverse; - } break; - case VG_IMJP_I: { - wallNormalVector[0] = -sqrt2Inverse; - wallNormalVector[1] = sqrt2Inverse; - } break; - case VG_IMJM_I: { - wallNormalVector[0] = -sqrt2Inverse; - wallNormalVector[1] = -sqrt2Inverse; - } break; - case VG_IPJP_O: { - wallNormalVector[0] = sqrt2Inverse; - wallNormalVector[1] = sqrt2Inverse; - } break; - case VG_IPJM_O: { - wallNormalVector[0] = sqrt2Inverse; - wallNormalVector[1] = -sqrt2Inverse; - } break; - case VG_IMJP_O: { - wallNormalVector[0] = -sqrt2Inverse; - wallNormalVector[1] = sqrt2Inverse; - } break; - case VG_IMJM_O: { - wallNormalVector[0] = -sqrt2Inverse; - wallNormalVector[1] = -sqrt2Inverse; - } break; - - default: - break; - } - Real outFlux = 0; // flow into wall - Real inFlux = 0; // flow into fluid bulk - for (int xiIndex = 0; xiIndex < NUMXI; xiIndex++) { - const Real cx = XI[xiIndex * LATTDIM]; - const Real cy = XI[xiIndex * LATTDIM + 1]; - Real cDotNormal = (CS * cx - u) * wallNormalVector[0] + - (CS * cy - v) * wallNormalVector[1]; - if (cDotNormal < 0) { - outFlux += (-cDotNormal * f(xiIndex, 0, 0)); - } - if (cDotNormal > 0) { - Real cu = (CS * cx * u + CS * cy * v); - inFlux += - (cDotNormal * WEIGHTS[xiIndex] * - (1 + cu + - 0.5 * (cu * cu - (u * u + v * v)))); // i.e., the - // equilibrium - } - } - Real rho = outFlux / inFlux; - for (int xiIndex = 0; xiIndex < NUMXI; xiIndex++) { - const Real cx = XI[xiIndex * LATTDIM]; - const Real cy = (int)XI[xiIndex * LATTDIM + 1]; - Real cDotNormal = (CS * cx - u) * wallNormalVector[0] + - (CS * cy - v) * wallNormalVector[1]; - if (cDotNormal >= 0) { - Real cu = (CS * cx * u + CS * cy * v); - f(xiIndex, 0, 0) = - (rho * WEIGHTS[xiIndex] * - (1 + cu + 0.5 * (cu * cu - (u * u + v * v)))); - } - } - } // case Vertex_KineticDiffuseWall - break; - default: -#ifdef debug - ops_printf("%s\n", - "Warning: KerCutCellImmersedBoundary: there seems a " - "boundary condition that has note been implemented"); -#endif - break; - } - } -#endif OPS_2D -} - -void KerCutCellExtrapolPressure1ST(const Real *givenBoundaryVars, - const ACC &nodeType, - const ACC &geometryProperty, - ACC &f) { -#ifdef OPS_2D - - VertexGeometryTypes vg = (VertexGeometryTypes)geometryProperty(0, 0); - Real rhoGiven = givenBoundaryVars[0]; - Real rho = 0; - for (int xiIndex = 0; xiIndex < NUMXI; xiIndex++) { - const int cx = (int)XI[xiIndex * LATTDIM]; - const int cy = (int)XI[xiIndex * LATTDIM + 1]; - switch (vg) { - case VG_IP: { - if (cx > 0) { - f(xiIndex, 0, 0) = f(xiIndex, 1, 0); - } - } break; - case VG_IM: { - if (cx < 0) { - f(xiIndex, 0, 0) = f(xiIndex, -1, 0); - } - } break; - case VG_JP: { - if (cy > 0) { - f(xiIndex, 0, 0) = f(xiIndex, 0, 1); - } - } break; - case VG_JM: { - if (cy < 0) { - f(xiIndex, 0, 0) = f(xiIndex, 0, -1); - } - - } break; - - case VG_IPJP_I: { - // VG_IP - if (vt == nodeType(0, 1)) { - if ((cx > 0 && cy >= 0) || (cx >= 0 && cy > 0)) { - f(xiIndex, 0, 0) = f(xiIndex, 1, 0); - } - } - // VG_JP - if (vt == nodeType(1, 0)) { - if ((cx > 0 && cy >= 0) || (cx >= 0 && cy > 0)) { - f(xiIndex, 0, 0) = f(xiIndex, 0, 1); - } - } - } break; - case VG_IPJM_I: { - // VG_IP - if (vt == nodeType(0, -1)) { - if ((cx > 0 && cy <= 0) || (cx >= 0 && cy < 0)) { - f(xiIndex, 0, 0) = f(xiIndex, 1, 0); - } - } - // VG_JM - if (vt == nodeType(1, 0)) { - if ((cx > 0 && cy <= 0) || (cx >= 0 && cy < 0)) { - f(xiIndex, 0, 0) = f(xiIndex, 0, -1); - } - } - } break; - - case VG_IMJP_I: { - // VG_IM - if (vt == nodeType(0, 1)) { - if ((cx < 0 && cy >= 0) || (cx <= 0 && cy > 0)) { - f(xiIndex, 0, 0) = f(xiIndex, -1, 0); - } - } - // VG_JP - if (vt == nodeType(-1, 0)) { - if ((cx < 0 && cy >= 0) || (cx <= 0 && cy > 0)) { - f(xiIndex, 0, 0) = f(xiIndex, 0, 1); - } - } - } break; - case VG_IMJM_I: { - // VG_IM - if (vt == nodeType(0, -1)) { - if ((cx < 0 && cy <= 0) || (cx <= 0 && cy < 0)) { - f(xiIndex, 0, 0) = f(xiIndex, -1, 0); - } - } - // VG_JM - if (vt == nodeType(-1, 0)) { - if ((cx < 0 && cy <= 0) || (cx <= 0 && cy < 0)) { - f(xiIndex, 0, 0) = f(xiIndex, 0, -1); - } - } - } break; - default: - break; - } - rho += f(xiIndex, 0, 0); - } - Real ratio = rhoGiven / rho; - for (int xiIndex = 0; xiIndex < NUMXI; xiIndex++) { - f(xiIndex, 0, 0) *= ratio; - } +// void KerCutCellEmbeddedBoundary(const ACC &nodeType, +// const ACC &geometryProperty, +// ACC &f) { +// #ifdef OPS_2D +// /*! +// For the bounce back scheme,We consider zero velocity boundary first. +// To make sure the velocity at boundary is zero, the implementation +// is lattice specific. +// */ +// VertexType vt = (VertexType)nodeType(0, 0); +// VertexGeometryTypes vg = (VertexGeometryTypes)geometryProperty(0, 0); +// // TODO to be changed for embeded boundary +// if (vt == VertexType::ImmersedBoundary) { +// switch (vt) { +// case Vertex_EQMDiffuseRefl: { +// Real u{0}; +// Real v{0}; +// const Real sqrt3 = sqrt(3); +// switch (vg) { +// case VG_IP: { +// const Real f3 = f(3, 0, 0); +// const Real f7 = f(7, 0, 0); +// const Real f6 = f(6, 0, 0); +// const Real rhow = +// 6 * (f3 + f6 + f7) / (u * u - sqrt3 * u + 1); +// f(5, 0, 0) = f7 + rhow * (u + v) / (6 * sqrt3); +// f(1, 0, 0) = f3 + 2 * rhow * u / (3 * sqrt3); +// f(8, 0, 0) = f6 + rhow * (u - v) / (6 * sqrt3); +// f(0, 0, 0) = 2 * rhow * (2 - u * u - v * v) / 9; +// f(2, 0, 0) = +// -(u * u - 2 * (1 + sqrt3 * v + v * v) * rhow) / 18; +// f(4, 0, 0) = +// -((-2 + u * u + 2 * sqrt3 * v - 2 * v * v) * rhow) / +// 18; +// } break; +// case VG_IM: { +// const Real f5 = f(5, 0, 0); +// const Real f1 = f(1, 0, 0); +// const Real f8 = f(8, 0, 0); +// const Real rhow = +// 6 * (f1 + f5 + f8) / (u * u + sqrt3 * u + 1); +// f(7, 0, 0) = f5 - rhow * (u + v) / (6 * sqrt3); +// f(3, 0, 0) = f1 - 2 * rhow * u / (3 * sqrt3); +// f(6, 0, 0) = f8 + rhow * (v - u) / (6 * sqrt3); +// f(0, 0, 0) = 2 * rhow * (2 - u * u - v * v) / 9; +// f(2, 0, 0) = +// -(u * u - 2 * (1 + sqrt3 * v + v * v) * rhow) / 18; +// f(4, 0, 0) = +// -((-2 + u * u + 2 * sqrt3 * v - 2 * v * v) * rhow) / +// 18; +// } break; +// case VG_JP: { +// const Real f4 = f(4, 0, 0); +// const Real f8 = f(8, 0, 0); +// const Real f7 = f(7, 0, 0); +// const Real rhow = +// 6 * (f4 + f8 + f7) / (v * v - sqrt3 * v + 1); +// f(2, 0, 0) = f4 + 2 * rhow * v / (3 * sqrt3); +// f(6, 0, 0) = f8 + rhow * (v - u) / (6 * sqrt3); +// f(5, 0, 0) = f7 + rhow * (u + v) / (6 * sqrt3); +// f(1, 0, 0) = +// ((2 + 2 * sqrt3 * u + 2 * u * u - v * v) * rhow) / +// 18; +// f(3, 0, 0) = +// -((-2 + 2 * sqrt3 * u - 2 * u * u + v * v) * rhow) / +// 18; +// f(0, 0, 0) = 2 * rhow * (2 - u * u - v * v) / 9; +// } break; +// case VG_JM: { +// const Real f2 = f(2, 0, 0); +// const Real f5 = f(5, 0, 0); +// const Real f6 = f(6, 0, 0); +// const Real rhow = +// 6 * (f2 + f5 + f6) / (v * v + sqrt3 * v + 1); +// f(4, 0, 0) = f2 - 2 * rhow * v / (3 * sqrt3); +// f(8, 0, 0) = f6 + rhow * (u - v) / (6 * sqrt3); +// f(7, 0, 0) = f5 - rhow * (u + v) / (6 * sqrt3); +// f(1, 0, 0) = +// ((2 + 2 * sqrt3 * u + 2 * u * u - v * v) * rhow) / +// 18; +// f(3, 0, 0) = +// -((-2 + 2 * sqrt3 * u - 2 * u * u + v * v) * rhow) / +// 18; +// f(0, 0, 0) = 2 * rhow * (2 - u * u - v * v) / 9; +// } break; +// case VG_IPJP_I: { +// const Real f3 = f(3, 0, 0); +// const Real f7 = f(7, 0, 0); +// const Real f4 = f(4, 0, 0); +// const Real rhow = +// (-36 * (f3 + f4 + f7)) / +// (-9 + 5 * sqrt3 * u - 3 * u * u + 5 * sqrt3 * v - +// 3 * u * v - 3 * v * v); +// f(1, 0, 0) = f3 + 2 * rhow * u / (3 * sqrt3); +// f(5, 0, 0) = f7 + rhow * (u + v) / (6 * sqrt3); +// f(2, 0, 0) = f4 + 2 * rhow * v / (3 * sqrt3); +// f(0, 0, 0) = 2 * rhow * (2 - u * u - v * v) / 9; +// f(6, 0, 0) = (1 + u * u + sqrt3 * v + v * v - +// u * (sqrt3 + 3 * v)) * +// rhow / 36; +// f(8, 0, 0) = (1 + u * u + u * (sqrt3 - 3 * v) - +// sqrt3 * v + v * v) * +// rhow / 36; +// } break; +// case VG_IPJM_I: { +// const Real f2 = f(2, 0, 0); +// const Real f3 = f(3, 0, 0); +// const Real f6 = f(6, 0, 0); +// const Real rhow = +// (-36 * (f2 + f3 + f6)) / +// (-9 + 5 * sqrt3 * u - 3 * u * u - 5 * sqrt3 * v + +// 3 * u * v - 3 * v * v); +// f(8, 0, 0) = f6 + rhow * (u - v) / (6 * sqrt3); +// f(1, 0, 0) = f3 + 2 * rhow * u / (3 * sqrt3); +// f(4, 0, 0) = f2 - 2 * rhow * v / (3 * sqrt3); +// f(0, 0, 0) = 2 * rhow * (2 - u * u - v * v) / 9; +// f(5, 0, 0) = (1 + u * u + sqrt3 * v + v * v + +// u * (sqrt3 + 3 * v)) * +// rhow / 36; +// f(7, 0, 0) = (1 - sqrt3 * u + u * u - sqrt3 * v + +// 3 * u * v + v * v) * +// rhow / 36; +// } break; +// case VG_IMJP_I: { +// const Real f1 = f(1, 0, 0); +// const Real f4 = f(4, 0, 0); +// const Real f8 = f(8, 0, 0); +// const Real rhow = +// (36 * (f1 + f4 + f8)) / +// (9 + 5 * sqrt3 * u + 3 * u * u - 5 * sqrt3 * v - +// 3 * u * v + 3 * v * v); +// f(6, 0, 0) = f8 + rhow * (v - u) / (6 * sqrt3); +// f(3, 0, 0) = f1 - 2 * rhow * u / (3 * sqrt3); +// f(2, 0, 0) = f4 + 2 * rhow * v / (3 * sqrt3); +// f(0, 0, 0) = 2 * rhow * (2 - u * u - v * v) / 9; +// f(5, 0, 0) = (1 + u * u + sqrt3 * v + v * v + +// u * (sqrt3 + 3 * v)) * +// rhow / 36; +// f(7, 0, 0) = (1 - sqrt3 * u + u * u - sqrt3 * v + +// 3 * u * v + v * v) * +// rhow / 36; +// } break; +// case VG_IMJM_I: { +// const Real f1 = f(1, 0, 0); +// const Real f5 = f(5, 0, 0); +// const Real f2 = f(2, 0, 0); +// const Real rhow = +// (36 * (f1 + f2 + f5)) / +// (9 + 5 * sqrt3 * u + 3 * u * u + 5 * sqrt3 * v + +// 3 * u * v + 3 * v * v); +// f(3, 0, 0) = f1 - 2 * rhow * u / (3 * sqrt3); +// f(4, 0, 0) = f2 - 2 * rhow * v / (3 * sqrt3); +// f(7, 0, 0) = f5 - rhow * (u + v) / (6 * sqrt3); +// f(0, 0, 0) = 2 * rhow * (2 - u * u - v * v) / 9; +// f(6, 0, 0) = (1 + u * u + sqrt3 * v + v * v - +// u * (sqrt3 + 3 * v)) * +// rhow / 36; +// f(8, 0, 0) = (1 + u * u + u * (sqrt3 - 3 * v) - +// sqrt3 * v + v * v) * +// rhow / 36; +// } break; +// case VG_IPJP_O: { // outter corner point +// const Real f3 = f(3, 0, 0); +// const Real f7 = f(7, 0, 0); +// const Real f4 = f(4, 0, 0); +// const Real rhow = +// (-36 * (f3 + f4 + f7)) / +// (-9 + 5 * sqrt3 * u - 3 * u * u + 5 * sqrt3 * v - +// 3 * u * v - 3 * v * v); +// f(1, 0, 0) = f3 + 2 * rhow * u / (3 * sqrt3); +// f(5, 0, 0) = f7 + rhow * (u + v) / (6 * sqrt3); +// f(2, 0, 0) = f4 + 2 * rhow * v / (3 * sqrt3); +// f(0, 0, 0) = 2 * rhow * (2 - u * u - v * v) / 9; +// f(6, 0, 0) = (1 + u * u + sqrt3 * v + v * v - +// u * (sqrt3 + 3 * v)) * +// rhow / 36; +// f(8, 0, 0) = (1 + u * u + u * (sqrt3 - 3 * v) - +// sqrt3 * v + v * v) * +// rhow / 36; +// } break; +// case VG_IPJM_O: { // outter corner point +// const Real f2 = f(2, 0, 0); +// const Real f3 = f(3, 0, 0); +// const Real f6 = f(6, 0, 0); +// const Real rhow = +// (-36 * (f2 + f3 + f6)) / +// (-9 + 5 * sqrt3 * u - 3 * u * u - 5 * sqrt3 * v + +// 3 * u * v - 3 * v * v); +// f(8, 0, 0) = f6 + rhow * (u - v) / (6 * sqrt3); +// f(1, 0, 0) = f3 + 2 * rhow * u / (3 * sqrt3); +// f(4, 0, 0) = f2 - 2 * rhow * v / (3 * sqrt3); +// f(0, 0, 0) = 2 * rhow * (2 - u * u - v * v) / 9; +// f(5, 0, 0) = (1 + u * u + sqrt3 * v + v * v + +// u * (sqrt3 + 3 * v)) * +// rhow / 36; +// f(7, 0, 0) = (1 - sqrt3 * u + u * u - sqrt3 * v + +// 3 * u * v + v * v) * +// rhow / 36; +// } break; +// case VG_IMJP_O: { // outter corner point +// const Real f1 = f(1, 0, 0); +// const Real f4 = f(4, 0, 0); +// const Real f8 = f(8, 0, 0); +// const Real rhow = +// (36 * (f1 + f4 + f8)) / +// (9 + 5 * sqrt3 * u + 3 * u * u - 5 * sqrt3 * v - +// 3 * u * v + 3 * v * v); +// f(6, 0, 0) = f8 + rhow * (v - u) / (6 * sqrt3); +// f(3, 0, 0) = f1 - 2 * rhow * u / (3 * sqrt3); +// f(2, 0, 0) = f4 + 2 * rhow * v / (3 * sqrt3); +// f(0, 0, 0) = 2 * rhow * (2 - u * u - v * v) / 9; +// f(5, 0, 0) = (1 + u * u + sqrt3 * v + v * v + +// u * (sqrt3 + 3 * v)) * +// rhow / 36; +// f(7, 0, 0) = (1 - sqrt3 * u + u * u - sqrt3 * v + +// 3 * u * v + v * v) * +// rhow / 36; +// } break; +// case VG_IMJM_O: { // outter corner point +// const Real f1 = f(1, 0, 0); +// const Real f5 = f(5, 0, 0); +// const Real f2 = f(2, 0, 0); +// const Real rhow = +// (36 * (f1 + f2 + f5)) / +// (9 + 5 * sqrt3 * u + 3 * u * u + 5 * sqrt3 * v + +// 3 * u * v + 3 * v * v); +// f(3, 0, 0) = f1 - 2 * rhow * u / (3 * sqrt3); +// f(4, 0, 0) = f2 - 2 * rhow * v / (3 * sqrt3); +// f(7, 0, 0) = f5 - rhow * (u + v) / (6 * sqrt3); +// f(0, 0, 0) = 2 * rhow * (2 - u * u - v * v) / 9; +// f(6, 0, 0) = (1 + u * u + sqrt3 * v + v * v - +// u * (sqrt3 + 3 * v)) * +// rhow / 36; +// f(8, 0, 0) = (1 + u * u + u * (sqrt3 - 3 * v) - +// sqrt3 * v + v * v) * +// rhow / 36; +// } break; +// default: +// break; +// } // vg +// } // case Vertex_EQMDiffRefl +// break; +// case Vertex_KineticDiffuseWall: { +// Real u = 0; +// Real v = 0; // means non-moving wall +// Real wallNormalVector[]{0, 0}; +// const Real sqrt2Inverse = 1 / sqrt(2); +// switch (vg) { +// case VG_IP: { +// wallNormalVector[0] = 1; +// wallNormalVector[1] = 0; +// } break; +// case VG_IM: { +// wallNormalVector[0] = -1; +// wallNormalVector[1] = 0; +// } break; +// case VG_JP: { +// wallNormalVector[0] = 0; +// wallNormalVector[1] = 1; +// } break; +// case VG_JM: { +// wallNormalVector[0] = 0; +// wallNormalVector[1] = -1; +// } break; +// case VG_IPJP_I: { +// wallNormalVector[0] = sqrt2Inverse; +// wallNormalVector[1] = sqrt2Inverse; +// } break; +// case VG_IPJM_I: { +// wallNormalVector[0] = sqrt2Inverse; +// wallNormalVector[1] = -sqrt2Inverse; +// } break; +// case VG_IMJP_I: { +// wallNormalVector[0] = -sqrt2Inverse; +// wallNormalVector[1] = sqrt2Inverse; +// } break; +// case VG_IMJM_I: { +// wallNormalVector[0] = -sqrt2Inverse; +// wallNormalVector[1] = -sqrt2Inverse; +// } break; +// case VG_IPJP_O: { +// wallNormalVector[0] = sqrt2Inverse; +// wallNormalVector[1] = sqrt2Inverse; +// } break; +// case VG_IPJM_O: { +// wallNormalVector[0] = sqrt2Inverse; +// wallNormalVector[1] = -sqrt2Inverse; +// } break; +// case VG_IMJP_O: { +// wallNormalVector[0] = -sqrt2Inverse; +// wallNormalVector[1] = sqrt2Inverse; +// } break; +// case VG_IMJM_O: { +// wallNormalVector[0] = -sqrt2Inverse; +// wallNormalVector[1] = -sqrt2Inverse; +// } break; -#endif // OPS_2D -} +// default: +// break; +// } +// Real outFlux = 0; // flow into wall +// Real inFlux = 0; // flow into fluid bulk +// for (int xiIndex = 0; xiIndex < NUMXI; xiIndex++) { +// const Real cx = XI[xiIndex * LATTDIM]; +// const Real cy = XI[xiIndex * LATTDIM + 1]; +// Real cDotNormal = (CS * cx - u) * wallNormalVector[0] + +// (CS * cy - v) * wallNormalVector[1]; +// if (cDotNormal < 0) { +// outFlux += (-cDotNormal * f(xiIndex, 0, 0)); +// } +// if (cDotNormal > 0) { +// Real cu = (CS * cx * u + CS * cy * v); +// inFlux += +// (cDotNormal * WEIGHTS[xiIndex] * +// (1 + cu + +// 0.5 * (cu * cu - (u * u + v * v)))); // i.e., the +// // equilibrium +// } +// } +// Real rho = outFlux / inFlux; +// for (int xiIndex = 0; xiIndex < NUMXI; xiIndex++) { +// const Real cx = XI[xiIndex * LATTDIM]; +// const Real cy = (int)XI[xiIndex * LATTDIM + 1]; +// Real cDotNormal = (CS * cx - u) * wallNormalVector[0] + +// (CS * cy - v) * wallNormalVector[1]; +// if (cDotNormal >= 0) { +// Real cu = (CS * cx * u + CS * cy * v); +// f(xiIndex, 0, 0) = +// (rho * WEIGHTS[xiIndex] * +// (1 + cu + 0.5 * (cu * cu - (u * u + v * v)))); +// } +// } +// } // case Vertex_KineticDiffuseWall +// break; +// default: +// #ifdef debug +// ops_printf("%s\n", +// "Warning: KerCutCellImmersedBoundary: there seems a " +// "boundary condition that has note been implemented"); +// #endif +// break; +// } +// } +// #endif OPS_2D +// } -void KerCutCellExtrapolPressure2ND(const Real *givenBoundaryVars, - const ACC &nodeType, +void KerCutCellExtrapolPressure1ST(ACC &f, const ACC &nodeType, const ACC &geometryProperty, - ACC &f) { + const Real *givenBoundaryVars, + const int *surface, const int *lattIdx) { #ifdef OPS_2D - VertexGeometryTypes vg = (VertexGeometryTypes)geometryProperty(0, 0); + const BoundarySurface boundarySurface{(BoundarySurface)(*surface)}; + VertexGeometryType vg = (VertexGeometryType)geometryProperty(0, 0); Real rhoGiven = givenBoundaryVars[0]; Real rho = 0; - for (int xiIndex = 0; xiIndex < NUMXI; xiIndex++) { - const int cx = (int)XI[xiIndex * LATTDIM]; - const int cy = (int)XI[xiIndex * LATTDIM + 1]; + for (int xiIdx = lattIdx[0]; xiIdx <= lattIdx[1]; xiIdx++) { + Real cx{CS * XI[xiIdx * LATTDIM]}; + Real cy{CS * XI[xiIdx * LATTDIM + 1]}; switch (vg) { case VG_IP: { if (cx > 0) { - f(xiIndex, 0, 0) = 2 * f(xiIndex, 1, 0) - f(xiIndex, 2, 0); + f(xiIdx, 0, 0) = f(xiIdx, 1, 0); } } break; case VG_IM: { if (cx < 0) { - f(xiIndex, 0, 0) = - 2 * f(xiIndex, -1, 0) - f(xiIndex, -2, 0); + f(xiIdx, 0, 0) = f(xiIdx, -1, 0); } } break; case VG_JP: { if (cy > 0) { - f(xiIndex, 0, 0) = 2 * f(xiIndex, 0, 1) - f(xiIndex, 0, 2); + f(xiIdx, 0, 0) = f(xiIdx, 0, 1); } } break; case VG_JM: { if (cy < 0) { - f(xiIndex, 0, 0) = - 2 * f(xiIndex, 0, -1) - f(xiIndex, 0, -2); + f(xiIdx, 0, 0) = f(xiIdx, 0, -1); } } break; - // inner corner point case VG_IPJP_I: { - // VG_IP - if (vt == nodeType(0, 1)) { - if ((cx > 0 && cy >= 0) || (cx >= 0 && cy > 0)) { - f(xiIndex, 0, 0) = - 2 * f(xiIndex, 1, 0) - f(xiIndex, 2, 0); + if ((cx >= 0 && cy > 0) || (cx > 0 && cy == 0)) { + if (boundarySurface == BoundarySurface::Left) { + f(xiIdx, 0, 0) = f(xiIdx, 1, 0); } - } - // VG_JP - if (vt == nodeType(1, 0)) { - if ((cx > 0 && cy >= 0) || (cx >= 0 && cy > 0)) { - f(xiIndex, 0, 0) = - 2 * f(xiIndex, 0, 1) - f(xiIndex, 0, 2); + if (boundarySurface == BoundarySurface::Bottom) { + f(xiIdx, 0, 0) = f(xiIdx, 0, 1); } } } break; case VG_IPJM_I: { - // VG_IP - if (vt == nodeType(0, -1)) { - if ((cx > 0 && cy <= 0) || (cx >= 0 && cy < 0)) { - f(xiIndex, 0, 0) = - 2 * f(xiIndex, 1, 0) - f(xiIndex, 2, 0); + if ((cx >= 0 && cy < 0) || (cx > 0 && cy == 0)) { + if (boundarySurface == BoundarySurface::Left) { + f(xiIdx, 0, 0) = f(xiIdx, 1, 0); } - } - // VG_JM - if (vt == nodeType(1, 0)) { - if ((cx > 0 && cy <= 0) || (cx >= 0 && cy < 0)) { - f(xiIndex, 0, 0) = - 2 * f(xiIndex, 0, -1) - f(xiIndex, 0, -2); + if (boundarySurface == BoundarySurface::Top) { + f(xiIdx, 0, 0) = f(xiIdx, 0, -1); } } } break; case VG_IMJP_I: { - // VG_IM - if (vt == nodeType(0, 1)) { - if ((cx < 0 && cy >= 0) || (cx <= 0 && cy > 0)) { - f(xiIndex, 0, 0) = - 2 * f(xiIndex, -1, 0) - f(xiIndex, -2, 0); + if ((cx <= 0 && cy > 0) || (cx < 0 && cy == 0)) { + if (boundarySurface == BoundarySurface::Right) { + f(xiIdx, 0, 0) = f(xiIdx, -1, 0); } - } - // VG_JP - if (vt == nodeType(-1, 0)) { - if ((cx < 0 && cy >= 0) || (cx <= 0 && cy > 0)) { - f(xiIndex, 0, 0) = - 2 * f(xiIndex, 0, 1) - f(xiIndex, 0, 2); + if (boundarySurface == BoundarySurface::Bottom) { + f(xiIdx, 0, 0) = f(xiIdx, 0, 1); } } } break; case VG_IMJM_I: { - // VG_IM - if (vt == nodeType(0, -1)) { - if ((cx < 0 && cy <= 0) || (cx <= 0 && cy < 0)) { - f(xiIndex, 0, 0) = - 2 * f(xiIndex, -1, 0) - f(xiIndex, -2, 0); + if ((cx <= 0 && cy < 0) || (cx < 0 && cy == 0)) { + if (boundarySurface == BoundarySurface::Right) { + f(xiIdx, 0, 0) = f(xiIdx, -1, 0); } - } - // VG_JM - if (vt == nodeType(-1, 0)) { - if ((cx < 0 && cy <= 0) || (cx <= 0 && cy < 0)) { - f(xiIndex, 0, 0) = - 2 * f(xiIndex, 0, -1) - f(xiIndex, 0, -2); + if (boundarySurface == BoundarySurface::Top) { + f(xiIdx, 0, 0) = f(xiIdx, 0, -1); } } } break; default: break; } - rho += f(xiIndex, 0, 0); + rho += f(xiIdx, 0, 0); } Real ratio = rhoGiven / rho; - for (int xiIndex = 0; xiIndex < NUMXI; xiIndex++) { - f(xiIndex, 0, 0) *= ratio; + for (int xiIdx = lattIdx[0]; xiIdx < lattIdx[1]; xiIdx++) { + f(xiIdx, 0, 0) *= ratio; } - #endif // OPS_2D } +// void KerCutCellBounceBack(const ACC &nodeType, +// const ACC &geometryProperty, ACC &f) { +// #ifdef OPS_2D -// void KerCutCellKinetic(const Real *givenMacroVars, const int *nodeType, -// const int *geometryProperty, Real *f) { -// VertexType vt = (VertexType)nodeType(0, 0); -// if (vt == Vertex_KineticDiffuseWall) { -// VertexGeometryTypes vg = -// (VertexGeometryTypes)geometryProperty(0, 0); -// Real u = givenMacroVars[1]; -// Real v = givenMacroVars[2]; -// Real T = givenMacroVars[3]; -// Real primaryVector[]{0, 0}; -// Real secondVector[]{0, 0}; -// int boundaryType{0}; // 0 normal boundary 1 inner corner 2 outter -// // corner -// switch (vg) { -// case VG_IP: { -// boundaryType = 0; -// primaryVector[0] = 1; -// primaryVector[1] = 0; -// } break; -// case VG_IM: { -// boundaryType = 0; -// primaryVector[0] = -1; -// primaryVector[1] = 0; -// } break; -// case VG_JP: { -// boundaryType = 0; -// primaryVector[0] = 0; -// primaryVector[1] = 1; -// } break; -// case VG_JM: { -// boundaryType = 0; -// primaryVector[0] = 0; -// primaryVector[1] = -1; -// } break; -// case VG_IPJP_I: // inner corner point -// { -// boundaryType = 1; -// primaryVector[0] = 1; -// primaryVector[1] = 0; -// secondVector[0] = 0; -// secondVector[1] = 1; -// } break; -// case VG_IPJM_I: // inner corner point -// { -// boundaryType = 1; -// primaryVector[0] = 1; -// primaryVector[1] = 0; -// secondVector[0] = 0; -// secondVector[1] = -1; -// } break; -// case VG_IMJP_I: // inner corner point -// { -// boundaryType = 1; -// primaryVector[0] = -1; -// primaryVector[1] = 0; -// secondVector[0] = 0; -// secondVector[1] = 1; -// } break; -// case VG_IMJM_I: // inner corner point -// { -// boundaryType = 1; -// primaryVector[0] = -1; -// primaryVector[1] = 0; -// secondVector[0] = 0; -// secondVector[1] = -1; -// } break; -// default: -// break; -// } -// Real outFlux = 0; // flow into wall -// Real inFlux = 0; // flow into fluid bulk -// for (int xiIndex = 0; xiIndex < NUMXI; xiIndex++) { -// const Real relVeloX = CS * XI[xiIndex * LATTDIM] - u; -// const Real relVeloY = CS * XI[xiIndex * LATTDIM + 1] - v; -// const Real speed = relVeloX * relVeloX + relVeloY * relVeloY; -// Real cDotPrimary = -// relVeloX * primaryVector[0] + relVeloY * primaryVector[1]; -// bool isInflux = cDotPrimary > 0; -// bool isOutFlux = cDotPrimary < 0; -// } -// if (2 == boundaryType) { -// Real cDotSecond = -// relVeloX * secondVector[0] + relVeloY * secondVector[1]; -// isInflux = isInflux && (cDotSecond > 0); -// isOutFlux = isOutFlux && (cDotSecond < 0); -// } - -// if (isOutFlux) { -// outFlux += (speed * f(xiIndex, 0, 0)); -// } -// if (isInflux) { -// inFlux += (speed * CalcBGKFeq(xiIndex, 1, u, v, T, FEQORDER)); -// } +// VertexGeometryTypes vg = (VertexGeometryTypes)geometryProperty(0, 0); +// switch (vg) { +// case VG_IP: { +// f(5, 0, 0) = f(7, 0, 0); +// f(1, 0, 0) = f(3, 0, 0); +// f(8, 0, 0) = f(6, 0, 0); +// } break; +// case VG_IM: { +// f(7, 0, 0) = f(5, 0, 0); +// f(3, 0, 0) = f(1, 0, 0); +// f(6, 0, 0) = f(8, 0, 0); +// } break; +// case VG_JP: { +// f(2, 0, 0) = f(4, 0, 0); +// f(6, 0, 0) = f(8, 0, 0); +// f(5, 0, 0) = f(7, 0, 0); +// } break; +// case VG_JM: { +// f(4, 0, 0) = f(2, 0, 0); +// f(8, 0, 0) = f(6, 0, 0); +// f(7, 0, 0) = f(5, 0, 0); +// } break; +// case VG_IPJP_I: // inner corner point +// f(5, 0, 0) = f(7, 0, 0); +// break; +// case VG_IPJM_I: // inner corner point +// f(8, 0, 0) = f(6, 0, 0); +// break; +// case VG_IMJP_I: // inner corner point +// f(6, 0, 0) = f(8, 0, 0); +// break; +// case VG_IMJM_I: // inner corner point +// f(7, 0, 0) = f(5, 0, 0); +// break; +// default: +// break; // } -// Real rho = outFlux / inFlux; - -// for (int xiIndex = 0; xiIndex < NUMXI; xiIndex++) { -// const Real relVeloX = CS * XI[xiIndex * LATTDIM] - u; -// const Real relVeloY = CS * XI[xiIndex * LATTDIM + 1] - v; -// Real cDotPrimary = -// relVeloX * primaryVector[0] + relVeloY * primaryVector[1]; -// bool isInflux = cDotPrimary >= 0; -// if (0 == boundaryType) { -// Real cDotSecond = -// relVeloX * secondVector[0] + relVeloY * secondVector[1]; -// isInflux = isInflux && (cDotSecond >= 0); -// } -// if (1 == boundaryType) { -// Real cDotSecond = -// relVeloX * secondVector[0] + relVeloY * secondVector[1]; -// isInflux = isInflux || (cDotSecond > 0); -// } -// if (isInflux) { -// f(xiIndex, 0, 0) = -// CalcBGKFeq(xiIndex, rho, u, v, T, FEQORDER); -// } -// } -// } - -// else { -// #ifdef debug -// ops_printf("%s\n", -// "Warning: this node is not a kinetic boundary " -// "point: KerCutCellKinetic"); -// #endif -// } +// #endif // OPS_2D // } - -void KerCutCellCorrectedKinetic(const Real *givenMacroVars, const Real *dt, - const ACC &nodeType, +void KerCutCellEQMDiffuseRefl(ACC &f, const ACC &nodeType, const ACC &geometryProperty, - const ACC &tau, const ACC &feq, - ACC &f) { -#ifdef OPS_2D - VertexGeometryTypes vg = (VertexGeometryTypes)geometryProperty(0, 0); - const Real u = givenMacroVars[1]; - const Real v = givenMacroVars[2]; - // only for single component - const Real kn = tau(0, 0, 0); - Real wallNormalVector[]{0, 0}; - const Real sqrt2Inverse = 1 / sqrt(2); - switch (vg) { - case VG_IP: { - wallNormalVector[0] = 1; - wallNormalVector[1] = 0; - } break; - case VG_IM: { - wallNormalVector[0] = -1; - wallNormalVector[1] = 0; - } break; - case VG_JP: { - wallNormalVector[0] = 0; - wallNormalVector[1] = 1; - } break; - case VG_JM: { - wallNormalVector[0] = 0; - wallNormalVector[1] = -1; - } break; - case VG_IPJP_I: // inner corner point - { - wallNormalVector[0] = sqrt2Inverse; - wallNormalVector[1] = sqrt2Inverse; - } break; - case VG_IPJM_I: // inner corner point - { - wallNormalVector[0] = sqrt2Inverse; - wallNormalVector[1] = -sqrt2Inverse; - } break; - case VG_IMJP_I: // inner corner point - { - wallNormalVector[0] = -sqrt2Inverse; - wallNormalVector[1] = sqrt2Inverse; - } break; - case VG_IMJM_I: // inner corner point - { - wallNormalVector[0] = -sqrt2Inverse; - wallNormalVector[1] = -sqrt2Inverse; - } break; - default: - break; - } - Real outFlux = 0; // flow into wall - Real inFlux = 0; // flow into fluid bulk - for (int xiIndex = 0; xiIndex < NUMXI; xiIndex++) { - const Real cx = XI[xiIndex * LATTDIM]; - const Real cy = XI[xiIndex * LATTDIM + 1]; - Real cDotNormal = (CS * cx - u) * wallNormalVector[0] + - (CS * cy - v) * wallNormalVector[1]; - if (cDotNormal < 0) { - outFlux += - (-cDotNormal * - ((f(xiIndex, 0, 0) + (*dt) * feq(xiIndex, 0, 0) / (2 * kn)) / - (1 + (*dt) / (2 * kn)))); - } - if (cDotNormal > 0) { - Real cu = (CS * cx * u + CS * cy * v); - inFlux += - (cDotNormal * WEIGHTS[xiIndex] * - (1 + cu + 0.5 * (cu * cu - (u * u + v * v)))); // i.e., the - // equilibrium - } - } - Real rho = outFlux / inFlux; - for (int xiIndex = 0; xiIndex < NUMXI; xiIndex++) { - const Real cx = XI[xiIndex * LATTDIM]; - const Real cy = (int)XI[xiIndex * LATTDIM + 1]; - Real cDotNormal = (CS * cx - u) * wallNormalVector[0] + - (CS * cy - v) * wallNormalVector[1]; - if (cDotNormal >= 0) { - Real cu = (CS * cx * u + CS * cy * v); - f(xiIndex, 0, 0) = (rho * WEIGHTS[xiIndex] * - (1 + cu + 0.5 * (cu * cu - (u * u + v * v)))); - } - } - -#endif // OPS_2D -} - -void KerCutCellBounceBack(const ACC &nodeType, - const ACC &geometryProperty, ACC &f) { -#ifdef OPS_2D - - VertexGeometryTypes vg = (VertexGeometryTypes)geometryProperty(0, 0); - switch (vg) { - case VG_IP: { - f(5, 0, 0) = f(7, 0, 0); - f(1, 0, 0) = f(3, 0, 0); - f(8, 0, 0) = f(6, 0, 0); - } break; - case VG_IM: { - f(7, 0, 0) = f(5, 0, 0); - f(3, 0, 0) = f(1, 0, 0); - f(6, 0, 0) = f(8, 0, 0); - } break; - case VG_JP: { - f(2, 0, 0) = f(4, 0, 0); - f(6, 0, 0) = f(8, 0, 0); - f(5, 0, 0) = f(7, 0, 0); - } break; - case VG_JM: { - f(4, 0, 0) = f(2, 0, 0); - f(8, 0, 0) = f(6, 0, 0); - f(7, 0, 0) = f(5, 0, 0); - } break; - case VG_IPJP_I: // inner corner point - f(5, 0, 0) = f(7, 0, 0); - break; - case VG_IPJM_I: // inner corner point - f(8, 0, 0) = f(6, 0, 0); - break; - case VG_IMJP_I: // inner corner point - f(6, 0, 0) = f(8, 0, 0); - break; - case VG_IMJM_I: // inner corner point - f(7, 0, 0) = f(5, 0, 0); - break; - default: - break; - } - -#endif // OPS_2D -} - -void KerCutCellEQMDiffuseRefl(const Real *givenMacroVars, - const ACC &nodeType, - const ACC &geometryProperty, ACC &f, - const int *componentId) { + const Real *givenMacroVars, + const int *lattIdx) { #ifdef OPS_2D // This kernel is suitable for a single-speed lattice // but only for the second-order expansion at this moment // Therefore, the equilibrium function order is fixed at 2 const int equilibriumOrder{2}; - const int compoIdx{*componentId}; - - VertexGeometryTypes vg = (VertexGeometryTypes)geometryProperty(0, 0); - Real u = givenMacroVars[1]; - Real v = givenMacroVars[2]; + VertexGeometryType vg = (VertexGeometryType)geometryProperty(0, 0); + Real u = givenMacroVars[0]; + Real v = givenMacroVars[1]; #ifdef CPU #if DebugLevel >= 2 ops_printf( "KerCutCellEQMDiffuseRefl: We received the following " "conditions for the surface %i:\n", geometryProperty(0, 0)); - ops_printf("U=%f, V=%f, for the component %i\n", u, v, *componentId); + ops_printf("U=%f, V=%f\n", u, v); #endif #endif int numOutgoing{0}; int numIncoming{0}; int numParallel{0}; - int *outgoing = - new int[COMPOINDEX[2 * compoIdx + 1] - COMPOINDEX[2 * compoIdx] + 1]; - int *incoming = - new int[COMPOINDEX[2 * compoIdx + 1] - COMPOINDEX[2 * compoIdx] + 1]; - int *parallel = - new int[COMPOINDEX[2 * compoIdx + 1] - COMPOINDEX[2 * compoIdx] + 1]; + int lattStart{lattIdx[0]}; + int lattEnd{lattIdx[1]}; + int *outgoing = new int[lattEnd - lattStart + 1]; + int *incoming = new int[lattEnd - lattStart + 1]; + int *parallel = new int[lattEnd - lattStart + 1]; Real rhoIncoming{0}; Real rhoParallel{0}; Real deltaRho{0}; - for (int xiIdx = COMPOINDEX[2 * compoIdx]; - xiIdx <= COMPOINDEX[2 * compoIdx + 1]; xiIdx++) { + for (int xiIdx = lattStart; xiIdx <= lattEnd; xiIdx++) { + Real cx{CS * XI[xiIdx * LATTDIM]}; + Real cy{CS * XI[xiIdx * LATTDIM + 1]}; BndryDvType bdt = FindBdyDvType(vg, &XI[xiIdx * LATTDIM]); switch (bdt) { case BndryDv_Incoming: { @@ -1017,8 +572,6 @@ void KerCutCellEQMDiffuseRefl(const Real *givenMacroVars, } break; case BndryDv_Outgoing: { outgoing[numOutgoing] = xiIdx; - Real cx{CS * XI[xiIdx * LATTDIM]}; - Real cy{CS * XI[xiIdx * LATTDIM + 1]}; deltaRho += (2 * WEIGHTS[xiIdx]) * (cx * u + cy * v); numOutgoing++; } break; @@ -1056,136 +609,108 @@ void KerCutCellEQMDiffuseRefl(const Real *givenMacroVars, #endif // OPS_2D } -void KerCutCellPeriodic(const ACC &nodeType, - const ACC &geometryProperty, ACC &f) { +void KerCutCellPeriodic(ACC &f, const ACC &nodeType, + const ACC &geometryProperty, const int *lattIdx, + const int *surface) { #ifdef OPS_2D + const int xiStartPos{lattIdx[0]}; + const int xiEndPos{lattIdx[1]}; + const BoundarySurface boundarySurface{(BoundarySurface)(*surface)}; - VertexGeometryTypes vg = (VertexGeometryTypes)geometryProperty(0, 0); + VertexGeometryType vg = (VertexGeometryType)geometryProperty(0, 0); switch (vg) { case VG_IP: - for (int xiIndex = 0; xiIndex < NUMXI; xiIndex++) { - Real cx = XI[xiIndex * LATTDIM]; - if (cx > 0) { - f(xiIndex, 0, 0) = f(xiIndex, -1, 0); - } + for (int xiIndex = xiStartPos; xiIndex <= xiEndPos; xiIndex++) { + f(xiIndex, 0, 0) = f(xiIndex, -1, 0); } break; case VG_IM: - for (int xiIndex = 0; xiIndex < NUMXI; xiIndex++) { - Real cx = XI[xiIndex * LATTDIM]; - if (cx < 0) { - f(xiIndex, 0, 0) = f(xiIndex, 1, 0); - } + for (int xiIndex = xiStartPos; xiIndex <= xiEndPos; xiIndex++) { + f(xiIndex, 0, 0) = f(xiIndex, 1, 0); } break; case VG_JP: - for (int xiIndex = 0; xiIndex < NUMXI; xiIndex++) { - Real cy = XI[xiIndex * LATTDIM + 1]; - if (cy > 0) { - f(xiIndex, 0, 0) = f(xiIndex, 0, -1); - } + for (int xiIndex = xiStartPos; xiIndex <= xiEndPos; xiIndex++) { + f(xiIndex, 0, 0) = f(xiIndex, 0, -1); } break; case VG_JM: - for (int xiIndex = 0; xiIndex < NUMXI; xiIndex++) { - Real cy = XI[xiIndex * LATTDIM + 1]; - if (cy < 0) { - f(xiIndex, 0, 0) = f(xiIndex, 0, 1); - } + for (int xiIndex = xiStartPos; xiIndex <= xiEndPos; xiIndex++) { + f(xiIndex, 0, 0) = f(xiIndex, 0, 1); } break; - // There are only inner corners for block boundaries + // There are only inner corners for block boundaries case VG_IPJP_I: { // VG_IP - if (vt == nodeType(0, 1)) { - for (int xiIndex = 0; xiIndex < NUMXI; xiIndex++) { - Real cx = XI[xiIndex * LATTDIM]; - Real cy = XI[xiIndex * LATTDIM + 1]; - if (cy > 0 || cx > 0) { - f(xiIndex, 0, 0) = f(xiIndex, -1, 0); - } + if (boundarySurface == BoundarySurface::Left) { + for (int xiIndex = xiStartPos; xiIndex <= xiEndPos; xiIndex++) { + f(xiIndex, 0, 0) = f(xiIndex, -1, 0); } } // VG_JP - if (vt == nodeType(1, 0)) { - for (int xiIndex = 0; xiIndex < NUMXI; xiIndex++) { - Real cx = XI[xiIndex * LATTDIM]; - Real cy = XI[xiIndex * LATTDIM + 1]; - if (cy > 0 || cx > 0) { - f(xiIndex, 0, 0) = f(xiIndex, 0, -1); - } + if (boundarySurface == BoundarySurface::Bottom) { + for (int xiIndex = xiStartPos; xiIndex <= xiEndPos; xiIndex++) { + f(xiIndex, 0, 0) = f(xiIndex, 0, -1); } } } break; case VG_IPJM_I: { // VG_IP - if (vt == nodeType(0, -1)) { - for (int xiIndex = 0; xiIndex < NUMXI; xiIndex++) { - Real cx = XI[xiIndex * LATTDIM]; - Real cy = XI[xiIndex * LATTDIM + 1]; - if (cy < 0 || cx > 0) { - f(xiIndex, 0, 0) = f(xiIndex, -1, 0); - } + if (boundarySurface == BoundarySurface::Left) { + for (int xiIndex = xiStartPos; xiIndex <= xiEndPos; xiIndex++) { + f(xiIndex, 0, 0) = f(xiIndex, -1, 0); } } // VG_JM - if (vt == nodeType(1, 0)) { - for (int xiIndex = 0; xiIndex < NUMXI; xiIndex++) { - Real cx = XI[xiIndex * LATTDIM]; - Real cy = XI[xiIndex * LATTDIM + 1]; - if (cy < 0 || cx > 0) { - f(xiIndex, 0, 0) = f(xiIndex, 0, 1); - } + if (boundarySurface == BoundarySurface::Top) { + for (int xiIndex = xiStartPos; xiIndex <= xiEndPos; xiIndex++) { + f(xiIndex, 0, 0) = f(xiIndex, 0, 1); } } } break; case VG_IMJP_I: { // VG_IM - if (vt == nodeType(0, 1)) { - for (int xiIndex = 0; xiIndex < NUMXI; xiIndex++) { - Real cx = XI[xiIndex * LATTDIM]; - Real cy = XI[xiIndex * LATTDIM + 1]; - if (cy > 0 || cx < 0) { - f(xiIndex, 0, 0) = f(xiIndex, 1, 0); - } + if (boundarySurface == BoundarySurface::Right) { + for (int xiIndex = xiStartPos; xiIndex <= xiEndPos; xiIndex++) { + f(xiIndex, 0, 0) = f(xiIndex, 1, 0); } } // VG_JP - if (vt == nodeType(-1, 0)) { - for (int xiIndex = 0; xiIndex < NUMXI; xiIndex++) { - Real cx = XI[xiIndex * LATTDIM]; - Real cy = XI[xiIndex * LATTDIM + 1]; - if (cy > 0 || cx < 0) { - f(xiIndex, 0, 0) = f(xiIndex, 0, -1); - } + if (boundarySurface == BoundarySurface::Bottom) { + for (int xiIndex = xiStartPos; xiIndex <= xiEndPos; xiIndex++) { + f(xiIndex, 0, 0) = f(xiIndex, 0, -1); } } } break; case VG_IMJM_I: { // VG_IM - if (vt == nodeType(0, -1)) { - for (int xiIndex = 0; xiIndex < NUMXI; xiIndex++) { - Real cx = XI[xiIndex * LATTDIM]; - Real cy = XI[xiIndex * LATTDIM + 1]; - if (cy < 0 || cx < 0) { - f(xiIndex, 0, 0) = f(xiIndex, 1, 0); - } + if (boundarySurface == BoundarySurface::Right) { + for (int xiIndex = xiStartPos; xiIndex <= xiEndPos; xiIndex++) { + f(xiIndex, 0, 0) = f(xiIndex, 1, 0); } } // VG_JM - if (vt == nodeType(-1, 0)) { - for (int xiIndex = 0; xiIndex < NUMXI; xiIndex++) { - Real cx = XI[xiIndex * LATTDIM]; - Real cy = XI[xiIndex * LATTDIM + 1]; - if (cy < 0 || cx < 0) { - f(xiIndex, 0, 0) = f(xiIndex, 0, 1); - } + if (boundarySurface == BoundarySurface::Top) { + for (int xiIndex = xiStartPos; xiIndex <= xiEndPos; xiIndex++) { + f(xiIndex, 0, 0) = f(xiIndex, 0, 1); } } } break; default: break; } +#ifdef CPU + for (int xiIndex = xiStartPos; xiIndex <= xiEndPos; xiIndex++) { + const Real res{f(xiIndex, 0, 0)}; + if (isnan(res) || res <= 0 || isinf(res)) { + ops_printf( + "Error! Distribution function %f becomes invalid at the " + "lattice %i\n at the surface %i\n", + res, xiIndex, geometryProperty(0, 0)); + assert(!(isnan(res) || res <= 0 || isinf(res))); + } + } +#endif #endif // OPS_2D } @@ -1201,7 +726,7 @@ void KerCutCellZouHeVelocity(const Real *givenMacroVars, Note: This boundary condition is lattice specific. */ - VertexGeometryTypes vg = (VertexGeometryTypes)geometryProperty(0, 0); + VertexGeometryType vg = (VertexGeometryType)geometryProperty(0, 0); Real rho{0}; Real u{givenMacroVars[1]}; Real v{givenMacroVars[2]}; @@ -1362,7 +887,7 @@ void KerCutCellZouHeVelocity(const Real *givenMacroVars, #endif // OPS_2D outer // Boundary conditions for three-dimensional problems - +#ifdef OPS_3D void KerCutCellExtrapolPressure1ST3D(ACC &f, const ACC &nodeType, const ACC &geometryProperty, const Real *givenBoundaryVars, @@ -1747,66 +1272,6 @@ void KerCutCellEQMDiffuseRefl3D(ACC &f, const ACC &nodeType, #endif //OPS_3D } -void KerCutCellNoslipEQN3D(ACC &f, const ACC &nodeType, - const Real *givenMacroVars, const int *lattIdx) { -#ifdef OPS_3D - Real u = givenMacroVars[0]; - Real v = givenMacroVars[1]; - Real w = givenMacroVars[2]; -#ifdef CPU -#if DebugLevel >= 2 - ops_printf( - "KerCutCellNoslipEQN3D: We received the following " - "conditions:\n"); - ops_printf("U=%f, V=%f, W=%f\n", u, v, w); -#endif -#endif - Real rhoIntermidate{0}; - Real uIntermidate{0}; - Real vIntermidate{0}; - Real wIntermidate{0}; - - for (int xiIdx = lattIdx[0]; xiIdx <= lattIdx[1]; xiIdx++) { - Real cx{CS * XI[xiIdx * LATTDIM]}; - Real cy{CS * XI[xiIdx * LATTDIM + 1]}; - Real cz{CS * XI[xiIdx * LATTDIM + 2]}; - rhoIntermidate += f(xiIdx, 0, 0, 0); - uIntermidate += (cx * f(xiIdx, 0, 0, 0)); - vIntermidate += (cy * f(xiIdx, 0, 0, 0)); - wIntermidate += (cz * f(xiIdx, 0, 0, 0)); - } - uIntermidate /= rhoIntermidate; - vIntermidate /= rhoIntermidate; - wIntermidate /= rhoIntermidate; -#ifdef CPU -#if DebugLevel >= 2 - ops_printf("Calculated intermidate density = %f\n", rhoIntermidate); - ops_printf("Calculated intermidate U = %f\n", uIntermidate); - ops_printf("Calculated intermidate V = %f\n", vIntermidate); - ops_printf("Calculated intermidate W = %f\n", wIntermidate); -#endif -#endif - for (int xiIdx = lattIdx[0]; xiIdx <= lattIdx[1]; xiIdx++) { - f(xiIdx, 0, 0, 0) = - - - - CalcBGKFeq(xiIdx, rhoIntermidate, u, v, w, 1, 2); -#ifdef CPU - const Real res{f(xiIdx, 0, 0, 0)}; - if (isnan(res) || res <= 0 || isinf(res)) { - ops_printf( - "Error! Distribution function %f becomes invalid at the " - "lattice %i\n", - res, xiIdx); - assert(!(isnan(res) || res <= 0 || isinf(res))); - } -#endif - } - -#endif // OPS_3D -} - void KerCutCellPeriodic3D(ACC &f, const ACC &nodeType, const ACC &geometryProperty, const int *lattIdx, const int* surface) { @@ -2196,4 +1661,5 @@ void KerCutCellPeriodic3D(ACC &f, const ACC &nodeType, #endif // OPS_3D } +#endif //OPS_3D #endif // BOUNDARY_KERNEL_INC \ No newline at end of file diff --git a/Src/boundary_wrapper.cpp b/Src/boundary_wrapper.cpp index 5a559a0..ade78c0 100644 --- a/Src/boundary_wrapper.cpp +++ b/Src/boundary_wrapper.cpp @@ -49,21 +49,70 @@ void TreatBlockBoundary3D(const Block& block, const int componentID, ops_arg_gbl(g_Components().at(componentID).index, 2, "int", OPS_READ)); } break; - case BoundaryScheme::EQNNoSlip: { + case BoundaryScheme::FDPeriodic: { ops_par_loop( - KerCutCellNoslipEQN3D, "KerCutCellNoslipEQN3D", block.Get(), + KerCutCellPeriodic3D, "KerCutCellPeriodic3D", block.Get(), SpaceDim(), range.data(), ops_arg_dat(g_f()[blockIndex], NUMXI, LOCALSTENCIL, "double", OPS_RW), ops_arg_dat(g_NodeType().at(componentID).at(blockIndex), 1, LOCALSTENCIL, "int", OPS_READ), - ops_arg_gbl(givenVars, 3, "double", OPS_READ), + ops_arg_dat(g_GeometryProperty()[blockIndex], 1, LOCALSTENCIL, + "int", OPS_READ), + ops_arg_gbl(g_Components().at(componentID).index, 2, "int", + OPS_READ), + ops_arg_gbl(&surface, 1, "int", OPS_READ)); + } break; + default: + break; + } +} +#endif //OPS_3D + +#ifdef OPS_2D +void TreatBlockBoundary(const Block& block, const int componentID, + const Real* givenVars, + const BoundaryScheme boundaryScheme, + const BoundarySurface boundarySurface) { + const int surface{(int)boundarySurface}; + const int blockIndex{block.ID()}; + std::vector range(2 * SpaceDim()); + range.assign(block.BoundarySurfaceRange().at(boundarySurface).begin(), + block.BoundarySurfaceRange().at(boundarySurface).end()); + switch (boundaryScheme) { + case BoundaryScheme::ExtrapolPressure1ST: { + ops_par_loop( + KerCutCellExtrapolPressure1ST, + "KerCutCellExtrapolPressure1ST", block.Get(), SpaceDim(), + range.data(), + ops_arg_dat(g_f()[blockIndex], NUMXI, ONEPTREGULARSTENCIL, + "double", OPS_RW), + ops_arg_dat(g_NodeType().at(componentID).at(blockIndex), 1, + ONEPTREGULARSTENCIL, "int", OPS_READ), + ops_arg_dat(g_GeometryProperty()[blockIndex], 1, LOCALSTENCIL, + "int", OPS_READ), + ops_arg_gbl(givenVars, 1, "double", OPS_READ), + ops_arg_gbl(&surface, 1, "int", OPS_READ), + ops_arg_gbl(g_Components().at(componentID).index, 2, "int", + OPS_READ)); + } break; + case BoundaryScheme::EQMDiffuseRefl: { + ops_par_loop( + KerCutCellEQMDiffuseRefl, "KerCutCellEQMDiffuseRefl", + block.Get(), SpaceDim(), range.data(), + ops_arg_dat(g_f()[blockIndex], NUMXI, LOCALSTENCIL, "double", + OPS_RW), + ops_arg_dat(g_NodeType().at(componentID).at(blockIndex), 1, + LOCALSTENCIL, "int", OPS_READ), + ops_arg_dat(g_GeometryProperty()[blockIndex], 1, LOCALSTENCIL, + "int", OPS_READ), + ops_arg_gbl(givenVars, 2, "double", OPS_READ), ops_arg_gbl(g_Components().at(componentID).index, 2, "int", OPS_READ)); } break; case BoundaryScheme::FDPeriodic: { ops_par_loop( - KerCutCellPeriodic3D, "KerCutCellPeriodic3D", block.Get(), + KerCutCellPeriodic, "KerCutCellPeriodic", block.Get(), SpaceDim(), range.data(), ops_arg_dat(g_f()[blockIndex], NUMXI, LOCALSTENCIL, "double", OPS_RW), @@ -79,4 +128,4 @@ void TreatBlockBoundary3D(const Block& block, const int componentID, break; } } -#endif //OPS_3D +#endif //OPS_2D diff --git a/Src/configuration.cpp b/Src/configuration.cpp index 283a61e..72778e7 100644 --- a/Src/configuration.cpp +++ b/Src/configuration.cpp @@ -101,9 +101,13 @@ NLOHMANN_JSON_SERIALIZE_ENUM(BoundarySurface, {{BoundarySurface::Left, "Left"}, {BoundarySurface::Right, "Right"}, {BoundarySurface::Top, "Top"}, - {BoundarySurface::Bottom, "Bottom"}, + {BoundarySurface::Bottom, "Bottom"} +#ifdef OPS_3D + , {BoundarySurface::Front, "Front"}, - {BoundarySurface::Back, "Back"}}); + {BoundarySurface::Back, "Back"} +#endif + }); NLOHMANN_JSON_SERIALIZE_ENUM(InitialType, {{Initial_BGKFeq2nd, "Initial_BGKFeq2nd"}}); diff --git a/Src/evolution.cpp b/Src/evolution.cpp index 02ee84f..db56f8c 100644 --- a/Src/evolution.cpp +++ b/Src/evolution.cpp @@ -28,15 +28,23 @@ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. -*/ + */ -/*! @brief Wrap functions for main evolution cycle. +/*! + * @brief Wrap functions for main evolution cycle. * @author Jianping Meng * @details Define wrap functions for implementing the main evolution * cycle */ - #include "evolution.h" +#include "type.h" +#include "scheme.h" +#include "block.h" +#include "field.h" +#include "boundary.h" +#include "flowfield.h" +#include "model.h" + /* * In the following routines, there are some variables are defined * for the convenience of the translator which may not be able to @@ -44,378 +52,43 @@ * Even though, a variable rather than a numerical literacy will need * some modifications in the Python translator. */ +// void TreatEmbeddedBoundary3D() { +// for (int blockIdx = 0; blockIdx < BlockNum(); blockIdx++) { +// int* iterRng.data() = BlockIterRng(blockIdx, IterRngBulk()); +// ops_par_loop( +// KerCutCellImmersedBoundary3D, "KerCutCellImmersedBoundary3D", +// g_Block[blockIdx], SpaceDim(), iterRng.data(), +// ops_arg_dat(g_NodeType()[blockIdx], 1, LOCALSTENCIL, "int", +// OPS_READ), ops_arg_dat(g_GeometryProperty[blockIdx], 1, +// LOCALSTENCIL, "int", +// OPS_READ), +// ops_arg_dat(g_f()[blockIdx], NUMXI, LOCALSTENCIL, "double", +// OPS_RW)); +// } +// } -#ifdef OPS_2D - -void Collision() { - for (int blockIndex = 0; blockIndex < BlockNum(); blockIndex++) { - int* iterRng = BlockIterRng(blockIndex, IterRngWhole()); - ops_par_loop(KerCollide, "KerCollide", g_Block[blockIndex], SPACEDIM, - iterRng, ops_arg_gbl(pTimeStep(), 1, "double", OPS_READ), - ops_arg_dat(g_NodeType[blockIndex], NUMCOMPONENTS, - LOCALSTENCIL, "int", OPS_READ), - ops_arg_dat(g_f[blockIndex], NUMXI, LOCALSTENCIL, "double", - OPS_READ), - ops_arg_dat(g_feq[blockIndex], NUMXI, LOCALSTENCIL, - "double", OPS_READ), - ops_arg_dat(g_Tau[blockIndex], NUMCOMPONENTS, LOCALSTENCIL, - "double", OPS_READ), - ops_arg_dat(g_Bodyforce[blockIndex], NUMXI, LOCALSTENCIL, - "double", OPS_READ), - ops_arg_dat(g_fStage[blockIndex], NUMXI, LOCALSTENCIL, - "double", OPS_WRITE)); - } -} - -void Stream() { - for (int blockIndex = 0; blockIndex < BlockNum(); blockIndex++) { - int* iterRng = BlockIterRng(blockIndex, IterRngWhole()); - ops_par_loop( - KerStream, "KerStream", g_Block[blockIndex], SPACEDIM, iterRng, - ops_arg_dat(g_f[blockIndex], NUMXI, LOCALSTENCIL, "double", OPS_RW), - ops_arg_dat(g_fStage[blockIndex], NUMXI, ONEPTLATTICESTENCIL, - "double", OPS_READ) - ops_arg_dat(g_NodeType[blockIndex], NUMCOMPONENTS, LOCALSTENCIL, - "int", OPS_READ), - ops_arg_dat(g_GeometryProperty[blockIndex], 1, LOCALSTENCIL, "int", - OPS_READ)); - } -} - -void UpdateMacroVars() { - for (int blockIndex = 0; blockIndex < BlockNum(); blockIndex++) { - int* iterRng = BlockIterRng(blockIndex, IterRngWhole()); - ops_par_loop(KerCalcMacroVars, "KerCalcMacroVars", g_Block[blockIndex], - SPACEDIM, iterRng, - ops_arg_dat(g_NodeType[blockIndex], NUMCOMPONENTS, - LOCALSTENCIL, "int", OPS_READ), - ops_arg_dat(g_f[blockIndex], NUMXI, LOCALSTENCIL, "double", - OPS_READ), - ops_arg_dat(g_MacroVars[blockIndex], NUMMACROVAR, - LOCALSTENCIL, "double", OPS_RW)); - } -} - -void UpdateFeqandBodyforce() { - for (int blockIndex = 0; blockIndex < BlockNum(); blockIndex++) { - int* iterRng = BlockIterRng(blockIndex, IterRngWhole()); - ops_par_loop(KerCalcFeq, "KerCalcPolyFeq", g_Block[blockIndex], - SPACEDIM, iterRng, - ops_arg_dat(g_NodeType[blockIndex], NUMCOMPONENTS, - LOCALSTENCIL, "int", OPS_READ), - ops_arg_dat(g_MacroVars[blockIndex], NUMMACROVAR, - LOCALSTENCIL, "double", OPS_READ), - ops_arg_dat(g_feq[blockIndex], NUMXI, LOCALSTENCIL, - "double", OPS_RW)); - // force term to be added - - // time is not used in the current force - Real* timeF{0}; - ops_par_loop(KerCalcBodyForce, "KerCalcBodyForce", - g_Block[blockIndex], SPACEDIM, iterRng, - ops_arg_gbl(&timeF, 1, "double", OPS_READ), - ops_arg_dat(g_NodeType[blockIndex], NUMCOMPONENTS, - LOCALSTENCIL, "int", OPS_READ), - ops_arg_dat(g_CoordinateXYZ[blockIndex], SPACEDIM, - LOCALSTENCIL, "double", OPS_READ), - ops_arg_dat(g_MacroVars[blockIndex], NUMMACROVAR, - LOCALSTENCIL, "double", OPS_READ), - ops_arg_dat(g_Bodyforce[blockIndex], NUMXI, LOCALSTENCIL, - "double", OPS_RW)); - } -} - -void TreatDomainBoundary(const int blockIndex, const int componentID, - const Real* givenVars, int* range, - const BoundaryType boundaryScheme) -{ - switch (boundaryScheme) { - case BoundaryScheme::ExtrapolPressure1ST: { - ops_par_loop( - KerCutCellExtrapolPressure1ST, "KerCutCellExtrapolPressure1ST", - g_Block[blockIndex], SPACEDIM, range, - ops_arg_gbl(givenVars, NUMMACROVAR, "double", OPS_READ), - ops_arg_dat(g_NodeType[blockIndex], NUMCOMPONENTS, - ONEPTREGULARSTENCIL, "int", OPS_READ), - ops_arg_dat(g_GeometryProperty[blockIndex], 1, LOCALSTENCIL, - "int", OPS_READ), - ops_arg_dat(g_f[blockIndex], NUMXI, ONEPTREGULARSTENCIL, "double", - OPS_RW)); - } break; - case BoundaryType__ExtrapolPressure2ND: { - ops_par_loop( - KerCutCellExtrapolPressure2ND, - "KerCutCellExtrapolPressure2ND", g_Block[blockIndex], - SPACEDIM, range, - ops_arg_gbl(givenVars, NUMMACROVAR, "double", OPS_READ), - ops_arg_dat(g_NodeType[blockIndex], 1, ONEPTREGULARSTENCIL, - "int", OPS_READ), - ops_arg_dat(g_GeometryProperty[blockIndex], 1, LOCALSTENCIL, - "int", OPS_READ), - ops_arg_dat(g_f[blockIndex], NUMXI, TWOPTREGULARSTENCIL, - "double", OPS_RW)); - } break; - case BoundaryType__BounceBackWall: { - ops_par_loop(KerCutCellBounceBack, "KerCutCellBounceBack", - g_Block[blockIndex], SPACEDIM, range, - ops_arg_dat(g_NodeType[blockIndex], 1, LOCALSTENCIL, - "int", OPS_READ), - ops_arg_dat(g_GeometryProperty[blockIndex], 1, - LOCALSTENCIL, "int", OPS_READ), - ops_arg_dat(g_f[blockIndex], NUMXI, LOCALSTENCIL, - "double", OPS_RW)); - } break; - case BoundaryType__ZouHeVelocity: { - ops_par_loop( - KerCutCellZouHeVelocity, "KerCutCellZouHeVelocity,", - g_Block[blockIndex], SPACEDIM, range, - ops_arg_gbl(givenVars, NUMMACROVAR, "double", OPS_READ), - ops_arg_dat(g_NodeType[blockIndex], 1, LOCALSTENCIL, "int", - OPS_READ), - ops_arg_dat(g_GeometryProperty[blockIndex], 1, LOCALSTENCIL, - "int", OPS_READ), - ops_arg_dat(g_MacroVars[blockIndex], NUMMACROVAR, - ONEPTLATTICESTENCIL, "double", OPS_READ), - ops_arg_dat(g_f[blockIndex], NUMXI, ONEPTLATTICESTENCIL, - "double", OPS_RW)); - } break; - case BoundaryType__KineticDiffuseWall: { - // ops_par_loop( - // KerCutCellKinetic, "KerCutCellKinetic", g_Block[blockIndex], - // SPACEDIM, range, - // ops_arg_gbl(givenVars, NUMMACROVAR, "double", OPS_READ), - // ops_arg_dat(g_NodeType[blockIndex], 1, LOCALSTENCIL, "int", - // OPS_READ), - // ops_arg_dat(g_GeometryProperty[blockIndex], 1, LOCALSTENCIL, - // "int", OPS_READ), - // ops_arg_dat(g_f[blockIndex], NUMXI, LOCALSTENCIL, "double", - // OPS_RW)); - } - case BoundaryType__EQMDiffuseRefl: { - ops_par_loop( - KerCutCellEQMDiffuseRefl, "KerCutCellEQMDiffuseRefl", - g_Block[blockIndex], SPACEDIM, range, - ops_arg_gbl(givenVars, NUMMACROVAR, "double", OPS_READ), - ops_arg_dat(g_NodeType[blockIndex], NUMCOMPONENTS, LOCALSTENCIL, - "int", OPS_READ), - ops_arg_dat(g_GeometryProperty[blockIndex], 1, LOCALSTENCIL, - "int", OPS_READ), - ops_arg_dat(g_f[blockIndex], NUMXI, LOCALSTENCIL, "double", - OPS_RW), - ops_arg_gbl(&componentID, 1, "int", OPS_READ)); - } break; - case BoundaryType__FreeFlux: { - ops_par_loop(KerCutCellZeroFlux, "KerCutCellZeroFlux", - g_Block[blockIndex], SPACEDIM, range, - ops_arg_dat(g_NodeType[blockIndex], 1, LOCALSTENCIL, - "int", OPS_READ), - ops_arg_dat(g_GeometryProperty[blockIndex], 1, - LOCALSTENCIL, "int", OPS_READ), - ops_arg_dat(g_f[blockIndex], NUMXI, LOCALSTENCIL, - "double", OPS_RW)); - } break; - case BoundaryType__Periodic: { - ops_par_loop(KerCutCellPeriodic, "KerCutCellPeriodic", - g_Block[blockIndex], SPACEDIM, range, - ops_arg_dat(g_NodeType[blockIndex], NUMCOMPONENTS, - LOCALSTENCIL, "int", OPS_READ), - ops_arg_dat(g_GeometryProperty[blockIndex], 1, - LOCALSTENCIL, "int", OPS_READ), - ops_arg_dat(g_f[blockIndex], NUMXI, LOCALSTENCIL, - "double", OPS_RW)); - } break; - default: - break; - } -} - -void ImplementBoundary() { - for (auto boundary : BlockBoundaries()) { - int* range{BoundarySurfaceRange(boundary.blockIndex, - boundary.boundarySurface)}; - TreatBlockBoundary(boundary.blockIndex, boundary.componentID, - boundary.givenVars.data(), range, - boundary.boundaryScheme, boundary.boundarySurface); - } -} - - -void TreatEmbeddedBoundary() { - for (int blockIdx = 0; blockIdx < BlockNum(); blockIdx++) { - int* iterRng = BlockIterRng(blockIdx, IterRngBulk()); - ops_par_loop( - KerCutCellEmbeddedBoundary, "KerCutCellImmersedBoundary", - g_Block[blockIdx], SPACEDIM, iterRng, - ops_arg_dat(g_NodeType[blockIdx], NUMCOMPONENTS, LOCALSTENCIL, - "int", OPS_READ), - ops_arg_dat(g_GeometryProperty[blockIdx], 1, LOCALSTENCIL, "int", - OPS_READ), - ops_arg_dat(g_f[blockIdx], NUMXI, LOCALSTENCIL, "double", OPS_RW)); - } -} -//TODO This function needs to be improved for different initialisation scheme -void InitialiseSolution() { - UpdateFeqandBodyforce(); - for (int blockIndex = 0; blockIndex < BlockNum(); blockIndex++) { - int* iterRng = BlockIterRng(blockIndex, IterRngWhole()); - const Real zero = 0; - ops_par_loop( - KerSetfFixValue, "KerSetfFixValue", g_Block[blockIndex], SPACEDIM, - iterRng, ops_arg_gbl(&zero, 1, "double", OPS_READ), - ops_arg_dat(g_Bodyforce[0], NUMXI, LOCALSTENCIL, "double", OPS_RW)); - } - CopyDistribution(g_feq, g_f); -} - -void CopyDistribution(const ops_dat* fSrc, ops_dat* fDest) { - for (int blockIndex = 0; blockIndex < BlockNum(); blockIndex++) { - int* iterRng = BlockIterRng(blockIndex, IterRngWhole()); - ops_par_loop(KerCopyf, "KerCopyf", g_Block[blockIndex], SPACEDIM, - iterRng, - ops_arg_dat(fSrc[blockIndex], NUMXI, LOCALSTENCIL, - "double", OPS_READ), - ops_arg_dat(fDest[blockIndex], NUMXI, LOCALSTENCIL, - "double", OPS_WRITE)); - } -} - -void CalcTotalMass(double* totalMass) { - ops_reduction massHandle = - ops_decl_reduction_handle(sizeof(double), "double", "massHandle"); - for (int blockIdx = 0; blockIdx < BlockNum(); blockIdx++) { - int* iterRng = BlockIterRng(blockIdx, IterRngWhole()); - ops_par_loop(KerCalcSumofDensity, "KerCalcSumofDensity", - g_Block[blockIdx], SPACEDIM, iterRng, - ops_arg_dat(g_MacroVars[blockIdx], NUMMACROVAR, - LOCALSTENCIL, "double", OPS_READ), - ops_arg_reduce(massHandle, 1, "double", OPS_INC)); - } - ops_reduction_result(massHandle, (double*)totalMass); -} - -void NormaliseF(Real* ratio) { - for (int blockIdx = 0; blockIdx < BlockNum(); blockIdx++) { - int* iterRng = BlockIterRng(blockIdx, IterRngWhole()); - ops_par_loop( - KerNormaliseF, "KerNormaliseF", g_Block[blockIdx], SPACEDIM, - iterRng, ops_arg_gbl(ratio, 1, "double", OPS_READ), - ops_arg_dat(g_f[blockIdx], NUMXI, LOCALSTENCIL, "double", OPS_RW)); - } -} - -void CalcResidualError() { - for (int macroVarIdx = 0; macroVarIdx < MacroVarsNum(); macroVarIdx++) { - for (int blockIdx = 0; blockIdx < BlockNum(); blockIdx++) { - int* iterRng = BlockIterRng(blockIdx, IterRngWhole()); - ops_par_loop(KerCalcMacroVarSquareofDifference, - "KerCalcMacroVarSquareofDifference", g_Block[blockIdx], - SPACEDIM, iterRng, - ops_arg_dat(g_MacroVars[blockIdx], NUMMACROVAR, - LOCALSTENCIL, "double", OPS_READ), - ops_arg_dat(g_MacroVarsCopy[blockIdx], NUMMACROVAR, - LOCALSTENCIL, "double", OPS_READ), - ops_arg_gbl(¯oVarIdx, 1, "int", OPS_READ), - ops_arg_reduce(g_ResidualErrorHandle[macroVarIdx], 1, - "double", OPS_INC)); - } - } - for (int macroVarIdx = 0; macroVarIdx < MacroVarsNum(); macroVarIdx++) { - ops_reduction_result(g_ResidualErrorHandle[macroVarIdx], - (double*)&g_ResidualError[2 * macroVarIdx]); - } - for (int blockIdx = 0; blockIdx < BlockNum(); blockIdx++) { - int* iterRng = BlockIterRng(blockIdx, IterRngWhole()); - ops_par_loop(KerCopyMacroVars, "KerCopyMacroVars", g_Block[blockIdx], - SPACEDIM, iterRng, - ops_arg_dat(g_MacroVars[blockIdx], NUMMACROVAR, - LOCALSTENCIL, "double", OPS_READ), - ops_arg_dat(g_MacroVarsCopy[blockIdx], NUMMACROVAR, - LOCALSTENCIL, "double", OPS_RW)); - } - for (int macroVarIdx = 0; macroVarIdx < MacroVarsNum(); macroVarIdx++) { - for (int blockIdx = 0; blockIdx < BlockNum(); blockIdx++) { - int* iterRng = BlockIterRng(blockIdx, IterRngWhole()); - ops_par_loop(KerCalcMacroVarSquare, "KerCalcMacroVarSquare", - g_Block[blockIdx], SPACEDIM, iterRng, - ops_arg_dat(g_MacroVars[blockIdx], NUMMACROVAR, - LOCALSTENCIL, "double", OPS_READ), - ops_arg_gbl(¯oVarIdx, 1, "int", OPS_READ), - ops_arg_reduce(g_ResidualErrorHandle[macroVarIdx], 1, - "double", OPS_INC)); - } - } - for (int macroVarIdx = 0; macroVarIdx < MacroVarsNum(); macroVarIdx++) { - ops_reduction_result(g_ResidualErrorHandle[macroVarIdx], - (double*)&g_ResidualError[2 * macroVarIdx + 1]); - } -} - -void ForwardEuler() { - for (int blockIndex = 0; blockIndex < BlockNum(); blockIndex++) { - int* iterRng = BlockIterRng(blockIndex, IterRngWhole()); -//TODO finite difference scheme needs to be revised for the new colliison manner - // ops_par_loop(KerCutCellCVTUpwind2nd, "KerCutCellCVTUpwind2nd", - // g_Block[blockIndex], SPACEDIM, iterRng, - // ops_arg_dat(g_CoordinateXYZ[blockIndex], SPACEDIM, - // ONEPTREGULARSTENCIL, "double", OPS_READ), - // ops_arg_dat(g_NodeType[blockIndex], NUMCOMPONENTS, - // LOCALSTENCIL, "int", OPS_READ), - // ops_arg_dat(g_GeometryProperty[blockIndex], 1, - // LOCALSTENCIL, "int", OPS_READ), - // ops_arg_dat(g_f[blockIndex], NUMXI, ONEPTREGULARSTENCIL, - // "double", OPS_READ), - // ops_arg_dat(g_fStage[blockIndex], NUMXI, LOCALSTENCIL, - // "double", OPS_RW)); - Real schemeCoeff{1}; - // ops_par_loop(KerCutCellExplicitTimeMach, "KerCutCellExplicitTimeMach", - // g_Block[blockIndex], SPACEDIM, iterRng, - // ops_arg_gbl(pTimeStep(), 1, "double", OPS_READ), - // ops_arg_gbl(&schemeCoeff, 1, "double", OPS_READ), - // ops_arg_dat(g_NodeType[blockIndex], NUMCOMPONENTS, - // LOCALSTENCIL, "int", OPS_READ), - // ops_arg_dat(g_GeometryProperty[blockIndex], 1, - // LOCALSTENCIL, "int", OPS_READ), - // ops_arg_dat(g_fStage[blockIndex], NUMXI, LOCALSTENCIL, - // "double", OPS_READ), - // ops_arg_dat(g_feq[blockIndex], NUMXI, LOCALSTENCIL, - // "double", OPS_READ), - // ops_arg_dat(g_Tau[blockIndex], NUMCOMPONENTS, LOCALSTENCIL, - // "double", OPS_READ), - // ops_arg_dat(g_Bodyforce[blockIndex], NUMXI, LOCALSTENCIL, - // "double", OPS_READ), - // ops_arg_dat(g_f[blockIndex], NUMXI, LOCALSTENCIL, "double", - // OPS_RW)); - // } -} -void DispResidualError(const int iter, const SizeType checkPeriod) { - ops_printf("##########Residual Error at %i time step##########\n", iter); - for (int macroVarIdx = 0; macroVarIdx < MacroVarsNum(); macroVarIdx++) { - Real residualError = g_ResidualError[2 * macroVarIdx] / - g_ResidualError[2 * macroVarIdx + 1] / - (checkPeriod * TimeStep()); - ops_printf("%s = %.17g\n", MacroVarName()[macroVarIdx].c_str(), - residualError); - } -} -void Iterate(const SizeType steps, const SizeType checkPointPeriod) { +void Iterate(const SizeType steps, const SizeType checkPointPeriod, + const SizeType start) { const SchemeType scheme = Scheme(); ops_printf("Starting the iteration...\n"); switch (scheme) { case Scheme_StreamCollision: { - for (int iter = 0; iter < steps; iter++) { + for (SizeType iter = start; iter < start + steps; iter++) { + const Real time{iter * TimeStep()}; + StreamCollision(time); + if (((iter + 1) % checkPointPeriod) == 0) { + ops_printf("%d iterations!\n", iter + 1); +#ifdef OPS_3D + UpdateMacroVars3D(); +#endif #ifdef OPS_2D - StreamCollision(); // Stream-Collision scheme - // TimeMarching();//Finite difference scheme + cutting cell - if ((iter % checkPointPeriod) == 0 && iter != 0) { UpdateMacroVars(); - CalcResidualError(); - DispResidualError(iter, checkPointPeriod); - WriteFlowfieldToHdf5(iter); - WriteDistributionsToHdf5(iter); - WriteNodePropertyToHdf5(iter); +#endif + WriteFlowfieldToHdf5((iter + 1)); + WriteDistributionsToHdf5((iter + 1)); + WriteNodePropertyToHdf5((iter + 1)); } -#endif // end of OPS_2D } } break; default: @@ -423,33 +96,35 @@ void Iterate(const SizeType steps, const SizeType checkPointPeriod) { } ops_printf("Simulation finished! Exiting...\n"); DestroyModel(); - DestroyFlowfield(); + } -void Iterate(const Real convergenceCriteria, const SizeType checkPointPeriod) { +void Iterate(const Real convergenceCriteria, const SizeType checkPointPeriod, + const SizeType start) { const SchemeType scheme = Scheme(); ops_printf("Starting the iteration...\n"); switch (scheme) { case Scheme_StreamCollision: { - int iter{0}; + SizeType iter{start}; Real residualError{1}; do { + const Real time{iter * TimeStep()}; + StreamCollision(time); + iter = iter + 1; + if ((iter % checkPointPeriod) == 0) { +#ifdef OPS_3D + UpdateMacroVars3D(); +#endif #ifdef OPS_2D - StreamCollision(); // Stream-Collision scheme - // TimeMarching();//Finite difference scheme + cutting cell - if ((iter % checkPointPeriod) == 0 && iter != 0) { UpdateMacroVars(); +#endif CalcResidualError(); - residualError = - GetMaximumResidual(checkPointPeriod); + residualError = GetMaximumResidual(checkPointPeriod); DispResidualError(iter, checkPointPeriod); WriteFlowfieldToHdf5(iter); WriteDistributionsToHdf5(iter); WriteNodePropertyToHdf5(iter); } - -#endif // end of OPS_2D - iter = iter + 1; } while (residualError >= convergenceCriteria); } break; default: @@ -457,54 +132,63 @@ void Iterate(const Real convergenceCriteria, const SizeType checkPointPeriod) { } ops_printf("Simulation finished! Exiting...\n"); DestroyModel(); - DestroyFlowfield(); } -//TODO Shall we introduce debug information mechanism similar to 3D version? -void StreamCollision() { - +void StreamCollision(const Real time) { #if DebugLevel >= 1 ops_printf("Calculating the macroscopic variables...\n"); #endif +#ifdef OPS_3D + UpdateMacroVars3D(); +#endif +#ifdef OPS_2D UpdateMacroVars(); - CopyDistribution(g_f, g_fStage); - +#endif + CopyBlockEnvelopDistribution(g_fStage(), g_f()); +#if DebugLevel >= 1 + ops_printf("Calculating the mesoscopic body force term...\n"); +#endif + UpdateMacroscopicBodyForce(time); +#ifdef OPS_3D + PreDefinedBodyForce3D(); +#endif +#ifdef OPS_2D + PreDefinedBodyForce(); +#endif #if DebugLevel >= 1 - ops_printf("Calculating the equilibrium function and the body force term...\n"); + ops_printf("Calculating the collision term...\n"); +#endif +#ifdef OPS_3D + PreDefinedCollision3D(); +#endif +#ifdef OPS_2D + PreDefinedCollision(); #endif - UpdateFeqandBodyforce(); #if DebugLevel >= 1 - ops_printf("Colliding...\n"); + ops_printf("Updating the halos...\n"); #endif - Collision(); + TransferHalos(); #if DebugLevel >= 1 ops_printf("Streaming...\n"); #endif +#ifdef OPS_3D + Stream3D(); +#endif +#ifdef OPS_2D Stream(); - -#if DebugLevel >= 1 - ops_printf("Updating the halos...\n"); #endif - if (nullptr != HaloGroup()) { - ops_halo_transfer(HaloGroup()); - } #if DebugLevel >= 1 ops_printf("Implementing the boundary conditions...\n"); #endif - ImplementBoundaryConditions(); -} -// TODO Will implememt later. -// void TimeMarching() { -// UpdateMacroVars(); -// UpdateFeqandBodyforce(); -// UpdateTau(); -// ForwardEuler(); -// //ops_halo_transfer(HaloGroups); -// ImplementBoundary(); -// } +#ifdef OPS_3D + ImplementBoundary3D(); +#endif +#ifdef OPS_2D + ImplementBoundary(); +#endif +} -#endif /* OPS_2D */ \ No newline at end of file diff --git a/Src/evolution.h b/Src/evolution.h index cc1d5bd..f8a6089 100644 --- a/Src/evolution.h +++ b/Src/evolution.h @@ -28,21 +28,24 @@ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. -*/ + */ -/*! @brief Head file for wrap functions. +/*! @brief Head file for wrap functions. * @author Jianping Meng - * @details Declare wrap functions for implementing the main evolution cycle + * @details Declare wrap functions for implementing the main evolution + * cycle */ -#ifndef EVOLUTION_H_ -#define EVOLUTION_H_ -#include "boundary.h" -#include "flowfield.h" -#include "scheme.h" +#ifndef EVOLUTION3D_H_ +#define EVOLUTION3D_H_ +//#include "boundary.h" +//#include "flowfield.h" +//#include "model.h" +//#include "scheme.h" #include "type.h" +#include "field.h" +#include "flowfield.h" #include "model.h" -#include "hilemms.h" /*! * In this module, we provide subroutines that implement a update system over @@ -59,27 +62,69 @@ * modules. Instead, the kernel functions should be used. This mechanism * allows the subroutines to directly read variables such as g_f */ -// Routines for the stream-collision scheme. -void Stream(); -void Collision(); -void ImplementBoundary(); -void CalcResidualError(); + +// Stream-collision scheme related /*! - * Routine for completing one full time step + * Overall wrap for stream-collision scheme */ -void StreamCollision(); -// Routines for the general finite-difference scheme. -void UpdateBoundary(); -void TimeMarching(); -// Common routines -void InitialiseSolution(); -void UpdateMacroVars(); -void UpdateFeqandBodyforce(); -void UpdateHalos(); -void CopyDistribution(const ops_dat *fSrc, ops_dat *fDest); -void DispResidualError(const int iter, const SizeType timePeriod); -void NormaliseF(Real *ratio); -void TreatDomainBoundary(const int blockIndex, const int componentID, - const Real *givenVars, int *range, - const BoundaryScheme boundaryScheme); -#endif /* EVOLUTION_H_ */ +void StreamCollision(const Real time); + +void Iterate(const SizeType steps, const SizeType checkPointPeriod, + const SizeType start = 0); +void Iterate(const Real convergenceCriteria, const SizeType checkPointPeriod, + const SizeType start = 0); + +template +void Iterate(void (*cycle)(T), const SizeType steps, + const SizeType checkPointPeriod, const SizeType start = 0) { + ops_printf("Starting the iteration...\n"); + for (SizeType iter = start; iter < start + steps; iter++) { + const Real time{iter * TimeStep()}; + cycle(time); + if (((iter + 1) % checkPointPeriod) == 0) { + ops_printf("%d iterations!\n", iter + 1); +#ifdef OPS_3D + UpdateMacroVars3D(); +#endif +#ifdef OPS_2D + UpdateMacroVars(); +#endif + WriteFlowfieldToHdf5((iter + 1)); + WriteDistributionsToHdf5((iter + 1)); + WriteNodePropertyToHdf5((iter + 1)); + } + } + ops_printf("Simulation finished! Exiting...\n"); + DestroyModel(); +} + +template +void Iterate(void (*cycle)(T), const Real convergenceCriteria, + const SizeType checkPointPeriod, const SizeType start = 0) { + SizeType iter{start}; + Real residualError{1}; + do { + const Real time{iter * TimeStep()}; + cycle(time); + iter = iter + 1; + if ((iter % checkPointPeriod) == 0) { +#ifdef OPS_3D + UpdateMacroVars3D(); +#endif +#ifdef OPS_2D + UpdateMacroVars(); +#endif + CalcResidualError(); + residualError = GetMaximumResidual(checkPointPeriod); + DispResidualError(iter, checkPointPeriod); + WriteFlowfieldToHdf5(iter); + WriteDistributionsToHdf5(iter); + WriteNodePropertyToHdf5(iter); + } + } while (residualError >= convergenceCriteria); + + ops_printf("Simulation finished! Exiting...\n"); + DestroyModel(); +} + +#endif /* EVOLUTION3D_H_ */ diff --git a/Src/evolution3d.cpp b/Src/evolution3d.cpp deleted file mode 100644 index c2479f9..0000000 --- a/Src/evolution3d.cpp +++ /dev/null @@ -1,165 +0,0 @@ -/** - * Copyright 2019 United Kingdom Research and Innovation - * - * Authors: See AUTHORS - * - * Contact: [jianping.meng@stfc.ac.uk and/or jpmeng@gmail.com] - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice - * this list of conditions and the following disclaimer in the documentation - * and or other materials provided with the distribution. - * 3. Neither the name of the copyright holder nor the names of its contributors - * may be used to endorse or promote products derived from this software - * without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * ANDANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ - -/*! - * @brief Wrap functions for main evolution cycle. - * @author Jianping Meng - * @details Define wrap functions for implementing the main evolution - * cycle - */ -#include "evolution3d.h" -#include "type.h" -#include "scheme.h" -#include "block.h" -#include "field.h" -#include "boundary.h" -#include "flowfield.h" -#include "model.h" - -/* - * In the following routines, there are some variables are defined - * for the convenience of the translator which may not be able to - * understand a function parameter in the ops_par_loop call - * Even though, a variable rather than a numerical literacy will need - * some modifications in the Python translator. - */ -#ifdef OPS_3D - - - -// void TreatEmbeddedBoundary3D() { -// for (int blockIdx = 0; blockIdx < BlockNum(); blockIdx++) { -// int* iterRng.data() = BlockIterRng(blockIdx, IterRngBulk()); -// ops_par_loop( -// KerCutCellImmersedBoundary3D, "KerCutCellImmersedBoundary3D", -// g_Block[blockIdx], SpaceDim(), iterRng.data(), -// ops_arg_dat(g_NodeType()[blockIdx], 1, LOCALSTENCIL, "int", -// OPS_READ), ops_arg_dat(g_GeometryProperty[blockIdx], 1, -// LOCALSTENCIL, "int", -// OPS_READ), -// ops_arg_dat(g_f()[blockIdx], NUMXI, LOCALSTENCIL, "double", -// OPS_RW)); -// } -// } - - -void Iterate(const SizeType steps, const SizeType checkPointPeriod, - const SizeType start) { - const SchemeType scheme = Scheme(); - ops_printf("Starting the iteration...\n"); - switch (scheme) { - case Scheme_StreamCollision: { - for (SizeType iter = start; iter < start + steps; iter++) { - const Real time{iter * TimeStep()}; - StreamCollision3D(time); - if (((iter + 1) % checkPointPeriod) == 0) { - ops_printf("%d iterations!\n", iter + 1); - UpdateMacroVars3D(); - WriteFlowfieldToHdf5((iter + 1)); - WriteDistributionsToHdf5((iter + 1)); - WriteNodePropertyToHdf5((iter + 1)); - } - } - } break; - default: - break; - } - ops_printf("Simulation finished! Exiting...\n"); - DestroyModel(); - -} - -void Iterate(const Real convergenceCriteria, const SizeType checkPointPeriod, - const SizeType start) { - const SchemeType scheme = Scheme(); - ops_printf("Starting the iteration...\n"); - switch (scheme) { - case Scheme_StreamCollision: { - SizeType iter{start}; - Real residualError{1}; - do { - const Real time{iter * TimeStep()}; - StreamCollision3D(time); - iter = iter + 1; - if ((iter % checkPointPeriod) == 0) { - UpdateMacroVars3D(); - CalcResidualError3D(); - residualError = GetMaximumResidual(checkPointPeriod); - DispResidualError3D(iter, checkPointPeriod); - WriteFlowfieldToHdf5(iter); - WriteDistributionsToHdf5(iter); - WriteNodePropertyToHdf5(iter); - } - } while (residualError >= convergenceCriteria); - } break; - default: - break; - } - ops_printf("Simulation finished! Exiting...\n"); - DestroyModel(); -} - -void StreamCollision3D(const Real time) { -#if DebugLevel >= 1 - ops_printf("Calculating the macroscopic variables...\n"); -#endif - UpdateMacroVars3D(); - CopyBlockEnvelopDistribution3D(g_fStage(), g_f()); -#if DebugLevel >= 1 - ops_printf("Calculating the mesoscopic body force term...\n"); -#endif - UpdateMacroscopicBodyForce(time); - PreDefinedBodyForce3D(); -#if DebugLevel >= 1 - ops_printf("Calculating the collision term...\n"); -#endif - PreDefinedCollision3D(); - -#if DebugLevel >= 1 - ops_printf("Updating the halos...\n"); -#endif - TransferHalos(); - -#if DebugLevel >= 1 - ops_printf("Streaming...\n"); -#endif - Stream3D(); - -#if DebugLevel >= 1 - ops_printf("Implementing the boundary conditions...\n"); -#endif - // TODO This function shall be inside evolution3D.cpp - // TODO The data structure for BCs shall be inside boundary module - ImplementBoundary3D(); -} - -#endif /* OPS_3D */ diff --git a/Src/evolution_ToBeDeleted.cpp b/Src/evolution_ToBeDeleted.cpp new file mode 100644 index 0000000..02ee84f --- /dev/null +++ b/Src/evolution_ToBeDeleted.cpp @@ -0,0 +1,510 @@ +/** + * Copyright 2019 United Kingdom Research and Innovation + * + * Authors: See AUTHORS + * + * Contact: [jianping.meng@stfc.ac.uk and/or jpmeng@gmail.com] + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice + * this list of conditions and the following disclaimer in the documentation + * and or other materials provided with the distribution. + * 3. Neither the name of the copyright holder nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * ANDANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. +*/ + +/*! @brief Wrap functions for main evolution cycle. + * @author Jianping Meng + * @details Define wrap functions for implementing the main evolution + * cycle + */ + +#include "evolution.h" +/* + * In the following routines, there are some variables are defined + * for the convenience of the translator which may not be able to + * understand a function parameter in the ops_par_loop call + * Even though, a variable rather than a numerical literacy will need + * some modifications in the Python translator. + */ + +#ifdef OPS_2D + +void Collision() { + for (int blockIndex = 0; blockIndex < BlockNum(); blockIndex++) { + int* iterRng = BlockIterRng(blockIndex, IterRngWhole()); + ops_par_loop(KerCollide, "KerCollide", g_Block[blockIndex], SPACEDIM, + iterRng, ops_arg_gbl(pTimeStep(), 1, "double", OPS_READ), + ops_arg_dat(g_NodeType[blockIndex], NUMCOMPONENTS, + LOCALSTENCIL, "int", OPS_READ), + ops_arg_dat(g_f[blockIndex], NUMXI, LOCALSTENCIL, "double", + OPS_READ), + ops_arg_dat(g_feq[blockIndex], NUMXI, LOCALSTENCIL, + "double", OPS_READ), + ops_arg_dat(g_Tau[blockIndex], NUMCOMPONENTS, LOCALSTENCIL, + "double", OPS_READ), + ops_arg_dat(g_Bodyforce[blockIndex], NUMXI, LOCALSTENCIL, + "double", OPS_READ), + ops_arg_dat(g_fStage[blockIndex], NUMXI, LOCALSTENCIL, + "double", OPS_WRITE)); + } +} + +void Stream() { + for (int blockIndex = 0; blockIndex < BlockNum(); blockIndex++) { + int* iterRng = BlockIterRng(blockIndex, IterRngWhole()); + ops_par_loop( + KerStream, "KerStream", g_Block[blockIndex], SPACEDIM, iterRng, + ops_arg_dat(g_f[blockIndex], NUMXI, LOCALSTENCIL, "double", OPS_RW), + ops_arg_dat(g_fStage[blockIndex], NUMXI, ONEPTLATTICESTENCIL, + "double", OPS_READ) + ops_arg_dat(g_NodeType[blockIndex], NUMCOMPONENTS, LOCALSTENCIL, + "int", OPS_READ), + ops_arg_dat(g_GeometryProperty[blockIndex], 1, LOCALSTENCIL, "int", + OPS_READ)); + } +} + +void UpdateMacroVars() { + for (int blockIndex = 0; blockIndex < BlockNum(); blockIndex++) { + int* iterRng = BlockIterRng(blockIndex, IterRngWhole()); + ops_par_loop(KerCalcMacroVars, "KerCalcMacroVars", g_Block[blockIndex], + SPACEDIM, iterRng, + ops_arg_dat(g_NodeType[blockIndex], NUMCOMPONENTS, + LOCALSTENCIL, "int", OPS_READ), + ops_arg_dat(g_f[blockIndex], NUMXI, LOCALSTENCIL, "double", + OPS_READ), + ops_arg_dat(g_MacroVars[blockIndex], NUMMACROVAR, + LOCALSTENCIL, "double", OPS_RW)); + } +} + +void UpdateFeqandBodyforce() { + for (int blockIndex = 0; blockIndex < BlockNum(); blockIndex++) { + int* iterRng = BlockIterRng(blockIndex, IterRngWhole()); + ops_par_loop(KerCalcFeq, "KerCalcPolyFeq", g_Block[blockIndex], + SPACEDIM, iterRng, + ops_arg_dat(g_NodeType[blockIndex], NUMCOMPONENTS, + LOCALSTENCIL, "int", OPS_READ), + ops_arg_dat(g_MacroVars[blockIndex], NUMMACROVAR, + LOCALSTENCIL, "double", OPS_READ), + ops_arg_dat(g_feq[blockIndex], NUMXI, LOCALSTENCIL, + "double", OPS_RW)); + // force term to be added + + // time is not used in the current force + Real* timeF{0}; + ops_par_loop(KerCalcBodyForce, "KerCalcBodyForce", + g_Block[blockIndex], SPACEDIM, iterRng, + ops_arg_gbl(&timeF, 1, "double", OPS_READ), + ops_arg_dat(g_NodeType[blockIndex], NUMCOMPONENTS, + LOCALSTENCIL, "int", OPS_READ), + ops_arg_dat(g_CoordinateXYZ[blockIndex], SPACEDIM, + LOCALSTENCIL, "double", OPS_READ), + ops_arg_dat(g_MacroVars[blockIndex], NUMMACROVAR, + LOCALSTENCIL, "double", OPS_READ), + ops_arg_dat(g_Bodyforce[blockIndex], NUMXI, LOCALSTENCIL, + "double", OPS_RW)); + } +} + +void TreatDomainBoundary(const int blockIndex, const int componentID, + const Real* givenVars, int* range, + const BoundaryType boundaryScheme) +{ + switch (boundaryScheme) { + case BoundaryScheme::ExtrapolPressure1ST: { + ops_par_loop( + KerCutCellExtrapolPressure1ST, "KerCutCellExtrapolPressure1ST", + g_Block[blockIndex], SPACEDIM, range, + ops_arg_gbl(givenVars, NUMMACROVAR, "double", OPS_READ), + ops_arg_dat(g_NodeType[blockIndex], NUMCOMPONENTS, + ONEPTREGULARSTENCIL, "int", OPS_READ), + ops_arg_dat(g_GeometryProperty[blockIndex], 1, LOCALSTENCIL, + "int", OPS_READ), + ops_arg_dat(g_f[blockIndex], NUMXI, ONEPTREGULARSTENCIL, "double", + OPS_RW)); + } break; + case BoundaryType__ExtrapolPressure2ND: { + ops_par_loop( + KerCutCellExtrapolPressure2ND, + "KerCutCellExtrapolPressure2ND", g_Block[blockIndex], + SPACEDIM, range, + ops_arg_gbl(givenVars, NUMMACROVAR, "double", OPS_READ), + ops_arg_dat(g_NodeType[blockIndex], 1, ONEPTREGULARSTENCIL, + "int", OPS_READ), + ops_arg_dat(g_GeometryProperty[blockIndex], 1, LOCALSTENCIL, + "int", OPS_READ), + ops_arg_dat(g_f[blockIndex], NUMXI, TWOPTREGULARSTENCIL, + "double", OPS_RW)); + } break; + case BoundaryType__BounceBackWall: { + ops_par_loop(KerCutCellBounceBack, "KerCutCellBounceBack", + g_Block[blockIndex], SPACEDIM, range, + ops_arg_dat(g_NodeType[blockIndex], 1, LOCALSTENCIL, + "int", OPS_READ), + ops_arg_dat(g_GeometryProperty[blockIndex], 1, + LOCALSTENCIL, "int", OPS_READ), + ops_arg_dat(g_f[blockIndex], NUMXI, LOCALSTENCIL, + "double", OPS_RW)); + } break; + case BoundaryType__ZouHeVelocity: { + ops_par_loop( + KerCutCellZouHeVelocity, "KerCutCellZouHeVelocity,", + g_Block[blockIndex], SPACEDIM, range, + ops_arg_gbl(givenVars, NUMMACROVAR, "double", OPS_READ), + ops_arg_dat(g_NodeType[blockIndex], 1, LOCALSTENCIL, "int", + OPS_READ), + ops_arg_dat(g_GeometryProperty[blockIndex], 1, LOCALSTENCIL, + "int", OPS_READ), + ops_arg_dat(g_MacroVars[blockIndex], NUMMACROVAR, + ONEPTLATTICESTENCIL, "double", OPS_READ), + ops_arg_dat(g_f[blockIndex], NUMXI, ONEPTLATTICESTENCIL, + "double", OPS_RW)); + } break; + case BoundaryType__KineticDiffuseWall: { + // ops_par_loop( + // KerCutCellKinetic, "KerCutCellKinetic", g_Block[blockIndex], + // SPACEDIM, range, + // ops_arg_gbl(givenVars, NUMMACROVAR, "double", OPS_READ), + // ops_arg_dat(g_NodeType[blockIndex], 1, LOCALSTENCIL, "int", + // OPS_READ), + // ops_arg_dat(g_GeometryProperty[blockIndex], 1, LOCALSTENCIL, + // "int", OPS_READ), + // ops_arg_dat(g_f[blockIndex], NUMXI, LOCALSTENCIL, "double", + // OPS_RW)); + } + case BoundaryType__EQMDiffuseRefl: { + ops_par_loop( + KerCutCellEQMDiffuseRefl, "KerCutCellEQMDiffuseRefl", + g_Block[blockIndex], SPACEDIM, range, + ops_arg_gbl(givenVars, NUMMACROVAR, "double", OPS_READ), + ops_arg_dat(g_NodeType[blockIndex], NUMCOMPONENTS, LOCALSTENCIL, + "int", OPS_READ), + ops_arg_dat(g_GeometryProperty[blockIndex], 1, LOCALSTENCIL, + "int", OPS_READ), + ops_arg_dat(g_f[blockIndex], NUMXI, LOCALSTENCIL, "double", + OPS_RW), + ops_arg_gbl(&componentID, 1, "int", OPS_READ)); + } break; + case BoundaryType__FreeFlux: { + ops_par_loop(KerCutCellZeroFlux, "KerCutCellZeroFlux", + g_Block[blockIndex], SPACEDIM, range, + ops_arg_dat(g_NodeType[blockIndex], 1, LOCALSTENCIL, + "int", OPS_READ), + ops_arg_dat(g_GeometryProperty[blockIndex], 1, + LOCALSTENCIL, "int", OPS_READ), + ops_arg_dat(g_f[blockIndex], NUMXI, LOCALSTENCIL, + "double", OPS_RW)); + } break; + case BoundaryType__Periodic: { + ops_par_loop(KerCutCellPeriodic, "KerCutCellPeriodic", + g_Block[blockIndex], SPACEDIM, range, + ops_arg_dat(g_NodeType[blockIndex], NUMCOMPONENTS, + LOCALSTENCIL, "int", OPS_READ), + ops_arg_dat(g_GeometryProperty[blockIndex], 1, + LOCALSTENCIL, "int", OPS_READ), + ops_arg_dat(g_f[blockIndex], NUMXI, LOCALSTENCIL, + "double", OPS_RW)); + } break; + default: + break; + } +} + +void ImplementBoundary() { + for (auto boundary : BlockBoundaries()) { + int* range{BoundarySurfaceRange(boundary.blockIndex, + boundary.boundarySurface)}; + TreatBlockBoundary(boundary.blockIndex, boundary.componentID, + boundary.givenVars.data(), range, + boundary.boundaryScheme, boundary.boundarySurface); + } +} + + +void TreatEmbeddedBoundary() { + for (int blockIdx = 0; blockIdx < BlockNum(); blockIdx++) { + int* iterRng = BlockIterRng(blockIdx, IterRngBulk()); + ops_par_loop( + KerCutCellEmbeddedBoundary, "KerCutCellImmersedBoundary", + g_Block[blockIdx], SPACEDIM, iterRng, + ops_arg_dat(g_NodeType[blockIdx], NUMCOMPONENTS, LOCALSTENCIL, + "int", OPS_READ), + ops_arg_dat(g_GeometryProperty[blockIdx], 1, LOCALSTENCIL, "int", + OPS_READ), + ops_arg_dat(g_f[blockIdx], NUMXI, LOCALSTENCIL, "double", OPS_RW)); + } +} +//TODO This function needs to be improved for different initialisation scheme +void InitialiseSolution() { + UpdateFeqandBodyforce(); + for (int blockIndex = 0; blockIndex < BlockNum(); blockIndex++) { + int* iterRng = BlockIterRng(blockIndex, IterRngWhole()); + const Real zero = 0; + ops_par_loop( + KerSetfFixValue, "KerSetfFixValue", g_Block[blockIndex], SPACEDIM, + iterRng, ops_arg_gbl(&zero, 1, "double", OPS_READ), + ops_arg_dat(g_Bodyforce[0], NUMXI, LOCALSTENCIL, "double", OPS_RW)); + } + CopyDistribution(g_feq, g_f); +} + +void CopyDistribution(const ops_dat* fSrc, ops_dat* fDest) { + for (int blockIndex = 0; blockIndex < BlockNum(); blockIndex++) { + int* iterRng = BlockIterRng(blockIndex, IterRngWhole()); + ops_par_loop(KerCopyf, "KerCopyf", g_Block[blockIndex], SPACEDIM, + iterRng, + ops_arg_dat(fSrc[blockIndex], NUMXI, LOCALSTENCIL, + "double", OPS_READ), + ops_arg_dat(fDest[blockIndex], NUMXI, LOCALSTENCIL, + "double", OPS_WRITE)); + } +} + +void CalcTotalMass(double* totalMass) { + ops_reduction massHandle = + ops_decl_reduction_handle(sizeof(double), "double", "massHandle"); + for (int blockIdx = 0; blockIdx < BlockNum(); blockIdx++) { + int* iterRng = BlockIterRng(blockIdx, IterRngWhole()); + ops_par_loop(KerCalcSumofDensity, "KerCalcSumofDensity", + g_Block[blockIdx], SPACEDIM, iterRng, + ops_arg_dat(g_MacroVars[blockIdx], NUMMACROVAR, + LOCALSTENCIL, "double", OPS_READ), + ops_arg_reduce(massHandle, 1, "double", OPS_INC)); + } + ops_reduction_result(massHandle, (double*)totalMass); +} + +void NormaliseF(Real* ratio) { + for (int blockIdx = 0; blockIdx < BlockNum(); blockIdx++) { + int* iterRng = BlockIterRng(blockIdx, IterRngWhole()); + ops_par_loop( + KerNormaliseF, "KerNormaliseF", g_Block[blockIdx], SPACEDIM, + iterRng, ops_arg_gbl(ratio, 1, "double", OPS_READ), + ops_arg_dat(g_f[blockIdx], NUMXI, LOCALSTENCIL, "double", OPS_RW)); + } +} + +void CalcResidualError() { + for (int macroVarIdx = 0; macroVarIdx < MacroVarsNum(); macroVarIdx++) { + for (int blockIdx = 0; blockIdx < BlockNum(); blockIdx++) { + int* iterRng = BlockIterRng(blockIdx, IterRngWhole()); + ops_par_loop(KerCalcMacroVarSquareofDifference, + "KerCalcMacroVarSquareofDifference", g_Block[blockIdx], + SPACEDIM, iterRng, + ops_arg_dat(g_MacroVars[blockIdx], NUMMACROVAR, + LOCALSTENCIL, "double", OPS_READ), + ops_arg_dat(g_MacroVarsCopy[blockIdx], NUMMACROVAR, + LOCALSTENCIL, "double", OPS_READ), + ops_arg_gbl(¯oVarIdx, 1, "int", OPS_READ), + ops_arg_reduce(g_ResidualErrorHandle[macroVarIdx], 1, + "double", OPS_INC)); + } + } + for (int macroVarIdx = 0; macroVarIdx < MacroVarsNum(); macroVarIdx++) { + ops_reduction_result(g_ResidualErrorHandle[macroVarIdx], + (double*)&g_ResidualError[2 * macroVarIdx]); + } + for (int blockIdx = 0; blockIdx < BlockNum(); blockIdx++) { + int* iterRng = BlockIterRng(blockIdx, IterRngWhole()); + ops_par_loop(KerCopyMacroVars, "KerCopyMacroVars", g_Block[blockIdx], + SPACEDIM, iterRng, + ops_arg_dat(g_MacroVars[blockIdx], NUMMACROVAR, + LOCALSTENCIL, "double", OPS_READ), + ops_arg_dat(g_MacroVarsCopy[blockIdx], NUMMACROVAR, + LOCALSTENCIL, "double", OPS_RW)); + } + for (int macroVarIdx = 0; macroVarIdx < MacroVarsNum(); macroVarIdx++) { + for (int blockIdx = 0; blockIdx < BlockNum(); blockIdx++) { + int* iterRng = BlockIterRng(blockIdx, IterRngWhole()); + ops_par_loop(KerCalcMacroVarSquare, "KerCalcMacroVarSquare", + g_Block[blockIdx], SPACEDIM, iterRng, + ops_arg_dat(g_MacroVars[blockIdx], NUMMACROVAR, + LOCALSTENCIL, "double", OPS_READ), + ops_arg_gbl(¯oVarIdx, 1, "int", OPS_READ), + ops_arg_reduce(g_ResidualErrorHandle[macroVarIdx], 1, + "double", OPS_INC)); + } + } + for (int macroVarIdx = 0; macroVarIdx < MacroVarsNum(); macroVarIdx++) { + ops_reduction_result(g_ResidualErrorHandle[macroVarIdx], + (double*)&g_ResidualError[2 * macroVarIdx + 1]); + } +} + +void ForwardEuler() { + for (int blockIndex = 0; blockIndex < BlockNum(); blockIndex++) { + int* iterRng = BlockIterRng(blockIndex, IterRngWhole()); +//TODO finite difference scheme needs to be revised for the new colliison manner + // ops_par_loop(KerCutCellCVTUpwind2nd, "KerCutCellCVTUpwind2nd", + // g_Block[blockIndex], SPACEDIM, iterRng, + // ops_arg_dat(g_CoordinateXYZ[blockIndex], SPACEDIM, + // ONEPTREGULARSTENCIL, "double", OPS_READ), + // ops_arg_dat(g_NodeType[blockIndex], NUMCOMPONENTS, + // LOCALSTENCIL, "int", OPS_READ), + // ops_arg_dat(g_GeometryProperty[blockIndex], 1, + // LOCALSTENCIL, "int", OPS_READ), + // ops_arg_dat(g_f[blockIndex], NUMXI, ONEPTREGULARSTENCIL, + // "double", OPS_READ), + // ops_arg_dat(g_fStage[blockIndex], NUMXI, LOCALSTENCIL, + // "double", OPS_RW)); + Real schemeCoeff{1}; + // ops_par_loop(KerCutCellExplicitTimeMach, "KerCutCellExplicitTimeMach", + // g_Block[blockIndex], SPACEDIM, iterRng, + // ops_arg_gbl(pTimeStep(), 1, "double", OPS_READ), + // ops_arg_gbl(&schemeCoeff, 1, "double", OPS_READ), + // ops_arg_dat(g_NodeType[blockIndex], NUMCOMPONENTS, + // LOCALSTENCIL, "int", OPS_READ), + // ops_arg_dat(g_GeometryProperty[blockIndex], 1, + // LOCALSTENCIL, "int", OPS_READ), + // ops_arg_dat(g_fStage[blockIndex], NUMXI, LOCALSTENCIL, + // "double", OPS_READ), + // ops_arg_dat(g_feq[blockIndex], NUMXI, LOCALSTENCIL, + // "double", OPS_READ), + // ops_arg_dat(g_Tau[blockIndex], NUMCOMPONENTS, LOCALSTENCIL, + // "double", OPS_READ), + // ops_arg_dat(g_Bodyforce[blockIndex], NUMXI, LOCALSTENCIL, + // "double", OPS_READ), + // ops_arg_dat(g_f[blockIndex], NUMXI, LOCALSTENCIL, "double", + // OPS_RW)); + // } +} + +void DispResidualError(const int iter, const SizeType checkPeriod) { + ops_printf("##########Residual Error at %i time step##########\n", iter); + for (int macroVarIdx = 0; macroVarIdx < MacroVarsNum(); macroVarIdx++) { + Real residualError = g_ResidualError[2 * macroVarIdx] / + g_ResidualError[2 * macroVarIdx + 1] / + (checkPeriod * TimeStep()); + ops_printf("%s = %.17g\n", MacroVarName()[macroVarIdx].c_str(), + residualError); + } +} +void Iterate(const SizeType steps, const SizeType checkPointPeriod) { + const SchemeType scheme = Scheme(); + ops_printf("Starting the iteration...\n"); + switch (scheme) { + case Scheme_StreamCollision: { + for (int iter = 0; iter < steps; iter++) { +#ifdef OPS_2D + StreamCollision(); // Stream-Collision scheme + // TimeMarching();//Finite difference scheme + cutting cell + if ((iter % checkPointPeriod) == 0 && iter != 0) { + UpdateMacroVars(); + CalcResidualError(); + DispResidualError(iter, checkPointPeriod); + WriteFlowfieldToHdf5(iter); + WriteDistributionsToHdf5(iter); + WriteNodePropertyToHdf5(iter); + } +#endif // end of OPS_2D + } + } break; + default: + break; + } + ops_printf("Simulation finished! Exiting...\n"); + DestroyModel(); + DestroyFlowfield(); +} + +void Iterate(const Real convergenceCriteria, const SizeType checkPointPeriod) { + const SchemeType scheme = Scheme(); + ops_printf("Starting the iteration...\n"); + switch (scheme) { + case Scheme_StreamCollision: { + int iter{0}; + Real residualError{1}; + do { +#ifdef OPS_2D + StreamCollision(); // Stream-Collision scheme + // TimeMarching();//Finite difference scheme + cutting cell + if ((iter % checkPointPeriod) == 0 && iter != 0) { + UpdateMacroVars(); + CalcResidualError(); + residualError = + GetMaximumResidual(checkPointPeriod); + DispResidualError(iter, checkPointPeriod); + WriteFlowfieldToHdf5(iter); + WriteDistributionsToHdf5(iter); + WriteNodePropertyToHdf5(iter); + } + +#endif // end of OPS_2D + iter = iter + 1; + } while (residualError >= convergenceCriteria); + } break; + default: + break; + } + ops_printf("Simulation finished! Exiting...\n"); + DestroyModel(); + DestroyFlowfield(); +} + +//TODO Shall we introduce debug information mechanism similar to 3D version? +void StreamCollision() { + +#if DebugLevel >= 1 + ops_printf("Calculating the macroscopic variables...\n"); +#endif + UpdateMacroVars(); + CopyDistribution(g_f, g_fStage); + +#if DebugLevel >= 1 + ops_printf("Calculating the equilibrium function and the body force term...\n"); +#endif + UpdateFeqandBodyforce(); + +#if DebugLevel >= 1 + ops_printf("Colliding...\n"); +#endif + Collision(); + +#if DebugLevel >= 1 + ops_printf("Streaming...\n"); +#endif + Stream(); + +#if DebugLevel >= 1 + ops_printf("Updating the halos...\n"); +#endif + if (nullptr != HaloGroup()) { + ops_halo_transfer(HaloGroup()); + } + +#if DebugLevel >= 1 + ops_printf("Implementing the boundary conditions...\n"); +#endif + ImplementBoundaryConditions(); +} + +// TODO Will implememt later. +// void TimeMarching() { +// UpdateMacroVars(); +// UpdateFeqandBodyforce(); +// UpdateTau(); +// ForwardEuler(); +// //ops_halo_transfer(HaloGroups); +// ImplementBoundary(); +// } + +#endif /* OPS_2D */ \ No newline at end of file diff --git a/Src/evolution3d.h b/Src/evolution_ToBeDeleted.h similarity index 54% rename from Src/evolution3d.h rename to Src/evolution_ToBeDeleted.h index c888dec..cc1d5bd 100644 --- a/Src/evolution3d.h +++ b/Src/evolution_ToBeDeleted.h @@ -28,24 +28,21 @@ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. - */ +*/ -/*! @brief Head file for wrap functions. +/*! @brief Head file for wrap functions. * @author Jianping Meng - * @details Declare wrap functions for implementing the main evolution - * cycle + * @details Declare wrap functions for implementing the main evolution cycle */ -#ifndef EVOLUTION3D_H_ -#define EVOLUTION3D_H_ -//#include "boundary.h" -//#include "flowfield.h" -//#include "model.h" -//#include "scheme.h" -#include "type.h" -#include "field.h" +#ifndef EVOLUTION_H_ +#define EVOLUTION_H_ +#include "boundary.h" #include "flowfield.h" +#include "scheme.h" +#include "type.h" #include "model.h" +#include "hilemms.h" /*! * In this module, we provide subroutines that implement a update system over @@ -62,62 +59,27 @@ * modules. Instead, the kernel functions should be used. This mechanism * allows the subroutines to directly read variables such as g_f */ - -#ifdef OPS_3D -// Stream-collision scheme related +// Routines for the stream-collision scheme. +void Stream(); +void Collision(); +void ImplementBoundary(); +void CalcResidualError(); /*! - * Overall wrap for stream-collision scheme + * Routine for completing one full time step */ -void StreamCollision3D(const Real time); - - -void Iterate(const SizeType steps, const SizeType checkPointPeriod, - const SizeType start = 0); -void Iterate(const Real convergenceCriteria, const SizeType checkPointPeriod, - const SizeType start = 0); - -template -void Iterate(void (*cycle)(T), const SizeType steps, - const SizeType checkPointPeriod, const SizeType start = 0) { - ops_printf("Starting the iteration...\n"); - for (SizeType iter = start; iter < start + steps; iter++) { - const Real time{iter * TimeStep()}; - cycle(time); - if (((iter + 1) % checkPointPeriod) == 0) { - ops_printf("%d iterations!\n", iter + 1); - UpdateMacroVars3D(); - WriteFlowfieldToHdf5((iter + 1)); - WriteDistributionsToHdf5((iter + 1)); - WriteNodePropertyToHdf5((iter + 1)); - } - } - ops_printf("Simulation finished! Exiting...\n"); - DestroyModel(); -} - -template -void Iterate(void (*cycle)(T), const Real convergenceCriteria, - const SizeType checkPointPeriod, const SizeType start = 0) { - SizeType iter{start}; - Real residualError{1}; - do { - const Real time{iter * TimeStep()}; - cycle(time); - iter = iter + 1; - if ((iter % checkPointPeriod) == 0) { - UpdateMacroVars3D(); - CalcResidualError3D(); - residualError = GetMaximumResidual(checkPointPeriod); - DispResidualError3D(iter, checkPointPeriod); - WriteFlowfieldToHdf5(iter); - WriteDistributionsToHdf5(iter); - WriteNodePropertyToHdf5(iter); - } - } while (residualError >= convergenceCriteria); - - ops_printf("Simulation finished! Exiting...\n"); - DestroyModel(); -} - -#endif /* OPS_3D */ -#endif /* EVOLUTION3D_H_ */ +void StreamCollision(); +// Routines for the general finite-difference scheme. +void UpdateBoundary(); +void TimeMarching(); +// Common routines +void InitialiseSolution(); +void UpdateMacroVars(); +void UpdateFeqandBodyforce(); +void UpdateHalos(); +void CopyDistribution(const ops_dat *fSrc, ops_dat *fDest); +void DispResidualError(const int iter, const SizeType timePeriod); +void NormaliseF(Real *ratio); +void TreatDomainBoundary(const int blockIndex, const int componentID, + const Real *givenVars, int *range, + const BoundaryScheme boundaryScheme); +#endif /* EVOLUTION_H_ */ diff --git a/Src/field.h b/Src/field.h index 69175e9..27f0767 100644 --- a/Src/field.h +++ b/Src/field.h @@ -133,7 +133,7 @@ Field::Field(const char* varName, const int dataDim, template void Field::CreateFieldFromScratch(const Block& block) { - T* temp = NULL; + T* temp{nullptr}; int* d_p = new int[spaceDim]; int* d_m = new int[spaceDim]; int* base = new int[spaceDim]; @@ -146,7 +146,7 @@ void Field::CreateFieldFromScratch(const Block& block) { std::string dataName{name + "_" + block.Name()}; std::vector size{block.Size()}; ops_dat localDat = - ops_decl_dat((ops_block)block.Get(), dim, size.data(), base, + ops_decl_dat(block.Get(), dim, size.data(), base, d_m, d_p, temp, type.c_str(), dataName.c_str()); data.emplace(blockId, localDat); dataBlock.emplace(blockId, block); @@ -229,7 +229,7 @@ void Field::CreateHalos() { #ifdef OPS_2D const int d_p[]{haloDepth, haloDepth}; const int d_m[]{-haloDepth, -haloDepth}; - const int dir[] { 1, 2 } + int dir[]{1, 2}; #endif for (const auto& surfaceNeighbor : block.Neighbors()) { const Neighbor& neighbor{surfaceNeighbor.second}; diff --git a/Src/flowfield.cpp b/Src/flowfield.cpp index 9f0833d..a6d86e1 100644 --- a/Src/flowfield.cpp +++ b/Src/flowfield.cpp @@ -81,6 +81,9 @@ RealFieldGroup& g_MacroBodyforce() { return MacroBodyforce; }; * DT: time step */ Real DT{1}; +Real DX{1}; + + RealField CoordinateXYZ{"CoordinateXYZ"}; RealField& g_CoordinateXYZ() { return CoordinateXYZ; }; @@ -137,7 +140,7 @@ const std::string& CaseName() { return CASENAME; } Real TotalMeshSize() { return 0; } - +Real GetDx() { return DX;} Real TimeStep() { return DT; } const Real* pTimeStep() { return &DT; } void SetTimeStep(Real dt) { DT = dt; } @@ -241,7 +244,7 @@ void PrepareFlowField() { SetBoundaryNodeType(); } -void DispResidualError3D(const int iter, const SizeType checkPeriod) { +void DispResidualError(const int iter, const SizeType checkPeriod) { ops_printf("##########Residual Error at %i time step##########\n", iter); for (auto& compo : g_Components()) { for (auto& macroVar : compo.second.macroVars) { diff --git a/Src/flowfield.h b/Src/flowfield.h index a159bcc..74835ea 100644 --- a/Src/flowfield.h +++ b/Src/flowfield.h @@ -68,6 +68,7 @@ IntFieldGroup& g_NodeType(); IntField& g_GeometryProperty(); Real TimeStep(); const Real* pTimeStep(); +Real GetDx(); const std::string& CaseName(); Real TotalMeshSize(); const std::map& HaloGroups(); @@ -107,14 +108,12 @@ void DefineBlocks(const std::vector& blockIds, const std::vector& blockSizes, const Real meshSize, const std::map>& startPos); bool IsTransient(); -#ifdef OPS_3D -void CalcResidualError3D(); -void DispResidualError3D(const int iter, const SizeType checkPeriod); -void CopyDistribution3D(RealField& fDest, RealField& fSrc); -void CopyBlockEnvelopDistribution3D(Field& fDest, Field& fSrc); -void NormaliseF3D(Real* ratio); -#endif // OPS_3D +void CalcResidualError(); +void DispResidualError(const int iter, const SizeType checkPeriod); +void CopyDistribution(RealField& fDest, RealField& fSrc); +void CopyBlockEnvelopDistribution(Field& fDest, Field& fSrc); +void NormaliseF(Real* ratio); void CopyCurrentMacroVar(); void SetBulkandHaloNodesType(const Block& block, int compoId); void SetBoundaryNodeType(); diff --git a/Src/flowfield_kernel.inc b/Src/flowfield_kernel.inc index 0649132..0c1f4d8 100644 --- a/Src/flowfield_kernel.inc +++ b/Src/flowfield_kernel.inc @@ -102,9 +102,10 @@ void KerCopyCoordinateXYZ(const ACC& src, ACC& dest) { #ifdef OPS_3D for (int idx = 0; idx < 3; idx++) { dest(idx, 0, 0, 0) = src(idx, 0, 0, 0); -#endif } +#endif } + void KerCopyDispf(const ACC& src, ACC& dest, const int* disp) { for (int xiIndex = 0; xiIndex < NUMXI; xiIndex++) { #ifdef OPS_2D diff --git a/Src/flowfield_wrapper.cpp b/Src/flowfield_wrapper.cpp index 35ee16c..5e86877 100644 --- a/Src/flowfield_wrapper.cpp +++ b/Src/flowfield_wrapper.cpp @@ -28,7 +28,7 @@ void CopyCurrentMacroVar() { } } -void CalcResidualError3D() { +void CalcResidualError() { std::map diff; for (const auto& pair : g_MacroVars()) { const int varId{pair.first}; @@ -88,7 +88,7 @@ void CalcResidualError3D() { } } -void CopyDistribution3D(RealField& fDest, RealField& fSrc) { +void CopyDistribution(RealField& fDest, RealField& fSrc) { for (const auto& idBlock : g_Block()) { const Block& block{idBlock.second}; std::vector iterRng; @@ -110,7 +110,7 @@ void CopyDistribution3D(RealField& fDest, RealField& fSrc) { // which needs the information at halo points. // The routine shall be removed if the stream process can be implemented in a // way that f_stage is not necessary. -void CopyBlockEnvelopDistribution3D(Field& fDest, Field& fSrc) { +void CopyBlockEnvelopDistribution(Field& fDest, Field& fSrc) { // int haloIterRng[]{0, 0, 0, 0, 0, 0}; for (const auto& idBlock : g_Block()) { const Block& block{idBlock.second}; @@ -190,6 +190,7 @@ void CopyBlockEnvelopDistribution3D(Field& fDest, Field& fSrc) { "double", OPS_WRITE), ops_arg_dat(fSrc[blockIndex], NUMXI, LOCALSTENCIL, "double", OPS_READ)); +#ifdef OPS_3D iterRng.assign( block.BoundarySurfaceRange().at(BoundarySurface::Back).begin(), block.BoundarySurfaceRange().at(BoundarySurface::Back).end()); @@ -226,10 +227,11 @@ void CopyBlockEnvelopDistribution3D(Field& fDest, Field& fSrc) { "double", OPS_WRITE), ops_arg_dat(fSrc[blockIndex], NUMXI, LOCALSTENCIL, "double", OPS_READ)); +#endif // OPS_3D } } -void NormaliseF3D(Real* ratio) { +void NormaliseF(Real* ratio) { for (auto idBlock : g_Block()) { Block& block{idBlock.second}; std::vector iterRng; @@ -254,7 +256,7 @@ void AssignCoordinates(const Block& block, const int sizeY{block.Size().at(1)}; ops_par_loop(KerSetCoordinates, "KerSetCoordinates", block.Get(), SpaceDim(), range.data(), - ops_arg_dat(g_CoordinateXYZ()[blockIndex], SpaceDim(), + ops_arg_dat(g_CoordinateXYZ()[block.ID()], SpaceDim(), LOCALSTENCIL, "double", OPS_WRITE), ops_arg_idx(), ops_arg_gbl(coordinateX, sizeX, "double", OPS_READ), @@ -306,10 +308,10 @@ void SetBlockGeometryProperty(const Block& block) { haloIterRng[1] = iterRange[1] + 1; haloIterRng[2] = iterRange[2] - 1; haloIterRng[3] = iterRange[3] - 1; - if (3 == SpaceDim()) { - haloIterRng[4] = iterRange[4] - 1; - haloIterRng[5] = iterRange[5] + 1; - } +#ifdef OPS_3D + haloIterRng[4] = iterRange[4] - 1; + haloIterRng[5] = iterRange[5] + 1; +#endif ops_par_loop(KerSetIntField, "KerSetIntField", block.Get(), SpaceDim(), haloIterRng, ops_arg_gbl(&geometryProperty, 1, "int", OPS_READ), @@ -322,10 +324,10 @@ void SetBlockGeometryProperty(const Block& block) { haloIterRng[1] = iterRange[1] + 1; haloIterRng[2] = iterRange[2] + 1; haloIterRng[3] = iterRange[3] + 1; - if (3 == SpaceDim()) { - haloIterRng[4] = iterRange[4] - 1; - haloIterRng[5] = iterRange[5] + 1; - } +#ifdef OPS_3D + haloIterRng[4] = iterRange[4] - 1; + haloIterRng[5] = iterRange[5] + 1; +#endif ops_par_loop(KerSetIntField, "KerSetIntField", block.Get(), SpaceDim(), haloIterRng, ops_arg_gbl(&geometryProperty, 1, "int", OPS_READ), @@ -338,10 +340,10 @@ void SetBlockGeometryProperty(const Block& block) { haloIterRng[1] = iterRange[1] - 1; haloIterRng[2] = iterRange[2] - 1; haloIterRng[3] = iterRange[3] + 1; - if (3 == SpaceDim()) { - haloIterRng[4] = iterRange[4] - 1; - haloIterRng[5] = iterRange[5] + 1; - } +#ifdef OPS_3D + haloIterRng[4] = iterRange[4] - 1; + haloIterRng[5] = iterRange[5] + 1; +#endif ops_par_loop(KerSetIntField, "KerSetIntField", block.Get(), SpaceDim(), haloIterRng, ops_arg_gbl(&geometryProperty, 1, "int", OPS_READ), @@ -354,45 +356,45 @@ void SetBlockGeometryProperty(const Block& block) { haloIterRng[1] = iterRange[1] + 1; haloIterRng[2] = iterRange[2] - 1; haloIterRng[3] = iterRange[3] + 1; - if (3 == SpaceDim()) { - haloIterRng[4] = iterRange[4] - 1; - haloIterRng[5] = iterRange[5] + 1; - } +#ifdef OPS_3D + haloIterRng[4] = iterRange[4] - 1; + haloIterRng[5] = iterRange[5] + 1; +#endif ops_par_loop(KerSetIntField, "KerSetIntField", block.Get(), SpaceDim(), haloIterRng, ops_arg_gbl(&geometryProperty, 1, "int", OPS_READ), ops_arg_dat(g_GeometryProperty()[block.ID()], 1, LOCALSTENCIL, "int", OPS_WRITE)); - if (3 == SpaceDim()) { - iterRange.assign( - block.BoundarySurfaceRange().at(BoundarySurface::Back).begin(), - block.BoundarySurfaceRange().at(BoundarySurface::Back).end()); - haloIterRng[0] = iterRange[0] - 1; - haloIterRng[1] = iterRange[1] + 1; - haloIterRng[2] = iterRange[2] - 1; - haloIterRng[3] = iterRange[3] + 1; - haloIterRng[4] = iterRange[4] - 1; - haloIterRng[5] = iterRange[5] - 1; - ops_par_loop(KerSetIntField, "KerSetIntField", block.Get(), SpaceDim(), - haloIterRng, - ops_arg_gbl(&geometryProperty, 1, "int", OPS_READ), - ops_arg_dat(g_GeometryProperty()[block.ID()], 1, - LOCALSTENCIL, "int", OPS_WRITE)); - iterRange.assign( - block.BoundarySurfaceRange().at(BoundarySurface::Front).begin(), - block.BoundarySurfaceRange().at(BoundarySurface::Front).end()); - haloIterRng[0] = iterRange[0] - 1; - haloIterRng[1] = iterRange[1] + 1; - haloIterRng[2] = iterRange[2] - 1; - haloIterRng[3] = iterRange[3] + 1; - haloIterRng[4] = iterRange[4] + 1; - haloIterRng[5] = iterRange[5] + 1; - ops_par_loop(KerSetIntField, "KerSetIntField", block.Get(), SpaceDim(), - haloIterRng, - ops_arg_gbl(&geometryProperty, 1, "int", OPS_READ), - ops_arg_dat(g_GeometryProperty()[block.ID()], 1, - LOCALSTENCIL, "int", OPS_WRITE)); - } +#ifdef OPS_3D + iterRange.assign( + block.BoundarySurfaceRange().at(BoundarySurface::Back).begin(), + block.BoundarySurfaceRange().at(BoundarySurface::Back).end()); + haloIterRng[0] = iterRange[0] - 1; + haloIterRng[1] = iterRange[1] + 1; + haloIterRng[2] = iterRange[2] - 1; + haloIterRng[3] = iterRange[3] + 1; + haloIterRng[4] = iterRange[4] - 1; + haloIterRng[5] = iterRange[5] - 1; + ops_par_loop(KerSetIntField, "KerSetIntField", block.Get(), SpaceDim(), + haloIterRng, + ops_arg_gbl(&geometryProperty, 1, "int", OPS_READ), + ops_arg_dat(g_GeometryProperty()[block.ID()], 1, LOCALSTENCIL, + "int", OPS_WRITE)); + iterRange.assign( + block.BoundarySurfaceRange().at(BoundarySurface::Front).begin(), + block.BoundarySurfaceRange().at(BoundarySurface::Front).end()); + haloIterRng[0] = iterRange[0] - 1; + haloIterRng[1] = iterRange[1] + 1; + haloIterRng[2] = iterRange[2] - 1; + haloIterRng[3] = iterRange[3] + 1; + haloIterRng[4] = iterRange[4] + 1; + haloIterRng[5] = iterRange[5] + 1; + ops_par_loop(KerSetIntField, "KerSetIntField", block.Get(), SpaceDim(), + haloIterRng, + ops_arg_gbl(&geometryProperty, 1, "int", OPS_READ), + ops_arg_dat(g_GeometryProperty()[block.ID()], 1, LOCALSTENCIL, + "int", OPS_WRITE)); +#endif // OPS_3D // specify domain geometryProperty = VG_JP; @@ -431,209 +433,193 @@ void SetBlockGeometryProperty(const Block& block) { ops_arg_gbl(&geometryProperty, 1, "int", OPS_READ), ops_arg_dat(g_GeometryProperty()[block.ID()], 1, LOCALSTENCIL, "int", OPS_WRITE)); - if (3 == SpaceDim()) { - geometryProperty = VG_KP; - iterRange.assign( - block.BoundarySurfaceRange().at(BoundarySurface::Back).begin(), - block.BoundarySurfaceRange().at(BoundarySurface::Back).end()); - ops_par_loop(KerSetIntField, "KerSetIntField", block.Get(), SpaceDim(), - iterRange.data(), - ops_arg_gbl(&geometryProperty, 1, "int", OPS_READ), - ops_arg_dat(g_GeometryProperty()[block.ID()], 1, - LOCALSTENCIL, "int", OPS_WRITE)); - geometryProperty = VG_KM; - iterRange.assign( - block.BoundarySurfaceRange().at(BoundarySurface::Front).begin(), - block.BoundarySurfaceRange().at(BoundarySurface::Front).end()); - ops_par_loop(KerSetIntField, "KerSetIntField", block.Get(), SpaceDim(), - iterRange.data(), - ops_arg_gbl(&geometryProperty, 1, "int", OPS_READ), - ops_arg_dat(g_GeometryProperty()[block.ID()], 1, - LOCALSTENCIL, "int", OPS_WRITE)); - } +#ifdef OPS_3D + geometryProperty = VG_KP; + iterRange.assign( + block.BoundarySurfaceRange().at(BoundarySurface::Back).begin(), + block.BoundarySurfaceRange().at(BoundarySurface::Back).end()); + ops_par_loop(KerSetIntField, "KerSetIntField", block.Get(), SpaceDim(), + iterRange.data(), + ops_arg_gbl(&geometryProperty, 1, "int", OPS_READ), + ops_arg_dat(g_GeometryProperty()[block.ID()], 1, LOCALSTENCIL, + "int", OPS_WRITE)); + geometryProperty = VG_KM; + iterRange.assign( + block.BoundarySurfaceRange().at(BoundarySurface::Front).begin(), + block.BoundarySurfaceRange().at(BoundarySurface::Front).end()); + ops_par_loop(KerSetIntField, "KerSetIntField", block.Get(), SpaceDim(), + iterRange.data(), + ops_arg_gbl(&geometryProperty, 1, "int", OPS_READ), + ops_arg_dat(g_GeometryProperty()[block.ID()], 1, LOCALSTENCIL, + "int", OPS_WRITE)); +#endif // OPS_3D const int nx{(int)block.Size().at(0)}; const int ny{(int)block.Size().at(1)}; // 2D Domain corner points four types - if (2 == SpaceDim()) { - int iminjmin[]{0, 1, 0, 1}; - geometryProperty = VG_IPJP_I; - ops_par_loop(KerSetIntField, "KerSetIntField", block.Get(), SpaceDim(), - iminjmin, - ops_arg_gbl(&geometryProperty, 1, "int", OPS_READ), - ops_arg_dat(g_GeometryProperty()[block.ID()], 1, - LOCALSTENCIL, "int", OPS_WRITE)); - int iminjmax[] = {0, 1, ny - 1, ny}; - geometryProperty = VG_IPJM_I; - ops_par_loop(KerSetIntField, "KerSetIntField", block.Get(), SpaceDim(), - iminjmax, - ops_arg_gbl(&geometryProperty, 1, "int", OPS_READ), - ops_arg_dat(g_GeometryProperty()[block.ID()], 1, - LOCALSTENCIL, "int", OPS_WRITE)); - int imaxjmax[] = {nx - 1, nx, ny - 1, ny}; - geometryProperty = VG_IMJM_I; - ops_par_loop(KerSetIntField, "KerSetIntField", block.Get(), SpaceDim(), - imaxjmax, - ops_arg_gbl(&geometryProperty, 1, "int", OPS_READ), - ops_arg_dat(g_GeometryProperty()[block.ID()], 1, - LOCALSTENCIL, "int", OPS_WRITE)); - int imaxjmin[] = {nx - 1, nx, 0, 1}; - geometryProperty = VG_IMJP_I; - ops_par_loop(KerSetIntField, "KerSetIntField", block.Get(), SpaceDim(), - imaxjmin, - ops_arg_gbl(&geometryProperty, 1, "int", OPS_READ), - ops_arg_dat(g_GeometryProperty()[block.ID()], 1, - LOCALSTENCIL, "int", OPS_WRITE)); - } +#ifdef OPS_2D + int iminjmin[]{0, 1, 0, 1}; + geometryProperty = VG_IPJP_I; + ops_par_loop(KerSetIntField, "KerSetIntField", block.Get(), SpaceDim(), + iminjmin, ops_arg_gbl(&geometryProperty, 1, "int", OPS_READ), + ops_arg_dat(g_GeometryProperty()[block.ID()], 1, LOCALSTENCIL, + "int", OPS_WRITE)); + int iminjmax[] = {0, 1, ny - 1, ny}; + geometryProperty = VG_IPJM_I; + ops_par_loop(KerSetIntField, "KerSetIntField", block.Get(), SpaceDim(), + iminjmax, ops_arg_gbl(&geometryProperty, 1, "int", OPS_READ), + ops_arg_dat(g_GeometryProperty()[block.ID()], 1, LOCALSTENCIL, + "int", OPS_WRITE)); + int imaxjmax[] = {nx - 1, nx, ny - 1, ny}; + geometryProperty = VG_IMJM_I; + ops_par_loop(KerSetIntField, "KerSetIntField", block.Get(), SpaceDim(), + imaxjmax, ops_arg_gbl(&geometryProperty, 1, "int", OPS_READ), + ops_arg_dat(g_GeometryProperty()[block.ID()], 1, LOCALSTENCIL, + "int", OPS_WRITE)); + int imaxjmin[] = {nx - 1, nx, 0, 1}; + geometryProperty = VG_IMJP_I; + ops_par_loop(KerSetIntField, "KerSetIntField", block.Get(), SpaceDim(), + imaxjmin, ops_arg_gbl(&geometryProperty, 1, "int", OPS_READ), + ops_arg_dat(g_GeometryProperty()[block.ID()], 1, LOCALSTENCIL, + "int", OPS_WRITE)); +#endif // OPS_2D - if (3 == SpaceDim()) { - const int nz{(int)block.Size().at(2)}; - // 3D Domain edges 12 types - int iminjmin[]{0, 1, 0, 1, 0, nz}; - geometryProperty = VG_IPJP_I; - ops_par_loop(KerSetIntField, "KerSetIntField", block.Get(), SpaceDim(), - iminjmin, - ops_arg_gbl(&geometryProperty, 1, "int", OPS_READ), - ops_arg_dat(g_GeometryProperty()[block.ID()], 1, - LOCALSTENCIL, "int", OPS_WRITE)); - int iminjmax[]{0, 1, ny - 1, ny, 0, nz}; - geometryProperty = VG_IPJM_I; - ops_par_loop(KerSetIntField, "KerSetIntField", block.Get(), SpaceDim(), - iminjmax, - ops_arg_gbl(&geometryProperty, 1, "int", OPS_READ), - ops_arg_dat(g_GeometryProperty()[block.ID()], 1, - LOCALSTENCIL, "int", OPS_WRITE)); - int imaxjmax[]{nx - 1, nx, ny - 1, ny, 0, nz}; - geometryProperty = VG_IMJM_I; - ops_par_loop(KerSetIntField, "KerSetIntField", block.Get(), SpaceDim(), - imaxjmax, - ops_arg_gbl(&geometryProperty, 1, "int", OPS_READ), - ops_arg_dat(g_GeometryProperty()[block.ID()], 1, - LOCALSTENCIL, "int", OPS_WRITE)); - int imaxjmin[]{nx - 1, nx, 0, 1, 0, nz}; - geometryProperty = VG_IMJP_I; - ops_par_loop(KerSetIntField, "KerSetIntField", block.Get(), SpaceDim(), - imaxjmin, - ops_arg_gbl(&geometryProperty, 1, "int", OPS_READ), - ops_arg_dat(g_GeometryProperty()[block.ID()], 1, - LOCALSTENCIL, "int", OPS_WRITE)); +#ifdef OPS_3D + const int nz{(int)block.Size().at(2)}; + // 3D Domain edges 12 types + int iminjmin[]{0, 1, 0, 1, 0, nz}; + geometryProperty = VG_IPJP_I; + ops_par_loop(KerSetIntField, "KerSetIntField", block.Get(), SpaceDim(), + iminjmin, ops_arg_gbl(&geometryProperty, 1, "int", OPS_READ), + ops_arg_dat(g_GeometryProperty()[block.ID()], 1, LOCALSTENCIL, + "int", OPS_WRITE)); + int iminjmax[]{0, 1, ny - 1, ny, 0, nz}; + geometryProperty = VG_IPJM_I; + ops_par_loop(KerSetIntField, "KerSetIntField", block.Get(), SpaceDim(), + iminjmax, ops_arg_gbl(&geometryProperty, 1, "int", OPS_READ), + ops_arg_dat(g_GeometryProperty()[block.ID()], 1, LOCALSTENCIL, + "int", OPS_WRITE)); + int imaxjmax[]{nx - 1, nx, ny - 1, ny, 0, nz}; + geometryProperty = VG_IMJM_I; + ops_par_loop(KerSetIntField, "KerSetIntField", block.Get(), SpaceDim(), + imaxjmax, ops_arg_gbl(&geometryProperty, 1, "int", OPS_READ), + ops_arg_dat(g_GeometryProperty()[block.ID()], 1, LOCALSTENCIL, + "int", OPS_WRITE)); + int imaxjmin[]{nx - 1, nx, 0, 1, 0, nz}; + geometryProperty = VG_IMJP_I; + ops_par_loop(KerSetIntField, "KerSetIntField", block.Get(), SpaceDim(), + imaxjmin, ops_arg_gbl(&geometryProperty, 1, "int", OPS_READ), + ops_arg_dat(g_GeometryProperty()[block.ID()], 1, LOCALSTENCIL, + "int", OPS_WRITE)); - int iminkmin[]{0, 1, 0, ny, 0, 1}; - geometryProperty = VG_IPKP_I; - ops_par_loop(KerSetIntField, "KerSetIntField", block.Get(), SpaceDim(), - iminkmin, - ops_arg_gbl(&geometryProperty, 1, "int", OPS_READ), - ops_arg_dat(g_GeometryProperty()[block.ID()], 1, - LOCALSTENCIL, "int", OPS_WRITE)); - int iminkmax[]{0, 1, 0, ny, nz - 1, nz}; - geometryProperty = VG_IPKM_I; - ops_par_loop(KerSetIntField, "KerSetIntField", block.Get(), SpaceDim(), - iminkmax, - ops_arg_gbl(&geometryProperty, 1, "int", OPS_READ), - ops_arg_dat(g_GeometryProperty()[block.ID()], 1, - LOCALSTENCIL, "int", OPS_WRITE)); - int imaxkmax[]{nx - 1, nx, 0, ny, nz - 1, nz}; - geometryProperty = VG_IMKM_I; - ops_par_loop(KerSetIntField, "KerSetIntField", block.Get(), SpaceDim(), - imaxkmax, - ops_arg_gbl(&geometryProperty, 1, "int", OPS_READ), - ops_arg_dat(g_GeometryProperty()[block.ID()], 1, - LOCALSTENCIL, "int", OPS_WRITE)); - int imaxkmin[]{nx - 1, nx, 0, ny, 0, 1}; - geometryProperty = VG_IMKP_I; - ops_par_loop(KerSetIntField, "KerSetIntField", block.Get(), SpaceDim(), - imaxkmin, - ops_arg_gbl(&geometryProperty, 1, "int", OPS_READ), - ops_arg_dat(g_GeometryProperty()[block.ID()], 1, - LOCALSTENCIL, "int", OPS_WRITE)); + int iminkmin[]{0, 1, 0, ny, 0, 1}; + geometryProperty = VG_IPKP_I; + ops_par_loop(KerSetIntField, "KerSetIntField", block.Get(), SpaceDim(), + iminkmin, ops_arg_gbl(&geometryProperty, 1, "int", OPS_READ), + ops_arg_dat(g_GeometryProperty()[block.ID()], 1, LOCALSTENCIL, + "int", OPS_WRITE)); + int iminkmax[]{0, 1, 0, ny, nz - 1, nz}; + geometryProperty = VG_IPKM_I; + ops_par_loop(KerSetIntField, "KerSetIntField", block.Get(), SpaceDim(), + iminkmax, ops_arg_gbl(&geometryProperty, 1, "int", OPS_READ), + ops_arg_dat(g_GeometryProperty()[block.ID()], 1, LOCALSTENCIL, + "int", OPS_WRITE)); + int imaxkmax[]{nx - 1, nx, 0, ny, nz - 1, nz}; + geometryProperty = VG_IMKM_I; + ops_par_loop(KerSetIntField, "KerSetIntField", block.Get(), SpaceDim(), + imaxkmax, ops_arg_gbl(&geometryProperty, 1, "int", OPS_READ), + ops_arg_dat(g_GeometryProperty()[block.ID()], 1, LOCALSTENCIL, + "int", OPS_WRITE)); + int imaxkmin[]{nx - 1, nx, 0, ny, 0, 1}; + geometryProperty = VG_IMKP_I; + ops_par_loop(KerSetIntField, "KerSetIntField", block.Get(), SpaceDim(), + imaxkmin, ops_arg_gbl(&geometryProperty, 1, "int", OPS_READ), + ops_arg_dat(g_GeometryProperty()[block.ID()], 1, LOCALSTENCIL, + "int", OPS_WRITE)); - int jminkmin[]{0, nx, 0, 1, 0, 1}; - geometryProperty = VG_JPKP_I; - ops_par_loop(KerSetIntField, "KerSetIntField", block.Get(), SpaceDim(), - jminkmin, - ops_arg_gbl(&geometryProperty, 1, "int", OPS_READ), - ops_arg_dat(g_GeometryProperty()[block.ID()], 1, - LOCALSTENCIL, "int", OPS_WRITE)); - int jminkmax[]{0, nx, 0, 1, nz - 1, nz}; - geometryProperty = VG_JPKM_I; - ops_par_loop(KerSetIntField, "KerSetIntField", block.Get(), SpaceDim(), - jminkmax, - ops_arg_gbl(&geometryProperty, 1, "int", OPS_READ), - ops_arg_dat(g_GeometryProperty()[block.ID()], 1, - LOCALSTENCIL, "int", OPS_WRITE)); - int jmaxkmax[]{0, nx, ny - 1, ny, nz - 1, nz}; - geometryProperty = VG_JMKM_I; - ops_par_loop(KerSetIntField, "KerSetIntField", block.Get(), SpaceDim(), - jmaxkmax, - ops_arg_gbl(&geometryProperty, 1, "int", OPS_READ), - ops_arg_dat(g_GeometryProperty()[block.ID()], 1, - LOCALSTENCIL, "int", OPS_WRITE)); - int jmaxkmin[]{0, nx, ny - 1, ny, 0, 1}; - geometryProperty = VG_JMKP_I; - ops_par_loop(KerSetIntField, "KerSetIntField", block.Get(), SpaceDim(), - jmaxkmin, - ops_arg_gbl(&geometryProperty, 1, "int", OPS_READ), - ops_arg_dat(g_GeometryProperty()[block.ID()], 1, - LOCALSTENCIL, "int", OPS_WRITE)); + int jminkmin[]{0, nx, 0, 1, 0, 1}; + geometryProperty = VG_JPKP_I; + ops_par_loop(KerSetIntField, "KerSetIntField", block.Get(), SpaceDim(), + jminkmin, ops_arg_gbl(&geometryProperty, 1, "int", OPS_READ), + ops_arg_dat(g_GeometryProperty()[block.ID()], 1, LOCALSTENCIL, + "int", OPS_WRITE)); + int jminkmax[]{0, nx, 0, 1, nz - 1, nz}; + geometryProperty = VG_JPKM_I; + ops_par_loop(KerSetIntField, "KerSetIntField", block.Get(), SpaceDim(), + jminkmax, ops_arg_gbl(&geometryProperty, 1, "int", OPS_READ), + ops_arg_dat(g_GeometryProperty()[block.ID()], 1, LOCALSTENCIL, + "int", OPS_WRITE)); + int jmaxkmax[]{0, nx, ny - 1, ny, nz - 1, nz}; + geometryProperty = VG_JMKM_I; + ops_par_loop(KerSetIntField, "KerSetIntField", block.Get(), SpaceDim(), + jmaxkmax, ops_arg_gbl(&geometryProperty, 1, "int", OPS_READ), + ops_arg_dat(g_GeometryProperty()[block.ID()], 1, LOCALSTENCIL, + "int", OPS_WRITE)); + int jmaxkmin[]{0, nx, ny - 1, ny, 0, 1}; + geometryProperty = VG_JMKP_I; + ops_par_loop(KerSetIntField, "KerSetIntField", block.Get(), SpaceDim(), + jmaxkmin, ops_arg_gbl(&geometryProperty, 1, "int", OPS_READ), + ops_arg_dat(g_GeometryProperty()[block.ID()], 1, LOCALSTENCIL, + "int", OPS_WRITE)); - // 3D domain corners 8 types - int iminjminkmin[]{0, 1, 0, 1, 0, 1}; - geometryProperty = VG_IPJPKP_I; - ops_par_loop(KerSetIntField, "KerSetIntField", block.Get(), SpaceDim(), - iminjminkmin, - ops_arg_gbl(&geometryProperty, 1, "int", OPS_READ), - ops_arg_dat(g_GeometryProperty()[block.ID()], 1, - LOCALSTENCIL, "int", OPS_WRITE)); - int iminjminkmax[]{0, 1, 0, 1, nz - 1, nz}; - geometryProperty = VG_IPJPKM_I; - ops_par_loop(KerSetIntField, "KerSetIntField", block.Get(), SpaceDim(), - iminjminkmax, - ops_arg_gbl(&geometryProperty, 1, "int", OPS_READ), - ops_arg_dat(g_GeometryProperty()[block.ID()], 1, - LOCALSTENCIL, "int", OPS_WRITE)); - int iminjmaxkmin[]{0, 1, ny - 1, ny, 0, 1}; - geometryProperty = VG_IPJMKP_I; - ops_par_loop(KerSetIntField, "KerSetIntField", block.Get(), SpaceDim(), - iminjmaxkmin, - ops_arg_gbl(&geometryProperty, 1, "int", OPS_READ), - ops_arg_dat(g_GeometryProperty()[block.ID()], 1, - LOCALSTENCIL, "int", OPS_WRITE)); - int iminjmaxkmax[]{0, 1, ny - 1, ny, nz - 1, nz}; - geometryProperty = VG_IPJMKM_I; - ops_par_loop(KerSetIntField, "KerSetIntField", block.Get(), SpaceDim(), - iminjmaxkmax, - ops_arg_gbl(&geometryProperty, 1, "int", OPS_READ), - ops_arg_dat(g_GeometryProperty()[block.ID()], 1, - LOCALSTENCIL, "int", OPS_WRITE)); - int imaxjminkmin[]{nx - 1, nx, 0, 1, 0, 1}; - geometryProperty = VG_IMJPKP_I; - ops_par_loop(KerSetIntField, "KerSetIntField", block.Get(), SpaceDim(), - imaxjminkmin, - ops_arg_gbl(&geometryProperty, 1, "int", OPS_READ), - ops_arg_dat(g_GeometryProperty()[block.ID()], 1, - LOCALSTENCIL, "int", OPS_WRITE)); - int imaxjminkmax[]{nx - 1, nx, 0, 1, nz - 1, nz}; - geometryProperty = VG_IMJPKM_I; - ops_par_loop(KerSetIntField, "KerSetIntField", block.Get(), SpaceDim(), - imaxjminkmax, - ops_arg_gbl(&geometryProperty, 1, "int", OPS_READ), - ops_arg_dat(g_GeometryProperty()[block.ID()], 1, - LOCALSTENCIL, "int", OPS_WRITE)); - int imaxjmaxkmin[]{nx - 1, nx, ny - 1, ny, 0, 1}; - geometryProperty = VG_IMJMKP_I; - ops_par_loop(KerSetIntField, "KerSetIntField", block.Get(), SpaceDim(), - imaxjmaxkmin, - ops_arg_gbl(&geometryProperty, 1, "int", OPS_READ), - ops_arg_dat(g_GeometryProperty()[block.ID()], 1, - LOCALSTENCIL, "int", OPS_WRITE)); - int imaxjmaxkmax[]{nx - 1, nx, ny - 1, ny, nz - 1, nz}; - geometryProperty = VG_IMJMKM_I; - ops_par_loop(KerSetIntField, "KerSetIntField", block.Get(), SpaceDim(), - imaxjmaxkmax, - ops_arg_gbl(&geometryProperty, 1, "int", OPS_READ), - ops_arg_dat(g_GeometryProperty()[block.ID()], 1, - LOCALSTENCIL, "int", OPS_WRITE)); - } + // 3D domain corners 8 types + int iminjminkmin[]{0, 1, 0, 1, 0, 1}; + geometryProperty = VG_IPJPKP_I; + ops_par_loop(KerSetIntField, "KerSetIntField", block.Get(), SpaceDim(), + iminjminkmin, + ops_arg_gbl(&geometryProperty, 1, "int", OPS_READ), + ops_arg_dat(g_GeometryProperty()[block.ID()], 1, LOCALSTENCIL, + "int", OPS_WRITE)); + int iminjminkmax[]{0, 1, 0, 1, nz - 1, nz}; + geometryProperty = VG_IPJPKM_I; + ops_par_loop(KerSetIntField, "KerSetIntField", block.Get(), SpaceDim(), + iminjminkmax, + ops_arg_gbl(&geometryProperty, 1, "int", OPS_READ), + ops_arg_dat(g_GeometryProperty()[block.ID()], 1, LOCALSTENCIL, + "int", OPS_WRITE)); + int iminjmaxkmin[]{0, 1, ny - 1, ny, 0, 1}; + geometryProperty = VG_IPJMKP_I; + ops_par_loop(KerSetIntField, "KerSetIntField", block.Get(), SpaceDim(), + iminjmaxkmin, + ops_arg_gbl(&geometryProperty, 1, "int", OPS_READ), + ops_arg_dat(g_GeometryProperty()[block.ID()], 1, LOCALSTENCIL, + "int", OPS_WRITE)); + int iminjmaxkmax[]{0, 1, ny - 1, ny, nz - 1, nz}; + geometryProperty = VG_IPJMKM_I; + ops_par_loop(KerSetIntField, "KerSetIntField", block.Get(), SpaceDim(), + iminjmaxkmax, + ops_arg_gbl(&geometryProperty, 1, "int", OPS_READ), + ops_arg_dat(g_GeometryProperty()[block.ID()], 1, LOCALSTENCIL, + "int", OPS_WRITE)); + int imaxjminkmin[]{nx - 1, nx, 0, 1, 0, 1}; + geometryProperty = VG_IMJPKP_I; + ops_par_loop(KerSetIntField, "KerSetIntField", block.Get(), SpaceDim(), + imaxjminkmin, + ops_arg_gbl(&geometryProperty, 1, "int", OPS_READ), + ops_arg_dat(g_GeometryProperty()[block.ID()], 1, LOCALSTENCIL, + "int", OPS_WRITE)); + int imaxjminkmax[]{nx - 1, nx, 0, 1, nz - 1, nz}; + geometryProperty = VG_IMJPKM_I; + ops_par_loop(KerSetIntField, "KerSetIntField", block.Get(), SpaceDim(), + imaxjminkmax, + ops_arg_gbl(&geometryProperty, 1, "int", OPS_READ), + ops_arg_dat(g_GeometryProperty()[block.ID()], 1, LOCALSTENCIL, + "int", OPS_WRITE)); + int imaxjmaxkmin[]{nx - 1, nx, ny - 1, ny, 0, 1}; + geometryProperty = VG_IMJMKP_I; + ops_par_loop(KerSetIntField, "KerSetIntField", block.Get(), SpaceDim(), + imaxjmaxkmin, + ops_arg_gbl(&geometryProperty, 1, "int", OPS_READ), + ops_arg_dat(g_GeometryProperty()[block.ID()], 1, LOCALSTENCIL, + "int", OPS_WRITE)); + int imaxjmaxkmax[]{nx - 1, nx, ny - 1, ny, nz - 1, nz}; + geometryProperty = VG_IMJMKM_I; + ops_par_loop(KerSetIntField, "KerSetIntField", block.Get(), SpaceDim(), + imaxjmaxkmax, + ops_arg_gbl(&geometryProperty, 1, "int", OPS_READ), + ops_arg_dat(g_GeometryProperty()[block.ID()], 1, LOCALSTENCIL, + "int", OPS_WRITE)); +#endif // OPS_3D } void SetBoundaryNodeType() { @@ -670,8 +656,8 @@ void SetBoundaryNodeType() { iterRange.at(1)--; iterRange.at(2)++; iterRange.at(3)--; -#endif } +#endif } // Specify general boundary type ops_par_loop(KerSetIntField, "KerSetIntField", block.Get(), SpaceDim(), @@ -701,10 +687,10 @@ void SetBulkandHaloNodesType(const Block& block, int compoId) { haloIterRng[1] = iterRange[1] + 1; haloIterRng[2] = iterRange[2] - 1; haloIterRng[3] = iterRange[3] - 1; - if (3 == SpaceDim()) { - haloIterRng[4] = iterRange[4] - 1; - haloIterRng[5] = iterRange[5] + 1; - } +#ifdef OPS_3D + haloIterRng[4] = iterRange[4] - 1; + haloIterRng[5] = iterRange[5] + 1; +#endif ops_par_loop(KerSetIntField, "KerSetIntField", block.Get(), SpaceDim(), haloIterRng, ops_arg_gbl(&immersedSolidType, 1, "int", OPS_READ), @@ -718,10 +704,10 @@ void SetBulkandHaloNodesType(const Block& block, int compoId) { haloIterRng[1] = iterRange[1] + 1; haloIterRng[2] = iterRange[2] + 1; haloIterRng[3] = iterRange[3] + 1; - if (3 == SpaceDim()) { - haloIterRng[4] = iterRange[4] - 1; - haloIterRng[5] = iterRange[5] + 1; - } +#ifdef OPS_3D + haloIterRng[4] = iterRange[4] - 1; + haloIterRng[5] = iterRange[5] + 1; +#endif // Specify halo points ops_par_loop(KerSetIntField, "KerSetIntField", block.Get(), SpaceDim(), haloIterRng, @@ -736,10 +722,10 @@ void SetBulkandHaloNodesType(const Block& block, int compoId) { haloIterRng[1] = iterRange[1] - 1; haloIterRng[2] = iterRange[2] - 1; haloIterRng[3] = iterRange[3] + 1; - if (3 == SpaceDim()) { - haloIterRng[4] = iterRange[4] - 1; - haloIterRng[5] = iterRange[5] + 1; - } +#ifdef OPS_3D + haloIterRng[4] = iterRange[4] - 1; + haloIterRng[5] = iterRange[5] + 1; +#endif // Specify halo points ops_par_loop(KerSetIntField, "KerSetIntField", block.Get(), SpaceDim(), haloIterRng, @@ -754,10 +740,10 @@ void SetBulkandHaloNodesType(const Block& block, int compoId) { haloIterRng[1] = iterRange[1] + 1; haloIterRng[2] = iterRange[2] - 1; haloIterRng[3] = iterRange[3] + 1; - if (3 == SpaceDim()) { - haloIterRng[4] = iterRange[4] - 1; - haloIterRng[5] = iterRange[5] + 1; - } +#ifdef OPS_3D + haloIterRng[4] = iterRange[4] - 1; + haloIterRng[5] = iterRange[5] + 1; +#endif // Specify halo points ops_par_loop(KerSetIntField, "KerSetIntField", block.Get(), SpaceDim(), haloIterRng, @@ -765,38 +751,38 @@ void SetBulkandHaloNodesType(const Block& block, int compoId) { ops_arg_dat(g_NodeType().at(compoId).at(block.ID()), 1, LOCALSTENCIL, "int", OPS_WRITE)); - if (3 == SpaceDim()) { - iterRange.assign( - block.BoundarySurfaceRange().at(BoundarySurface::Back).begin(), - block.BoundarySurfaceRange().at(BoundarySurface::Back).end()); - haloIterRng[0] = iterRange[0] - 1; - haloIterRng[1] = iterRange[1] + 1; - haloIterRng[2] = iterRange[2] - 1; - haloIterRng[3] = iterRange[3] + 1; - haloIterRng[4] = iterRange[4] - 1; - haloIterRng[5] = iterRange[5] - 1; - // Specify halo points - ops_par_loop(KerSetIntField, "KerSetIntField", block.Get(), SpaceDim(), - haloIterRng, - ops_arg_gbl(&immersedSolidType, 1, "int", OPS_READ), - ops_arg_dat(g_NodeType().at(compoId).at(block.ID()), 1, - LOCALSTENCIL, "int", OPS_WRITE)); +#ifdef OPS_3D + iterRange.assign( + block.BoundarySurfaceRange().at(BoundarySurface::Back).begin(), + block.BoundarySurfaceRange().at(BoundarySurface::Back).end()); + haloIterRng[0] = iterRange[0] - 1; + haloIterRng[1] = iterRange[1] + 1; + haloIterRng[2] = iterRange[2] - 1; + haloIterRng[3] = iterRange[3] + 1; + haloIterRng[4] = iterRange[4] - 1; + haloIterRng[5] = iterRange[5] - 1; + // Specify halo points + ops_par_loop(KerSetIntField, "KerSetIntField", block.Get(), SpaceDim(), + haloIterRng, + ops_arg_gbl(&immersedSolidType, 1, "int", OPS_READ), + ops_arg_dat(g_NodeType().at(compoId).at(block.ID()), 1, + LOCALSTENCIL, "int", OPS_WRITE)); - iterRange.assign( - block.BoundarySurfaceRange().at(BoundarySurface::Front).begin(), - block.BoundarySurfaceRange().at(BoundarySurface::Front).end()); - haloIterRng[0] = iterRange[0] - 1; - haloIterRng[1] = iterRange[1] + 1; - haloIterRng[2] = iterRange[2] - 1; - haloIterRng[3] = iterRange[3] + 1; - haloIterRng[4] = iterRange[4] + 1; - haloIterRng[5] = iterRange[5] + 1; - // Specify halo points - ops_par_loop(KerSetIntField, "KerSetIntField", block.Get(), SpaceDim(), - haloIterRng, - ops_arg_gbl(&immersedSolidType, 1, "int", OPS_READ), - ops_arg_dat(g_NodeType().at(compoId).at(block.ID()), 1, - LOCALSTENCIL, "int", OPS_WRITE)); - } + iterRange.assign( + block.BoundarySurfaceRange().at(BoundarySurface::Front).begin(), + block.BoundarySurfaceRange().at(BoundarySurface::Front).end()); + haloIterRng[0] = iterRange[0] - 1; + haloIterRng[1] = iterRange[1] + 1; + haloIterRng[2] = iterRange[2] - 1; + haloIterRng[3] = iterRange[3] + 1; + haloIterRng[4] = iterRange[4] + 1; + haloIterRng[5] = iterRange[5] + 1; + // Specify halo points + ops_par_loop(KerSetIntField, "KerSetIntField", block.Get(), SpaceDim(), + haloIterRng, + ops_arg_gbl(&immersedSolidType, 1, "int", OPS_READ), + ops_arg_dat(g_NodeType().at(compoId).at(block.ID()), 1, + LOCALSTENCIL, "int", OPS_WRITE)); +#endif // OPS_3D FreeArrayMemory(haloIterRng); } diff --git a/Src/hilemms.h b/Src/hilemms_ToBeDeleted.h similarity index 100% rename from Src/hilemms.h rename to Src/hilemms_ToBeDeleted.h diff --git a/Src/hilemms_ops.cpp b/Src/hilemms_ops_ToBeDeleted.cpp similarity index 100% rename from Src/hilemms_ops.cpp rename to Src/hilemms_ops_ToBeDeleted.cpp diff --git a/Src/hilemms_ops_kernel.h b/Src/hilemms_ops_kernel_ToBeDeleted.h similarity index 100% rename from Src/hilemms_ops_kernel.h rename to Src/hilemms_ops_kernel_ToBeDeleted.h diff --git a/Src/model.cpp b/Src/model.cpp index a6220dd..6f59850 100644 --- a/Src/model.cpp +++ b/Src/model.cpp @@ -298,7 +298,6 @@ void DefineComponents(const std::vector& compoNames, if (timeStep == 0) { g_f().CreateFieldFromScratch(g_Block()); for (auto& pair : g_NodeType()) { - ops_printf("Create Nodetype %d\n", pair.first); pair.second.CreateFieldFromScratch(g_Block()); } } else { diff --git a/Src/model.h b/Src/model.h index e110927..489b677 100644 --- a/Src/model.h +++ b/Src/model.h @@ -35,7 +35,7 @@ **/ #ifndef MODEL_H #define MODEL_H -#include +#include #include #include #include @@ -168,10 +168,11 @@ void DefineInitialCondition(std::vector types, void UpdateMacroVars3D(); void PreDefinedBodyForce3D(); void PreDefinedInitialCondition3D(); -/*! - * Ops_par_loop for the collision step - */ void PreDefinedCollision3D(); +void PreDefinedCollision3D(int* velID, int* loop, Real tauRef, + CollisionType collisionType, int compoId, int rhoId, int Tid); #endif //OPS_3D + + #endif diff --git a/Src/model_kernel.inc b/Src/model_kernel.inc index 1c7d4be..a6d5eb8 100644 --- a/Src/model_kernel.inc +++ b/Src/model_kernel.inc @@ -53,384 +53,302 @@ * In the macroVars, all variables are conserved, i.e., with density. * @todo how to deal with overflow in a kernel function? in particular, GPU */ - #ifdef OPS_2D - -// two dimensional code -void KerCalcFeq(const int* nodeType, const Real* macroVars, Real* feq) { - VertexType vt{(VertexType)nodeType(0, 0)}; +void KerCalcDensity(ACC& Rho, const ACC& f, + const ACC& nodeType, const int* lattIdx) { +#ifdef OPS_2D + VertexType vt = (VertexType)nodeType(0, 0); if (vt != VertexType::ImmersedSolid) { - for (int compoIndex = 0; compoIndex < NUMCOMPONENTS; compoIndex++) { - CollisionType CollisionType{ - (CollisionType)CollisionType[compoIndex]}; - const int startPos{VARIABLECOMPPOS[2 * compoIndex]}; - if (Equilibrium_BGKIsothermal2nd == CollisionType) { - Real rho{macroVars(startPos, 0, 0)}; - Real u{macroVars(startPos + 1, 0, 0)}; - Real v{macroVars(startPos + 2, 0, 0)}; - const Real T{1}; - const int polyOrder{2}; - for (int xiIndex = COMPOINDEX[2 * compoIndex]; - xiIndex <= COMPOINDEX[2 * compoIndex + 1]; xiIndex++) { - feq(xiIndex, 0, 0) = - CalcBGKFeq(xiIndex, rho, u, v, T, polyOrder); - } - } - if (Collision_BGKThermal4th == CollisionType) { - Real rho{macroVars(startPos, 0, 0)}; - Real u{macroVars(startPos + 1, 0, 0)}; - Real v{macroVars(startPos + 2, 0, 0)}; - Real T{macroVars(startPos + 3, 0, 0)}; - const int polyOrder{4}; - for (int xiIndex = COMPOINDEX[2 * compoIndex]; - xiIndex <= COMPOINDEX[2 * compoIndex + 1]; xiIndex++) { - feq(xiIndex, 0, 0) = - CalcBGKFeq(xiIndex, rho, u, v, T, polyOrder); - } - } - if (Collision_BGKSWE4th == CollisionType) { - Real h{macroVars(startPos, 0, 0)}; - Real u{macroVars(startPos + 1, 0, 0)}; - Real v{macroVars(startPos + 2, 0, 0)}; - const int polyOrder{4}; - for (int xiIndex = COMPOINDEX[2 * compoIndex]; - xiIndex <= COMPOINDEX[2 * compoIndex + 1]; xiIndex++) { - feq(xiIndex, 0, 0) = - CalcSWEFeq(xiIndex, h, u, v, polyOrder); - } - } + Real rho{0}; + for (int xiIdx = lattIdx[0]; xiIdx <= lattIdx[1]; xiIdx++) { + rho += f(xiIdx, 0, 0); } +#ifdef CPU + if (isnan(rho) || rho <= 0 || isinf(rho)) { + ops_printf( + "Error! Density %f becomes invalid！Something " + "wrong...", + rho); + assert(!(isnan(rho) || rho <= 0 || isinf(rho))); + } +#endif + Rho(0, 0) = rho; } +#endif // OPS_2D } -/*! - * If a Newton-Cotes quadrature is used, it can be converted to the way - * similar to the Gauss-Hermite quadrature * - */ -void KerCalcMacroVars(const ACC& nodeType, const ACC& f, - ACC& macroVars) { +void KerCalcU(ACC& U, const ACC& f, const ACC& nodeType, + const ACC& Rho, const int* lattIdx) { #ifdef OPS_2D VertexType vt = (VertexType)nodeType(0, 0); if (vt != VertexType::ImmersedSolid) { - bool rhoCalculated{false}; - bool veloCalculated[LATTDIM]; - for (int lattIdx = 0; lattIdx < LATTDIM; lattIdx++) { - veloCalculated[lattIdx] = false; + Real u{0}; + for (int xiIdx = lattIdx[0]; xiIdx <= lattIdx[1]; xiIdx++) { + u += CS * XI[xiIdx * LATTDIM] * f(xiIdx, 0, 0); + } + u /= Rho(0, 0); +#ifdef CPU + if (isnan(u) || isinf(u)) { + ops_printf( + "Error! Velocity U=%f becomes invalid! " + "Maybe something wrong...\n", + u); + assert(!(isnan(u) || isinf(u))); } - for (int m = 0; m < NUMMACROVAR; m++) { - macroVars(m, 0, 0) = 0; - VariableTypes varType = (VariableTypes)VARIABLETYPE[m]; - switch (varType) { - case Variable_Rho: { - rhoCalculated = true; - for (int xiIdx = 0; xiIdx < NUMXI; xiIdx++) { - macroVars(m, 0, 0) += - f(xiIdx, 0, 0); - } -#ifdef debug - Real rho{macroVars(m, 0, 0)}; - if (isnan(rho) || rho <= 0 || isinf(rho)) { - ops_printf( - "%sDensity=%f\n", - "Density becomes invalid! Maybe something wrong...", - rho); - } #endif - } break; - case Variable_U: { - if (rhoCalculated) { - veloCalculated[0] = true; - for (int xiIdx = 0; xiIdx < NUMXI; xiIdx++) { - macroVars(m, 0, 0) += - CS * XI[xiIdx * LATTDIM] * - f(xiIdx, 0, 0); - } - macroVars(m, 0, 0) /= - macroVars(0, 0, 0); -#ifdef debug - Real u{macroVars(m, 0, 0)}; - if (isnan(u) || isinf(u)) { - ops_printf("%sU=%f\n", - "Velocity becomes invalid! Maybe " - "something wrong...", - u); - } + U(0, 0) = u; + } +#endif // OPS_2D +} +void KerCalcV(ACC& V, const ACC& f, const ACC& nodeType, + const ACC& Rho, const int* lattIdx) { +#ifdef OPS_2D + VertexType vt = (VertexType)nodeType(0, 0); + if (vt != VertexType::ImmersedSolid) { + Real v{0}; + for (int xiIdx = lattIdx[0]; xiIdx <= lattIdx[1]; xiIdx++) { + v += CS * XI[xiIdx * LATTDIM + 1] * f(xiIdx, 0, 0); + } + v /= Rho(0, 0); +#ifdef CPU + if (isnan(v) || isinf(v)) { + ops_printf( + "Error! Velocity V=%f becomes invalid! " + "Maybe something wrong...\n", + v); + assert(!(isnan(v) || isinf(v))); + } #endif - } else { - ops_printf("%s\n", - "Density has not been calculated before " - "calculating U!"); - } - } break; - case Variable_V: { - if (rhoCalculated) { - veloCalculated[1] = true; - for (int xiIdx = 0; xiIdx < NUMXI; xiIdx++) { - macroVars(m, 0, 0) += - CS * XI[xiIdx * LATTDIM + 1] * - f(xiIdx, 0, 0); - } - macroVars(m, 0, 0) /= - macroVars(0, 0, 0); -#ifdef debug - Real v{macroVars(m, 0, 0)}; - if (isnan(v) || isinf(v)) { - ops_printf("%sV=%f\n", - "Velocity becomes invalid! Maybe " - "something wrong...", - v); - } -#endif - } else { - ops_printf("%s\n", - "Density has not been calculated before " - "calculating V!"); - } - } break; - case Variable_W: { - if (rhoCalculated) { - veloCalculated[2] = true; - for (int xiIdx = 0; xiIdx < NUMXI; xiIdx++) { - macroVars(m, 0, 0) += - CS * XI[xiIdx * LATTDIM + 2] * - f(xiIdx, 0, 0); - } - macroVars(m, 0, 0) /= - macroVars(0, 0, 0); -#ifdef debug - Real w{macroVars(m, 0, 0)}; - if (isnan(w) || isinf(w)) { - ops_printf("%sW=%f\n", - "Velocity becomes invalid! Maybe " - "something wrong...", - w); - } -#endif - } else { - ops_printf("%s\n", - "Density has not been calculated before " - "calculating W!"); - } - } break; - case Variable_Qx: { - bool ifCalc = true; - Real velo[LATTDIM]; - for (int d = 0; d < LATTDIM; d++) { - ifCalc = ifCalc && veloCalculated[d]; - velo[d] = macroVars(d + 1, 0, 0); - } - if (ifCalc) { - for (int xiIdx = 0; xiIdx < NUMXI; xiIdx++) { - Real T = 0; - for (int d = 0; d < LATTDIM; d++) { - T += (CS * XI[xiIdx * LATTDIM + d] - velo[d]) * - (CS * XI[xiIdx * LATTDIM + d] - velo[d]) * - f(xiIdx, 0, 0); - } - macroVars(m, 0, 0) += - (0.5 * (CS * XI[xiIdx * LATTDIM] - velo[0]) * - T); - } -#ifdef debug - Real qx{macroVars(m, 0, 0)}; - if (isnan(qx) || isinf(qx)) { - ops_printf("%sQx=%f\n", - "Heatflux becomes invalid! Maybe " - "something wrong...", - qx); - } -#endif - } else { - ops_printf("%s\n", - "The macroscopic velocity have " - "not been calculated before calculating the " - "temperature!"); - } - } break; - case Variable_Qy: { - bool ifCalc = true; - Real velo[LATTDIM]; - for (int d = 0; d < LATTDIM; d++) { - ifCalc = ifCalc && veloCalculated[d]; - velo[d] = macroVars(d + 1, 0, 0); - } - if (ifCalc) { - for (int xiIdx = 0; xiIdx < NUMXI; xiIdx++) { - Real T = 0; - for (int d = 0; d < LATTDIM; d++) { - T += (CS * XI[xiIdx * LATTDIM + d] - velo[d]) * - (CS * XI[xiIdx * LATTDIM + d] - velo[d]) * - f(xiIdx, 0, 0); - } - macroVars(m, 0, 0) += - (0.5 * - (CS * XI[xiIdx * LATTDIM + 1] - velo[1]) * T); - } -#ifdef debug - Real qy{macroVars(m, 0, 0)}; - if (isnan(qy) || isinf(qy)) { - ops_printf("%sQx=%f\n", - "Heatflux becomes invalid! Maybe " - "something wrong...", - qy); - } -#endif - } else { - ops_printf("%s\n", - "The density and macroscopic velocity have " - "not been calculated before calculating the " - "temperature!"); - } - } break; - case Variable_Qz: { - bool ifCalc = true; - Real velo[LATTDIM]; - for (int d = 0; d < LATTDIM; d++) { - ifCalc = ifCalc && veloCalculated[d]; - velo[d] = macroVars(d + 1, 0, 0); - } - if (ifCalc) { - for (int xiIdx = 0; xiIdx < NUMXI; xiIdx++) { - Real T = 0; - for (int d = 0; d < LATTDIM; d++) { - T += (CS * XI[xiIdx * LATTDIM + d] - velo[d]) * - (CS * XI[xiIdx * LATTDIM + d] - velo[d]) * - f(xiIdx, 0, 0); - } - macroVars(m, 0, 0) += - (0.5 * - (CS * XI[xiIdx * LATTDIM + 2] - velo[2]) * T); - } -#ifdef debug - Real qz{macroVars(m, 0, 0)}; - if (isnan(qz) || isinf(qz)) { - ops_printf("%sQx=%f\n", - "Heatflux becomes invalid! Maybe " - "something wrong...", - qz); - } + V(0, 0) = v; + } +#endif // OPS_2D +} + +void KerCalcUForce(ACC& U, const ACC& f, const ACC& nodeType, + const ACC& coordinates, const ACC& acceleration, + const ACC& Rho, const Real* dt, const int* lattIdx) { +#ifdef OPS_2D + const Real x{coordinates(0, 0, 0)}; + const Real y{coordinates(1, 0, 0)}; + VertexType vt = (VertexType)nodeType(0, 0); + if (vt != VertexType::ImmersedSolid) { + Real u{0}; + for (int xiIdx = lattIdx[0]; xiIdx <= lattIdx[1]; xiIdx++) { + u += CS * XI[xiIdx * LATTDIM] * f(xiIdx, 0, 0); + } + u /= Rho(0, 0); + if (VertexType::Fluid == vt || VertexType::MDPeriodic == vt) { + u += ((*dt) * acceleration(0, 0, 0) / 2); + } +#ifdef CPU + if (isnan(u) || isinf(u)) { + ops_printf( + "Error! Velocity U=%f becomes invalid! Maybe something wrong " + "at x=%f y=%f\n", + u, x, y); + assert(!(isnan(u) || isinf(u))); + } #endif - } else { - ops_printf("%s\n", - "The density and macroscopic velocity have " - "not been calculated before calculating the " - "temperature!"); - } - } break; - case Variable_T: { - bool ifCalc = rhoCalculated; - Real rho = macroVars(0, 0, 0); - Real velo[LATTDIM]; - for (int d = 0; d < LATTDIM; d++) { - ifCalc = ifCalc && veloCalculated[d]; - velo[d] = macroVars(d + 1, 0, 0); - } - if (ifCalc) { - for (int xiIdx = 0; xiIdx < NUMXI; xiIdx++) { - for (int d = 0; d < LATTDIM; d++) { - macroVars(m, 0, 0) += - (CS * XI[xiIdx * LATTDIM + d] - velo[d]) * - (CS * XI[xiIdx * LATTDIM + d] - velo[d]) * - f(xiIdx, 0, 0); - } - } - macroVars(m, 0, 0) /= (rho * LATTDIM); -#ifdef debug - Real T{macroVars(m, 0, 0)}; - if (isnan(T) || isinf(T)) { - ops_printf("%sW=%f\n", - "Temperature becomes invalid! Maybe " - "something wrong...", - T); - } + U(0, 0) = u; + } +#endif // OPS_2D +} + +void KerCalcVForce(ACC& V, const ACC& f, const ACC& nodeType, + const ACC& coordinates, const ACC& acceleration, + const ACC& Rho, const Real* dt, const int* lattIdx) { +#ifdef OPS_2D + const Real x{coordinates(0, 0, 0)}; + const Real y{coordinates(1, 0, 0)}; + VertexType vt = (VertexType)nodeType(0, 0); + if (vt != VertexType::ImmersedSolid) { + Real v{0}; + for (int xiIdx = lattIdx[0]; xiIdx <= lattIdx[1]; xiIdx++) { + v += CS * XI[xiIdx * LATTDIM + 1] * f(xiIdx, 0, 0); + } + v /= Rho(0, 0); + if (VertexType::Fluid == vt || VertexType::MDPeriodic == vt) { + v += ((*dt) * acceleration(1, 0, 0) / 2); + } +#ifdef CPU + if (isnan(v) || isinf(v)) { + ops_printf( + "Error! Velocity V=%f becomes invalid! Maybe something wrong " + "at x=%f y=%f\n", + v, x, y); + assert(!(isnan(v) || isinf(v))); + } #endif - } else { - ops_printf("%s\n", - "The density and macroscopic velocity have " - "not been calculated before calculating the " - "temperature!"); - } - } break; - default: - break; - } // Switch - } // m - } // isVertex -#endif //OPS_2D + V(0, 0) = v; + } +#endif // OPS_2D } +/*! + * If a Newton-Cotes quadrature is used, it can be converted to the way + * similar to the Gauss-Hermite quadrature * + */ -void KerCalcBodyForce(const Real* time, const ACC& nodeType, - const ACC& coordinates, const ACC macroVars, - ACC& bodyForce) { +void KerInitialiseBGK2nd(ACC& f, const ACC& nodeType, + const ACC& Rho, const ACC& U, + const ACC& V, const int* lattIdx) { #ifdef OPS_2D - // here we assume the force is constant - // user may introduce a function of g(r,t) for the body force - const Real g[]{0.0001, 0}; + VertexType vt = (VertexType)nodeType(0, 0); + if (vt != VertexType::ImmersedSolid) { + Real rho{Rho(0, 0)}; + Real u{U(0, 0)}; + Real v{V(0, 0)}; + const Real T{1}; + const int polyOrder{2}; + for (int xiIdx = lattIdx[0]; xiIdx <= lattIdx[1]; xiIdx++) { + f(xiIdx, 0, 0) = CalcBGKFeq(xiIdx, rho, u, v, T, polyOrder); +#ifdef CPU + const Real res{f(xiIdx, 0, 0)}; + if (isnan(res) || res <= 0 || isinf(res)) { + ops_printf( + "Error! Distribution function %f becomes invalid at the " + "lattice %i\n", + res, xiIdx); + assert(!(isnan(res) || res <= 0 || isinf(res))); + } +#endif // CPU + } + } +#endif // OPS_2D +} - for (int compoIndex = 0; compoIndex < NUMCOMPONENTS; compoIndex++) { - VertexType vt = - (VertexType)nodeType(compoIndex, 0, 0); - if (vt != VertexType::ImmersedSolid) { - BodyForceType forceType{(BodyForceType)FORCETYPE[compoIndex]}; - const int startPos{VARIABLECOMPPOS[2 * compoIndex]}; - switch (forceType) { - case BodyForce_1st: { - Real rho{macroVars(startPos, 0, 0)}; - for (int xiIndex = COMPOINDEX[2 * compoIndex]; - xiIndex <= COMPOINDEX[2 * compoIndex + 1]; xiIndex++) { - bodyForce(xiIndex, 0, 0) = - CalcBodyForce(xiIndex, rho, g); +void KerCollideBGKIsothermal(ACC& fStage, const ACC& f, + const ACC& coordinates, + const ACC& nodeType, const ACC& Rho, + const ACC& U, const ACC& V, + const Real* tauRef, + const Real* dt, const int* lattIdx) { +#ifdef OPS_2D + VertexType vt = (VertexType)nodeType(0, 0); + // collisionRequired: means if collision is required at boundary + // e.g., the ZouHe boundary condition explicitly requires collision + bool collisionRequired = (vt != VertexType::ImmersedSolid); + if (collisionRequired) { + Real rho{Rho(0, 0)}; + Real u{U(0, 0)}; + Real v{V(0, 0)}; + const Real T{1}; + const int polyOrder{2}; + Real tau = (*tauRef); + Real dtOvertauPlusdt = (*dt) / (tau + 0.5 * (*dt)); + for (int xiIndex = lattIdx[0]; xiIndex <= lattIdx[1]; xiIndex++) { + const Real feq{CalcBGKFeq(xiIndex, rho, u, v, T, polyOrder)}; + if (vt == VertexType::Fluid || vt == VertexType::MDPeriodic) { + fStage(xiIndex, 0, 0) = + feq + (1 - dtOvertauPlusdt) * (f(xiIndex, 0, 0) - feq) + + tau * dtOvertauPlusdt * fStage(xiIndex, 0, 0); + } else { + fStage(xiIndex, 0, 0) = + feq + (1 - dtOvertauPlusdt) * (f(xiIndex, 0, 0) - feq); + } #ifdef CPU - const Real res{ - bodyForce(xiIndex, 0, 0)}; - if (isnan(res) || isinf(res)) { - ops_printf( - "Error! Body force %f becomes " - "invalid for the component %i at the lattice " - "%i\n", - res, compoIndex, xiIndex); - assert(!(isnan(res) || res <= 0 || isinf(res))); - } -#endif - } - } break; - case BodyForce_None: { - for (int xiIndex = COMPOINDEX[2 * compoIndex]; - xiIndex <= COMPOINDEX[2 * compoIndex + 1]; xiIndex++) { - bodyForce(xiIndex, 0, 0) = 0; + const Real res{fStage(xiIndex, 0, 0)}; + if (isnan(res) || res <= 0 || isinf(res)) { + ops_printf( + "Error! Distribution function = %e becomes invalid at " + "the lattice %i where feq=%e and rho=%e u=%e v=%e at " + "x=%e y=%e\n", + res, xiIndex, feq, rho, u, v, coordinates(0, 0, 0), + coordinates(1, 0, 0)); + assert(!(isnan(res) || res <= 0 || isinf(res))); + } +#endif // CPU + } + } +#endif // OPS_2D +} + +void KerCollideBGKThermal(ACC& fStage, const ACC& f, + const ACC& nodeType, const ACC& Rho, + const ACC& U, const ACC& V, + const ACC& Temperature, const Real* tauRef, + const Real* dt, const int* lattIdx) { +#ifdef OPS_2D + VertexType vt = (VertexType)nodeType(0, 0); + // collisionRequired: means if collision is required at boundary + // e.g., the ZouHe boundary condition explicitly requires collision + bool collisionRequired = (vt != VertexType::ImmersedSolid); + if (collisionRequired) { + Real rho{Rho(0, 0)}; + Real u{U(0, 0)}; + Real v{V(0, 0)}; + Real T{Temperature(0, 0)}; + const int polyOrder{4}; + Real tau = (*tauRef) / (rho * sqrt(T)); + Real dtOvertauPlusdt = (*dt) / (tau + 0.5 * (*dt)); + for (int xiIndex = lattIdx[0]; xiIndex <= lattIdx[1]; xiIndex++) { + const Real feq{CalcBGKFeq(xiIndex, rho, u, v, T, polyOrder)}; + if (vt == VertexType::Fluid || vt == VertexType::MDPeriodic) { + fStage(xiIndex, 0, 0) = + f(xiIndex, 0, 0) - + dtOvertauPlusdt * (f(xiIndex, 0, 0) - feq) + + tau * dtOvertauPlusdt * fStage(xiIndex, 0, 0); + } else { + fStage(xiIndex, 0, 0) = + f(xiIndex, 0, 0) - + dtOvertauPlusdt * (f(xiIndex, 0, 0) - feq); + } #ifdef CPU - const Real res{ - bodyForce(xiIndex, 0, 0)}; - if (isnan(res) || isinf(res)) { - ops_printf( - "Error! Body force %f becomes " - "invalid for the component %i at the lattice " - "%i\n", - res, compoIndex, xiIndex); - assert(!(isnan(res) || res <= 0 || isinf(res))); - } -#endif - } - } break; - default: + const Real res{fStage(xiIndex, 0, 0)}; + if (isnan(res) || res <= 0 || isinf(res)) { + ops_printf( + "Error! Distribution function %f becomes invalid at the " + "lattice %i\n", + res, xiIndex); + assert(!(isnan(res) || res <= 0 || isinf(res))); + } +#endif // CPU + } + } +#endif // OPS_2D +} + +void KerCalcBodyForce1ST(ACC& fStage, const ACC& acceration, + const ACC& Rho, const ACC& nodeType, + const int* lattIdx) { +#ifdef OPS_2D + + VertexType vt = (VertexType)nodeType(0, 0); + if (vt == VertexType::Fluid || vt == VertexType::MDPeriodic) { + Real rho{Rho(0, 0)}; + Real g[]{acceration(0, 0, 0), acceration(1, 0, 0)}; + for (int xiIndex = lattIdx[0]; xiIndex <= lattIdx[1]; xiIndex++) { + const Real bodyForce{CalcBodyForce(xiIndex, rho, g)}; #ifdef CPU - ops_printf( - "Error! We don't deal with the chosen type of " - "force function at this moment!\n"); - assert(false); -#endif - break; + if (isnan(bodyForce) || isinf(bodyForce)) { + ops_printf( + "Error! Body force %f becomes invalid at the lattice " + "%i\n", + bodyForce, xiIndex); + assert(!(isnan(bodyForce) || isinf(bodyForce))); } +#endif + fStage(xiIndex, 0, 0) = bodyForce; } } - #endif //OPS_2D +#endif // OPS_2D } -#endif //OPS_2D outter -#ifdef OPS_3D +void KerCalcBodyForceNone(ACC& fStage, const ACC& acceration, + const ACC& nodeType, const int* lattIdx) { +#ifdef OPS_2D + VertexType vt = (VertexType)nodeType(0, 0); + if (vt == VertexType::Fluid || vt == VertexType::MDPeriodic) { + for (int xiIndex = lattIdx[0]; xiIndex <= lattIdx[1]; xiIndex++) { + fStage(xiIndex, 0, 0) = 0; + } + } +#endif // OPS_2D +} +#endif // OPS_2D outter +#ifdef OPS_3D void KerInitialiseBGK2nd3D(ACC& f, const ACC& nodeType, const ACC& Rho, const ACC& U, const ACC& V, const ACC& W, @@ -596,6 +514,7 @@ void KerCalcBodyForceNone3D(ACC& fStage, const ACC& acceration, void KerCalcDensity3D(ACC& Rho, const ACC& f, const ACC& nodeType, const int* lattIdx) { +#ifdef OPS_3D VertexType vt = (VertexType)nodeType(0, 0, 0); if (vt != VertexType::ImmersedSolid) { Real rho{0}; @@ -613,10 +532,12 @@ void KerCalcDensity3D(ACC& Rho, const ACC& f, #endif Rho(0, 0, 0) = rho; } +#endif // OPS_3D } void KerCalcU3D(ACC& U, const ACC& f, const ACC& nodeType, const ACC& Rho, const int* lattIdx) { +#ifdef OPS_3D VertexType vt = (VertexType)nodeType(0, 0, 0); if (vt != VertexType::ImmersedSolid) { Real u{0}; @@ -635,10 +556,12 @@ void KerCalcU3D(ACC& U, const ACC& f, const ACC& nodeType, #endif U(0, 0, 0) = u; } +#endif // OPS_3D } void KerCalcV3D(ACC& V, const ACC& f, const ACC& nodeType, const ACC& Rho, const int* lattIdx) { +#ifdef OPS_3D VertexType vt = (VertexType)nodeType(0, 0, 0); if (vt != VertexType::ImmersedSolid) { Real v{0}; @@ -657,10 +580,12 @@ void KerCalcV3D(ACC& V, const ACC& f, const ACC& nodeType, #endif V(0, 0, 0) = v; } +#endif // OPS_3D } void KerCalcW3D(ACC& W, const ACC& f, const ACC& nodeType, const ACC& Rho, const int* lattIdx) { +#ifdef OPS_3D VertexType vt = (VertexType)nodeType(0, 0, 0); if (vt != VertexType::ImmersedSolid) { Real w{0}; @@ -679,12 +604,14 @@ void KerCalcW3D(ACC& W, const ACC& f, const ACC& nodeType, #endif W(0, 0, 0) = w; } +#endif // OPS_3D } void KerCalcUForce3D(ACC& U, const ACC& f, const ACC& nodeType, const ACC& coordinates, const ACC& acceleration, const ACC& Rho, const Real* dt, const int* lattIdx) { +#ifdef OPS_3D const Real x{coordinates(0, 0, 0, 0)}; const Real y{coordinates(1, 0, 0, 0)}; const Real z{coordinates(2, 0, 0, 0)}; @@ -709,12 +636,14 @@ void KerCalcUForce3D(ACC& U, const ACC& f, const ACC& nodeType, #endif U(0, 0, 0) = u; } +#endif // OPS_3D } void KerCalcVForce3D(ACC& V, const ACC& f, const ACC& nodeType, const ACC& coordinates, const ACC& acceleration, const ACC& Rho, const Real* dt, const int* lattIdx) { +#ifdef OPS_3D const Real x{coordinates(0, 0, 0, 0)}; const Real y{coordinates(1, 0, 0, 0)}; const Real z{coordinates(2, 0, 0, 0)}; @@ -739,12 +668,14 @@ void KerCalcVForce3D(ACC& V, const ACC& f, const ACC& nodeType, #endif V(0, 0, 0) = v; } +#endif // OPS_3D } void KerCalcWForce3D(ACC& W, const ACC& f, const ACC& nodeType, const ACC& coordinates, const ACC& acceleration, const ACC& Rho, const Real* dt, const int* lattIdx) { +#ifdef OPS_3D const Real x{coordinates(0, 0, 0, 0)}; const Real y{coordinates(1, 0, 0, 0)}; const Real z{coordinates(2, 0, 0, 0)}; @@ -769,475 +700,8 @@ void KerCalcWForce3D(ACC& W, const ACC& f, const ACC& nodeType, #endif W(0, 0, 0) = w; } +#endif //OPS_3D } -/*! - * If a Newton-Cotes quadrature is used, it can be converted to the way - * similar to the Gauss-Hermite quadrature - * - */ -// void KerCalcMacroVars3D(ACC & macroVars, const ACC& f, -// const ACC& nodeType, -// const ACC& coordinates, -// const ACC& acceleration, const Real* dt) { -// #ifdef OPS_3D -// const Real x{coordinates(0, 0, 0, 0)}; -// const Real y{coordinates(1, 0, 0, 0)}; -// const Real z{coordinates(2, 0, 0, 0)}; -// for (int compoIndex = 0; compoIndex < NUMCOMPONENTS; compoIndex++) { -// VertexType vt = -// (VertexType)nodeType(compoIndex, 0, 0, 0); -// if (vt != VertexType::ImmersedSolid) { -// bool rhoCalculated{false}; -// Real rho{0}; -// bool* veloCalculated = new bool[LATTDIM]; -// Real* velo = new Real[LATTDIM]; -// for (int lattIdx = 0; lattIdx < LATTDIM; lattIdx++) { -// veloCalculated[lattIdx] = false; -// velo[lattIdx] = 0; -// } -// for (int m = VARIABLECOMPPOS[2 * compoIndex]; -// m <= VARIABLECOMPPOS[2 * compoIndex + 1]; m++) { -// macroVars(m, 0, 0, 0) = 0; -// VariableTypes varType = (VariableTypes)VARIABLETYPE[m]; -// switch (varType) { -// case Variable_Rho: { -// rhoCalculated = true; -// for (int xiIdx = COMPOINDEX[2 * compoIndex]; -// xiIdx <= COMPOINDEX[2 * compoIndex + 1]; xiIdx++) { -// macroVars(m, 0, 0, 0) += -// f(xiIdx, 0, 0, 0); -// } -// rho = macroVars(m, 0, 0, 0); -// #ifdef CPU -// if (isnan(rho) || rho <= 0 || isinf(rho)) { -// ops_printf( -// "Error! Density %f becomes invalid！Something " -// "wrong...", -// rho); -// ops_printf( -// "For the component %i at x=%f y=%f z=%f\n", -// compoIndex, x, y, z); -// assert(!(isnan(rho) || rho <= 0 || isinf(rho))); -// } -// #endif -// } break; -// case Variable_U: { -// if (rhoCalculated) { -// veloCalculated[0] = true; -// for (int xiIdx = COMPOINDEX[2 * compoIndex]; -// xiIdx <= COMPOINDEX[2 * compoIndex + 1]; -// xiIdx++) { -// macroVars(m, 0, 0, 0) += -// CS * XI[xiIdx * LATTDIM] * -// f(xiIdx, 0, 0, 0); -// } -// macroVars(m, 0, 0, 0) /= rho; -// velo[0] = macroVars(m, 0, 0, 0); -// #ifdef CPU -// if (isnan(velo[0]) || isinf(velo[0])) { -// ops_printf( -// "Error! Velocity U=%f becomes invalid! " -// "Maybe something wrong...\n", -// velo[0]); -// ops_printf( -// "For the component %i at x=%f y=%f z=%f\n", -// compoIndex, x, y, z); -// assert(!(isnan(velo[0]) || isinf(velo[0]))); -// } -// #endif -// } else { -// #ifdef CPU -// ops_printf( -// "Error! Density has not be calculated before " -// "calculating U! Macroscopic variables shall be " -// "defined in the order of [rho,u,v,w,...]\n"); -// assert(rhoCalculated); -// #endif -// } -// } break; -// case Variable_V: { -// if (rhoCalculated) { -// veloCalculated[1] = true; -// for (int xiIdx = COMPOINDEX[2 * compoIndex]; -// xiIdx <= COMPOINDEX[2 * compoIndex + 1]; -// xiIdx++) { -// macroVars(m, 0, 0, 0) += -// CS * XI[xiIdx * LATTDIM + 1] * -// f(xiIdx, 0, 0, 0); -// } -// macroVars(m, 0, 0, 0) /= rho; -// velo[1] = macroVars(m, 0, 0, 0); -// #ifdef CPU -// if (isnan(velo[1]) || isinf(velo[1])) { -// ops_printf( -// "Error! Velocity V=%f becomes invalid! " -// "Maybe something wrong...\n", -// velo[1]); -// ops_printf( -// "For the component %i at x=%f y=%f z=%f\n", -// compoIndex, x, y, z); -// assert(!(isnan(velo[1]) || isinf(velo[1]))); -// } -// #endif -// } else { -// #ifdef CPU -// ops_printf( -// "Error! Density has not be calculated before " -// "calculating V! Macroscopic variables shall be " -// "defined in the order of [rho,u,v,w,...]\n"); -// assert(rhoCalculated); -// #endif -// } -// } break; -// case Variable_W: { -// if (rhoCalculated) { -// veloCalculated[2] = true; -// for (int xiIdx = COMPOINDEX[2 * compoIndex]; -// xiIdx <= COMPOINDEX[2 * compoIndex + 1]; -// xiIdx++) { -// macroVars(m, 0, 0, 0) += -// CS * XI[xiIdx * LATTDIM + 2] * -// f(xiIdx, 0, 0, 0); -// } -// macroVars(m, 0, 0, 0) /= rho; -// velo[2] = macroVars(m, 0, 0, 0); -// #ifdef CPU -// if (isnan(velo[2]) || isinf(velo[2])) { -// ops_printf( -// "Error! Velocity W=%f becomes invalid! " -// "Maybe something wrong...\n", -// velo[2]); -// ops_printf( -// "For the component %i at x=%f y=%f z=%f\n", -// compoIndex, x, y, z); -// assert(!(isnan(velo[2]) || isinf(velo[2]))); -// } -// #endif -// } else { -// #ifdef CPU -// ops_printf( -// "Error! Density has not be calculated before " -// "calculating W! Macroscopic variables shall be " -// "defined in the order of [rho,u,v,w,...]\n"); -// assert(rhoCalculated); -// #endif -// } -// } break; -// case Variable_Qx: { -// bool ifCalc = true; -// for (int d = 0; d < LATTDIM; d++) { -// ifCalc = ifCalc && veloCalculated[d]; -// } -// if (ifCalc) { -// for (int xiIdx = COMPOINDEX[2 * compoIndex]; -// xiIdx <= COMPOINDEX[2 * compoIndex + 1]; -// xiIdx++) { -// Real T = 0; -// for (int d = 0; d < LATTDIM; d++) { -// T += (CS * XI[xiIdx * LATTDIM + d] - -// velo[d]) * -// (CS * XI[xiIdx * LATTDIM + d] - -// velo[d]) * -// f(xiIdx, 0, 0, 0); -// } -// macroVars(m, 0, 0, 0) += -// (0.5 * -// (CS * XI[xiIdx * LATTDIM] - velo[0]) * T); -// } -// #ifdef CPU -// Real qx{macroVars(m, 0, 0, 0)}; -// if (isnan(qx) || isinf(qx)) { -// ops_printf( -// "Error!Heat flux qx=%f becomes invalid! " -// "Maybe something wrong...\n", -// qx); -// ops_printf( -// "For the component %i at x=%f y=%f z=%f\n", -// compoIndex, x, y, z); -// assert(!(isnan(qx) || isinf(qx))); -// } -// #endif -// } else { -// #ifdef CPU -// ops_printf( -// "Error! The density and macroscopic velocity " -// "have not been calculated! Macroscopic " -// "variables shall be defined in the order of " -// "[rho,u,v,w,T,Qx,Qy...]\n"); -// assert(ifCalc); -// #endif -// } -// } break; -// case Variable_Qy: { -// bool ifCalc = true; -// for (int d = 0; d < LATTDIM; d++) { -// ifCalc = ifCalc && veloCalculated[d]; -// } -// if (ifCalc) { -// for (int xiIdx = COMPOINDEX[2 * compoIndex]; -// xiIdx <= COMPOINDEX[2 * compoIndex + 1]; -// xiIdx++) { -// Real T = 0; -// for (int d = 0; d < LATTDIM; d++) { -// T += (CS * XI[xiIdx * LATTDIM + d] - -// velo[d]) * -// (CS * XI[xiIdx * LATTDIM + d] - -// velo[d]) * -// f(xiIdx, 0, 0, 0); -// } -// macroVars(m, 0, 0, 0) += -// (0.5 * -// (CS * XI[xiIdx * LATTDIM + 1] - velo[1]) * -// T); -// } -// #ifdef CPU -// Real qy{macroVars(m, 0, 0, 0)}; -// if (isnan(qy) || isinf(qy)) { -// ops_printf( -// "Error! Heat flux qy=%f becomes invalid! " -// "Maybe something wrong...\n", -// qy); -// ops_printf( -// "For the component %i at x=%f y=%f z=%f\n", -// compoIndex, x, y, z); -// assert(!(isnan(qy) || isinf(qy))); -// } -// #endif -// } else { -// #ifdef CPU -// ops_printf( -// "Error! The density and macroscopic velocity " -// "have not been calculated! Macroscopic " -// "variables shall be defined in the order of " -// "[rho,u,v,w,T,Qx,Qy...]\n"); -// assert(ifCalc); -// #endif -// } -// } break; -// case Variable_Qz: { -// bool ifCalc = true; -// for (int d = 0; d < LATTDIM; d++) { -// ifCalc = ifCalc && veloCalculated[d]; -// } -// if (ifCalc) { -// for (int xiIdx = COMPOINDEX[2 * compoIndex]; -// xiIdx <= COMPOINDEX[2 * compoIndex + 1]; -// xiIdx++) { -// Real T = 0; -// for (int d = 0; d < LATTDIM; d++) { -// T += (CS * XI[xiIdx * LATTDIM + d] - -// velo[d]) * -// (CS * XI[xiIdx * LATTDIM + d] - -// velo[d]) * -// f(xiIdx, 0, 0, 0); -// } -// macroVars(m, 0, 0, 0) += -// (0.5 * -// (CS * XI[xiIdx * LATTDIM + 2] - velo[2]) * -// T); -// } -// #ifdef CPU -// Real qz{macroVars(m, 0, 0, 0)}; -// if (isnan(qz) || isinf(qz)) { -// ops_printf( -// "Error! Heat flux qz=%f becomes invalid! " -// "Maybe something wrong...\n", -// qz); -// ops_printf( -// "For the component %i at x=%f y=%f z=%f\n", -// compoIndex, x, y, z); -// assert(!(isnan(qz) || isinf(qz))); -// } -// #endif -// } else { -// #ifdef CPU -// ops_printf( -// "Error! The density and macroscopic velocity " -// "have not been calculated! Macroscopic " -// "variables shall be defined in the order of " -// "[rho,u,v,w,T,Qx,Qy...]\n"); -// assert(ifCalc); -// #endif -// } -// } break; -// case Variable_T: { -// bool ifCalc = rhoCalculated; -// for (int d = 0; d < LATTDIM; d++) { -// ifCalc = ifCalc && veloCalculated[d]; -// } -// if (ifCalc) { -// for (int xiIdx = COMPOINDEX[2 * compoIndex]; -// xiIdx <= COMPOINDEX[2 * compoIndex + 1]; -// xiIdx++) { -// for (int d = 0; d < LATTDIM; d++) { -// macroVars(m, 0, 0, 0) += -// (CS * XI[xiIdx * LATTDIM + d] - -// velo[d]) * -// (CS * XI[xiIdx * LATTDIM + d] - -// velo[d]) * -// f(xiIdx, 0, 0, 0); -// } -// } -// macroVars(m, 0, 0, 0) /= -// (rho * LATTDIM); -// #ifdef CPU -// Real T{macroVars(m, 0, 0, 0)}; -// if (isnan(T) || isinf(T)) { -// ops_printf( -// "Error! Temperature T=%f becomes invalid! " -// "Maybe something wrong...\n", -// T); -// ops_printf( -// "For the component %i at x=%f y=%f z=%f\n", -// compoIndex, x, y, z); -// assert(!(isnan(T) || isinf(T))); -// } -// #endif -// } else { -// #ifdef CPU -// ops_printf( -// "Error! The density and macroscopic velocity " -// "have not been calculated! Macroscopic " -// "variables shall be defined in the order of " -// "[rho,u,v,w,T,Qx,Qy...]\n"); -// assert(ifCalc); -// #endif -// } -// } break; -// case Variable_U_Force: { -// if (rhoCalculated) { -// veloCalculated[0] = true; -// for (int xiIdx = COMPOINDEX[2 * compoIndex]; -// xiIdx <= COMPOINDEX[2 * compoIndex + 1]; -// xiIdx++) { -// macroVars(m, 0, 0, 0) += -// CS * XI[xiIdx * LATTDIM] * -// f(xiIdx, 0, 0, 0); -// } -// macroVars(m, 0, 0, 0) /= rho; -// if (VertexType::Fluid == vt || -// VertexType::MDPeriodic == vt) { -// macroVars(m, 0, 0, 0) += -// ((*dt) * -// acceleration(3* compoIndex,0,0,0) / 2); -// } -// velo[0] = macroVars(m, 0, 0, 0); -// #ifdef CPU -// if (isnan(velo[0]) || isinf(velo[0])) { -// ops_printf( -// "Error! Velocity U=%f becomes invalid! " -// "Maybe something wrong...\n", -// velo[0]); -// ops_printf( -// "For the component %i at x=%f y=%f z=%f\n", -// compoIndex, x, y, z); -// assert(!(isnan(velo[0]) || isinf(velo[0]))); -// } -// #endif -// } else { -// #ifdef CPU -// ops_printf( -// "Error! Density has not be calculated " -// "before " -// "calculating U! Macroscopic variables " -// "shall be " -// "defined in the order of " -// "[rho,u,v,w,...]\n"); -// assert(rhoCalculated); -// #endif -// } -// } break; -// case Variable_V_Force: { -// if (rhoCalculated) { -// veloCalculated[1] = true; -// for (int xiIdx = COMPOINDEX[2 * compoIndex]; -// xiIdx <= COMPOINDEX[2 * compoIndex + 1]; -// xiIdx++) { -// macroVars(m, 0, 0, 0) += -// CS * XI[xiIdx * LATTDIM + 1] * -// f(xiIdx, 0, 0, 0); -// } -// macroVars(m, 0, 0, 0) /= rho; -// if (VertexType::Fluid == vt || -// VertexType::MDPeriodic == vt) { -// macroVars(m, 0, 0, 0) += -// ((*dt) * -// acceleration(3* compoIndex + 1,0,0,0) / -// 2); -// } -// velo[1] = macroVars(m, 0, 0, 0); -// #ifdef CPU -// if (isnan(velo[1]) || isinf(velo[1])) { -// ops_printf("%sV=%f\n", -// "Velocity becomes invalid! Maybe " -// "something wrong...", -// velo[1]); -// ops_printf( -// "For the component %i at x=%f y=%f z=%f\n", -// compoIndex, x, y, z); -// assert(!(isnan(velo[1]) || isinf(velo[1]))); -// } -// #endif -// } else { -// #ifdef CPU -// ops_printf( -// "Error! Density has not be calculated before " -// "calculating V! Macroscopic variables shall be " -// "defined in the order of [rho,u,v,w,...]\n"); -// assert(rhoCalculated); -// #endif -// } -// } break; -// case Variable_W_Force: { -// if (rhoCalculated) { -// veloCalculated[2] = true; -// for (int xiIdx = COMPOINDEX[2 * compoIndex]; -// xiIdx <= COMPOINDEX[2 * compoIndex + 1]; -// xiIdx++) { -// macroVars(m, 0, 0, 0) += -// CS * XI[xiIdx * LATTDIM + 2] * -// f(xiIdx, 0, 0, 0); -// } -// macroVars(m, 0, 0, 0) /= rho; -// if (VertexType::Fluid == vt || -// VertexType::MDPeriodic == vt) { -// macroVars(m, 0, 0, 0) += -// ((*dt) * -// acceleration(3 * compoIndex + 2, 0, 0, 0) / -// 2); -// } -// velo[2] = macroVars(m, 0, 0, 0); -// #ifdef CPU -// if (isnan(velo[2]) || isinf(velo[2])) { -// ops_printf( -// "Error! Velocity W=%f becomes invalid! " -// "Maybe something wrong...\n", -// velo[2]); -// ops_printf( -// "For the component %i at x=%f y=%f z=%f\n", -// compoIndex, x, y, z); -// assert(!(isnan(velo[2]) || isinf(velo[2]))); -// } -// #endif -// } else { -// #ifdef CPU -// ops_printf( -// "Error! Density has not be calculated before " -// "calculating W! Macroscopic variables shall be " -// "defined in the order of [rho,u,v,w,...]\n"); -// assert(rhoCalculated); -// #endif -// } -// } break; -// default: -// break; -// } // Switch -// } // m -// delete[] veloCalculated; -// delete[] velo; -// } // compoIdx -// } // isVertex -// #endif // OPS_3D -// } #endif //OPS_3D outter + #endif //MODEL_KERNEL_INC \ No newline at end of file diff --git a/Src/model_wrapper.cpp b/Src/model_wrapper.cpp index c85b7a4..29e7aca 100644 --- a/Src/model_wrapper.cpp +++ b/Src/model_wrapper.cpp @@ -7,7 +7,9 @@ #include "ops_seq_v2.h" #include "model_kernel.inc" #ifdef OPS_3D + void PreDefinedCollision3D() { +#ifdef OPS_3D for (const auto& idBlock : g_Block()) { const Block& block{idBlock.second}; std::vector iterRng; @@ -80,9 +82,11 @@ void PreDefinedCollision3D() { } } } +#endif // OPS_3D } void UpdateMacroVars3D() { +#ifdef OPS_3D for (const auto& idBlock : g_Block()) { const Block& block{idBlock.second}; std::vector iterRng; @@ -243,9 +247,11 @@ void UpdateMacroVars3D() { } } } +#endif // OPS_3D } void PreDefinedBodyForce3D() { +#ifdef OPS_3D for (const auto& idBlock : g_Block()) { const Block& block{idBlock.second}; std::vector iterRng; @@ -292,9 +298,11 @@ void PreDefinedBodyForce3D() { } } } +#endif // OPS_3D } void PreDefinedInitialCondition3D() { +#ifdef OPS_3D for (const auto& idBlock : g_Block()) { const Block& block{idBlock.second}; std::vector iterRng; @@ -336,5 +344,389 @@ void PreDefinedInitialCondition3D() { if (!IsTransient()) { CopyCurrentMacroVar(); } +#endif // OPS_3D } -#endif //OPS_3D \ No newline at end of file + +void PreDefinedCollision3D(int* velID, int* loop, Real tauRef, + CollisionType collisionType,int componentId, int rhoId, int Tid) { + + #ifdef OPS_3D + int iDx = velID[0]; + int iDy = velID[1]; + int iDz = velID[2]; + + for (const auto& idBlock : g_Block()) { + const Block& block{idBlock.second}; + std::vector iterRng; + iterRng.assign(block.WholeRange().begin(), block.WholeRange().end()); + const int blockIndex{block.ID()}; + const Real* pdt{pTimeStep()}; + switch (collisionType) { + case Collision_BGKIsothermal2nd: + ops_par_loop( + KerCollideBGKIsothermal3D, "KerCollideBGKIsothermal3D", + block.Get(), SpaceDim(), iterRng.data(), + ops_arg_dat(g_fStage()[blockIndex], NUMXI, LOCALSTENCIL, + "double", OPS_WRITE), + ops_arg_dat(g_f()[blockIndex], NUMXI, LOCALSTENCIL, + "double", OPS_READ), + ops_arg_dat(g_CoordinateXYZ()[blockIndex], SpaceDim(), + LOCALSTENCIL, "double", OPS_READ), + ops_arg_dat(g_NodeType().at(componentId).at(blockIndex), 1, + LOCALSTENCIL, "int", OPS_READ), + ops_arg_dat(g_MacroVars() + .at(rhoId).at(blockIndex), + 1, LOCALSTENCIL, "double", OPS_READ), + ops_arg_dat(g_MacroVars().at(iDx).at(blockIndex), + 1, LOCALSTENCIL, "double", OPS_READ), + ops_arg_dat(g_MacroVars().at(iDy).at(blockIndex), + 1, LOCALSTENCIL, "double", OPS_READ), + ops_arg_dat(g_MacroVars().at(iDz).at(blockIndex), + 1, LOCALSTENCIL, "double", OPS_READ), + ops_arg_gbl(&tauRef, 1, "double", OPS_READ), + ops_arg_gbl(pdt, 1, "double", OPS_READ), + ops_arg_gbl(loop, 2, "int", OPS_READ)); + break; + case Collision_BGKThermal4th: + ops_par_loop( + KerCollideBGKThermal3D, "KerCollideBGKThermal3D", + block.Get(), SpaceDim(), iterRng.data(), + ops_arg_dat(g_fStage()[blockIndex], NUMXI, LOCALSTENCIL, + "double", OPS_WRITE), + ops_arg_dat(g_f()[blockIndex], NUMXI, LOCALSTENCIL, + "double", OPS_READ), + ops_arg_dat(g_NodeType().at(componentId).at(blockIndex), 1, + LOCALSTENCIL, "int", OPS_READ), + ops_arg_dat(g_MacroVars() + .at(rhoId).at(blockIndex), + 1, LOCALSTENCIL, "double", OPS_READ), + ops_arg_dat(g_MacroVars().at(iDx).at(blockIndex), + 1, LOCALSTENCIL, "double", OPS_READ), + ops_arg_dat(g_MacroVars().at(iDy).at(blockIndex), + 1, LOCALSTENCIL, "double", OPS_READ), + ops_arg_dat(g_MacroVars().at(iDz).at(blockIndex), + 1, LOCALSTENCIL, "double", OPS_READ), + ops_arg_dat(g_MacroVars() + .at(Tid).at(blockIndex), + 1, LOCALSTENCIL, "double", OPS_READ), + ops_arg_gbl(&tauRef, 1, "double", OPS_READ), + ops_arg_gbl(pdt, 1, "double", OPS_READ), + ops_arg_gbl(loop, 2, "int", OPS_READ)); + break; + default: + ops_printf( + "The specified collision type is not implemented!\n"); + break; + + } + + } + + + + +#endif //OPS_3D +} + + + + +#endif // OPS_3D outter +#ifdef OPS_2D +void PreDefinedCollision() { +#ifdef OPS_2D + for (const auto& idBlock : g_Block()) { + const Block& block{idBlock.second}; + std::vector iterRng; + iterRng.assign(block.WholeRange().begin(), block.WholeRange().end()); + const int blockIndex{block.ID()}; + for (const auto& idCompo : g_Components()) { + const Component& compo{idCompo.second}; + const CollisionType collisionType{compo.collisionType}; + const Real tau{compo.tauRef}; + const Real* pdt{pTimeStep()}; + switch (collisionType) { + case Collision_BGKIsothermal2nd: + ops_par_loop( + KerCollideBGKIsothermal, "KerCollideBGKIsothermal", + block.Get(), SpaceDim(), iterRng.data(), + ops_arg_dat(g_fStage()[blockIndex], NUMXI, LOCALSTENCIL, + "double", OPS_WRITE), + ops_arg_dat(g_f()[blockIndex], NUMXI, LOCALSTENCIL, + "double", OPS_READ), + ops_arg_dat(g_CoordinateXYZ()[blockIndex], SpaceDim(), + LOCALSTENCIL, "double", OPS_READ), + ops_arg_dat(g_NodeType().at(compo.id).at(blockIndex), 1, + LOCALSTENCIL, "int", OPS_READ), + ops_arg_dat(g_MacroVars() + .at(compo.macroVars.at(Variable_Rho).id) + .at(blockIndex), + 1, LOCALSTENCIL, "double", OPS_READ), + ops_arg_dat(g_MacroVars().at(compo.uId).at(blockIndex), + 1, LOCALSTENCIL, "double", OPS_READ), + ops_arg_dat(g_MacroVars().at(compo.vId).at(blockIndex), + 1, LOCALSTENCIL, "double", OPS_READ), + ops_arg_gbl(&tau, 1, "double", OPS_READ), + ops_arg_gbl(pdt, 1, "double", OPS_READ), + ops_arg_gbl(compo.index, 2, "int", OPS_READ)); + break; + case Collision_BGKThermal4th: + ops_par_loop( + KerCollideBGKThermal, "KerCollideBGKThermal", + block.Get(), SpaceDim(), iterRng.data(), + ops_arg_dat(g_fStage()[blockIndex], NUMXI, LOCALSTENCIL, + "double", OPS_WRITE), + ops_arg_dat(g_f()[blockIndex], NUMXI, LOCALSTENCIL, + "double", OPS_READ), + ops_arg_dat(g_NodeType().at(compo.id).at(blockIndex), 1, + LOCALSTENCIL, "int", OPS_READ), + ops_arg_dat(g_MacroVars() + .at(compo.macroVars.at(Variable_Rho).id) + .at(blockIndex), + 1, LOCALSTENCIL, "double", OPS_READ), + ops_arg_dat(g_MacroVars().at(compo.uId).at(blockIndex), + 1, LOCALSTENCIL, "double", OPS_READ), + ops_arg_dat(g_MacroVars().at(compo.vId).at(blockIndex), + 1, LOCALSTENCIL, "double", OPS_READ), + ops_arg_dat(g_MacroVars() + .at(compo.macroVars.at(Variable_T).id) + .at(blockIndex), + 1, LOCALSTENCIL, "double", OPS_READ), + ops_arg_gbl(&tau, 1, "double", OPS_READ), + ops_arg_gbl(pdt, 1, "double", OPS_READ), + ops_arg_gbl(compo.index, 2, "int", OPS_READ)); + break; + default: + ops_printf( + "The specified collision type is not implemented!\n"); + break; + } + } + } +#endif // OPS_2D +} + + + +void UpdateMacroVars() { +#ifdef OPS_2D + for (const auto& idBlock : g_Block()) { + const Block& block{idBlock.second}; + std::vector iterRng; + iterRng.assign(block.WholeRange().begin(), block.WholeRange().end()); + const int blockIndex{block.ID()}; + const Real* pdt{pTimeStep()}; + for (const auto& idCompo : g_Components()) { + const Component& compo{idCompo.second}; + for (auto& macroVar : compo.macroVars) { + const int varId{macroVar.second.id}; + const VariableTypes varType{macroVar.first}; + switch (varType) { + case Variable_Rho: + ops_par_loop( + KerCalcDensity, "KerCalcDensity", block.Get(), + SpaceDim(), iterRng.data(), + ops_arg_dat(g_MacroVars().at(varId).at(blockIndex), + 1, LOCALSTENCIL, "double", OPS_RW), + ops_arg_dat(g_f()[blockIndex], NUMXI, LOCALSTENCIL, + "double", OPS_READ), + ops_arg_dat( + g_NodeType().at(compo.id).at(blockIndex), 1, + LOCALSTENCIL, "int", OPS_READ), + ops_arg_gbl(compo.index, 2, "int", OPS_READ)); + break; + case Variable_U: + ops_par_loop( + KerCalcU, "KerCalcU", block.Get(), SpaceDim(), + iterRng.data(), + ops_arg_dat(g_MacroVars().at(varId).at(blockIndex), + 1, LOCALSTENCIL, "double", OPS_RW), + ops_arg_dat(g_f()[blockIndex], NUMXI, LOCALSTENCIL, + "double", OPS_READ), + ops_arg_dat( + g_NodeType().at(compo.id).at(blockIndex), 1, + LOCALSTENCIL, "int", OPS_READ), + ops_arg_dat( + g_MacroVars() + .at(compo.macroVars.at(Variable_Rho).id) + .at(blockIndex), + 1, LOCALSTENCIL, "double", OPS_READ), + ops_arg_gbl(compo.index, 2, "int", OPS_READ)); + break; + case Variable_V: + ops_par_loop( + KerCalcV, "KerCalcV", block.Get(), SpaceDim(), + iterRng.data(), + ops_arg_dat(g_MacroVars().at(varId).at(blockIndex), + 1, LOCALSTENCIL, "double", OPS_RW), + ops_arg_dat(g_f()[blockIndex], NUMXI, LOCALSTENCIL, + "double", OPS_READ), + ops_arg_dat( + g_NodeType().at(compo.id).at(blockIndex), 1, + LOCALSTENCIL, "int", OPS_READ), + ops_arg_dat( + g_MacroVars() + .at(compo.macroVars.at(Variable_Rho).id) + .at(blockIndex), + 1, LOCALSTENCIL, "double", OPS_READ), + ops_arg_gbl(compo.index, 2, "int", OPS_READ)); + break; + case Variable_U_Force: + ops_par_loop( + KerCalcUForce, "KerCalcUForce", block.Get(), + SpaceDim(), iterRng.data(), + ops_arg_dat(g_MacroVars().at(varId).at(blockIndex), + 1, LOCALSTENCIL, "double", OPS_RW), + ops_arg_dat(g_f()[blockIndex], NUMXI, LOCALSTENCIL, + "double", OPS_READ), + ops_arg_dat( + g_NodeType().at(compo.id).at(blockIndex), 1, + LOCALSTENCIL, "int", OPS_READ), + ops_arg_dat(g_CoordinateXYZ()[blockIndex], + SpaceDim(), LOCALSTENCIL, "double", + OPS_READ), + ops_arg_dat( + g_MacroBodyforce().at(compo.id).at(blockIndex), + SpaceDim(), LOCALSTENCIL, "double", OPS_READ), + ops_arg_dat( + g_MacroVars() + .at(compo.macroVars.at(Variable_Rho).id) + .at(blockIndex), + 1, LOCALSTENCIL, "double", OPS_READ), + ops_arg_gbl(pdt, 1, "double", OPS_READ), + ops_arg_gbl(compo.index, 2, "int", OPS_READ)); + break; + case Variable_V_Force: + ops_par_loop( + KerCalcVForce, "KerCalcVForce", block.Get(), + SpaceDim(), iterRng.data(), + ops_arg_dat(g_MacroVars().at(varId).at(blockIndex), + 1, LOCALSTENCIL, "double", OPS_RW), + ops_arg_dat(g_f()[blockIndex], NUMXI, LOCALSTENCIL, + "double", OPS_READ), + ops_arg_dat( + g_NodeType().at(compo.id).at(blockIndex), 1, + LOCALSTENCIL, "int", OPS_READ), + ops_arg_dat(g_CoordinateXYZ()[blockIndex], + SpaceDim(), LOCALSTENCIL, "double", + OPS_READ), + ops_arg_dat( + g_MacroBodyforce().at(compo.id).at(blockIndex), + SpaceDim(), LOCALSTENCIL, "double", OPS_READ), + ops_arg_dat( + g_MacroVars() + .at(compo.macroVars.at(Variable_Rho).id) + .at(blockIndex), + 1, LOCALSTENCIL, "double", OPS_READ), + ops_arg_gbl(pdt, 1, "double", OPS_READ), + ops_arg_gbl(compo.index, 2, "int", OPS_READ)); + break; + default: + break; + } + } + } + } +#endif // OPS_2D +} + + +void PreDefinedBodyForce() { +#ifdef OPS_2D + for (const auto& idBlock : g_Block()) { + const Block& block{idBlock.second}; + std::vector iterRng; + iterRng.assign(block.WholeRange().begin(), block.WholeRange().end()); + const int blockIndex{block.ID()}; + for (const auto& idCompo : g_Components()) { + const Component& compo{idCompo.second}; + const BodyForceType forceType{compo.bodyForceType}; + switch (forceType) { + case BodyForce_1st: + ops_par_loop( + KerCalcBodyForce1ST, "KerCalcBodyForce1ST", + block.Get(), SpaceDim(), iterRng.data(), + ops_arg_dat(g_fStage()[blockIndex], NUMXI, LOCALSTENCIL, + "double", OPS_WRITE), + ops_arg_dat( + g_MacroBodyforce().at(compo.id).at(blockIndex), + SpaceDim(), LOCALSTENCIL, "double", OPS_READ), + ops_arg_dat(g_MacroVars() + .at(compo.macroVars.at(Variable_Rho).id) + .at(blockIndex), + 1, LOCALSTENCIL, "double", OPS_RW), + ops_arg_dat(g_NodeType().at(compo.id).at(blockIndex), 1, + LOCALSTENCIL, "int", OPS_READ), + ops_arg_gbl(compo.index, 2, "int", OPS_READ)); + break; + case BodyForce_None: + ops_par_loop( + KerCalcBodyForceNone, "KerCalcBodyForceNone", + block.Get(), SpaceDim(), iterRng.data(), + ops_arg_dat(g_fStage()[blockIndex], NUMXI, LOCALSTENCIL, + "double", OPS_WRITE), + ops_arg_dat( + g_MacroBodyforce().at(compo.id).at(blockIndex), + SpaceDim(), LOCALSTENCIL, "double", OPS_READ), + ops_arg_dat(g_NodeType().at(compo.id).at(blockIndex), 1, + LOCALSTENCIL, "int", OPS_READ), + ops_arg_gbl(compo.index, 2, "int", OPS_READ)); + break; + default: + ops_printf( + "The specified force type is not implemented!\n"); + break; + } + } + } +#endif // OPS_2D +} + + + +void PreDefinedInitialCondition() { +#ifdef OPS_2D + for (const auto& idBlock : g_Block()) { + const Block& block{idBlock.second}; + std::vector iterRng; + iterRng.assign(block.WholeRange().begin(), block.WholeRange().end()); + const int blockIndex{block.ID()}; + for (const auto& idCompo : g_Components()) { + const Component& compo{idCompo.second}; + const int compoId{compo.id}; + const InitialType initialType{compo.initialType}; + switch (initialType) { + case Initial_BGKFeq2nd: { + ops_par_loop( + KerInitialiseBGK2nd, "KerInitialiseBGK2nd", + block.Get(), SpaceDim(), iterRng.data(), + ops_arg_dat(g_f()[blockIndex], NUMXI, LOCALSTENCIL, + "double", OPS_WRITE), + ops_arg_dat(g_NodeType().at(compoId).at(blockIndex), 1, + LOCALSTENCIL, "int", OPS_READ), + ops_arg_dat(g_MacroVars() + .at(compo.macroVars.at(Variable_Rho).id) + .at(blockIndex), + 1, LOCALSTENCIL, "double", OPS_READ), + ops_arg_dat(g_MacroVars().at(compo.uId).at(blockIndex), + 1, LOCALSTENCIL, "double", OPS_READ), + ops_arg_dat(g_MacroVars().at(compo.vId).at(blockIndex), + 1, LOCALSTENCIL, "double", OPS_READ), + ops_arg_gbl(compo.index, 2, "int", OPS_READ)); + } break; + default: + ops_printf( + "The specified initial type is not implemented!\n"); + break; + } + } + } + // TODO this may be better arranged. + if (!IsTransient()) { + CopyCurrentMacroVar(); + } +#endif // OPS_2D +} + +#endif //OPS_2D outter + diff --git a/Src/mplb.h b/Src/mplb.h index 137f33f..4f3f077 100644 --- a/Src/mplb.h +++ b/Src/mplb.h @@ -39,7 +39,7 @@ #include "type.h" #include "flowfield.h" #ifdef OPS_3D -#include "evolution3d.h" +#include "evolution.h" #endif #ifdef OPS_2D #include "evolution.h" diff --git a/Src/scheme.h b/Src/scheme.h index e9d7deb..51a7a54 100644 --- a/Src/scheme.h +++ b/Src/scheme.h @@ -75,4 +75,8 @@ const SchemeType Scheme(); #ifdef OPS_3D void Stream3D(); #endif //OPS_3D + +#ifdef OPS_2D +void Stream(); +#endif //OPS_2D #endif diff --git a/Src/scheme_kernel.inc b/Src/scheme_kernel.inc index 645e934..7d0bcf0 100644 --- a/Src/scheme_kernel.inc +++ b/Src/scheme_kernel.inc @@ -39,7 +39,6 @@ * supported currently **/ - #ifndef SCHEME_KERNEL_INC #define SCHEME_KERNEL_INC #include "ops_lib_core.h" @@ -51,896 +50,913 @@ #include "model.h" #ifdef OPS_2D // two dimensional code -void KerCollide(const Real* dt, const int* nodeType, const Real* f, - const Real* feq, const Real* relaxationTime, - const Real* bodyForce, Real* fStage) { - VertexType vt = (VertexType)nodeType(0, 0); - // collisionRequired: means if collision is required at boundary - // e.g., the ZouHe boundary condition explicitly requires collision - bool collisionRequired = (vt != VertexType::ImmersedSolid); - if (collisionRequired) { - for (int compoIndex = 0; compoIndex < NUMCOMPONENTS; compoIndex++) { - Real tau = relaxationTime(compoIndex, 0, 0); - Real dtOvertauPlusdt = (*dt) / (tau + 0.5 * (*dt)); - for (int xiIndex = COMPOINDEX[2 * compoIndex]; - xiIndex <= COMPOINDEX[2 * compoIndex + 1]; xiIndex++) { - fStage(xiIndex, 0, 0) = - f(xiIndex, 0, 0) - - dtOvertauPlusdt * (f(xiIndex, 0, 0) - feq(xiIndex, 0, 0)) + - tau * dtOvertauPlusdt * bodyForce(xiIndex, 0, 0); - } - } - } -} - -void KerStream(ACC& f, const ACC& fStage, const ACC& nodeType, - const ACC& geometry) { +void KerStream(ACC&f, const ACC&fStage, const ACC&nodeType, + const ACC&geometry, const int *lattIdx) { #ifdef OPS_2D - // ops_printf("Inside stream kernel!!!!!. \n"); VertexType vt = (VertexType)nodeType(0, 0); - VertexGeometryTypes vg = (VertexGeometryTypes)geometry(0, 0); - for (int compoIndex = 0; compoIndex < NUMCOMPONENTS; compoIndex++) { - for (int xiIndex = COMPOINDEX[2 * compoIndex]; - xiIndex <= COMPOINDEX[2 * compoIndex + 1]; xiIndex++) { - int cx = (int)XI[xiIndex * LATTDIM]; - int cy = (int)XI[xiIndex * LATTDIM + 1]; + VertexGeometryType vg = (VertexGeometryType)geometry(0, 0); + for (int xiIndex = lattIdx[0]; xiIndex <= lattIdx[1]; xiIndex++) { + int cx = (int)XI[xiIndex * LATTDIM]; + int cy = (int)XI[xiIndex * LATTDIM + 1]; - if (vt == VertexType::Fluid) { - f(xiIndex, 0, 0) = fStage(xiIndex, -cx, -cy); - continue; - } - //Block boundary and immersed boundary - if (vt != VertexType::ImmersedSolid && vt != VertexType::Fluid) { - // TODO te be determined if necessary - bool streamRequired{true}; + if (vt == VertexType::Fluid || vt == VertexType::MDPeriodic || + vt == VertexType::VirtualBoundary) { + f(xiIndex, 0, 0) = fStage(xiIndex, -cx, -cy); + continue; + } + // Block boundary and immersed boundary + if (vt != VertexType::ImmersedSolid && vt != VertexType::Fluid && + vt != VertexType::VirtualBoundary && vt != VertexType::MDPeriodic) { + // this is reserved in case some boundary conditions + // might not allow stream + bool streamRequired{true}; - if (streamRequired) { - if ((cx == 0) && (cy == 0)) { - f(xiIndex, 0, 0) = fStage(xiIndex, 0, 0); - continue; - } + if (streamRequired) { + if ((cx == 0) && (cy == 0)) { + f(xiIndex, 0, 0) = fStage(xiIndex, 0, 0); + continue; } - switch (vg) { - case VG_IP: - // (cx=0 means stream is implemented at i=0,so here we - // disable the step at boundary) - if (streamRequired) { - if (cx <= 0) { - f(xiIndex, 0, 0) = fStage(xiIndex, -cx, -cy); - } - } else { - if (cx < 0) { - f(xiIndex, 0, 0) = fStage(xiIndex, -cx, -cy); - } - } - break; - case VG_IM: - if (streamRequired) { - if (cx >= 0) { - f(xiIndex, 0, 0) = fStage(xiIndex, -cx, -cy); - } - } else { - if (cx > 0) { - f(xiIndex, 0, 0) = fStage(xiIndex, -cx, -cy); - } - } - break; - case VG_JP: - if (streamRequired) { - if (cy <= 0) { - f(xiIndex, 0, 0) = fStage(xiIndex, -cx, -cy); - } - } else { - if (cy < 0) { - f(xiIndex, 0, 0) = fStage(xiIndex, -cx, -cy); - } - } - break; - case VG_JM: - if (streamRequired) { - if (cy >= 0) { - f(xiIndex, 0, 0) = fStage(xiIndex, -cx, -cy); - } - } else { - if (cy > 0) { - f(xiIndex, 0, 0) = fStage(xiIndex, -cx, -cy); - } - } - break; - case VG_IPJP_I: - if (streamRequired) { - if (cy <= 0 && cx <= 0) { - f(xiIndex, 0, 0) = fStage(xiIndex, -cx, -cy); - } - } else { - if (cy < 0 && cx < 0) { - f(xiIndex, 0, 0) = fStage(xiIndex, -cx, -cy); - } - } - break; - case VG_IPJM_I: - if (streamRequired) { - if (cy >= 0 && cx <= 0) { - f(xiIndex, 0, 0) = fStage(xiIndex, -cx, -cy); - } - } else { - if (cy > 0 && cx < 0) { - f(xiIndex, 0, 0) = fStage(xiIndex, -cx, -cy); - } - } - break; - case VG_IMJP_I: - if (streamRequired) { - if (cy <= 0 && cx >= 0) { - f(xiIndex, 0, 0) = fStage(xiIndex, -cx, -cy); - } - } else { - if (cy < 0 && cx > 0) { - f(xiIndex, 0, 0) = fStage(xiIndex, -cx, -cy); - } - } - break; - case VG_IMJM_I: - if (streamRequired) { - if (cy >= 0 && cx >= 0) { - f(xiIndex, 0, 0) = fStage(xiIndex, -cx, -cy); - } - } else { - if (cy > 0 && cx > 0) { - f(xiIndex, 0, 0) = fStage(xiIndex, -cx, -cy); - } - } - break; - case VG_IPJP_O: - if (cy < 0 || cx < 0) { - f(xiIndex, 0, 0) = fStage(xiIndex, -cx, -cy); - } - break; - case VG_IPJM_O: - if (cy > 0 || cx < 0) { + } + switch (vg) { + case VG_IP: + // (cx=0 means stream is implemented at i=0,so here we + // disable the step at boundary) + if (streamRequired) { + if (cx <= 0) { f(xiIndex, 0, 0) = fStage(xiIndex, -cx, -cy); } - break; - case VG_IMJP_O: - if (cy < 0 || cx > 0) { + } else { + if (cx < 0) { f(xiIndex, 0, 0) = fStage(xiIndex, -cx, -cy); } - break; - case VG_IMJM_O: - if (cy > 0 || cx > 0) { + } + break; + case VG_IM: + if (streamRequired) { + if (cx >= 0) { f(xiIndex, 0, 0) = fStage(xiIndex, -cx, -cy); } - break; - default: - break; - } - } - } - -#endif // OPS_2D - } -} - -void KerCutCellCVTUpwind1st(const ACC& coordinateXYZ, - const ACC& nodeType, const ACC& geometry, - const ACC& f, ACC& fGradient) { -#ifdef OPS_2D - VertexType vt = (VertexType)nodeType(0, 0); - VertexGeometryTypes vg = (VertexGeometryTypes)geometry(0, 0); - for (int compoIndex = 0; compoIndex < NUMCOMPONENTS; compoIndex++) { - for (int xiIndex = COMPOINDEX[2 * compoIndex]; - xiIndex <= COMPOINDEX[2 * compoIndex + 1]; xiIndex++) { - Real cx{XI[xiIndex * LATTDIM]}; - Real cy{XI[xiIndex * LATTDIM + 1]}; - // setting a initial value - fGradient(xiIndex, 0, 0) = 0; - // make sure no calculation occurring at boundary when unnecessary - // this can avoid access undefined memory if the halo point is set - // incorrectly - bool needCalc{true}; - if (vt == VertexType::ImmersedSolid) { - needCalc = false; - } - //Block and immersed boundary nodes - if (vt != VertexType::ImmersedSolid && vt != VertexType::Fluid) { - switch (vg) { - case VG_IP: + } else { if (cx > 0) { - needCalc = false; - } - break; - case VG_IM: - if (cx < 0) { - needCalc = false; + f(xiIndex, 0, 0) = fStage(xiIndex, -cx, -cy); } - break; - case VG_JP: - if (cy > 0) { - needCalc = false; + } + break; + case VG_JP: + if (streamRequired) { + if (cy <= 0) { + f(xiIndex, 0, 0) = fStage(xiIndex, -cx, -cy); } - break; - case VG_JM: + } else { if (cy < 0) { - needCalc = false; - } - break; - case VG_IPJP_I: - if (cy > 0 || cx > 0) { - needCalc = false; - } - break; - case VG_IPJM_I: - if (cy < 0 || cx > 0) { - needCalc = false; + f(xiIndex, 0, 0) = fStage(xiIndex, -cx, -cy); } - break; - case VG_IMJP_I: - if (cy > 0 || cx < 0) { - needCalc = false; + } + break; + case VG_JM: + if (streamRequired) { + if (cy >= 0) { + f(xiIndex, 0, 0) = fStage(xiIndex, -cx, -cy); } - break; - case VG_IMJM_I: - if (cy < 0 || cx < 0) { - needCalc = false; + } else { + if (cy > 0) { + f(xiIndex, 0, 0) = fStage(xiIndex, -cx, -cy); } - break; - // It appears that the information is well defined for the - // outer corners - // which is quite different from the inner corners - case VG_IPJP_O: - if (cy >= 0 && cx >= 0) { - needCalc = false; + } + break; + case VG_IPJP_I: + if (streamRequired) { + if (cy <= 0 && cx <= 0) { + f(xiIndex, 0, 0) = fStage(xiIndex, -cx, -cy); } - break; - case VG_IPJM_O: - if (cy <= 0 && cx >= 0) { - needCalc = false; + } else { + if (cy < 0 && cx < 0) { + f(xiIndex, 0, 0) = fStage(xiIndex, -cx, -cy); } - break; - case VG_IMJP_O: + } + break; + case VG_IPJM_I: + if (streamRequired) { if (cy >= 0 && cx <= 0) { - needCalc = false; + f(xiIndex, 0, 0) = fStage(xiIndex, -cx, -cy); } - break; - case VG_IMJM_O: - if (cy <= 0 && cx <= 0) { - needCalc = false; + } else { + if (cy > 0 && cx < 0) { + f(xiIndex, 0, 0) = fStage(xiIndex, -cx, -cy); } - break; - default: - break; - } - } - if (needCalc) { - if (cx > 0) { - fGradient(xiIndex, 0, 0) += - CS * cx * - ((f(xiIndex, 0, 0) - f(xiIndex, -1, 0)) / - (coordinateXYZ(0, 0, 0) - coordinateXYZ(0, -1, 0))); - } - if (cx < 0) { - fGradient(xiIndex, 0, 0) += - CS * cx * - ((f(xiIndex, 1, 0) - f(xiIndex, 0, 0)) / - (coordinateXYZ(0, 1, 0) - coordinateXYZ(0, 0, 0))); - } - if (cy > 0) { - fGradient(xiIndex, 0, 0) += - CS * cy * - ((f(xiIndex, 0, 0) - f(xiIndex, 0, -1)) / - (coordinateXYZ(1, 0, 0) - coordinateXYZ(1, 0, -1))); - } - if (cy < 0) { - fGradient(xiIndex, 0, 0) += - CS * cy * - ((f(xiIndex, 0, 1) - f(xiIndex, 0, 0)) / - (coordinateXYZ(1, 0, 1) - coordinateXYZ(1, 0, 0))); - } - } - } - } -#endif // OPS_2D -} - -void KerCutCellCVTUpwind2nd(const ACC& coordinateXYZ, - const ACC nodeType, const ACC& geometry, - const ACC& f, ACC& fGradient) { -#ifdef OPS_2D - VertexType vt = (VertexType)nodeType(0, 0); - VertexGeometryTypes vg = (VertexGeometryTypes)geometry(0, 0); - for (int compoIndex = 0; compoIndex < NUMCOMPONENTS; compoIndex++) { - for (int xiIndex = COMPOINDEX[2 * compoIndex]; - xiIndex <= COMPOINDEX[2 * compoIndex + 1]; xiIndex++) { - Real cx{XI[xiIndex * LATTDIM]}; - Real cy{XI[xiIndex * LATTDIM + 1]}; - bool reduceOrderX{false}; - bool reduceOrderY{false}; - bool needCalc{true}; - // setting a initial value - fGradient(xiIndex, 0, 0) = 0; - if (vt == VertexType::ImmersedSolid) { - needCalc = false; - } - // make sure no calculation occurring at boundary when unnecessary - // this can avoid access undefined memory if the halo point is set - // incorrectly - if (vt == VertexType::Fluid) { - // current node is a fluid point - if (cx > 0) { - VertexType vtUpwind = (VertexType)nodeType(-1, 0); - VertexGeometryTypes vgUpwind = - (VertexGeometryTypes)geometry(-1, 0); - if (vtUpwind != VertexType::Fluid && vgUpwind == VG_IP) { - // if the upwind node is boundary and VG_IP point - reduceOrderX = true; - } - } // cx >0 - if (cx < 0) { - VertexType vtUpwind = (VertexType)nodeType(1, 0); - VertexGeometryTypes vgUpwind = - (VertexGeometryTypes)geometry(1, 0); - if (vtUpwind != VertexType::Fluid && vgUpwind == VG_IM) { - // if the upwind node is boundary and VG_IM point - reduceOrderX = true; } - } // Cx<0 - if (cy > 0) { - VertexType vtUpwind = (VertexType)nodeType(0, -1); - VertexGeometryTypes vgUpwind = - (VertexGeometryTypes)geometry(0, -1); - if (vtUpwind != VertexType::Fluid && vgUpwind == VG_JP) { - reduceOrderY = true; - } - } // cy >0 - if (cy < 0) { - VertexType vtUpwind = (VertexType)nodeType(0, 1); - VertexGeometryTypes vgUpwind = - (VertexGeometryTypes)geometry(0, 1); - if (vtUpwind != VertexType::Fluid && vgUpwind == VG_JM) { - reduceOrderY = true; - } - } // cy < 0 - } // current node is a fluid point - // Current node is a boundary node or a immersed boundary node - if (vt != VertexType::Fluid && vt != VertexType::ImmersedSolid) { - switch (vg) { - case VG_IP: - if (cx > 0) { - needCalc = false; - } else { - if (cy > 0) { - VertexGeometryTypes vgUpwind = - (VertexGeometryTypes)geometry(0, -1); - if (((vgUpwind == VG_IPJM_O || - vgUpwind == VG_IPJP_O) && - (((VertexType)nodeType(0, -2)) == - VertexType::ImmersedSolid)) || - (vgUpwind == VG_IPJM_I) || - (vgUpwind == VG_IPJP_I)) { - reduceOrderY = true; - } - } // cy > 0 VG_IP - if (cy < 0) { - VertexGeometryTypes vgUpwind = - (VertexGeometryTypes)geometry(0, 1); - if (((vgUpwind == VG_IPJM_O || - vgUpwind == VG_IPJP_O) && - (((VertexType)nodeType(0, 2)) == - VertexType::ImmersedSolid)) || - (vgUpwind == VG_IPJM_I) || - (vgUpwind == VG_IPJP_I)) { - reduceOrderY = true; - } - } // Cy < 0 VG_IP - } - break; - case VG_IM: - if (cx < 0) { - needCalc = false; - } else { - if (cy > 0) { - VertexGeometryTypes vgUpwind = - (VertexGeometryTypes)geometry(0, -1); - if (((vgUpwind == VG_IMJM_O || - vgUpwind == VG_IMJP_O) && - (((VertexType)nodeType(0, -2)) == - VertexType::ImmersedSolid)) || - (vgUpwind == VG_IMJM_I) || - (vgUpwind == VG_IMJP_I)) { - reduceOrderY = true; - } - } // cy > 0 VG_IM - if (cy < 0) { - VertexGeometryTypes vgUpwind = - (VertexGeometryTypes)geometry(0, 1); - if (((vgUpwind == VG_IMJM_O || - vgUpwind == VG_IMJP_O) && - (((VertexType)nodeType(0, 2)) == - VertexType::ImmersedSolid)) || - (vgUpwind == VG_IMJM_I) || - (vgUpwind == VG_IMJP_I)) { - reduceOrderY = true; - } - } // cy < 0 VG_IM - } - break; - case VG_JP: - if (cy > 0) { - needCalc = false; - } else { - if (cx > 0) { - VertexGeometryTypes vgUpwind = - (VertexGeometryTypes)geometry(-1, 0); - if (((vgUpwind == VG_IMJP_O || - vgUpwind == VG_IPJP_O) && - (((VertexType)nodeType(-2, 0)) == - VertexType::ImmersedSolid)) || - (vgUpwind == VG_IMJP_I) || - (vgUpwind == VG_IPJP_I)) { - reduceOrderX = true; - } - } // cx > 0 VG_JP - if (cx < 0) { - VertexGeometryTypes vgUpwind = - (VertexGeometryTypes)geometry(1, 0); - if (((vgUpwind == VG_IMJP_O || - vgUpwind == VG_IPJP_O) && - (((VertexType)nodeType(2, 0)) == - VertexType::ImmersedSolid)) || - (vgUpwind == VG_IMJP_I) || - (vgUpwind == VG_IPJP_I)) { - reduceOrderX = true; - } - } // cx <0 VG_JP - } - break; - case VG_JM: - if (cy < 0) { - needCalc = false; - } else { - if (cx > 0) { - VertexGeometryTypes vgUpwind = - (VertexGeometryTypes)geometry(-1, 0); - if (((vgUpwind == VG_IMJM_O || - vgUpwind == VG_IPJM_O) && - (((VertexType)nodeType(-2, 0)) == - VertexType::ImmersedSolid)) || - (vgUpwind == VG_IMJM_I) || - (vgUpwind == VG_IPJM_I)) { - reduceOrderX = true; - } - } // cx > 0 VG_JM - if (cx < 0) { - VertexGeometryTypes vgUpwind = - (VertexGeometryTypes)geometry(1, 0); - if (((vgUpwind == VG_IPJM_O || - vgUpwind == VG_IMJM_O) && - (((VertexType)nodeType(2, 0)) == - VertexType::ImmersedSolid)) || - (vgUpwind == VG_IMJM_I) || - (vgUpwind == VG_IPJM_I)) { - reduceOrderX = true; - } - } // cx > 0 VG_JM - } - break; - case VG_IPJP_I: - if (cy > 0 || cx > 0) { - needCalc = false; - } else { - VertexGeometryTypes vgUpwind = - (VertexGeometryTypes)geometry(1, 0); - if (vgUpwind == VG_IMJP_I) { - reduceOrderX = true; - } - vgUpwind = (VertexGeometryTypes)geometry(0, 1); - if (vgUpwind == VG_IPJM_I) { - reduceOrderY = true; - } - } - break; - case VG_IPJM_I: - if (cy < 0 || cx > 0) { - needCalc = false; - } else { - VertexGeometryTypes vgUpwind = - (VertexGeometryTypes)geometry(1, 0); - if (vgUpwind == VG_IMJM_I) { - reduceOrderX = true; - } - vgUpwind = (VertexGeometryTypes)geometry(0, -1); - if (vgUpwind == VG_IPJP_I) { - reduceOrderY = true; - } - } - break; - case VG_IMJP_I: - if (cy > 0 || cx < 0) { - needCalc = false; - } else { - VertexGeometryTypes vgUpwind = - (VertexGeometryTypes)geometry(-1, 0); - if (vgUpwind == VG_IPJP_I) { - reduceOrderX = true; - } - vgUpwind = (VertexGeometryTypes)geometry(0, 1); - if (vgUpwind == VG_IMJM_I) { - reduceOrderY = true; - } - } - break; - case VG_IMJM_I: - if (cy < 0 || cx < 0) { - needCalc = false; - } else { - VertexGeometryTypes vgUpwind = - (VertexGeometryTypes)geometry(-1, 0); - if (vgUpwind == VG_IPJM_I) { - reduceOrderX = true; - } - vgUpwind = (VertexGeometryTypes)geometry(0, -1); - if (vgUpwind == VG_IMJP_I) { - reduceOrderY = true; - } - } - break; - case VG_IPJP_O: - if (cy >= 0 && cx >= 0) { - needCalc = false; - } else { - VertexGeometryTypes vgUpwind = - (VertexGeometryTypes)geometry(1, 0); - if (vgUpwind == VG_IM) { - reduceOrderX = true; - } - vgUpwind = (VertexGeometryTypes)geometry(0, 1); - if (vgUpwind == VG_JM) { - reduceOrderY = true; - } - } - break; - case VG_IPJM_O: + break; + case VG_IMJP_I: + if (streamRequired) { if (cy <= 0 && cx >= 0) { - needCalc = false; - } else { - VertexGeometryTypes vgUpwind = - (VertexGeometryTypes)geometry(1, 0); - if (vgUpwind == VG_IM) { - reduceOrderX = true; - } - vgUpwind = (VertexGeometryTypes)geometry(0, -1); - if (vgUpwind == VG_JP) { - reduceOrderY = true; - } - } - break; - case VG_IMJP_O: - if (cy >= 0 && cx <= 0) { - needCalc = false; - } else { - VertexGeometryTypes vgUpwind = - (VertexGeometryTypes)geometry(-1, 0); - if (vgUpwind == VG_IP) { - reduceOrderX = true; - } - vgUpwind = (VertexGeometryTypes)geometry(0, 1); - if (vgUpwind == VG_JM) { - reduceOrderY = true; - } - } - break; - case VG_IMJM_O: - if (cy <= 0 && cx <= 0) { - needCalc = false; - } else { - VertexGeometryTypes vgUpwind = - (VertexGeometryTypes)geometry(-1, 0); - if (vgUpwind == VG_IP) { - reduceOrderX = true; - } - vgUpwind = (VertexGeometryTypes)geometry(0, -1); - if (vgUpwind == VG_JP) { - reduceOrderY = true; - } - } - break; - default: - break; - } - } // if the current node is a boundary node - if (needCalc) { - if (cx > 0) { - if (reduceOrderX) { - fGradient(xiIndex, 0, 0) += - CS * cx * - ((f(xiIndex, 0, 0) - f(xiIndex, -1, 0)) / - (coordinateXYZ(0, 0, 0) - - coordinateXYZ(0, -1, 0))); - } else { - Real x_2{coordinateXYZ(0, -2, 0)}; - Real x_1{coordinateXYZ(0, -1, 0)}; - Real x_0{coordinateXYZ(0, 0, 0)}; - // ops_printf("x_2=%f x_1=%f x_0=%f\n",x_2,x_1,x_0); - fGradient(xiIndex, 0, 0) += - CS * cx * - (f(xiIndex, -2, 0) * - (1 / (x_2 - x_0) - 1 / (x_2 - x_1)) + - f(xiIndex, -1, 0) * - (1 / (x_1 - x_0) - 1 / (x_1 - x_2)) + - f(xiIndex, 0, 0) * - (1 / (x_0 - x_1) + 1 / (x_0 - x_2))); - } - } - if (cx < 0) { - if (reduceOrderX) { - fGradient(xiIndex, 0, 0) += - CS * cx * - ((f(xiIndex, 1, 0) - f(xiIndex, 0, 0)) / - (coordinateXYZ(0, 1, 0) - coordinateXYZ(0, 0, 0))); - } else { - Real xp2{coordinateXYZ(0, 2, 0)}; - Real xp1{coordinateXYZ(0, 1, 0)}; - Real x0{coordinateXYZ(0, 0, 0)}; - fGradient(xiIndex, 0, 0) += - CS * cx * - (f(xiIndex, 0, 0) * - (1 / (x0 - xp2) + 1 / (x0 - xp1)) + - f(xiIndex, 1, 0) * - (1 / (xp2 - xp1) + 1 / (xp1 - x0)) + - f(xiIndex, 2, 0) * - (1 / (xp2 - x0) + 1 / (xp1 - xp2))); - } - } - if (cy > 0) { - if (reduceOrderY) { - fGradient(xiIndex, 0, 0) += - CS * cy * - ((f(xiIndex, 0, 0) - f(xiIndex, 0, -1)) / - (coordinateXYZ(1, 0, 0) - - coordinateXYZ(1, 0, -1))); - } else { - Real y_2{coordinateXYZ(1, 0, -2)}; - Real y_1{coordinateXYZ(1, 0, -1)}; - Real y_0{coordinateXYZ(1, 0, 0)}; - fGradient(xiIndex, 0, 0) += - CS * cy * - (f(xiIndex, 0, -2) * - (1 / (y_2 - y_0) - 1 / (y_2 - y_1)) + - f(xiIndex, 0, -1) * - (1 / (y_1 - y_0) - 1 / (y_1 - y_2)) + - f(xiIndex, 0, 0) * - (1 / (y_0 - y_1) + 1 / (y_0 - y_2))); - } - } - if (cy < 0) { - if (reduceOrderY) { - fGradient(xiIndex, 0, 0) += - CS * cy * - ((f(xiIndex, 0, 1) - f(xiIndex, 0, 0)) / - (coordinateXYZ(1, 0, 1) - coordinateXYZ(1, 0, 0))); - } else { - Real yp2{coordinateXYZ(1, 0, 2)}; - Real yp1{coordinateXYZ(1, 0, 1)}; - Real y0{coordinateXYZ(1, 0, 0)}; - fGradient(xiIndex, 0, 0) += - CS * cy * - (f(xiIndex, 0, 0) * - (1 / (y0 - yp2) + 1 / (y0 - yp1)) + - f(xiIndex, 0, 1) * - (1 / (yp2 - yp1) + 1 / (yp1 - y0)) + - f(xiIndex, 0, 2) * - (1 / (yp2 - y0) + 1 / (yp1 - yp2))); - } - } - } - } - } -#endif // OPS_2D -} - -void KerCutCellSemiImplicitTimeMach( - const Real* dt, const Real* schemeCoeff, const ACC& nodeType, - const ACC& geometry, const ACC& fGradient, const ACC& feq, - const ACC& relaxationTime, const ACC& bodyForce, Real* f) { - /* - dt(0), schemeCoeff(1),nodeType(2),geometry(3), fGradient (4), - feq(5), - relaxationTime(6), bodyForce(7), f(8) - */ - VertexType vt = (VertexType)nodeType(0, 0); - VertexGeometryTypes vg = (VertexGeometryTypes)geometry(0, 0); - for (int compoIndex = 0; compoIndex < NUMCOMPONENTS; compoIndex++) { - Real tau = relaxationTime(compoIndex, 0, 0); - for (int xiIndex = COMPOINDEX[2 * compoIndex]; - xiIndex <= COMPOINDEX[2 * compoIndex + 1]; xiIndex++) { - Real cx{XI[xiIndex * LATTDIM]}; - Real cy{XI[xiIndex * LATTDIM + 1]}; - bool needMarch{true}; - if (vt == VertexType::ImmersedSolid) { - needMarch = false; - } - //Block or immersed boundary - if (vt != VertexType::ImmersedSolid && vt != VertexType::Fluid) { - switch (vg) { - case VG_IP: - if (cx > 0) { - needMarch = false; - } - break; - case VG_IM: - if (cx < 0) { - needMarch = false; - } - break; - case VG_JP: - if (cy > 0) { - needMarch = false; - } - break; - case VG_JM: - if (cy < 0) { - needMarch = false; - } - break; - case VG_IPJP_I: - if (cy > 0 || cx > 0) { - needMarch = false; - } - break; - case VG_IPJM_I: - if (cy < 0 || cx > 0) { - needMarch = false; - } - break; - case VG_IMJP_I: - if (cy > 0 || cx < 0) { - needMarch = false; + f(xiIndex, 0, 0) = fStage(xiIndex, -cx, -cy); } - break; - case VG_IMJM_I: - if (cy < 0 || cx < 0) { - needMarch = false; + } else { + if (cy < 0 && cx > 0) { + f(xiIndex, 0, 0) = fStage(xiIndex, -cx, -cy); } - break; - case VG_IPJP_O: + } + break; + case VG_IMJM_I: + if (streamRequired) { if (cy >= 0 && cx >= 0) { - needMarch = false; - } - break; - case VG_IPJM_O: - if (cy <= 0 && cx >= 0) { - needMarch = false; - } - break; - case VG_IMJP_O: - if (cy >= 0 && cx <= 0) { - needMarch = false; - } - break; - case VG_IMJM_O: - if (cy <= 0 && cx <= 0) { - needMarch = false; - } - break; - default: - break; - } - } - if (needMarch) { - f(xiIndex, 0, 0) = (feq(xiIndex, 0, 0) * (*dt) + - bodyForce(xiIndex, 0, 0) * (*dt) * tau - - fGradient(xiIndex, 0, 0) * (*dt) * tau + - f(xiIndex, 0, 0) * tau) / - ((*dt) + tau); - } - } - } -} - -void KerCutCellExplicitTimeMach( - const Real* dt, const Real* schemeCoeff, const ACC& nodeType, - const ACC& geometry, const ACC& fGradient, const ACC& feq, - const ACC& relaxationTime, const ACC& bodyForce, ACC& f) { - /* - dt(0), schemeCoeff(1),nodeType(2),geometry(3), fGradient (4), - feq(5), - relaxationTime(6), bodyForce(7), f(8) - */ - VertexType vt = (VertexType)nodeType(0, 0); - VertexGeometryTypes vg = (VertexGeometryTypes)geometry(0, 0); - for (int compoIndex = 0; compoIndex < NUMCOMPONENTS; compoIndex++) { - Real tau = relaxationTime(compoIndex, 0, 0); - for (int xiIndex = COMPOINDEX[2 * compoIndex]; - xiIndex <= COMPOINDEX[2 * compoIndex + 1]; xiIndex++) { - Real cx{XI[xiIndex * LATTDIM]}; - Real cy{XI[xiIndex * LATTDIM + 1]}; - bool needMarch{true}; - if (vt == VertexType::ImmersedSolid) { - needMarch = false; - } - //Block or immersed boundary - if (vt != VertexType::ImmersedSolid && vt != VertexType::Fluid) { { - switch (vg) { - case VG_IP: - if (cx > 0) { - needMarch = false; - } - break; - case VG_IM: - if (cx < 0) { - needMarch = false; + f(xiIndex, 0, 0) = fStage(xiIndex, -cx, -cy); } - break; - case VG_JP: - if (cy > 0) { - needMarch = false; + } else { + if (cy > 0 && cx > 0) { + f(xiIndex, 0, 0) = fStage(xiIndex, -cx, -cy); } - break; - case VG_JM: - if (cy < 0) { - needMarch = false; + } + break; + case VG_IPJP_O: + if (streamRequired) { + if (cy <= 0 || cx <= 0) { + f(xiIndex, 0, 0) = fStage(xiIndex, -cx, -cy); } - break; - case VG_IPJP_I: - if (cy > 0 || cx > 0) { - needMarch = false; + } else { + if (cy < 0 || cx < 0) { + f(xiIndex, 0, 0) = fStage(xiIndex, -cx, -cy); } - break; - case VG_IPJM_I: - if (cy < 0 || cx > 0) { - needMarch = false; + } + break; + case VG_IPJM_O: + if (streamRequired) { + if (cy >= 0 || cx <= 0) { + f(xiIndex, 0, 0) = fStage(xiIndex, -cx, -cy); } - break; - case VG_IMJP_I: + } else { if (cy > 0 || cx < 0) { - needMarch = false; - } - break; - case VG_IMJM_I: - if (cy < 0 || cx < 0) { - needMarch = false; + f(xiIndex, 0, 0) = fStage(xiIndex, -cx, -cy); } - break; - case VG_IPJP_O: - if (cy >= 0 && cx >= 0) { - needMarch = false; + } + break; + case VG_IMJP_O: + if (streamRequired) { + if (cy <= 0 || cx >= 0) { + f(xiIndex, 0, 0) = fStage(xiIndex, -cx, -cy); } - break; - case VG_IPJM_O: - if (cy <= 0 && cx >= 0) { - needMarch = false; + } else { + if (cy < 0 || cx > 0) { + f(xiIndex, 0, 0) = fStage(xiIndex, -cx, -cy); } - break; - case VG_IMJP_O: - if (cy >= 0 && cx <= 0) { - needMarch = false; + } + break; + case VG_IMJM_O: + if (streamRequired) { + if (cy >= 0 || cx >= 0) { + f(xiIndex, 0, 0) = fStage(xiIndex, -cx, -cy); } - break; - case VG_IMJM_O: - if (cy <= 0 && cx <= 0) { - needMarch = false; + } else { + if (cy > 0 || cx > 0) { + f(xiIndex, 0, 0) = fStage(xiIndex, -cx, -cy); } - break; - default: - break; - } - } - if (needMarch) { - f(xiIndex, 0, 0) = - f(xiIndex, 0, 0) + - (*dt) * (*schemeCoeff) * - (bodyForce(xiIndex, 0, 0) - fGradient(xiIndex, 0, 0) + - (feq(xiIndex, 0, 0) - f(xiIndex, 0, 0)) / tau); + } + break; + default: + break; } } } +#endif // OPS_2D } +// void KerCutCellCVTUpwind1st(const ACC& coordinateXYZ, +// const ACC& nodeType, const ACC& +// geometry, const ACC& f, ACC& +// fGradient) { +// #ifdef OPS_2D +// VertexType vt = (VertexType)nodeType(0, 0); +// VertexGeometryTypes vg = (VertexGeometryTypes)geometry(0, 0); +// for (int compoIndex = 0; compoIndex < NUMCOMPONENTS; compoIndex++) { +// for (int xiIndex = COMPOINDEX[2 * compoIndex]; +// xiIndex <= COMPOINDEX[2 * compoIndex + 1]; xiIndex++) { +// Real cx{XI[xiIndex * LATTDIM]}; +// Real cy{XI[xiIndex * LATTDIM + 1]}; +// // setting a initial value +// fGradient(xiIndex, 0, 0) = 0; +// // make sure no calculation occurring at boundary when +// unnecessary +// // this can avoid access undefined memory if the halo point is +// set +// // incorrectly +// bool needCalc{true}; +// if (vt == VertexType::ImmersedSolid) { +// needCalc = false; +// } +// // Block and immersed boundary nodes +// if (vt != VertexType::ImmersedSolid && vt != VertexType::Fluid) { +// switch (vg) { +// case VG_IP: +// if (cx > 0) { +// needCalc = false; +// } +// break; +// case VG_IM: +// if (cx < 0) { +// needCalc = false; +// } +// break; +// case VG_JP: +// if (cy > 0) { +// needCalc = false; +// } +// break; +// case VG_JM: +// if (cy < 0) { +// needCalc = false; +// } +// break; +// case VG_IPJP_I: +// if (cy > 0 || cx > 0) { +// needCalc = false; +// } +// break; +// case VG_IPJM_I: +// if (cy < 0 || cx > 0) { +// needCalc = false; +// } +// break; +// case VG_IMJP_I: +// if (cy > 0 || cx < 0) { +// needCalc = false; +// } +// break; +// case VG_IMJM_I: +// if (cy < 0 || cx < 0) { +// needCalc = false; +// } +// break; +// // It appears that the information is well defined for +// the +// // outer corners +// // which is quite different from the inner corners +// case VG_IPJP_O: +// if (cy >= 0 && cx >= 0) { +// needCalc = false; +// } +// break; +// case VG_IPJM_O: +// if (cy <= 0 && cx >= 0) { +// needCalc = false; +// } +// break; +// case VG_IMJP_O: +// if (cy >= 0 && cx <= 0) { +// needCalc = false; +// } +// break; +// case VG_IMJM_O: +// if (cy <= 0 && cx <= 0) { +// needCalc = false; +// } +// break; +// default: +// break; +// } +// } +// if (needCalc) { +// if (cx > 0) { +// fGradient(xiIndex, 0, 0) += +// CS * cx * +// ((f(xiIndex, 0, 0) - f(xiIndex, -1, 0)) / +// (coordinateXYZ(0, 0, 0) - coordinateXYZ(0, -1, 0))); +// } +// if (cx < 0) { +// fGradient(xiIndex, 0, 0) += +// CS * cx * +// ((f(xiIndex, 1, 0) - f(xiIndex, 0, 0)) / +// (coordinateXYZ(0, 1, 0) - coordinateXYZ(0, 0, 0))); +// } +// if (cy > 0) { +// fGradient(xiIndex, 0, 0) += +// CS * cy * +// ((f(xiIndex, 0, 0) - f(xiIndex, 0, -1)) / +// (coordinateXYZ(1, 0, 0) - coordinateXYZ(1, 0, -1))); +// } +// if (cy < 0) { +// fGradient(xiIndex, 0, 0) += +// CS * cy * +// ((f(xiIndex, 0, 1) - f(xiIndex, 0, 0)) / +// (coordinateXYZ(1, 0, 1) - coordinateXYZ(1, 0, 0))); +// } +// } +// } +// } +// #endif // OPS_2D +// } + +// void KerCutCellCVTUpwind2nd(const ACC& coordinateXYZ, +// const ACC nodeType, const ACC& +// geometry, const ACC& f, ACC& +// fGradient) { +// #ifdef OPS_2D +// VertexType vt = (VertexType)nodeType(0, 0); +// VertexGeometryTypes vg = (VertexGeometryTypes)geometry(0, 0); +// for (int compoIndex = 0; compoIndex < NUMCOMPONENTS; compoIndex++) { +// for (int xiIndex = COMPOINDEX[2 * compoIndex]; +// xiIndex <= COMPOINDEX[2 * compoIndex + 1]; xiIndex++) { +// Real cx{XI[xiIndex * LATTDIM]}; +// Real cy{XI[xiIndex * LATTDIM + 1]}; +// bool reduceOrderX{false}; +// bool reduceOrderY{false}; +// bool needCalc{true}; +// // setting a initial value +// fGradient(xiIndex, 0, 0) = 0; +// if (vt == VertexType::ImmersedSolid) { +// needCalc = false; +// } +// // make sure no calculation occurring at boundary when +// unnecessary +// // this can avoid access undefined memory if the halo point is +// set +// // incorrectly +// if (vt == VertexType::Fluid) { +// // current node is a fluid point +// if (cx > 0) { +// VertexType vtUpwind = (VertexType)nodeType(-1, 0); +// VertexGeometryTypes vgUpwind = +// (VertexGeometryTypes)geometry(-1, 0); +// if (vtUpwind != VertexType::Fluid && vgUpwind == VG_IP) { +// // if the upwind node is boundary and VG_IP point +// reduceOrderX = true; +// } +// } // cx >0 +// if (cx < 0) { +// VertexType vtUpwind = (VertexType)nodeType(1, 0); +// VertexGeometryTypes vgUpwind = +// (VertexGeometryTypes)geometry(1, 0); +// if (vtUpwind != VertexType::Fluid && vgUpwind == VG_IM) { +// // if the upwind node is boundary and VG_IM point +// reduceOrderX = true; +// } +// } // Cx<0 +// if (cy > 0) { +// VertexType vtUpwind = (VertexType)nodeType(0, -1); +// VertexGeometryTypes vgUpwind = +// (VertexGeometryTypes)geometry(0, -1); +// if (vtUpwind != VertexType::Fluid && vgUpwind == VG_JP) { +// reduceOrderY = true; +// } +// } // cy >0 +// if (cy < 0) { +// VertexType vtUpwind = (VertexType)nodeType(0, 1); +// VertexGeometryTypes vgUpwind = +// (VertexGeometryTypes)geometry(0, 1); +// if (vtUpwind != VertexType::Fluid && vgUpwind == VG_JM) { +// reduceOrderY = true; +// } +// } // cy < 0 +// } // current node is a fluid point +// // Current node is a boundary node or a immersed boundary node +// if (vt != VertexType::Fluid && vt != VertexType::ImmersedSolid) { +// switch (vg) { +// case VG_IP: +// if (cx > 0) { +// needCalc = false; +// } else { +// if (cy > 0) { +// VertexGeometryTypes vgUpwind = +// (VertexGeometryTypes)geometry(0, -1); +// if (((vgUpwind == VG_IPJM_O || +// vgUpwind == VG_IPJP_O) && +// (((VertexType)nodeType(0, -2)) == +// VertexType::ImmersedSolid)) || +// (vgUpwind == VG_IPJM_I) || +// (vgUpwind == VG_IPJP_I)) { +// reduceOrderY = true; +// } +// } // cy > 0 VG_IP +// if (cy < 0) { +// VertexGeometryTypes vgUpwind = +// (VertexGeometryTypes)geometry(0, 1); +// if (((vgUpwind == VG_IPJM_O || +// vgUpwind == VG_IPJP_O) && +// (((VertexType)nodeType(0, 2)) == +// VertexType::ImmersedSolid)) || +// (vgUpwind == VG_IPJM_I) || +// (vgUpwind == VG_IPJP_I)) { +// reduceOrderY = true; +// } +// } // Cy < 0 VG_IP +// } +// break; +// case VG_IM: +// if (cx < 0) { +// needCalc = false; +// } else { +// if (cy > 0) { +// VertexGeometryTypes vgUpwind = +// (VertexGeometryTypes)geometry(0, -1); +// if (((vgUpwind == VG_IMJM_O || +// vgUpwind == VG_IMJP_O) && +// (((VertexType)nodeType(0, -2)) == +// VertexType::ImmersedSolid)) || +// (vgUpwind == VG_IMJM_I) || +// (vgUpwind == VG_IMJP_I)) { +// reduceOrderY = true; +// } +// } // cy > 0 VG_IM +// if (cy < 0) { +// VertexGeometryTypes vgUpwind = +// (VertexGeometryTypes)geometry(0, 1); +// if (((vgUpwind == VG_IMJM_O || +// vgUpwind == VG_IMJP_O) && +// (((VertexType)nodeType(0, 2)) == +// VertexType::ImmersedSolid)) || +// (vgUpwind == VG_IMJM_I) || +// (vgUpwind == VG_IMJP_I)) { +// reduceOrderY = true; +// } +// } // cy < 0 VG_IM +// } +// break; +// case VG_JP: +// if (cy > 0) { +// needCalc = false; +// } else { +// if (cx > 0) { +// VertexGeometryTypes vgUpwind = +// (VertexGeometryTypes)geometry(-1, 0); +// if (((vgUpwind == VG_IMJP_O || +// vgUpwind == VG_IPJP_O) && +// (((VertexType)nodeType(-2, 0)) == +// VertexType::ImmersedSolid)) || +// (vgUpwind == VG_IMJP_I) || +// (vgUpwind == VG_IPJP_I)) { +// reduceOrderX = true; +// } +// } // cx > 0 VG_JP +// if (cx < 0) { +// VertexGeometryTypes vgUpwind = +// (VertexGeometryTypes)geometry(1, 0); +// if (((vgUpwind == VG_IMJP_O || +// vgUpwind == VG_IPJP_O) && +// (((VertexType)nodeType(2, 0)) == +// VertexType::ImmersedSolid)) || +// (vgUpwind == VG_IMJP_I) || +// (vgUpwind == VG_IPJP_I)) { +// reduceOrderX = true; +// } +// } // cx <0 VG_JP +// } +// break; +// case VG_JM: +// if (cy < 0) { +// needCalc = false; +// } else { +// if (cx > 0) { +// VertexGeometryTypes vgUpwind = +// (VertexGeometryTypes)geometry(-1, 0); +// if (((vgUpwind == VG_IMJM_O || +// vgUpwind == VG_IPJM_O) && +// (((VertexType)nodeType(-2, 0)) == +// VertexType::ImmersedSolid)) || +// (vgUpwind == VG_IMJM_I) || +// (vgUpwind == VG_IPJM_I)) { +// reduceOrderX = true; +// } +// } // cx > 0 VG_JM +// if (cx < 0) { +// VertexGeometryTypes vgUpwind = +// (VertexGeometryTypes)geometry(1, 0); +// if (((vgUpwind == VG_IPJM_O || +// vgUpwind == VG_IMJM_O) && +// (((VertexType)nodeType(2, 0)) == +// VertexType::ImmersedSolid)) || +// (vgUpwind == VG_IMJM_I) || +// (vgUpwind == VG_IPJM_I)) { +// reduceOrderX = true; +// } +// } // cx > 0 VG_JM +// } +// break; +// case VG_IPJP_I: +// if (cy > 0 || cx > 0) { +// needCalc = false; +// } else { +// VertexGeometryTypes vgUpwind = +// (VertexGeometryTypes)geometry(1, 0); +// if (vgUpwind == VG_IMJP_I) { +// reduceOrderX = true; +// } +// vgUpwind = (VertexGeometryTypes)geometry(0, 1); +// if (vgUpwind == VG_IPJM_I) { +// reduceOrderY = true; +// } +// } +// break; +// case VG_IPJM_I: +// if (cy < 0 || cx > 0) { +// needCalc = false; +// } else { +// VertexGeometryTypes vgUpwind = +// (VertexGeometryTypes)geometry(1, 0); +// if (vgUpwind == VG_IMJM_I) { +// reduceOrderX = true; +// } +// vgUpwind = (VertexGeometryTypes)geometry(0, -1); +// if (vgUpwind == VG_IPJP_I) { +// reduceOrderY = true; +// } +// } +// break; +// case VG_IMJP_I: +// if (cy > 0 || cx < 0) { +// needCalc = false; +// } else { +// VertexGeometryTypes vgUpwind = +// (VertexGeometryTypes)geometry(-1, 0); +// if (vgUpwind == VG_IPJP_I) { +// reduceOrderX = true; +// } +// vgUpwind = (VertexGeometryTypes)geometry(0, 1); +// if (vgUpwind == VG_IMJM_I) { +// reduceOrderY = true; +// } +// } +// break; +// case VG_IMJM_I: +// if (cy < 0 || cx < 0) { +// needCalc = false; +// } else { +// VertexGeometryTypes vgUpwind = +// (VertexGeometryTypes)geometry(-1, 0); +// if (vgUpwind == VG_IPJM_I) { +// reduceOrderX = true; +// } +// vgUpwind = (VertexGeometryTypes)geometry(0, -1); +// if (vgUpwind == VG_IMJP_I) { +// reduceOrderY = true; +// } +// } +// break; +// case VG_IPJP_O: +// if (cy >= 0 && cx >= 0) { +// needCalc = false; +// } else { +// VertexGeometryTypes vgUpwind = +// (VertexGeometryTypes)geometry(1, 0); +// if (vgUpwind == VG_IM) { +// reduceOrderX = true; +// } +// vgUpwind = (VertexGeometryTypes)geometry(0, 1); +// if (vgUpwind == VG_JM) { +// reduceOrderY = true; +// } +// } +// break; +// case VG_IPJM_O: +// if (cy <= 0 && cx >= 0) { +// needCalc = false; +// } else { +// VertexGeometryTypes vgUpwind = +// (VertexGeometryTypes)geometry(1, 0); +// if (vgUpwind == VG_IM) { +// reduceOrderX = true; +// } +// vgUpwind = (VertexGeometryTypes)geometry(0, -1); +// if (vgUpwind == VG_JP) { +// reduceOrderY = true; +// } +// } +// break; +// case VG_IMJP_O: +// if (cy >= 0 && cx <= 0) { +// needCalc = false; +// } else { +// VertexGeometryTypes vgUpwind = +// (VertexGeometryTypes)geometry(-1, 0); +// if (vgUpwind == VG_IP) { +// reduceOrderX = true; +// } +// vgUpwind = (VertexGeometryTypes)geometry(0, 1); +// if (vgUpwind == VG_JM) { +// reduceOrderY = true; +// } +// } +// break; +// case VG_IMJM_O: +// if (cy <= 0 && cx <= 0) { +// needCalc = false; +// } else { +// VertexGeometryTypes vgUpwind = +// (VertexGeometryTypes)geometry(-1, 0); +// if (vgUpwind == VG_IP) { +// reduceOrderX = true; +// } +// vgUpwind = (VertexGeometryTypes)geometry(0, -1); +// if (vgUpwind == VG_JP) { +// reduceOrderY = true; +// } +// } +// break; +// default: +// break; +// } +// } // if the current node is a boundary node +// if (needCalc) { +// if (cx > 0) { +// if (reduceOrderX) { +// fGradient(xiIndex, 0, 0) += +// CS * cx * +// ((f(xiIndex, 0, 0) - f(xiIndex, -1, 0)) / +// (coordinateXYZ(0, 0, 0) - +// coordinateXYZ(0, -1, 0))); +// } else { +// Real x_2{coordinateXYZ(0, -2, 0)}; +// Real x_1{coordinateXYZ(0, -1, 0)}; +// Real x_0{coordinateXYZ(0, 0, 0)}; +// // ops_printf("x_2=%f x_1=%f x_0=%f\n",x_2,x_1,x_0); +// fGradient(xiIndex, 0, 0) += +// CS * cx * +// (f(xiIndex, -2, 0) * +// (1 / (x_2 - x_0) - 1 / (x_2 - x_1)) + +// f(xiIndex, -1, 0) * +// (1 / (x_1 - x_0) - 1 / (x_1 - x_2)) + +// f(xiIndex, 0, 0) * +// (1 / (x_0 - x_1) + 1 / (x_0 - x_2))); +// } +// } +// if (cx < 0) { +// if (reduceOrderX) { +// fGradient(xiIndex, 0, 0) += +// CS * cx * +// ((f(xiIndex, 1, 0) - f(xiIndex, 0, 0)) / +// (coordinateXYZ(0, 1, 0) - coordinateXYZ(0, 0, +// 0))); +// } else { +// Real xp2{coordinateXYZ(0, 2, 0)}; +// Real xp1{coordinateXYZ(0, 1, 0)}; +// Real x0{coordinateXYZ(0, 0, 0)}; +// fGradient(xiIndex, 0, 0) += +// CS * cx * +// (f(xiIndex, 0, 0) * +// (1 / (x0 - xp2) + 1 / (x0 - xp1)) + +// f(xiIndex, 1, 0) * +// (1 / (xp2 - xp1) + 1 / (xp1 - x0)) + +// f(xiIndex, 2, 0) * +// (1 / (xp2 - x0) + 1 / (xp1 - xp2))); +// } +// } +// if (cy > 0) { +// if (reduceOrderY) { +// fGradient(xiIndex, 0, 0) += +// CS * cy * +// ((f(xiIndex, 0, 0) - f(xiIndex, 0, -1)) / +// (coordinateXYZ(1, 0, 0) - +// coordinateXYZ(1, 0, -1))); +// } else { +// Real y_2{coordinateXYZ(1, 0, -2)}; +// Real y_1{coordinateXYZ(1, 0, -1)}; +// Real y_0{coordinateXYZ(1, 0, 0)}; +// fGradient(xiIndex, 0, 0) += +// CS * cy * +// (f(xiIndex, 0, -2) * +// (1 / (y_2 - y_0) - 1 / (y_2 - y_1)) + +// f(xiIndex, 0, -1) * +// (1 / (y_1 - y_0) - 1 / (y_1 - y_2)) + +// f(xiIndex, 0, 0) * +// (1 / (y_0 - y_1) + 1 / (y_0 - y_2))); +// } +// } +// if (cy < 0) { +// if (reduceOrderY) { +// fGradient(xiIndex, 0, 0) += +// CS * cy * +// ((f(xiIndex, 0, 1) - f(xiIndex, 0, 0)) / +// (coordinateXYZ(1, 0, 1) - coordinateXYZ(1, 0, +// 0))); +// } else { +// Real yp2{coordinateXYZ(1, 0, 2)}; +// Real yp1{coordinateXYZ(1, 0, 1)}; +// Real y0{coordinateXYZ(1, 0, 0)}; +// fGradient(xiIndex, 0, 0) += +// CS * cy * +// (f(xiIndex, 0, 0) * +// (1 / (y0 - yp2) + 1 / (y0 - yp1)) + +// f(xiIndex, 0, 1) * +// (1 / (yp2 - yp1) + 1 / (yp1 - y0)) + +// f(xiIndex, 0, 2) * +// (1 / (yp2 - y0) + 1 / (yp1 - yp2))); +// } +// } +// } +// } +// } +// #endif // OPS_2D +// } + +// void KerCutCellSemiImplicitTimeMach( +// const Real* dt, const Real* schemeCoeff, const ACC& nodeType, +// const ACC& geometry, const ACC& fGradient, const ACC& +// feq, const ACC& relaxationTime, const ACC& bodyForce, Real* +// f) { +// #ifdef OPS_2D +// /* +// dt(0), schemeCoeff(1),nodeType(2),geometry(3), fGradient (4), +// feq(5), +// relaxationTime(6), bodyForce(7), f(8) +// */ +// VertexType vt = (VertexType)nodeType(0, 0); +// VertexGeometryTypes vg = (VertexGeometryTypes)geometry(0, 0); +// for (int compoIndex = 0; compoIndex < NUMCOMPONENTS; compoIndex++) { +// Real tau = relaxationTime(compoIndex, 0, 0); +// for (int xiIndex = COMPOINDEX[2 * compoIndex]; +// xiIndex <= COMPOINDEX[2 * compoIndex + 1]; xiIndex++) { +// Real cx{XI[xiIndex * LATTDIM]}; +// Real cy{XI[xiIndex * LATTDIM + 1]}; +// bool needMarch{true}; +// if (vt == VertexType::ImmersedSolid) { +// needMarch = false; +// } +// // Block or immersed boundary +// if (vt != VertexType::ImmersedSolid && vt != VertexType::Fluid) { +// switch (vg) { +// case VG_IP: +// if (cx > 0) { +// needMarch = false; +// } +// break; +// case VG_IM: +// if (cx < 0) { +// needMarch = false; +// } +// break; +// case VG_JP: +// if (cy > 0) { +// needMarch = false; +// } +// break; +// case VG_JM: +// if (cy < 0) { +// needMarch = false; +// } +// break; +// case VG_IPJP_I: +// if (cy > 0 || cx > 0) { +// needMarch = false; +// } +// break; +// case VG_IPJM_I: +// if (cy < 0 || cx > 0) { +// needMarch = false; +// } +// break; +// case VG_IMJP_I: +// if (cy > 0 || cx < 0) { +// needMarch = false; +// } +// break; +// case VG_IMJM_I: +// if (cy < 0 || cx < 0) { +// needMarch = false; +// } +// break; +// case VG_IPJP_O: +// if (cy >= 0 && cx >= 0) { +// needMarch = false; +// } +// break; +// case VG_IPJM_O: +// if (cy <= 0 && cx >= 0) { +// needMarch = false; +// } +// break; +// case VG_IMJP_O: +// if (cy >= 0 && cx <= 0) { +// needMarch = false; +// } +// break; +// case VG_IMJM_O: +// if (cy <= 0 && cx <= 0) { +// needMarch = false; +// } +// break; +// default: +// break; +// } +// } +// if (needMarch) { +// f(xiIndex, 0, 0) = (feq(xiIndex, 0, 0) * (*dt) + +// bodyForce(xiIndex, 0, 0) * (*dt) * tau - +// fGradient(xiIndex, 0, 0) * (*dt) * tau + +// f(xiIndex, 0, 0) * tau) / +// ((*dt) + tau); +// } +// } +// } +// #endif // OPS_2D +// } + +// void KerCutCellExplicitTimeMach( +// const Real* dt, const Real* schemeCoeff, const ACC& nodeType, +// const ACC& geometry, const ACC& fGradient, const ACC& +// feq, const ACC& relaxationTime, const ACC& bodyForce, +// ACC& f) { +// #ifdef OPS_2D +// /* +// dt(0), schemeCoeff(1),nodeType(2),geometry(3), fGradient (4), +// feq(5), +// relaxationTime(6), bodyForce(7), f(8) +// */ +// VertexType vt = (VertexType)nodeType(0, 0); +// VertexGeometryTypes vg = (VertexGeometryTypes)geometry(0, 0); +// for (int compoIndex = 0; compoIndex < NUMCOMPONENTS; compoIndex++) { +// Real tau = relaxationTime(compoIndex, 0, 0); +// for (int xiIndex = COMPOINDEX[2 * compoIndex]; +// xiIndex <= COMPOINDEX[2 * compoIndex + 1]; xiIndex++) { +// Real cx{XI[xiIndex * LATTDIM]}; +// Real cy{XI[xiIndex * LATTDIM + 1]}; +// bool needMarch{true}; +// if (vt == VertexType::ImmersedSolid) { +// needMarch = false; +// } +// // Block or immersed boundary +// if (vt != VertexType::ImmersedSolid && vt != VertexType::Fluid) { +// { +// switch (vg) { +// case VG_IP: +// if (cx > 0) { +// needMarch = false; +// } +// break; +// case VG_IM: +// if (cx < 0) { +// needMarch = false; +// } +// break; +// case VG_JP: +// if (cy > 0) { +// needMarch = false; +// } +// break; +// case VG_JM: +// if (cy < 0) { +// needMarch = false; +// } +// break; +// case VG_IPJP_I: +// if (cy > 0 || cx > 0) { +// needMarch = false; +// } +// break; +// case VG_IPJM_I: +// if (cy < 0 || cx > 0) { +// needMarch = false; +// } +// break; +// case VG_IMJP_I: +// if (cy > 0 || cx < 0) { +// needMarch = false; +// } +// break; +// case VG_IMJM_I: +// if (cy < 0 || cx < 0) { +// needMarch = false; +// } +// break; +// case VG_IPJP_O: +// if (cy >= 0 && cx >= 0) { +// needMarch = false; +// } +// break; +// case VG_IPJM_O: +// if (cy <= 0 && cx >= 0) { +// needMarch = false; +// } +// break; +// case VG_IMJP_O: +// if (cy >= 0 && cx <= 0) { +// needMarch = false; +// } +// break; +// case VG_IMJM_O: +// if (cy <= 0 && cx <= 0) { +// needMarch = false; +// } +// break; +// default: +// break; +// } +// } +// if (needMarch) { +// f(xiIndex, 0, 0) = +// f(xiIndex, 0, 0) + +// (*dt) * (*schemeCoeff) * +// (bodyForce(xiIndex, 0, 0) - +// fGradient(xiIndex, 0, 0) + +// (feq(xiIndex, 0, 0) - f(xiIndex, 0, 0)) / tau); +// } +// } +// } +// #endif // OPS_2D +// } + #endif // OPS_2D outter #ifdef OPS_3D // three dimensional code -void KerStream3D(ACC & f, const ACC& fStage, +void KerStream3D(ACC& f, const ACC& fStage, const ACC& nodeType, const ACC& geometry, const int* lattIdx) { #ifdef OPS_3D @@ -960,7 +976,6 @@ void KerStream3D(ACC & f, const ACC& fStage, if (vt != VertexType::ImmersedSolid && vt != VertexType::Fluid && vt != VertexType::VirtualBoundary && vt != VertexType::MDPeriodic) { - // TODO to be determined if necessary bool streamRequired{true}; if (streamRequired) { if ((cx == 0) && (cy == 0) && (cz == 0)) { @@ -1597,4 +1612,4 @@ void KerStream3D(ACC & f, const ACC& fStage, #endif // OPS_3D outter -#endif // SCHEME_KERNEL.inc +#endif // SCHEME_KERNEL.inc diff --git a/Src/scheme_wrapper.cpp b/Src/scheme_wrapper.cpp index 7690568..c6b84b6 100644 --- a/Src/scheme_wrapper.cpp +++ b/Src/scheme_wrapper.cpp @@ -6,6 +6,7 @@ #include "scheme_kernel.inc" #ifdef OPS_3D void Stream3D() { +#ifdef OPS_3D for (const auto& idBlock : g_Block()) { const Block& block{idBlock.second}; std::vector iterRng; @@ -26,5 +27,33 @@ void Stream3D() { ops_arg_gbl(compo.second.index, 2, "int", OPS_READ)); } } +#endif // OPS_3Ds +} +#endif // OPS_3D + +#ifdef OPS_2D +void Stream() { +#ifdef OPS_2D + for (const auto& idBlock : g_Block()) { + const Block& block{idBlock.second}; + std::vector iterRng; + iterRng.assign(block.WholeRange().begin(), block.WholeRange().end()); + const int blockIndex{block.ID()}; + for (const auto& compo : g_Components()) { + ops_par_loop( + KerStream, "KerStream", block.Get(), SpaceDim(), + iterRng.data(), + ops_arg_dat(g_f().at(blockIndex), NUMXI, LOCALSTENCIL, "double", + OPS_RW), + ops_arg_dat(g_fStage().at(blockIndex), NUMXI, + ONEPTLATTICESTENCIL, "double", OPS_READ), + ops_arg_dat(g_NodeType().at(compo.first).at(blockIndex), 1, + LOCALSTENCIL, "int", OPS_READ), + ops_arg_dat(g_GeometryProperty().at(blockIndex), 1, + LOCALSTENCIL, "int", OPS_READ), + ops_arg_gbl(compo.second.index, 2, "int", OPS_READ)); + } + } +#endif // OPS_2D } -#endif // OPS_3D \ No newline at end of file +#endif // OPS_2D \ No newline at end of file diff --git a/Tests/FieldBlock/CMakeLists.txt b/Tests/FieldBlock/CMakeLists.txt new file mode 100644 index 0000000..3c2d779 --- /dev/null +++ b/Tests/FieldBlock/CMakeLists.txt @@ -0,0 +1,29 @@ + +cmake_minimum_required(VERSION 3.18) +set(AppName FieldBlock) +set(AppSrc field_block.cpp) +set(LibSrc block.cpp) +set(LibHeadList "") +set(SpaceDim 3) +if (NOT OPTIMISE) + # Set the path for LibSrc + set(LibSrcPath "") + foreach(Src IN LISTS LibSrc) + list(APPEND LibSrcPath ${LibDir}/${Src}) + endforeach(Src IN LISTS LibSrc) + SeqDevTarget("${SpaceDim}" 0) + MpiDevTarget("${SpaceDim}" 0) +else() + #set the files needed to be translated by ops.py from the app side + set(AppSrcGenList field_block.cpp) + set(AppKernelGenList setvalue_kernel.inc) + set(AppHeadList "") + CreateTempDir() + set(TMP_SOURCE_DIR ${CMAKE_CURRENT_BINARY_DIR}/tmp) + set(HeadList ${LibHeadList} ${AppHeadList}) + WriteJsonConfig(${TMP_SOURCE_DIR} ${AppName} "${LibSrc}" "${AppSrcGenList}" "${AppKernelGenList}" "${HeadList}" ${SpaceDim}) + TranslateSourceCodes(${LibDir} "${LibSrcGenList}" "${AppSrcGenList}" ${TMP_SOURCE_DIR}) + SeqTarget("${SpaceDim}") + MpiTarget("${SpaceDim}") + CudaTarget("${SpaceDim}") +endif () diff --git a/Utility/InstallHDF5.py b/Utility/InstallHDF5.py new file mode 100644 index 0000000..abc6e70 --- /dev/null +++ b/Utility/InstallHDF5.py @@ -0,0 +1,85 @@ +import sys +import platform +import os +import shutil +import argparse +sys.path.append(os.getcwd()) +from utility import DownloadUrl, UnTar, RunShellCmd, CloneGitRepo + + +parser = argparse.ArgumentParser(description=""" +Install the HDF5 library manually for the OPS libary!\n +Please set the compiler environment before calling this tool, e.g., +source /opt/intel/oneapi/setvars.sh for using the Intel oneAPI compiler.\n +Sudo might be needed when the INSTALLDIR is pointed to a non-user folder\n +""",formatter_class=argparse.RawTextHelpFormatter) +parser.add_argument("-v", "--verbose", help="Display detailed output during compilation.", default=False, + action="store_true") +parser.add_argument("-p", "--parallel", help="Enable the parallel mode.", default=False, + action="store_true") +parser.add_argument("-f", "--fortran", help="Enable the Fortran library.", default=False, + action="store_true") +parser.add_argument("-I", "--InstallDir", type=str, default="/usr/local/", + help="Set the directory for installation.") +parser.add_argument("-B", "--BuildDir", type=str, default="BuildHDF5", + help="Set the directory for building.") +parser.add_argument("-T", "--test", default=False, action="store_true", help="Run testings.") +parser.add_argument("-D", "--download", default=True, action="store_false", help="Download the library.") +args = parser.parse_args() + +system = platform.system() +parentPath = os.getcwd() +cpuNum = os.cpu_count() +buildPath = args.BuildDir +installDir = args.InstallDir +fortran = args.fortran +verbose = args.verbose +parallel = args.parallel +test = args.test +download = args.download +if ("Windows" in system): + print("Windows is not supported at this moment") +try: + if (not os.path.exists(buildPath)): + os.mkdir(buildPath) + print ("Successfully created the directory %s " % buildPath) +except OSError: + print ("Creation of the directory %s failed" % buildPath) + +os.chdir(buildPath) +hdf5Dir = "CMake-hdf5-1.10.7" +url="https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-1.10/hdf5-1.10.7/src/CMake-hdf5-1.10.7.tar.gz" +if download: + UnTar(DownloadUrl(url)) +os.chdir(hdf5Dir) +if not os.path.exists("build"): + os.mkdir("build") +os.chdir("build") + +# here the Z_LIB and SZIP support are disabled since CMake cannot find them. +baseCmd = "-DCMAKE_BUILD_TYPE:STRING=Release -DHDF5_ENABLE_SZIP_SUPPORT:BOOL=OFF -DHDF5_ENABLE_Z_LIB_SUPPORT:BOOL=OFF -DCMAKE_INSTALL_PREFIX='" + installDir +cmakeCmd = 'cmake ../hdf5-1.10.7' +cmakeCmd = cmakeCmd.split() +cmakeCmd.append('-G') +cmakeCmd.append("Unix Makefiles") +for para in baseCmd.split(): + cmakeCmd.append(para) +if parallel: + cmakeCmd.append("-DHDF5_ENABLE_PARALLEL=ON") + cmakeCmd.append("-DHDF5_BUILD_CPP_LIB=OFF") +if verbose: + cmakeCmd.append("-DCMAKE_VERBOSE_MAKEFILE=ON") +if fortran: + cmakeCmd.append("-DHDF5_BUILD_FORTRAN=ON") +print(cmakeCmd) +import subprocess +RunShellCmd(cmakeCmd) +RunShellCmd("cmake --build . --config Release -j "+ str(cpuNum)) +if test: + RunShellCmd("ctest . -C Release -j "+ str(cpuNum)) +RunShellCmd("cmake --install . --config Release") +os.chdir(parentPath) +try: + shutil.rmtree(buildPath) +except: + raise diff --git a/Utility/utility.py b/Utility/utility.py new file mode 100644 index 0000000..1cbebcb --- /dev/null +++ b/Utility/utility.py @@ -0,0 +1,69 @@ + +from git.remote import RemoteProgress + +class Progress(RemoteProgress): + def update(self, *args): + print(self._cur_line) + +def UnTar(fileName, dir=""): + try: + import tarfile + except ImportError: + print("Please install tarfile package first!") + return None + print("Extacting the package",fileName) + if fileName.endswith("tar.gz"): + tar = tarfile.open(fileName, "r:gz") + elif fileName.endswith("tar"): + tar = tarfile.open(fileName, "r:") + if dir != "": + tar.extractall(dir) + else: + tar.extractall() + tar.close() + +def DownloadUrl(link): + try: + import requests + import urllib + except ImportError: + print("Please install requests and urllib package first!") + return None + fileName = link.split('/')[-1] + url = urllib.request.urlopen(link) + file = open(fileName, 'wb') + fileSize = int(url.info()["Content-Length"]) + print("Downloading: %s Bytes: %s" % (fileName, fileSize)) + fileSizeDownloaded = 0 + blockSize = 8192 + while True: + buffer = url.read(blockSize) + if not buffer: + break + fileSizeDownloaded += len(buffer) + file.write(buffer) + print('%10d [%3.2f%%]\r' % (fileSizeDownloaded, fileSizeDownloaded * 100. / fileSize),end="") + file.close() + return fileName + +def CloneGitRepo(link,dir): + try: + from git.repo.base import Repo + except ImportError: + print("Please install GitPython package first!") + return None + Repo.clone_from(link, dir, progress=Progress()) + +def RunShellCmd(cmd,silent=False): + try: + import subprocess + import sys + except ImportError: + print("Please install subprocess and sys package first!") + return None + if isinstance(cmd,str): + cmd = cmd.split() + if not silent: + subprocess.run(cmd, stdout=sys.stdout, stderr=subprocess.STDOUT) + else: + subprocess.run(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT)