diff --git a/CMakeLists.txt b/CMakeLists.txt index 359ebc7..8c9586d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -20,7 +20,7 @@ endif() set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake/Modules/") set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${PROJECT_SOURCE_DIR}/findFFTW") -set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${PROJECT_SOURCE_DIR}/findFFTMPI") +set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${PROJECT_SOURCE_DIR}/findHEFFTE") set(LIBS) set(CMAKE_CXX_STANDARD 20) # C++20... @@ -38,7 +38,7 @@ include_directories(${Boost_INCLUDE_DIRS}) option(USE_MPI "Use MPI" ON) option(USE_OMP "Use OpenMP" OFF) -option(USE_FFTMPI "Use FFTMPI" ON) +option(USE_HEFFTE "Use Heffte" ON) ###mpi if(USE_MPI) @@ -64,13 +64,13 @@ if(USE_OMP) message(SEND_ERROR "OMP not yet supported / experimental.") endif() -if(USE_FFTMPI) +if(USE_HEFFTE) + add_definitions(-DHAVE_HEFFTE) + find_package(Heffte REQUIRED) find_package(FFTW REQUIRED) - find_package(FFTMPI REQUIRED) - add_definitions(-DHAVE_FFTMPI) + include_directories(${HEFFTE_INCLUDES}) include_directories(${FFTW_INCLUDE_DIRS}) - include_directories(${FFTMPI_INCLUDES}) - set(LIBS ${LIBS} "${FFTMPI_LIBRARIES}" "${FFTW_LIBRARIES}" ) + set(LIBS ${LIBS} "${HEFFTE_LIBRARIES}" "${FFTW_LIBRARIES}") endif() option(PROFILE "Use gperftools" OFF) @@ -81,6 +81,12 @@ if(PROFILE) set(LIBS ${LIBS} "${GPERFTOOLS_LIBRARIES}") endif() +find_package(HDF5 COMPONENTS C CXX HL REQUIRED) +include_directories( ${HDF5_INCLUDES} ) +include_directories (${HDF5_INCLUDE_DIR}) +set (LIBS ${LIBS} "${HDF5_CXX_LIBRARIES}") + + include_directories(src) add_subdirectory (src) diff --git a/app/CMakeLists.txt b/app/CMakeLists.txt index 3fa2ced..663d8bb 100644 --- a/app/CMakeLists.txt +++ b/app/CMakeLists.txt @@ -1,15 +1,24 @@ add_executable (TG ./TG/TG.cpp) -target_link_libraries (TG NOVA ${Boost_LIBRARIES} ${MPI_LIBRARIES} ${FFTMPI_LIBRARIES} ${LIBS}) +target_link_libraries (TG NOVA ${Boost_LIBRARIES} ${MPI_LIBRARIES} ${HEFFTE_LIBRARIES} ${LIBS}) install(TARGETS TG DESTINATION ${CMAKE_INSTALL_PREFIX}/bin) add_executable (TG_Poisson ./TG_Piosson/TG.cpp) -target_link_libraries (TG_Poisson NOVA ${Boost_LIBRARIES} ${MPI_LIBRARIES} ${FFTMPI_LIBRARIES} ${LIBS}) +target_link_libraries (TG_Poisson NOVA ${Boost_LIBRARIES} ${MPI_LIBRARIES} ${LIBS}) install(TARGETS TG_Poisson DESTINATION ${CMAKE_INSTALL_PREFIX}/bin) add_executable (OT ./OT/OT.cpp) -target_link_libraries (OT NOVA ${Boost_LIBRARIES} ${MPI_LIBRARIES} ${FFTMPI_LIBRARIES} ${LIBS}) +target_link_libraries (OT NOVA ${Boost_LIBRARIES} ${MPI_LIBRARIES} ${HEFFTE_LIBRARIES} ${LIBS}) install(TARGETS OT DESTINATION ${CMAKE_INSTALL_PREFIX}/bin) add_executable (OT_Poisson ./OT_Poisson/OT.cpp) -target_link_libraries (OT_Poisson NOVA ${Boost_LIBRARIES} ${MPI_LIBRARIES} ${FFTMPI_LIBRARIES} ${LIBS}) +target_link_libraries (OT_Poisson NOVA ${Boost_LIBRARIES} ${MPI_LIBRARIES} ${HEFFTE_LIBRARIES} ${LIBS}) install(TARGETS OT DESTINATION ${CMAKE_INSTALL_PREFIX}/bin) + +add_executable (shearLayer ./shearLayer/shearLayer.cpp) +target_link_libraries (shearLayer NOVA ${Boost_LIBRARIES} ${MPI_LIBRARIES} ${LIBS}) +install(TARGETS shearLayer DESTINATION ${CMAKE_INSTALL_PREFIX}/bin) + +add_executable (mhdjet ./mhdjet/mhdjet.cpp) +target_link_libraries (mhdjet NOVA ${Boost_LIBRARIES} ${MPI_LIBRARIES} ${LIBS}) +install(TARGETS mhdjet DESTINATION ${CMAKE_INSTALL_PREFIX}/bin) + diff --git a/app/OT/BEqn.H b/app/OT/BEqn.H index 9f6224d..f988aae 100644 --- a/app/OT/BEqn.H +++ b/app/OT/BEqn.H @@ -29,12 +29,12 @@ } - Bstar.correctBoundaryConditions(); - pBEqn.rhs( ex::div(Bstar) ); // /time.dt() - pBEqn.solve(); - pB.correctBoundaryConditions(); - B = Bstar - ex::grad(pB); //*time.dt() - B.correctBoundaryConditions(); +// Bstar.correctBoundaryConditions(); +// pBEqn.rhs( ex::div(Bstar) ); // /time.dt() +// pBEqn.solve(); +// pB.correctBoundaryConditions(); +// B = Bstar - ex::grad(pB); //*time.dt() +// B.correctBoundaryConditions(); if( time.writelog() ) { diff --git a/app/OT_Poisson/OT.cpp b/app/OT_Poisson/OT.cpp index 1fe3e67..3bd3cd7 100644 --- a/app/OT_Poisson/OT.cpp +++ b/app/OT_Poisson/OT.cpp @@ -23,7 +23,7 @@ int main(int argc, char* argv[]) { settings::process( argc, argv ); - Time time( 0.001, 8, 50 ); //args: dt, endT, write interval / steps + Time time( 0.001, 2.01, 10 ); //args: dt, endT, write interval / steps const scalar pi = tools::pi; parallelCom::decompose( settings::zoneName()+"/"+"mesh" ); @@ -45,12 +45,35 @@ int main(int argc, char* argv[]) std::shared_ptr > pB_ptr( std::make_shared >( mesh, 0, "pB" ) ); auto& pB = (*pB_ptr); - scalar mu = 0.01; - scalar eta = 0.01; + scalar mu = 0.057; + scalar eta = 0.057; poisson pEqn(p_ptr); poisson pBEqn(pB_ptr); + std::default_random_engine eng( parallelCom::myProcNo() ); + std::uniform_real_distribution dist(-1.0,1.0); + +/* + //initial conditions + for( int i=settings::m()/2; i() ); reduce( Jmax, maxOp() ); + reduce( Umax, maxOp() ); reduce( Em, plusOp() ); reduce( Jav, plusOp() ); reduce( omega, plusOp() ); @@ -141,15 +170,18 @@ int main(int argc, char* argv[]) Jav /= n; omega /= n; +// Jmax=Jmax*17.54; + scalar epsilon=0.0; epsilon = -mu*(omega*omega) - eta*(Jav*Jav); if( time.curTime() > time.dt() && parallelCom::master() ) { - data< >, 4> dU; for( int i=0; i<4; i++ ) { @@ -25,16 +26,18 @@ { Ustar = Ustar+settings::RK4a()[rk][j]*dU[j]; } - + } Ustar.correctBoundaryConditions(); - pEqn.rhs(ex::div(Ustar) / time.dt() / settings::RK4c()[rk]); + + pEqn.rhs(ex::div(Ustar) / time.dt() ); + pEqn.solve(); p.correctBoundaryConditions(); - U = Ustar - settings::RK4c()[rk]*time.dt() * ex::grad(p); + U = Ustar - time.dt() * ex::grad(p); U.correctBoundaryConditions(); - } - + + if( time.writelog() ) { diff --git a/app/TG_Piosson/TG.cpp b/app/TG_Piosson/TG.cpp index 9ff8dc2..8c85fb6 100644 --- a/app/TG_Piosson/TG.cpp +++ b/app/TG_Piosson/TG.cpp @@ -22,7 +22,7 @@ int main(int argc, char* argv[]) { settings::process( argc, argv ); - Time time( 0.01, 20, 1.0 ); //args: dt, endT, write interval / steps + Time time( 0.01, 20, 1 ); //args: dt, endT, write interval / steps const scalar pi = tools::pi; parallelCom::decompose( settings::zoneName()+"/"+"mesh" ); @@ -53,6 +53,7 @@ int main(int argc, char* argv[]) scalar z = -pi+(k-settings::m()/2)*mesh.dz()+mesh.origin().z(); U(i, j, k) = vector(std::sin(x)*std::cos(y)*std::cos(z),-std::cos(x)*std::sin(y)*std::cos(z),0); +// U(i, j, k) = vector(0.0, 0.0, 0.0); } } } @@ -79,8 +80,8 @@ int main(int argc, char* argv[]) p.write(settings::zoneName()+"/data", "p"); - if( time.writelog() ) - { +// if( time.writelog() ) +// { if( parallelCom::master() ) { std::cout << "Step: " << time.timeStep() << ". Time: " << time.curTime() << std::endl; @@ -88,7 +89,7 @@ int main(int argc, char* argv[]) } tools::CFL( U, mesh ); - } +// } scalar Ek=0.0; int n=0; diff --git a/app/mhdjet/BEqn.H b/app/mhdjet/BEqn.H new file mode 100644 index 0000000..f7861ab --- /dev/null +++ b/app/mhdjet/BEqn.H @@ -0,0 +1,56 @@ +{ + std::array >, 4> dB; + for( int i=0; i<4; i++ ) + { + reuseTmp dBt( mesh ); + dB[i]= dBt(); + } + + + reuseTmp Boldt( mesh ); + std::shared_ptr > Bold( Boldt() ); + + *Bold = B; + + for( int rk=0; rk<4; rk++ ) + { + + dB[rk] = + time.dt() * + ( + -ex::div( U, B ) + +ex::div( B, U ) + +ex::laplacian( eta, B ) + ); + + Bstar = Bold; + for( int j=0; j<=rk; j++ ) + { + Bstar = Bstar + settings::RK4a()[rk][j]*dB[j]; + } + + B = Bstar; + B.correctBoundaryConditions(); + + } + + pBEqn.rhs(ex::div(B)); + pBEqn.solve(); + pB.correctBoundaryConditions(); + B -= ex::grad(pB); + B.correctBoundaryConditions(); + + J = ex::curl(B); + J.correctBoundaryConditions(); + + if( time.writelog() ) + { + auto divB = ex::div(B); + auto maxdivB = tools::maximum(divB); + if( parallelCom::master() ) + { + std::cout<<"Magnetic field divergence: " << maxdivB< >, 4> dU; + for( int i=0; i<4; i++ ) + { + reuseTmp dUt( mesh ); + dU[i]= dUt(); + } + + reuseTmp UOldt( mesh ); + std::shared_ptr > UOld( UOldt() ); + + reuseTmp Ustartt( mesh ); + std::shared_ptr > Ustart( Ustartt() ); + *Ustart = Ustart; + + *UOld = U; + + for( int rk=0; rk<4; rk++ ) + { + dU[rk] = + time.dt() * + ( + -ex::div( U, U ) + +ex::laplacian( mu, U ) + +ex::div( B, B ) + -ex::grad( 0.5*(B&B) ) + ); + + Ustar = UOld; + for( int j=0; j<=rk; j++ ) + { + Ustar = Ustar+settings::RK4a()[rk][j]*dU[j]; + } + + } + + Ustar.correctBoundaryConditions(); + pEqn.rhs(ex::div(Ustar) / time.dt()); + pEqn.solve(); + p.correctBoundaryConditions(); + U = Ustar - time.dt() * ex::grad(p); + + U.correctBoundaryConditions(); + + Umag = (U&U); + + if( time.writelog() ) + { + auto divU = ex::div(U); + auto maxdivU = tools::maximum(divU); + + if( parallelCom::master() ) + { + std::cout<<"Velocity divergence: " << maxdivU< +#include "Types/vector/vector.h" +#include "ex/grad/grad.h" +#include "ex/div/div.h" +#include "ex/curl/curl.h" +#include "ex/laplacian/laplacian.h" +#include "Time/Time.h" +#include "parallelCom/parallelCom.h" +#include "settings/settings.h" +#include "BC/BC.h" +#include +#include "Tools/Tools.h" +#include "poisson/poisson.h" +#include + +#include "H5cpp.h" + +#include +#include +#include + + //H5 Create + //H5 Write dataset using MPI + void writeVectorFieldToHDF5( MPI_Comm comm, const std::string &fileName, const std::vector>> &vectorField) { + + MPI_Comm world = parallelCom::world(); + int rank, size; + MPI_Comm_rank(comm, &rank); + MPI_Comm_size(comm, &size); + + // Dimensions of the vector field + hsize_t hni = vectorField.size(); + hsize_t hnj = vectorField[0].size(); + hsize_t hnk = vectorField[0][0].size(); + + //prepare data + std::vector data(hni * hnj * hnk ); + int n=0; + for (int i = 0; i < hni; ++i) { + for (int j = 0; j < hnj; ++j) { + for (int k = 0; k < hnk; ++k) { + data[n] = vectorField[i][j][k]; + n++; + } + } + } + + // Create an HDF5 file + hid_t plist_id = H5Pcreate(H5P_FILE_ACCESS); + H5Pset_fapl_mpio(plist_id, parallelCom::world(), MPI_INFO_NULL); + hid_t file_id = H5Fcreate(fileName.c_str(), H5F_ACC_TRUNC, H5P_DEFAULT, plist_id); + + // Create a data space + hsize_t dims[] = { static_cast((hni)*parallelCom::ni() ), + static_cast((hnj)*parallelCom::nj() ), + static_cast((hnk)*parallelCom::nk() ) }; // 3 for the vector components (x, y, z) + hid_t dataspace_id = H5Screate_simple(3, dims, NULL); + + //Create a dataset + hid_t dataset_id = H5Dcreate2(file_id, "/3DArray", H5T_NATIVE_DOUBLE, dataspace_id, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); + + // Create the local dataspace for each MPI rank + hsize_t local_dims[3] = {hni, hnj, hnk}; + hsize_t start[3] = {static_cast((hni*parallelCom::i()) ) , static_cast((hnj*parallelCom::j()) ) , static_cast((hnk*parallelCom::k()) ) }; + hsize_t count[3] = {hni, hnj, hnk}; + hid_t memspace_id = H5Screate_simple(3, local_dims, NULL); + + + //Sanity Checks for data indexing +// std::cout << " ni: " << parallelCom::ni() << " nj: " << parallelCom::nj() << " nk: " << parallelCom::nk() << " i: " < &vectorField) { + // Dimensions of the vector field + hsize_t hni = vectorField[0]-settings::m(); + hsize_t hnj = vectorField[1]-settings::m(); + hsize_t hnk = vectorField[2]-settings::m(); + + // Create an HDF5 file + H5::H5File file(fileName, H5F_ACC_TRUNC); + + // Create a data space + hsize_t dims[3] = {hni, hnj, hnk }; // 3 for the vector components (x, y, z) + H5::DataSpace dataspace(3, dims); + + // Create a dataset + H5::DataSet dataset = file.createDataSet("VectorField", H5::PredType::IEEE_F64LE, dataspace); + + // Prepare data to write + std::vector data(hni * hnj * hnk * 3); + for (int i = 0; i < hni; ++i) { + for (int j = 0; j < hnj; ++j) { + for (int k = 0; k < hnk; ++k) { + data[(i * hnj * hnk + j * hnk + k) + 0] = vectorField[0]; // x component + data[(i * hnj * hnk + j * hnk + k) + 1] = vectorField[1]; // y component + data[(i * hnj * hnk + j * hnk + k) + 2] = vectorField[2]; // z component + + } + } + } + + // Write data to the dataset + dataset.write(data.data(), H5::PredType::IEEE_F64LE); + + // Close the dataset and file + dataset.close(); + file.close(); +} +*/ + + + +int main(int argc, char* argv[]) +{ + settings::restart() = true; + + settings::process( argc, argv ); + Time time( 0.02, 160.02, 50 ); //args: dt, endT, write interval / steps + + const scalar pi = tools::pi; + parallelCom::decompose( settings::zoneName()+"/"+"mesh" ); + + Mesh mesh( settings::zoneName()+"/"+"mesh", time ); + + mesh.write(settings::zoneName()+"/data"); + + + Field U( mesh, vector(0,0,0), "U" ); + Field Ustar( mesh, vector(0,0,0), "U" ); + Field Ustart( mesh, vector(0,0,0), "U" ); + Field B( mesh, vector(0,0,0), "B" ); + Field Bstar( mesh, vector(0,0,0), "B" ); + Field J( mesh, vector(0,0,0), "B" ); + Field omegav( mesh, vector(0,0,0), "U" ); + + std::shared_ptr > Umag_ptr(std::make_shared >(mesh, 0, "U") ); + auto& Umag = (*Umag_ptr); + + std::shared_ptr > p_ptr( std::make_shared >( mesh, 0, "p" ) ); + auto& p = (*p_ptr); + + std::shared_ptr > pB_ptr( std::make_shared >( mesh, 0, "pB" ) ); + auto& pB = (*pB_ptr); + + std::shared_ptr > fftx_ptr( std::make_shared >( mesh, 0, "U" ) ); + auto& fftx = (*fftx_ptr); + + scalar mu = 0.000285714; + scalar eta = 0.000285714; + + poisson fftxDo(fftx_ptr); + poisson pEqn(p_ptr); + poisson pBEqn(pB_ptr); + + std::default_random_engine eng( parallelCom::myProcNo() ); + std::uniform_real_distribution dist(-1.0,1.0); + + //initial conditions + for( int i=settings::m()/2; i>> vectorField(hni, std::vector>(hnj, std::vector(hnk))); + //write Ux + int n=0; + for (int k = 0; k < hnk; ++k) { + for (int j = 0; j < hnj; ++j) { + for (int i = 0; i < hni; ++i) { + vectorField[i][j][k] = U( (i+settings::m()/2), (j+settings::m()/2), (k+settings::m()/2) ).x(); // x component + n++; + } + } + } + std::string H5title = "./h5data/Ux" + std::to_string( (time.curTime()-time.dt()) ) + ".h5"; + writeVectorFieldToHDF5(parallelCom::world(), H5title, vectorField); + + //write Uy + n=0; + for (int k = 0; k < hnk; ++k) { + for (int j = 0; j < hnj; ++j) { + for (int i = 0; i < hni; ++i) { + vectorField[i][j][k] = U( (i+settings::m()/2), (j+settings::m()/2), (k+settings::m()/2) ).y(); // x component + n++; + } + } + } + H5title = "./h5data/Uy_" + std::to_string( (time.curTime()-time.dt()) ) + ".h5"; + writeVectorFieldToHDF5(parallelCom::world(), H5title, vectorField); + + //write Uz + n=0; + for (int k = 0; k < hnk; ++k) { + for (int j = 0; j < hnj; ++j) { + for (int i = 0; i < hni; ++i) { + vectorField[i][j][k] = U( (i+settings::m()/2), (j+settings::m()/2), (k+settings::m()/2) ).z(); // x component + n++; + } + } + } + H5title = "./h5data/Uz_" + std::to_string( (time.curTime()-time.dt()) ) + ".h5"; + writeVectorFieldToHDF5(parallelCom::world(), H5title, vectorField); + + //write p + n=0; + for (int k = 0; k < hnk; ++k) { + for (int j = 0; j < hnj; ++j) { + for (int i = 0; i < hni; ++i) { + vectorField[i][j][k] = p( (i+settings::m()/2), (j+settings::m()/2), (k+settings::m()/2) ); // x component + n++; + } + } + } + H5title = "./h5data/p_" + std::to_string( (time.curTime()-time.dt()) ) + ".h5"; + writeVectorFieldToHDF5(parallelCom::world(), H5title, vectorField); + + //write Bx + n=0; + for (int k = 0; k < hnk; ++k) { + for (int j = 0; j < hnj; ++j) { + for (int i = 0; i < hni; ++i) { + vectorField[i][j][k] = B( (i+settings::m()/2), (j+settings::m()/2), (k+settings::m()/2) ).x(); // x component + n++; + } + } + } + H5title = "./h5data/Bx_" + std::to_string( (time.curTime()-time.dt()) ) + ".h5"; + writeVectorFieldToHDF5(parallelCom::world(), H5title, vectorField); + + //write By + n=0; + for (int k = 0; k < hnk; ++k) { + for (int j = 0; j < hnj; ++j) { + for (int i = 0; i < hni; ++i) { + vectorField[i][j][k] = B( (i+settings::m()/2), (j+settings::m()/2), (k+settings::m()/2) ).y(); // x component + n++; + } + } + } + H5title = "./h5data/By_" + std::to_string( (time.curTime()-time.dt()) ) + ".h5"; + writeVectorFieldToHDF5(parallelCom::world(), H5title, vectorField); + + //write Bz + n=0; + for (int k = 0; k < hnk; ++k) { + for (int j = 0; j < hnj; ++j) { + for (int i = 0; i < hni; ++i) { + vectorField[i][j][k] = B( (i+settings::m()/2), (j+settings::m()/2), (k+settings::m()/2) ).z(); // x component + n++; + } + } + } + H5title = "./h5data/Bz_" + std::to_string( (time.curTime()-time.dt()) ) + ".h5"; + writeVectorFieldToHDF5(parallelCom::world(), H5title, vectorField); + + //write Jx + n=0; + for (int k = 0; k < hnk; ++k) { + for (int j = 0; j < hnj; ++j) { + for (int i = 0; i < hni; ++i) { + vectorField[i][j][k] = J( (i+settings::m()/2), (j+settings::m()/2), (k+settings::m()/2) ).x(); // x component + n++; + } + } + } + H5title = "./h5data/Jx_" + std::to_string( (time.curTime()-time.dt()) ) + ".h5"; + writeVectorFieldToHDF5(parallelCom::world(), H5title, vectorField); + + //write Jy + n=0; + for (int k = 0; k < hnk; ++k) { + for (int j = 0; j < hnj; ++j) { + for (int i = 0; i < hni; ++i) { + vectorField[i][j][k] = J( (i+settings::m()/2), (j+settings::m()/2), (k+settings::m()/2) ).y(); // x component + n++; + } + } + } + H5title = "./h5data/Jy_" + std::to_string( (time.curTime()-time.dt()) ) + ".h5"; + writeVectorFieldToHDF5(parallelCom::world(), H5title, vectorField); + + //write Jz + n=0; + for (int k = 0; k < hnk; ++k) { + for (int j = 0; j < hnj; ++j) { + for (int i = 0; i < hni; ++i) { + vectorField[i][j][k] = J( (i+settings::m()/2), (j+settings::m()/2), (k+settings::m()/2) ).z(); // x component + n++; + } + } + } + H5title = "./h5data/Jz_" + std::to_string( (time.curTime()-time.dt()) ) + ".h5"; + writeVectorFieldToHDF5(parallelCom::world(), H5title, vectorField); + + + + + + if( parallelCom::master() ) + { + std::cout << "Vector field written to vector_field.h5" << std::endl; + } + } + + + scalar Ek=0.0; + scalar Em=0.0; + scalar Jmax=0.0; + scalar Umax=0.0; + omegav = ex::curl(U); + scalar Jav=0.0; + scalar omega=0.0; + int n=0; + + for( int i=settings::m()/2; i() ); + reduce( Jmax, maxOp() ); + reduce( Umax, maxOp() ); + reduce( Em, plusOp() ); + reduce( Jav, plusOp() ); + reduce( omega, plusOp() ); + reduce( n, plusOp() ); + + Ek /= n; + Em /= n; + Jav /= n; + omega /= n; + + scalar epsilon=0.0; + epsilon = -mu*(omega*omega) - eta*(Jav*Jav); + + if( time.curTime() > time.dt() && parallelCom::master() ) + { + data< >, 4> dB; + for( int i=0; i<4; i++ ) + { + reuseTmp dBt( mesh ); + dB[i]= dBt(); + } + + + reuseTmp Boldt( mesh ); + std::shared_ptr > Bold( Boldt() ); + + *Bold = B; + + for( int rk=0; rk<4; rk++ ) + { + + dB[rk] = + time.dt() * + ( + -ex::div( U, B ) + +ex::div( B, U ) + +ex::laplacian( eta, B ) + ); + + Bstar = Bold; + for( int j=0; j<=rk; j++ ) + { + Bstar = Bstar + settings::RK4a()[rk][j]*dB[j]; + } + + B = Bstar; + B.correctBoundaryConditions(); + + } + + pBEqn.rhs(ex::div(B)); + pBEqn.solve(); + pB.correctBoundaryConditions(); + B -= ex::grad(pB); + B.correctBoundaryConditions(); + +// J = ex::curl(B); +// J.correctBoundaryConditions(); + + if( time.writelog() ) + { + auto divB = ex::div(B); + auto maxdivB = tools::maximum(divB); + if( parallelCom::master() ) + { + std::cout<<"Magnetic field divergence: " << maxdivB< dTt( mesh ); + std::shared_ptr > dT( dTt() ); + + reuseTmp Tct( mesh ); + std::shared_ptr > Tc( Tct() ); + + reuseTmp TOldt( mesh ); + std::shared_ptr > TOld( TOldt() ); + + *Tc = T; + *TOld = T; + + for( int rk=0; rk<4; rk++ ) + { + *dT = + time.dt() * + ( + -ex::div( U, T ) + +ex::laplacian( ka, T ) + ); + +// drhoT->correctBoundaryConditions(); + + *Tc = Tc+RK4[rk]*dT; + + if( rk<3 ) + { + T = TOld + (int(rk/2)+1)/2.0 * (dT); + } + else + { + T=Tc; + } + + T.correctBoundaryConditions(); + } +} diff --git a/app/shearLayer/UEqn.H b/app/shearLayer/UEqn.H new file mode 100644 index 0000000..b19fa90 --- /dev/null +++ b/app/shearLayer/UEqn.H @@ -0,0 +1,52 @@ +{ + reuseTmp dUt( mesh ); + std::shared_ptr > dU( dUt() ); + + reuseTmp UOldt( mesh ); + std::shared_ptr > UOld( UOldt() ); + + reuseTmp Uct( mesh ); + std::shared_ptr > Uc( Uct() ); + + *UOld = U; + + + for( int rk=0; rk<4; rk++ ) + { + *dU = + time.dt() * + ( + -ex::div( U, U ) + +ex::laplacian( mu, U ) +// +mu*ex::div( ( dev2(transpose(gradU) ) ) ) +// -( (rho0*T)*(g) ) + +ex::div( B, B ) + -ex::grad( 0.5*(B&B) ) + ); + + + Ustar = UOld; + Ustar = Ustar+RK4[rk]*dU; + + Ustar.correctBoundaryConditions(); + pEqn.rhs(ex::div(Ustar) / time.dt() / settings::RK4c()[rk]); + pEqn.solve(); + p.correctBoundaryConditions(); + U = Ustar - settings::RK4c()[rk]*time.dt() * ex::grad(p); + + U.correctBoundaryConditions(); + } + +// if( time.writelog() ) +// { + auto divU = ex::div(U); + auto maxdivU = tools::maximum(divU); + + if( parallelCom::master() ) + { + std::cout<<"Velocity divergence: " << maxdivU< +#include "Types/vector/vector.h" +#include "ex/grad/grad.h" +#include "ex/div/div.h" +#include "ex/laplacian/laplacian.h" +#include "Time/Time.h" +#include "parallelCom/parallelCom.h" +#include "settings/settings.h" +#include "BC/BC.h" +#include +#include "Tools/Tools.h" +#include "poisson/poisson.h" + +#include + +#include +#include +#include + +int main(int argc, char* argv[]) +{ + settings::process( argc, argv ); + Time time( 0.001, 420, 1 ); //args: dt, endT, write interval / steps + + const scalar pi = tools::pi; + parallelCom::decompose( settings::zoneName()+"/"+"mesh" ); + + Mesh mesh( settings::zoneName()+"/"+"mesh", time ); + + mesh.write(settings::zoneName()+"/data"); +/*-------------------- + scalar Mach=0.2; + scalar Re0=1967.0; + scalar Ri0=0.08; + scalar Pr=7; + + scalar rho0=999.57; + scalar c=0.0417; + + scalar u0=c*Mach; + scalar h0=0.2;//Re0*mu/rho0/u0; + scalar t0=0.1;//-Ri0*u0*u0/g[2]/h0; + scalar mu = rho0*u0*h0/Re0; + scalar ka = mu/Pr; + scalar eta = mu; +// vector g( 0, 0, -Ri0*u0*u0/t0/h0 ); + vector g( 0, 0, -0.000011129 ); + + Field U( mesh, vector(1,0,0), "U" ); + Field T( mesh, 0, "T" ); + Field pHyd( mesh,0, "pHyd" ); + Field Ustar( mesh, vector(0,0,0), "U" ); +// Field B( mesh, vector(0,0,0), "B" ); +// Field Bstar( mesh, vector(0,0,0), "B" ); + +----------------------*/ + + Field U( mesh, vector(0,0,0), "U" ); + Field Ustar( mesh, vector(0,0,0), "U" ); + Field B( mesh, vector(0,0,0), "B" ); + Field Bstar( mesh, vector(0,0,0), "B" ); + + + scalar mu = 1.0/28000.0; + scalar u0=1.0; + scalar delta = 1.0/28.0; + scalar cn = 0.001; + scalar eta = mu*10.0; + + + std::default_random_engine eng( parallelCom::myProcNo() ); + std::uniform_real_distribution dist(-1.0,1.0); + + std::shared_ptr > p_ptr( std::make_shared >( mesh, (0.87), "p" ) ); + auto& p = (*p_ptr); + + std::shared_ptr > pB_ptr( std::make_shared >( mesh, 0, "pB" ) ); + auto& pB = (*pB_ptr); + + poisson pEqn(p_ptr); + poisson pBEqn(pB_ptr); + + { + //initial conditions + for( int i=settings::m()/2; i upt( mesh ); +// std::shared_ptr > up( upt() ); +// reuseTmp Tpt( mesh ); +// std::shared_ptr > Tp( Tpt() ); +// #include "layerAverage.H" +// +// reuseTmp epst( mesh ); +// std::shared_ptr > eps( epst() ); +// *eps = 2.0*( ex::grad( up ) && ex::grad( up ) )*mu/rho0; +// (*eps).write("e"); +// +// reuseTmp xit( mesh ); +// std::shared_ptr > xi( xit() ); +// *xi = 2.0*( ex::grad( Tp ) & ex::grad( Tp ) )*ka/rho0; +// (*xi).write("x"); +// +// reuseTmp vortt( mesh ); +// std::shared_ptr > vort( vortt() ); +// *vort = ex::curl( U ); +// (*vort).write("v"); +// +// reuseTmp gpt( mesh ); +// std::shared_ptr > gp( gpt() ); +// *gp = ex::grad(p); +// (*gp).write("z"); +// +// mesh.writeSOS(); +// } + } + + + std::cout<< timer.elapsed().wall / 1e9 < drhoTt( mesh ); + std::shared_ptr > drhoT( drhoTt() ); + + reuseTmp rhoTct( mesh ); + std::shared_ptr > rhoTc( rhoTct() ); + + reuseTmp rhoTOldt( mesh ); + std::shared_ptr > rhoTOld( rhoTOldt() ); + + *rhoTc = rho*T; + *rhoTOld = rho*T; + + for( int rk=0; rk<4; rk++ ) + { + *drhoT = + time.dt() * + ( + -ex::div( rho*U, T ) + +ex::laplacian( ka, T ) + ); + +// drhoT->correctBoundaryConditions(); + + *rhoTc = rhoTc+RK4[rk]*drhoT; + + if( rk<3 ) + { + T = rhoTOld/rho + (int(rk/2)+1)/2.0 * (drhoT/rho); + } + else + { + T=rhoTc/rho; + } + + T.correctBoundaryConditions(); + } +} diff --git a/app/shearLayerWC/UEqn.H b/app/shearLayerWC/UEqn.H new file mode 100644 index 0000000..025635c --- /dev/null +++ b/app/shearLayerWC/UEqn.H @@ -0,0 +1,42 @@ +{ + reuseTmp drhoUt( mesh ); + std::shared_ptr > drhoU( drhoUt() ); + + reuseTmp rhoUOldt( mesh ); + std::shared_ptr > rhoUOld( rhoUOldt() ); + + reuseTmp rhoUct( mesh ); + std::shared_ptr > rhoUc( rhoUct() ); + + *rhoUc = rho*U; + *rhoUOld = rho*U; + +// std::shared_ptr > gradU( fdc::grad( U ) ); +// gradU->correctBoundaryConditions(); + + for( int rk=0; rk<4; rk++ ) + { + *drhoU = + time.dt() * + ( + -ex::div( rho*U, U ) + -ex::grad( p ) + +ex::laplacian( mu, U ) +// +mu*ex::div( ( dev2(transpose(gradU) ) ) ) + -( (rho*T)*g ) + ); + + *rhoUc = rhoUc+RK4[rk]*drhoU; + + if( rk<3 ) + { + U = rhoUOld/rho + (int(rk/2)+1)/2.0 * (drhoU/rho); + } + else + { + U=rhoUc/rho; + } + + U.correctBoundaryConditions(); + } +} diff --git a/app/shearLayerWC/hydrostaticP.H b/app/shearLayerWC/hydrostaticP.H new file mode 100644 index 0000000..9f636e3 --- /dev/null +++ b/app/shearLayerWC/hydrostaticP.H @@ -0,0 +1,39 @@ +{ + for( int i=settings::m()/2; i cumTot( parallelCom::worldSize(), scalar(0.0) ); + cumTot[parallelCom::myProcNo()] = pHydc; + + all_reduce( cumTot, std::plus() ); + + pHydc = 0.0; + double aboveProcs=0.0; + + for( int pk=parallelCom::k()+1; pk=settings::m()/2; k-- ) + { + pHydc += T(i, j, k)*mesh.dz()*g[2]*rho0; + pHyd(i, j, k) = pHydc; // + aboveProcs; + } + + } + } +//add field and implement null boundary condition +// pHyd.correctBoundaryConditions(); + + p += pHyd; +} + diff --git a/app/shearLayerWC/layerAverage.H b/app/shearLayerWC/layerAverage.H new file mode 100644 index 0000000..0e7087e --- /dev/null +++ b/app/shearLayerWC/layerAverage.H @@ -0,0 +1,52 @@ +{ + for( int k=0; k proc_avgU( parallelCom::worldSize(), 0.0 ); + std::vector proc_avgT( parallelCom::worldSize(), 0.0 ); + std::vector proc_N( parallelCom::worldSize(), 0 ); + + for( int j=settings::m()/2; j()); + all_reduce(proc_avgT, std::plus()); + all_reduce(proc_N, std::plus()); + + vector avgU(0,0,0); + scalar avgT=0.0; + int N=0; + + for( int proci=0; procicorrectBoundaryConditions(); + //Tp->correctBoundaryConditions(); + } +} + diff --git a/app/shearLayerWC/pEqn.H b/app/shearLayerWC/pEqn.H new file mode 100644 index 0000000..cd6ff37 --- /dev/null +++ b/app/shearLayerWC/pEqn.H @@ -0,0 +1,4 @@ +{ + p = (c*c/2.0/rho0)*rho*rho; +} + diff --git a/app/shearLayerWC/rhoEqn.H b/app/shearLayerWC/rhoEqn.H new file mode 100644 index 0000000..4d4334c --- /dev/null +++ b/app/shearLayerWC/rhoEqn.H @@ -0,0 +1,38 @@ +{ + reuseTmp drhot( mesh ); + std::shared_ptr > drho( drhot() ); + + reuseTmp rhoct( mesh ); + std::shared_ptr > rhoc( rhoct() ); + + reuseTmp rhoOldt( mesh ); + std::shared_ptr > rhoOld( rhoOldt() ); + + *rhoOld = rho; + *rhoc = rho; + + for( int rk=0; rk<4; rk++ ) + { + *drho = + time.dt() * + ( + -ex::div( rho*U ) + ); + + //drho->correctBoundaryConditions(); + + *rhoc = rhoc+RK4[rk]*drho; + + if( rk<3 ) + { + rho = rhoOld + (int(rk/2)+1)/2.0 * drho; + } + else + { + rho=rhoc; + } + + rho.correctBoundaryConditions(); + } + +} diff --git a/app/shearLayerWC/shearLayer.cpp b/app/shearLayerWC/shearLayer.cpp new file mode 100644 index 0000000..3c04145 --- /dev/null +++ b/app/shearLayerWC/shearLayer.cpp @@ -0,0 +1,247 @@ +#include "Field/Field.h" +#include +#include "Types/vector/vector.h" +#include "ex/grad/grad.h" +#include "ex/div/div.h" +#include "ex/curl/curl.h" +#include "ex/laplacian/laplacian.h" +#include "Time/Time.h" +#include "parallelCom/parallelCom.h" +#include "settings/settings.h" +#include "BC/BC.h" +#include +#include "Tools/Tools.h" + +#include + +#include +#include + +int main(int argc, char* argv[]) +{ + settings::process( argc, argv ); + Time time( 0.01, 1000000, 100 ); //args: dt, endT, write interval / steps + + parallelCom::decompose( settings::zoneName()+"/"+"mesh" ); + + Mesh mesh( settings::zoneName()+"/"+"mesh", time ); + + mesh.write(settings::zoneName()+"/data"); + + scalar Mach=0.2; + scalar Re0=1967.0; + scalar Ri0=0.08; + scalar Pr=7; + + scalar rho0=999.57; + scalar c=0.0417; + + scalar u0=c*Mach; + scalar h0=0.2;//Re0*mu/rho0/u0; + scalar t0=0.1;//-Ri0*u0*u0/g[2]/h0; + scalar mu = rho0*u0*h0/Re0; + scalar ka = mu/Pr; + vector g( 0, 0, -Ri0*u0*u0/t0/h0 ); + + Field U( mesh, vector(1,0,0), "U" ); + Field T( mesh, 0, "T" ); + Field p( mesh,0, "p" ); + Field pHyd( mesh,0, "pHyd" ); + Field rho( mesh, rho0, "rho" ); + +//------------------------------------------- +// Time time( 5e-9, 1000000, scalar(6*h0/u0) ); +// +// int ni=840*1.5; +// int nj=102*1.5; +// int nk=900*1.5; +// +// parallelCom::decompose( ni, nj, nk ); +// +// Mesh mesh( ni, nj, nk, 14.0*h0, 1.7*h0, 15*h0, time ); +// +// +// time.dt() = 0.5*std::min(mesh.dx(), std::min( mesh.dy(), mesh.dz() ) )/(u0/Mach); +// +// if( parallelCom::master() ) +// { +// std::cout<<"Speed of sound: "< dist(-1.0,1.0); + + +// if( settings::restart() ) +// { +// U.read("U"); +// rho.read("r"); +// T.read("T"); +// p.read("p"); +// +// U.correctBoundaryConditions(); +// p.correctBoundaryConditions(); +// rho.correctBoundaryConditions(); +// T.correctBoundaryConditions(); +// } +// else + { + #include "pEqn.H" + //initial conditions + for( int i=settings::m()/2; i upt( mesh ); +// std::shared_ptr > up( upt() ); +// reuseTmp Tpt( mesh ); +// std::shared_ptr > Tp( Tpt() ); +// #include "layerAverage.H" +// +// reuseTmp epst( mesh ); +// std::shared_ptr > eps( epst() ); +// *eps = 2.0*( ex::grad( up ) && ex::grad( up ) )*mu/rho0; +// (*eps).write("e"); +// +// reuseTmp xit( mesh ); +// std::shared_ptr > xi( xit() ); +// *xi = 2.0*( ex::grad( Tp ) & ex::grad( Tp ) )*ka/rho0; +// (*xi).write("x"); +// +// reuseTmp vortt( mesh ); +// std::shared_ptr > vort( vortt() ); +// *vort = ex::curl( U ); +// (*vort).write("v"); +// +// reuseTmp gpt( mesh ); +// std::shared_ptr > gp( gpt() ); +// *gp = ex::grad(p); +// (*gp).write("z"); +// +// mesh.writeSOS(); +// } + } + + + std::cout<< timer.elapsed().wall / 1e9 <::update Field* fld_ptr ) { + //bottom if( this->dir() == bottom && parallelCom::k() == 0 ) { for( int i=0; ini(); i++ ) @@ -11,20 +12,116 @@ void dirichletBC::update for( int j=0; jnj(); j++ ) { int k=settings::m()/2; - fld_ptr->operator()(i,j,k) = this->val(); - } + component(fld_ptr->operator()(i, j, k), this->comp()) = this->val(); + + for( int l=0; loperator()(i, j, k-(l+1)), this->comp()) = 2.0*(l+1)*this->val() - component(fld_ptr->operator()(i, j, k+(l+1) ), this->comp()); + } + + } } } - else if( this->dir() == top && parallelCom::k() == parallelCom::nk()-1 ) + + + //top + if( this->dir() == top && parallelCom::k() == parallelCom::nk()-1 ) { for( int i=0; ini(); i++ ) { for( int j=0; jnj(); j++ ) { int k=fld_ptr->nk()-settings::m()/2-1; - fld_ptr->operator()(i,j,k) = this->val(); + component(fld_ptr->operator()(i, j, k), this->comp()) = this->val(); + + for( int l=0; loperator()(i, j, k+(l+1)), this->comp()) = 2.0*(l+1)*this->val() - component(fld_ptr->operator()(i, j, k-(l+1) ), this->comp()); + } + } } } + + //east + if( this->dir() == east && parallelCom::i() == 0 ) + { + for( int j=0; jnj(); j++ ) + { + for( int k=0; knk(); k++ ) + { + int i=settings::m()/2; + component(fld_ptr->operator()(i, j, k), this->comp()) = this->val(); + + for( int l=0; loperator()(i-(l+1), j, k), this->comp()) = 2.0*(l+1)*this->val() - component(fld_ptr->operator()(i+(l+1), j, k ), this->comp()); + } + + } + } + } + + + //west + if( this->dir() == west && parallelCom::i() == parallelCom::ni()-1 ) + { + for( int j=0; jnj(); j++ ) + { + for( int k=0; knk(); k++ ) + { + int i=fld_ptr->ni()-settings::m()/2-1; + component(fld_ptr->operator()(i, j, k), this->comp()) = this->val(); + + for( int l=0; loperator()(i+(l+1), j, k), this->comp()) = 2.0*(l+1)*this->val() - component(fld_ptr->operator()(i-(l+1), j, k ), this->comp()); + } + + } + } + } + + + //south + if( this->dir() == south && parallelCom::j() == 0 ) + { + for( int i=0; ini(); i++ ) + { + for( int k=0; knk(); k++ ) + { + int j=settings::m()/2; + component(fld_ptr->operator()(i, j, k), this->comp()) = this->val(); + + for( int l=0; loperator()(i, j-(l+1), k), this->comp()) = 2.0*(l+1)*this->val() - component(fld_ptr->operator()(i, j+(l+1), k ), this->comp()); + } + + } + } + } + + + //north + if( this->dir() == north && parallelCom::j() == parallelCom::nj()-1 ) + { + for( int i=0; ini(); i++ ) + { + for( int k=0; knk(); k++ ) + { + int j=fld_ptr->nj()-settings::m()/2-1; + component(fld_ptr->operator()(i, j, k), this->comp()) = this->val(); + + for( int l=0; loperator()(i, j+(l+1), k), this->comp()) = 2.0*(l+1)*this->val() - component(fld_ptr->operator()(i, j-(l+1), k ), this->comp()); + } + + } + } + } + + } diff --git a/src/BC/neumannBC.hxx b/src/BC/neumannBC.hxx index 947b45f..43386ed 100644 --- a/src/BC/neumannBC.hxx +++ b/src/BC/neumannBC.hxx @@ -4,27 +4,131 @@ void neumannBC::update Field* fld_ptr ) { - if( this->dir() == bottom && parallelCom::k() == 0 ) - { - for( int i=0; ini(); i++ ) + //bottom + if( this->dir() == bottom && parallelCom::k() == 0 ) + { + for( int i=0; ini(); i++ ) + { + for( int j=0; jnj(); j++ ) + { + int k=settings::m()/2; + + component(fld_ptr->operator()(i, j, k), this->comp()) = component(fld_ptr->operator()(i, j, k+1), this->comp()) + -this->val()*fld_ptr->mesh().dz(); + + for( int l=0; loperator()(i, j, k-(l+1)), this->comp()) = component(fld_ptr->operator()(i, j, k+(l+1)), this->comp()) + -2.0*(l+1)*this->val()*fld_ptr->mesh().dz(); + } + } + } + } + + //top + if( this->dir() == top && parallelCom::k() == parallelCom::nk()-1 ) + { + for( int i=0; ini(); i++ ) + { + for( int j=0; jnj(); j++ ) + { + int k=fld_ptr->nk()-settings::m()/2-1; + + component(fld_ptr->operator()(i, j, k), this->comp()) = component(fld_ptr->operator()(i, j, k-1), this->comp()) + + this->val()*fld_ptr->mesh().dz(); + + for( int l=0; loperator()(i, j, k+(l+1)), this->comp()) = component(fld_ptr->operator()(i, j, k-(l+1)), this->comp()) + -2.0*(l+1)*this->val()*fld_ptr->mesh().dz(); + } + } + } + } + + + //east + if( this->dir() == east && parallelCom::i() == 0 ) { for( int j=0; jnj(); j++ ) - { - int k=settings::m()/2; - fld_ptr->operator()(i,j,k) = this->val(); - } + { + for( int k=0; knk(); k++ ) + { + int i=settings::m()/2; + + component(fld_ptr->operator()(i, j, k), this->comp()) = component(fld_ptr->operator()(i+1, j, k), this->comp()) + -this->val()*fld_ptr->mesh().dx(); + + for( int l=0; loperator()(i-(l+1), j, k), this->comp()) = component(fld_ptr->operator()(i+(l+1), j, k), this->comp()) + -2.0*(l+1)*this->val()*fld_ptr->mesh().dx(); + } + } + } } - } - else if( this->dir() == top && parallelCom::k() == parallelCom::nk()-1 ) - { - for( int i=0; ini(); i++ ) + + //west + if( this->dir() == west && parallelCom::i() == parallelCom::ni()-1 ) { for( int j=0; jnj(); j++ ) - { - int k=fld_ptr->nk()-settings::m()/2-1; - fld_ptr->operator()(i,j,k) = this->val(); + { + for( int k=0; knk(); k++ ) + { + int i=fld_ptr->ni()-settings::m()/2-1; + + component(fld_ptr->operator()(i, j, k), this->comp()) = component(fld_ptr->operator()(i-1, j, k), this->comp()) + + this->val()*fld_ptr->mesh().dx(); + + for( int l=0; loperator()(i+(l+1), j, k), this->comp()) = component(fld_ptr->operator()(i-(l+1), j, k), this->comp()) + -2.0*(l+1)*this->val()*fld_ptr->mesh().dx(); + } + } + } + } + + //south + if( this->dir() == south && parallelCom::j() == 0 ) + { + for( int i=0; ini(); i++ ) + { + for( int k=0; knk(); k++ ) + { + int j=settings::m()/2; + + component(fld_ptr->operator()(i, j, k), this->comp()) = component(fld_ptr->operator()(i, j+1, k), this->comp()) + -this->val()*fld_ptr->mesh().dy(); + + for( int l=0; loperator()(i, j-(l+1), k), this->comp()) = component(fld_ptr->operator()(i, j+(l+1), k), this->comp()) + -2.0*(l+1)*this->val()*fld_ptr->mesh().dy(); + } + } } } - } -} + //north + if( this->dir() == north && parallelCom::j() == parallelCom::nj()-1 ) + { + for( int i=0; ini(); i++ ) + { + for( int k=0; knk(); k++ ) + { + int j=fld_ptr->nj()-settings::m()/2-1; + + component(fld_ptr->operator()(i, j, k), this->comp()) = component(fld_ptr->operator()(i, j-1, k), this->comp()) + + this->val()*fld_ptr->mesh().dy(); + + for( int l=0; loperator()(i, j+(l+1), k), this->comp()) = component(fld_ptr->operator()(i, j-(l+1), k), this->comp()) + -2.0*(l+1)*this->val()*fld_ptr->mesh().dy(); + } + } + } + } + +} diff --git a/src/Field/Field.h b/src/Field/Field.h index 57edb6c..223b8b8 100644 --- a/src/Field/Field.h +++ b/src/Field/Field.h @@ -18,7 +18,6 @@ */ #ifndef FIELD_H #define FIELD_H - #if defined(_OPENMP) #include #endif @@ -129,6 +128,7 @@ class Field void correctBoundaryConditions(); void write(std::string, std::string); +// void H5write(std::string, std::string); void read(std::string, std::string); void read(std::string, std::string, int); void setBC( std::string ); diff --git a/src/Mesh/Mesh.cpp b/src/Mesh/Mesh.cpp index 4c1dd5e..043deca 100644 --- a/src/Mesh/Mesh.cpp +++ b/src/Mesh/Mesh.cpp @@ -18,7 +18,8 @@ */ #include "Mesh/Mesh.h" #include - +#include +#include "H5Cpp.h" Mesh::Mesh( std::string fileName, Time& runTime ) : @@ -103,7 +104,7 @@ Mesh::Mesh( std::string fileName, Time& runTime ) } origin_ = vector( (ni_-1)*dx_*parallelCom::i()+ox, (nj_-1)*dy_*parallelCom::j()+oy, (nk_-1)*dz_*parallelCom::k()+oz ); -#ifdef HAVE_FFTMPI +#ifdef HAVE_HEFFTE localStart_[0] = origin_[0]; localStart_[1] = origin_[1]; localStart_[2] = origin_[2]; @@ -324,8 +325,12 @@ void Mesh::writeCase( std::string path ) fout << "VARIABLE" << std::endl; fout << "vector per node: velocity " << boost::format("U.%|04|.*****.dat") % parallelCom::myProcNo() << std::endl ; + fout << "vector per node: Uprime " << boost::format("Uprime.%|04|.*****.dat") % parallelCom::myProcNo() << std::endl ; + fout << "vector per node: Udash " << boost::format("Udash.%|04|.*****.dat") % parallelCom::myProcNo() << std::endl ; fout << "scalar per node: p " << boost::format("p.%|04|.*****.dat") % parallelCom::myProcNo() << std::endl ; fout << "scalar per node: T " << boost::format("T.%|04|.*****.dat") % parallelCom::myProcNo() << std::endl ; + fout << "vector per node: B " << boost::format("B.%|04|.*****.dat") % parallelCom::myProcNo() << std::endl ; + fout << "vector per node: J " << boost::format("J.%|04|.*****.dat") % parallelCom::myProcNo() << std::endl ; fout << "TIME" << std::endl ; fout << "time set: 1" << std::endl ; @@ -343,7 +348,7 @@ void Mesh::writeCase( std::string path ) fout.close() ; - this->writeRestartInfo( path ); + this->writeRestartInfo( path ); } void Mesh::writeRestartInfo( std::string path ) @@ -364,13 +369,14 @@ void Mesh::writeRestartInfo( std::string path ) } + scalar Mesh::z( int k ) const { scalar z; if( hyperbollic_ ) { scalar A = std::pow( s1_/s2_, 0.5 ); -#ifdef HAVE_FFTMPI +#ifdef HAVE_HEFFTE int globK = k-settings::m()/2 + localStart_[2]; #else int globK = k-settings::m()/2 + (parallelCom::k())*(nk_-settings::m()-1); @@ -438,3 +444,4 @@ void Mesh::sendMUIPoints() } } #endif + diff --git a/src/Mesh/Mesh.h b/src/Mesh/Mesh.h index 17794a3..c655d10 100644 --- a/src/Mesh/Mesh.h +++ b/src/Mesh/Mesh.h @@ -28,6 +28,7 @@ #include #include #include +#include "H5Cpp.h" class Time; @@ -57,7 +58,7 @@ class Mesh vector origin_; - Time& runTime_; +// Time& runTime_; Box bbox_; Box bboxHalo_; @@ -67,13 +68,14 @@ class Mesh bool hyperbollic_; -#ifdef HAVE_FFTMPI +#ifdef HAVE_HEFFTE ptrdiff_t localStart_[3]; #endif public: Mesh( int, int, int, scalar, scalar, scalar, Time& ); Mesh( std::string, Time& ); + Time& runTime_; //access int ni() const @@ -156,7 +158,7 @@ class Mesh return bboxHalo_; } -#ifdef HAVE_FFTMPI +#ifdef HAVE_HEFFTE ptrdiff_t* glob_n() { return glob_n_; @@ -172,6 +174,7 @@ class Mesh void writeCase( std::string ); void writeSOS(); void writeRestartInfo( std::string ); +// void H5write( std::string ); }; #endif diff --git a/src/parallelCom/parallelCom.cpp b/src/parallelCom/parallelCom.cpp index 7911b28..4f16883 100644 --- a/src/parallelCom/parallelCom.cpp +++ b/src/parallelCom/parallelCom.cpp @@ -60,13 +60,13 @@ void parallelCom::decompose(int ni, int nj, int nk) { int np=worldSize_; - double lsq=1e10; + double lsq=1e10; for( int ii=1; ii<=np; ii++ ) { for( int jj=1; jj<=np/ii; jj++ ) { - if( std::fabs( double(np)/double(ii)/double(jj) - np/ii/jj ) < 1e-5 ) + if( std::fabs( double(np)/double(ii)/double(jj) - np/ii/jj ) < 1e-5 ) { int kk=np/ii/jj; @@ -82,6 +82,28 @@ void parallelCom::decompose(int ni, int nj, int nk) } } } +// pencil decomposition template + +// int ii=1; +// for( int jj=1; jj<=np; jj++ ) +// { +// if( std::fabs( double(np)/double(jj) - np/jj ) < 1e-5 ) +// { +// int kk=np/jj; +// +// double lsqc = std::pow( double(nj)/double(jj), 2 ) + std::pow( double(nk)/double(kk), 2 ); +// +// if( lsqc < lsq ) +// { +// lsq=lsqc; +// ni_=ii; +// nj_=jj; +// nk_=kk; +// } +// } +// } + + if( master() ) { diff --git a/src/poisson/poisson.cpp b/src/poisson/poisson.cpp index 4acbe98..4833fa7 100644 --- a/src/poisson/poisson.cpp +++ b/src/poisson/poisson.cpp @@ -12,22 +12,17 @@ poisson::poisson( std::shared_ptr > pptr ) nk_(pptr->mesh().nk()-settings::m()-1), localStart_(pptr->mesh().localStart()) { -fft = std::make_unique(parallelCom::world(), 2); - //fft options - //tflag=0; - permute = 0; + //Define FFT grid. This can be trimmed now HeFFTe is the prefered library. + int NFAST = n_[0]; + int NMID = n_[1]; + int NSLOW = n_[2]; - NFAST = n_[0]; - NMID = n_[1]; - NSLOW = n_[2]; - -// int MPI_Init( int *argc, char ***argv ); - MPI_Comm world = parallelCom::world(); - nfast = NFAST; nmid = NMID; nslow = NSLOW; + MPI_Comm world = parallelCom::world(); + npfast = parallelCom::ni(); npmid = parallelCom::nj(); npslow = parallelCom::nk(); @@ -42,7 +37,7 @@ fft = std::make_unique(parallelCom::world(), 2); klo = (int) 1.0*(ipslow)*nslow/npslow; khi = (int) 1.0*(ipslow+1)*nslow/npslow - 1; - //define local coords for wavenumbers + //Define local coords for wavenumber calculations inxlo = static_cast (1.0 * ipfast * nfast / npfast); inxhi = static_cast (1.0 * (ipfast+1) * nfast / npfast) - 1; @@ -54,6 +49,7 @@ fft = std::make_unique(parallelCom::world(), 2); nfft_in = (inxhi-inxlo+1) * (inyhi-inylo+1) * (inzhi-inzlo+1); +//Indexing for MPI running { for (int iproc = 0; iproc < parallelCom::worldSize(); iproc++) { if (parallelCom::myProcNo() != iproc) continue; @@ -69,9 +65,9 @@ fft = std::make_unique(parallelCom::world(), 2); iglobal = inxlo + ilocal; jglobal = inylo + jlocal; kglobal = inzlo + klocal; - printf("Value (%d,%d,%d) on proc %d \n", - iglobal,jglobal,kglobal, - parallelCom::myProcNo()); +// printf("Value (%d,%d,%d) on proc %d \n", +// iglobal,jglobal,kglobal, +// parallelCom::myProcNo()); if (m == 0) { localStart_[0] = iglobal; @@ -83,29 +79,88 @@ fft = std::make_unique(parallelCom::world(), 2); if (parallelCom::myProcNo() < parallelCom::worldSize()) MPI_Send(&tmp,0,MPI_INT,parallelCom::myProcNo()+1,0,world); } } + //Sanity check for fft coordinates. Uncomment for debugging + //printf("Assigned local start coordinates (%d,%d,%d) on proc %d \n", + // localStart_[0],localStart_[1],localStart_[2],parallelCom::myProcNo()); + //printf("local range ijk (%d,%d,%d,%d,%d,%d)\n", inxlo,inxhi,inylo,inyhi,inzlo,inzhi); -std::cout<setup(nfast, nmid, nslow, // 3d verion - ilo, ihi, jlo, - jhi, klo, khi, - ilo, ihi, jlo, - jhi, klo, khi, - permute, fftsize, sendsize, recvsize); + //initialize local fft domain + up_N = std::make_unique>(std::array ({inxlo, inylo, inzlo}) ,std::array ({inxhi, inyhi, inzhi})); + N = up_N.get(); + + up_NDCT = std::make_unique>(std::array ({inxlo, inylo, inzlo}) ,std::array ({inxhi, inyhi, inzhi})); + NDCT = up_NDCT.get(); + + //Construct and allocate memory for fft and dct + //HeFFTe::fftw_cos1 selects FFTW DCT TYPE REDFT00 + dct = std::make_unique>(*NDCT, *NDCT, world); + fft2d = std::make_unique>(*NDCT, *NDCT, world); + dct->nullify_executor(0); + dct->nullify_executor(2); + fft2d->nullify_executor(1); + + + fftx = std::make_unique>(*N, *N, world); + fftx->nullify_executor(1); + fftx->nullify_executor(2); + + fft = std::make_unique>(*N, *N, world); + + //plan 3d DCT-FFT-FFT + //perform 1d r2r_DCT + //translate domain from std::double to std::complex + //perform 2d r2c_DFT + //convolute and solve + //perform 2d c2r_IDFT + //translate domain from std::complex to std::double + //perform 1d r2r_DCT + //std::double cosphi + //std::complex dctphi + //std::complex dctphihat + //heffte::fft1d> + //heffte::fft2d> - //fft allocate memory - std::cout<(2*fftsize); - phi_ = up_phi_.get(); - up_k2_ = std::make_unique(2*fftsize); + //Create and allocate memory for FFT/DCT inputs & outputs. + up_k2_ = std::make_unique((fft->size_inbox())); k2_ = up_k2_.get(); + + up_k2c_ = std::make_unique((2*dct->size_inbox())); + k2c_ = up_k2c_.get(); + + up_k2cy_ = std::make_unique((2*dct->size_inbox())); + k2cy_ = up_k2cy_.get(); + + up_dctworkspace_ = std::make_unique>(2*dct->size_workspace()); + dctworkspace_ = up_dctworkspace_.get(); + + up_cosphihat_ = std::make_unique>((2*dct->size_outbox())); + cosphihat_ = up_cosphihat_.get(); + + up_cosphi_ = std::make_unique>((2*dct->size_inbox())); + cosphi_ = up_cosphi_.get(); + + up_cosphicomplex_ = std::make_unique>>((2*fft2d->size_outbox())); + cosphicomplex_ = up_cosphicomplex_.get(); - wavenumbers_(); + up_phi_ = std::make_unique>((fft->size_inbox())); + phi_ = up_phi_.get(); + + up_phihat_ = std::make_unique>>((fft->size_outbox())); + phihat_ = up_phihat_.get(); + + up_psi_ = std::make_unique>((fftx->size_inbox())); + psi_ = up_psi_.get(); + + up_psihat_ = std::make_unique>>((fftx->size_outbox())); + psihat_ = up_psihat_.get(); + + + + wavenumbersFFT_(); + wavenumbersDCT_(); } @@ -118,51 +173,81 @@ void poisson::rhs( std::shared_ptr > f ) { for( int i=(settings::m()/2); i<(pptr_->ni()-1-settings::m()/2); i++ ) { - phi_[l] = (double) f->operator()(i, j, k); - l++; - phi_[l] = 0.0; + phi_[0][l] = f->operator()(i, j, k) ; l++; } + } + } +} + +void poisson::rhsDCT( std::shared_ptr > f ) +{ + + int l=0; + for( int k=(settings::m()/2); k<(pptr_->nk()-1-settings::m()/2); k++ ) + { + for( int j=(settings::m()/2); j<(pptr_->nj()-1-settings::m()/2); j++ ) + { + for( int i=(settings::m()/2); i<(pptr_->ni()-1-settings::m()/2); i++ ) + { + cosphi_[0][l] = f->operator()(i, j, k) ; + l++; + } } - } - fft->compute(phi_,phi_,1); + } + } +void poisson::x_FFT( std::shared_ptr > f ) +{ + int l=0; + for( int k=(settings::m()/2)+1; k<(settings::m()/2)+2; k++ ) + { + for( int j=(settings::m()/2); j<(pptr_->nj()-1-settings::m()/2); j++ ) + { + for( int i=(settings::m()/2); i<(pptr_->ni()-1-settings::m()/2); i++ ) + { + psi_[0][l] = f->operator()(i, j, k) ; + l++; + } + } + } + fftx->forward(psi_->data(), psihat_->data()); + + std::ofstream data("fftxdat.dat"); + for(size_t ll=0; ll<((fftx->size_outbox())/nk_); ll++) { + data<forward(phi_->data(), phihat_->data()); + + //dealiasing +// scalar kk = (n_[0]*n_[1]*n_[2]) * 2.0/3.0; +// + int l=0; + for( int k=0; k(0.0, 0.0); } else - { - phi_[l] = -phi_[l] / k2_[l]; - } - l++; - - if( k2_[l] < tools::eps ) - { - phi_[l] = 0.0; - } - else - { - phi_[l] = -phi_[l] / k2_[l]; - } - l++; + { + phihat_[0][l] = std::complex(( -std::real(phihat_[0][l]) / k2_[l]) , -std::imag(phihat_[0][l]) / k2_[l]); + } + l++; } - } + } } - - fft->compute(phi_,phi_,-1); + fft->backward(phihat_->data(), phi_->data()); l=0; for( int k=(settings::m()/2); k<(pptr_->nk()-1-settings::m()/2); k++ ) @@ -171,32 +256,98 @@ void poisson::solve() { for( int i=(settings::m()/2); i<(pptr_->ni()-1-settings::m()/2); i++ ) { - pptr_->operator()(i,j,k) = phi_[l]; + pptr_->operator()(i,j,k) = phi_[0][l] / (n_[0]*n_[1]*n_[2]); l++; - l++; } } } +} + + +void poisson::solveDCT() +{ + dct->forward(cosphi_->data(), cosphihat_->data()); + + int l=0; + for( int k=0; kforward(cosphihat_->data(), cosphicomplex_->data()); + + l=0; + for( int k=0; k(0.0 , 0.0); + } + else + { + cosphicomplex_[0][l] =( std::complex(( -std::real(cosphicomplex_[0][l]) / k2c_[l]) , -std::imag(cosphicomplex_[0][l]) / k2c_[l] ) ); + } + l++; + } + } + } + + fft2d->backward(cosphicomplex_->data(), cosphihat_->data()); + + dct->backward(cosphihat_->data(), cosphi_->data(), dctworkspace_->data()); + + scalar pmax=0.0; + l=0; + for( int k=(settings::m()/2); k<(pptr_->nk()-1-settings::m()/2); k++ ) + { + for( int j=(settings::m()/2); j<(pptr_->nj()-1-settings::m()/2); j++ ) + { + for( int i=(settings::m()/2); i<(pptr_->ni()-1-settings::m()/2); i++ ) + { +// pmax = std::max( pmax, (cosphi_[0][l] / (2.0*n_[0]*n_[1]*(n_[2]-1) ))); + pptr_->operator()(i,j,k) = cosphi_[0][l] / (2*(n_[0])*(n_[1]-1)*n_[2]); + l++; + + } + } + } +// std::cout<<" "<size_inbox()<size_outbox()<size_outbox()<size_outbox()<size_outbox()<size_inbox()<mesh(); auto pi = tools::pi; - printf("Assigned local start coordinates (%d,%d,%d) on proc %d \n", - localStart_[0],localStart_[1],localStart_[2],parallelCom::myProcNo()); scalar kx, ky, kz; //TODO: will be complex for non-central schemes int l=0; - for( int k=0; kmesh(); + auto pi = tools::pi; + + + scalar kx, ky, kz; //TODO: will be complex for non-central schemes + int l=0; + kx=0; + ky=0; + kz=0; + for( int k=0; k tools::eps ) + { + for( int ll=0; ll<(settings::m()/2); ll++ ) + { + kpx+=kx*settings::coef()[ll]/(ll+1)*std::sin((ll+1)*kx*m.dx())/(kx*m.dx()); + } + } + if(ky > tools::eps ) + { + for( int ll=0; ll<(settings::m()/2); ll++ ) + { + kpy+=ky*settings::coef()[ll]/(ll+1)*std::sin((ll+1)*ky*m.dy())/(ky*m.dy()); + } + } + if(kz > tools::eps ) + { + for( int ll=0; ll<(settings::m()/2); ll++ ) + { + kpz+=kz*settings::coef()[ll]/(ll+1)*std::sin((ll+1)*kz*m.dz())/(kz*m.dz()); + } + } + + k2cy_[l] = (kpy/(n_[1]-1)) * (kpy/(n_[1]-1) ); + k2c_[l] = kpx*kpx + kpz*kpz; + l++; + } + } + } +} + +//Destructor no longer needed, can remove poisson::~poisson() { - free(phi_); + free(N); + fftw_free(phi_); + fftw_free(cosphi_); + fftw_free(phihat_); + } + diff --git a/src/poisson/poisson.h b/src/poisson/poisson.h index 08ef128..78ed9c5 100644 --- a/src/poisson/poisson.h +++ b/src/poisson/poisson.h @@ -21,19 +21,25 @@ #define POISSON_H #include -#include "fft3d.h" +#include "heffte.h" +#include + #include "Mesh/Mesh.h" #include "Field/Field.h" #include "Tools/Tools.h" #include #include +#include -using namespace FFTMPI_NS; +using namespace heffte; class poisson { protected: - std::unique_ptr fft; + std::unique_ptr> fft; + std::unique_ptr> fft2d; + std::unique_ptr> dct; + std::unique_ptr> fftx; std::shared_ptr > pptr_; int fftsize; unsigned ni_, nj_, nk_; @@ -51,27 +57,63 @@ class poisson int nfft_in; int nfft_out; int ilo,ihi,jlo,jhi,klo,khi; - int permute,sendsize,recvsize; + int permute,exchange,sendsize,recvsize; int flag,niter,tflag; double tmax; int memorysize; + + std::unique_ptr>> up_phihat_; + std::vector>* phihat_; + + std::unique_ptr> up_cosphi_; + std::vector* cosphi_; + + std::unique_ptr>> up_cosphicomplex_; + std::vector>* cosphicomplex_; + + std::unique_ptr> up_cosphihat_; + std::vector* cosphihat_; + + std::unique_ptr> up_dctworkspace_; + std::vector* dctworkspace_; + + std::unique_ptr> up_phi_; + std::vector* phi_; + + std::unique_ptr>> up_psihat_; + std::vector>* psihat_; + + std::unique_ptr> up_psi_; + std::vector* psi_; + + std::unique_ptr up_k2cy_; + double* k2cy_; - std::unique_ptr up_phi_; - FFT_SCALAR* phi_; std::unique_ptr up_k2_; double* k2_; + std::unique_ptr up_k2c_; + double* k2c_; + + heffte::box3d<>* N; + std::unique_ptr> up_N; + + heffte::box3d<>* NDCT; + std::unique_ptr> up_NDCT; public: + poisson( std::shared_ptr > ); ~poisson(); - void wavenumbers_(); - + void wavenumbersFFT_(); + void wavenumbersDCT_(); + void x_FFT( std::shared_ptr > f ); void rhs(std::shared_ptr > f ); + void rhsDCT(std::shared_ptr > f ); void solve(); + void solveDCT(); }; #endif - diff --git a/src/reuseTmp/reuseTmp.hxx b/src/reuseTmp/reuseTmp.hxx index be178e9..4b8b1aa 100644 --- a/src/reuseTmp/reuseTmp.hxx +++ b/src/reuseTmp/reuseTmp.hxx @@ -32,7 +32,8 @@ reuseTmp::reuseTmp for( int i=0; i<=20; i++ ) { assert( i!=20 ); - if( tmp_[i].unique() || tmp_[i] == NULL) +// if( tmp_[i].unique() || tmp_[i] == NULL) + if( tmp_[i].use_count() == 1 || tmp_[i] == NULL) { if( tmp_[i] == NULL ) {