/*
This file is part of the Snoopy code.
Snoopy code is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
Snoopy code is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with Snoopy code. If not, see .
*/
#include
#include "common.h"
#include "timestep.h"
#include "output/output.h"
#include "output/output_dump.h"
#include "interface.h"
#include "gfft.h"
#include "shear.h"
#include "transpose.h"
#include "symmetries.h"
#include "initflow.h"
#ifdef BOUNDARY_C
#include "boundary.h"
#endif
#include "debug.h"
const double gammaRK[3] = {8.0 / 15.0 , 5.0 / 12.0 , 3.0 / 4.0};
const double xiRK[2] = {-17.0 / 60.0 , -5.0 / 12.0};
double forcing_last_time;
/***************************************************************/
/**
generate a timestep (dt) as a function of the current flow configuration/velocity
This routine is essentially an application of the CFL condition. it returns
a timestep (dt)
@param tremap: when using shear, the current remap time of the frame
@param fldi: Field structure containing the flow status
*/
/***************************************************************/
double newdt(struct Field fldi, double tremap) {
int i;
double gamma_v;
double maxfx , maxfy, maxfz;
#ifdef MHD
double gamma_b;
double maxbx , maxby, maxbz;
#endif
#ifdef COMPRESSIBLE
double q0, dmin;
#endif
double dt;
DEBUG_START_FUNC;
#ifdef _OPENMP
#pragma omp parallel for private(i) schedule(static)
#endif
for( i = 0 ; i < NTOTAL_COMPLEX ; i++) {
w1[i] = fldi.vx[i];
w2[i] = fldi.vy[i];
w3[i] = fldi.vz[i];
}
gfft_c2r_t(w1);
gfft_c2r_t(w2);
gfft_c2r_t(w3);
#ifdef COMPRESSIBLE
// When compressible is active, we are vj is the linear momentum
// Wave speeds are however computed as velocity, we therefore
// need a conversion
for( i = 0 ; i < NTOTAL_COMPLEX ; i++) {
w4[i] = fldi.d[i];
}
gfft_c2r_t(w4);
// Compute the minimum density (used to determine the viscous CFL condition)
dmin=wr4[0];
// Exclude 0.0 (due to dump zone of in place ffts)
for( i = 0 ; i < 2*NTOTAL_COMPLEX ; i++) {
if( (wr4[i] < dmin) && (wr4[i] != 0.0)) dmin = wr4[i];
}
dmin = dmin / ((double) NTOTAL);
reduce(&dmin, 3);
if(dmin <= 0.0) ERROR_HANDLER(ERROR_CRITICAL, "Negative density detected: panic");
// Compute the velocity field (used in advection CFL condition
check_positivity(wr4);
for( i = 0 ; i < NTOTAL_COMPLEX*2 ; i++) {
q0=((double) NTOTAL) / wr4[i];
wr1[i] = wr1[i] * q0;
wr2[i] = wr2[i] * q0;
wr3[i] = wr3[i] * q0;
}
#endif
maxfx=0.0;
maxfy=0.0;
maxfz=0.0;
for( i = 0 ; i < NTOTAL_COMPLEX * 2 ; i++) {
if( fabs( wr1[i] ) > maxfx ) maxfx = fabs( wr1[i] );
if( fabs( wr2[i] ) > maxfy ) maxfy = fabs( wr2[i] );
if( fabs( wr3[i] ) > maxfz ) maxfz = fabs( wr3[i] );
}
maxfx = maxfx / ((double) NTOTAL);
maxfy = maxfy / ((double) NTOTAL);
maxfz = maxfz / ((double) NTOTAL);
#ifdef MPI_SUPPORT
reduce(&maxfx,2);
reduce(&maxfy,2);
reduce(&maxfz,2);
#endif
#ifdef COMPRESSIBLE
maxfx=maxfx+param.cs;
maxfy=maxfy+param.cs;
maxfz=maxfz+param.cs;
#endif
gamma_v = (kxmax + fabs(tremap)*kymax) * maxfx + kymax * maxfy + kzmax * maxfz;
#ifdef WITH_ROTATION
gamma_v += fabs(param.omega) / param.safety_source;
#endif
#ifdef WITH_SHEAR
gamma_v += fabs(param.shear) / param.safety_source;
#endif
#ifdef BOUSSINESQ
gamma_v += pow(fabs(param.N2), 0.5) / param.safety_source;
#ifdef WITH_EXPLICIT_DISSIPATION
gamma_v += ((kxmax+fabs(tremap)*kymax)*(kxmax+fabs(tremap)*kymax)+kymax*kymax+kzmax*kzmax) * nu_th; // NB: this is very conservative. It should be combined with the condition on nu
#endif
#endif
#ifdef TIME_DEPENDANT_SHEAR
gamma_v += fabs(param.omega_shear) / param.safety_source;
#endif
#ifdef COMPRESSIBLE
gamma_v += ((kxmax+fabs(tremap)*kymax)*(kxmax+fabs(tremap)*kymax)+kymax*kymax+kzmax*kzmax) * nu / dmin; // CFL condition on viscosity
#endif
#ifndef COMPRESSIBLE
#ifdef WITH_EXPLICIT_DISSIPATION
gamma_v += ((kxmax+fabs(tremap)*kymax)*(kxmax+fabs(tremap)*kymax)+kymax*kymax+kzmax*kzmax) * nu; // CFL condition on viscosity in incompressible regime
#endif
#endif
#ifdef WITH_PARTICLES
gamma_v += 1.0 / (fabs(param.particles_stime) * param.safety_source );
maxfx=0.0;
maxfy=0.0;
maxfz=0.0;
for(i = 0 ; i < param.particles_n/NPROC ; i++) {
if( fabs( fld.part[i].vx ) > maxfx ) maxfx = fabs( fld.part[i].vx );
if( fabs( fld.part[i].vy ) > maxfy ) maxfy = fabs( fld.part[i].vy );
if( fabs( fld.part[i].vz ) > maxfz ) maxfz = fabs( fld.part[i].vz );
}
#ifdef MPI_SUPPORT
reduce(&maxfx,2);
reduce(&maxfy,2);
reduce(&maxfz,2);
#endif
gamma_v += param.lx/(NX)*maxfx+param.ly/(NY)*maxfy+param.lz/(NZ)*maxfz;
#endif
#ifdef MHD
#ifdef WITH_BRAGINSKII
gamma_v += ((kxmax+fabs(tremap)*kymax)*(kxmax+fabs(tremap)*kymax)+kymax*kymax+kzmax*kzmax)/param.reynolds_B;
#endif
/* Compute the magnetic CFL condition */
#ifdef _OPENMP
#pragma omp parallel for private(i) schedule(static)
#endif
for( i = 0 ; i < NTOTAL_COMPLEX ; i++) {
w1[i] = fldi.bx[i];
w2[i] = fldi.by[i];
w3[i] = fldi.bz[i];
}
gfft_c2r_t(w1);
gfft_c2r_t(w2);
gfft_c2r_t(w3);
#ifdef COMPRESSIBLE
// When compressible is active, the alfven speed depends on the density
// We considers V_a=B/sqrt(rho)
for( i = 0 ; i < NTOTAL_COMPLEX*2 ; i++) {
q0=pow(((double) NTOTAL) / wr4[i],0.5);
wr1[i] = wr1[i] * q0;
wr2[i] = wr2[i] * q0;
wr3[i] = wr3[i] * q0;
}
#endif
maxbx=0.0;
maxby=0.0;
maxbz=0.0;
for( i = 0 ; i < NTOTAL_COMPLEX * 2 ; i++) {
if( fabs( wr1[i] ) > maxbx ) maxbx = fabs( wr1[i] );
if( fabs( wr2[i] ) > maxby ) maxby = fabs( wr2[i] );
if( fabs( wr3[i] ) > maxbz ) maxbz = fabs( wr3[i] );
}
maxbx = maxbx / ((double) NTOTAL);
maxby = maxby / ((double) NTOTAL);
maxbz = maxbz / ((double) NTOTAL);
#ifdef MPI_SUPPORT
reduce(&maxbx,2);
reduce(&maxby,2);
reduce(&maxbz,2);
#endif
#ifdef COMPRESSIBLE
// we need the phase speed of the fast magnetosonic wave, not of the torsional alfven wave
maxbx = pow( maxbx * maxbx + param.cs*param.cs , 0.5);
maxby = pow( maxby * maxby + param.cs*param.cs , 0.5);
maxbz = pow( maxbz * maxbz + param.cs*param.cs , 0.5);
#endif
gamma_b = (kxmax + fabs(tremap)*kymax) * maxbx + kymax * maxby + kzmax * maxbz;
#ifdef WITH_HALL
gamma_b += ((kxmax+fabs(tremap)*kymax)*(kxmax+fabs(tremap)*kymax)+kymax*kymax+kzmax*kzmax) * pow(maxbx*maxbx + maxby*maxby + maxbz*maxbz, 0.5) / param.x_hall;
#endif
#ifdef WITH_EXPLICIT_DISSIPATION
gamma_b += ((kxmax+fabs(tremap)*kymax)*(kxmax+fabs(tremap)*kymax)+kymax*kymax+kzmax*kzmax) * eta; // CFL condition on resistivity
#endif
dt = param.cfl / (gamma_v + gamma_b);
#else
dt = param.cfl / gamma_v;
#endif
#ifdef DEBUG
#ifdef MHD
MPI_Printf("newdt: maxbx=%e, maxby=%e, maxbz=%e\n",maxbx,maxby, maxbz);
#endif
MPI_Printf("newdt: maxfx=%e, maxfy=%e, maxfz=%e, dt=%e\n",maxfx,maxfy, maxfz, dt);
#endif
CHECK_NAN(dt);
DEBUG_END_FUNC;
return(dt);
}
/***************************************************************/
/**
Integrate in time the physical system from t_start to t_end.
Outputs are done according to gvars.h
@param t_start: initial time of the simulation (usually 0...)
@param t_end: final time of the simulation (will stop precisely at that time).
*/
/***************************************************************/
void mainloop(double t_start, double t_end) {
struct Field fld, dfld, fld1;
double dt = 0.0;
double t = 0.0;
double tremap = 0.0;
double timer_end, timer_start;
int i,n,nloop;
DEBUG_START_FUNC;
// We first init mainloop structures
allocate_field(&fld);
allocate_field(&dfld);
allocate_field(&fld1);
// Init the flow structure (aka initial conditions)
init_flow(fld);
nloop=0;
// Read restart file if needed
if(param.restart) {
#ifdef DEBUG
MPI_Printf("Reading dump file\n");
#endif
read_dump(fld,&t,OUTPUT_DUMP);
}
else {
t = t_start;
// Go for an output
output(fld,t);
}
// Init shear parameters
#ifdef WITH_SHEAR
tremap = time_shift(t);
kvolve(tremap);
#else
tremap = 0.0;
#endif
timer_start = get_c_time();
while (t < t_end) {
#ifdef DEBUG
MPI_Printf("Begining of loop:\n");
MPI_Printf("fld:\n");
D_show_all(fld);
MPI_Printf("**************************************************************************************\n");
#endif
nloop++;
if(!(nloop % param.interface_check)) check_interface(fld,t,dt,nloop,timer_start);
dt = newdt(fld, tremap);
// Let's try to stop exactly at t_final
if(dt > (t_end - t)) dt = t_end - t;
// Stop if elpased time is larger than MAX_ELAPSED_TIME (in hours)
if((get_c_time()-timer_start) > 3600 * param.max_t_elapsed) {
MPI_Printf("Maximum elapsed time reached. Terminating.\n");
dump_immediate(fld,t);
break;
}
// This is an order 3 runge Kutta scheme with low storage
// 1st RK3 step
timestep(dfld, fld, t, tremap, dt );
#ifdef _OPENMP
#pragma omp parallel private(i,n)
{
#endif
for( n = 0 ; n < fld.nfield ; n++) {
#ifdef _OPENMP
#pragma omp for schedule(static)
#endif
for( i = 0 ; i < NTOTAL_COMPLEX ; i++) {
fld.farray[n][i] = fld.farray[n][i] + gammaRK[0] * dfld.farray[n][i] * dt;
fld1.farray[n][i] = fld.farray[n][i] + xiRK[0] * dfld.farray[n][i] * dt;
}
}
#ifdef WITH_PARTICLES
#ifdef _OPENMP
#pragma omp for schedule(static)
#endif
for( i = 0 ; i < param.particles_n ; i++) {
fld.part[i].x = fld.part[i].x + gammaRK[0] * dfld.part[i].x * dt;
fld.part[i].y = fld.part[i].y + gammaRK[0] * dfld.part[i].y * dt;
fld.part[i].z = fld.part[i].z + gammaRK[0] * dfld.part[i].z * dt;
fld.part[i].vx = fld.part[i].vx + gammaRK[0] * dfld.part[i].vx * dt;
fld.part[i].vy = fld.part[i].vy + gammaRK[0] * dfld.part[i].vy * dt;
fld.part[i].vz = fld.part[i].vz + gammaRK[0] * dfld.part[i].vz * dt;
fld1.part[i].x = fld.part[i].x + xiRK[0] * dfld.part[i].x * dt;
fld1.part[i].y = fld.part[i].y + xiRK[0] * dfld.part[i].y * dt;
fld1.part[i].z = fld.part[i].z + xiRK[0] * dfld.part[i].z * dt;
fld1.part[i].vx = fld.part[i].vx + xiRK[0] * dfld.part[i].vx * dt;
fld1.part[i].vy = fld.part[i].vy + xiRK[0] * dfld.part[i].vy * dt;
fld1.part[i].vz = fld.part[i].vz + xiRK[0] * dfld.part[i].vz * dt;
}
#endif
#ifdef _OPENMP
}
#endif
#ifdef DEBUG
MPI_Printf("RK, 1st Step:\n");
MPI_Printf("fld:\n");
D_show_all(fld);
MPI_Printf("fld1:\n");
D_show_all(fld1);
MPI_Printf("dfld:\n");
D_show_all(dfld);
MPI_Printf("**************************************************************************************\n");
#endif
// 2nd RK3 step
#ifdef WITH_SHEAR
#ifdef TIME_DEPENDANT_SHEAR
kvolve(time_shift(t+gammaRK[0]*dt));
#else
kvolve(tremap+gammaRK[0]*dt);
#endif
#endif
timestep(dfld, fld, t+gammaRK[0]*dt, tremap+gammaRK[0]*dt, dt);
#ifdef _OPENMP
#pragma omp parallel private(i,n)
{
#endif
for( n = 0 ; n < fld.nfield ; n++) {
#ifdef _OPENMP
#pragma omp for schedule(static)
#endif
for( i = 0 ; i < NTOTAL_COMPLEX ; i++) {
fld.farray[n][i] = fld1.farray[n][i] + gammaRK[1] * dfld.farray[n][i] * dt;
fld1.farray[n][i] = fld.farray[n][i] + xiRK[1] * dfld.farray[n][i] * dt;
}
}
#ifdef WITH_PARTICLES
#ifdef _OPENMP
#pragma omp for schedule(static)
#endif
for( i = 0 ; i < param.particles_n ; i++) {
fld.part[i].x = fld1.part[i].x + gammaRK[1] * dfld.part[i].x * dt;
fld.part[i].y = fld1.part[i].y + gammaRK[1] * dfld.part[i].y * dt;
fld.part[i].z = fld1.part[i].z + gammaRK[1] * dfld.part[i].z * dt;
fld.part[i].vx = fld1.part[i].vx + gammaRK[1] * dfld.part[i].vx * dt;
fld.part[i].vy = fld1.part[i].vy + gammaRK[1] * dfld.part[i].vy * dt;
fld.part[i].vz = fld1.part[i].vz + gammaRK[1] * dfld.part[i].vz * dt;
fld1.part[i].x = fld.part[i].x + xiRK[1] * dfld.part[i].x * dt;
fld1.part[i].y = fld.part[i].y + xiRK[1] * dfld.part[i].y * dt;
fld1.part[i].z = fld.part[i].z + xiRK[1] * dfld.part[i].z * dt;
fld1.part[i].vx = fld.part[i].vx + xiRK[1] * dfld.part[i].vx * dt;
fld1.part[i].vy = fld.part[i].vy + xiRK[1] * dfld.part[i].vy * dt;
fld1.part[i].vz = fld.part[i].vz + xiRK[1] * dfld.part[i].vz * dt;
}
#endif
#ifdef _OPENMP
}
#endif
#ifdef DEBUG
MPI_Printf("RK, 2nd Step:\n");
MPI_Printf("fld:\n");
D_show_all(fld);
MPI_Printf("fld1:\n");
D_show_all(fld1);
MPI_Printf("dfld:\n");
D_show_all(dfld);
MPI_Printf("**************************************************************************************\n");
#endif
// 3rd RK3 Step
#ifdef WITH_SHEAR
#ifdef TIME_DEPENDANT_SHEAR
kvolve(time_shift(t + (gammaRK[0] + xiRK[0] + gammaRK[1]) * dt));
#else
kvolve(tremap + (gammaRK[0] + xiRK[0] + gammaRK[1]) * dt );
#endif
#endif
timestep(dfld, fld, t + (gammaRK[0] + xiRK[0] + gammaRK[1]) * dt, tremap + (gammaRK[0] + xiRK[0] + gammaRK[1]) * dt, dt);
#ifdef _OPENMP
#pragma omp parallel private(i,n)
{
#endif
for( n = 0 ; n < fld.nfield ; n++) {
#ifdef _OPENMP
#pragma omp for schedule(static)
#endif
for( i = 0 ; i < NTOTAL_COMPLEX ; i++) {
fld.farray[n][i] = fld1.farray[n][i] + gammaRK[2] * dfld.farray[n][i] * dt;
}
}
#ifdef WITH_PARTICLES
#ifdef _OPENMP
#pragma omp for schedule(static)
#endif
for( i = 0 ; i < param.particles_n ; i++) {
fld.part[i].x = fld1.part[i].x + gammaRK[2] * dfld.part[i].x * dt;
fld.part[i].y = fld1.part[i].y + gammaRK[2] * dfld.part[i].y * dt;
fld.part[i].z = fld1.part[i].z + gammaRK[2] * dfld.part[i].z * dt;
fld.part[i].vx = fld1.part[i].vx + gammaRK[2] * dfld.part[i].vx * dt;
fld.part[i].vy = fld1.part[i].vy + gammaRK[2] * dfld.part[i].vy * dt;
fld.part[i].vz = fld1.part[i].vz + gammaRK[2] * dfld.part[i].vz * dt;
}
#endif
#ifdef _OPENMP
}
#endif
#ifdef DEBUG
MPI_Printf("RK, 3rd Step:\n");
MPI_Printf("fld:\n");
D_show_all(fld);
MPI_Printf("fld1:\n");
D_show_all(fld1);
MPI_Printf("dfld:\n");
D_show_all(dfld);
MPI_Printf("**************************************************************************************\n");
#endif
// Runge Kutta finished
// Implicit step
implicitstep(fld, t, dt);
// evolving the frame
t = t + dt;
#ifdef WITH_SHEAR
#ifdef TIME_DEPENDANT_SHEAR
tremap = time_shift(t);
#else
tremap = tremap + dt;
// Check if a remap is needed
if(tremap > param.ly / (2.0 * param.shear * param.lx)) {
tremap = time_shift(t); // Recompute tremap from current time, assuming all the remaps have been done
for( n = 0 ; n < fld.nfield ; n++) {
remap(fld.farray[n]);
}
}
#endif
kvolve(tremap);
#endif
// Symmetries cleaning
if(param.force_symmetries) {
if(!(nloop % param.symmetries_step)) enforce_complex_symm(fld);
}
// Divergence cleaning
#ifndef COMPRESSIBLE
projector(fld.vx,fld.vy,fld.vz);
#endif
#ifdef MHD
projector(fld.bx,fld.by,fld.bz);
#endif
// The boundary conditions arises naturally from the initial conditions (the relevant symmetries are conserved by the eq. of motion)
// We keep this instruction here to enforce these boundary conditions at the end of each loop to remove numerical noise.
// Nevertheless, it is not required to call it so often...
#ifdef BOUNDARY_C
boundary_c(fld);
#endif
output(fld,t);
}
timer_end=get_c_time();
MPI_Printf("mainloop finished in %d loops and %f seconds (%f sec/loop)\n",nloop,timer_end-timer_start,(timer_end-timer_start)/nloop);
MPI_Printf("fft time=%f s (%f pc)\n",read_fft_timer(), read_fft_timer()/(timer_end-timer_start)*100.0);
MPI_Printf("I/O time=%f s (%f pc)\n",read_output_timer(), read_output_timer()/(timer_end-timer_start)*100.0);
#ifdef MPI_SUPPORT
#ifndef FFTW3_MPI_SUPPORT
MPI_Printf("Time used for transpose: %f seconds, or %f pc of total computation time\n",read_transpose_timer(), read_transpose_timer()/(timer_end-timer_start)*100.0);
#endif
#endif
// Close everything
deallocate_field(&fld);
deallocate_field(&fld1);
deallocate_field(&dfld);
return;
}