Field3DParallel + more FCI changes

.git-blame-ignore-revs

@@ -3,3 +3,4 @@ d8f14fdddb5ca0fbb32d8e2bf5ac2960d6ac5ce6
 ed2117e6d6826a98b6988e2f18c0c34e408563b6
 # Added by the bot
 4b010b7634aee1045743be80c268d4644522cd29
+67d8ac88a9f1ebc33f3db76f39abea1e7db11d8a

.github/workflows/docker.yml

@@ -6,7 +6,7 @@ on:
       - master
       - next
       # Add your branch here if you want containers for it
-      - fix3121
+      - f3dwy
       - docker-ci
 
 env:
@@ -26,11 +26,11 @@ jobs:
       matrix:
         mpi: [mpich]
         metric3d:
-          - name: "With OpenMP"
+          - name: "With 3D Metrics"
             cmake: ON
-            base_prefix: "openmp-"
+            base_prefix: ""
             tag_prefix: "3d-"
-          - name: "Without OpenMP"
+          - name: "Without 3D Metrics"
             cmake: OFF
             base_prefix: ""
             tag_prefix: ""
@@ -75,7 +75,7 @@ jobs:
           build-args: |
             BASE=${{ matrix.mpi }}-${{ matrix.metric3d.base_prefix }}${{ matrix.config.base_postfix }}-main
             MPI=${{ matrix.mpi }}
-            CMAKE_OPTIONS=${{ matrix.config.options }} -DBOUT_ENABLE_METRIC_3D=${{ matrix.metric3d.cmake }} -DBOUT_ENABLE_OPENMP=${{ matrix.metric3d.cmake }}
+            CMAKE_OPTIONS=${{ matrix.config.options }} -DBOUT_ENABLE_METRIC_3D=${{ matrix.metric3d.cmake }}
             COMMIT=${{ github.sha }}
             URL=${{ github.server_url }}/${{ github.repository }}
           context: .docker/fedora/

.github/workflows/tests.yml

@@ -179,6 +179,7 @@ jobs:
       - name: Install pip packages
         run: |
           python -m pip install --upgrade pip setuptools
+          python -m pip install git+https://github.com/boutproject/zoidberg@better-metric
           python -m pip install -r requirements.txt
 
       - name: Cache SUNDIALS build

CMakeLists.txt

@@ -271,11 +271,12 @@ set(BOUT_SOURCES
   ./src/mesh/interpolation/hermite_spline_z.cxx
   ./src/mesh/interpolation/interpolation_z.cxx
   ./src/mesh/interpolation/lagrange_4pt_xz.cxx
-  ./src/mesh/interpolation/monotonic_hermite_spline_xz.cxx
   ./src/mesh/invert3x3.hxx
   ./src/mesh/mesh.cxx
   ./src/mesh/parallel/fci.cxx
   ./src/mesh/parallel/fci.hxx
+  ./src/mesh/parallel/fci_comm.cxx
+  ./src/mesh/parallel/fci_comm.hxx
   ./src/mesh/parallel/identity.cxx
   ./src/mesh/parallel/shiftedmetric.cxx
   ./src/mesh/parallel/shiftedmetricinterp.cxx
@@ -376,6 +377,7 @@ if (zoidberg_FOUND EQUAL 0)
 else()
   set(zoidberg_FOUND OFF)
 endif()
+message(STATUS "Found Zoidberg for FCI tests: ${zoidberg_FOUND}")
 
 option(BOUT_GENERATE_FIELDOPS "Automatically re-generate the Field arithmetic operators from the Python templates. \
 Requires Python3, clang-format, and Jinja2. Turn this OFF to skip generating them if, for example, \
@@ -582,6 +584,9 @@ else()
 endif()
 set(BOUT_USE_METRIC_3D ${BOUT_ENABLE_METRIC_3D})
 
+option(BOUT_ENABLE_FCI_AUTOMAGIC "Enable (slow?) automatic features for FCI" ON)
+set(BOUT_USE_FCI_AUTOMAGIC ${BOUT_ENABLE_FCI_AUTOMAGIC})
+
 include(CheckCXXSourceCompiles)
 check_cxx_source_compiles("int main() { const char* name = __PRETTY_FUNCTION__; }"
   HAS_PRETTY_FUNCTION)

bin/bout-build-deps.sh

@@ -10,7 +10,7 @@ NCVER=${NCVER:-4.7.4}
 NCCXXVER=${NCCXXVER:-4.3.1}
 FFTWVER=${FFTWVER:-3.3.9}
 SUNVER=${SUNVER:-5.7.0}
-PETSCVER=${PETSCVER:-3.15.0}
+PETSCVER=${PETSCVER:-3.21.4}
 
 
 HDF5FLAGS=${HDF5FLAGS:-}
@@ -147,7 +147,7 @@ petsc() {
     test -z $PETSC_DIR || error "\$PETSC_DIR is set ($PETSC_DIR) - please unset"
     test -z $PETSC_ARCH || error "\$PETSC_ARCH is set ($PETSC_ARCH) - please unset"
     cd $BUILD
-    wget -c https://ftp.mcs.anl.gov/pub/petsc/release-snapshots/petsc-$PETSCVER.tar.gz || :
+    wget -c https://web.cels.anl.gov/projects/petsc/download/release-snapshots/petsc-$PETSCVER.tar.gz || :
     tar -xf petsc-$PETSCVER.tar.gz
     cd petsc-$PETSCVER
     unset PETSC_DIR

bout++Config.cmake.in

@@ -8,7 +8,7 @@ set(BOUT_USE_TRACK @BOUT_USE_TRACK@)
 set(BOUT_USE_SIGFPE @BOUT_USE_SIGFPE@)
 set(BOUT_USE_OPENMP @BOUT_USE_OPENMP@)
 set(BOUT_HAS_CUDA @BOUT_HAS_CUDA@)
-set(BOUT_HAS_OUTPUT_DEBUG @BOUT_HAS_OUTPUT_DEBUG@)
+set(BOUT_USE_OUTPUT_DEBUG @BOUT_USE_OUTPUT_DEBUG@)
 set(BOUT_CHECK_LEVEL @BOUT_CHECK_LEVEL@)
 set(BOUT_USE_METRIC_3D @BOUT_USE_METRIC_3D@)
 
@@ -182,3 +182,30 @@ if (BOUT_HAS_ADIOS2)
 endif()
 
 include("${CMAKE_CURRENT_LIST_DIR}/bout++Targets.cmake")
+
+add_library(bout++ INTERFACE IMPORTED)
+set_target_properties(bout++
+  PROPERTIES BOUT_USE_SIGNAL @BOUT_USE_SIGNAL@
+  BOUT_USE_COLOR @BOUT_USE_COLOR@
+  BOUT_USE_TRACK @BOUT_USE_TRACK@
+  BOUT_USE_SIGFPE @BOUT_USE_SIGFPE@
+  BOUT_USE_OPENMP @BOUT_USE_OPENMP@
+  BOUT_HAS_CUDA @BOUT_HAS_CUDA@
+  BOUT_USE_OUTPUT_DEBUG @BOUT_USE_OUTPUT_DEBUG@
+  BOUT_CHECK_LEVEL @BOUT_CHECK_LEVEL@
+  BOUT_USE_METRIC_3D @BOUT_USE_METRIC_3D@
+  BOUT_HAS_PVODE @BOUT_HAS_PVODE@
+  BOUT_HAS_NETCDF @BOUT_HAS_NETCDF@
+  BOUT_HAS_ADIOS2 @BOUT_HAS_ADIOS2@
+  BOUT_HAS_FFTW @BOUT_HAS_FFTW@
+  BOUT_HAS_LAPACK @BOUT_HAS_LAPACK@
+  BOUT_HAS_PETSC @BOUT_HAS_PETSC@
+  BOUT_HAS_SLEPC @BOUT_HAS_SLEPC@
+  BOUT_HAS_SCOREP @BOUT_HAS_SCOREP@
+  BOUT_USE_UUID_SYSTEM_GENERATOR @BOUT_USE_UUID_SYSTEM_GENERATOR@
+  BOUT_HAS_SUNDIALS @BOUT_HAS_SUNDIALS@
+  BOUT_HAS_HYPRE @BOUT_HAS_HYPRE@
+  BOUT_HAS_GETTEXT @BOUT_HAS_GETTEXT@
+  BOUT_HAS_UMPIRE @BOUT_HAS_UMPIRE@
+  BOUT_HAS_RAJA @BOUT_HAS_RAJA@
+)

cmake/SetupBOUTThirdParty.cmake

@@ -291,7 +291,7 @@ if (BOUT_USE_SUNDIALS)
     set(EXAMPLES_ENABLE_C OFF CACHE BOOL "" FORCE)
     set(EXAMPLES_INSTALL OFF CACHE BOOL "" FORCE)
     set(ENABLE_MPI ${BOUT_USE_MPI} CACHE BOOL "" FORCE)
-    set(ENABLE_OPENMP OFF CACHE BOOL "" FORCE)
+    set(ENABLE_OPENMP ${BOUT_USE_OPENMP} CACHE BOOL "" FORCE)
     if (BUILD_SHARED_LIBS)
       set(BUILD_STATIC_LIBS OFF CACHE BOOL "" FORCE)
     else()

cmake_build_defines.hxx.in

@@ -34,6 +34,7 @@
 #cmakedefine BOUT_METRIC_TYPE @BOUT_METRIC_TYPE@
 #cmakedefine01 BOUT_USE_METRIC_3D
 #cmakedefine01 BOUT_USE_MSGSTACK
+#cmakedefine01 BOUT_USE_FCI_AUTOMAGIC
 
 // CMake build does not support legacy interface
 #define BOUT_HAS_LEGACY_NETCDF 0

examples/fci-wave/div-integrate/BOUT.inp

@@ -38,7 +38,7 @@ bndry_par_ydown = parallel_neumann_o2
 
 [v]
 
-bndry_par_all = parallel_dirichlet
+bndry_par_all = parallel_dirichlet_o2
 
 [solver]
 output_step = 0.1

examples/fci-wave/div/BOUT.inp

@@ -38,7 +38,7 @@ bndry_par_ydown = parallel_neumann_o2
 
 [v]
 
-bndry_par_all = parallel_dirichlet
+bndry_par_all = parallel_dirichlet_o2
 
 [solver]
 output_step = 0.1

examples/fci-wave/logn/BOUT.inp

@@ -38,7 +38,7 @@ bndry_par_ydown = parallel_neumann_o2
 
 [nv]
 
-bndry_par_all = parallel_dirichlet
+bndry_par_all = parallel_dirichlet_o2
 
 [solver]
 output_step = 0.1

externalpackages/PVODE/include/pvode/band.h

@@ -138,7 +138,7 @@ typedef struct bandmat_type {
  *                                                                *
  ******************************************************************/
 
-#define PVODE_BAND_ELEM(A,i,j) ((A->data)[j][i-j+(A->smu)])
+#define PVODE_BAND_ELEM(A, i, j) (((A)->data)[j][(i) - (j) + ((A)->smu)])
 
 /******************************************************************
  *                                                                *
@@ -153,7 +153,7 @@ typedef struct bandmat_type {
  *                                                                *
  ******************************************************************/
 
-#define PVODE_BAND_COL(A,j) (((A->data)[j])+(A->smu))
+#define PVODE_BAND_COL(A, j) ((((A)->data)[j]) + ((A)->smu))
 
 /******************************************************************
  *                                                                *

externalpackages/PVODE/precon/band.h

@@ -153,7 +153,7 @@ typedef struct bandmat_type {
  *                                                                *
  ******************************************************************/
 
-#define BAND_COL(A, j) (((A->data)[j]) + (A->smu))
+#define PVODE_BAND_COL(A, j) ((((A)->data)[j]) + ((A)->smu))
 
 /******************************************************************
  *                                                                *