PETSc Update (#32)

.github/scripts/_typos.toml

@@ -0,0 +1,19 @@
+[default]
+extend-ignore-identifiers-re = [
+    "AttributeID.*Supress.*",
+]
+
+[default.extend-identifiers]
+AttributeIDSupressMenu = "AttributeIDSupressMenu"
+
+[default.extend-words]
+iy = "iy"
+ths = "ths"
+asend = "asend"
+alph = "alph"
+fo = "fo"
+thi = "thi"
+doub = "doub"
+Doub = "Doub"
+Numer = "Numer"
+thr = "thr"

.github/workflows/pretty.yml

@@ -2,9 +2,7 @@ name: Pretty
 
 on:
   push:
-
   pull_request:
-
   workflow_dispatch:
 
 jobs:

.github/workflows/spelling.yml

@@ -0,0 +1,18 @@
+name: Spell Check
+on:
+  push:
+  pull_request:
+  workflow_dispatch:
+
+jobs:
+  run:
+    name: Spell Check
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+
+    - name: Spell Check
+      uses: crate-ci/typos@master
+      with:
+        config: ${{github.workspace}}/.github/scripts/_typos.toml

.gitignore

@@ -1,6 +1,7 @@
 */*/v.*
 */*/D/*
 !*/*/D/.gittouch
+*/*/*/*.log
 *.o
 *.mod
 .depend
@@ -11,4 +12,5 @@ libcommon.a
 .vscode
 field
 packages
-traction
+traction
+

Makefile.in

@@ -1,10 +1,12 @@
-# Directories (change this to where you put RBC3D!)
-WORK_DIR = /storage/home/hcoda1/6/sbryngelson3/p-sbryngelson3-0/test-rbc/RBC3D
+# Get RBC3D root directory
+WORK_DIR := $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))))
 
 # Package directories (shouldn't need to change these)
-PETSC_DIR = $(WORK_DIR)/packages/mypetsc
-include $(PETSC_DIR)/conf/variables
+PETSC_DIR = $(WORK_DIR)/packages/petsc-3.19.6
+PETSC_ARCH = $(PETSC_DIR)/petsc_configure
+
 SPHEREPACK_DIR = $(WORK_DIR)/packages/spherepack3.2
+
 LAPACK_DIR = $(WORK_DIR)/packages/lapack-3.11
 
 # Makedependf90 binary
@@ -18,19 +20,21 @@ vpath $(WORK_DIR)/common
 
 # Includes
 PETSC_INCLUDE = $(PETSC_DIR)/include
+PETSC_ARCH_INCLUDE = $(PETSC_ARCH)/include
 NETCDF_INCLUDE = $(NETCDF_DIR)/include
 
 COMMON_INCLUDE = -I$(WORK_DIR)/common
-INCLUDE = $(COMMON_INCLUDE) -I$(PETSC_INCLUDE) -I$(NETCDF_INCLUDE)
+INCLUDE = $(COMMON_INCLUDE) -I$(PETSC_INCLUDE) -I$(PETSC_ARCH_INCLUDE) -I$(NETCDF_INCLUDE)
 
 # Libraries
 COMMON_LIB = $(WORK_DIR)/common/libcommon.a
 SPHPK_LIB = -L$(SPHEREPACK_DIR)/lib -lspherepack
 FFTW_LIB = -L$(FFTW_DIR)/lib -lfftw3
 NETCDF_LIB = -L$(NETCDF_DIR)/lib -lnetcdff
-PETSC_LIB = -L$(PETSC_DIR)/lib $(PETSC_KSP_LIB_BASIC)
-MKL_LIB = -L$(MKL_ROOT)lib/intel64/ -lmkl_gf_lp64 -lmkl_sequential -lmkl_core -lpthread -lm -ldl
-LAPACK_LIB = -L$(LAPACK_DIR) -llapack
+PETSC_LIB = -Wl,-rpath,$(PETSC_ARCH)/lib -L$(PETSC_ARCH)/lib -lpetsc -lstdc++
+MKL_LIB = -L$(MKL_ROOT)lib/intel64/ -lmkl_gf_lp64 -lmkl_core -lmkl_sequential -lpthread -lm -ldl
+BLAS_LIB = -L$(BLAS_DIR) -lrefblas
+LAPACK_LIB = -L$(LAPACK_DIR) -llapack -lrefblas
 
 # Compiler and linker
 FC = mpif90

README.md

@@ -23,8 +23,22 @@ This codebase solves the boundary integral form of the Stokes equations via an a
 
 ### Installation
 
-Building RBC3D can be a fragile process and we are working on improving it.
-In the meantime, a careful documentation of build instructions [is available here](https://github.com/comp-physics/RBC3D/blob/master/install/readme.md).
+To install on PACE Phoenix, you can salloc a node and then run: 
+
+```shell
+bash rbc.sh install-with-mkl
+```
+
+Then to execute and run a case, you can:
+```shell
+cd examples/case
+make .depend
+make
+srun -n 1 ./initcond
+srun ./tube
+```
+
+On other supercomputing clusters, it should be easy to replace line of 7 `./install/install-with-mkl.sh` with the modules available on your system and change the directories in `Makefile.in` to point to those modules. If one of these isn't available, you can follow the manual build instructions [available here](https://github.com/comp-physics/RBC3D/blob/master/install/readme.md).
 
 ### Papers that use RBC3D
 

common/ModBasicMath.F90

@@ -1,4 +1,4 @@
-! Collection of basic math operation
+! Collection of basic math operations
 module ModBasicMath
 
   use ModDataTypes
@@ -7,7 +7,7 @@ module ModBasicMath
 
   private
 
-  public :: VecNorm, &
+  public :: VecNormL2, &
             CrossProd, &
             InvMat2, &
             InvMat3, &
@@ -27,12 +27,12 @@ module ModBasicMath
 !**********************************************************************
 ! L2 norm of a vector
 ! |a|
-  function VecNorm(a) result(c)
+  function VecNormL2(a) result(c)
     real(WP) :: a(:), c
 
     c = sqrt(sum(a*a))
 
-  end function VecNorm
+  end function VecNormL2
 
 !**********************************************************************
 ! Cross product of two vectors
@@ -167,7 +167,9 @@ subroutine Matrix_PseudoInvert(A, B)
     AtA = matmul(B, A)
 
     ! Solve (AtA)X = B and store solution in B
-    call DPOSV('U', N, M, Ata, N, B, N, ierr)
+    call dposv('U', N, M, Ata, N, B, N, ierr)
+
+    if (ierr .ne. 0) write (*, *) "Matrix_PseudoInvert error code: ", ierr
 
     ! Deallocate working arrays
     deallocate (AtA)
@@ -212,7 +214,9 @@ subroutine QuadFit_1D(x, f, a0, a1, a2)
     lhs(3, 1) = lhs(1, 3)
     lhs(3, 2) = lhs(2, 3)
 
-    call DPOSV('U', 3, 1, lhs, 3, rhs, 3, ierr)
+    call dposv('U', 3, 1, lhs, 3, rhs, 3, ierr)
+
+    if (ierr .ne. 0) write (*, *) "QuadFit_1D error code: ", ierr
 
     a0 = rhs(1)
     a1 = rhs(2)
@@ -222,6 +226,7 @@ end subroutine QuadFit_1D
 
 !**********************************************************************
 ! 2D Quadratic fit
+! Uses least squares normal equation method
 ! Arguments:
 !  x(i,:), f(i) -- coordinates and functional value of the i-th point
 ! Note:
@@ -265,7 +270,9 @@ subroutine QuadFit_2D(x, f, a0, a1, a2, a11, a12, a22)
       end do ! jj
     end do ! ii
 
-    call DPOSV('U', 6, 1, lhs, 6, rhs, 6, ierr)
+    call dposv('U', 6, 1, lhs, 6, rhs, 6, ierr)
+
+    if (ierr .ne. 0) write (*, *) "QuadFit_2D error code: ", ierr
 
     a0 = rhs(1)
     a1 = rhs(2)
@@ -410,7 +417,7 @@ subroutine BsplineFunc(xc, P, imin, w)
   end subroutine BsplineFunc
 
 !**********************************************************************
-! Random number generator, copied from Numerical Recipies
+! Random number generator, copied from Numerical Recipes
   function RandomNumber(idum)
     integer, intent(IN), optional :: idum
     real(WP)        :: RandomNumber

common/ModConf.F90

@@ -5,6 +5,9 @@ module ModConf
   use ModDataTypes
   use ModDataStruct
 
+#include "petsc/finclude/petsc.h"
+  use petsc
+
   implicit none
 
   ! Domain size
@@ -108,7 +111,6 @@ module ModConf
   subroutine InitMPI(split_comm)
     logical, optional :: split_comm
 
-#include "../petsc_include.h"
     integer :: numNodes, nodeNum
     character(MPI_Max_Processor_Name) :: machinename
     integer :: lenname
@@ -207,8 +209,6 @@ end subroutine InitMPI
 !**********************************************************************
 ! Finalize MPI and PETSc
   subroutine FinalizeMPI
-
-#include "../petsc_include.h"
     integer :: ierr
 
     if (MPI_COMM_Ewald /= MPI_COMM_WORLD) then
@@ -437,7 +437,7 @@ subroutine DomainDecomp
   end subroutine DomainDecomp
 
 !**********************************************************************
-! Wheather a source point make contribution to the local domain
+! Whether a source point make contribution to the local domain
   function Is_Source(x)
     real(WP) :: x(3)
     logical :: Is_Source
@@ -493,7 +493,7 @@ function Cell_Has_Source(cell) result(hasSource)
   end function Cell_Has_Source
 
 !**********************************************************************
-! Wheather a triangle has active source points
+! Whether a triangle has active source points
 ! Arguments:
 !  x(i,:) -- the coordinates of the i-th vertex
   function Tri_Has_Source(x) result(hasSource)

common/ModDataStruct.F90

@@ -2,8 +2,10 @@ module ModDataStruct
 
   use ModDataTypes
 
+#include "petsc/finclude/petsc.h"
+  use petsc
+
   implicit none
-#include "../petsc_include.h"
 
   private
 
@@ -140,8 +142,8 @@ module ModDataStruct
 ! radius -- patch radius
 ! nrad, nazm -- number of patch points along radial and
 !       azimuthal directions
-! thL, phiL -- local coordiantes of patch mesh points
-! w -- weight of integration weight (including masking fucntion)
+! thL, phiL -- local coordinates of patch mesh points
+! w -- weight of integration weight (including masking function)
 ! thG(:,:,i,j), phiG(:,:,i,j) -- global latitudinal and longitudinal
 !       coordinates of a patch centered at (i,j)-th mesh point
   type t_RbcPolarPatch
@@ -172,7 +174,7 @@ module ModDataStruct
 !
 ! a3 -- element normal
 ! area -- element area
-! epsDist -- threshhold distance for singular integration
+! epsDist -- threshold distance for singular integration
 ! areaTot -- total surface area
 !
 ! lhs -- lhs matrix for no-slip boundary condition
@@ -198,7 +200,7 @@ module ModDataStruct
 !  nPoint  -- number of points
 !  x(i,:) -- coordinate of the i-th point
 !  f(i,:) -- force singularity at the i-th point
-!  g(i,:) -- double-layer potential source singularity at the i-th piont
+!  g(i,:) -- double-layer potential source singularity at the i-th point
 !  a3(i,:) -- normal direction
 !  lam(i) -- viscRatio
 !
@@ -207,7 +209,7 @@ module ModDataStruct
 !     for wall surface, (surfId, iele, -1)
 !
 !  Nc -- number of cells
-!  iLbNc -- Nc/Lb, for indentifying which cell a point lies in
+!  iLbNc -- Nc/Lb, for identifying which cell a point lies in
 !  hoc -- first point in the cell
 !  next -- next point
   type t_SourceList
@@ -227,7 +229,7 @@ module ModDataStruct
 !  nPoint -- number of points
 !  x -- coordinate and velocities of points
 !  indx -- same as those in t_SourceList
-!  active -- wheather the target point is active
+!  active -- Whether the target point is active
   type t_TargetList
     integer :: nPoint
     real(WP), pointer, dimension(:, :) :: x

common/ModHashTable.F90

@@ -60,7 +60,7 @@ end subroutine HashTable_Build
 !**********************************************************************
 ! Compute the Hash table index of a point
 ! Argument:
-!  Nc -- nunber of cell blocks
+!  Nc -- number of cell blocks
 !  iLbNc -- Nc/(local domain size)
 !  x -- point coordinate
 !  i1, i2, i3 -- Hash index of x(:)

common/ModIO.F90

@@ -210,9 +210,9 @@ subroutine WriteManyRBCs(fn, nrbc, rbcs)
       write (cell_unit, '(A,I9,A,I9,A)') 'ZONE I=', nlat + 1, '  J=', nlon + 1, '  F=POINT'
 
       do ilon = 1, nlon
-      do ilat = 0, nlat
-        write (cell_unit, '(3F20.10)') x(ilat, ilon, :)
-      end do ! ilat
+        do ilat = 0, nlat
+          write (cell_unit, '(3F20.10)') x(ilat, ilon, :)
+        end do ! ilat
       end do ! ilon
       do ilat = 0, nlat
         write (cell_unit, '(3F20.10)') x(ilat, 1, :)
@@ -230,7 +230,7 @@ end subroutine WriteManyRBCs
 ! Write the shape of blood cells of specified type to file
 ! Arguments:
 !  fn -- file suffix name
-!  nrbc -- nubmer of cells
+!  nrbc -- number of cells
 !  rbcs -- blood cells
 !  type -- type filter (1: rbc, 2: leukocyte, 3: sickle cell)
   subroutine WriteManyRBCsByType(fn, nrbc, rbcs, type)