An Example of an Uninstrumented FORTRAN Code Using MPI IO (Corresponding, instrumented FORTRAN code)

C************************************************************************
C* mpioExampleF.f:							*
C* This is an example of a FORTRAN code which uses MPIO.		*
C************************************************************************
      program main
      implicit none
      include 'mpif.h'
C*    *******************************************************************
C*    * # of times to repeat write/read cycle				*
C*    *******************************************************************
      integer NTIMES 
      parameter (NTIMES=5)
C*    *******************************************************************
C*    * read/write size in bytes					*
C*    *******************************************************************
      integer NBYTES 
      parameter (NBYTES=1024*1024*4)
      integer nprocs, myrank, i
      character*1024 str 
      integer buf(NBYTES/4)
      integer fh, status(MPI_STATUS_SIZE)
      integer ierr, argc, iargc
      integer*8 offset
      call MPI_INIT(ierr)
      call MPI_COMM_SIZE(MPI_COMM_WORLD, nprocs, ierr)
      call MPI_COMM_RANK(MPI_COMM_WORLD, myrank, ierr)
C*    *******************************************************************
C*    * process 0 takes the input file name as a command-line		*
C*    * argument and broadcasts it to other processes			*
C*    *******************************************************************
      if (myrank .eq. 0) then
         argc = iargc()
         i = 0
         call getarg(i,str)
         do while ((i .lt. argc) .and. (str .ne. '-fname'))
            i = i + 1
            call getarg(i,str)
         end do
         i = i + 1
         call getarg(i,str)
         call MPI_BCAST(str, 1024, MPI_CHARACTER, 0, MPI_COMM_WORLD, 
     1                  ierr)
         print*, 'Access size per process = ', NBYTES,'  bytes, ',
     1           ' NTIMES= ', NTIMES
      else
         call MPI_BCAST(str, 1024, MPI_CHARACTER, 0, MPI_COMM_WORLD, 
     1                  ierr)
      end if
      offset = myrank*NBYTES
      do i=1, NTIMES
C*        ***************************************************************
C*        * open the file, seek, write, and then close			*
C*        ***************************************************************
         call MPI_FILE_OPEN(MPI_COMM_WORLD, str, 
     1                     MPI_MODE_CREATE+MPI_MODE_RDWR, MPI_INFO_NULL, 
     2                     fh, ierr)
         call MPI_FILE_SEEK(fh, offset, MPI_SEEK_SET, ierr)
         call MPI_FILE_WRITE(fh, buf, NBYTES, MPI_BYTE, status, ierr)
         call MPI_FiLE_CLOSE(fh, ierr)
C*       ****************************************************************
C*       *  wait for all nodes to close file before proceeding		*
C*       ****************************************************************
         call MPI_BARRIER(MPI_COMM_WORLD, ierr)
  
C*       ****************************************************************
C*       *  open the file again, seek, read, and then close		*
C*       ****************************************************************
         call MPI_FILE_OPEN(MPI_COMM_WORLD, str, 
     1                     MPI_MODE_CREATE+MPI_MODE_RDWR, MPI_INFO_NULL, 
     2                     fh, ierr)
         call MPI_FILE_SEEK(fh, offset, MPI_SEEK_SET, ierr)
         call MPI_FILE_READ(fh, buf, NBYTES, MPI_BYTE, status, ierr)
         call MPI_FILE_CLOSE(fh, ierr)
C*       ****************************************************************
C*       *  wait for all nodes to close file before proceeding		*
C*       ****************************************************************
         call MPI_BARRIER(MPI_COMM_WORLD, ierr)
      end do
      call MPI_FINALIZE(ierr)
      stop
      end

Instrumented FORTRAN Code Using MPI IO (Corresponding, Uninstrumented FORTRAN Code)

C************************************************************************
C* mpioExampleF.f:							*
C* This is an example of a FORTRAN code which uses MPIO.		*
C************************************************************************
      program main
      implicit none
      include 'fMPIOTrace.h'
      include 'mpif.h'
C*    *******************************************************************
C*    * # of times to repeat write/read cycle				*
C*    *******************************************************************
      integer NTIMES 
      parameter (NTIMES=5)
C*    *******************************************************************
C*    * read/write size in bytes					*
C*    *******************************************************************
      integer NBYTES 
      parameter (NBYTES=1024*1024*4)
      integer nprocs, myrank, i
      character*1024 str 
      integer buf(NBYTES/4)
      integer fh, status(MPI_STATUS_SIZE)
      integer ierr, argc, iargc
      integer*8 offset
      call MPI_INIT(ierr)
      call MPI_COMM_SIZE(MPI_COMM_WORLD, nprocs, ierr)
      call MPI_COMM_RANK(MPI_COMM_WORLD, myrank, ierr)
C*    *******************************************************************
C*    * Pablo real-time tracing; trace output to samplef.run.nd? 	*
C*    *******************************************************************
      call initmpiotrace( 'fsample.real', 12, 1 )
C*    *******************************************************************
C*    * process 0 takes the input file name as a command-line		*
C*    * argument and broadcasts it to other processes			*
C*    *******************************************************************
      if (myrank .eq. 0) then
         argc = iargc()
         i = 0
         call getarg(i,str)
         do while ((i .lt. argc) .and. (str .ne. '-fname'))
            i = i + 1
            call getarg(i,str)
         end do
         i = i + 1
         call getarg(i,str)
         call MPI_BCAST(str, 1024, MPI_CHARACTER, 0, MPI_COMM_WORLD, 
     1                  ierr)
         print*, 'Access size per process = ', NBYTES,'  bytes, ',
     1           ' NTIMES= ', NTIMES
      else
         call MPI_BCAST(str, 1024, MPI_CHARACTER, 0, MPI_COMM_WORLD, 
     1                  ierr)
      end if
      offset = myrank*NBYTES
      do i=1, NTIMES
C*        ***************************************************************
C*        * open the file, seek, write, and then close			*
C*        ***************************************************************
         call MPI_FILE_OPEN(MPI_COMM_WORLD, str, 
     1                     MPI_MODE_CREATE+MPI_MODE_RDWR, MPI_INFO_NULL, 
     2                     fh, ierr)
         call MPI_FILE_SEEK(fh, offset, MPI_SEEK_SET, ierr)
         call MPI_FILE_WRITE(fh, buf, NBYTES, MPI_BYTE, status, ierr)
         call MPI_FiLE_CLOSE(fh, ierr)
C*       ****************************************************************
C*       *  wait for all nodes to close file before proceeding		*
C*       ****************************************************************
         call MPI_BARRIER(MPI_COMM_WORLD, ierr)
  
C*       ****************************************************************
C*       *  open the file again, seek, read, and then close		*
C*       ****************************************************************
         call MPI_FILE_OPEN(MPI_COMM_WORLD, str, 
     1                     MPI_MODE_CREATE+MPI_MODE_RDWR, MPI_INFO_NULL, 
     2                     fh, ierr)
         call MPI_FILE_SEEK(fh, offset, MPI_SEEK_SET, ierr)
         call MPI_FILE_READ(fh, buf, NBYTES, MPI_BYTE, status, ierr)
         call MPI_FILE_CLOSE(fh, ierr)
C*       ****************************************************************
C*       *  wait for all nodes to close file before proceeding		*
C*       ****************************************************************
         call MPI_BARRIER(MPI_COMM_WORLD, ierr)
      end do
C*    *******************************************************************
C*    *  call this routine to wrap up MPI I/O tracing			*
C*    *******************************************************************
      call endmpiotrace()
      call MPI_FINALIZE(ierr)
      stop
      end

Compiling, Linking, and Executing

Suppose the Pablo software is installed in the directory <PabloDir> and the MPI library is installed in the directory <mpiDir>.

Note: In order to obtain information about the UNIX I/O activity in the reprot, the I/O portion of the MPI library you are using must be instrumented.  Useful data about the MPI I/O calls can still be obtained even if an uninstrumented version of the MPI library is used. 

To compile the code, use the following command:

mpif77 -c mpioExampleF.f -I<PabloDir>/include -I<mpiDir>/include

To link the code, use the following command:

mpif77 -o mpioExampleF.o -L<PabloDir>/lib -lPabloTrace -lPabloTraceExt -L<mpiDir> -lmpi [other libraries as necessary]

To execute the code, use the following command:

mpirun -np <nprocs> myEXE -fname outFile

In this example, the application will access the file outFile.   The Pablo library will produce real-time trace output files named fsample.real.nd<p> where <p> is the processor number. 

Click on the name below to download either of the binary trace files produced for the case <nprocs> = 2. 

  • fsample.real.nd0
  • fsample.real.nd1
  • No post-processing of these files is required because realtime (summary) tracing was performed instead of runtime tracing.

    Merging the Trace Output Files

    Before doing any further processing, the trace files should be merged. This can be done by issuing the following commands:

    MergePabloTraces -o fsample.mrg fsamples.real.nd*

    This will produce a tracefile fsample.mrg.

    Trace Output Files

    Click to download the trace output file fsample.mrg  produced by following the steps above with nprocs equal to two (2).. The file is in binary. To convert it to ASCII, use the command SDDFconverter. This command will prompt users for input. The following is the session used to convert fsample.mrg from a binary format to the ASCII file fsample.mrg.ascii. User responses are in bold. Click to download the converted trace output file fsample.mrg.ascii.

    % SDDFconverter

    Please enter name of the input SDDF file: fsample.mrg

    File is in SDDF Non-Native Binary format

    Output in Ascii, Binary or Converted (reverse byte order) format [A, B, C]: A

    Please entery name of the output SDDF file: fsample.mrg.ascii

    Do you want diagnostic messages printed [Y or N}: N

    Processing Data: Sample Reports

    Issue the command MPIOstatsTable fsample.mrg to produce the report for all processors on the screen.