An Example of an Uninstrumented C Code Using MPI IO (Corresponding, instrumented C code)

#include "mpi.h"
#include "mpio.h"              /* not necessary with MPICH1.1.1 or HPMPI1.4 */
#include <stdio.h>
#include <string.h>
#include <stdlib.h>
/*======================================================================*/
/* mpioExample.c:							*/
/* This is an example of a C code using MPI I/O calls.                  */
/* It is a simple code using MPI I/O calls to write and read data. 	*/
/* The file name is specified as a command-line argument.    		*/
/*======================================================================*/
#define NTIMES 5               /* # of times to repeat write/read cycle */
#define NBYTES (1024*1024*4)   /* write/read size in bytes 		*/
int main( int argc, char **argv )
{
   int nprocs, myrank, i, len;
   char *filename;
   int *buf;
   MPI_File fh;
   MPI_Status status;
   MPI_Init( &argc, &argv );
   MPI_Comm_size( MPI_COMM_WORLD, &nprocs );
   MPI_Comm_rank( MPI_COMM_WORLD, &myrank );
   /*===================================================================*/
   /* process 0 takes the input file name as a command-line argument 	*/
   /* and broadcasts it to other processes 				*/ 
   /*===================================================================*/
   if ( myrank == 0 ) 
   {
      i = 1;
      while ( (i < argc) && strcmp("-fname", *argv) )
      {
         i++;
         argv++;
      }
      argv++;
      len = strlen( *argv );
      filename = (char *) malloc(len + 1);
      strcpy( filename, *argv );
      MPI_Bcast( &len, 1, MPI_INT, 0, MPI_COMM_WORLD );
      MPI_Bcast( filename, len+1, MPI_CHAR, 0, MPI_COMM_WORLD );
      printf( "%d write/read cycles per process, ", NTIMES );
      printf( "each transfering %d bytes.\n", NBYTES );
   } 
   else 
   {
      MPI_Bcast( &len, 1, MPI_INT, 0, MPI_COMM_WORLD );
      filename = (char *) malloc( len+1 );
      MPI_Bcast( filename, len+1, MPI_CHAR, 0, MPI_COMM_WORLD );
   }
   buf = (int *) malloc( NBYTES );
               
   /*===================================================================*/
   /* Repeat the open/write/close/open/read/close cycle NTIMES 		*/
   /*===================================================================*/
   for ( i = 0; i < NTIMES; i++ ) 
   {
      /*================================================================*/
      /* open the file, seek to offset for this node, write data, and 	*/
      /* then close the file 						*/
      /*================================================================*/
      MPI_File_open( MPI_COMM_WORLD, 
                     filename,
                     MPI_MODE_CREATE | MPI_MODE_RDWR, 
                     MPI_INFO_NULL, 
                     &fh );
      MPI_File_seek( fh, myrank*NBYTES, MPI_SEEK_SET );
      MPI_File_write( fh, buf, NBYTES, MPI_BYTE, &status );
      MPI_File_close( &fh );
      /*================================================================*/
      /* wait for all nodes to close file before proceeding 		*/
      /*================================================================*/
      MPI_Barrier( MPI_COMM_WORLD );
      /*================================================================*/
      /* open the file again, seek to offset for this node, read data, 	*/
      /* and then close the file 					*/
      /*================================================================*/
      MPI_File_open( MPI_COMM_WORLD, 
                     filename,
                     MPI_MODE_CREATE | MPI_MODE_RDWR, 
                     MPI_INFO_NULL, 
                     &fh );
      MPI_File_seek( fh, myrank*NBYTES, MPI_SEEK_SET );
      MPI_File_read( fh, buf, NBYTES, MPI_BYTE, &status );
      MPI_File_close(&fh);
      /*================================================================*/
      /* wait for all nodes to close file before proceeding 		*/
      /*================================================================*/
      MPI_Barrier( MPI_COMM_WORLD );
   }
   free( buf );
   free(filename);
   MPI_Finalize();
}

Instrumented C Code Using MPI IO (Corresponding, Uninstrumented C Code)

#include "MPIO_Trace.h"
#include "mpi.h"
#include "mpio.h"              /* not necessary with MPICH1.1.1 or HPMPI1.4 */
#include <stdio.h>
#include <string.h>
#include <stdlib.h>
/*======================================================================*/
/* mpioExample.c:							*/
/* This is an example of a C code using MPI I/O calls after Pablo	*/
/* instrumentation.							*/
/* It is a simple code using MPI I/O calls to write and read data. 	*/
/* The file name is specified as a command-line argument.    		*/
/*======================================================================*/
#define NTIMES 5               /* # of times to repeat write/read cycle */
#define NBYTES (1024*1024*4)   /* write/read size in bytes 		*/
int main( int argc, char **argv )
{
   int nprocs, myrank, i, len;
   char *filename;
   int *buf;
   MPI_File fh;
   MPI_Status status;
   MPI_Init( &argc, &argv );
   MPI_Comm_size( MPI_COMM_WORLD, &nprocs );
   MPI_Comm_rank( MPI_COMM_WORLD, &myrank );
   /*===================================================================*/
   /* runtime tracing; trace output to sample.run.nd? 			*/
   /*===================================================================*/
   initMPIOTrace( "sample.run", 0 );
   /*===================================================================*/
   /* process 0 takes the input file name as a command-line argument 	*/
   /* and broadcasts it to other processes 				*/ 
   /*===================================================================*/
   if ( myrank == 0 ) 
   {
      i = 1;
      while ( (i < argc) && strcmp("-fname", *argv) )
      {
         i++;
         argv++;
      }
      argv++;
      len = strlen( *argv );
      filename = (char *) malloc(len + 1);
      strcpy( filename, *argv );
      MPI_Bcast( &len, 1, MPI_INT, 0, MPI_COMM_WORLD );
      MPI_Bcast( filename, len+1, MPI_CHAR, 0, MPI_COMM_WORLD );
      printf( "%d write/read cycles per process, ", NTIMES );
      printf( "each transfering %d bytes.\n", NBYTES );
   } 
   else 
   {
      MPI_Bcast( &len, 1, MPI_INT, 0, MPI_COMM_WORLD );
      filename = (char *) malloc( len+1 );
      MPI_Bcast( filename, len+1, MPI_CHAR, 0, MPI_COMM_WORLD );
   }
   buf = (int *) malloc( NBYTES );
               
   /*===================================================================*/
   /* Repeat the open/write/close/open/read/close cycle NTIMES 		*/
   /*===================================================================*/
   for ( i = 0; i < NTIMES; i++ ) 
   {
      /*================================================================*/
      /* open the file, seek to offset for this node, write data, and 	*/
      /* then close the file 						*/
      /*================================================================*/
      MPI_File_open( MPI_COMM_WORLD, 
                     filename,
                     MPI_MODE_CREATE | MPI_MODE_RDWR, 
                     MPI_INFO_NULL, 
                     &fh );
      MPI_File_seek( fh, myrank*NBYTES, MPI_SEEK_SET );
      MPI_File_write( fh, buf, NBYTES, MPI_BYTE, &status );
      MPI_File_close( &fh );
      /*================================================================*/
      /* wait for all nodes to close file before proceeding 		*/
      /*================================================================*/
      MPI_Barrier( MPI_COMM_WORLD );
      /*================================================================*/
      /* open the file again, seek to offset for this node, read data, 	*/
      /* and then close the file 					*/
      /*================================================================*/
      MPI_File_open( MPI_COMM_WORLD, 
                     filename,
                     MPI_MODE_CREATE | MPI_MODE_RDWR, 
                     MPI_INFO_NULL, 
                     &fh );
      MPI_File_seek( fh, myrank*NBYTES, MPI_SEEK_SET );
      MPI_File_read( fh, buf, NBYTES, MPI_BYTE, &status );
      MPI_File_close(&fh);
      /*================================================================*/
      /* wait for all nodes to close file before proceeding 		*/
      /*================================================================*/
      MPI_Barrier( MPI_COMM_WORLD );
   }
   free( buf );
   free(filename);
   /*===================================================================*/
   /* call this routine to wrap up MPI I/O tracing. 			*/
   /*===================================================================*/
   endMPIOTrace();
   MPI_Finalize();
}

Compiling, Linking, and Executing

Suppose the Pablo software is installed in the directory <PabloDir> and the MPI library is installed in the directory <mpiDir>.

Note: In order to obtain information about the UNIX I/O activity in the reprot, the I/O portion of the MPI library you are using must be instrumented.  Useful data about the MPI I/O calls can still be obtained even if an uninstrumented version of the MPI library is used. 

To compile the code, use the following command:

mpicc -c mpioExample.c -I<PabloDir>/include -I<mpidir>/include

To link the code, use the following command:

mpicc -o myEXE mpioExample.o -L<PabloDir>/lib -lPabloTrace -lPabloTraceExt -lmpi [other libraries as necessary]

To execute the code, use the following command:

mpirun -np <nprocs> myEXE -fname outFile

This will produce trace output files named sample.run.nd<p> where <p> is the processor number.

Click on the name below to download either of the binary trace files produced for the case <nprocs> = 2.  See the Trace Output Files section below for the steps required to convert these files to ascii.

Postprocessing of the Trace Output Files

For each of the trace output files sample.run.nd.<p>, produced by following the procedure outlined above, issue the following command:

CreateMPIORecords -o out.<p> sample.run.nd<p>

This will produce the output files out.<p> containing records summarizing the Unix I/O activity which occurred during the execution of each MPI I/O procedure.

Click on the name below to download either of the binary trace files produced for the case <nprocs> = 2.  See the Trace Output Files section below for the steps required to convert these files to ascii.

Merging the Trace Output Files

Before doing any further processing, the trace files should be merged and the file ids synchronized. This can be done by issuing the following commands:

MergePabloTraces -o out.mrg  out.*

SyncIOfileIDs out.mrg

This will produce a tracefile out.mrg.syncFiles and a map file out.mrg.syncFiles.map. This last file is a mapping of file IDs to file names in the application.

Trace Output Files

Click to download the trace output file out.mrg.syncFiles  produced by following the steps above with nprocs equal to two (2). The file is in binary. To convert it to ASCII, use the command SDDFconverter. This command will prompt users for input. The following is the session used to convert out.mrg.syncFiles from a binary format to the ASCII file out.mrg.syncFiles.ascii. User responses are in bold. Click to download the converted trace output file User responses are in bold. Click to download the converted trace output file out.mrg.syncFiles.ascii.

% SDDFconverter

Please enter name of the input SDDF file: out.mrg.syncFiles

File is in SDDF Non-Native Binary format

Output in Ascii, Binary or Converted (reverse byte order) format [A, B, C]: A

Please entery name of the output SDDF file: out.mrg.syncFiles.ascii

Do you want diagnostic messages printed [Y or N}: N

Processing Data: Sample Reports

Issue the command MPIOstatsTable myTraces.mrg.syncFiles to produce the report for all processors on the screen.