/* CVS info   */
/* $Date: 2007/03/13 17:06:56 $     */
/* $Revision: 1.5 $ */
/* $RCSfile: io_b_1thru4.c,v $  */
/* $Name: rel_7 $     */
/*
   This file contains all of the actual benchmarks. The makefile will
   produce 5 programs by conditionally compiling the same source. Bench2
   and Bench3 are virtually the same. The main difference has to do with
   the size of the input files. The comments at the beginning of each
   subroutine have a brief description of what each program does.
   The benchmarks measures the following:
       Caching local to a compute node
       Caching local to an IO server
       Reading from disc with no caching
       How well mmaps performs
       How well the system can handle a large number of writes
*/

#include <unistd.h>
#include <stdio.h>
#include <stdlib.h>
#include <time.h>
#include <sys/time.h>
#include <errno.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <sys/types.h>
#include <sys/mman.h>
#include "data_sizes.h"
#include "iobench.h"
 
// Global
extern long jobid;

// Permissions 0666 everone can read and write
#define BM_PERMS  (S_IRUSR|S_IWUSR|S_IRGRP|S_IWGRP|S_IROTH|S_IWOTH)

// Open for read and write
#define BM_FLAGS  (O_RDWR)

#define W_BM_PERMS  (S_IRUSR|S_IWUSR|S_IRGRP|S_IWGRP|S_IROTH|S_IWOTH)
#define W_BM_FLAGS  (O_WRONLY|O_CREAT|O_EXCL)

static void syserror(int line, char* file, const char* func, char* msg){
  fprintf(stderr,"%04ld %s", jobid, msg);
  fprintf(stderr,"%s ", strerror( errno ) );
  fprintf(stderr,"in function %s in file %s at line %d\n",
          func, file, line );
}

#define SYSERROR(MSG) syserror(__LINE__, __FILE__, __func__, MSG)

char  io_dir_name[4096];
char  io_file_name[4096];

char* part = "abc";

double wall(void);
static void mk_unique_nm(char* , char* , int);


#if  defined( __BM_RFCC )

// Each job reads same 150M file from its ioserver
// File name is .../cnode_caching_150M


int read_from_cnode_cache(char* io_dir_name, double* el_time){

  ssize_t nr;
  int par_rd;
  double t0;
  int fd;
  int i;

  char version[] = "FIOPB_VER_4/06";
  static char cvs_info[] =
   "BMARKGRP $Date: 2007/03/13 17:06:56 $ $Revision: 1.5 $ $RCSfile: io_b_1thru4.c,v $ $Name: rel_7 $";

  sprintf(io_file_name,"%s/cnode_caching_150M", io_dir_name);

#ifdef DBG
  printf("dir name = %s  file name = %s\n", io_dir_name, io_file_name);
#endif

  t0 = wall();

  if( (fd = bm_open(io_file_name , BM_FLAGS, BM_PERMS) ) == -1){
    fprintf(stderr,"%04ld Read_from_cnode_cache, initial open of file %s failed\n",
	    jobid,io_file_name );
    return -1;
  }

  for(i=0; i<64; i++){
#ifdef PRG
    fprintf(stderr,"file number %d\n", i);
#endif
    if ( bm_read( fd, io_buffer, BF_SZ ) == -1 ){
      SYSERROR("READING FILE:  ");
      ch_p_reads();                  //check for partial reads (not and error)
      fprintf(stderr,"%04ld ,Read_from_cnode_cache read number %d, file %s\n", jobid,
	      i, io_file_name );
      return -1;
    }
  }

  if( bm_close( fd ) == -1 ){
    SYSERROR("B1A CLOSE FILE:  ");
    fprintf(stderr,"bench 1a , file %s\n", io_file_name );
    return -1;
  }

  *el_time = wall() - t0;
  return 0;
}

#elif   defined( __BM_MFI )


int mmap_file_in(char* io_dir_name, double* el_time){
  double t0;
  // Each job mmaps the 750M file 'mmap_file_input_750M'  from its ioserver and reads
  // a byte from each page. This is done 16 times

  int fdin;
  int i, j;
  char *src, *dst, sum=0;
  long pgsz, pages;

  struct stat statbuf;

  static char cvs_info[] =
    "BMARKGRP $Date: 2007/03/13 17:06:56 $ $Revision: 1.5 $ $RCSfile: io_b_1thru4.c,v $ $Name: rel_7 $";

  if ( (pgsz = sysconf( _SC_PAGE_SIZE )) == -1L){
    fprintf(stderr,"%04ld mmap_file_in: sysconf can't find page size\n",
	    jobid );
    exit( 1 );
  }

  sprintf(io_file_name, "%s/mmap_file_input_750M", io_dir_name);
#ifdef DBG
  printf("dir name = %s  file name = %s\n", io_dir_name, io_file_name);
#endif

  t0 = wall();

  for (i=0; i<16; i++){
#ifdef PRG
    fprintf(stderr,"map repetition %d\n", i);
#endif
    if ( (fdin = open(io_file_name, O_RDONLY)) < 0){
      fprintf(stderr,"%04ld mmap_file_in , can't open file %s\n",
	      jobid, io_file_name );
      return -1;
    }

    if (fstat(fdin, &statbuf) < 0){
      fprintf(stderr,"%04ld mmap_file_in , fstat error on file %s failed\n",
	      jobid, io_file_name );
      return -1;
    }

    pages = statbuf.st_size/pgsz;

    errno = 0;
    if (( src = mmap( 0, statbuf.st_size, PROT_READ,
		    MAP_FILE|MAP_SHARED , fdin, 0)) == MAP_FAILED ){
      SYSERROR("mmap_file_input_750M:  ");
      fprintf(stderr,"mmap_file_in  , file %s\n", io_file_name );
      return -1;
    }

    // Touch each page of memory by reading a byte
    for(j=0; j<statbuf.st_size; j+=pgsz  ){
#ifdef PRG
      fprintf(stderr,"%d  %d  %d\n",j, pgsz, statbuf.st_size);
#endif
      sum += src[ j ];
    }

    errno = 0;
    if (munmap( src, statbuf.st_size ) == -1){
      SYSERROR("mmap_file_input_750M:  ");
      fprintf(stderr,"%04ld ,mmap_file_in munmap error on file %s failed \n",
	      jobid, io_file_name );
      return -1;
    }
  }

  *el_time = wall() - t0;

  return 0;
}

#elif   defined( __BM_RFIOSC ) || defined( __BM_RFD )

#   if defined( __BM_RFIOSC )
       int read_from_ioserver_cache(char* io_dir_name, double* el_time){
#   else
       int read_from_disk(char* io_dir_name, double* el_time){
#   endif
  // read_from_ioserver_cache   ( cached on IO server)
  // Each job reads parts of the 4G file 'read_from_IO_server_cache_4G' from its ioserver

  //   read 18000 chunks    of     size     4k
  //   read 150   chunks    of     size     1m
  //   read 1     chunks    of     size   128m

  // read_from_disk   ( uncached on IO server)
  // Each job reads parts of the 512G file 'read_disk_512G' from its ioserver

  //   read 80   chunks    of     size     1MB
  //   read  1   chunks    of     size    96MB

  double t0;
  off_t blocks;
  off_t whichblock;
  int i, j;
  double d_rand_max = RAND_MAX;
  off_t ls_stat;
  ssize_t rd_stat;
  size_t howbig[R];
  int    howmany[R];
  int fd;

  static char cvs_info[] =
    "BMARKGRP $Date: 2007/03/13 17:06:56 $ $Revision: 1.5 $ $RCSfile: io_b_1thru4.c,v $ $Name: rel_7 $";


#if  defined( __BM_RFIOSC )
  sprintf(io_file_name,"%s/read_from_IO_server_cache_4G", io_dir_name);
#else
  sprintf(io_file_name,"%s/read_disk_512G", io_dir_name);
#endif

#ifdef DBG
  printf("dir name = %s  file name = %s\n", io_dir_name, io_file_name);
#endif

  howmany[0] = B23_HM0;  howbig[0]  = B23_HB0;
  howmany[1] = B23_HM1;  howbig[1]  = B23_HB1;
#if (R == 3)
  howmany[2] = B23_HM2;  howbig[2]  = B23_HB2;
#endif
  t0 = wall();
  if( (fd = bm_open(io_file_name , BM_FLAGS, BM_PERMS)) == -1){
    fprintf(stderr,"%04ld ,read_from_IO_server_cache initial open of file %s\n",
	    jobid, io_file_name );
    return -1;
  }

  for(i=0; i<R; i++){
    blocks = FILE_SZ/howbig[i];
#ifdef PRG
    fprintf(stderr,"reading %d blocks of size %d\n", howmany[i], howbig[i]);
#endif
    for(j=0; j<howmany[i]; j++){
      whichblock = (off_t)(blocks*((double)rand()/d_rand_max)  );
#ifdef PRG
      fprintf(stderr,"block number %d of %d total\n", whichblock, blocks);
#endif
      errno = 0;
      ls_stat = bm_lseek(fd, whichblock*howbig[i], SEEK_SET);
      if ( ls_stat == ( off_t) -1  ){

#if  defined( __BM_RFIOSC )
	SYSERROR("read_from_IO_server:  ");
	fprintf(stderr,"%04ld , read_from_IO_server: part 2%c, lseek number %d, file %s\n",
		jobid, part[i], j, io_file_name );
#else
	SYSERROR("read_from_disk:  ");
	fprintf(stderr,"%04ld read_from_disk, part 2%c, lseek number %d, file %s\n",
		jobid, part[i], j, io_file_name );
#endif
	return -1;
      }

      errno = 0;
      rd_stat = bm_read(fd, io_buffer, howbig[i]);

      if ( rd_stat == (ssize_t) -1){
	SYSERROR("B23 READ:  ");
	fprintf(stderr,"%04ld bench 2|3, part 2%c, read number %d, file %s\n",
		jobid, part[i], j, io_file_name );
	return -1;
      }
    }
  }

  errno = 0;
  if( bm_close( fd ) == -1 ){
    SYSERROR("B2 CLOSE FILE:  ");
    fprintf(stderr,"bench 2|3 , file %s\n", io_file_name );
    return -1;
  }
  *el_time = wall() - t0;
  return 0;
}

#elif   defined( __BM_WMF )


int write_many_files(char* io_dir_name, double* el_time){

  // Each job writes  N  files to its ioserver
  // write   1200 files of size  4k
  // write     60 files of size  1m
  // write      1 files of size 64m

  double t0;
  size_t howbig[3];
  int    howmany[3];
  char*  fn[] = { "write_many_files_a", "write_many_files_b", "write_many_files_c" };
  ssize_t written;
  int i, j;
  int fd;

  static char cvs_info[] =
    "BMARKGRP $Date: 2007/03/13 17:06:56 $ $Revision: 1.5 $ $RCSfile: io_b_1thru4.c,v $ $Name: rel_7 $";

  howmany[0] = B4_HM0;  howbig[0]  = B4_HB0;
  howmany[1] = B4_HM1;  howbig[1]  = B4_HB1;
  howmany[2] = B4_HM2;  howbig[2]  = B4_HB2;

  t0 = wall();

  for(i=0; i<3; i++){
    for(j=0; j<howmany[i]; j++){
      // Create a file name unique to this RES session
      mk_unique_nm(io_dir_name, fn[i], j );

#ifdef PRG
      printf("dir name = %s  file name = %s\n", io_dir_name, io_file_name);
#endif

      fd = bm_open(io_file_name, W_BM_FLAGS, W_BM_PERMS);

      if ( (written = bm_write(fd, io_buffer, howbig[i] )) == (ssize_t) -1){
	fprintf(stderr,"%04ld write_many_files_a, part 4%c, write number %d, file %s\n",
		jobid, part[i], j, io_file_name );
	ch_p_writes();
	return -1;
      }
      if( bm_close( fd ) == -1 ){
	SYSERROR("write_many_files_a CLOSE FILE:  ");
	fprintf(stderr,"write_many_files_a , file %s\n", io_file_name );
	return -1;
      }
    }
  }
  *el_time = wall() - t0;

  return 0;
}

#endif

static void mk_unique_nm(char* io_dir_name, char* basename, int n){
  long f_suffix;

  // Assume jobid <= 4096     b4_4096000000, b4_4096000001, ...b4_4096000002,
  f_suffix = jobid*1000000 + n;

  sprintf(io_file_name,"%s/%s_%010ld", io_dir_name, basename, f_suffix );
}


/* wall-clock time */

double wall(void){
  struct timeval tp;
  gettimeofday (&tp, NULL);
  return
    tp.tv_sec + tp.tv_usec/(double)1.0e6;
}