/**************************************************************************
*
*   R11 is the output routine for Benchmark 11. 
*
*   This will need to be modified to contain the particular
*   configuration information requested.
*
*   Parameters:
*     Provided by calling routine:
*       C       =   Product matrix
*       MODULUS =   Modulus for the arithmetic
*       N       =   Size of matrices
*       M       =   Size of sub-matrices on each PE
*       Q       =   Size of grid of Q x Q PEs
*       NUMSIZE =   Number of words used to hold an MP integer
*       ERR     =   Error flag
*       CSET    =   CPU time required to generate the data
*       WSET    =   Wallclock time required to generate the data
*       CRUN    =   CPU time required to do the benchmark
*       WRUN    =   Wallclock time required to do the benchmark
*       CCHECK  =   CPU time required to check the results
*       WCHECK  =   Wallclock time required to check the results
*
**************************************************************************/
/* CVS info   */
/* $Date: 2005/01/10 21:05:28 $     */
/* $Revision: 1.2 $ */
/* $RCSfile: r11.c,v $  */
/* $Name: rel_5 $     */

#include "bench11.h"

#include <mpi.h>

static char cvs_info[] = "BMARKGRP $Date: 2005/01/10 21:05:28 $ $Revision: 1.2 $ $RCSfile: r11.c,v $ $Name: rel_5 $"; 


/* use macros to address 3-dim arrays with variable dimension */

#define c(I,J,K)        c[ (K) + numsize * ( (J) + m * (I) ) ]

void r11 ( mplong c[], mplong modulus[], mplong checksum,
	int n, int q, int m, int numsize, int err,
	double cset, double wset, double crun, double wrun,
	double ccheck, double wcheck )
{
   /* Variables for time statistics */
   double crunmax, crunmin, crunavg, wrunmax;

   int i, j, k;
   int perow, pecol, locali, localj, crank;
   int tag, IER, sendsize;

   extern int npes, mype;

   mplong tempc [ maxnumsize ];

   char * date();

   MPI_Status status;
   MPI_Comm world = MPI_COMM_WORLD;

   /* Collect-add run times in AVGCRUN, max in MAXCRUN on PE0 */

   if (npes > 1)
   {
      MPI_Reduce(&crun, &crunmin, 1, MPI_DOUBLE, MPI_MIN, 0, world);
      MPI_Reduce(&crun, &crunmax, 1, MPI_DOUBLE, MPI_MAX, 0, world);
      MPI_Reduce(&crun, &crunavg, 1, MPI_DOUBLE, MPI_SUM, 0, world);
      if (mype == 0) crunavg = crunavg / npes;
      MPI_Reduce(&wrun, &wrunmax, 1, MPI_DOUBLE, MPI_MAX, 0, world);
   }

   /* PE0 prints out header and times */

   if (mype == 0)
   {
      printf ("\n\n\nBenchmark #11 -- Multiprecision Arithmetic - MPI\n");
      printf("Date: %s", date());

      printf ( "Number of processors = %d \n", npes );
      printf ( "Size of matrices =     %d \n", n );
      printf ( "Modulus: \n" );
      mpwrite(modulus);
      printf ("\n");

      printf ( "Time for set up: \n" );
      printf ( "CPU        = %f seconds \n", cset );
      printf ( "Wall Clock = %f seconds \n\n", wset );

      printf ( "Time to run: \n" );
      if (npes > 1)
      {
         printf ( "CPU MIN    = %f seconds \n", crunmin );
         printf ( "CPU MAX    = %f seconds \n", crunmax );
         printf ( "CPU AVG    = %f seconds \n", crunavg );
         printf ( "Wall Clock = %f seconds \n\n", wrunmax );
      }
      else
      {
         printf ( "CPU        = %f seconds \n", crun );
         printf ( "Wall Clock = %f seconds \n\n", wrun );
      }

      printf ( "Time to check results: \n" );
      printf ( "CPU        = %f seconds \n", ccheck );
      printf ( "Wall Clock = %f seconds \n\n", wcheck );

      printf ( "Checksum   = %ld\n\n", checksum );

      /* status returned by check routine */

      if (err == 0)
         printf ( "NO ERRORS FOUND\n" );

      if (err < 0)
      {
         printf ( "The results for given input parameters are unknown. \n" );
         printf ( "No checking is possible. \n" );
      }

      if (err == 1)
         printf ( "ERROR: Results do not agree with previous results. \n" );

      if (err == 2)
         printf ( "ERROR: MPI_Gather failed, unable to check results. \n" );

   } /* IF (MYPE == 0) print out header and times */

   // PE0 finished first half of output.
   // If PrintAll flag is defined in makefile,
   //   print selected entries in product matrix C

#ifdef PrintAll

   sendsize = numsize * intspermplong;

   /* Several PEs must cooperate to get the data to PE0 */

   for ( i = 0; i < n; i += n/3 )
   for ( j = 0; j < n; j += n/4 )
   {
      /* figure where C(I,J) is and get it to PE0 and print */

      perow = i / m;
      pecol = j / m;
      crank = q * perow + pecol;

      if (crank > 0)
      {
         /* get C(I,J) from other PE to TEMPC on PE0 */
         locali = i % m;
         localj = j % m;
         tag = crank;

         if (mype == 0)
            MPI_Recv ( tempc, sendsize, MPI_INT, crank, tag, world, &status );

         if (mype == crank)
            MPI_Send ( &c(locali,localj,0), sendsize, MPI_INT, 0, tag, world);
      }
      else
      {
         /* it`s already on PE0, just move it from C(I,J) to TEMPC */

         for ( k = 0; k < numsize; k++)
            tempc[k] = c(i,j,k);
      }

      if (mype == 0)
      {
# ifdef GMP
         tempc[MPPTR] = (mplong) & tempc[MPDATA]; /* moved, fix ptr */
# endif
         printf ( "\nC(%4d,%4d ) =:\n", i+1, j+1 );
         mpwrite( tempc );
      }

   } /* end loop on i and j */

   /* get C(N,N) */

   /* crank = q^2 -1 -- (n-1)/m is q-1 -- unless q > n */

   crank = (q+1) * ((n-1)/m);

   if ( crank > 0 )
   {
      /* get C(N,N) from last PE into TEMPC on PE0 */

      locali = (n-1) % m;
      localj = locali;
      tag = crank;

      if (mype == 0)
         MPI_Recv ( tempc, sendsize, MPI_INT, crank, tag, world, &status );

      if (mype == crank)
         MPI_Send ( &c(locali,localj,0), sendsize, MPI_INT, 0, tag, world );
   }
   else
   {
      /* it`s already on PE0, just move it from C(N,N) to TEMPC */

      for ( k = 0; k < numsize; k++)
         tempc[k] = c(n-1,n-1,k);
   }

   if (mype == 0)
   {
# ifdef GMP
      tempc[MPPTR] = (mplong) & tempc[MPDATA]; /* moved, fix ptr */
# endif
      printf ( "\nC(%4d,%4d ) =:\n", n, n );
      mpwrite( tempc );
      printf ( "\n" );
   }

#endif

/* End printing selected values of C if PrintAll is defined in makefile */

}