





















































#if defined(MPASS_UDPIP)  || defined(MPASS_3C905B)
#include "../library_udpip/bspudpl.h"
#else

#ifdef MPASS_TCPIP
#include "../library_tcpip/bsptcpl.h"
#else

#ifdef MPASS_MPL
#include "mpproto.h"
#else

#ifdef MPASS_MPI
#include "mpi.h"
#else

#ifdef MPASS_PARIX
#include <sys/root.h>
#include <sys/rrouter.h>
#include <virt_top.h>
#include <sys/comm.h>
#else

#ifdef MPASS_EXPRESS
#include "express.h"
#else

#if MPASS_PARMACS
#ifdef MPASS_PARMACS_HOST
#include <pmhost.h>
#else
#include <pmnode.h>
#endif 
#endif /* MPASS_PARMACS */
#endif /* MPASS_EXPRESS */
#endif /* MPASS_PARIX   */
#endif /* MPASS_MPI     */
#endif /* MPASS_MPL     */
#endif /* MPASS_UDPIP   */
#endif /* MPASS_TCPIP   */







#ifdef BSP_ONLINE_PROCS
#define BSP_MAX_PROCS BSP_ONLINE_PROCS
#else
#define BSP_MAX_PROCS 128
#endif



#define BSP_MPASS_FORK                   1
#define BSP_MPASS_SSTEP_COMM_ODD         2
#define BSP_MPASS_SSTEP_COMM_EVEN        3
#define BSP_MPASS_SSTEP_EXCHANGE_ODD     4
#define BSP_MPASS_SSTEP_EXCHANGE_EVEN    5
#define BSP_MPASS_DEBUG_WAIT             6
#define BSP_MPASS_SOFTWARE_BCAST         7
#define BSP_MPASS_SOFTWARE_GATHER        8
#define BSP_MPASS_SOFTWARE_CONSTANT_ODD  9
#define BSP_MPASS_SOFTWARE_CONSTANT_EVEN 10
#define BSP_MPASS_ARGV_SIZE              11
#define BSP_MPASS_ARGV_DATA              12
#define BSP_MPASS_STATISTICS_DATA        13
#define BSP_MPASS_ABORT                  14
#define BSP_MPASS_LAST_DEFINE            15




#define BSP_MPASS_MAXBACKOFF 4194304



























































#if defined(MPASS_UDPIP) || defined(MPASS_3C905B)
#define PID_DONTCARE (-1)
#define comm_handle   int
#define status_handle int

#define cwhatis_nprocs_mypid(refargc,refargv,refnprocs,refpid)        \
  (getenv("BSPTCP_PID")?                                              \
    (bspudp_init(refnprocs),(*(refpid)=atoi(getenv("BSPTCP_PID")),0)):\
    (*(refnprocs)=1,(*(refpid)=0),0))
  
#define f77whatis_nprocs_mypid(refnprocs,refpid) \
     cwhatis_nprocs_mypid(ohshit,ohshit,refnprocs,refpid)

#define setup_group_before_kill_unwanted(nprocs,refgids) /*donothing*/

#define setup_group_after_kill_unwanted(nprocs,refgids)  /*donothing*/

#define setup_nonblocksend_buffersize(refsize,refnbuffers,slot,round,latency) \
     (*refsize)=bspudp_set_buffer_size(refsize,refnbuffers,slot,round,latency)

#define block_recv(msg,size,from,type,nbytes) \
     bspudp_block_recv((char*)(msg),size,from,type,nbytes,            	\
			  (*type!=BSP_MPASS_SSTEP_COMM_ODD)     && 	\
			  (*type!=BSP_MPASS_SSTEP_COMM_EVEN)    && 	\
			  (*type!=BSP_MPASS_SSTEP_EXCHANGE_ODD) && 	\
			  (*type!=BSP_MPASS_SSTEP_EXCHANGE_EVEN))

#define nonblock_send(msg,size,to,type,lastone)                        	\
     bspudp_nonblock_send((char*)(msg),size,to,type,              	\
			  (type!=BSP_MPASS_SSTEP_COMM_ODD)     && 	\
			  (type!=BSP_MPASS_SSTEP_COMM_EVEN)    && 	\
			  (type!=BSP_MPASS_SSTEP_EXCHANGE_ODD) && 	\
			  (type!=BSP_MPASS_SSTEP_EXCHANGE_EVEN),lastone)

#define nonblocksend_blockrecv(o,os,d,t,i,is,s,n) \
  _bsp_software_nonblocksend_blockrecv(o,os,d,t,i,is,s,n)

#define wait_sends() bspudp_wait_sends() 

#define nonblock_probe(src,type,wait) bspudp_probe(src,type,0,wait,   	\
			  (*type!=BSP_MPASS_SSTEP_COMM_ODD)     && 	\
			  (*type!=BSP_MPASS_SSTEP_COMM_EVEN)    && 	\
			  (*type!=BSP_MPASS_SSTEP_EXCHANGE_ODD) && 	\
			  (*type!=BSP_MPASS_SSTEP_EXCHANGE_EVEN))

#define block_probe(src,type) bspudp_probe(src,type,1,0,                \
			  (*type!=BSP_MPASS_SSTEP_COMM_ODD)     && 	\
			  (*type!=BSP_MPASS_SSTEP_COMM_EVEN)    && 	\
			  (*type!=BSP_MPASS_SSTEP_EXCHANGE_ODD) && 	\
			  (*type!=BSP_MPASS_SSTEP_EXCHANGE_EVEN))

#ifdef MPASS_3C905B
#define messg_comm_status(x) /* bspkern_comm_status(x) */
#else
#define messg_comm_status(x) /* do nothing */
#endif

extern int bspudp_active_nprocs;
#define messg_active_procs(x)                \
  do {                                       \
    bspudp_active_nprocs=x;                  \
    if (bspudp_active_nprocs>_bsp_noslots)   \
      bspudp_active_nprocs=_bsp_noslots;     \
  } while(0)

#define messg_stopall()    bspudp_messg_stopall()

#define MPASS_ATEXIT
#define messg_exit(status)      \
  do {                          \
    bspudp_messg_exit(status);  \
    if (_bsp_pid) exit(status); \
  } while(0)

#define barrier_sync() _bsp_software_barrier()
#else






#ifdef MPASS_TCPIP
#define PID_DONTCARE (-1)
#define comm_handle   int
#define status_handle int

#define cwhatis_nprocs_mypid(refargc,refargv,refnprocs,refpid)        \
  (getenv("BSPTCP_PID")?                                              \
    (bsptcp_init(refnprocs),(*(refpid)=atoi(getenv("BSPTCP_PID")),0)):\
    (*(refnprocs)=1,(*(refpid)=0),0))
  
#define f77whatis_nprocs_mypid(refnprocs,refpid) \
     cwhatis_nprocs_mypid(ohshit,ohshit,refnprocs,refpid)

#define setup_group_before_kill_unwanted(nprocs,refgids) /*donothing*/

#define setup_group_after_kill_unwanted(nprocs,refgids)  /*donothing*/

#define setup_nonblocksend_buffersize(refsize,refnbuffers,slot,round,latency) \
     (*refsize)=bsptcp_set_buffer_size(refsize,refnbuffers,slot)

#define block_recv(msg,size,from,type,nbytes) \
     bsptcp_block_recv((char*)(msg),size,from,type,nbytes)

#define nonblock_send(msg,size,to,type,lastone)                        	\
     bsptcp_nonblock_send((char*)(msg),size,to,type,              	\
			  (type!=BSP_MPASS_SSTEP_COMM_ODD)     && 	\
			  (type!=BSP_MPASS_SSTEP_COMM_EVEN)    && 	\
			  (type!=BSP_MPASS_SSTEP_EXCHANGE_ODD) && 	\
			  (type!=BSP_MPASS_SSTEP_EXCHANGE_EVEN))

#define nonblocksend_blockrecv(o,os,d,t,i,is,s,n) \
  _bsp_software_nonblocksend_blockrecv(o,os,d,t,i,is,s,n)

#define wait_sends() bsptcp_wait_sends() 

#define nonblock_probe(src,type,wait) bsptcp_probe(src,type,0,wait)

#define block_probe(src,type) bsptcp_probe(src,type,1,0)

#define messg_comm_status(x) /* do nothing */

extern int bsptcp_active_nprocs;
#define messg_active_procs(x)                \
  do {                                       \
    bsptcp_active_nprocs=x;                  \
    if (bsptcp_active_nprocs>_bsp_noslots)   \
      bsptcp_active_nprocs=_bsp_noslots;     \
  } while(0)

#define messg_stopall()    bsptcp_messg_stopall()

#define MPASS_ATEXIT
#define messg_exit(status)      \
  do {                          \
    bsptcp_messg_exit(status);  \
    if (_bsp_pid) exit(status); \
  } while(0)

#define barrier_sync() _bsp_software_barrier()
#else






#ifdef MPASS_MPL
#define PID_DONTCARE   DONTCARE
#define comm_handle   int
#define status_handle int

#define cwhatis_nprocs_mypid(refargc,refargv,refnprocs,refpid) \
  mpc_environ(refnprocs,refpid)

#define f77whatis_nprocs_mypid(refnprocs,refpid) \
     cwhatis_nprocs_mypid(ohshit,ohshit,refnprocs,refpid)

#define setup_group_before_kill_unwanted(nprocs,refgids) /*donothing*/

#define setup_group_after_kill_unwanted(nprocs,refgids)              \
  do {                                                               \
    _bspmpass_errno = mpc_group(nprocs,refgids,1,&_mpass.group);    \
    if (_bspmpass_errno)                                             \
      bsp_abort("{bsp_start}: unable to create communication group");\
  } while(0)





#define setup_nonblocksend_buffersize(refsize,refnbuffers,slot,round,latency) \
  do {                                                          \
    _bspmpl_nbuf[0] = 0;                                        \
    if (mpc_task_set(_bspmpl_nbuf,2))                           \
      bsp_abort("{bsp_start}: cannot set run mode for Sp2");    \
    _bspmpl_nbuf[0] = *(refsize);                               \
    _bspmpl_nbuf[1] = 1;                                        \
    if (mpc_task_set(_bspmpl_nbuf,3))                           \
      bsp_abort("{bsp_start}: cannot set buffer size for Sp2"); \
  } while(0)



#ifdef SANITY_CHECK
#define block_recv(msg,size,from,type,nbytes)                   \
  do {                                                          \
    if (mpc_brecv(msg,size,from,type,nbytes))                   \
      bsp_abort("{block_recv} failed(%d) at line %d \"%s\"\n"   \
                "msg =%d\n *( size )=%d\n *( from )=%d\n"       \
                " *( type )=%d\n *( nbytes )=%d",               \
                mperrno, __LINE__,__FILE__,                     \
                msg,size,*(from),*(type),*(nbytes));            \
  } while(0)
#else
#define block_recv(msg,size,from,type,nbytes) \
  mpc_brecv(msg,size,from,type,nbytes)
#endif






#define nonblock_send(msg,size,to,type,lastone)           \
  ( (_bspmpass_msgid>10000)?                              \
    (  (_bspmpass_msgid=DONTCARE,                         \
        mpc_wait(&_bspmpass_msgid,&_bspmpl_nbytes)),      \
       (mpc_send(msg,size,to,type,&_bspmpass_msgid),1)):  \
    (mpc_send(msg,size,to,type,&_bspmpass_msgid),1))



#ifdef TURBO
#define nonblocksend_blockrecv(o,os,d,t,i,is,s,n) \
  do {                                            \
    nonblock_send(o,os,d,t,0);                    \
    _bspmpass_type = t;                           \
    block_recv(i,is,s,&_bspmpass_type,n);         \
  } while (0)

#else
#define nonblocksend_blockrecv(o,os,d,t,i,is,s,n) \
  _bsp_software_nonblocksend_blockrecv(o,os,d,t,i,is,s,n)
#endif




#define wait_sends()                          \
  do {                                         \
    _bspmpass_msgid = ALLMSG;                  \
    mpc_wait(&_bspmpass_msgid,&_bspmpl_nbytes);\
  } while(0)



#define nonblock_probe(src,type,wait)          \
  (mpc_probe(src,type,&_bspmpass_status)?      \
     (bsp_abort("{nonblock_probe} failed"),0): \
     (_bspmpass_status!=-1))

#define block_probe(src,type)                       \
  do {                                              \
    _bspmpass_errno = -1;                           \
    while(_bspmpass_errno==-1) {                    \
      if (mpc_probe(src,type,&_bspmpass_errno)==-1) \
        bsp_abort("{block_probe} failed");          \
    }                                               \
  } while(0);
                                                
#define messg_comm_status(x) /* do nothing */

#define messg_active_procs(x) /* Do nothing */

#define messg_stopall()    mpc_stopall(0)

#define messg_exit(status) exit(0); 





#define barrier_sync() _bsp_software_barrier()
#else






#ifdef MPASS_PARIX
#define PID_DONTCARE   (-1)
#define comm_handle int
#define status_handle int

#define cwhatis_nprocs_mypid(refargc,refargv,refnprocs,refpid) \
  (*(refnprocs) = GET_ROOT ()->ProcRoot->nProcs,              \
  (*(refpid)    = GET_ROOT ()->ProcRoot->MyProcID,            \
   0))

#define f77whatis_nprocs_mypid(refnprocs,refpid) \
     cwhatis_nprocs_mypid(ohshit,ohshit,refnprocs,refpid)

#define setup_group_before_kill_unwanted(nprocs,refgids) /*donothing*/
#define setup_group_after_kill_unwanted(nprocs,refgids)  /*donothing*/
#define setup_nonblocksend_buffersize(size,refnbuffers,slot,round,latency) \
   *(size) = MAX_MESSAGE_SIZE



#define nonblock_send(msg,size,to,type,lastone)            \
  (PutMessage(to,type,MSG_TYPE_USER_START,42,-1,msg,size),1)

#define block_recv(msg,size,from,type,nbytes)              \
  do {                                                     \
    if (_bspparix_received==NULL) {                        \
      if (GetMessage(*(from),*(type),MSG_TYPE_USER_START,  \
                     -1,&_bspparix_message)==-1)           \
        bsp_abort("{block_recv} failed");                  \
      *(from)  =_bspparix_message.Header.SourceProcId;     \
      *(type)  =_bspparix_message.Header.ReqId;            \
      *(nbytes)=_bspparix_message.Header.Size;             \
      memcpy(msg,_bspparix_message.Body,*(nbytes));        \
    } else {                                               \
      *(from)  =_bspparix_received->Header.SourceProcId;   \
      *(type)  =_bspparix_received->Header.ReqId;          \
      *(nbytes)=_bspparix_received->Header.Size;           \
      memcpy(msg,_bspparix_received->Body,*(nbytes));      \
      _bspparix_received=NULL;                             \
    }                                                      \
  } while(0)



#define nonblocksend_blockrecv(o,os,d,t,i,is,s,n) \
  _bsp_software_nonblocksend_blockrecv(o,os,d,t,i,is,s,n)



#define wait_sends() /* donothing*/



#define nonblock_probe(src,type,wait)                              \
  ((_bspmpass_status=GetMessage(*(src),*(type),MSG_TYPE_USER_START,\
                                0,&_bspparix_message)),            \
   (((_bspmpass_status==GM_TIMEOUT) || (_bspmpass_status!=42))?    \
       (_bspparix_received=NULL,0):                                \
       (_bspparix_received=&_bspparix_message,                     \
       (*(src)=_bspparix_message.Header.SourceProcId,              \
       (*(type)=_bspparix_message.Header.ReqId)))                  \
   ))

#define block_probe(src,type)                                      \
  do {                                                             \
    _bspmpass_status=GetMessage(*(src),*(type),MSG_TYPE_USER_START,\
                                -1,&_bspparix_message);            \
    if (_bspmpass_status==-1)                                      \
      bsp_abort("{block_probe} failed");                           \
    _bspparix_received = &_bspparix_message;                       \
    *(src) =_bspparix_message.Header.SourceProcId;                 \
    *(type)=_bspparix_message.Header.ReqId;                        \
  } while(0)



#define messg_comm_status(x) /* do nothing */

#define messg_active_procs(x) /* Do nothing */

#define messg_stopall()    AbortServer(0)  

#define messg_exit(status) exit(0); 



#define barrier_sync() _bsp_software_barrier()

#else






#ifdef MPASS_PARMACS
#define MPASS_NEEDS_HOST_PROGRAM
#define MPASS_HOST_SWAP_BYTE_ORDERING
#define PID_DONTCARE   (PM_ANYPID)
#define PID_HOSTID     (pm_hostpid())
#define comm_handle int
#define status_handle int

#define cwhatis_nprocs_mypid(refargc,refargv,refnprocs,refpid)      \
  ((pm_init()<0)?1:                                                 \
   (((*(refpid)=pm_mypid())==PM_NOPID)?                             \
     (bsp_abort("{bsp_begin} unable to calculate process id"),1):   \
     (*(refnprocs)=pm_nrnodes(),0)))

#define f77whatis_nprocs_mypid(refnprocs,refpid) \
     cwhatis_nprocs_mypid(ohshit,ohshit,refnprocs,refpid)

#define setup_group_before_kill_unwanted(nprocs,refgids) /*donothing*/
#define setup_group_after_kill_unwanted(nprocs,refgids)  /*donothing*/
#define setup_nonblocksend_buffersize(size,refnbuffers,slot,round,latency) \
   /* Do nothing */



#ifdef MPASS_PARMACS_HOST
#define nonblock_send(msg,size,to,type,lastone)          \
  ((pm_send(to,type,size,(char*) (msg))==(size))?1:      \
   (fprintf(stderr,"{nonblock_send(%s)} line %d of %s\n",\
            pm_strerror(pm_error()),__LINE__,__FILE__),0))
#else
#define nonblock_send(msg,size,to,type,lastone)          \
  ((pm_send(to,type,size,(char*) (msg))==(size))?1:      \
   (bsp_debug("{nonblock_send(%s)} line %d of %s\n",     \
              pm_strerror(pm_error()),__LINE__,__FILE__),0))
#endif

#define block_recv(msg,size,from,type,nbytes)                        \
  do {                                                               \
    char *msg_ptr=(char*) (msg);                                     \
    *(nbytes)=pm_recv(*(from),*(type),_bsp_buffer_size,               \
                      &msg_ptr,from,type);                           \
    if (*(nbytes)<0)                                                 \
      bsp_abort("{block_recv} line %d of %s\n"                       \
                "FROM=%d TYPE=%d NBYTES=%d "                         \
                "failed with \"%s\"",                                \
                __LINE__,__FILE__,                                   \
                *(from),*(type),*(nbytes),                           \
                pm_strerror(pm_error()));                            \
  } while(0)



#define nonblocksend_blockrecv(o,os,d,t,i,is,s,n) \
  _bsp_software_nonblocksend_blockrecv(o,os,d,t,i,is,s,n)



#define wait_sends() /* donothing*/



#define nonblock_probe(src,type,wait)                              \
  pm_probe(*(src),*(type),src,type)

#define block_probe(src,type)                                      \
  pm_wait(1,src,type,src,type)



#define messg_comm_status(x)  /* do nothing */

#define messg_active_procs(x) /* Do nothing */

#define messg_stopall()                         \
  do {                                          \
    int exit_status=1;                          \
    nonblock_send(&exit_status,sizeof(int),     \
                  PID_HOSTID,BSP_MPASS_ABORT,0);\
    pm_delete();                                \
    pm_end();                                   \
  } while(0)

#define MPASS_ATEXIT
#define messg_exit(status)                        \
  do {                                            \
    int exit_status=0;                            \
    if (_bsp_pid==0)                              \
      nonblock_send(&exit_status,sizeof(int),     \
                    PID_HOSTID,BSP_MPASS_ABORT,0);\
    pm_end();                                     \
  } while(0)



#define barrier_sync() _bsp_software_barrier()

#else






#ifdef MPASS_EXPRESS
#define PID_DONTCARE   DONTCARE
#define comm_handle   int
#define status_handle int

#define cwhatis_nprocs_mypid(refargc,refargv,refnprocs,refpid)       \
  (exparam(&_bspexpress_params),                                     \
   (*(refnprocs) = _bspexpress_params.nprocs,                        \
    (*(refpid)  = _bspexpress_params.procnum,                        \
     (((fasync(stdout)<0)?(bsp_abort("fasync(stdout) failed"),1):1), \
      (((fasync(stderr)<0)?(bsp_abort("fasync(stdout) failed"),1):1),\
      0)))))

#define f77whatis_nprocs_mypid(refnprocs,refpid) \
     cwhatis_nprocs_mypid(ohshit,ohshit,refnprocs,refpid)
          
#define setup_group_before_kill_unwanted(nprocs,refgids) /*donothing*/

#define setup_group_after_kill_unwanted(nprocs,refgids)  /*donothing*/



#define setup_nonblocksend_buffersize(refsize,refnbuffers,slot,round,latency)\
 /*empty*/



#ifdef SANITY_CHECK
#define block_recv(msg,size,from,type,nbytes) \
  do {                                        \
    *(nbytes) = exread(msg,size,from,type);   \
    if (*(nbytes) < 0)                        \
      bsp_abort("{block_recv} failed");       \
  } while(0)
#else
#define block_recv(msg,size,from,type,nbytes) \
  do {                                        \
    *(nbytes) = exread(msg,size,from,type);   \
  } while(0)
#endif




#define nonblock_send(msg,size,to,type,lastone)                    \
  ((exwrite(msg,size,(_bspexpress_dst = to,   &_bspexpress_dst),   \
                     (_bspmpass_type  = type, &_bspmpass_type)))<0?\
     (bsp_abort("{nonblock_send} failed"),0):                      \
     (1))



#ifdef TURBO
#define nonblocksend_blockrecv(o,os,d,t,i,is,s,n)           \
  do {                                                      \
    _bspmpass_type  = t;                                    \
    _bspexpress_dst = d;                                    \
    *(n) = exchange(i,is,s,&_bspmpass_type,                 \
                    o,os,&_bspexpress_dst,&_bspmpass_type); \
    if ((_bspmpass_type!=t) || (_bspexpress_dst != d))      \
      bsp_abort("{nonblocksend_blockrecv} failed");         \
  } while(0)
#else
#define nonblocksend_blockrecv(o,os,d,t,i,is,s,n) \
  do {                                            \
    nonblock_send(o,os,d,t,0);                    \
    _bspmpass_type = t;                           \
    block_recv(i,is,s,&_bspmpass_type,n);         \
  } while (0)
#endif




#define wait_sends()  /* donothing */



#define nonblock_probe(src,type,wait) (extest(src,type)>=0)

#ifdef SANITY_CHECK
#define block_probe(src,type)               \
  do {                                      \
    if (exbtest(src,type)<0)                \
        bsp_abort("{block_probe} failed");  \
  } while(0)
#else  
#define block_probe(src,type) \
  do {                        \
    exbtest(src,type);        \
  } while(0)
#endif


                                     
#define messg_comm_status(x) /* do nothing */

#define messg_active_procs(x) /* Do nothing */

#define messg_stopall()    abort(0)

#define messg_exit(status) exit(0); 



 
#define barrier_sync() _bsp_software_barrier()

#else






#ifdef MPASS_MPI
#define PID_DONTCARE   MPI_ANY_SOURCE
#define comm_handle MPI_Request*
#define status_handle MPI_Status

#define cwhatis_nprocs_mypid(refargc,refargv,refnprocs,refpid) \
  (MPI_Init(refargc,refargv)                               || \
   MPI_Comm_rank(MPI_COMM_WORLD,refpid)                    || \
   (_bspmpass_errno=MPI_Comm_size(MPI_COMM_WORLD,refnprocs),  \
     (_mpass.group = MPI_COMM_WORLD,_bspmpass_errno)))

#ifdef UNDERSCORE
#define f77MPI_Init mpi_init_
#else
#ifdef DOUBLEUNDERSCORE
#define f77MPI_Init mpi_init__
#else
#ifdef UPPERCASE
#define f77MPI_Init MPI_INIT
#else
#define f77MPI_Init mpi_init
#endif
#endif
#endif

#define f77whatis_nprocs_mypid(refnprocs,refpid) \
  ((f77MPI_Init(&_bspmpass_errno),_bspmpass_errno)         || \
   MPI_Comm_rank(MPI_COMM_WORLD,refpid)                    || \
   (_bspmpass_errno=MPI_Comm_size(MPI_COMM_WORLD,refnprocs),  \
     (_mpass.group = MPI_COMM_WORLD,_bspmpass_errno)))

#define setup_group_before_kill_unwanted(nprocs,refgids)                  \
  do {                                                                    \
    if (MPI_Comm_group(MPI_COMM_WORLD,&_bspmpi_group_world))              \
      bsp_abort("{bsp_start} failed to create an MPI comm group");        \
    if (MPI_Group_incl(_bspmpi_group_world,nprocs,refgids,&_bspmpi_group))\
      bsp_abort("{bsp_start} failed to add to an MPI comm group");        \
    if (MPI_Comm_create(MPI_COMM_WORLD,_bspmpi_group,&_mpass.group))      \
      bsp_abort("{bsp_start}: unable to create communication group");     \
  } while(0)

#define setup_group_after_kill_unwanted(nprocs,refgids) /*donothing*/
#define setup_nonblocksend_buffersize(size,refnbuffers,slot,round,latency)\
  do {                                                                   \
    _bspmpass_msgid    = CALLOC(_bsp_comm_fifo_size,SIZEOF(MPI_Request));\
    _bspmpi_status_arr = CALLOC(_bsp_comm_fifo_size,SIZEOF(MPI_Status)); \
    if (_bspmpass_msgid==NULL || _bspmpi_status_arr==NULL)              \
      bsp_abort("{setup_nonblocksend_buffersize} failed");              \
    _bspmpi_next_msgid=0;                                               \
  } while(0)



#define block_recv(msg,size,from,type,nbytes)             \
 do {                                                     \
   if (MPI_Recv(msg,size,MPI_BYTE,*(from),*(type),        \
                _mpass.group,&_bspmpass_status))          \
     bsp_abort("{block_recv} failed to MPI_Recv");        \
   if (MPI_Get_count(&_bspmpass_status,MPI_BYTE,nbytes))  \
     bsp_abort("{block_recv} failed to MPI_Get_count");   \
   *(from) = _bspmpass_status.MPI_SOURCE;                 \
   *(type) = _bspmpass_status.MPI_TAG;                    \
 } while(0)



#define nonblock_send(msg,size,to,type,lastone)            \
  (MPI_Isend(msg,size,MPI_BYTE,to,type,_mpass.group,       \
                  &_bspmpass_msgid[_bspmpi_next_msgid++])? \
    (bsp_abort("{nonblock_send} failed"),0):1)



#define nonblocksend_blockrecv(o,os,d,t,i,is,s,n) \
  _bsp_software_nonblocksend_blockrecv(o,os,d,t,i,is,s,n)



#define wait_sends()                                     \
  do {                                                   \
    if (MPI_Waitall(_bspmpi_next_msgid,_bspmpass_msgid,  \
                    _bspmpi_status_arr))                 \
      bsp_abort("{wait_sends} failed");                  \
    _bspmpi_next_msgid=0;                                \
  } while(0)




#define nonblock_probe(src,type,wait)                \
  (MPI_Iprobe(*(src),*(type),_mpass.group,           \
              &_bspmpass_errno,&_bspmpass_status)?   \
     (bsp_abort("{nonblock_probe} failed"),0):       \
     (*(src) = _bspmpass_status.MPI_SOURCE,          \
     (*(type)= _bspmpass_status.MPI_TAG,             \
     _bspmpass_errno)))                              \
    
#define block_probe(src,type)                                   \
  do {                                                          \
    if (MPI_Probe(*(src),*(type),_mpass.group,&_bspmpass_status))\
      bsp_abort("{block_probe} failed");                        \
    *(src) = _bspmpass_status.MPI_SOURCE;                         \
    *(type)= _bspmpass_status.MPI_TAG;                            \
  } while(0)



#define messg_comm_status(x) /* do nothing */

#define messg_active_procs(x) /* Do nothing */

#define messg_stopall() MPI_Abort(MPI_COMM_WORLD,0)

#define MPASS_ATEXIT
#define messg_exit(status) MPI_Finalize()



#ifdef SANITY_CHECK
#define barrier_sync()                                    \
  do {                                                    \
    if (MPI_Barrier(_mpass.group))                        \
      bsp_abort("{barrier-sync}: synchronisation failed");\
  } while(0)
#else
#define barrier_sync() MPI_Barrier(_mpass.group); 
#endif 
#endif /* MPASS_MPI */
#endif /* MPASS_EXPRESS */
#endif /* MPASS_PARMACS */
#endif /* MPASS_PARIX   */
#endif /* MPASS_MPL     */
#endif /* MPASS_TCPIP   */
#endif /* MPASS_UDPIP   */






typedef unsigned int bsp_bitvec_t;

#define set_bit_in_type(type,x,i)    (x|=(((type) 1) << i))
#define unset_bit_in_type(type,x,i)  (x&=~(((type) 1) << i))
#define test_bit_in_type(type,x,i)   ((int) ((x >> i) & 1))

#define messg_matrix_set(i)                                           \
  do {                                                                \
    int i_=(i);                                                       \
    set_bit_in_type(                                                  \
       bsp_bitvec_t,                                                  \
       _mpass.proc_comm_exchange_matrix[i_/(sizeof(bsp_bitvec_t)*8)], \
       MOD_POW2(i_,(sizeof(bsp_bitvec_t)*8)));                        \
  } while(0) 

#define messg_matrix_unset(i)                                         \
  do {                                                                \
    int i_ = (i);                                                     \
    unset_bit_in_type(                                                \
       bsp_bitvec_t,                                                  \
       _mpass.proc_comm_exchange_matrix[i_/(sizeof(bsp_bitvec_t)*8)], \
       MOD_POW2(i_,(sizeof(bsp_bitvec_t)*8)));                        \
  } while(0) 


#ifdef __GNUC__
#define messg_matrix_isset(i)                                       \
  ({int i_ = (i);                                                   \
    int res_;                                                       \
    res_ = test_bit_in_type(bsp_bitvec_t,                           \
                   _mpass.proc_comm_exchange_matrix[i_/             \
                      (sizeof(bsp_bitvec_t)*8)],                    \
                   MOD_POW2(i_,(sizeof(bsp_bitvec_t)*8)));          \
    res_;})
    
#else

#define messg_matrix_isset(i)                                       \
  test_bit_in_type(bsp_bitvec_t,                                    \
                   _mpass.proc_comm_exchange_matrix[(i)/            \
                      (sizeof(bsp_bitvec_t)*8)],                    \
                   MOD_POW2((i),(sizeof(bsp_bitvec_t)*8)))
#endif








typedef struct {
#ifdef MPASS_MPI
   MPI_Comm         group;
#else
   int              group;
#endif
   bsp_bitvec_t    *proc_comm_exchange_matrix;
   bsp_bitvec_t    *proc_comm_exchange_matrix_copy;

   _BSPcomm_thunk **proc_comm_out_fifo;    /* pending comms per proc  */
   _BSPcomm_thunk **proc_comm_in_fifo;     /* comm coming in          */
   int             *proc_comm_out_next;    /* next position out       */ 
   int             *proc_comm_in_next;     /* next position in        */
   int             *proc_comm_out_size;    /* size of out fifo        */
   int             *proc_comm_in_size;     /* size of out fifo        */
} _BSPinfo;

extern _BSPinfo _mpass;
extern comm_handle   _bspmpass_msgid; 
extern status_handle _bspmpass_status;
extern int           _bspmpass_errno;
extern int           _bspmpass_type;

#ifdef MPASS_MPL
extern int           _bspmpl_nbytes;
#endif

#ifdef MPASS_MPI
extern status_handle *_bspmpi_status_arr;
extern int           _bspmpi_next_msgid;
#endif

#ifdef MPASS_PARIX
extern RR_Message_t _bspparix_message,*_bspparix_received;
#endif

#ifdef MPASS_EXPRESS
extern int          _bspexpress_dst; 
#endif    

