#include "BSprivate.h"

/* ******************************************************************* */
/* Routines for building, manipulating, and using the compiled message */
/* passing routines.                                                   */
/* ******************************************************************* */

/*+ BMcomp_init - Initialize a compiled message structure.  

	Input Parameters: 
.   base_type - The base number for message types in this structure.  
                 See BMmsg.h for more information on base_type.

	Returns: 
    pointer to the compiled message structure
 +*/
BMcomp_msg *BMcomp_init(int base_type)
{
	BMcomp_msg *ph_ptr;

	MY_MALLOCN(ph_ptr,(BMcomp_msg *),sizeof(BMcomp_msg),1);
	ph_ptr->num_phases = 0;
	ph_ptr->num_msgs = 0;
	ph_ptr->base_type = base_type;
	ph_ptr->phase_list = NULL;
	return(ph_ptr);
}

/*+ BMfree_comp_msg - Free a compiled message structure.

	Input Parameters: 
.   comp_msg - Pointer to compiled message structure to free

	Returns: void
 +*/
void BMfree_comp_msg(BMcomp_msg *comp_msg)
{
	BMphase *phase_ptr;
	BMmsg *msg_ptr;

	while (comp_msg->phase_list != NULL) {
		/* free a phase */
		phase_ptr = comp_msg->phase_list;
		comp_msg->phase_list = comp_msg->phase_list->next;
		while (phase_ptr->msg_list != NULL) {
			/* free a msg */
			msg_ptr = phase_ptr->msg_list;
			phase_ptr->msg_list = phase_ptr->msg_list->next;
			BMfree_user_data(msg_ptr); CHKERR(0);
			BMfree_setup_data(msg_ptr); CHKERR(0);
			if (msg_ptr->msg != NULL) {
				MY_FREE(msg_ptr->msg);
			}
			MY_FREE(msg_ptr);
		}
		MY_FREE(phase_ptr);
	}
	MY_FREE(comp_msg);
}

/*+ BMget_phase - Get a pointer to a particular message phase.

	Input Parameters:  
.   comp_msg - The compiled message structure
.   phase_num - the phase number point to find

	Returns: 
    pointer to the phase list or NULL if not found
 +*/
BMphase *BMget_phase(BMcomp_msg *comp_msg, int phase_num)
{
	BMphase *phase_ptr;

	phase_ptr = comp_msg->phase_list;
	while (phase_ptr != NULL) {
		if (phase_ptr->phase == phase_num) {
			return(phase_ptr);
		}
		phase_ptr = phase_ptr->next;
	}
	return(phase_ptr);
}

/*+ BMadd_msg - Insert a message into the structure.

	Input Parameters:  
.   comp_msg - The compiled message structure
.   phase_num - a pointer to the message
.   procinfo - information about the processor number, etc.

	Returns: 
    void
 +*/
void BMadd_msg(BMcomp_msg *comp_msg, BMmsg *msg_ptr, BSprocinfo *procinfo)
{
	BMphase *phase_ptr, *prev;
	int	found;
	BMmsg *cur_msg;

	/* do we need to generate a msg number? */
	if (msg_ptr->msg_type < 0) {
		if (comp_msg->num_msgs >= MAX_NUM_MSGS) {
			MY_SETERRC(MSG_ERROR,"Too many messages in comp_msg\n");
		}
		msg_ptr->msg_type = comp_msg->base_type + comp_msg->num_msgs +	
			procinfo->my_id*MAX_NUM_MSGS;
	}
	comp_msg->num_msgs++;

	/* find the phase, if possible */
	phase_ptr = comp_msg->phase_list;
	prev = phase_ptr;
	found = FALSE;
	while (phase_ptr != NULL) {
		if (phase_ptr->phase == msg_ptr->phase) {
			/* put the msg at the end of the list */
			cur_msg = phase_ptr->end_msg_list;
			phase_ptr->end_msg_list = msg_ptr;
			cur_msg->next = msg_ptr;
			msg_ptr->next = NULL;
			found = TRUE;
			break;
		}
		if (phase_ptr->phase > msg_ptr->phase) break;
		prev = phase_ptr;
		phase_ptr = phase_ptr->next;
	}
	/* did not find phase, must create it */
	if (!found) {
		comp_msg->num_phases++;
		if (phase_ptr == comp_msg->phase_list) {
			MY_MALLOC(comp_msg->phase_list,(BMphase *),sizeof(BMphase),1);
			comp_msg->phase_list->phase = msg_ptr->phase;
			comp_msg->phase_list->msg_list = msg_ptr;
			comp_msg->phase_list->end_msg_list = msg_ptr;
			comp_msg->phase_list->cur_msg = NULL;
			comp_msg->phase_list->cur_list_ptr = NULL;
			comp_msg->phase_list->cur_recv_msg = NULL;
			comp_msg->phase_list->async_list = NULL;
			msg_ptr->next = NULL;
			comp_msg->phase_list->next = prev;
		} else {
			MY_MALLOC(phase_ptr,(BMphase *),sizeof(BMphase),1);
			phase_ptr->phase = msg_ptr->phase;
			phase_ptr->msg_list = msg_ptr;
			phase_ptr->end_msg_list = msg_ptr;
			phase_ptr->next = prev->next;
			phase_ptr->cur_msg = NULL;
			phase_ptr->cur_list_ptr = NULL;
			phase_ptr->async_list = NULL;
			phase_ptr->cur_recv_msg = NULL;
			prev->next = phase_ptr;
			msg_ptr->next = NULL;
		}
	}
}

/*+ BMcreate_msg - Create a message

	Input Parameters:  
.   phase - the message phase number
.   msg_type - the message type
.   msg_data_type - the data type of the message (see tools)
.   proc - the processor to send to/from

	Returns: 
    pointer to the new message data structure
 +*/
BMmsg *BMcreate_msg(int phase, int msg_type, MPI_Datatype msg_data_type,
					int proc)
{
	BMmsg *msg;

	MY_MALLOCN(msg,(BMmsg *),sizeof(BMmsg),1);
	msg->phase = phase;
	msg->msg_type = msg_type;
	msg->msg_data_type = msg_data_type;
	msg->status = COMPLETED;
	msg->proc = proc;
	msg->msg = NULL;
	BMinit_user(&(msg->user_data)); CHKERRN(0);
	BMinit_user(&(msg->setup_data)); CHKERRN(0);
	return(msg);
}

/*+ BMget_msg_size - Return the size of a message (in number of elements)

	Input Parameters:  
.   msg - pointer to a message

	Returns: 
    the message size in number of elements
 +*/
int BMget_msg_size(BMmsg *msg)
{
	return(msg->size);
}

/*+ BMset_msg_size - Set the size of a message (in number of elements)

	Input Parameters:  
.   msg - pointer to a message
.   size - the message size (in number of elements)

	Returns: 
    void
 +*/
void BMset_msg_size(BMmsg *msg, int size)
{
	msg->size = size;
}

/*+ BMinit_user - Initialize the user data structure

	Input Parameters:  
.   user_data - pointer to the user_data structure

	Returns: 
    void
 +*/
void BMinit_user(BMuser_data *user_data)
{
	user_data->data = NULL;
	user_data->length = 0;
	user_data->free_data = NULL;
}

/*+ BMfree_user - Free the user data structure

	Input Parameters:  
.   user_data - pointer to the user_data structure

	Returns: 
    void
 +*/
void BMfree_user(BMuser_data *user_data)
{
	if (user_data->free_data != NULL) {
		(*user_data->free_data)(user_data->data); CHKERR(0);
	}
	user_data->data = NULL;
	user_data->length = 0;
	user_data->free_data = NULL;
}

/*+ BMfree_user_data - Free the user's data (from the message structure)

	Input Parameters:  
.   msg - pointer to the message

	Returns: 
    void
 +*/
void BMfree_user_data(BMmsg *msg)
{
	BMfree_user(&(msg->user_data)); CHKERR(0);
}

/*+ BMfree_setup_data - Free the user's setup data (from the message structure)

	Input Parameters:  
.   msg - pointer to the message

	Returns: 
    void
 +*/
void BMfree_setup_data(BMmsg *msg)
{
	BMfree_user(&(msg->setup_data)); CHKERR(0);
}

/*+ BMset_user - Set the user data structure

	Input Parameters:  
.   user_data - pointer to the user data structure
.   data - pointer to the user's data
.   length - the length of the user data
.   fcn - the function to free the user data

	Returns: 
    void
 +*/
void BMset_user(BMuser_data *user_data, int *data, int length, void (*fcn)(int *))
{
	user_data->data = data;
	user_data->length = length;
	user_data->free_data = fcn;
}

/*+ BMset_user_data - Set the user data structure (in a message)

	Input Parameters:  
.   msg - pointer to the message
.   data - pointer to the user's data
.   length - the length of the user data
.   fcn - the function to free the user data

	Returns: 
    void
 +*/
void BMset_user_data(BMmsg *msg, int *data, int length, void (*fcn)(int *))
{
	BMset_user(&(msg->user_data),data,length,fcn); CHKERR(0);
}

/*+ BMset_setup_data - Set the user's setup data structure (in a message)

	Input Parameters:  
.   msg - pointer to the message
.   data - pointer to the user's data
.   length - the length of the user data
.   fcn - the function to free the user data

	Returns: 
    void
 +*/
void BMset_setup_data(BMmsg *msg, int *data, int length, void (*fcn)(int *))
{
	BMset_user(&(msg->setup_data),data,length,fcn); CHKERR(0);
}

/*+ BMget_user - Return the user's data and it's length 

	Input Parameters:  
.   msg - pointer to a message

	Output Parameters:  
.   length - pointer to the length of the message

	Returns: 
    pointer to the user data
 +*/
int *BMget_user(BMmsg *msg, int *length)
{
	(*length) = msg->user_data.length;
	return(msg->user_data.data);
}

/*+ BMget_setup - Return the user's setup data and it's length 

	Input Parameters:  
.   msg - pointer to a message

	Output Parameters:  
.   length - pointer to the length of the message

	Returns: 
    pointer to the user data
 +*/
int *BMget_setup(BMmsg *msg, int *length)
{
	(*length) = msg->setup_data.length;
	return(msg->setup_data.data);
}

/*+ BMset_msg_ptr - Give the message structure a pointer to the space
                    allocated for the message data.

	Input Parameters:  
.   msg - pointer to a message
.   msg_ptr - pointer to the space for the message

	Returns: 
    void
 +*/
void BMset_msg_ptr(BMmsg *msg, void *msg_ptr)
{
	msg->msg = msg_ptr;
}

/*+ BMget_msg_ptr - Get the pointer to the message data.

	Input Parameters:  
.   msg - pointer to a message

	Returns: 
    pointer to the message data
 +*/
void *BMget_msg_ptr(BMmsg *msg)
{
	return(msg->msg);
}

/*+ BMnext_msg - Get the next message in this phase

	Input Parameters:  
.   phase_ptr - pointer to the phase
.   msg_ptr - pointer to a message

	Returns: 
    pointer to the next message (NULL if none)
 +*/
BMmsg *BMnext_msg(BMphase *phase_ptr, BMmsg *msg_ptr)
{

	if (phase_ptr == NULL) return(NULL);
	if (msg_ptr == NULL) {
		msg_ptr = phase_ptr->msg_list;
	} else {
		msg_ptr = msg_ptr->next;
	}
	return(msg_ptr);
}

/*+ BMalloc_msg - Pre-allocate messages for the compiled message structure

	Input Parameters:  
.   comp_msg - pointer to the compiled message structure

 +*/
void BMalloc_msg(BMcomp_msg *comp_msg)
{
	BMphase *phase_ptr;
	BMmsg *msg_ptr;
	int	type_size;

	for (phase_ptr=comp_msg->phase_list;phase_ptr != NULL;
		phase_ptr=phase_ptr->next) {
		for (msg_ptr=phase_ptr->msg_list;msg_ptr!=NULL;msg_ptr=msg_ptr->next) {
			if (msg_ptr->msg == NULL) {
				MPI_Type_size(msg_ptr->msg_data_type,(MPI_Aint *)&type_size);
				MY_MALLOC(msg_ptr->msg,(char *),
					type_size*msg_ptr->size,1);
			}
		}
	}
}

/*+ BMfix_send - Decide what must be sent when

	Input Parameters:  
.   msg_type - message type for the use of BMfix_send
.   comp_msg_type - base type for the compiled message structure
.   comp_msg_data_type - message data type for the compiled message structure
.                        (see tools)
.   comp_msg - pointer to the compiled message structure
.   phase_ptr - pointer to the phase to work on
.   free_fcn - pointer to function to free user data
.   procinfo - the usual processor information

	Returns: 
    the number of messages to be sent
 +*/
int BMfix_send(int msg_type, int comp_msg_type, MPI_Datatype comp_msg_data_type,
	BMcomp_msg *comp_msg, BMphase *phase_ptr, void (*free_fcn)(int *),
	BSprocinfo *procinfo)
{
	BMmsg	*msg_ptr;
	int	size, from, msg_len;
	int	i, source_len, *t_msg;
	int	*addr, *source;
	BSmsg_list *msg_list;
	int	num_to_recv, num_recved;
	int	*tot_num_msg, *tot_num_msg2, *iwork, *tot_len;
	int	**msg, **cur_msg_ptr;
	int	cur_size;
	int	*int_msg;
	int	consumed;
	int	np;
	MPI_Status	mpi_status;
	void *in_msg;

	msg_list = NULL;

	/* sync up to avoid problems */
	GSYNC(procinfo->procset);
	np = procinfo->nprocs;
	MY_MALLOCN(tot_num_msg,(int *),sizeof(int)*np,0);
	MY_MALLOCN(tot_num_msg2,(int *),sizeof(int)*np,0);
	MY_MALLOCN(tot_len,(int *),sizeof(int)*np,1);
	MY_MALLOCN(msg,(int **),sizeof(int *)*np,3);
	MY_MALLOCN(cur_msg_ptr,(int **),sizeof(int *)*np,3);
	MY_MALLOCN(iwork,(int *),sizeof(int)*np,2);
	for (i=0;i<np;i++) {
		tot_num_msg[i] = 0;
		tot_num_msg2[i] = 0;
		tot_len[i] = 0;
		msg[i] = NULL;
	}
	if (phase_ptr != NULL) {
		for (msg_ptr=phase_ptr->msg_list;msg_ptr!=NULL;msg_ptr=msg_ptr->next) {
			source = BMget_setup(msg_ptr,&source_len); CHKERRN(0);
			msg_len = source_len + 3;
			tot_num_msg[msg_ptr->proc]++;
			tot_num_msg2[msg_ptr->proc]++;
			tot_len[msg_ptr->proc] += msg_len;
		}
	}
	GISUM(tot_num_msg2,np,iwork,procinfo->procset);
	num_to_recv = tot_num_msg2[procinfo->my_id];
	MY_FREE(iwork);
	MY_FREE(tot_num_msg2);

	/* tell the people who you want stuff from, that you want it */
	if (phase_ptr != NULL) {
		for (msg_ptr=phase_ptr->msg_list;msg_ptr!=NULL;msg_ptr=msg_ptr->next) {
			source = BMget_setup(msg_ptr,&source_len); CHKERRN(0);
			msg_len = source_len + 3;
			if (msg[msg_ptr->proc] == NULL) {
				MY_MALLOCN(t_msg,(int *),tot_len[msg_ptr->proc]*sizeof(int),5);
				msg[msg_ptr->proc] = t_msg;
				cur_msg_ptr[msg_ptr->proc] = t_msg;
			} else {
				t_msg = cur_msg_ptr[msg_ptr->proc];
			}
			cur_msg_ptr[msg_ptr->proc] += msg_len;
			t_msg[0] = msg_ptr->phase;
			t_msg[1] = msg_ptr->msg_type;
			t_msg[2] = msg_len;
			for (i=0;i<source_len;i++) {
				t_msg[3+i] = source[i];
			}
			tot_num_msg[msg_ptr->proc]--;
			if (tot_num_msg[msg_ptr->proc] == 0) {
				MY_SEND_SYNC(msg_list,msg_type,msg[msg_ptr->proc],
					tot_len[msg_ptr->proc],(msg_ptr->proc),MPI_INT,procinfo);
				MY_FREE(msg[msg_ptr->proc]);
			}
		}
	}
	MY_FREE(tot_num_msg);
	MY_FREE(tot_len);
	MY_FREE(msg);
	MY_FREE(cur_msg_ptr);

	/* now wait for msgs from people */
	num_recved = 0;
	while (num_recved < num_to_recv) {
		/* receive a message */
		RECVSYNCUNSZN(msg_type,in_msg,size,MPI_INT,procinfo,mpi_status); 
		CHKERRN(0);
		from = mpi_status.MPI_SOURCE;
		CHECK_SEND_LIST(msg_list);
		int_msg = (int *)in_msg;
		consumed = 0;
		while (consumed < size) {
			cur_size = int_msg[2];
			/* if no message number, then create one */
			if (int_msg[1] < 0) {
				int_msg[1] = num_recved + comp_msg_type;
			}
			msg_ptr = BMcreate_msg(int_msg[0],int_msg[1],comp_msg_data_type,
				from); CHKERRN(0);
			BMadd_msg(comp_msg,msg_ptr,procinfo); CHKERRN(0);
			/* copy user data from message */
			MY_MALLOCN(addr,(int *),(cur_size-3)*sizeof(int),6);
			for (i=0;i<cur_size-3;i++) {
				addr[i] = int_msg[i+3];
			}
			BMset_user_data(msg_ptr,addr,cur_size-3,free_fcn);
			CHKERRN(0);
			consumed += cur_size;
			int_msg += cur_size;
			num_recved++;
		}
		MSGFREERECV(in_msg);CHKERRN(0);
	}
	FINISH_SEND_LIST(msg_list);

	/* return the number of messages that were created */
	return(num_recved);
}

/*+ BMinit_comp_msg - Post receives for all messages in this structure

	Input Parameters:  
.   comp_msg - pointer to the message structure
.   procinfo - the usual processor information

	Returns: 
    void
 +*/
void BMinit_comp_msg(BMcomp_msg *comp_msg, BSprocinfo *procinfo)
{
	BMphase *phase_ptr;
	BMmsg *msg_ptr;

	for (phase_ptr=comp_msg->phase_list;phase_ptr!=NULL;
		phase_ptr=phase_ptr->next) {
		for (msg_ptr=phase_ptr->msg_list;msg_ptr!=NULL;msg_ptr=msg_ptr->next) {
			msg_ptr->status = NOT_COMPLETED;
			if (msg_ptr->msg != NULL) {
				RECVASYNCNOMEMFORCE(msg_ptr->msg_type,msg_ptr->msg,
					msg_ptr->size,msg_ptr->msg_data_type,
					msg_ptr->recv_id,procinfo); CHKERR(0);
			}
		}
	}
	GSYNC(procinfo->procset);
}

/*+ BMfinish_comp_msg - Clean up after using message structure

	Input Parameters:  
.   comp_msg - pointer to the message structure
.   procinfo - the usual processor information

	Returns: 
    void
 +*/
void BMfinish_comp_msg(BMcomp_msg *comp_msg, BSprocinfo *procinfo)
{
	BMphase *phase_ptr;
	BMmsg *msg_ptr;

	for (phase_ptr=comp_msg->phase_list;phase_ptr!=NULL;
		phase_ptr=phase_ptr->next) {
		for (msg_ptr=phase_ptr->msg_list;msg_ptr!=NULL;msg_ptr=msg_ptr->next) {
			if ((msg_ptr->msg != NULL) && (msg_ptr->status != COMPLETED)) {
				SENDWAITNOMEM(msg_ptr->msg_type,msg_ptr->msg,msg_ptr->size,
					msg_ptr->proc,msg_ptr->msg_data_type,msg_ptr->send_id); 
				CHKERR(0);
				msg_ptr->status = COMPLETED;
			}
		}
	}
}

/*+ BMsendf_msg - Send a message as a "forced" message

	Input Parameters:  
.   msg - pointer to a message
.   procinfo - information about the processor number, etc.

	Returns: 
    void
 +*/
void BMsendf_msg(BMmsg *msg, BSprocinfo *procinfo)
{
	SENDASYNCNOMEMFORCE(msg->msg_type,msg->msg,msg->size,msg->proc,
		msg->msg_data_type,msg->send_id,procinfo);
	msg->status = NOT_COMPLETED;
	CHKERR(0);
}

/*+ BMcheck_on_async_block(phase_ptr)
    Input Parameters:
.   phase_ptr -- pointer to the current phase

	Returns: 
    void
 +*/
void BMcheck_on_async_block(BMphase *phase_ptr)
{
	if (phase_ptr != NULL) {
		MCHECK_SEND_LIST(phase_ptr->async_list);
	}
}
    
/*+ BMfinish_on_async_block(phase_ptr)
    Input Parameters:
.   phase_ptr -- pointer to the current phase

	Returns: 
    void
 +*/
void BMfinish_on_async_block(BMphase *phase_ptr)
{
	if (phase_ptr != NULL) {
		MFINISH_SEND_LIST(phase_ptr->async_list);
	}
}
    
#define TRUE_MSG_LEN(msg_len) \
(((int)((msg_len+sizeof(int))/sizeof(FLOAT)))*sizeof(FLOAT))
/*+ BMsend_block_msg - Send a message and try to group with others

	Input Parameters:  
.   phase_ptr -- pointer to the current phase
.   msg - pointer to a message
.   procinfo - information about the processor number, etc.

	Returns: 
    void
 +*/
void BMsend_block_msg(BMphase *phase_ptr, BMmsg *msg, BSprocinfo *procinfo)
{
	int	*int_msg;
	char	*char_msg, *ochar_msg;
	BSmsg_list	*msg_list_node;
	int	i;
	int	*tint1; 

	/* this really shouldn't happen */
	if (phase_ptr == NULL) return;

	/* this means we should send off the pending message, if it exists */
	if (msg == NULL) {
		if (phase_ptr->cur_msg != NULL) {
			msg_list_node = phase_ptr->cur_msg;
			phase_ptr->cur_msg = NULL;
			msg_list_node->next = phase_ptr->async_list;
			phase_ptr->async_list = msg_list_node;
			SENDASYNCNOMEM(msg_list_node->msg_type,msg_list_node->msg_buf,
				msg_list_node->msg_len,msg_list_node->msg_to,
				msg_list_node->msg_data_type,msg_list_node->msg_id,procinfo); 
			CHKERR(0);
		}
		return;
	}

	/* check to see if any of the messages have been sent */
	MCHECK_SEND_LIST(phase_ptr->async_list);

	if (phase_ptr->cur_msg != NULL) {
		msg_list_node = phase_ptr->cur_msg;
		/* check to see if the destination of the message in cur_msg is the */
		/* same as the message to be sent, if not, then clear cur_msg */
		/* same thing if there isn't enough room for the message */
		if ((msg->proc != msg_list_node->msg_to) ||
			(msg->size+sizeof(FLOAT)+TRUE_MSG_LEN(msg_list_node->msg_len) >
			PREFER_MAX_MSG_SIZE)) {
			msg_list_node = phase_ptr->cur_msg;
			phase_ptr->cur_msg = NULL;
			msg_list_node->next = phase_ptr->async_list;
			phase_ptr->async_list = msg_list_node;
			SENDASYNCNOMEM(msg_list_node->msg_type,msg_list_node->msg_buf,
				msg_list_node->msg_len,msg_list_node->msg_to,
				msg_list_node->msg_data_type,msg_list_node->msg_id,procinfo); 
			CHKERR(0);
			/* fall through... */
		} else {
			/* we have enough room put it here and return */
			/* find my place in the message */
			char_msg = &(msg_list_node->msg_buf[msg_list_node->msg_len]);
			msg_list_node->msg_len += (TRUE_MSG_LEN(msg->size) + sizeof(FLOAT));
			tint1 = (int *) msg_list_node->msg_buf;
			(*tint1)++;
			int_msg = (int *) char_msg;
			int_msg[0] = msg->size;
			char_msg += sizeof(FLOAT);
			ochar_msg = (char *)msg->msg;
			for (i=0;i<msg->size;i++) {
				char_msg[i] = ochar_msg[i];
			}
			return;
		}
	}

	/* the type of the block message is the type of the 1st message */
	/* extra data in each message */
	/* 1 int -- the number of messages in the block */
	/* 1 int per sub message -- the type of a single message */
	/* 1 int per sub message -- the length of a single message */
	if (phase_ptr->cur_msg == NULL) {
		if (TRUE_MSG_LEN(msg->size)+sizeof(FLOAT)*2 <= PREFER_MAX_MSG_SIZE) {
			MY_MALLOC(msg_list_node,(BSmsg_list *),sizeof(BSmsg_list),1);
			phase_ptr->cur_msg = msg_list_node;
			msg_list_node->msg_type = msg->msg_type;
			msg_list_node->msg_len = TRUE_MSG_LEN(msg->size)+2*sizeof(FLOAT); 
			msg_list_node->msg_to = msg->proc;
			msg_list_node->msg_data_type = msg->msg_data_type;
			MY_MALLOC(msg_list_node->msg_buf,(char *),PREFER_MAX_MSG_SIZE,1);
			char_msg = (char *) msg_list_node->msg_buf;
			int_msg = (int *) char_msg;
			int_msg[0] = 1;
			char_msg += sizeof(FLOAT);
			int_msg = (int *) char_msg;
			int_msg[0] = msg->size;
			char_msg += sizeof(FLOAT);
			ochar_msg = (char *)msg->msg;
			for (i=0;i<msg->size;i++) {
				char_msg[i] = ochar_msg[i];
			}
		} else {
			MY_MALLOC(msg_list_node,(BSmsg_list *),sizeof(BSmsg_list),1);
			msg_list_node->next = phase_ptr->async_list;
			phase_ptr->async_list = msg_list_node;
			msg_list_node->msg_type = msg->msg_type;
			msg_list_node->msg_len = TRUE_MSG_LEN(msg->size) + sizeof(FLOAT)*2;
			msg_list_node->msg_to = msg->proc;
			msg_list_node->msg_data_type = msg->msg_data_type;
			MY_MALLOC(msg_list_node->msg_buf,(char *),msg_list_node->msg_len,1);
			char_msg = (char *) msg_list_node->msg_buf;
			int_msg = (int *) char_msg;
			int_msg[0] = 1;
			char_msg += sizeof(FLOAT);
			int_msg = (int *) char_msg;
			int_msg[0] = msg->size;
			char_msg += sizeof(FLOAT);
			ochar_msg = (char *)msg->msg;
			for (i=0;i<msg->size;i++) {
				char_msg[i] = ochar_msg[i];
			}
			SENDASYNCNOMEM(msg_list_node->msg_type,msg_list_node->msg_buf,
				msg_list_node->msg_len,msg_list_node->msg_to,
				msg_list_node->msg_data_type,msg_list_node->msg_id,procinfo); 
			CHKERR(0);
		}
	}
}

/*+ BMrecv_block_msg - Receive any unreceived blocked message in this phase

	Input Parameters:  
.   phase_ptr - pointer to the phase
.   procinfo - information about the processor number, etc.

	Returns: 
    pointer to the message (NULL if no more messages)
 +*/
BMmsg *BMrecv_block_msg(BMphase *phase_ptr, BSprocinfo  *procinfo)
{
	BMmsg *msg_ptr, *tmsg_ptr;
	int	i;
	int	*int_msg;
	char	*char_msg;
	int	num_msg;
	MPI_Status	mpi_status;

	if (phase_ptr == NULL) return(NULL);
	if (phase_ptr->cur_list_ptr == NULL) {
		phase_ptr->cur_list_ptr = phase_ptr->msg_list;
	} else {
		phase_ptr->cur_list_ptr = phase_ptr->cur_list_ptr->next;
	}
	msg_ptr = phase_ptr->cur_list_ptr;
	if (msg_ptr == NULL) {
		return(NULL);
	} else {
		if (msg_ptr->status == NOT_COMPLETED) {
			/* this means we *really* have to receive it */
			/* first, free up the current received message */
			BMfree_block_msg(phase_ptr);
			RECVSYNCUNSZN(msg_ptr->msg_type,phase_ptr->cur_recv_msg,
				msg_ptr->size,msg_ptr->msg_data_type,procinfo,mpi_status); 
			CHKERRN(0);
			/* now break up the message */
			char_msg = (char *)phase_ptr->cur_recv_msg;
			int_msg = (int *) char_msg;
			tmsg_ptr = msg_ptr;
			num_msg = int_msg[0];
			char_msg += sizeof(FLOAT);
			for (i=0;i<num_msg;i++) {
				int_msg = (int *) char_msg;
				tmsg_ptr->size = int_msg[0];
				char_msg += sizeof(FLOAT);
				tmsg_ptr->msg = char_msg;
				tmsg_ptr->status = COMPLETED;
				tmsg_ptr = tmsg_ptr->next;
				char_msg += TRUE_MSG_LEN(int_msg[0]);
			}
			return(msg_ptr);
		} else {
			/* we just have to return with it */
			return(msg_ptr);
		}
	}
}


/*+ BMrecv_msg - Receive any unreceived message in this phase

	Input Parameters:  
.   phase_ptr - pointer to the phase

	Returns: 
    pointer to the message (NULL if no more messages)
 +*/
BMmsg *BMrecv_msg(BMphase *phase_ptr)
{
	BMmsg *msg_ptr;
	int	waiting;
	int	finished;
	MPI_Status	mpi_status;

	if (phase_ptr == NULL) return(NULL);
	waiting = TRUE;
	while (waiting) {
		waiting = FALSE;
		for (msg_ptr=phase_ptr->msg_list;msg_ptr!=NULL;msg_ptr=msg_ptr->next) {
			if (msg_ptr->status == NOT_COMPLETED) {
				if (msg_ptr->msg != NULL) {
					MPI_Test(&(msg_ptr->recv_id),&finished,&mpi_status);
					if (finished) {
						msg_ptr->alloced = FALSE;
						msg_ptr->status = COMPLETED;
						return(msg_ptr);
					} else {
						waiting = TRUE;
					}
				} else {
					printf("Major error in BMrecv_msg\n");
				}
			}
		}
	}
	return(NULL);
}

/*+ BMfree_block_msg - Free the current blocked message, if any

	Input Parameters:  
.   phase_ptr - pointer to the current phase

	Returns: 
    void
 +*/
void BMfree_block_msg(BMphase *phase_ptr)
{
	if (phase_ptr != NULL) {
		if (phase_ptr->cur_recv_msg != NULL) {
			MSGFREERECV(phase_ptr->cur_recv_msg); CHKERR(0);
			phase_ptr->cur_recv_msg = NULL;
		}
	}
}

/*+ BMfree_msg - Free a received message

	Input Parameters:  
.   msg_ptr - pointer to a message

	Returns: 
    void
 +*/
void BMfree_msg(BMmsg *msg_ptr)
{
	if (msg_ptr->alloced) {
		if (msg_ptr->msg != NULL) {
			MSGFREERECV(msg_ptr->msg); CHKERR(0);
		}
		msg_ptr->msg = NULL;
		msg_ptr->alloced = FALSE;
	}
}