From: Dimitri Sivanich Use no-op messages in place of cross-partition interrupts when nacking a put message in the GRU. This allows us to remove MMR's as a destination from the GRU driver. Signed-off-by: Dimitri Sivanich Signed-off-by: Mike Travis Tested-by: John Estabrook Tested-by: Gary Kroening --- arch/x86/include/asm/uv/uv_hub.h | 10 ---------- drivers/misc/sgi-gru/grukservices.c | 36 ++++++++++++++++++++---------------- 2 files changed, 20 insertions(+), 26 deletions(-) --- linux.orig/arch/x86/include/asm/uv/uv_hub.h +++ linux/arch/x86/include/asm/uv/uv_hub.h @@ -642,16 +642,6 @@ static inline unsigned long uv_read_glob return readq(uv_global_mmr64_address(pnode, offset)); } -/* - * Global MMR space addresses when referenced by the GRU. (GRU does - * NOT use socket addressing). - */ -static inline unsigned long uv_global_gru_mmr_address(int pnode, unsigned long offset) -{ - return UV_GLOBAL_GRU_MMR_BASE | offset | - ((unsigned long)pnode << uv_hub_info->m_val); -} - static inline void uv_write_global_mmr8(int pnode, unsigned long offset, unsigned char val) { writeb(val, uv_global_mmr64_address(pnode, offset)); --- linux.orig/drivers/misc/sgi-gru/grukservices.c +++ linux/drivers/misc/sgi-gru/grukservices.c @@ -718,8 +718,8 @@ cberr: static int send_message_put_nacked(void *cb, struct gru_message_queue_desc *mqd, void *mesg, int lines) { - unsigned long m, *val = mesg, gpa, save; - int ret; + unsigned long m; + int ret, loops = 200; /* experimentally determined */ m = mqd->mq_gpa + (gru_get_amo_value_head(cb) << 6); if (lines == 2) { @@ -735,22 +735,26 @@ static int send_message_put_nacked(void return MQE_OK; /* - * Send a cross-partition interrupt to the SSI that contains the target - * message queue. Normally, the interrupt is automatically delivered by - * hardware but some error conditions require explicit delivery. - * Use the GRU to deliver the interrupt. Otherwise partition failures + * Send a noop message in order to deliver a cross-partition interrupt + * to the SSI that contains the target message queue. Normally, the + * interrupt is automatically delivered by hardware following mesq + * operations, but some error conditions require explicit delivery. + * The noop message will trigger delivery. Otherwise partition failures * could cause unrecovered errors. */ - gpa = uv_global_gru_mmr_address(mqd->interrupt_pnode, UVH_IPI_INT); - save = *val; - *val = uv_hub_ipi_value(mqd->interrupt_apicid, mqd->interrupt_vector, - dest_Fixed); - gru_vstore_phys(cb, gpa, gru_get_tri(mesg), IAA_REGISTER, IMA); - ret = gru_wait(cb); - *val = save; - if (ret != CBS_IDLE) - return MQE_UNEXPECTED_CB_ERR; - return MQE_OK; + do { + ret = send_noop_message(cb, mqd, mesg); + } while ((ret == MQIE_AGAIN || ret == MQE_CONGESTION) && (loops-- > 0)); + + if (ret == MQIE_AGAIN || ret == MQE_CONGESTION) { + /* Don't indicate to the app to resend the message, as it's */ + /* already been successfully sent. We simply send an OK */ + /* (rather than fail the send with MQE_UNEXPECTED_CB_ERR), */ + /* assuming that the other side is receiving enough */ + /* interrupts to get this message processed anyway. */ + ret = MQE_OK; + } + return ret; } /* --