From 165a1287424cde3f5a35e9db05066b01bdc73c51 Mon Sep 17 00:00:00 2001
From: Simone Bna <s.bn@cineca.it>
Date: Thu, 19 Nov 2020 18:59:21 +0100
Subject: [PATCH] STY: formatted AmgXConsolidation.cu

---
 src/AmgXConsolidation.cu | 80 +++++++++++++++++++++++++++++++++++-----
 1 file changed, 71 insertions(+), 9 deletions(-)

diff --git a/src/AmgXConsolidation.cu b/src/AmgXConsolidation.cu
index 87e7681..6cb9d93 100644
--- a/src/AmgXConsolidation.cu
+++ b/src/AmgXConsolidation.cu
@@ -32,7 +32,11 @@
     Changes local row offsets to describe the consolidated row space on
     the root rank.
 */
-__global__ void fixConsolidatedRowOffsets(int nLocalRows, int offset, int* rowOffsets)
+__global__ void fixConsolidatedRowOffsets
+(
+    int nLocalRows, 
+    int offset, 
+    int* rowOffsets)
 {
     for(int i = threadIdx.x + blockIdx.x*blockDim.x; i < nLocalRows; i += blockDim.x*gridDim.x)
     {
@@ -51,10 +55,12 @@ struct ConsolidationHandles
 };
 
 /* \implements AmgXSolver::initializeConsolidation */
-PetscErrorCode AmgXSolver::initializeConsolidation(
+PetscErrorCode AmgXSolver::initializeConsolidation
+(
     const PetscInt nLocalRows,
     const PetscInt nLocalNz,
-    const PetscScalar* values)
+    const PetscScalar* values
+)
 {
     PetscFunctionBeginUser;
 
@@ -72,10 +78,26 @@ PetscErrorCode AmgXSolver::initializeConsolidation(
 
     // Fetch to all the number of local rows on each rank
     MPI_Request req[2];
-    int ierr = MPI_Iallgather(&nLocalRows, 1, MPI_INT, nRowsInDevWorld.data(), 1, MPI_INT, devWorld, &req[0]); CHK;
+    int ierr = MPI_Iallgather(
+        &nLocalRows, 
+        1,
+        MPI_INT, 
+        nRowsInDevWorld.data(), 
+        1, 
+        MPI_INT, 
+        devWorld, 
+        &req[0]); CHK;
 
     // Fetch to all the number of non zeros on each rank
-    ierr = MPI_Iallgather(&nLocalNz, 1, MPI_INT, nnzInDevWorld.data(), 1, MPI_INT, devWorld, &req[1]); CHK;
+    ierr = MPI_Iallgather(
+        &nLocalNz, 
+        1, 
+        MPI_INT, 
+        nnzInDevWorld.data(), 
+        1, 
+        MPI_INT, 
+        devWorld, 
+        &req[1]); CHK;
     MPI_Waitall(2, req, MPI_STATUSES_IGNORE);
 
     // Calculate consolidate number of rows, non-zeros, and calculate row, non-zero displacements
@@ -224,9 +246,39 @@ PetscErrorCode AmgXSolver::consolidateMatrix(
     {
         // Gather the matrix data to the root rank for consolidation
         MPI_Request req[3];
-        int ierr = MPI_Igatherv(rowOffsets, nLocalRows, MPI_INT, rowOffsetsCons, nRowsInDevWorld.data(), rowDispls.data(), MPI_INT, 0, devWorld, &req[0]); CHK;
-        ierr = MPI_Igatherv(colIndicesGlobal, nLocalNz, MPI_INT, colIndicesGlobalCons, nnzInDevWorld.data(), nzDispls.data(), MPI_INT, 0, devWorld, &req[1]); CHK;
-        ierr = MPI_Igatherv(values, nLocalNz, MPI_DOUBLE, valuesCons, nnzInDevWorld.data(), nzDispls.data(), MPI_DOUBLE, 0, devWorld, &req[2]); CHK;
+        int ierr = MPI_Igatherv(
+            rowOffsets, 
+            nLocalRows, 
+            MPI_INT, 
+            rowOffsetsCons, 
+            nRowsInDevWorld.data(), 
+            rowDispls.data(), 
+            MPI_INT,
+            0, 
+            devWorld, 
+            &req[0]); CHK;
+        ierr = MPI_Igatherv(
+            colIndicesGlobal, 
+            nLocalNz, 
+            MPI_INT, 
+            colIndicesGlobalCons, 
+            nnzInDevWorld.data(), 
+            nzDispls.data(), 
+            MPI_INT, 
+            0, 
+            devWorld, 
+            &req[1]); CHK;
+        ierr = MPI_Igatherv(
+            values, 
+            nLocalNz, 
+            MPI_DOUBLE, 
+            valuesCons, 
+            nnzInDevWorld.data(), 
+            nzDispls.data(), 
+            MPI_DOUBLE, 
+            0, 
+            devWorld, 
+            &req[2]); CHK;
         MPI_Waitall(3, req, MPI_STATUSES_IGNORE);
 
         if (gpuProc == 0)
@@ -296,7 +348,16 @@ PetscErrorCode AmgXSolver::reconsolidateValues(
     case ConsolidationStatus::Host:
     {
         // Gather the matrix values to the root rank for consolidation
-        int ierr = MPI_Gatherv(values, nLocalNz, MPI_DOUBLE, valuesCons, nnzInDevWorld.data(), nzDispls.data(), MPI_DOUBLE, 0, devWorld); CHK;
+        int ierr = MPI_Gatherv(
+            values, 
+            nLocalNz, 
+            MPI_DOUBLE, 
+            valuesCons, 
+            nnzInDevWorld.data(), 
+            nzDispls.data(), 
+            MPI_DOUBLE, 
+            0, 
+            devWorld); CHK;
         break;
     }
     default:
@@ -426,3 +487,4 @@ PetscErrorCode AmgXSolver::finalizeConsolidation()
 
     PetscFunctionReturn(0);
 }
+
-- 
GitLab