/beegfs/hackathon/users/eoseret/qaas_runs_test/178-663-0018/intel/TeaLeaf/build/TeaLeaf/driver/remote_halo_driver.cpp: 10 - 161
--------------------------------------------------------------------------------

10:   for (int ii = 0; ii < NUM_FIELDS; ++ii) {
11:     if (!settings.fields_to_exchange[ii]) {
12:       continue;
13:     }
14: 
15:     FieldBufferType field;
16:     switch (ii) {
17:       case FIELD_DENSITY: field = chunk->density; break;
18:       case FIELD_ENERGY0: field = chunk->energy0; break;
19:       case FIELD_ENERGY1: field = chunk->energy; break;
20:       case FIELD_U: field = chunk->u; break;
21:       case FIELD_P: field = chunk->p; break;
22:       case FIELD_SD: field = chunk->sd; break;
[...]
34:     if (settings.kernel_language == Kernel_Language::C) {
35:       run_pack_or_unpack(chunk, settings, depth, face, pack, field, buffer, buffer_len);
36:     } else if (settings.kernel_language == Kernel_Language::FORTRAN) {
37:     }
38:     buffer_len += depth * offset;
[...]
45: void remote_halo_driver(Chunk *chunks, Settings &settings, int depth) {
46: #ifndef NO_MPI
47:   // Two sends and two receives
48:   int max_messages = settings.num_chunks_per_rank * 4;
49:   MPI_Request requests[max_messages];
[...]
56:   for (int cc = 0; cc < settings.num_chunks_per_rank; ++cc) {
57:     if (chunks[cc].neighbours[CHUNK_LEFT] != EXTERNAL_FACE) {
58:       int buffer_len = invoke_pack_or_unpack(&(chunks[cc]), settings, CHUNK_LEFT, depth, chunks[cc].y, true, chunks[cc].left_send);
59:       run_send_recv_halo(&chunks[cc], settings,                                      //
60:                          chunks[cc].left_send, chunks[cc].left_recv,                 //
61:                          chunks[cc].staging_left_send, chunks[cc].staging_left_recv, //
62:                          buffer_len, chunks[cc].neighbours[CHUNK_LEFT], 0, 1,        //
63:                          &(requests[num_messages]), &(requests[num_messages + 1]));
64: 
65:       num_messages += 2;
66:     }
67: 
68:     if (chunks[cc].neighbours[CHUNK_RIGHT] != EXTERNAL_FACE) {
69:       int buffer_len = invoke_pack_or_unpack(&(chunks[cc]), settings, CHUNK_RIGHT, depth, chunks[cc].y, true, chunks[cc].right_send);
70:       run_send_recv_halo(&chunks[cc], settings,                                        //
71:                          chunks[cc].right_send, chunks[cc].right_recv,                 //
72:                          chunks[cc].staging_right_send, chunks[cc].staging_right_recv, //
73:                          buffer_len, chunks[cc].neighbours[CHUNK_RIGHT], 1, 0,         //
74:                          &(requests[num_messages]), &(requests[num_messages + 1]));
75: 
76:       num_messages += 2;
77:     }
78:   }
79: 
80:   for (int cc = 0; cc < settings.num_chunks_per_rank; ++cc) {
81:     run_before_waitall_halo(&chunks[cc], settings);
82:   }
83:   wait_for_requests(settings, num_messages, requests);
84:   for (int cc = 0; cc < settings.num_chunks_per_rank; ++cc) {
85:     int buffer_len = 0;
86:     for (int ii = 0; ii < NUM_FIELDS; ++ii) {
87:       if (!settings.fields_to_exchange[ii]) continue;
88:       buffer_len += depth * chunks[cc].y;
89:     }
90:     if (chunks[cc].neighbours[CHUNK_LEFT] != EXTERNAL_FACE)
91:       run_restore_recv_halo(&chunks[cc], settings, chunks[cc].left_recv, chunks[cc].staging_left_recv, buffer_len);
92:     if (chunks[cc].neighbours[CHUNK_RIGHT] != EXTERNAL_FACE)
93:       run_restore_recv_halo(&chunks[cc], settings, chunks[cc].right_recv, chunks[cc].staging_right_recv, buffer_len);
94:   }
95: 
96:   // Unpack lr buffers
97:   for (int cc = 0; cc < settings.num_chunks_per_rank; ++cc) {
98:     if (chunks[cc].neighbours[CHUNK_LEFT] != EXTERNAL_FACE) {
99:       invoke_pack_or_unpack(&(chunks[cc]), settings, CHUNK_LEFT, depth, chunks[cc].y, false, chunks[cc].left_recv);
100:     }
101: 
102:     if (chunks[cc].neighbours[CHUNK_RIGHT] != EXTERNAL_FACE) {
103:       invoke_pack_or_unpack(&(chunks[cc]), settings, CHUNK_RIGHT, depth, chunks[cc].y, false, chunks[cc].right_recv);
[...]
110:   for (int cc = 0; cc < settings.num_chunks_per_rank; ++cc) {
111:     if (chunks[cc].neighbours[CHUNK_BOTTOM] != EXTERNAL_FACE) {
112:       int buffer_len = invoke_pack_or_unpack(&(chunks[cc]), settings, CHUNK_BOTTOM, depth, chunks[cc].x, true, chunks[cc].bottom_send);
113:       run_send_recv_halo(&chunks[cc], settings,                                          //
114:                          chunks[cc].bottom_send, chunks[cc].bottom_recv,                 //
115:                          chunks[cc].staging_bottom_send, chunks[cc].staging_bottom_recv, //
116:                          buffer_len, chunks[cc].neighbours[CHUNK_BOTTOM], 0, 1,          //
117:                          &(requests[num_messages]), &(requests[num_messages + 1]));
118: 
119:       num_messages += 2;
120:     }
121: 
122:     if (chunks[cc].neighbours[CHUNK_TOP] != EXTERNAL_FACE) {
123:       int buffer_len = invoke_pack_or_unpack(&(chunks[cc]), settings, CHUNK_TOP, depth, chunks[cc].x, true, chunks[cc].top_send);
124:       run_send_recv_halo(&chunks[cc], settings,                                    //
125:                          chunks[cc].top_send, chunks[cc].top_recv,                 //
126:                          chunks[cc].staging_top_send, chunks[cc].staging_top_recv, //
127:                          buffer_len, chunks[cc].neighbours[CHUNK_TOP], 1, 0,       //
128:                          &(requests[num_messages]), &(requests[num_messages + 1]));
129: 
130:       num_messages += 2;
131:     }
132:   }
133:   for (int cc = 0; cc < settings.num_chunks_per_rank; ++cc) {
134:     run_before_waitall_halo(&chunks[cc], settings);
135:   }
136:   wait_for_requests(settings, num_messages, requests);
137:   for (int cc = 0; cc < settings.num_chunks_per_rank; ++cc) {
138:     int buffer_len = 0;
139:     for (int ii = 0; ii < NUM_FIELDS; ++ii) {
140:       if (!settings.fields_to_exchange[ii]) continue;
141:       buffer_len += depth * chunks[cc].x;
142:     }
143:     if (chunks[cc].neighbours[CHUNK_BOTTOM] != EXTERNAL_FACE)
144:       run_restore_recv_halo(&chunks[cc], settings, chunks[cc].bottom_recv, chunks[cc].staging_bottom_recv, buffer_len);
145:     if (chunks[cc].neighbours[CHUNK_TOP] != EXTERNAL_FACE)
146:       run_restore_recv_halo(&chunks[cc], settings, chunks[cc].top_recv, chunks[cc].staging_top_recv, buffer_len);
147:   }
148: 
149:   // Unpack tb buffers
150:   for (int cc = 0; cc < settings.num_chunks_per_rank; ++cc) {
151:     if (chunks[cc].neighbours[CHUNK_BOTTOM] != EXTERNAL_FACE) {
152:       invoke_pack_or_unpack(&(chunks[cc]), settings, CHUNK_BOTTOM, depth, chunks[cc].x, false, chunks[cc].bottom_recv);
153:     }
154: 
155:     if (chunks[cc].neighbours[CHUNK_TOP] != EXTERNAL_FACE) {
156:       invoke_pack_or_unpack(&(chunks[cc]), settings, CHUNK_TOP, depth, chunks[cc].x, false, chunks[cc].top_recv);
157:     }
158:   }
159: 
160: #endif
161: }
