OV - exec - Loop 2488

	Loop Id: 2488	Module: exec	Source: forall.hpp:59-59 [...]	Coverage: 1.64%

0x4ae740 LDR	D3, [X13]    [12]

0x4ae744 ADD	X4, X4, #1

0x4ae748 LDR	D20, [X14]    [11]

0x4ae74c LDR	D6, [X12]    [1]

0x4ae750 FADD	D1, D3, D3

0x4ae754 LDR	D5, [X8]    [7]

0x4ae758 FADD	D21, D20, D20

0x4ae75c LDR	D28, [X15]    [5]

0x4ae760 LDR	D0, [X2]    [8]

0x4ae764 FDIV	D2, D1, D6

0x4ae768 LDR	D16, [X1]    [4]

0x4ae76c FADD	D29, D28, D28

0x4ae770 LDR	D17, [X5]    [6]

0x4ae774 ADD	X5, X5, X6

0x4ae778 LDR	D18, [X10, X0]    [2]

0x4ae77c FDIV	D22, D21, D5

0x4ae780 LDR	D7, [X9, X0]    [9]

0x4ae784 LDR	D27, [X3]    [10]

0x4ae788 FDIV	D30, D29, D17

0x4ae78c FMUL	D19, D2, D0

0x4ae790 FMADD	D23, D22, D16, D19

0x4ae794 FADD	D24, D22, D2

0x4ae798 FADD	D25, D24, D7

0x4ae79c FADD	D26, D23, D18

0x4ae7a0 FADD	D31, D25, D30

0x4ae7a4 FMADD	D7, D30, D27, D26

0x4ae7a8 FDIV	D16, D7, D31

0x4ae7ac STR	D16, [X7, X0]    [3]

0x4ae7b0 ADD	X0, X0, X27

0x4ae7b4 LDR	D0, [X3]    [10]

0x4ae7b8 FNMSUB	D17, D16, D4, D0

0x4ae7bc STR	D17, [X3]    [10]

0x4ae7c0 LDR	D5, [X1]    [4]

0x4ae7c4 FNMSUB	D6, D16, D4, D5

0x4ae7c8 STR	D6, [X1]    [4]

0x4ae7cc ADD	X1, X1, X6

0x4ae7d0 LDR	D3, [X2]    [8]

0x4ae7d4 FNMSUB	D1, D16, D4, D3

0x4ae7d8 STR	D1, [X2]    [8]

0x4ae7dc ADD	X2, X2, X6

0x4ae7e0 CMP	X4, X11

0x4ae7e4 B.NE	4ae740

/home/hbollore/qaas-runs/170-289-7893/intel/Kripke/build/Kripke/src/Kripke/Kernel/SweepSubdomain.cpp: 87 - 105

--------------------------------------------------------------------------------

87:             double xcos_dxi = 2.0 * xcos(d) / dx(i);

88:             double ycos_dyj = 2.0 * ycos(d) / dy(j);

89:             double zcos_dzk = 2.0 * zcos(d) / dz(k);

90:

91:             Zone z(zone_layout(*i, *j, *k));

92:

93:             /* Calculate new zonal flux */

94:             double psi_d_g_z = (rhs(d,g,z)

95:                 + psi_lf(d, g, j, k) * xcos_dxi

96:                 + psi_fr(d, g, i, k) * ycos_dyj

97:                 + psi_bo(d, g, i, j) * zcos_dzk)

98:                 / (xcos_dxi + ycos_dyj + zcos_dzk + sigt(g, z));

99:

100:             psi(d, g, z) = psi_d_g_z;

101:

102:             /* Apply diamond-difference relationships */

103:             psi_lf(d, g, j, k) = 2.0 * psi_d_g_z - psi_lf(d, g, j, k);

104:             psi_fr(d, g, i, k) = 2.0 * psi_d_g_z - psi_fr(d, g, i, k);

105:             psi_bo(d, g, i, j) = 2.0 * psi_d_g_z - psi_bo(d, g, i, j);

/home/hbollore/qaas-runs/170-289-7893/intel/Kripke/build/Kripke/tpl/raja/include/RAJA/policy/loop/forall.hpp: 59 - 59

--------------------------------------------------------------------------------

59:   for (decltype(distance_it) i = 0; i < distance_it; ++i) {

Coverage (%)	Name	Source Location	Module
►98.43+	__kmp_GOMP_microtask_wrapper(i[...]		libomp.so
○	__kmp_invoke_microtask		libomp.so
►1.57+	GOMP_parallel		libomp.so
○	void Kripke::DispatchHelper<Kr[...]	plugins.hpp:31	exec
○	Kripke::Kernel::sweepSubdomain[...]	ArchLayout.h:145	exec
○	Kripke::SweepSolver(Kripke::Co[...]	SweepSolver.cpp:78	exec
○	Kripke::SteadyStateSolver(Krip[...]	stl_vector.h:680	exec
○	main	new_allocator.h:79	exec
○	__libc_start_main		libc-2.31.so
○	_start	iostream:74	exec

Path /

Metric	Value
CQA speedup if no scalar integer	1.00
CQA speedup if FP arith vectorized	1.00
CQA speedup if fully vectorized	3.00 - 4.00
CQA speedup if no inter-iteration dependency	NA
CQA speedup if next bottleneck killed	1.14
Bottlenecks	P10, P11, P12,
Function	void RAJA::internal::StatementExecutor, RAJA::statement::For<2l, RAJA::policy::loop::loop_exec, RAJA::statement::For<3l, RAJA::policy::loop::loop_exec, RAJA::statement::For<4l, RAJA::policy::loop::loop_exec, RAJA::statement::Lambda<0l> > > > > >::exec, RAJA::statement::For<2l, RAJA::policy::loop::loop_exec, RAJA::statement::For<3l, RAJA::policy::loop::loop_exec, RAJA::statement::For<4l, RAJA::policy::loop::loop_exec, RAJA::statement::Lambda<0l> > > > > >, camp::tuple, Kripke::Direction>, RAJA::impl::Span, Kripke::Group>, RAJA::impl::Span, Kripke::ZoneK>, RAJA::impl::Span, Kripke::ZoneJ>, RAJA::impl::Span, Kripke::ZoneI> >, camp::tuple<>, SweepSdom::operator() >(Kripke::ArchLayoutT, Kripke::Core::DataStore&, Kripke::SdomId) const::{lambda(Kripke::Direction, Kripke::Group, Kripke::ZoneK, Kripke::ZoneJ, Kripke::ZoneI)#1}>&>(RAJA::internal::LoopData, RAJA::statement::For<2l, RAJA::policy::loop::loop_exec, RAJA::statement::For<3l, RAJA::policy::loop::loop_exec, RAJA::statement::For<4l, RAJA::policy::loop::loop_exec, RAJA::statement::Lambda<0l> > > > > >, camp::tuple, Kripke::Direction>, RAJA::impl::Span, Kripke::Group>, RAJA::impl::Span, Kripke::ZoneK>, RAJA::impl::Span, Kripke::ZoneJ>, RAJA::impl::Span, Kripke::ZoneI> >, camp::tuple<>, SweepSdom::operator() >(Kripke::ArchLayoutT, Kripke::Core::DataStore&, Kripke::SdomId) const::{lambda(Kripke::Direction, Kripke::Group, Kripke::ZoneK, Kripke::ZoneJ, Kripke::ZoneI)#1}>&) [clone ._omp_fn.0]
Source	SweepSubdomain.cpp:87-105,forall.hpp:59-59
Source loop unroll info	unrolled by 2
Source loop unroll confidence level	max
Unroll/vectorization loop type	NA
Unroll factor	1
CQA cycles	6.00
CQA cycles if no scalar integer	6.00
CQA cycles if FP arith vectorized	6.00
CQA cycles if fully vectorized	2.00 - 1.50
Front-end cycles	5.25
DIV/SQRT cycles	0.50
P0 cycles	0.50
P1 cycles	1.50
P2 cycles	1.50
P3 cycles	1.50
P4 cycles	1.50
P5 cycles	5.25
P6 cycles	5.25
P7 cycles	5.25
P8 cycles	5.25
P9 cycles	6.00
P10 cycles	6.00
P11 cycles	6.00
P12 cycles	0.00
P13 cycles	0.00
P14 cycles	4.00 - 2.00
Inter-iter dependencies cycles	1
FE+BE cycles (UFS)	NA
Stall cycles (UFS)	NA
Nb insns	42.00
Nb uops	42.00
Nb loads	NA
Nb stores	4.00
Nb stack references	0.00
FLOP/cycle	3.67
Nb FLOP add-sub	7.00
Nb FLOP mul	1.00
Nb FLOP fma	5.00
Nb FLOP div	4.00
Nb FLOP rcp	0.00
Nb FLOP sqrt	0.00
Nb FLOP rsqrt	0.00
Bytes/cycle	24.00
Bytes prefetched	0.00
Bytes loaded	112.00
Bytes stored	32.00
Stride 0	NA
Stride 1	NA
Stride n	NA
Stride unknown	NA
Stride indirect	NA
Vectorization ratio all	0.00
Vectorization ratio load	0.00
Vectorization ratio store	0.00
Vectorization ratio mul	NA
Vectorization ratio add_sub	NA
Vectorization ratio fma	NA
Vectorization ratio div_sqrt	NA
Vectorization ratio other	NA
Vector-efficiency ratio all	25.00
Vector-efficiency ratio load	25.00
Vector-efficiency ratio store	25.00
Vector-efficiency ratio mul	NA
Vector-efficiency ratio add_sub	NA
Vector-efficiency ratio fma	NA
Vector-efficiency ratio div_sqrt	NA
Vector-efficiency ratio other	NA

Metric	Value
CQA speedup if no scalar integer	1.00
CQA speedup if FP arith vectorized	1.00
CQA speedup if fully vectorized	3.00 - 4.00
CQA speedup if no inter-iteration dependency	NA
CQA speedup if next bottleneck killed	1.14
Bottlenecks	P10, P11, P12,
Function	void RAJA::internal::StatementExecutor, RAJA::statement::For<2l, RAJA::policy::loop::loop_exec, RAJA::statement::For<3l, RAJA::policy::loop::loop_exec, RAJA::statement::For<4l, RAJA::policy::loop::loop_exec, RAJA::statement::Lambda<0l> > > > > >::exec, RAJA::statement::For<2l, RAJA::policy::loop::loop_exec, RAJA::statement::For<3l, RAJA::policy::loop::loop_exec, RAJA::statement::For<4l, RAJA::policy::loop::loop_exec, RAJA::statement::Lambda<0l> > > > > >, camp::tuple, Kripke::Direction>, RAJA::impl::Span, Kripke::Group>, RAJA::impl::Span, Kripke::ZoneK>, RAJA::impl::Span, Kripke::ZoneJ>, RAJA::impl::Span, Kripke::ZoneI> >, camp::tuple<>, SweepSdom::operator() >(Kripke::ArchLayoutT, Kripke::Core::DataStore&, Kripke::SdomId) const::{lambda(Kripke::Direction, Kripke::Group, Kripke::ZoneK, Kripke::ZoneJ, Kripke::ZoneI)#1}>&>(RAJA::internal::LoopData, RAJA::statement::For<2l, RAJA::policy::loop::loop_exec, RAJA::statement::For<3l, RAJA::policy::loop::loop_exec, RAJA::statement::For<4l, RAJA::policy::loop::loop_exec, RAJA::statement::Lambda<0l> > > > > >, camp::tuple, Kripke::Direction>, RAJA::impl::Span, Kripke::Group>, RAJA::impl::Span, Kripke::ZoneK>, RAJA::impl::Span, Kripke::ZoneJ>, RAJA::impl::Span, Kripke::ZoneI> >, camp::tuple<>, SweepSdom::operator() >(Kripke::ArchLayoutT, Kripke::Core::DataStore&, Kripke::SdomId) const::{lambda(Kripke::Direction, Kripke::Group, Kripke::ZoneK, Kripke::ZoneJ, Kripke::ZoneI)#1}>&) [clone ._omp_fn.0]
Source	SweepSubdomain.cpp:87-105,forall.hpp:59-59
Source loop unroll info	unrolled by 2
Source loop unroll confidence level	max
Unroll/vectorization loop type	NA
Unroll factor	1
CQA cycles	6.00
CQA cycles if no scalar integer	6.00
CQA cycles if FP arith vectorized	6.00
CQA cycles if fully vectorized	2.00 - 1.50
Front-end cycles	5.25
DIV/SQRT cycles	0.50
P0 cycles	0.50
P1 cycles	1.50
P2 cycles	1.50
P3 cycles	1.50
P4 cycles	1.50
P5 cycles	5.25
P6 cycles	5.25
P7 cycles	5.25
P8 cycles	5.25
P9 cycles	6.00
P10 cycles	6.00
P11 cycles	6.00
P12 cycles	0.00
P13 cycles	0.00
P14 cycles	4.00 - 2.00
Inter-iter dependencies cycles	1
FE+BE cycles (UFS)	NA
Stall cycles (UFS)	NA
Nb insns	42.00
Nb uops	42.00
Nb loads	NA
Nb stores	4.00
Nb stack references	0.00
FLOP/cycle	3.67
Nb FLOP add-sub	7.00
Nb FLOP mul	1.00
Nb FLOP fma	5.00
Nb FLOP div	4.00
Nb FLOP rcp	0.00
Nb FLOP sqrt	0.00
Nb FLOP rsqrt	0.00
Bytes/cycle	24.00
Bytes prefetched	0.00
Bytes loaded	112.00
Bytes stored	32.00
Stride 0	NA
Stride 1	NA
Stride n	NA
Stride unknown	NA
Stride indirect	NA
Vectorization ratio all	0.00
Vectorization ratio load	0.00
Vectorization ratio store	0.00
Vectorization ratio mul	NA
Vectorization ratio add_sub	NA
Vectorization ratio fma	NA
Vectorization ratio div_sqrt	NA
Vectorization ratio other	NA
Vector-efficiency ratio all	25.00
Vector-efficiency ratio load	25.00
Vector-efficiency ratio store	25.00
Vector-efficiency ratio mul	NA
Vector-efficiency ratio add_sub	NA
Vector-efficiency ratio fma	NA
Vector-efficiency ratio div_sqrt	NA
Vector-efficiency ratio other	NA

Path /

Average path: Display a virtual path defined by average values of all real paths

Function	void RAJA::internal::StatementExecutor, RAJA::statement::For<2l, RAJA::policy::loop::loop_exec, RAJA::statement::For<3l, RAJA::policy::loop::loop_exec, RAJA::statement::For<4l, RAJA::policy::loop::loop_exec, RAJA::statement::Lambda<0l> > > > > >::exec, RAJA::statement::For<2l, RAJA::policy::loop::loop_exec, RAJA::statement::For<3l, RAJA::policy::loop::loop_exec, RAJA::statement::For<4l, RAJA::policy::loop::loop_exec, RAJA::statement::Lambda<0l> > > > > >, camp::tuple, Kripke::Direction>, RAJA::impl::Span, Kripke::Group>, RAJA::impl::Span, Kripke::ZoneK>, RAJA::impl::Span, Kripke::ZoneJ>, RAJA::impl::Span, Kripke::ZoneI> >, camp::tuple<>, SweepSdom::operator() >(Kripke::ArchLayoutT, Kripke::Core::DataStore&, Kripke::SdomId) const::{lambda(Kripke::Direction, Kripke::Group, Kripke::ZoneK, Kripke::ZoneJ, Kripke::ZoneI)#1}>&>(RAJA::internal::LoopData, RAJA::statement::For<2l, RAJA::policy::loop::loop_exec, RAJA::statement::For<3l, RAJA::policy::loop::loop_exec, RAJA::statement::For<4l, RAJA::policy::loop::loop_exec, RAJA::statement::Lambda<0l> > > > > >, camp::tuple, Kripke::Direction>, RAJA::impl::Span, Kripke::Group>, RAJA::impl::Span, Kripke::ZoneK>, RAJA::impl::Span, Kripke::ZoneJ>, RAJA::impl::Span, Kripke::ZoneI> >, camp::tuple<>, SweepSdom::operator() >(Kripke::ArchLayoutT, Kripke::Core::DataStore&, Kripke::SdomId) const::{lambda(Kripke::Direction, Kripke::Group, Kripke::ZoneK, Kripke::ZoneJ, Kripke::ZoneI)#1}>&) [clone ._omp_fn.0]
Source file and lines	forall.hpp:59-59
Module	exec

The loop is defined in:

/home/hbollore/qaas-runs/170-289-7893/intel/Kripke/build/Kripke/src/Kripke/Kernel/SweepSubdomain.cpp:87-105
/home/hbollore/qaas-runs/170-289-7893/intel/Kripke/build/Kripke/tpl/raja/include/RAJA/policy/loop/forall.hpp:59

The related source loop is unrolled by 2 (including vectorization).

gain
potential
hint
expert

Unrolling/vectorization cost

This loop is peel/tail of a unrolled/vectorized loop. If its cost is not negligible compared to the main (unrolled/vectorized) loop, unrolling/vectorization is counterproductive due to low trip count.

Details

The more iterations the main loop is processing, the higher the trip count must be to amortize peel/tail overhead.

Workaround

recompile with -fprofile-generate, execute and recompile with -fprofile-use (profile-guided optimization)
hardcode most frequent values of loop bounds by adding specialized paths.:
- For instance, replace for (i=0; i<n; i++) foo(i) with: switch (n) { case (4): for (i=0; i<4; i++) foo(i); break; case (6): for (i=0; i<6; i++) foo(i); break; default : for (i=0; i<n; i++) foo(i); break; }

Vectorization

Your loop is not vectorized. 4 data elements could be processed at once in vector registers.

Details

All VPU instructions are used in scalar version (process only one data element in vector registers).

FMA

Detected 5 FMA (fused multiply-add) operations. Presence of both ADD/SUB and MUL operations.

Workaround

Try to change order in which elements are evaluated (using parentheses) in arithmetic expressions containing both ADD/SUB and MUL operations to enable your compiler to generate FMA instructions wherever possible. For instance a + b*c is a valid FMA (MUL then ADD). However (a+b)* c cannot be translated into an FMA (ADD then MUL).

Matching between your loop (in the source code) and the binary loop

The binary loop is composed of 22 FP arithmetical operations:

12: addition or subtraction (5 inside FMA instructions)
6: multiply (5 inside FMA instructions)
4: divide

The binary loop is loading 112 bytes. The binary loop is storing 32 bytes.

Arithmetic intensity

Arithmetic intensity is 0.15 FP operations per loaded or stored byte.

General properties

nb instructions	42
loop length	168
nb stack references	0

Front-end

FIT IN UOP CACHE

front end

5.25 cycles

Back-end

	P0	P1	P2	P3	P4	P5	P6	P7	P8	P9	P10	P11	P12	P13	P14
uops	0.50	0.50	1.50	1.50	1.50	1.50	5.25	5.25	5.25	5.25	6.00	6.00	6.00	0.00	0.00
cycles	0.50	0.50	1.50	1.50	1.50	1.50	5.25	5.25	5.25	5.25	6.00	6.00	6.00	0.00	0.00

Cycles executing div or sqrt instructions	4.00-2.00
Longest recurrence chain latency (RecMII)	1.00

Cycles summary

Front-end	5.25
Data deps.	1.00
Overall L1	6.00

Vectorization ratios

all	0%
load	0%
store	0%
mul	NA (no mul vectorizable/vectorized instructions)
add-sub	NA (no add-sub vectorizable/vectorized instructions)
fma	NA (no fma vectorizable/vectorized instructions)
div/sqrt	NA (no div/sqrt vectorizable/vectorized instructions)
other	NA (no other vectorizable/vectorized instructions)

Cycles and memory resources usage

Assuming all data fit into the L1 cache, each iteration of the binary loop takes 6.00 cycles. At this rate:

19% of peak load performance is reached (18.67 out of 96.00 bytes loaded per cycle (GB/s @ 1GHz))
11% of peak store performance is reached (5.33 out of 48.00 bytes stored per cycle (GB/s @ 1GHz))

ASM code

In the binary file, the address of the loop is: 4ae740

Instruction	Nb FU	P0	P1	P2	P3	P4	P5	P6	P7	P8	P9	P10	P11	P12	Latency	Recip. throughput
LDR D3, [X13]	1	0	0	0	0	0	0	0	0	0	0	0.33	0.33	0.33	6	0.33
ADD X4, X4, #1	1	0	0	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	1	0.25
LDR D20, [X14]	1	0	0	0	0	0	0	0	0	0	0	0.33	0.33	0.33	6	0.33
LDR D6, [X12]	1	0	0	0	0	0	0	0	0	0	0	0.33	0.33	0.33	6	0.33
FADD D1, D3, D3	1	0	0	0	0	0	0	0.25	0.25	0.25	0.25	0	0	0	2	0.25
LDR D5, [X8]	1	0	0	0	0	0	0	0	0	0	0	0.33	0.33	0.33	6	0.33
FADD D21, D20, D20	1	0	0	0	0	0	0	0.25	0.25	0.25	0.25	0	0	0	2	0.25
LDR D28, [X15]	1	0	0	0	0	0	0	0	0	0	0	0.33	0.33	0.33	6	0.33
LDR D0, [X2]	1	0	0	0	0	0	0	0	0	0	0	0.33	0.33	0.33	6	0.33
FDIV D2, D1, D6	1	0	0	0	0	0	0	0.50	0	0.50	0	0	0	0	7-15	1-0.50
LDR D16, [X1]	1	0	0	0	0	0	0	0	0	0	0	0.33	0.33	0.33	6	0.33
FADD D29, D28, D28	1	0	0	0	0	0	0	0.25	0.25	0.25	0.25	0	0	0	2	0.25
LDR D17, [X5]	1	0	0	0	0	0	0	0	0	0	0	0.33	0.33	0.33	6	0.33
ADD X5, X5, X6	1	0	0	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	1	0.25
LDR D18, [X10, X0]	1	0	0	0	0	0	0	0	0	0	0	0.33	0.33	0.33	6	0.33
FDIV D22, D21, D5	1	0	0	0	0	0	0	0.50	0	0.50	0	0	0	0	7-15	1-0.50
LDR D7, [X9, X0]	1	0	0	0	0	0	0	0	0	0	0	0.33	0.33	0.33	6	0.33
LDR D27, [X3]	1	0	0	0	0	0	0	0	0	0	0	0.33	0.33	0.33	6	0.33
FDIV D30, D29, D17	1	0	0	0	0	0	0	0.50	0	0.50	0	0	0	0	7-15	1-0.50
FMUL D19, D2, D0	1	0	0	0	0	0	0	0.25	0.25	0.25	0.25	0	0	0	3	0.25
FMADD D23, D22, D16, D19	1	0	0	0	0	0	0	0.25	0.25	0.25	0.25	0	0	0	4	0.25
FADD D24, D22, D2	1	0	0	0	0	0	0	0.25	0.25	0.25	0.25	0	0	0	2	0.25
FADD D25, D24, D7	1	0	0	0	0	0	0	0.25	0.25	0.25	0.25	0	0	0	2	0.25
FADD D26, D23, D18	1	0	0	0	0	0	0	0.25	0.25	0.25	0.25	0	0	0	2	0.25
FADD D31, D25, D30	1	0	0	0	0	0	0	0.25	0.25	0.25	0.25	0	0	0	2	0.25
FMADD D7, D30, D27, D26	1	0	0	0	0	0	0	0.25	0.25	0.25	0.25	0	0	0	4	0.25
FDIV D16, D7, D31	1	0	0	0	0	0	0	0.50	0	0.50	0	0	0	0	7-15	1-0.50
STR D16, [X7, X0]	1	0	0	0	0	0	0	0.50	0.50	0	0	0.50	0.50	0	2	0.50
ADD X0, X0, X27	1	0	0	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	1	0.25
LDR D0, [X3]	1	0	0	0	0	0	0	0	0	0	0	0.33	0.33	0.33	6	0.33
FNMSUB D17, D16, D4, D0	1	0	0	0	0	0	0	0.25	0.25	0.25	0.25	0	0	0	4	0.25
STR D17, [X3]	1	0	0	0	0	0	0	0.50	0.50	0	0	0.50	0.50	0	2	0.50
LDR D5, [X1]	1	0	0	0	0	0	0	0	0	0	0	0.33	0.33	0.33	6	0.33
FNMSUB D6, D16, D4, D5	1	0	0	0	0	0	0	0.25	0.25	0.25	0.25	0	0	0	4	0.25
STR D6, [X1]	1	0	0	0	0	0	0	0.50	0.50	0	0	0.50	0.50	0	2	0.50
ADD X1, X1, X6	1	0	0	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	1	0.25
LDR D3, [X2]	1	0	0	0	0	0	0	0	0	0	0	0.33	0.33	0.33	6	0.33
FNMSUB D1, D16, D4, D3	1	0	0	0	0	0	0	0.25	0.25	0.25	0.25	0	0	0	4	0.25
STR D1, [X2]	1	0	0	0	0	0	0	0.50	0.50	0	0	0.50	0.50	0	2	0.50
ADD X2, X2, X6	1	0	0	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	1	0.25
CMP X4, X11	1	0	0	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	1	0.33
B.NE 4ae740 <_ZN4RAJA8internal17StatementExecutorINS_9statement8CollapseINS_26omp_parallel_collapse_execEN4camp7int_seqIlJLl0ELl1EEEEJNS2_3ForILl2ENS_6policy4loop9loop_execEJNS8_ILl3ESB_JNS8_ILl4ESB_JNS2_6LambdaILl0EJEEEEEEEEEEEEEEEE4execIRNS0_8LoopDataINS5_4listIJSH_EEENS5_5tupleIJNS_4impl4SpanINS_9Iterators16numeric_iteratorIN6Kripke9DirectionElPST_EEST_EENSP_INSR_INSS_5GroupElPSX_EESX_EENSP_INSQ_24strided_numeric_iteratorINSS_5ZoneKElPS12_EES12_EENSP_INS11_INSS_5ZoneJElPS16_EES16_EENSP_INS11_INSS_5ZoneIElPS1A_EES1A_EEEEENSN_IJEEEJZNK9SweepSdomclINSS_11ArchLayoutTINSS_12ArchT_OpenMPENSS_11LayoutT_DGZEEEEEvT_RNSS_4Core9DataStoreENSS_6SdomIdEEUlST_SX_S12_S16_S1A_E_EEEEEvOS1M_._omp_fn.0+0x87c>	1	0.50	0.50	0	0	0	0	0	0	0	0	0	0	0	1	0.50

Function	void RAJA::internal::StatementExecutor, RAJA::statement::For<2l, RAJA::policy::loop::loop_exec, RAJA::statement::For<3l, RAJA::policy::loop::loop_exec, RAJA::statement::For<4l, RAJA::policy::loop::loop_exec, RAJA::statement::Lambda<0l> > > > > >::exec, RAJA::statement::For<2l, RAJA::policy::loop::loop_exec, RAJA::statement::For<3l, RAJA::policy::loop::loop_exec, RAJA::statement::For<4l, RAJA::policy::loop::loop_exec, RAJA::statement::Lambda<0l> > > > > >, camp::tuple, Kripke::Direction>, RAJA::impl::Span, Kripke::Group>, RAJA::impl::Span, Kripke::ZoneK>, RAJA::impl::Span, Kripke::ZoneJ>, RAJA::impl::Span, Kripke::ZoneI> >, camp::tuple<>, SweepSdom::operator() >(Kripke::ArchLayoutT, Kripke::Core::DataStore&, Kripke::SdomId) const::{lambda(Kripke::Direction, Kripke::Group, Kripke::ZoneK, Kripke::ZoneJ, Kripke::ZoneI)#1}>&>(RAJA::internal::LoopData, RAJA::statement::For<2l, RAJA::policy::loop::loop_exec, RAJA::statement::For<3l, RAJA::policy::loop::loop_exec, RAJA::statement::For<4l, RAJA::policy::loop::loop_exec, RAJA::statement::Lambda<0l> > > > > >, camp::tuple, Kripke::Direction>, RAJA::impl::Span, Kripke::Group>, RAJA::impl::Span, Kripke::ZoneK>, RAJA::impl::Span, Kripke::ZoneJ>, RAJA::impl::Span, Kripke::ZoneI> >, camp::tuple<>, SweepSdom::operator() >(Kripke::ArchLayoutT, Kripke::Core::DataStore&, Kripke::SdomId) const::{lambda(Kripke::Direction, Kripke::Group, Kripke::ZoneK, Kripke::ZoneJ, Kripke::ZoneI)#1}>&) [clone ._omp_fn.0]
Source file and lines	forall.hpp:59-59
Module	exec

The loop is defined in:

/home/hbollore/qaas-runs/170-289-7893/intel/Kripke/build/Kripke/src/Kripke/Kernel/SweepSubdomain.cpp:87-105
/home/hbollore/qaas-runs/170-289-7893/intel/Kripke/build/Kripke/tpl/raja/include/RAJA/policy/loop/forall.hpp:59

The related source loop is unrolled by 2 (including vectorization).

gain
potential
hint
expert

Unrolling/vectorization cost

Details

The more iterations the main loop is processing, the higher the trip count must be to amortize peel/tail overhead.

Workaround

recompile with -fprofile-generate, execute and recompile with -fprofile-use (profile-guided optimization)
hardcode most frequent values of loop bounds by adding specialized paths.:
- For instance, replace for (i=0; i<n; i++) foo(i) with: switch (n) { case (4): for (i=0; i<4; i++) foo(i); break; case (6): for (i=0; i<6; i++) foo(i); break; default : for (i=0; i<n; i++) foo(i); break; }

Vectorization

Your loop is not vectorized. 4 data elements could be processed at once in vector registers.

Details

All VPU instructions are used in scalar version (process only one data element in vector registers).

FMA

Detected 5 FMA (fused multiply-add) operations. Presence of both ADD/SUB and MUL operations.

Workaround

Matching between your loop (in the source code) and the binary loop

The binary loop is composed of 22 FP arithmetical operations:

12: addition or subtraction (5 inside FMA instructions)
6: multiply (5 inside FMA instructions)
4: divide

The binary loop is loading 112 bytes. The binary loop is storing 32 bytes.

Arithmetic intensity

Arithmetic intensity is 0.15 FP operations per loaded or stored byte.

General properties

nb instructions	42
loop length	168
nb stack references	0

Front-end

FIT IN UOP CACHE

front end

5.25 cycles

Back-end

	P0	P1	P2	P3	P4	P5	P6	P7	P8	P9	P10	P11	P12	P13	P14
uops	0.50	0.50	1.50	1.50	1.50	1.50	5.25	5.25	5.25	5.25	6.00	6.00	6.00	0.00	0.00
cycles	0.50	0.50	1.50	1.50	1.50	1.50	5.25	5.25	5.25	5.25	6.00	6.00	6.00	0.00	0.00

Cycles executing div or sqrt instructions	4.00-2.00
Longest recurrence chain latency (RecMII)	1.00

Cycles summary

Front-end	5.25
Data deps.	1.00
Overall L1	6.00

Vectorization ratios

all	0%
load	0%
store	0%
mul	NA (no mul vectorizable/vectorized instructions)
add-sub	NA (no add-sub vectorizable/vectorized instructions)
fma	NA (no fma vectorizable/vectorized instructions)
div/sqrt	NA (no div/sqrt vectorizable/vectorized instructions)
other	NA (no other vectorizable/vectorized instructions)

Cycles and memory resources usage

Assuming all data fit into the L1 cache, each iteration of the binary loop takes 6.00 cycles. At this rate:

19% of peak load performance is reached (18.67 out of 96.00 bytes loaded per cycle (GB/s @ 1GHz))
11% of peak store performance is reached (5.33 out of 48.00 bytes stored per cycle (GB/s @ 1GHz))

ASM code

In the binary file, the address of the loop is: 4ae740

Instruction	Nb FU	P0	P1	P2	P3	P4	P5	P6	P7	P8	P9	P10	P11	P12	Latency	Recip. throughput
LDR D3, [X13]	1	0	0	0	0	0	0	0	0	0	0	0.33	0.33	0.33	6	0.33
ADD X4, X4, #1	1	0	0	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	1	0.25
LDR D20, [X14]	1	0	0	0	0	0	0	0	0	0	0	0.33	0.33	0.33	6	0.33
LDR D6, [X12]	1	0	0	0	0	0	0	0	0	0	0	0.33	0.33	0.33	6	0.33
FADD D1, D3, D3	1	0	0	0	0	0	0	0.25	0.25	0.25	0.25	0	0	0	2	0.25
LDR D5, [X8]	1	0	0	0	0	0	0	0	0	0	0	0.33	0.33	0.33	6	0.33
FADD D21, D20, D20	1	0	0	0	0	0	0	0.25	0.25	0.25	0.25	0	0	0	2	0.25
LDR D28, [X15]	1	0	0	0	0	0	0	0	0	0	0	0.33	0.33	0.33	6	0.33
LDR D0, [X2]	1	0	0	0	0	0	0	0	0	0	0	0.33	0.33	0.33	6	0.33
FDIV D2, D1, D6	1	0	0	0	0	0	0	0.50	0	0.50	0	0	0	0	7-15	1-0.50
LDR D16, [X1]	1	0	0	0	0	0	0	0	0	0	0	0.33	0.33	0.33	6	0.33
FADD D29, D28, D28	1	0	0	0	0	0	0	0.25	0.25	0.25	0.25	0	0	0	2	0.25
LDR D17, [X5]	1	0	0	0	0	0	0	0	0	0	0	0.33	0.33	0.33	6	0.33
ADD X5, X5, X6	1	0	0	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	1	0.25
LDR D18, [X10, X0]	1	0	0	0	0	0	0	0	0	0	0	0.33	0.33	0.33	6	0.33
FDIV D22, D21, D5	1	0	0	0	0	0	0	0.50	0	0.50	0	0	0	0	7-15	1-0.50
LDR D7, [X9, X0]	1	0	0	0	0	0	0	0	0	0	0	0.33	0.33	0.33	6	0.33
LDR D27, [X3]	1	0	0	0	0	0	0	0	0	0	0	0.33	0.33	0.33	6	0.33
FDIV D30, D29, D17	1	0	0	0	0	0	0	0.50	0	0.50	0	0	0	0	7-15	1-0.50
FMUL D19, D2, D0	1	0	0	0	0	0	0	0.25	0.25	0.25	0.25	0	0	0	3	0.25
FMADD D23, D22, D16, D19	1	0	0	0	0	0	0	0.25	0.25	0.25	0.25	0	0	0	4	0.25
FADD D24, D22, D2	1	0	0	0	0	0	0	0.25	0.25	0.25	0.25	0	0	0	2	0.25
FADD D25, D24, D7	1	0	0	0	0	0	0	0.25	0.25	0.25	0.25	0	0	0	2	0.25
FADD D26, D23, D18	1	0	0	0	0	0	0	0.25	0.25	0.25	0.25	0	0	0	2	0.25
FADD D31, D25, D30	1	0	0	0	0	0	0	0.25	0.25	0.25	0.25	0	0	0	2	0.25
FMADD D7, D30, D27, D26	1	0	0	0	0	0	0	0.25	0.25	0.25	0.25	0	0	0	4	0.25
FDIV D16, D7, D31	1	0	0	0	0	0	0	0.50	0	0.50	0	0	0	0	7-15	1-0.50
STR D16, [X7, X0]	1	0	0	0	0	0	0	0.50	0.50	0	0	0.50	0.50	0	2	0.50
ADD X0, X0, X27	1	0	0	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	1	0.25
LDR D0, [X3]	1	0	0	0	0	0	0	0	0	0	0	0.33	0.33	0.33	6	0.33
FNMSUB D17, D16, D4, D0	1	0	0	0	0	0	0	0.25	0.25	0.25	0.25	0	0	0	4	0.25
STR D17, [X3]	1	0	0	0	0	0	0	0.50	0.50	0	0	0.50	0.50	0	2	0.50
LDR D5, [X1]	1	0	0	0	0	0	0	0	0	0	0	0.33	0.33	0.33	6	0.33
FNMSUB D6, D16, D4, D5	1	0	0	0	0	0	0	0.25	0.25	0.25	0.25	0	0	0	4	0.25
STR D6, [X1]	1	0	0	0	0	0	0	0.50	0.50	0	0	0.50	0.50	0	2	0.50
ADD X1, X1, X6	1	0	0	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	1	0.25
LDR D3, [X2]	1	0	0	0	0	0	0	0	0	0	0	0.33	0.33	0.33	6	0.33
FNMSUB D1, D16, D4, D3	1	0	0	0	0	0	0	0.25	0.25	0.25	0.25	0	0	0	4	0.25
STR D1, [X2]	1	0	0	0	0	0	0	0.50	0.50	0	0	0.50	0.50	0	2	0.50
ADD X2, X2, X6	1	0	0	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	1	0.25
CMP X4, X11	1	0	0	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	1	0.33
B.NE 4ae740 <_ZN4RAJA8internal17StatementExecutorINS_9statement8CollapseINS_26omp_parallel_collapse_execEN4camp7int_seqIlJLl0ELl1EEEEJNS2_3ForILl2ENS_6policy4loop9loop_execEJNS8_ILl3ESB_JNS8_ILl4ESB_JNS2_6LambdaILl0EJEEEEEEEEEEEEEEEE4execIRNS0_8LoopDataINS5_4listIJSH_EEENS5_5tupleIJNS_4impl4SpanINS_9Iterators16numeric_iteratorIN6Kripke9DirectionElPST_EEST_EENSP_INSR_INSS_5GroupElPSX_EESX_EENSP_INSQ_24strided_numeric_iteratorINSS_5ZoneKElPS12_EES12_EENSP_INS11_INSS_5ZoneJElPS16_EES16_EENSP_INS11_INSS_5ZoneIElPS1A_EES1A_EEEEENSN_IJEEEJZNK9SweepSdomclINSS_11ArchLayoutTINSS_12ArchT_OpenMPENSS_11LayoutT_DGZEEEEEvT_RNSS_4Core9DataStoreENSS_6SdomIdEEUlST_SX_S12_S16_S1A_E_EEEEEvOS1M_._omp_fn.0+0x87c>	1	0.50	0.50	0	0	0	0	0	0	0	0	0	0	0	1	0.50

Report Configuration

Unrolling/vectorization cost

Details

Workaround

Vectorization

Details

FMA

Workaround

Matching between your loop (in the source code) and the binary loop

Arithmetic intensity

General properties

Front-end

Back-end

Cycles summary

Vectorization ratios

Cycles and memory resources usage

ASM code

Unrolling/vectorization cost

Details

Workaround

Vectorization

Details

FMA

Workaround

Matching between your loop (in the source code) and the binary loop

Arithmetic intensity

General properties

Front-end

Back-end

Cycles summary

Vectorization ratios

Cycles and memory resources usage

ASM code