OV - - Loop 1075 - libkripke.so

	Loop Id: 1075	Module: libkripke.so	Source: forall.hpp:59-59 [...]	Coverage: 1.69%

0x7fea0 MOV	-0x178(%RBP),%RAX    [12]

0x7fea7 VMOVSD	(%RAX,%R8,8),%XMM1    [13]

0x7fead MOV	-0x170(%RBP),%RAX    [12]

0x7feb4 VADDSD	%XMM1,%XMM1,%XMM1

0x7feb8 VDIVSD	(%RAX,%R15,8),%XMM1,%XMM1    [1]

0x7febe MOV	-0x180(%RBP),%RAX    [12]

0x7fec5 VMOVSD	(%RAX,%R8,8),%XMM2    [11]

0x7fecb MOV	-0x168(%RBP),%RAX    [12]

0x7fed2 VMOVHPD	(%RAX,%R8,8),%XMM2,%XMM2    [6]

0x7fed8 MOV	-0x50(%RBP),%RAX    [12]

0x7fedc VMOVSD	(%RAX,%R10,1),%XMM3    [5]

0x7fee2 MOV	-0x160(%RBP),%RAX    [12]

0x7fee9 VADDPD	%XMM2,%XMM2,%XMM2

0x7feed VMOVHPD	(%RAX,%RCX,8),%XMM3,%XMM3    [3]

0x7fef2 VDIVPD	%XMM3,%XMM2,%XMM2

0x7fef6 VMOVSD	(%R14,%R12,8),%XMM3    [10]

0x7fefc VFMADD213SD	(%RDX,%R13,1),%XMM2,%XMM3    [7]

0x7ff02 VPERMILPD	$0x1,%XMM2,%XMM4

0x7ff08 VFMADD231SD	(%RSI,%R10,1),%XMM1,%XMM3    [2]

0x7ff0e VADDSD	%XMM1,%XMM2,%XMM1

0x7ff12 VADDSD	%XMM4,%XMM1,%XMM1

0x7ff16 VADDSD	(%RDI,%R13,1),%XMM1,%XMM1    [8]

0x7ff1c VFMADD231SD	(%RBX,%R10,1),%XMM4,%XMM3    [4]

0x7ff22 VDIVSD	%XMM1,%XMM3,%XMM1

0x7ff26 VMOVSD	%XMM1,(%R9,%R13,1)    [9]

0x7ff2c ADD	-0x58(%RBP),%R13    [12]

0x7ff30 VMOVSD	(%R14,%R12,8),%XMM2    [10]

0x7ff36 VFMSUB231SD	%XMM0,%XMM1,%XMM2

0x7ff3b VMOVSD	%XMM2,(%R14,%R12,8)    [10]

0x7ff41 VMOVSD	(%RSI,%R10,1),%XMM2    [2]

0x7ff47 VFMSUB231SD	%XMM0,%XMM1,%XMM2

0x7ff4c VMOVSD	%XMM2,(%RSI,%R10,1)    [2]

0x7ff52 VFMSUB213SD	(%RBX,%R10,1),%XMM0,%XMM1    [4]

0x7ff58 VMOVSD	%XMM1,(%RBX,%R10,1)    [4]

0x7ff5e ADD	-0x48(%RBP),%R10    [12]

0x7ff62 DEC	%R11

0x7ff65 JNE	7fea0

/beegfs/hackathon/users/eoseret/qaas_runs/170-850-6313/intel/Kripke/build/Kripke/src/Kripke/Kernel/SweepSubdomain.cpp: 87 - 105

--------------------------------------------------------------------------------

87:             double xcos_dxi = 2.0 * xcos(d) / dx(i);

88:             double ycos_dyj = 2.0 * ycos(d) / dy(j);

89:             double zcos_dzk = 2.0 * zcos(d) / dz(k);

[...]

95:                 + psi_lf(d, g, j, k) * xcos_dxi

96:                 + psi_fr(d, g, i, k) * ycos_dyj

97:                 + psi_bo(d, g, i, j) * zcos_dzk)

98:                 / (xcos_dxi + ycos_dyj + zcos_dzk + sigt(g, z));

99:

100:             psi(d, g, z) = psi_d_g_z;

101:

102:             /* Apply diamond-difference relationships */

103:             psi_lf(d, g, j, k) = 2.0 * psi_d_g_z - psi_lf(d, g, j, k);

104:             psi_fr(d, g, i, k) = 2.0 * psi_d_g_z - psi_fr(d, g, i, k);

105:             psi_bo(d, g, i, j) = 2.0 * psi_d_g_z - psi_bo(d, g, i, j);

/beegfs/hackathon/users/eoseret/qaas_runs/170-850-6313/intel/Kripke/build/Kripke/tpl/raja/include/RAJA/policy/loop/forall.hpp: 59 - 59

--------------------------------------------------------------------------------

59:   for (decltype(distance_it) i = 0; i < distance_it; ++i) {

Path /

Metric	Value
CQA speedup if no scalar integer	1.00
CQA speedup if FP arith vectorized	2.25
CQA speedup if fully vectorized	3.00
CQA speedup if no inter-iteration dependency	NA
CQA speedup if next bottleneck killed	1.73
Bottlenecks	P8, P9,
Function	.omp_outlined..239
Source	SweepSubdomain.cpp:87-89,SweepSubdomain.cpp:95-105,forall.hpp:59-59
Source loop unroll info	multi-versionned
Source loop unroll confidence level	max
Unroll/vectorization loop type	NA
Unroll factor	NA
CQA cycles	15.00
CQA cycles if no scalar integer	15.00
CQA cycles if FP arith vectorized	6.67
CQA cycles if fully vectorized	5.00
Front-end cycles	6.00
DIV/SQRT cycles	0.75
P0 cycles	0.75
P1 cycles	0.50
P2 cycles	0.50
P3 cycles	0.50
P4 cycles	8.67
P5 cycles	8.67
P6 cycles	8.67
P7 cycles	4.50
P8 cycles	4.50
P9 cycles	4.17
P10 cycles	3.83
P11 cycles	2.00
P12 cycles	2.00
P13 cycles	15.00
Inter-iter dependencies cycles	1
FE+BE cycles (UFS)	NA
Stall cycles (UFS)	NA
Nb insns	37.00
Nb uops	36.00
Nb loads	22.00
Nb stores	4.00
Nb stack references	8.00
FLOP/cycle	1.47
Nb FLOP add-sub	6.00
Nb FLOP mul	0.00
Nb FLOP fma	6.00
Nb FLOP div	4.00
Nb FLOP rcp	0.00
Nb FLOP sqrt	0.00
Nb FLOP rsqrt	0.00
Bytes/cycle	13.87
Bytes prefetched	0.00
Bytes loaded	176.00
Bytes stored	32.00
Stride 0	2.00
Stride 1	0.00
Stride n	0.00
Stride unknown	0.00
Stride indirect	9.00
Vectorization ratio all	11.11
Vectorization ratio load	0.00
Vectorization ratio store	0.00
Vectorization ratio mul	NA
Vectorization ratio add_sub	20.00
Vectorization ratio fma	0.00
Vectorization ratio div_sqrt	33.33
Vectorization ratio other	100.00
Vector-efficiency ratio all	13.89
Vector-efficiency ratio load	12.50
Vector-efficiency ratio store	12.50
Vector-efficiency ratio mul	NA
Vector-efficiency ratio add_sub	15.00
Vector-efficiency ratio fma	12.50
Vector-efficiency ratio div_sqrt	16.67
Vector-efficiency ratio other	25.00

Metric	Value
CQA speedup if no scalar integer	1.00
CQA speedup if FP arith vectorized	2.25
CQA speedup if fully vectorized	3.00
CQA speedup if no inter-iteration dependency	NA
CQA speedup if next bottleneck killed	1.73
Bottlenecks	P8, P9,
Function	.omp_outlined..239
Source	SweepSubdomain.cpp:87-89,SweepSubdomain.cpp:95-105,forall.hpp:59-59
Source loop unroll info	multi-versionned
Source loop unroll confidence level	max
Unroll/vectorization loop type	NA
Unroll factor	NA
CQA cycles	15.00
CQA cycles if no scalar integer	15.00
CQA cycles if FP arith vectorized	6.67
CQA cycles if fully vectorized	5.00
Front-end cycles	6.00
DIV/SQRT cycles	0.75
P0 cycles	0.75
P1 cycles	0.50
P2 cycles	0.50
P3 cycles	0.50
P4 cycles	8.67
P5 cycles	8.67
P6 cycles	8.67
P7 cycles	4.50
P8 cycles	4.50
P9 cycles	4.17
P10 cycles	3.83
P11 cycles	2.00
P12 cycles	2.00
P13 cycles	15.00
Inter-iter dependencies cycles	1
FE+BE cycles (UFS)	NA
Stall cycles (UFS)	NA
Nb insns	37.00
Nb uops	36.00
Nb loads	22.00
Nb stores	4.00
Nb stack references	8.00
FLOP/cycle	1.47
Nb FLOP add-sub	6.00
Nb FLOP mul	0.00
Nb FLOP fma	6.00
Nb FLOP div	4.00
Nb FLOP rcp	0.00
Nb FLOP sqrt	0.00
Nb FLOP rsqrt	0.00
Bytes/cycle	13.87
Bytes prefetched	0.00
Bytes loaded	176.00
Bytes stored	32.00
Stride 0	2.00
Stride 1	0.00
Stride n	0.00
Stride unknown	0.00
Stride indirect	9.00
Vectorization ratio all	11.11
Vectorization ratio load	0.00
Vectorization ratio store	0.00
Vectorization ratio mul	NA
Vectorization ratio add_sub	20.00
Vectorization ratio fma	0.00
Vectorization ratio div_sqrt	33.33
Vectorization ratio other	100.00
Vector-efficiency ratio all	13.89
Vector-efficiency ratio load	12.50
Vector-efficiency ratio store	12.50
Vector-efficiency ratio mul	NA
Vector-efficiency ratio add_sub	15.00
Vector-efficiency ratio fma	12.50
Vector-efficiency ratio div_sqrt	16.67
Vector-efficiency ratio other	25.00

Path /

Average path: Display a virtual path defined by average values of all real paths

Function	.omp_outlined..239
Source file and lines	forall.hpp:59-59
Module	libkripke.so

The loop is defined in:

/beegfs/hackathon/users/eoseret/qaas_runs/170-850-6313/intel/Kripke/build/Kripke/src/Kripke/Kernel/SweepSubdomain.cpp:87-89,95-105
/beegfs/hackathon/users/eoseret/qaas_runs/170-850-6313/intel/Kripke/build/Kripke/tpl/raja/include/RAJA/policy/loop/forall.hpp:59

The related source loop is multi-versionned.

gain
potential
hint
expert

Vectorization

Your loop is not vectorized. Only 13% of vector register length is used (average across all SSE/AVX instructions). By fully vectorizing your loop, you can lower the cost of an iteration from 15.00 to 5.00 cycles (3.00x speedup).

Details

11% of SSE/AVX instructions are used in vector version (process two or more data elements in vector registers):

0% of SSE/AVX loads are used in vector version.
0% of SSE/AVX stores are used in vector version.
20% of SSE/AVX addition or subtraction instructions are used in vector version.
0% of SSE/AVX fused multiply-add instructions are used in vector version.
33% of SSE/AVX divide and square root instructions are used in vector version.

Since your execution units are vector units, only a fully vectorized loop can use their full power.

Workaround

Try another compiler or update/tune your current one:
- recompile with ffast-math (included in Ofast) to extend loop vectorization to FP reductions.
Remove inter-iterations dependences from your loop and make it unit-stride:
- If your arrays have 2 or more dimensions, check whether elements are accessed contiguously and, otherwise, try to permute loops accordingly: C storage order is row-major: for(i) for(j) a[j][i] = b[j][i]; (slow, non stride 1) => for(i) for(j) a[i][j] = b[i][j]; (fast, stride 1)
- If your loop streams arrays of structures (AoS), try to use structures of arrays instead (SoA): for(i) a[i].x = b[i].x; (slow, non stride 1) => for(i) a.x[i] = b.x[i]; (fast, stride 1)

Execution units bottlenecks

Performance is limited by execution of divide and square root operations (the divide/square root unit is a bottleneck). By removing all these bottlenecks, you can lower the cost of an iteration from 15.00 to 8.67 cycles (1.73x speedup).

Workaround

Reduce the number of division or square root instructions:
- If denominator is constant over iterations, use reciprocal (replace x/y with x*(1/y)). Check precision impact. This will be done by your compiler with ffast-math or Ofast
Check whether you really need double precision. If not, switch to single precision to speedup execution

Expensive FP math instructions/calls

Detected performance impact from expensive FP math instructions/calls. By removing/reexpressing them, you can lower the cost of an iteration from 15.00 to 8.33 cycles (1.80x speedup).

FMA

Detected 6 FMA (fused multiply-add) operations.

Slow data structures access

Detected data structures (typically arrays) that cannot be efficiently read/written

Details

Irregular (variable stride) or indirect: 9 occurrence(s)

Non-unit stride (uncontiguous) accesses are not efficiently using data caches

Workaround

Try to remove indirect accesses. If applicable, precompute elements out of the innermost loop.

Type of elements and instruction set

12 SSE or AVX instructions are processing arithmetic or math operations on double precision FP elements in scalar mode (one at a time). 2 SSE or AVX instructions are processing arithmetic or math operations on double precision FP elements in vector mode (two at a time).

Matching between your loop (in the source code) and the binary loop

The binary loop is composed of 22 FP arithmetical operations:

12: addition or subtraction (6 inside FMA instructions)
6: multiply (all inside FMA instructions)
4: divide

The binary loop is loading 176 bytes (22 double precision FP elements). The binary loop is storing 32 bytes (4 double precision FP elements).

Arithmetic intensity

Arithmetic intensity is 0.11 FP operations per loaded or stored byte.

Unroll opportunity

Loop is potentially data access bound.

Workaround

Unroll your loop if trip count is significantly higher than target unroll factor and if some data references are common to consecutive iterations. This can be done manually. Or with the unroll (resp. unroll_and_jam) directive on top of the inner (resp. surrounding) loop. You can enforce an unroll factor: #pragma unroll_and_jam N, unroll_and_jam(N), unroll N or unroll(N)

General properties

nb instructions	37
nb uops	36
loop length	203
used x86 registers	15
used mmx registers	0
used xmm registers	5
used ymm registers	0
used zmm registers	0
nb stack references	8

Front-end

ASSUMED MACRO FUSION FIT IN UOP CACHE

micro-operation queue	6.00 cycles
front end	6.00 cycles

Back-end

	ALU0/BRU0	ALU1	ALU2	ALU3	BRU1	AGU0	AGU1	AGU2	FP0	FP1	FP2	FP3	FP4	FP5
uops	0.75	0.75	0.50	0.50	0.50	8.67	8.67	8.67	4.50	4.50	4.17	3.83	2.00	2.00
cycles	0.75	0.75	0.50	0.50	0.50	8.67	8.67	8.67	4.50	4.50	4.17	3.83	2.00	2.00

Execution ports to units layout:

ALU0/BRU0: ALU
ALU1: ALU
ALU2: ALU
ALU3: ALU
BRU1:
AGU0 (256 bits): store address, load
AGU1 (256 bits): store address, load
AGU2 (256 bits): store address, load
FP0 (256 bits): VPU, DIV/SQRT
FP1 (256 bits): VPU, DIV/SQRT
FP2 (256 bits): VPU
FP3 (256 bits): VPU
FP4 (256 bits): FP store data
FP5 (256 bits): FP store data

Cycles executing div or sqrt instructions	15.00
Longest recurrence chain latency (RecMII)	1.00

Cycles summary

Front-end	6.00
Dispatch	8.67
DIV/SQRT	15.00
Data deps.	1.00
Overall L1	15.00

Vectorization ratios

all	11%
load	0%
store	0%
mul	NA (no mul vectorizable/vectorized instructions)
add-sub	20%
fma	0%
div/sqrt	33%
other	100%

Vector efficiency ratios

all	13%
load	12%
store	12%
mul	NA (no mul vectorizable/vectorized instructions)
add-sub	15%
fma	12%
div/sqrt	16%
other	25%

Cycles and memory resources usage

Assuming all data fit into the L1 cache, each iteration of the binary loop takes 15.00 cycles. At this rate:

12% of peak load performance is reached (11.73 out of 96.00 bytes loaded per cycle (GB/s @ 1GHz))
3% of peak store performance is reached (2.13 out of 64.00 bytes stored per cycle (GB/s @ 1GHz))

Front-end bottlenecks

Found no such bottlenecks.

ASM code

In the binary file, the address of the loop is: 7fea0

Instruction	Nb FU	ALU0/BRU0	ALU1	ALU2	ALU3	BRU1	AGU0	AGU1	AGU2	FP0	FP1	FP2	FP3	FP4	FP5	Latency	Recip. throughput
MOV -0x178(%RBP),%RAX	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	3	0.33
VMOVSD (%RAX,%R8,8),%XMM1	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	1	0.50
MOV -0x170(%RBP),%RAX	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	3	0.33
VADDSD %XMM1,%XMM1,%XMM1	1	0	0	0	0	0	0	0	0	0	0	0.50	0.50	0	0	3	0.50
VDIVSD (%RAX,%R15,8),%XMM1,%XMM1	1	0	0	0	0	0	0.33	0.33	0.33	0.50	0.50	0	0	0	0	13	5
MOV -0x180(%RBP),%RAX	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	3	0.33
VMOVSD (%RAX,%R8,8),%XMM2	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	1	0.50
MOV -0x168(%RBP),%RAX	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	3	0.33
VMOVHPD (%RAX,%R8,8),%XMM2,%XMM2	1	0	0	0	0	0	0.33	0.33	0.33	0	0.33	0.33	0.33	0	0	9	0.50
MOV -0x50(%RBP),%RAX	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	3	0.33
VMOVSD (%RAX,%R10,1),%XMM3	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	1	0.50
MOV -0x160(%RBP),%RAX	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	3	0.33
VADDPD %XMM2,%XMM2,%XMM2	1	0	0	0	0	0	0	0	0	0	0	0.50	0.50	0	0	3	0.50
VMOVHPD (%RAX,%RCX,8),%XMM3,%XMM3	1	0	0	0	0	0	0.33	0.33	0.33	0	0.33	0.33	0.33	0	0	9	0.50
VDIVPD %XMM3,%XMM2,%XMM2	1	0	0	0	0	0	0	0	0	0.50	0.50	0	0	0	0	13	5
VMOVSD (%R14,%R12,8),%XMM3	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	1	0.50
VFMADD213SD (%RDX,%R13,1),%XMM2,%XMM3	1	0	0	0	0	0	0.33	0.33	0.33	0.50	0.50	0	0	0	0	4	0.50
VPERMILPD $0x1,%XMM2,%XMM4	1	0	0	0	0	0	0	0	0	0	0.33	0.33	0.33	0	0	1	0.33
VFMADD231SD (%RSI,%R10,1),%XMM1,%XMM3	1	0	0	0	0	0	0.33	0.33	0.33	0.50	0.50	0	0	0	0	4	0.50
VADDSD %XMM1,%XMM2,%XMM1	1	0	0	0	0	0	0	0	0	0	0	0.50	0.50	0	0	3	0.50
VADDSD %XMM4,%XMM1,%XMM1	1	0	0	0	0	0	0	0	0	0	0	0.50	0.50	0	0	3	0.50
VADDSD (%RDI,%R13,1),%XMM1,%XMM1	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0.50	0.50	0	0	3	0.50
VFMADD231SD (%RBX,%R10,1),%XMM4,%XMM3	1	0	0	0	0	0	0.33	0.33	0.33	0.50	0.50	0	0	0	0	4	0.50
VDIVSD %XMM1,%XMM3,%XMM1	1	0	0	0	0	0	0	0	0	0.50	0.50	0	0	0	0	13	5
VMOVSD %XMM1,(%R9,%R13,1)	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0.50	0.50	1	1
ADD -0x58(%RBP),%R13	1	0.25	0.25	0.25	0.25	0	0.33	0.33	0.33	0	0	0	0	0	0	1	0.33
VMOVSD (%R14,%R12,8),%XMM2	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	1	0.50
VFMSUB231SD %XMM0,%XMM1,%XMM2	1	0	0	0	0	0	0	0	0	0.50	0.50	0	0	0	0	4	0.50
VMOVSD %XMM2,(%R14,%R12,8)	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0.50	0.50	1	1
VMOVSD (%RSI,%R10,1),%XMM2	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	1	0.50
VFMSUB231SD %XMM0,%XMM1,%XMM2	1	0	0	0	0	0	0	0	0	0.50	0.50	0	0	0	0	4	0.50
VMOVSD %XMM2,(%RSI,%R10,1)	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0.50	0.50	1	1
VFMSUB213SD (%RBX,%R10,1),%XMM0,%XMM1	1	0	0	0	0	0	0.33	0.33	0.33	0.50	0.50	0	0	0	0	4	0.50
VMOVSD %XMM1,(%RBX,%R10,1)	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0.50	0.50	1	1
ADD -0x48(%RBP),%R10	1	0.25	0.25	0.25	0.25	0	0.33	0.33	0.33	0	0	0	0	0	0	1	0.33
DEC %R11	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
JNE 7fea0 <.omp_outlined..239+0x7f0>	1	0.50	0	0	0	0.50	0	0	0	0	0	0	0	0	0	1	0.50-1

Function	.omp_outlined..239
Source file and lines	forall.hpp:59-59
Module	libkripke.so

The loop is defined in:

/beegfs/hackathon/users/eoseret/qaas_runs/170-850-6313/intel/Kripke/build/Kripke/src/Kripke/Kernel/SweepSubdomain.cpp:87-89,95-105
/beegfs/hackathon/users/eoseret/qaas_runs/170-850-6313/intel/Kripke/build/Kripke/tpl/raja/include/RAJA/policy/loop/forall.hpp:59

The related source loop is multi-versionned.

gain
potential
hint
expert

Vectorization

Details

11% of SSE/AVX instructions are used in vector version (process two or more data elements in vector registers):

0% of SSE/AVX loads are used in vector version.
0% of SSE/AVX stores are used in vector version.
20% of SSE/AVX addition or subtraction instructions are used in vector version.
0% of SSE/AVX fused multiply-add instructions are used in vector version.
33% of SSE/AVX divide and square root instructions are used in vector version.

Since your execution units are vector units, only a fully vectorized loop can use their full power.

Workaround

Try another compiler or update/tune your current one:
- recompile with ffast-math (included in Ofast) to extend loop vectorization to FP reductions.
Remove inter-iterations dependences from your loop and make it unit-stride:
- If your arrays have 2 or more dimensions, check whether elements are accessed contiguously and, otherwise, try to permute loops accordingly: C storage order is row-major: for(i) for(j) a[j][i] = b[j][i]; (slow, non stride 1) => for(i) for(j) a[i][j] = b[i][j]; (fast, stride 1)
- If your loop streams arrays of structures (AoS), try to use structures of arrays instead (SoA): for(i) a[i].x = b[i].x; (slow, non stride 1) => for(i) a.x[i] = b.x[i]; (fast, stride 1)

Execution units bottlenecks

Workaround

Reduce the number of division or square root instructions:
- If denominator is constant over iterations, use reciprocal (replace x/y with x*(1/y)). Check precision impact. This will be done by your compiler with ffast-math or Ofast
Check whether you really need double precision. If not, switch to single precision to speedup execution

Expensive FP math instructions/calls

Detected performance impact from expensive FP math instructions/calls. By removing/reexpressing them, you can lower the cost of an iteration from 15.00 to 8.33 cycles (1.80x speedup).

FMA

Detected 6 FMA (fused multiply-add) operations.

Slow data structures access

Detected data structures (typically arrays) that cannot be efficiently read/written

Details

Irregular (variable stride) or indirect: 9 occurrence(s)

Non-unit stride (uncontiguous) accesses are not efficiently using data caches

Workaround

Try to remove indirect accesses. If applicable, precompute elements out of the innermost loop.

Type of elements and instruction set

Matching between your loop (in the source code) and the binary loop

The binary loop is composed of 22 FP arithmetical operations:

12: addition or subtraction (6 inside FMA instructions)
6: multiply (all inside FMA instructions)
4: divide

The binary loop is loading 176 bytes (22 double precision FP elements). The binary loop is storing 32 bytes (4 double precision FP elements).

Arithmetic intensity

Arithmetic intensity is 0.11 FP operations per loaded or stored byte.

Unroll opportunity

Loop is potentially data access bound.

Workaround

General properties

nb instructions	37
nb uops	36
loop length	203
used x86 registers	15
used mmx registers	0
used xmm registers	5
used ymm registers	0
used zmm registers	0
nb stack references	8

Front-end

ASSUMED MACRO FUSION FIT IN UOP CACHE

micro-operation queue	6.00 cycles
front end	6.00 cycles

Back-end

	ALU0/BRU0	ALU1	ALU2	ALU3	BRU1	AGU0	AGU1	AGU2	FP0	FP1	FP2	FP3	FP4	FP5
uops	0.75	0.75	0.50	0.50	0.50	8.67	8.67	8.67	4.50	4.50	4.17	3.83	2.00	2.00
cycles	0.75	0.75	0.50	0.50	0.50	8.67	8.67	8.67	4.50	4.50	4.17	3.83	2.00	2.00

Execution ports to units layout:

ALU0/BRU0: ALU
ALU1: ALU
ALU2: ALU
ALU3: ALU
BRU1:
AGU0 (256 bits): store address, load
AGU1 (256 bits): store address, load
AGU2 (256 bits): store address, load
FP0 (256 bits): VPU, DIV/SQRT
FP1 (256 bits): VPU, DIV/SQRT
FP2 (256 bits): VPU
FP3 (256 bits): VPU
FP4 (256 bits): FP store data
FP5 (256 bits): FP store data

Cycles executing div or sqrt instructions	15.00
Longest recurrence chain latency (RecMII)	1.00

Cycles summary

Front-end	6.00
Dispatch	8.67
DIV/SQRT	15.00
Data deps.	1.00
Overall L1	15.00

Vectorization ratios

all	11%
load	0%
store	0%
mul	NA (no mul vectorizable/vectorized instructions)
add-sub	20%
fma	0%
div/sqrt	33%
other	100%

Vector efficiency ratios

all	13%
load	12%
store	12%
mul	NA (no mul vectorizable/vectorized instructions)
add-sub	15%
fma	12%
div/sqrt	16%
other	25%

Cycles and memory resources usage

Assuming all data fit into the L1 cache, each iteration of the binary loop takes 15.00 cycles. At this rate:

12% of peak load performance is reached (11.73 out of 96.00 bytes loaded per cycle (GB/s @ 1GHz))
3% of peak store performance is reached (2.13 out of 64.00 bytes stored per cycle (GB/s @ 1GHz))

Front-end bottlenecks

Found no such bottlenecks.

ASM code

In the binary file, the address of the loop is: 7fea0

Instruction	Nb FU	ALU0/BRU0	ALU1	ALU2	ALU3	BRU1	AGU0	AGU1	AGU2	FP0	FP1	FP2	FP3	FP4	FP5	Latency	Recip. throughput
MOV -0x178(%RBP),%RAX	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	3	0.33
VMOVSD (%RAX,%R8,8),%XMM1	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	1	0.50
MOV -0x170(%RBP),%RAX	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	3	0.33
VADDSD %XMM1,%XMM1,%XMM1	1	0	0	0	0	0	0	0	0	0	0	0.50	0.50	0	0	3	0.50
VDIVSD (%RAX,%R15,8),%XMM1,%XMM1	1	0	0	0	0	0	0.33	0.33	0.33	0.50	0.50	0	0	0	0	13	5
MOV -0x180(%RBP),%RAX	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	3	0.33
VMOVSD (%RAX,%R8,8),%XMM2	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	1	0.50
MOV -0x168(%RBP),%RAX	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	3	0.33
VMOVHPD (%RAX,%R8,8),%XMM2,%XMM2	1	0	0	0	0	0	0.33	0.33	0.33	0	0.33	0.33	0.33	0	0	9	0.50
MOV -0x50(%RBP),%RAX	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	3	0.33
VMOVSD (%RAX,%R10,1),%XMM3	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	1	0.50
MOV -0x160(%RBP),%RAX	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	3	0.33
VADDPD %XMM2,%XMM2,%XMM2	1	0	0	0	0	0	0	0	0	0	0	0.50	0.50	0	0	3	0.50
VMOVHPD (%RAX,%RCX,8),%XMM3,%XMM3	1	0	0	0	0	0	0.33	0.33	0.33	0	0.33	0.33	0.33	0	0	9	0.50
VDIVPD %XMM3,%XMM2,%XMM2	1	0	0	0	0	0	0	0	0	0.50	0.50	0	0	0	0	13	5
VMOVSD (%R14,%R12,8),%XMM3	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	1	0.50
VFMADD213SD (%RDX,%R13,1),%XMM2,%XMM3	1	0	0	0	0	0	0.33	0.33	0.33	0.50	0.50	0	0	0	0	4	0.50
VPERMILPD $0x1,%XMM2,%XMM4	1	0	0	0	0	0	0	0	0	0	0.33	0.33	0.33	0	0	1	0.33
VFMADD231SD (%RSI,%R10,1),%XMM1,%XMM3	1	0	0	0	0	0	0.33	0.33	0.33	0.50	0.50	0	0	0	0	4	0.50
VADDSD %XMM1,%XMM2,%XMM1	1	0	0	0	0	0	0	0	0	0	0	0.50	0.50	0	0	3	0.50
VADDSD %XMM4,%XMM1,%XMM1	1	0	0	0	0	0	0	0	0	0	0	0.50	0.50	0	0	3	0.50
VADDSD (%RDI,%R13,1),%XMM1,%XMM1	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0.50	0.50	0	0	3	0.50
VFMADD231SD (%RBX,%R10,1),%XMM4,%XMM3	1	0	0	0	0	0	0.33	0.33	0.33	0.50	0.50	0	0	0	0	4	0.50
VDIVSD %XMM1,%XMM3,%XMM1	1	0	0	0	0	0	0	0	0	0.50	0.50	0	0	0	0	13	5
VMOVSD %XMM1,(%R9,%R13,1)	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0.50	0.50	1	1
ADD -0x58(%RBP),%R13	1	0.25	0.25	0.25	0.25	0	0.33	0.33	0.33	0	0	0	0	0	0	1	0.33
VMOVSD (%R14,%R12,8),%XMM2	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	1	0.50
VFMSUB231SD %XMM0,%XMM1,%XMM2	1	0	0	0	0	0	0	0	0	0.50	0.50	0	0	0	0	4	0.50
VMOVSD %XMM2,(%R14,%R12,8)	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0.50	0.50	1	1
VMOVSD (%RSI,%R10,1),%XMM2	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	1	0.50
VFMSUB231SD %XMM0,%XMM1,%XMM2	1	0	0	0	0	0	0	0	0	0.50	0.50	0	0	0	0	4	0.50
VMOVSD %XMM2,(%RSI,%R10,1)	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0.50	0.50	1	1
VFMSUB213SD (%RBX,%R10,1),%XMM0,%XMM1	1	0	0	0	0	0	0.33	0.33	0.33	0.50	0.50	0	0	0	0	4	0.50
VMOVSD %XMM1,(%RBX,%R10,1)	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0.50	0.50	1	1
ADD -0x48(%RBP),%R10	1	0.25	0.25	0.25	0.25	0	0.33	0.33	0.33	0	0	0	0	0	0	1	0.33
DEC %R11	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
JNE 7fea0 <.omp_outlined..239+0x7f0>	1	0.50	0	0	0	0.50	0	0	0	0	0	0	0	0	0	1	0.50-1

▶Scalability Runs Description

Run 2x1	Number processes: 2Number nodes: 1Number processes per node: 2Run Command: <executable> --groups 1024 --procs 2,1,1MPI Command: mpirun -np <number_processes>Dataset: Run Directory: /beegfs/hackathon/users/eoseret/qaas_runs/170-850-6313/intel/Kripke/run/oneview_runs/compilers/aocc_10/oneview_run_1708510900I_MPI_PIN_DOMAIN: auto:scatterOMP_PLACES: threadsOMP_PROC_BIND: spreadOMP_NUM_THREADS: 1
Run 2x2	OMP_NUM_THREADS: 2I_MPI_PIN_DOMAIN: auto:scatterOMP_PLACES: threadsOMP_PROC_BIND: spread
Run 2x4	OMP_NUM_THREADS: 4I_MPI_PIN_DOMAIN: auto:scatterOMP_PLACES: threadsOMP_PROC_BIND: spread
Run 2x8	OMP_NUM_THREADS: 8I_MPI_PIN_DOMAIN: auto:scatterOMP_PLACES: threadsOMP_PROC_BIND: spread
Run 2x16	OMP_NUM_THREADS: 16I_MPI_PIN_DOMAIN: auto:scatterOMP_PLACES: threadsOMP_PROC_BIND: spread
Run 2x32	OMP_NUM_THREADS: 32I_MPI_PIN_DOMAIN: auto:scatterOMP_PLACES: threadsOMP_PROC_BIND: spread
Run 2x64	OMP_NUM_THREADS: 64I_MPI_PIN_DOMAIN: auto:scatterOMP_PLACES: threadsOMP_PROC_BIND: spread
Run 2x96	OMP_NUM_THREADS: 96I_MPI_PIN_DOMAIN: auto:scatterOMP_PLACES: threadsOMP_PROC_BIND: spread

(2x1) Efficiency	(2x1) Potential Speed-Up (%)	(2x2) Efficiency	(2x2) Potential Speed-Up (%)	(2x4) Efficiency	(2x4) Potential Speed-Up (%)	(2x8) Efficiency	(2x8) Potential Speed-Up (%)	(2x16) Efficiency	(2x16) Potential Speed-Up (%)	(2x32) Efficiency	(2x32) Potential Speed-Up (%)	(2x64) Efficiency	(2x64) Potential Speed-Up (%)	(2x96) Efficiency	(2x96) Potential Speed-Up (%)
1	0	0.96	0.07	0.93	0.13	0.92	0.14	0.9	0.17	0.89	0.19	0.59	0.72	0.4	1.15

Speed-Up and Efficiency

▶Detailed Speed-Up and Efficiency

Run	Number of threads	Efficiency (ideal is 1)	Speedup	Ideal Speedup	Time (s)	Coverage (%)
2x1	2	1	1	1	18.78	1.69
2x2	4	0.96	1.92	2	10.4	1.73
2x4	8	0.93	3.72	4	5.33	1.77
2x8	16	0.92	7.36	8	2.72	1.79
2x16	32	0.9	14.44	16	1.41	1.77
2x32	64	0.89	28.44	32	0.73	1.68
2x64	128	0.59	37.54	64	0.6	1.74
2x96	192	0.4	38.31	96	0.69	1.91

Report Configuration

Vectorization

Details

Workaround

Execution units bottlenecks

Workaround

Expensive FP math instructions/calls

FMA

Slow data structures access

Details

Workaround

Type of elements and instruction set

Matching between your loop (in the source code) and the binary loop

Arithmetic intensity

Unroll opportunity

Workaround

General properties

Front-end

Back-end

Cycles summary

Vectorization ratios

Vector efficiency ratios

Cycles and memory resources usage

Front-end bottlenecks

ASM code

Vectorization

Details

Workaround

Execution units bottlenecks

Workaround

Expensive FP math instructions/calls

FMA

Slow data structures access

Details

Workaround

Type of elements and instruction set

Matching between your loop (in the source code) and the binary loop

Arithmetic intensity

Unroll opportunity

Workaround

General properties

Front-end

Back-end

Cycles summary

Vectorization ratios

Vector efficiency ratios

Cycles and memory resources usage

Front-end bottlenecks

ASM code

▶Scalability Runs Description

▼Colums Filter

Speed-Up and Efficiency

▶Detailed Speed-Up and Efficiency