OV - exec - Loop 219 - libqmcparticle

	Loop Id: 219	Module: libqmcparticle_omptarget.so	Source: ParticleBConds3DSoa.h:234-255	Coverage: 0.7%

0x2c900 MOV	%R13,%RSI

0x2c903 MOV	%R12,%R13

0x2c906 MOV	-0x50(%RBP),%R12    [7]

0x2c90a VMOVAPD	(%RCX),%ZMM8    [8]

0x2c910 VMOVDDUP	0x78(%RDI),%XMM13    [2]

0x2c915 VMOVAPD	(%RCX,%R12,8),%ZMM9    [9]

0x2c91c MOV	%R13,%R12

0x2c91f VMOVAPD	(%RCX,%R12,1),%ZMM10    [1]

0x2c926 ADD	$0x40,%RCX

0x2c92a MOV	%RSI,%R13

0x2c92d VSUBPD	%ZMM3,%ZMM8,%ZMM8

0x2c933 VMULPD	0x48(%RDI){1to8},%ZMM8,%ZMM11    [2]

0x2c93a VMULPD	0x60(%RDI){1to8},%ZMM8,%ZMM12    [2]

0x2c941 VBROADCASTSD	%XMM13,%ZMM13

0x2c947 VMULPD	%ZMM8,%ZMM13,%ZMM8

0x2c94d VMOVDQA64	%ZMM7,%ZMM13

0x2c953 VSUBPD	%ZMM4,%ZMM9,%ZMM9

0x2c959 VSUBPD	%ZMM5,%ZMM10,%ZMM10

0x2c95f VFMADD231PD	0x50(%RDI){1to8},%ZMM9,%ZMM11    [2]

0x2c966 VFMADD231PD	0x68(%RDI){1to8},%ZMM9,%ZMM12    [2]

0x2c96d VFMADD231PD	0x80(%RDI){1to8},%ZMM9,%ZMM8    [2]

0x2c974 VMOVDQA64	%ZMM7,%ZMM9

0x2c97a VFMADD231PD	0x58(%RDI){1to8},%ZMM10,%ZMM11    [2]

0x2c981 VFMADD231PD	0x70(%RDI){1to8},%ZMM10,%ZMM12    [2]

0x2c988 VFMADD231PD	0x88(%RDI){1to8},%ZMM10,%ZMM8    [2]

0x2c98f VMOVDQA64	%ZMM7,%ZMM10

0x2c995 VPTERNLOGQ	$-0x8,%ZMM6,%ZMM11,%ZMM9

0x2c99c VPTERNLOGQ	$-0x8,%ZMM6,%ZMM12,%ZMM10

0x2c9a3 VPTERNLOGQ	$-0x8,%ZMM6,%ZMM8,%ZMM13

0x2c9aa VADDPD	%ZMM9,%ZMM11,%ZMM9

0x2c9b0 VADDPD	%ZMM10,%ZMM12,%ZMM10

0x2c9b6 VRNDSCALEPD	$0xb,%ZMM9,%ZMM9

0x2c9bd VRNDSCALEPD	$0xb,%ZMM10,%ZMM10

0x2c9c4 VSUBPD	%ZMM9,%ZMM11,%ZMM9

0x2c9ca VADDPD	%ZMM13,%ZMM8,%ZMM11

0x2c9d0 VSUBPD	%ZMM10,%ZMM12,%ZMM10

0x2c9d6 VMULPD	(%RDI){1to8},%ZMM9,%ZMM12    [2]

0x2c9dc VRNDSCALEPD	$0xb,%ZMM11,%ZMM11

0x2c9e3 VFMADD231PD	0x8(%RDI){1to8},%ZMM10,%ZMM12    [2]

0x2c9ea VSUBPD	%ZMM11,%ZMM8,%ZMM8

0x2c9f0 VFMADD231PD	0x10(%RDI){1to8},%ZMM8,%ZMM12    [2]

0x2c9f7 VMOVAPD	%ZMM12,(%R10)    [3]

0x2c9fd VMULPD	0x18(%RDI){1to8},%ZMM9,%ZMM11    [2]

0x2ca04 VFMADD231PD	0x20(%RDI){1to8},%ZMM10,%ZMM11    [2]

0x2ca0b VFMADD231PD	0x28(%RDI){1to8},%ZMM8,%ZMM11    [2]

0x2ca12 VMOVAPD	%ZMM11,(%R10,%R9,8)    [6]

0x2ca19 VMULPD	0x30(%RDI){1to8},%ZMM9,%ZMM9    [2]

0x2ca20 VFMADD231PD	0x38(%RDI){1to8},%ZMM10,%ZMM9    [2]

0x2ca27 VFMADD231PD	0x40(%RDI){1to8},%ZMM8,%ZMM9    [2]

0x2ca2e VMOVAPD	%ZMM9,(%R10,%RSI,1)    [5]

0x2ca35 VMOVAPD	(%R10),%ZMM8    [3]

0x2ca3b VMOVAPD	(%R10,%R9,8),%ZMM10    [6]

0x2ca42 ADD	$0x40,%R10

0x2ca46 VMULPD	%ZMM8,%ZMM8,%ZMM8

0x2ca4c VFMADD231PD	%ZMM9,%ZMM9,%ZMM8

0x2ca52 VFMADD231PD	%ZMM10,%ZMM10,%ZMM8

0x2ca58 VSQRTPD	%ZMM8,%ZMM8

0x2ca5e VMOVAPD	%ZMM8,(%R8,%RDX,1)    [4]

0x2ca65 ADD	$0x40,%RDX

0x2ca69 CMP	%RDX,-0x48(%RBP)    [7]

0x2ca6d JNE	2c900

/beegfs/hackathon/users/eoseret/qaas_runs/170-855-3059/intel/miniqmc/build/miniqmc/src/Particle/Lattice/ParticleBConds3DSoa.h: 234 - 255

--------------------------------------------------------------------------------

234: #pragma omp simd aligned(temp_r, px, py, pz, dx, dy, dz: QMC_SIMD_ALIGNMENT)

235:     for (int iat = first; iat < last; ++iat)

236:     {

237:       T displ_0 = px[iat] - x0;

238:       T displ_1 = py[iat] - y0;

239:       T displ_2 = pz[iat] - z0;

240:

241:       T ar_0 = displ_0 * g00 + displ_1 * g10 + displ_2 * g20;

242:       T ar_1 = displ_0 * g01 + displ_1 * g11 + displ_2 * g21;

243:       T ar_2 = displ_0 * g02 + displ_1 * g12 + displ_2 * g22;

244:

245:       //put them in the box

246:       ar_0 -= round(ar_0);

247:       ar_1 -= round(ar_1);

248:       ar_2 -= round(ar_2);

249:

250:       //unit2cart

251:       dx[iat] = ar_0 * r00 + ar_1 * r10 + ar_2 * r20;

252:       dy[iat] = ar_0 * r01 + ar_1 * r11 + ar_2 * r21;

253:       dz[iat] = ar_0 * r02 + ar_1 * r12 + ar_2 * r22;

254:

255:       temp_r[iat] = std::sqrt(dx[iat] * dx[iat] + dy[iat] * dy[iat] + dz[iat] * dz[iat]);

Path /

Metric	Value
CQA speedup if no scalar integer	1.00
CQA speedup if FP arith vectorized	1.00
CQA speedup if fully vectorized	1.05
CQA speedup if no inter-iteration dependency	NA
CQA speedup if next bottleneck killed	1.07
Bottlenecks	P8,
Function	_ZNK11qmcplusplus10DTD_BCondsIdLj3ELi40EE16computeDistancesINS_10TinyVectorIdLj3EEENS_18VectorSoAContainerIdLj3ENS_10MallocatorIdLm64EEEEES8_EEvRKT_RKT0_PdRT1_iii
Source	ParticleBConds3DSoa.h:234-255
Source loop unroll info	unrolled by 8
Source loop unroll confidence level	high
Unroll/vectorization loop type	main
Unroll factor	8
CQA cycles	23.00
CQA cycles if no scalar integer	23.00
CQA cycles if FP arith vectorized	23.00
CQA cycles if fully vectorized	22.00
Front-end cycles	10.67
DIV/SQRT cycles	1.00
P0 cycles	1.00
P1 cycles	0.75
P2 cycles	0.75
P3 cycles	0.50
P4 cycles	17.00
P5 cycles	17.00
P6 cycles	17.00
P7 cycles	23.00
P8 cycles	21.50
P9 cycles	12.25
P10 cycles	12.25
P11 cycles	4.00
P12 cycles	4.00
P13 cycles	16.00
Inter-iter dependencies cycles	1
FE+BE cycles (UFS)	NA
Stall cycles (UFS)	NA
Nb insns	61.00
Nb uops	64.00
Nb loads	25.00
Nb stores	4.00
Nb stack references	2.00
FLOP/cycle	15.65
Nb FLOP add-sub	72.00
Nb FLOP mul	56.00
Nb FLOP fma	112.00
Nb FLOP div	0.00
Nb FLOP rcp	0.00
Nb FLOP sqrt	8.00
Nb FLOP rsqrt	0.00
Bytes/cycle	32.00
Bytes prefetched	0.00
Bytes loaded	480.00
Bytes stored	256.00
Stride 0	2.00
Stride 1	7.00
Stride n	0.00
Stride unknown	0.00
Stride indirect	0.00
Vectorization ratio all	92.45
Vectorization ratio load	95.65
Vectorization ratio store	100.00
Vectorization ratio mul	100.00
Vectorization ratio add_sub	100.00
Vectorization ratio fma	100.00
Vectorization ratio div_sqrt	100.00
Vectorization ratio other	75.00
Vector-efficiency ratio all	93.40
Vector-efficiency ratio load	96.20
Vector-efficiency ratio store	100.00
Vector-efficiency ratio mul	100.00
Vector-efficiency ratio add_sub	100.00
Vector-efficiency ratio fma	100.00
Vector-efficiency ratio div_sqrt	100.00
Vector-efficiency ratio other	78.13

Metric	Value
CQA speedup if no scalar integer	1.00
CQA speedup if FP arith vectorized	1.00
CQA speedup if fully vectorized	1.05
CQA speedup if no inter-iteration dependency	NA
CQA speedup if next bottleneck killed	1.07
Bottlenecks	P8,
Function	_ZNK11qmcplusplus10DTD_BCondsIdLj3ELi40EE16computeDistancesINS_10TinyVectorIdLj3EEENS_18VectorSoAContainerIdLj3ENS_10MallocatorIdLm64EEEEES8_EEvRKT_RKT0_PdRT1_iii
Source	ParticleBConds3DSoa.h:234-255
Source loop unroll info	unrolled by 8
Source loop unroll confidence level	high
Unroll/vectorization loop type	main
Unroll factor	8
CQA cycles	23.00
CQA cycles if no scalar integer	23.00
CQA cycles if FP arith vectorized	23.00
CQA cycles if fully vectorized	22.00
Front-end cycles	10.67
DIV/SQRT cycles	1.00
P0 cycles	1.00
P1 cycles	0.75
P2 cycles	0.75
P3 cycles	0.50
P4 cycles	17.00
P5 cycles	17.00
P6 cycles	17.00
P7 cycles	23.00
P8 cycles	21.50
P9 cycles	12.25
P10 cycles	12.25
P11 cycles	4.00
P12 cycles	4.00
P13 cycles	16.00
Inter-iter dependencies cycles	1
FE+BE cycles (UFS)	NA
Stall cycles (UFS)	NA
Nb insns	61.00
Nb uops	64.00
Nb loads	25.00
Nb stores	4.00
Nb stack references	2.00
FLOP/cycle	15.65
Nb FLOP add-sub	72.00
Nb FLOP mul	56.00
Nb FLOP fma	112.00
Nb FLOP div	0.00
Nb FLOP rcp	0.00
Nb FLOP sqrt	8.00
Nb FLOP rsqrt	0.00
Bytes/cycle	32.00
Bytes prefetched	0.00
Bytes loaded	480.00
Bytes stored	256.00
Stride 0	2.00
Stride 1	7.00
Stride n	0.00
Stride unknown	0.00
Stride indirect	0.00
Vectorization ratio all	92.45
Vectorization ratio load	95.65
Vectorization ratio store	100.00
Vectorization ratio mul	100.00
Vectorization ratio add_sub	100.00
Vectorization ratio fma	100.00
Vectorization ratio div_sqrt	100.00
Vectorization ratio other	75.00
Vector-efficiency ratio all	93.40
Vector-efficiency ratio load	96.20
Vector-efficiency ratio store	100.00
Vector-efficiency ratio mul	100.00
Vector-efficiency ratio add_sub	100.00
Vector-efficiency ratio fma	100.00
Vector-efficiency ratio div_sqrt	100.00
Vector-efficiency ratio other	78.13

Path /

Average path: Display a virtual path defined by average values of all real paths

Function	_ZNK11qmcplusplus10DTD_BCondsIdLj3ELi40EE16computeDistancesINS_10TinyVectorIdLj3EEENS_18VectorSoAContainerIdLj3ENS_10MallocatorIdLm64EEEEES8_EEvRKT_RKT0_PdRT1_iii
Source file and lines	ParticleBConds3DSoa.h:234-255
Module	libqmcparticle_omptarget.so

The loop is defined in /beegfs/hackathon/users/eoseret/qaas_runs/170-855-3059/intel/miniqmc/build/miniqmc/src/Particle/Lattice/ParticleBConds3DSoa.h:234-255.

It is main loop of related source loop which is unrolled by 8 (including vectorization).

gain
potential
hint
expert

Vectorization

Your loop is highly vectorized. 93% of vector register length is used (average across all SSE/AVX instructions). By fully vectorizing your loop, you can lower the cost of an iteration from 23.00 to 22.00 cycles (1.05x speedup).

Details

92% of SSE/AVX instructions are used in vector version (process two or more data elements in vector registers):

95% of SSE/AVX loads are used in vector version.
75% of SSE/AVX instructions that are not load, store, addition, subtraction nor multiply instructions are used in vector version.

Since your execution units are vector units, only a fully vectorized loop can use their full power.

Workaround

Pass to your compiler a micro-architecture specialization option:
- use march=native
Use vector aligned instructions:
1. align your arrays on 64 bytes boundaries: replace { void *p = malloc (size); } with { void *p; posix_memalign (&p, 64, size); }.
2. inform your compiler that your arrays are vector aligned: if array 'foo' is 64 bytes-aligned, define a pointer 'p_foo' as __builtin_assume_aligned (foo, 64) and use it instead of 'foo' in the loop.

Execution units bottlenecks

Performance is limited by execution of FP multiply or FMA (fused multiply-add) operations (the FP multiply/FMA unit is a bottleneck). By removing all these bottlenecks, you can lower the cost of an iteration from 23.00 to 21.50 cycles (1.07x speedup).

Workaround

Reduce the number of FP multiply/FMA instructions

FMA

Detected 112 FMA (fused multiply-add) operations. Presence of both ADD/SUB and MUL operations.

Workaround

Try to change order in which elements are evaluated (using parentheses) in arithmetic expressions containing both ADD/SUB and MUL operations to enable your compiler to generate FMA instructions wherever possible. For instance a + b*c is a valid FMA (MUL then ADD). However (a+b)* c cannot be translated into an FMA (ADD then MUL).

Complex instructions

Detected COMPLEX INSTRUCTIONS.

Details

These instructions generate more than one micro-operation and only one of them can be decoded during a cycle and the extra micro-operations increase pressure on execution units.

VMOVAPD: 4 occurrences

Type of elements and instruction set

34 AVX-512 instructions are processing arithmetic or math operations on double precision FP elements in vector mode (eight at a time).

Matching between your loop (in the source code) and the binary loop

The binary loop is composed of 360 FP arithmetical operations:

184: addition or subtraction (112 inside FMA instructions)
168: multiply (112 inside FMA instructions)
8: square root

The binary loop is loading 480 bytes (60 double precision FP elements). The binary loop is storing 256 bytes (32 double precision FP elements).

Arithmetic intensity

Arithmetic intensity is 0.49 FP operations per loaded or stored byte.

General properties

nb instructions	61
nb uops	64
loop length	371
used x86 registers	10
used mmx registers	0
used xmm registers	1
used ymm registers	0
used zmm registers	11
nb stack references	2
ADD-SUB / MUL ratio	1.29

Front-end

ASSUMED MACRO FUSION FIT IN UOP CACHE

micro-operation queue	10.67 cycles
front end	10.67 cycles

Back-end

	ALU0/BRU0	ALU1	ALU2	ALU3	BRU1	AGU0	AGU1	AGU2	FP0	FP1	FP2	FP3	FP4	FP5
uops	1.00	1.00	0.75	0.75	0.50	9.67	9.67	9.67	11.50	11.00	7.75	7.75	4.00	4.00
cycles	1.00	1.00	0.75	0.75	0.50	17.00	17.00	17.00	23.00	21.50	12.25	12.25	4.00	4.00

Execution ports to units layout:

ALU0/BRU0: ALU
ALU1: ALU
ALU2: ALU
ALU3: ALU
BRU1:
AGU0 (256 bits): store address, load
AGU1 (256 bits): store address, load
AGU2 (256 bits): store address, load
FP0 (256 bits): VPU, DIV/SQRT
FP1 (256 bits): VPU, DIV/SQRT
FP2 (256 bits): VPU
FP3 (256 bits): VPU
FP4 (256 bits): FP store data
FP5 (256 bits): FP store data

Cycles executing div or sqrt instructions	16.00
Longest recurrence chain latency (RecMII)	1.00

Cycles summary

Front-end	10.67
Dispatch	23.00
DIV/SQRT	16.00
Data deps.	1.00
Overall L1	23.00

Vectorization ratios

INT

all	75%
load	NA (no load vectorizable/vectorized instructions)
store	NA (no store vectorizable/vectorized instructions)
mul	NA (no mul vectorizable/vectorized instructions)
add-sub	NA (no add-sub vectorizable/vectorized instructions)
fma	NA (no fma vectorizable/vectorized instructions)
other	75%

all	95%
load	95%
store	100%
mul	100%
add-sub	100%
fma	100%
div/sqrt	100%
other	75%

INT+FP

all	92%
load	95%
store	100%
mul	100%
add-sub	100%
fma	100%
div/sqrt	100%
other	75%

Vector efficiency ratios

INT

all	78%
load	NA (no load vectorizable/vectorized instructions)
store	NA (no store vectorizable/vectorized instructions)
mul	NA (no mul vectorizable/vectorized instructions)
add-sub	NA (no add-sub vectorizable/vectorized instructions)
fma	NA (no fma vectorizable/vectorized instructions)
other	78%

all	96%
load	96%
store	100%
mul	100%
add-sub	100%
fma	100%
div/sqrt	100%
other	78%

INT+FP

all	93%
load	96%
store	100%
mul	100%
add-sub	100%
fma	100%
div/sqrt	100%
other	78%

Cycles and memory resources usage

Assuming all data fit into the L1 cache, each iteration of the binary loop takes 23.00 cycles. At this rate:

21% of peak load performance is reached (20.87 out of 96.00 bytes loaded per cycle (GB/s @ 1GHz))
17% of peak store performance is reached (11.13 out of 64.00 bytes stored per cycle (GB/s @ 1GHz))

Front-end bottlenecks

Found no such bottlenecks.

ASM code

In the binary file, the address of the loop is: 2c900

Instruction	Nb FU	ALU0/BRU0	ALU1	ALU2	ALU3	BRU1	AGU0	AGU1	AGU2	FP0	FP1	FP2	FP3	FP4	FP5	Latency	Recip. throughput
MOV %R13,%RSI	1	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0.17
MOV %R12,%R13	1	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0.17
MOV -0x50(%RBP),%R12	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	3	0.33
VMOVAPD (%RCX),%ZMM8	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	3	1
VMOVDDUP 0x78(%RDI),%XMM13	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	1	0.50
VMOVAPD (%RCX,%R12,8),%ZMM9	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	3	1
MOV %R13,%R12	1	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0.17
VMOVAPD (%RCX,%R12,1),%ZMM10	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	3	1
ADD $0x40,%RCX	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
MOV %RSI,%R13	1	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0.17
VSUBPD %ZMM3,%ZMM8,%ZMM8	1	0	0	0	0	0	0	0	0	0	0	0.50	0.50	0	0	3	1
VMULPD 0x48(%RDI){1to8},%ZMM8,%ZMM11	1	0	0	0	0	0	0.33	0.33	0.33	0.50	0.50	0	0	0	0	3	1
VMULPD 0x60(%RDI){1to8},%ZMM8,%ZMM12	1	0	0	0	0	0	0.33	0.33	0.33	0.50	0.50	0	0	0	0	3	1
VBROADCASTSD %XMM13,%ZMM13	1	0	0	0	0	0	0	0	0	0	0.50	0.50	0	0	0	1	1
VMULPD %ZMM8,%ZMM13,%ZMM8	1	0	0	0	0	0	0	0	0	0.50	0.50	0	0	0	0	3	1
VMOVDQA64 %ZMM7,%ZMM13	1	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0.17
VSUBPD %ZMM4,%ZMM9,%ZMM9	1	0	0	0	0	0	0	0	0	0	0	0.50	0.50	0	0	3	1
VSUBPD %ZMM5,%ZMM10,%ZMM10	1	0	0	0	0	0	0	0	0	0	0	0.50	0.50	0	0	3	1
VFMADD231PD 0x50(%RDI){1to8},%ZMM9,%ZMM11	1	0	0	0	0	0	0.33	0.33	0.33	0.50	0.50	0	0	0	0	4	1
VFMADD231PD 0x68(%RDI){1to8},%ZMM9,%ZMM12	1	0	0	0	0	0	0.33	0.33	0.33	0.50	0.50	0	0	0	0	4	1
VFMADD231PD 0x80(%RDI){1to8},%ZMM9,%ZMM8	1	0	0	0	0	0	0.33	0.33	0.33	0.50	0.50	0	0	0	0	4	1
VMOVDQA64 %ZMM7,%ZMM9	1	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0.17
VFMADD231PD 0x58(%RDI){1to8},%ZMM10,%ZMM11	1	0	0	0	0	0	0.33	0.33	0.33	0.50	0.50	0	0	0	0	4	1
VFMADD231PD 0x70(%RDI){1to8},%ZMM10,%ZMM12	1	0	0	0	0	0	0.33	0.33	0.33	0.50	0.50	0	0	0	0	4	1
VFMADD231PD 0x88(%RDI){1to8},%ZMM10,%ZMM8	1	0	0	0	0	0	0.33	0.33	0.33	0.50	0.50	0	0	0	0	4	1
VMOVDQA64 %ZMM7,%ZMM10	1	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0.17
VPTERNLOGQ $-0x8,%ZMM6,%ZMM11,%ZMM9	1	0	0	0	0	0	0	0	0	0.25	0.25	0.25	0.25	0	0	1	0.50
VPTERNLOGQ $-0x8,%ZMM6,%ZMM12,%ZMM10	1	0	0	0	0	0	0	0	0	0.25	0.25	0.25	0.25	0	0	1	0.50
VPTERNLOGQ $-0x8,%ZMM6,%ZMM8,%ZMM13	1	0	0	0	0	0	0	0	0	0.25	0.25	0.25	0.25	0	0	1	0.50
VADDPD %ZMM9,%ZMM11,%ZMM9	1	0	0	0	0	0	0	0	0	0	0	0.50	0.50	0	0	3	1
VADDPD %ZMM10,%ZMM12,%ZMM10	1	0	0	0	0	0	0	0	0	0	0	0.50	0.50	0	0	3	1
VRNDSCALEPD $0xb,%ZMM9,%ZMM9	1	0	0	0	0	0	0	0	0	0	0	0.50	0.50	0	0	3	1
VRNDSCALEPD $0xb,%ZMM10,%ZMM10	1	0	0	0	0	0	0	0	0	0	0	0.50	0.50	0	0	3	1
VSUBPD %ZMM9,%ZMM11,%ZMM9	1	0	0	0	0	0	0	0	0	0	0	0.50	0.50	0	0	3	1
VADDPD %ZMM13,%ZMM8,%ZMM11	1	0	0	0	0	0	0	0	0	0	0	0.50	0.50	0	0	3	1
VSUBPD %ZMM10,%ZMM12,%ZMM10	1	0	0	0	0	0	0	0	0	0	0	0.50	0.50	0	0	3	1
VMULPD (%RDI){1to8},%ZMM9,%ZMM12	1	0	0	0	0	0	0.33	0.33	0.33	0.50	0.50	0	0	0	0	3	1
VRNDSCALEPD $0xb,%ZMM11,%ZMM11	1	0	0	0	0	0	0	0	0	0	0	0.50	0.50	0	0	3	1
VFMADD231PD 0x8(%RDI){1to8},%ZMM10,%ZMM12	1	0	0	0	0	0	0.33	0.33	0.33	0.50	0.50	0	0	0	0	4	1
VSUBPD %ZMM11,%ZMM8,%ZMM8	1	0	0	0	0	0	0	0	0	0	0	0.50	0.50	0	0	3	1
VFMADD231PD 0x10(%RDI){1to8},%ZMM8,%ZMM12	1	0	0	0	0	0	0.33	0.33	0.33	0.50	0.50	0	0	0	0	4	1
VMOVAPD %ZMM12,(%R10)	2	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	1	1	5	2
VMULPD 0x18(%RDI){1to8},%ZMM9,%ZMM11	1	0	0	0	0	0	0.33	0.33	0.33	0.50	0.50	0	0	0	0	3	1
VFMADD231PD 0x20(%RDI){1to8},%ZMM10,%ZMM11	1	0	0	0	0	0	0.33	0.33	0.33	0.50	0.50	0	0	0	0	4	1
VFMADD231PD 0x28(%RDI){1to8},%ZMM8,%ZMM11	1	0	0	0	0	0	0.33	0.33	0.33	0.50	0.50	0	0	0	0	4	1
VMOVAPD %ZMM11,(%R10,%R9,8)	2	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	1	1	5	2
VMULPD 0x30(%RDI){1to8},%ZMM9,%ZMM9	1	0	0	0	0	0	0.33	0.33	0.33	0.50	0.50	0	0	0	0	3	1
VFMADD231PD 0x38(%RDI){1to8},%ZMM10,%ZMM9	1	0	0	0	0	0	0.33	0.33	0.33	0.50	0.50	0	0	0	0	4	1
VFMADD231PD 0x40(%RDI){1to8},%ZMM8,%ZMM9	1	0	0	0	0	0	0.33	0.33	0.33	0.50	0.50	0	0	0	0	4	1
VMOVAPD %ZMM9,(%R10,%RSI,1)	2	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	1	1	5	2
VMOVAPD (%R10),%ZMM8	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	3	1
VMOVAPD (%R10,%R9,8),%ZMM10	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	3	1
ADD $0x40,%R10	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
VMULPD %ZMM8,%ZMM8,%ZMM8	1	0	0	0	0	0	0	0	0	0.50	0.50	0	0	0	0	3	1
VFMADD231PD %ZMM9,%ZMM9,%ZMM8	1	0	0	0	0	0	0	0	0	0.50	0.50	0	0	0	0	4	1
VFMADD231PD %ZMM10,%ZMM10,%ZMM8	1	0	0	0	0	0	0	0	0	0.50	0.50	0	0	0	0	4	1
VSQRTPD %ZMM8,%ZMM8	1	0	0	0	0	0	0	0	0	0.50	0.50	0	0	0	0	21	16
VMOVAPD %ZMM8,(%R8,%RDX,1)	2	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	1	1	5	2
ADD $0x40,%RDX	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
CMP %RDX,-0x48(%RBP)	1	0.25	0.25	0.25	0.25	0	0.33	0.33	0.33	0	0	0	0	0	0	1	0.33
JNE 2c900 <_ZNK11qmcplusplus10DTD_BCondsIdLj3ELi40EE16computeDistancesINS_10TinyVectorIdLj3EEENS_18VectorSoAContainerIdLj3ENS_10MallocatorIdLm64EEEEES8_EEvRKT_RKT0_PdRT1_iii+0xc0>	1	0.50	0	0	0	0.50	0	0	0	0	0	0	0	0	0	1	0.50-1

Function	_ZNK11qmcplusplus10DTD_BCondsIdLj3ELi40EE16computeDistancesINS_10TinyVectorIdLj3EEENS_18VectorSoAContainerIdLj3ENS_10MallocatorIdLm64EEEEES8_EEvRKT_RKT0_PdRT1_iii
Source file and lines	ParticleBConds3DSoa.h:234-255
Module	libqmcparticle_omptarget.so

gain
potential
hint
expert

Vectorization

Details

92% of SSE/AVX instructions are used in vector version (process two or more data elements in vector registers):

95% of SSE/AVX loads are used in vector version.
75% of SSE/AVX instructions that are not load, store, addition, subtraction nor multiply instructions are used in vector version.

Since your execution units are vector units, only a fully vectorized loop can use their full power.

Workaround

Pass to your compiler a micro-architecture specialization option:
- use march=native
Use vector aligned instructions:
1. align your arrays on 64 bytes boundaries: replace { void *p = malloc (size); } with { void *p; posix_memalign (&p, 64, size); }.
2. inform your compiler that your arrays are vector aligned: if array 'foo' is 64 bytes-aligned, define a pointer 'p_foo' as __builtin_assume_aligned (foo, 64) and use it instead of 'foo' in the loop.

Execution units bottlenecks

Workaround

Reduce the number of FP multiply/FMA instructions

FMA

Detected 112 FMA (fused multiply-add) operations. Presence of both ADD/SUB and MUL operations.

Workaround

Complex instructions

Detected COMPLEX INSTRUCTIONS.

Details

These instructions generate more than one micro-operation and only one of them can be decoded during a cycle and the extra micro-operations increase pressure on execution units.

VMOVAPD: 4 occurrences

Type of elements and instruction set

34 AVX-512 instructions are processing arithmetic or math operations on double precision FP elements in vector mode (eight at a time).

Matching between your loop (in the source code) and the binary loop

The binary loop is composed of 360 FP arithmetical operations:

184: addition or subtraction (112 inside FMA instructions)
168: multiply (112 inside FMA instructions)
8: square root

The binary loop is loading 480 bytes (60 double precision FP elements). The binary loop is storing 256 bytes (32 double precision FP elements).

Arithmetic intensity

Arithmetic intensity is 0.49 FP operations per loaded or stored byte.

General properties

nb instructions	61
nb uops	64
loop length	371
used x86 registers	10
used mmx registers	0
used xmm registers	1
used ymm registers	0
used zmm registers	11
nb stack references	2
ADD-SUB / MUL ratio	1.29

Front-end

ASSUMED MACRO FUSION FIT IN UOP CACHE

micro-operation queue	10.67 cycles
front end	10.67 cycles

Back-end

	ALU0/BRU0	ALU1	ALU2	ALU3	BRU1	AGU0	AGU1	AGU2	FP0	FP1	FP2	FP3	FP4	FP5
uops	1.00	1.00	0.75	0.75	0.50	9.67	9.67	9.67	11.50	11.00	7.75	7.75	4.00	4.00
cycles	1.00	1.00	0.75	0.75	0.50	17.00	17.00	17.00	23.00	21.50	12.25	12.25	4.00	4.00

Execution ports to units layout:

ALU0/BRU0: ALU
ALU1: ALU
ALU2: ALU
ALU3: ALU
BRU1:
AGU0 (256 bits): store address, load
AGU1 (256 bits): store address, load
AGU2 (256 bits): store address, load
FP0 (256 bits): VPU, DIV/SQRT
FP1 (256 bits): VPU, DIV/SQRT
FP2 (256 bits): VPU
FP3 (256 bits): VPU
FP4 (256 bits): FP store data
FP5 (256 bits): FP store data

Cycles executing div or sqrt instructions	16.00
Longest recurrence chain latency (RecMII)	1.00

Cycles summary

Front-end	10.67
Dispatch	23.00
DIV/SQRT	16.00
Data deps.	1.00
Overall L1	23.00

Vectorization ratios

INT

all	75%
load	NA (no load vectorizable/vectorized instructions)
store	NA (no store vectorizable/vectorized instructions)
mul	NA (no mul vectorizable/vectorized instructions)
add-sub	NA (no add-sub vectorizable/vectorized instructions)
fma	NA (no fma vectorizable/vectorized instructions)
other	75%

all	95%
load	95%
store	100%
mul	100%
add-sub	100%
fma	100%
div/sqrt	100%
other	75%

INT+FP

all	92%
load	95%
store	100%
mul	100%
add-sub	100%
fma	100%
div/sqrt	100%
other	75%

Vector efficiency ratios

INT

all	78%
load	NA (no load vectorizable/vectorized instructions)
store	NA (no store vectorizable/vectorized instructions)
mul	NA (no mul vectorizable/vectorized instructions)
add-sub	NA (no add-sub vectorizable/vectorized instructions)
fma	NA (no fma vectorizable/vectorized instructions)
other	78%

all	96%
load	96%
store	100%
mul	100%
add-sub	100%
fma	100%
div/sqrt	100%
other	78%

INT+FP

all	93%
load	96%
store	100%
mul	100%
add-sub	100%
fma	100%
div/sqrt	100%
other	78%

Cycles and memory resources usage

Assuming all data fit into the L1 cache, each iteration of the binary loop takes 23.00 cycles. At this rate:

21% of peak load performance is reached (20.87 out of 96.00 bytes loaded per cycle (GB/s @ 1GHz))
17% of peak store performance is reached (11.13 out of 64.00 bytes stored per cycle (GB/s @ 1GHz))

Front-end bottlenecks

Found no such bottlenecks.

ASM code

In the binary file, the address of the loop is: 2c900

Instruction	Nb FU	ALU0/BRU0	ALU1	ALU2	ALU3	BRU1	AGU0	AGU1	AGU2	FP0	FP1	FP2	FP3	FP4	FP5	Latency	Recip. throughput
MOV %R13,%RSI	1	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0.17
MOV %R12,%R13	1	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0.17
MOV -0x50(%RBP),%R12	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	3	0.33
VMOVAPD (%RCX),%ZMM8	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	3	1
VMOVDDUP 0x78(%RDI),%XMM13	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	1	0.50
VMOVAPD (%RCX,%R12,8),%ZMM9	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	3	1
MOV %R13,%R12	1	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0.17
VMOVAPD (%RCX,%R12,1),%ZMM10	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	3	1
ADD $0x40,%RCX	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
MOV %RSI,%R13	1	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0.17
VSUBPD %ZMM3,%ZMM8,%ZMM8	1	0	0	0	0	0	0	0	0	0	0	0.50	0.50	0	0	3	1
VMULPD 0x48(%RDI){1to8},%ZMM8,%ZMM11	1	0	0	0	0	0	0.33	0.33	0.33	0.50	0.50	0	0	0	0	3	1
VMULPD 0x60(%RDI){1to8},%ZMM8,%ZMM12	1	0	0	0	0	0	0.33	0.33	0.33	0.50	0.50	0	0	0	0	3	1
VBROADCASTSD %XMM13,%ZMM13	1	0	0	0	0	0	0	0	0	0	0.50	0.50	0	0	0	1	1
VMULPD %ZMM8,%ZMM13,%ZMM8	1	0	0	0	0	0	0	0	0	0.50	0.50	0	0	0	0	3	1
VMOVDQA64 %ZMM7,%ZMM13	1	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0.17
VSUBPD %ZMM4,%ZMM9,%ZMM9	1	0	0	0	0	0	0	0	0	0	0	0.50	0.50	0	0	3	1
VSUBPD %ZMM5,%ZMM10,%ZMM10	1	0	0	0	0	0	0	0	0	0	0	0.50	0.50	0	0	3	1
VFMADD231PD 0x50(%RDI){1to8},%ZMM9,%ZMM11	1	0	0	0	0	0	0.33	0.33	0.33	0.50	0.50	0	0	0	0	4	1
VFMADD231PD 0x68(%RDI){1to8},%ZMM9,%ZMM12	1	0	0	0	0	0	0.33	0.33	0.33	0.50	0.50	0	0	0	0	4	1
VFMADD231PD 0x80(%RDI){1to8},%ZMM9,%ZMM8	1	0	0	0	0	0	0.33	0.33	0.33	0.50	0.50	0	0	0	0	4	1
VMOVDQA64 %ZMM7,%ZMM9	1	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0.17
VFMADD231PD 0x58(%RDI){1to8},%ZMM10,%ZMM11	1	0	0	0	0	0	0.33	0.33	0.33	0.50	0.50	0	0	0	0	4	1
VFMADD231PD 0x70(%RDI){1to8},%ZMM10,%ZMM12	1	0	0	0	0	0	0.33	0.33	0.33	0.50	0.50	0	0	0	0	4	1
VFMADD231PD 0x88(%RDI){1to8},%ZMM10,%ZMM8	1	0	0	0	0	0	0.33	0.33	0.33	0.50	0.50	0	0	0	0	4	1
VMOVDQA64 %ZMM7,%ZMM10	1	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0.17
VPTERNLOGQ $-0x8,%ZMM6,%ZMM11,%ZMM9	1	0	0	0	0	0	0	0	0	0.25	0.25	0.25	0.25	0	0	1	0.50
VPTERNLOGQ $-0x8,%ZMM6,%ZMM12,%ZMM10	1	0	0	0	0	0	0	0	0	0.25	0.25	0.25	0.25	0	0	1	0.50
VPTERNLOGQ $-0x8,%ZMM6,%ZMM8,%ZMM13	1	0	0	0	0	0	0	0	0	0.25	0.25	0.25	0.25	0	0	1	0.50
VADDPD %ZMM9,%ZMM11,%ZMM9	1	0	0	0	0	0	0	0	0	0	0	0.50	0.50	0	0	3	1
VADDPD %ZMM10,%ZMM12,%ZMM10	1	0	0	0	0	0	0	0	0	0	0	0.50	0.50	0	0	3	1
VRNDSCALEPD $0xb,%ZMM9,%ZMM9	1	0	0	0	0	0	0	0	0	0	0	0.50	0.50	0	0	3	1
VRNDSCALEPD $0xb,%ZMM10,%ZMM10	1	0	0	0	0	0	0	0	0	0	0	0.50	0.50	0	0	3	1
VSUBPD %ZMM9,%ZMM11,%ZMM9	1	0	0	0	0	0	0	0	0	0	0	0.50	0.50	0	0	3	1
VADDPD %ZMM13,%ZMM8,%ZMM11	1	0	0	0	0	0	0	0	0	0	0	0.50	0.50	0	0	3	1
VSUBPD %ZMM10,%ZMM12,%ZMM10	1	0	0	0	0	0	0	0	0	0	0	0.50	0.50	0	0	3	1
VMULPD (%RDI){1to8},%ZMM9,%ZMM12	1	0	0	0	0	0	0.33	0.33	0.33	0.50	0.50	0	0	0	0	3	1
VRNDSCALEPD $0xb,%ZMM11,%ZMM11	1	0	0	0	0	0	0	0	0	0	0	0.50	0.50	0	0	3	1
VFMADD231PD 0x8(%RDI){1to8},%ZMM10,%ZMM12	1	0	0	0	0	0	0.33	0.33	0.33	0.50	0.50	0	0	0	0	4	1
VSUBPD %ZMM11,%ZMM8,%ZMM8	1	0	0	0	0	0	0	0	0	0	0	0.50	0.50	0	0	3	1
VFMADD231PD 0x10(%RDI){1to8},%ZMM8,%ZMM12	1	0	0	0	0	0	0.33	0.33	0.33	0.50	0.50	0	0	0	0	4	1
VMOVAPD %ZMM12,(%R10)	2	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	1	1	5	2
VMULPD 0x18(%RDI){1to8},%ZMM9,%ZMM11	1	0	0	0	0	0	0.33	0.33	0.33	0.50	0.50	0	0	0	0	3	1
VFMADD231PD 0x20(%RDI){1to8},%ZMM10,%ZMM11	1	0	0	0	0	0	0.33	0.33	0.33	0.50	0.50	0	0	0	0	4	1
VFMADD231PD 0x28(%RDI){1to8},%ZMM8,%ZMM11	1	0	0	0	0	0	0.33	0.33	0.33	0.50	0.50	0	0	0	0	4	1
VMOVAPD %ZMM11,(%R10,%R9,8)	2	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	1	1	5	2
VMULPD 0x30(%RDI){1to8},%ZMM9,%ZMM9	1	0	0	0	0	0	0.33	0.33	0.33	0.50	0.50	0	0	0	0	3	1
VFMADD231PD 0x38(%RDI){1to8},%ZMM10,%ZMM9	1	0	0	0	0	0	0.33	0.33	0.33	0.50	0.50	0	0	0	0	4	1
VFMADD231PD 0x40(%RDI){1to8},%ZMM8,%ZMM9	1	0	0	0	0	0	0.33	0.33	0.33	0.50	0.50	0	0	0	0	4	1
VMOVAPD %ZMM9,(%R10,%RSI,1)	2	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	1	1	5	2
VMOVAPD (%R10),%ZMM8	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	3	1
VMOVAPD (%R10,%R9,8),%ZMM10	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	3	1
ADD $0x40,%R10	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
VMULPD %ZMM8,%ZMM8,%ZMM8	1	0	0	0	0	0	0	0	0	0.50	0.50	0	0	0	0	3	1
VFMADD231PD %ZMM9,%ZMM9,%ZMM8	1	0	0	0	0	0	0	0	0	0.50	0.50	0	0	0	0	4	1
VFMADD231PD %ZMM10,%ZMM10,%ZMM8	1	0	0	0	0	0	0	0	0	0.50	0.50	0	0	0	0	4	1
VSQRTPD %ZMM8,%ZMM8	1	0	0	0	0	0	0	0	0	0.50	0.50	0	0	0	0	21	16
VMOVAPD %ZMM8,(%R8,%RDX,1)	2	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	1	1	5	2
ADD $0x40,%RDX	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
CMP %RDX,-0x48(%RBP)	1	0.25	0.25	0.25	0.25	0	0.33	0.33	0.33	0	0	0	0	0	0	1	0.33
JNE 2c900 <_ZNK11qmcplusplus10DTD_BCondsIdLj3ELi40EE16computeDistancesINS_10TinyVectorIdLj3EEENS_18VectorSoAContainerIdLj3ENS_10MallocatorIdLm64EEEEES8_EEvRKT_RKT0_PdRT1_iii+0xc0>	1	0.50	0	0	0	0.50	0	0	0	0	0	0	0	0	0	1	0.50-1

Report Configuration

Vectorization

Details

Workaround

Execution units bottlenecks

Workaround

FMA

Workaround

Complex instructions

Details

Type of elements and instruction set

Matching between your loop (in the source code) and the binary loop

Arithmetic intensity

General properties

Front-end

Back-end

Cycles summary

Vectorization ratios

Vector efficiency ratios

Cycles and memory resources usage

Front-end bottlenecks

ASM code

Vectorization

Details

Workaround

Execution units bottlenecks

Workaround

FMA

Workaround

Complex instructions

Details

Type of elements and instruction set

Matching between your loop (in the source code) and the binary loop

Arithmetic intensity

General properties

Front-end

Back-end

Cycles summary

Vectorization ratios

Vector efficiency ratios

Cycles and memory resources usage

Front-end bottlenecks

ASM code