OV - exec - Loop 128

	Loop Id: 128	Module: exec	Source: advec_mom_kernel.f90:215-241 [...]	Coverage: 3.87%

0x41dd90 VPMULLQ	%YMM26,%YMM2,%YMM26

0x41dd96 VPADDQ	%YMM27,%YMM13,%YMM27

0x41dd9c VPADDQ	%YMM26,%YMM27,%YMM26

0x41dda2 VPXORD	%XMM27,%XMM27,%XMM27

0x41dda8 VPCMPEQB	%XMM0,%XMM0,%K3

0x41ddae VGATHERQPD	(,%YMM26,1),%YMM27{%K3}    [2]

0x41ddb9 VANDPD	%YMM12,%YMM23,%YMM26

0x41ddbf VDIVPD	%YMM27,%YMM26,%YMM26

0x41ddc5 VPBLENDMD	%XMM21,%XMM20,%XMM27{%K1}

0x41ddcb VANDPD	%YMM12,%YMM29,%YMM29

0x41ddd1 VANDPD	%YMM12,%YMM25,%YMM30

0x41ddd7 VSUBPD	%YMM26,%YMM15,%YMM31

0x41dddd VMULPD	%YMM31,%YMM30,%YMM31

0x41dde3 VDIVSD	%XMM28,%XMM10,%XMM7

0x41dde9 VBROADCASTSD	%XMM7,%YMM7

0x41ddee VPMOVSXDQ	%XMM27,%YMM27

0x41ddf4 VPSUBQ	%YMM1,%YMM27,%YMM27

0x41ddfa VXORPD	%XMM9,%XMM9,%XMM9

0x41ddff KMOVQ	%K2,%K1

0x41de04 VGATHERQPD	(%RCX,%YMM27,8),%YMM9{%K1}    [7]

0x41de0b VMINPD	%YMM30,%YMM29,%YMM27

0x41de11 VFMADD213PD	%YMM29,%YMM26,%YMM29

0x41de17 VDIVPD	%YMM9,%YMM29,%YMM9

0x41de1d VFMADD231PD	%YMM7,%YMM31,%YMM9

0x41de23 VMULSD	%XMM8,%XMM28,%XMM7

0x41de29 VBROADCASTSD	%XMM7,%YMM7

0x41de2e VMULPD	%YMM7,%YMM9,%YMM7

0x41de32 VMINPD	%YMM27,%YMM7,%YMM7

0x41de38 VCMPPD	$0x2,%YMM11,%YMM25,%K1

0x41de3f VXORPD	%YMM18,%YMM7,%YMM7{%K1}

0x41de45 VMOVAPD	%YMM7,%YMM7{%K2}{z}

0x41de4b VSUBPD	%YMM26,%YMM16,%YMM9

0x41de51 VFMADD213PD	%YMM24,%YMM7,%YMM9

0x41de57 VMULPD	%YMM23,%YMM9,%YMM7

0x41de5d MOV	-0x78(%RBP),%R8    [3]

0x41de61 VMOVUPD	%YMM7,(%R8,%R11,8)    [8]

0x41de67 ADD	$0x4,%R11

0x41de6b CMP	-0xb0(%RBP),%R11    [3]

0x41de72 JA	41df80

0x41de78 LEA	(%R12,%R11,1),%R8

0x41de7c VMOVUPD	(%R14,%R11,8),%YMM23    [4]

0x41de83 VCMPPD	$0x1,%YMM11,%YMM23,%K1

0x41de8a VPBLENDMD	%XMM21,%XMM22,%XMM24{%K1}

0x41de90 VPMOVSXDQ	%XMM24,%YMM24

0x41de96 VPSUBQ	%YMM1,%YMM24,%YMM26

0x41de9c VPBROADCASTQ	%R8,%YMM24

0x41dea2 VPSUBQ	%YMM0,%YMM24,%YMM24

0x41dea8 VPSLLQ	$0x3,%YMM24,%YMM24

0x41deaf VPADDQ	%YMM14,%YMM24,%YMM27

0x41deb5 VPMULLQ	%YMM26,%YMM3,%YMM24

0x41debb VPADDQ	%YMM27,%YMM17,%YMM25

0x41dec1 VPADDQ	%YMM24,%YMM25,%YMM28

0x41dec7 VPXORD	%XMM24,%XMM24,%XMM24

0x41decd VPCMPEQB	%XMM0,%XMM0,%K2

0x41ded3 VGATHERQPD	(,%YMM28,1),%YMM24{%K2}    [6]

0x41dede VPBLENDMD	%XMM19,%XMM20,%XMM28{%K1}

0x41dee4 VPBLENDMD	%XMM22,%XMM21,%XMM29{%K1}

0x41deea VPMOVSXDQ	%XMM28,%YMM28

0x41def0 VPSUBQ	%YMM1,%YMM28,%YMM28

0x41def6 VPMULLQ	%YMM28,%YMM3,%YMM28

0x41defc VPADDQ	%YMM28,%YMM25,%YMM28

0x41df02 VXORPD	%XMM30,%XMM30,%XMM30

0x41df08 VPCMPEQB	%XMM0,%XMM0,%K2

0x41df0e VGATHERQPD	(,%YMM28,1),%YMM30{%K2}    [5]

0x41df19 VPMOVSXDQ	%XMM29,%YMM28

0x41df1f VPSUBQ	%YMM1,%YMM28,%YMM28

0x41df25 VPMULLQ	%YMM28,%YMM3,%YMM28

0x41df2b VPADDQ	%YMM28,%YMM25,%YMM25

0x41df31 VPXORD	%XMM28,%XMM28,%XMM28

0x41df37 VPCMPEQB	%XMM0,%XMM0,%K2

0x41df3d VGATHERQPD	(,%YMM25,1),%YMM28{%K2}    [1]

0x41df48 VSUBPD	%YMM30,%YMM24,%YMM29

0x41df4e VSUBPD	%YMM24,%YMM28,%YMM25

0x41df54 VMULPD	%YMM29,%YMM25,%YMM28

0x41df5a VCMPPD	$0x1,%YMM28,%YMM11,%K2

0x41df61 KORTESTB	%K2,%K2

0x41df65 JE	41dd90

0x41df6b VMOVSD	(%RCX,%R15,8),%XMM28    [7]

0x41df72 JMP	41dd90

/home/eoseret/qaas_runs_GNR/173-814-3702/intel/CloverLeaf1.3-FC/build/CloverLeaf1.3-FC/CloverLeaf_ref/kernels/advec_mom_kernel.f90: 215 - 241

--------------------------------------------------------------------------------

215:           IF(node_flux(j,k).LT.0.0)THEN

[...]

227:           sigma=ABS(node_flux(j,k))/(node_mass_pre(j,donor))

228:           width=celldy(k)

229:           vdiffuw=vel1(j,donor)-vel1(j,upwind)

230:           vdiffdw=vel1(j,downwind)-vel1(j,donor)

231:           limiter=0.0

232:           IF(vdiffuw*vdiffdw.GT.0.0)THEN

233:             auw=ABS(vdiffuw)

234:             adw=ABS(vdiffdw)

235:             wind=1.0_8

236:             IF(vdiffdw.LE.0.0) wind=-1.0_8

237:             limiter=wind*MIN(width*((2.0_8-sigma)*adw/width+(1.0_8+sigma)*auw/celldy(dif))/6.0_8,auw,adw)

238:           ENDIF

239:           advec_vel_s=vel1(j,donor)+(1.0_8-sigma)*limiter

240:           mom_flux(j,k)=advec_vel_s*node_flux(j,k)

241:         ENDDO

Coverage (%)	Name	Source Location	Module
►100.00+	__kmp_invoke_microtask		libiomp5.so
○	__kmp_invoke_task_func		libiomp5.so

Path /

Metric	Value
CQA speedup if no scalar integer	1.02
CQA speedup if FP arith vectorized	1.31
CQA speedup if fully vectorized	1.56
CQA speedup if no inter-iteration dependency	NA
CQA speedup if next bottleneck killed	1.01
Bottlenecks
Function	advec_mom_kernel_mod_mp_advec_mom_kernel_.DIR.OMP.PARALLEL.2
Source	advec_mom_kernel.f90:215-215,advec_mom_kernel.f90:227-241
Source loop unroll info	unrolled by 4
Source loop unroll confidence level	high
Unroll/vectorization loop type	main
Unroll factor	4
CQA cycles	28.17
CQA cycles if no scalar integer	27.50
CQA cycles if FP arith vectorized	21.46
CQA cycles if fully vectorized	18.00
Front-end cycles	19.00
P0 cycles	28.17
P1 cycles	27.83
P2 cycles	7.83
P3 cycles	7.83
P4 cycles	0.50
P5 cycles	28.00
P6 cycles	2.00
P7 cycles	0.50
P8 cycles	0.50
P9 cycles	0.50
P10 cycles	1.00
P11 cycles	7.83
DIV/SQRT cycles	20.00
Inter-iter dependencies cycles	0
FE+BE cycles (UFS)	NA
Stall cycles (UFS)	NA
Nb insns	78.00
Nb uops	114.00
Nb loads	8.50
Nb stores	1.00
Nb stack references	2.00
FLOP/cycle	2.34
Nb FLOP add-sub	16.00
Nb FLOP mul	17.00
Nb FLOP fma	12.00
Nb FLOP div	9.00
Nb FLOP rcp	0.00
Nb FLOP sqrt	0.00
Nb FLOP rsqrt	0.00
Bytes/cycle	8.66
Bytes prefetched	0.00
Bytes loaded	212.00
Bytes stored	32.00
Stride 0	1.50
Stride 1	2.00
Stride n	0.00
Stride unknown	0.00
Stride indirect	4.50
Vectorization ratio all	92.09
Vectorization ratio load	92.86
Vectorization ratio store	100.00
Vectorization ratio mul	88.89
Vectorization ratio add_sub	100.00
Vectorization ratio fma	100.00
Vectorization ratio div_sqrt	66.67
Vectorization ratio other	91.67
Vector-efficiency ratio all	40.92
Vector-efficiency ratio load	47.32
Vector-efficiency ratio store	50.00
Vector-efficiency ratio mul	45.83
Vector-efficiency ratio add_sub	50.00
Vector-efficiency ratio fma	50.00
Vector-efficiency ratio div_sqrt	37.50
Vector-efficiency ratio other	35.07

Metric	Value
CQA speedup if no scalar integer	1.02
CQA speedup if FP arith vectorized	1.31
CQA speedup if fully vectorized	1.56
CQA speedup if no inter-iteration dependency	NA
CQA speedup if next bottleneck killed	1.01
Bottlenecks	P0,
Function	advec_mom_kernel_mod_mp_advec_mom_kernel_.DIR.OMP.PARALLEL.2
Source	advec_mom_kernel.f90:215-215,advec_mom_kernel.f90:227-241
Source loop unroll info	unrolled by 4
Source loop unroll confidence level	high
Unroll/vectorization loop type	main
Unroll factor	4
CQA cycles	28.17
CQA cycles if no scalar integer	27.50
CQA cycles if FP arith vectorized	21.46
CQA cycles if fully vectorized	18.00
Front-end cycles	19.17
P0 cycles	28.17
P1 cycles	27.83
P2 cycles	8.00
P3 cycles	8.00
P4 cycles	0.50
P5 cycles	28.00
P6 cycles	2.00
P7 cycles	0.50
P8 cycles	0.50
P9 cycles	0.50
P10 cycles	1.00
P11 cycles	8.00
DIV/SQRT cycles	20.00
Inter-iter dependencies cycles	0
FE+BE cycles (UFS)	NA
Stall cycles (UFS)	NA
Nb insns	79.00
Nb uops	115.00
Nb loads	9.00
Nb stores	1.00
Nb stack references	2.00
FLOP/cycle	2.34
Nb FLOP add-sub	16.00
Nb FLOP mul	17.00
Nb FLOP fma	12.00
Nb FLOP div	9.00
Nb FLOP rcp	0.00
Nb FLOP sqrt	0.00
Nb FLOP rsqrt	0.00
Bytes/cycle	8.80
Bytes prefetched	0.00
Bytes loaded	216.00
Bytes stored	32.00
Stride 0	2.00
Stride 1	2.00
Stride n	0.00
Stride unknown	0.00
Stride indirect	4.00
Vectorization ratio all	91.43
Vectorization ratio load	85.71
Vectorization ratio store	100.00
Vectorization ratio mul	88.89
Vectorization ratio add_sub	100.00
Vectorization ratio fma	100.00
Vectorization ratio div_sqrt	66.67
Vectorization ratio other	91.67
Vector-efficiency ratio all	40.71
Vector-efficiency ratio load	44.64
Vector-efficiency ratio store	50.00
Vector-efficiency ratio mul	45.83
Vector-efficiency ratio add_sub	50.00
Vector-efficiency ratio fma	50.00
Vector-efficiency ratio div_sqrt	37.50
Vector-efficiency ratio other	35.07

Metric	Value
CQA speedup if no scalar integer	1.02
CQA speedup if FP arith vectorized	1.31
CQA speedup if fully vectorized	1.56
CQA speedup if no inter-iteration dependency	NA
CQA speedup if next bottleneck killed	1.01
Bottlenecks	P0,
Function	advec_mom_kernel_mod_mp_advec_mom_kernel_.DIR.OMP.PARALLEL.2
Source	advec_mom_kernel.f90:215-215,advec_mom_kernel.f90:227-241
Source loop unroll info	unrolled by 4
Source loop unroll confidence level	high
Unroll/vectorization loop type	main
Unroll factor	4
CQA cycles	28.17
CQA cycles if no scalar integer	27.50
CQA cycles if FP arith vectorized	21.46
CQA cycles if fully vectorized	18.00
Front-end cycles	18.83
P0 cycles	28.17
P1 cycles	27.83
P2 cycles	7.67
P3 cycles	7.67
P4 cycles	0.50
P5 cycles	28.00
P6 cycles	2.00
P7 cycles	0.50
P8 cycles	0.50
P9 cycles	0.50
P10 cycles	1.00
P11 cycles	7.67
DIV/SQRT cycles	20.00
Inter-iter dependencies cycles	0
FE+BE cycles (UFS)	NA
Stall cycles (UFS)	NA
Nb insns	77.00
Nb uops	113.00
Nb loads	8.00
Nb stores	1.00
Nb stack references	2.00
FLOP/cycle	2.34
Nb FLOP add-sub	16.00
Nb FLOP mul	17.00
Nb FLOP fma	12.00
Nb FLOP div	9.00
Nb FLOP rcp	0.00
Nb FLOP sqrt	0.00
Nb FLOP rsqrt	0.00
Bytes/cycle	8.52
Bytes prefetched	0.00
Bytes loaded	208.00
Bytes stored	32.00
Stride 0	1.00
Stride 1	2.00
Stride n	0.00
Stride unknown	0.00
Stride indirect	5.00
Vectorization ratio all	92.75
Vectorization ratio load	100.00
Vectorization ratio store	100.00
Vectorization ratio mul	88.89
Vectorization ratio add_sub	100.00
Vectorization ratio fma	100.00
Vectorization ratio div_sqrt	66.67
Vectorization ratio other	91.67
Vector-efficiency ratio all	41.12
Vector-efficiency ratio load	50.00
Vector-efficiency ratio store	50.00
Vector-efficiency ratio mul	45.83
Vector-efficiency ratio add_sub	50.00
Vector-efficiency ratio fma	50.00
Vector-efficiency ratio div_sqrt	37.50
Vector-efficiency ratio other	35.07

Path /

Average path: Display a virtual path defined by average values of all real paths

Function	advec_mom_kernel_mod_mp_advec_mom_kernel_.DIR.OMP.PARALLEL.2
Source file and lines	advec_mom_kernel.f90:215-241
Module	exec

The loop is defined in /home/eoseret/qaas_runs_GNR/173-814-3702/intel/CloverLeaf1.3-FC/build/CloverLeaf1.3-FC/CloverLeaf_ref/kernels/advec_mom_kernel.f90:215,227-241.

It is main loop of related source loop which is unrolled by 4 (including vectorization).
The structure of this loop is probably <if then [else] end>.

The presence of multiple execution paths is typically the main/first bottleneck.
Try to simplify control inside loop: ideally, try to remove all conditional expressions, for example by (if applicable):

hoisting them (moving them outside the loop)
turning them into conditional moves, MIN or MAX

Ex: if (x<0) x=0 => x = max(0,x) (Fortran instrinsic procedure)

gain
potential
hint
expert

Vectorization

Your loop is highly vectorized. Only 40% of vector register length is used (average across all SSE/AVX instructions).

Details

92% of SSE/AVX instructions are used in vector version (process two or more data elements in vector registers):

92% of SSE/AVX loads are used in vector version.
88% of SSE/AVX multiply instructions are used in vector version.
66% of SSE/AVX divide and square root instructions are used in vector version.
91% of SSE/AVX instructions that are not load, store, addition, subtraction nor multiply instructions are used in vector version.

Workaround

Read the "512-bits vectorization" report at "Potential" confidence level.

512-bits vectorization

On some x86 processors supporting 512-bits vectorization, compilers are often too conservative and limit vectorization to 256 bits. Performance can then be improved by enforcing 512-bits vectorization, especially with many vectorized and high trip count loops. 512-bits vectorization performance overhead (compared to 256-bits) is generally lower on newer processors.

Workaround

Recompile with -qopt-zmm-usage=high

Masked instructions

Detected masked instructions.

Details

Vector registers are partially exploited, which is expected if your loop is irregular or mixes elements of different sizes.

Workaround

If your loop is irregular, try to remove or hoist conditional structures out of your loop. If it mixes elements of different sizes, try to uniformize them.

FMA

Detected 12 FMA (fused multiply-add) operations. Presence of both ADD/SUB and MUL operations.

Workaround

Try to change order in which elements are evaluated (using parentheses) in arithmetic expressions containing both ADD/SUB and MUL operations to enable your compiler to generate FMA instructions wherever possible. For instance a + b*c is a valid FMA (MUL then ADD). However (a+b)* c cannot be translated into an FMA (ADD then MUL).

Complex instructions

Detected COMPLEX INSTRUCTIONS.

Details

These instructions generate more than one micro-operation and only one of them can be decoded during a cycle and the extra micro-operations increase pressure on execution units.

VGATHERQPD: 0 occurrences
VPMULLQ: 0 occurrences

Slow data structures access

Detected data structures (typically arrays) that cannot be efficiently read/written

Details

Irregular (variable stride) or indirect: 4 occurrence(s)

Non-unit stride (uncontiguous) accesses are not efficiently using data caches

Workaround

Try to remove indirect accesses. If applicable, precompute elements out of the innermost loop.

Gather/scatter instructions

Detected gather/scatter instructions (typically caused by indirect accesses).

Details

VGATHERQPD: 0 occurrences

Workaround

Try to simplify your code and/or replace indirect accesses with unit-stride ones.

Type of elements and instruction set

2 SSE or AVX instructions are processing arithmetic or math operations on double precision FP elements in scalar mode (one at a time). 18 AVX instructions are processing arithmetic or math operations on double precision FP elements in vector mode (four at a time).

Matching between your loop (in the source code) and the binary loop

The binary loop is composed of 66 FP arithmetical operations:

28: addition or subtraction (12 inside FMA instructions)
29: multiply (12 inside FMA instructions)
9: divide

The binary loop is loading 212 bytes (26 double precision FP elements). The binary loop is storing 32 bytes (4 double precision FP elements).

Arithmetic intensity

Arithmetic intensity is 0.27 FP operations per loaded or stored byte.

General properties

nb instructions	78
nb uops	114
loop length	481
used x86 registers	6.50
used mmx registers	0
used xmm registers	14
used ymm registers	23
used zmm registers	0
nb stack references	2
ADD-SUB / MUL ratio	0.80

Front-end

MACRO FUSION NOT POSSIBLE

micro-operation queue	19.00 cycles
front end	19.00 cycles

Back-end

	P0	P1	P2	P3	P4	P5	P6	P7	P8	P9	P10	P11
uops	28.17	27.83	7.83	7.83	0.50	28.00	2.00	0.50	0.50	0.50	1.00	7.83
cycles	28.17	27.83	7.83	7.83	0.50	28.00	2.00	0.50	0.50	0.50	1.00	7.83

Execution ports to units layout:

P0 (256 bits): VPU, BRU, ALU, DIV/SQRT
P1 (256 bits): ALU, VPU
P2 (512 bits): load
P3 (512 bits): load
P4 (512 bits): store data
P5 (512 bits): ALU, VPU
P6: ALU, BRU
P7: store address
P8: store address
P9 (512 bits): store data
P10: ALU
P11 (512 bits): load

Cycles executing div or sqrt instructions	20.00
Longest recurrence chain latency (RecMII)	0.00

Cycles summary

Front-end	19.00
Dispatch	28.17
DIV/SQRT	20.00
Data deps.	0.00
Overall L1	28.17

Vectorization ratios

INT

all	96%
load	NA (no load vectorizable/vectorized instructions)
store	NA (no store vectorizable/vectorized instructions)
mul	100%
add-sub	100%
fma	NA (no fma vectorizable/vectorized instructions)
other	94%

all	87%
load	92%
store	100%
mul	80%
add-sub	100%
fma	100%
div/sqrt	66%
other	89%

INT+FP

all	92%
load	92%
store	100%
mul	88%
add-sub	100%
fma	100%
div/sqrt	66%
other	91%

Vector efficiency ratios

INT

all	37%
load	NA (no load vectorizable/vectorized instructions)
store	NA (no store vectorizable/vectorized instructions)
mul	50%
add-sub	50%
fma	NA (no fma vectorizable/vectorized instructions)
other	25%

all	44%
load	47%
store	50%
mul	42%
add-sub	50%
fma	50%
div/sqrt	37%
other	43%

INT+FP

all	40%
load	47%
store	50%
mul	45%
add-sub	50%
fma	50%
div/sqrt	37%
other	35%

Cycles and memory resources usage

Detected masked instructions: assuming all mask elements are active. Assuming all data fit into the L1 cache, each iteration of the binary loop takes 28.17 cycles. At this rate:

3% of peak load performance is reached (7.53 out of 192.00 bytes loaded per cycle (GB/s @ 1GHz))
0% of peak store performance is reached (1.14 out of 128.00 bytes stored per cycle (GB/s @ 1GHz))

Function	advec_mom_kernel_mod_mp_advec_mom_kernel_.DIR.OMP.PARALLEL.2
Source file and lines	advec_mom_kernel.f90:215-241
Module	exec

hoisting them (moving them outside the loop)
turning them into conditional moves, MIN or MAX

Ex: if (x<0) x=0 => x = max(0,x) (Fortran instrinsic procedure)

gain
potential
hint
expert

Vectorization

Your loop is highly vectorized. Only 40% of vector register length is used (average across all SSE/AVX instructions).

Details

91% of SSE/AVX instructions are used in vector version (process two or more data elements in vector registers):

85% of SSE/AVX loads are used in vector version.
88% of SSE/AVX multiply instructions are used in vector version.
66% of SSE/AVX divide and square root instructions are used in vector version.
91% of SSE/AVX instructions that are not load, store, addition, subtraction nor multiply instructions are used in vector version.

Workaround

Read the "512-bits vectorization" report at "Potential" confidence level.

Execution units bottlenecks

Performance is limited by execution of FP multiply or FMA (fused multiply-add) operations (the FP multiply/FMA unit is a bottleneck).

Workaround

Reduce the number of FP multiply/FMA instructions

512-bits vectorization

Workaround

Recompile with -qopt-zmm-usage=high

Masked instructions

Detected masked instructions.

Details

Vector registers are partially exploited, which is expected if your loop is irregular or mixes elements of different sizes.

Workaround

If your loop is irregular, try to remove or hoist conditional structures out of your loop. If it mixes elements of different sizes, try to uniformize them.

FMA

Detected 12 FMA (fused multiply-add) operations. Presence of both ADD/SUB and MUL operations.

Workaround

Complex instructions

Detected COMPLEX INSTRUCTIONS.

Details

These instructions generate more than one micro-operation and only one of them can be decoded during a cycle and the extra micro-operations increase pressure on execution units.

VGATHERQPD: 5 occurrences►
- /home/eoseret/qaas_runs_GNR/173-814-3702/intel/CloverLeaf1.3-FC/build/CloverLeaf1.3-FC/CloverLeaf_ref/kernels/advec_mom_kernel.f90:227
- /home/eoseret/qaas_runs_GNR/173-814-3702/intel/CloverLeaf1.3-FC/build/CloverLeaf1.3-FC/CloverLeaf_ref/kernels/advec_mom_kernel.f90:237
- /home/eoseret/qaas_runs_GNR/173-814-3702/intel/CloverLeaf1.3-FC/build/CloverLeaf1.3-FC/CloverLeaf_ref/kernels/advec_mom_kernel.f90:229
- /home/eoseret/qaas_runs_GNR/173-814-3702/intel/CloverLeaf1.3-FC/build/CloverLeaf1.3-FC/CloverLeaf_ref/kernels/advec_mom_kernel.f90:229
- /home/eoseret/qaas_runs_GNR/173-814-3702/intel/CloverLeaf1.3-FC/build/CloverLeaf1.3-FC/CloverLeaf_ref/kernels/advec_mom_kernel.f90:230
VPMULLQ: 4 occurrences►
- /home/eoseret/qaas_runs_GNR/173-814-3702/intel/CloverLeaf1.3-FC/build/CloverLeaf1.3-FC/CloverLeaf_ref/kernels/advec_mom_kernel.f90:227
- /home/eoseret/qaas_runs_GNR/173-814-3702/intel/CloverLeaf1.3-FC/build/CloverLeaf1.3-FC/CloverLeaf_ref/kernels/advec_mom_kernel.f90:229
- /home/eoseret/qaas_runs_GNR/173-814-3702/intel/CloverLeaf1.3-FC/build/CloverLeaf1.3-FC/CloverLeaf_ref/kernels/advec_mom_kernel.f90:229
- /home/eoseret/qaas_runs_GNR/173-814-3702/intel/CloverLeaf1.3-FC/build/CloverLeaf1.3-FC/CloverLeaf_ref/kernels/advec_mom_kernel.f90:230

Slow data structures access

Detected data structures (typically arrays) that cannot be efficiently read/written

Details

Irregular (variable stride) or indirect: 4 occurrence(s)

Non-unit stride (uncontiguous) accesses are not efficiently using data caches

Workaround

Try to remove indirect accesses. If applicable, precompute elements out of the innermost loop.

Vector unaligned load/store instructions

Detected 2 optimal vector unaligned load/store instructions.

Details

VMOVUPD: 2 occurrences►
- /home/eoseret/qaas_runs_GNR/173-814-3702/intel/CloverLeaf1.3-FC/build/CloverLeaf1.3-FC/CloverLeaf_ref/kernels/advec_mom_kernel.f90:240
- /home/eoseret/qaas_runs_GNR/173-814-3702/intel/CloverLeaf1.3-FC/build/CloverLeaf1.3-FC/CloverLeaf_ref/kernels/advec_mom_kernel.f90:215

Workaround

Use vector aligned instructions:

align your arrays on 64 bytes boundaries: compile with -align array64byte (remark: not affects arrays in COMMON blocks).
inform your compiler that your arrays are vector aligned: Append !DIR$ VECTOR ALIGNED to the loop if all accessed arrays are aligned, or !DIR$ ASSUME_ALIGNED FOO: 64 if only FOO is aligned.

Gather/scatter instructions

Detected gather/scatter instructions (typically caused by indirect accesses).

Details

VGATHERQPD: 5 occurrences►
- /home/eoseret/qaas_runs_GNR/173-814-3702/intel/CloverLeaf1.3-FC/build/CloverLeaf1.3-FC/CloverLeaf_ref/kernels/advec_mom_kernel.f90:227
- /home/eoseret/qaas_runs_GNR/173-814-3702/intel/CloverLeaf1.3-FC/build/CloverLeaf1.3-FC/CloverLeaf_ref/kernels/advec_mom_kernel.f90:237
- /home/eoseret/qaas_runs_GNR/173-814-3702/intel/CloverLeaf1.3-FC/build/CloverLeaf1.3-FC/CloverLeaf_ref/kernels/advec_mom_kernel.f90:229
- /home/eoseret/qaas_runs_GNR/173-814-3702/intel/CloverLeaf1.3-FC/build/CloverLeaf1.3-FC/CloverLeaf_ref/kernels/advec_mom_kernel.f90:229
- /home/eoseret/qaas_runs_GNR/173-814-3702/intel/CloverLeaf1.3-FC/build/CloverLeaf1.3-FC/CloverLeaf_ref/kernels/advec_mom_kernel.f90:230

Workaround

Try to simplify your code and/or replace indirect accesses with unit-stride ones.

Type of elements and instruction set

Matching between your loop (in the source code) and the binary loop

The binary loop is composed of 66 FP arithmetical operations:

28: addition or subtraction (12 inside FMA instructions)
29: multiply (12 inside FMA instructions)
9: divide

The binary loop is loading 216 bytes (27 double precision FP elements). The binary loop is storing 32 bytes (4 double precision FP elements).

Arithmetic intensity

Arithmetic intensity is 0.27 FP operations per loaded or stored byte.

General properties

nb instructions	79
nb uops	115
loop length	487
used x86 registers	7
used mmx registers	0
used xmm registers	14
used ymm registers	23
used zmm registers	0
nb stack references	2
ADD-SUB / MUL ratio	0.80

Front-end

MACRO FUSION NOT POSSIBLE FIT IN UOP CACHE

micro-operation queue	19.17 cycles
front end	19.17 cycles

Back-end

	P0	P1	P2	P3	P4	P5	P6	P7	P8	P9	P10	P11
uops	28.17	27.83	8.00	8.00	0.50	28.00	2.00	0.50	0.50	0.50	1.00	8.00
cycles	28.17	27.83	8.00	8.00	0.50	28.00	2.00	0.50	0.50	0.50	1.00	8.00

Execution ports to units layout:

P0 (256 bits): VPU, BRU, ALU, DIV/SQRT
P1 (256 bits): ALU, VPU
P2 (512 bits): load
P3 (512 bits): load
P4 (512 bits): store data
P5 (512 bits): ALU, VPU
P6: ALU, BRU
P7: store address
P8: store address
P9 (512 bits): store data
P10: ALU
P11 (512 bits): load

Cycles executing div or sqrt instructions	20.00
Longest recurrence chain latency (RecMII)	0.00

Cycles summary

Front-end	19.17
Dispatch	28.17
DIV/SQRT	20.00
Data deps.	0.00
Overall L1	28.17

Vectorization ratios

INT

all	96%
load	NA (no load vectorizable/vectorized instructions)
store	NA (no store vectorizable/vectorized instructions)
mul	100%
add-sub	100%
fma	NA (no fma vectorizable/vectorized instructions)
other	94%

all	86%
load	85%
store	100%
mul	80%
add-sub	100%
fma	100%
div/sqrt	66%
other	89%

INT+FP

all	91%
load	85%
store	100%
mul	88%
add-sub	100%
fma	100%
div/sqrt	66%
other	91%

Vector efficiency ratios

INT

all	37%
load	NA (no load vectorizable/vectorized instructions)
store	NA (no store vectorizable/vectorized instructions)
mul	50%
add-sub	50%
fma	NA (no fma vectorizable/vectorized instructions)
other	25%

all	43%
load	44%
store	50%
mul	42%
add-sub	50%
fma	50%
div/sqrt	37%
other	43%

INT+FP

all	40%
load	44%
store	50%
mul	45%
add-sub	50%
fma	50%
div/sqrt	37%
other	35%

Cycles and memory resources usage

Detected masked instructions: assuming all mask elements are active. Assuming all data fit into the L1 cache, each iteration of the binary loop takes 28.17 cycles. At this rate:

3% of peak load performance is reached (7.67 out of 192.00 bytes loaded per cycle (GB/s @ 1GHz))
0% of peak store performance is reached (1.14 out of 128.00 bytes stored per cycle (GB/s @ 1GHz))

Front-end bottlenecks

Found no such bottlenecks.

ASM code

In the binary file, the address of the loop is: 41de78

Instruction	Nb FU	P0	P1	P2	P3	P4	P5	P6	P7	P8	P9	P10	P11	Latency	Recip. throughput	Vectorization
VPMULLQ %YMM26,%YMM2,%YMM26	5	1.50	1.50	0	0	0	0	0	0	0	0	0	0	15	1.50	vect (50.0%)
VPADDQ %YMM27,%YMM13,%YMM27	1	0.33	0.33	0	0	0	0.33	0	0	0	0	0	0	1	0.33	vect (50.0%)
VPADDQ %YMM26,%YMM27,%YMM26	1	0.33	0.33	0	0	0	0.33	0	0	0	0	0	0	1	0.33	vect (50.0%)
VPXORD %XMM27,%XMM27,%XMM27	1	0.33	0.33	0	0	0	0.33	0	0	0	0	0	0	0-1	0.33	vect (25.0%)
VPCMPEQB %XMM0,%XMM0,%K3	1	0	0	0	0	0	1	0	0	0	0	0	0	3	1	vect (25.0%)
VGATHERQPD (,%YMM26,1),%YMM27{%K3}	5	1	1	1.33	1.33	0	1	0	0	0	0	0	1.33	28	2	vect (50.0%)
VANDPD %YMM12,%YMM23,%YMM26	1	0.33	0.33	0	0	0	0.33	0	0	0	0	0	0	1	0.33	vect (50.0%)
VDIVPD %YMM27,%YMM26,%YMM26	1	1	0	0	0	0	0	0	0	0	0	0	0	13-15	8	vect (50.0%)
VPBLENDMD %XMM21,%XMM20,%XMM27{%K1}	1	0.33	0.33	0	0	0	0.33	0	0	0	0	0	0	1	0.33	vect (25.0%)
VANDPD %YMM12,%YMM29,%YMM29	1	0.33	0.33	0	0	0	0.33	0	0	0	0	0	0	1	0.33	vect (50.0%)
VANDPD %YMM12,%YMM25,%YMM30	1	0.33	0.33	0	0	0	0.33	0	0	0	0	0	0	1	0.33	vect (50.0%)
VSUBPD %YMM26,%YMM15,%YMM31	1	0	0.50	0	0	0	0.50	0	0	0	0	0	0	3	0.50	vect (50.0%)
VMULPD %YMM31,%YMM30,%YMM31	1	0.50	0.50	0	0	0	0	0	0	0	0	0	0	4	0.50	vect (50.0%)
VDIVSD %XMM28,%XMM10,%XMM7	1	1	0	0	0	0	0	0	0	0	0	0	0	13-15	4	scal (12.5%)
VBROADCASTSD %XMM7,%YMM7	1	0	0	0	0	0	1	0	0	0	0	0	0	3	1	scal (12.5%)
VPMOVSXDQ %XMM27,%YMM27	1	0	0	0	0	0	1	0	0	0	0	0	0	3	1	vect (25.0%)
VPSUBQ %YMM1,%YMM27,%YMM27	1	0.33	0.33	0	0	0	0.33	0	0	0	0	0	0	0-1	0.33	vect (50.0%)
VXORPD %XMM9,%XMM9,%XMM9	1	0	0	0	0	0	0	0	0	0	0	0	0	0	0.17	vect (25.0%)
KMOVQ %K2,%K1	1	1	0	0	0	0	0	0	0	0	0	0	0	0	1	N/A
VGATHERQPD (%RCX,%YMM27,8),%YMM9{%K1}	5	1	1	1.33	1.33	0	1	0	0	0	0	0	1.33	28	2	vect (50.0%)
VMINPD %YMM30,%YMM29,%YMM27	1	0.50	0.50	0	0	0	0	0	0	0	0	0	0	4	0.50	vect (50.0%)
VFMADD213PD %YMM29,%YMM26,%YMM29	1	0.50	0.50	0	0	0	0	0	0	0	0	0	0	4	0.50	vect (50.0%)
VDIVPD %YMM9,%YMM29,%YMM9	1	1	0	0	0	0	0	0	0	0	0	0	0	13-15	8	vect (50.0%)
VFMADD231PD %YMM7,%YMM31,%YMM9	1	0.50	0.50	0	0	0	0	0	0	0	0	0	0	4	0.50	vect (50.0%)
VMULSD %XMM8,%XMM28,%XMM7	1	0.50	0.50	0	0	0	0	0	0	0	0	0	0	4	0.50	scal (12.5%)
VBROADCASTSD %XMM7,%YMM7	1	0	0	0	0	0	1	0	0	0	0	0	0	3	1	scal (12.5%)
VMULPD %YMM7,%YMM9,%YMM7	1	0.50	0.50	0	0	0	0	0	0	0	0	0	0	4	0.50	vect (50.0%)
VMINPD %YMM27,%YMM7,%YMM7	1	0.50	0.50	0	0	0	0	0	0	0	0	0	0	4	0.50	vect (50.0%)
VCMPPD $0x2,%YMM11,%YMM25,%K1	1	0	0	0	0	0	1	0	0	0	0	0	0	3	1	vect (50.0%)
VXORPD %YMM18,%YMM7,%YMM7{%K1}	1	0.33	0.33	0	0	0	0.33	0	0	0	0	0	0	0-1	0.33	vect (50.0%)
VMOVAPD %YMM7,%YMM7{%K2}{z}	1	0	0	0	0	0	0	0	0	0	0	0	0	0-1	0.17	vect (50.0%)
VSUBPD %YMM26,%YMM16,%YMM9	1	0	0.50	0	0	0	0.50	0	0	0	0	0	0	3	0.50	vect (50.0%)
VFMADD213PD %YMM24,%YMM7,%YMM9	1	0.50	0.50	0	0	0	0	0	0	0	0	0	0	4	0.50	vect (50.0%)
VMULPD %YMM23,%YMM9,%YMM7	1	0.50	0.50	0	0	0	0	0	0	0	0	0	0	4	0.50	vect (50.0%)
MOV -0x78(%RBP),%R8	1	0	0	0.33	0.33	0	0	0	0	0	0	0	0.33	1	0.33	N/A
VMOVUPD %YMM7,(%R8,%R11,8)	1	0	0	0	0	0.50	0	0	0.50	0.50	0.50	0	0	0-1	0.50	vect (50.0%)
ADD $0x4,%R11	1	0	0	0	0	0	0	0	0	0	0	0	0	1	0.17	N/A
CMP -0xb0(%RBP),%R11	1	0.20	0.20	0.33	0.33	0	0.20	0.20	0	0	0	0.20	0.33	1	0.33	N/A
JA 41df80 <advec_mom_kernel_mod_mp_advec_mom_kernel_.DIR.OMP.PARALLEL.2+0x25d0>	1	0.50	0	0	0	0	0	0.50	0	0	0	0	0	0	0.50	N/A
LEA (%R12,%R11,1),%R8	1	0	0	0	0	0	0	0	0	0	0	0	0	0	0.17	N/A
VMOVUPD (%R14,%R11,8),%YMM23	1	0	0	0.33	0.33	0	0	0	0	0	0	0	0.33	0-1	0.33	vect (50.0%)
VCMPPD $0x1,%YMM11,%YMM23,%K1	1	0	0	0	0	0	1	0	0	0	0	0	0	3	1	vect (50.0%)
VPBLENDMD %XMM21,%XMM22,%XMM24{%K1}	1	0.33	0.33	0	0	0	0.33	0	0	0	0	0	0	1	0.33	vect (25.0%)
VPMOVSXDQ %XMM24,%YMM24	1	0	0	0	0	0	1	0	0	0	0	0	0	3	1	vect (25.0%)
VPSUBQ %YMM1,%YMM24,%YMM26	1	0.33	0.33	0	0	0	0.33	0	0	0	0	0	0	0-1	0.33	vect (50.0%)
VPBROADCASTQ %R8,%YMM24	1	0	0	0	0	0	1	0	0	0	0	0	0	3	1	scal (12.5%)
VPSUBQ %YMM0,%YMM24,%YMM24	1	0.33	0.33	0	0	0	0.33	0	0	0	0	0	0	0-1	0.33	vect (50.0%)
VPSLLQ $0x3,%YMM24,%YMM24	1	0.50	0.50	0	0	0	0	0	0	0	0	0	0	2-4	0.50	vect (50.0%)
VPADDQ %YMM14,%YMM24,%YMM27	1	0.33	0.33	0	0	0	0.33	0	0	0	0	0	0	1	0.33	vect (50.0%)
VPMULLQ %YMM26,%YMM3,%YMM24	5	1.50	1.50	0	0	0	0	0	0	0	0	0	0	15	1.50	vect (50.0%)
VPADDQ %YMM27,%YMM17,%YMM25	1	0.33	0.33	0	0	0	0.33	0	0	0	0	0	0	1	0.33	vect (50.0%)
VPADDQ %YMM24,%YMM25,%YMM28	1	0.33	0.33	0	0	0	0.33	0	0	0	0	0	0	1	0.33	vect (50.0%)
VPXORD %XMM24,%XMM24,%XMM24	1	0.33	0.33	0	0	0	0.33	0	0	0	0	0	0	0-1	0.33	vect (25.0%)
VPCMPEQB %XMM0,%XMM0,%K2	1	0	0	0	0	0	1	0	0	0	0	0	0	3	1	vect (25.0%)
VGATHERQPD (,%YMM28,1),%YMM24{%K2}	5	1	1	1.33	1.33	0	1	0	0	0	0	0	1.33	28	2	vect (50.0%)
VPBLENDMD %XMM19,%XMM20,%XMM28{%K1}	1	0.33	0.33	0	0	0	0.33	0	0	0	0	0	0	1	0.33	vect (25.0%)
VPBLENDMD %XMM22,%XMM21,%XMM29{%K1}	1	0.33	0.33	0	0	0	0.33	0	0	0	0	0	0	1	0.33	vect (25.0%)
VPMOVSXDQ %XMM28,%YMM28	1	0	0	0	0	0	1	0	0	0	0	0	0	3	1	vect (25.0%)
VPSUBQ %YMM1,%YMM28,%YMM28	1	0.33	0.33	0	0	0	0.33	0	0	0	0	0	0	0-1	0.33	vect (50.0%)
VPMULLQ %YMM28,%YMM3,%YMM28	5	1.50	1.50	0	0	0	0	0	0	0	0	0	0	15	1.50	vect (50.0%)
VPADDQ %YMM28,%YMM25,%YMM28	1	0.33	0.33	0	0	0	0.33	0	0	0	0	0	0	1	0.33	vect (50.0%)
VXORPD %XMM30,%XMM30,%XMM30	1	0	0	0	0	0	0	0	0	0	0	0	0	0	0.17	vect (25.0%)
VPCMPEQB %XMM0,%XMM0,%K2	1	0	0	0	0	0	1	0	0	0	0	0	0	3	1	vect (25.0%)
VGATHERQPD (,%YMM28,1),%YMM30{%K2}	5	1	1	1.33	1.33	0	1	0	0	0	0	0	1.33	28	2	vect (50.0%)
VPMOVSXDQ %XMM29,%YMM28	1	0	0	0	0	0	1	0	0	0	0	0	0	3	1	vect (25.0%)
VPSUBQ %YMM1,%YMM28,%YMM28	1	0.33	0.33	0	0	0	0.33	0	0	0	0	0	0	0-1	0.33	vect (50.0%)
VPMULLQ %YMM28,%YMM3,%YMM28	5	1.50	1.50	0	0	0	0	0	0	0	0	0	0	15	1.50	vect (50.0%)
VPADDQ %YMM28,%YMM25,%YMM25	1	0.33	0.33	0	0	0	0.33	0	0	0	0	0	0	1	0.33	vect (50.0%)
VPXORD %XMM28,%XMM28,%XMM28	1	0.33	0.33	0	0	0	0.33	0	0	0	0	0	0	0-1	0.33	vect (25.0%)
VPCMPEQB %XMM0,%XMM0,%K2	1	0	0	0	0	0	1	0	0	0	0	0	0	3	1	vect (25.0%)
VGATHERQPD (,%YMM25,1),%YMM28{%K2}	5	1	1	1.33	1.33	0	1	0	0	0	0	0	1.33	28	2	vect (50.0%)
VSUBPD %YMM30,%YMM24,%YMM29	1	0	0.50	0	0	0	0.50	0	0	0	0	0	0	3	0.50	vect (50.0%)
VSUBPD %YMM24,%YMM28,%YMM25	1	0	0.50	0	0	0	0.50	0	0	0	0	0	0	3	0.50	vect (50.0%)
VMULPD %YMM29,%YMM25,%YMM28	1	0.50	0.50	0	0	0	0	0	0	0	0	0	0	4	0.50	vect (50.0%)
VCMPPD $0x1,%YMM28,%YMM11,%K2	1	0	0	0	0	0	1	0	0	0	0	0	0	3	1	vect (50.0%)
KORTESTB %K2,%K2	1	1	0	0	0	0	0	0	0	0	0	0	0	0	1	N/A
JE 41dd90 <advec_mom_kernel_mod_mp_advec_mom_kernel_.DIR.OMP.PARALLEL.2+0x23e0>	1	0.50	0	0	0	0	0	0.50	0	0	0	0	0	0	0.50	N/A
VMOVSD (%RCX,%R15,8),%XMM28	1	0	0	0.33	0.33	0	0	0	0	0	0	0	0.33	1	0.33	scal (12.5%)
JMP 41dd90 <advec_mom_kernel_mod_mp_advec_mom_kernel_.DIR.OMP.PARALLEL.2+0x23e0>	1	0	0	0	0	0	0	0	0	0	0	0	0	0	2.08	N/A

Function	advec_mom_kernel_mod_mp_advec_mom_kernel_.DIR.OMP.PARALLEL.2
Source file and lines	advec_mom_kernel.f90:215-241
Module	exec

hoisting them (moving them outside the loop)
turning them into conditional moves, MIN or MAX

Ex: if (x<0) x=0 => x = max(0,x) (Fortran instrinsic procedure)

gain
potential
hint
expert

Vectorization

Your loop is highly vectorized. Only 41% of vector register length is used (average across all SSE/AVX instructions).

Details

92% of SSE/AVX instructions are used in vector version (process two or more data elements in vector registers):

88% of SSE/AVX multiply instructions are used in vector version.
66% of SSE/AVX divide and square root instructions are used in vector version.
91% of SSE/AVX instructions that are not load, store, addition, subtraction nor multiply instructions are used in vector version.

Workaround

Read the "512-bits vectorization" report at "Potential" confidence level.

Execution units bottlenecks

Performance is limited by execution of FP multiply or FMA (fused multiply-add) operations (the FP multiply/FMA unit is a bottleneck).

Workaround

Reduce the number of FP multiply/FMA instructions

512-bits vectorization

Workaround

Recompile with -qopt-zmm-usage=high

Masked instructions

Detected masked instructions.

Details

Vector registers are partially exploited, which is expected if your loop is irregular or mixes elements of different sizes.

Workaround

If your loop is irregular, try to remove or hoist conditional structures out of your loop. If it mixes elements of different sizes, try to uniformize them.

FMA

Detected 12 FMA (fused multiply-add) operations. Presence of both ADD/SUB and MUL operations.

Workaround

Complex instructions

Detected COMPLEX INSTRUCTIONS.

Details

These instructions generate more than one micro-operation and only one of them can be decoded during a cycle and the extra micro-operations increase pressure on execution units.

VGATHERQPD: 5 occurrences►
- /home/eoseret/qaas_runs_GNR/173-814-3702/intel/CloverLeaf1.3-FC/build/CloverLeaf1.3-FC/CloverLeaf_ref/kernels/advec_mom_kernel.f90:227
- /home/eoseret/qaas_runs_GNR/173-814-3702/intel/CloverLeaf1.3-FC/build/CloverLeaf1.3-FC/CloverLeaf_ref/kernels/advec_mom_kernel.f90:237
- /home/eoseret/qaas_runs_GNR/173-814-3702/intel/CloverLeaf1.3-FC/build/CloverLeaf1.3-FC/CloverLeaf_ref/kernels/advec_mom_kernel.f90:229
- /home/eoseret/qaas_runs_GNR/173-814-3702/intel/CloverLeaf1.3-FC/build/CloverLeaf1.3-FC/CloverLeaf_ref/kernels/advec_mom_kernel.f90:229
- /home/eoseret/qaas_runs_GNR/173-814-3702/intel/CloverLeaf1.3-FC/build/CloverLeaf1.3-FC/CloverLeaf_ref/kernels/advec_mom_kernel.f90:230
VPMULLQ: 4 occurrences►
- /home/eoseret/qaas_runs_GNR/173-814-3702/intel/CloverLeaf1.3-FC/build/CloverLeaf1.3-FC/CloverLeaf_ref/kernels/advec_mom_kernel.f90:227
- /home/eoseret/qaas_runs_GNR/173-814-3702/intel/CloverLeaf1.3-FC/build/CloverLeaf1.3-FC/CloverLeaf_ref/kernels/advec_mom_kernel.f90:229
- /home/eoseret/qaas_runs_GNR/173-814-3702/intel/CloverLeaf1.3-FC/build/CloverLeaf1.3-FC/CloverLeaf_ref/kernels/advec_mom_kernel.f90:229
- /home/eoseret/qaas_runs_GNR/173-814-3702/intel/CloverLeaf1.3-FC/build/CloverLeaf1.3-FC/CloverLeaf_ref/kernels/advec_mom_kernel.f90:230

Slow data structures access

Detected data structures (typically arrays) that cannot be efficiently read/written

Details

Irregular (variable stride) or indirect: 5 occurrence(s)

Non-unit stride (uncontiguous) accesses are not efficiently using data caches

Workaround

Try to remove indirect accesses. If applicable, precompute elements out of the innermost loop.

Vector unaligned load/store instructions

Detected 2 optimal vector unaligned load/store instructions.

Details

VMOVUPD: 2 occurrences►
- /home/eoseret/qaas_runs_GNR/173-814-3702/intel/CloverLeaf1.3-FC/build/CloverLeaf1.3-FC/CloverLeaf_ref/kernels/advec_mom_kernel.f90:240
- /home/eoseret/qaas_runs_GNR/173-814-3702/intel/CloverLeaf1.3-FC/build/CloverLeaf1.3-FC/CloverLeaf_ref/kernels/advec_mom_kernel.f90:215

Workaround

Use vector aligned instructions:

align your arrays on 64 bytes boundaries: compile with -align array64byte (remark: not affects arrays in COMMON blocks).
inform your compiler that your arrays are vector aligned: Append !DIR$ VECTOR ALIGNED to the loop if all accessed arrays are aligned, or !DIR$ ASSUME_ALIGNED FOO: 64 if only FOO is aligned.

Gather/scatter instructions

Detected gather/scatter instructions (typically caused by indirect accesses).

Details

VGATHERQPD: 5 occurrences►
- /home/eoseret/qaas_runs_GNR/173-814-3702/intel/CloverLeaf1.3-FC/build/CloverLeaf1.3-FC/CloverLeaf_ref/kernels/advec_mom_kernel.f90:227
- /home/eoseret/qaas_runs_GNR/173-814-3702/intel/CloverLeaf1.3-FC/build/CloverLeaf1.3-FC/CloverLeaf_ref/kernels/advec_mom_kernel.f90:237
- /home/eoseret/qaas_runs_GNR/173-814-3702/intel/CloverLeaf1.3-FC/build/CloverLeaf1.3-FC/CloverLeaf_ref/kernels/advec_mom_kernel.f90:229
- /home/eoseret/qaas_runs_GNR/173-814-3702/intel/CloverLeaf1.3-FC/build/CloverLeaf1.3-FC/CloverLeaf_ref/kernels/advec_mom_kernel.f90:229
- /home/eoseret/qaas_runs_GNR/173-814-3702/intel/CloverLeaf1.3-FC/build/CloverLeaf1.3-FC/CloverLeaf_ref/kernels/advec_mom_kernel.f90:230

Workaround

Try to simplify your code and/or replace indirect accesses with unit-stride ones.

Type of elements and instruction set

Matching between your loop (in the source code) and the binary loop

The binary loop is composed of 66 FP arithmetical operations:

28: addition or subtraction (12 inside FMA instructions)
29: multiply (12 inside FMA instructions)
9: divide

The binary loop is loading 208 bytes (26 double precision FP elements). The binary loop is storing 32 bytes (4 double precision FP elements).

Arithmetic intensity

Arithmetic intensity is 0.28 FP operations per loaded or stored byte.

General properties

nb instructions	77
nb uops	113
loop length	475
used x86 registers	6
used mmx registers	0
used xmm registers	14
used ymm registers	23
used zmm registers	0
nb stack references	2
ADD-SUB / MUL ratio	0.80

Front-end

MACRO FUSION NOT POSSIBLE FIT IN UOP CACHE

micro-operation queue	18.83 cycles
front end	18.83 cycles

Back-end

	P0	P1	P2	P3	P4	P5	P6	P7	P8	P9	P10	P11
uops	28.17	27.83	7.67	7.67	0.50	28.00	2.00	0.50	0.50	0.50	1.00	7.67
cycles	28.17	27.83	7.67	7.67	0.50	28.00	2.00	0.50	0.50	0.50	1.00	7.67

Execution ports to units layout:

P0 (256 bits): VPU, BRU, ALU, DIV/SQRT
P1 (256 bits): ALU, VPU
P2 (512 bits): load
P3 (512 bits): load
P4 (512 bits): store data
P5 (512 bits): ALU, VPU
P6: ALU, BRU
P7: store address
P8: store address
P9 (512 bits): store data
P10: ALU
P11 (512 bits): load

Cycles executing div or sqrt instructions	20.00
Longest recurrence chain latency (RecMII)	0.00

Cycles summary

Front-end	18.83
Dispatch	28.17
DIV/SQRT	20.00
Data deps.	0.00
Overall L1	28.17

Vectorization ratios

INT

all	96%
load	NA (no load vectorizable/vectorized instructions)
store	NA (no store vectorizable/vectorized instructions)
mul	100%
add-sub	100%
fma	NA (no fma vectorizable/vectorized instructions)
other	94%

all	88%
load	100%
store	100%
mul	80%
add-sub	100%
fma	100%
div/sqrt	66%
other	89%

INT+FP

all	92%
load	100%
store	100%
mul	88%
add-sub	100%
fma	100%
div/sqrt	66%
other	91%

Vector efficiency ratios

INT

all	37%
load	NA (no load vectorizable/vectorized instructions)
store	NA (no store vectorizable/vectorized instructions)
mul	50%
add-sub	50%
fma	NA (no fma vectorizable/vectorized instructions)
other	25%

all	44%
load	50%
store	50%
mul	42%
add-sub	50%
fma	50%
div/sqrt	37%
other	43%

INT+FP

all	41%
load	50%
store	50%
mul	45%
add-sub	50%
fma	50%
div/sqrt	37%
other	35%

Cycles and memory resources usage

Detected masked instructions: assuming all mask elements are active. Assuming all data fit into the L1 cache, each iteration of the binary loop takes 28.17 cycles. At this rate:

3% of peak load performance is reached (7.38 out of 192.00 bytes loaded per cycle (GB/s @ 1GHz))
0% of peak store performance is reached (1.14 out of 128.00 bytes stored per cycle (GB/s @ 1GHz))

Front-end bottlenecks

Found no such bottlenecks.

ASM code

In the binary file, the address of the loop is: 41de78

Instruction	Nb FU	P0	P1	P2	P3	P4	P5	P6	P7	P8	P9	P10	P11	Latency	Recip. throughput	Vectorization
VPMULLQ %YMM26,%YMM2,%YMM26	5	1.50	1.50	0	0	0	0	0	0	0	0	0	0	15	1.50	vect (50.0%)
VPADDQ %YMM27,%YMM13,%YMM27	1	0.33	0.33	0	0	0	0.33	0	0	0	0	0	0	1	0.33	vect (50.0%)
VPADDQ %YMM26,%YMM27,%YMM26	1	0.33	0.33	0	0	0	0.33	0	0	0	0	0	0	1	0.33	vect (50.0%)
VPXORD %XMM27,%XMM27,%XMM27	1	0.33	0.33	0	0	0	0.33	0	0	0	0	0	0	0-1	0.33	vect (25.0%)
VPCMPEQB %XMM0,%XMM0,%K3	1	0	0	0	0	0	1	0	0	0	0	0	0	3	1	vect (25.0%)
VGATHERQPD (,%YMM26,1),%YMM27{%K3}	5	1	1	1.33	1.33	0	1	0	0	0	0	0	1.33	28	2	vect (50.0%)
VANDPD %YMM12,%YMM23,%YMM26	1	0.33	0.33	0	0	0	0.33	0	0	0	0	0	0	1	0.33	vect (50.0%)
VDIVPD %YMM27,%YMM26,%YMM26	1	1	0	0	0	0	0	0	0	0	0	0	0	13-15	8	vect (50.0%)
VPBLENDMD %XMM21,%XMM20,%XMM27{%K1}	1	0.33	0.33	0	0	0	0.33	0	0	0	0	0	0	1	0.33	vect (25.0%)
VANDPD %YMM12,%YMM29,%YMM29	1	0.33	0.33	0	0	0	0.33	0	0	0	0	0	0	1	0.33	vect (50.0%)
VANDPD %YMM12,%YMM25,%YMM30	1	0.33	0.33	0	0	0	0.33	0	0	0	0	0	0	1	0.33	vect (50.0%)
VSUBPD %YMM26,%YMM15,%YMM31	1	0	0.50	0	0	0	0.50	0	0	0	0	0	0	3	0.50	vect (50.0%)
VMULPD %YMM31,%YMM30,%YMM31	1	0.50	0.50	0	0	0	0	0	0	0	0	0	0	4	0.50	vect (50.0%)
VDIVSD %XMM28,%XMM10,%XMM7	1	1	0	0	0	0	0	0	0	0	0	0	0	13-15	4	scal (12.5%)
VBROADCASTSD %XMM7,%YMM7	1	0	0	0	0	0	1	0	0	0	0	0	0	3	1	scal (12.5%)
VPMOVSXDQ %XMM27,%YMM27	1	0	0	0	0	0	1	0	0	0	0	0	0	3	1	vect (25.0%)
VPSUBQ %YMM1,%YMM27,%YMM27	1	0.33	0.33	0	0	0	0.33	0	0	0	0	0	0	0-1	0.33	vect (50.0%)
VXORPD %XMM9,%XMM9,%XMM9	1	0	0	0	0	0	0	0	0	0	0	0	0	0	0.17	vect (25.0%)
KMOVQ %K2,%K1	1	1	0	0	0	0	0	0	0	0	0	0	0	0	1	N/A
VGATHERQPD (%RCX,%YMM27,8),%YMM9{%K1}	5	1	1	1.33	1.33	0	1	0	0	0	0	0	1.33	28	2	vect (50.0%)
VMINPD %YMM30,%YMM29,%YMM27	1	0.50	0.50	0	0	0	0	0	0	0	0	0	0	4	0.50	vect (50.0%)
VFMADD213PD %YMM29,%YMM26,%YMM29	1	0.50	0.50	0	0	0	0	0	0	0	0	0	0	4	0.50	vect (50.0%)
VDIVPD %YMM9,%YMM29,%YMM9	1	1	0	0	0	0	0	0	0	0	0	0	0	13-15	8	vect (50.0%)
VFMADD231PD %YMM7,%YMM31,%YMM9	1	0.50	0.50	0	0	0	0	0	0	0	0	0	0	4	0.50	vect (50.0%)
VMULSD %XMM8,%XMM28,%XMM7	1	0.50	0.50	0	0	0	0	0	0	0	0	0	0	4	0.50	scal (12.5%)
VBROADCASTSD %XMM7,%YMM7	1	0	0	0	0	0	1	0	0	0	0	0	0	3	1	scal (12.5%)
VMULPD %YMM7,%YMM9,%YMM7	1	0.50	0.50	0	0	0	0	0	0	0	0	0	0	4	0.50	vect (50.0%)
VMINPD %YMM27,%YMM7,%YMM7	1	0.50	0.50	0	0	0	0	0	0	0	0	0	0	4	0.50	vect (50.0%)
VCMPPD $0x2,%YMM11,%YMM25,%K1	1	0	0	0	0	0	1	0	0	0	0	0	0	3	1	vect (50.0%)
VXORPD %YMM18,%YMM7,%YMM7{%K1}	1	0.33	0.33	0	0	0	0.33	0	0	0	0	0	0	0-1	0.33	vect (50.0%)
VMOVAPD %YMM7,%YMM7{%K2}{z}	1	0	0	0	0	0	0	0	0	0	0	0	0	0-1	0.17	vect (50.0%)
VSUBPD %YMM26,%YMM16,%YMM9	1	0	0.50	0	0	0	0.50	0	0	0	0	0	0	3	0.50	vect (50.0%)
VFMADD213PD %YMM24,%YMM7,%YMM9	1	0.50	0.50	0	0	0	0	0	0	0	0	0	0	4	0.50	vect (50.0%)
VMULPD %YMM23,%YMM9,%YMM7	1	0.50	0.50	0	0	0	0	0	0	0	0	0	0	4	0.50	vect (50.0%)
MOV -0x78(%RBP),%R8	1	0	0	0.33	0.33	0	0	0	0	0	0	0	0.33	1	0.33	N/A
VMOVUPD %YMM7,(%R8,%R11,8)	1	0	0	0	0	0.50	0	0	0.50	0.50	0.50	0	0	0-1	0.50	vect (50.0%)
ADD $0x4,%R11	1	0	0	0	0	0	0	0	0	0	0	0	0	1	0.17	N/A
CMP -0xb0(%RBP),%R11	1	0.20	0.20	0.33	0.33	0	0.20	0.20	0	0	0	0.20	0.33	1	0.33	N/A
JA 41df80 <advec_mom_kernel_mod_mp_advec_mom_kernel_.DIR.OMP.PARALLEL.2+0x25d0>	1	0.50	0	0	0	0	0	0.50	0	0	0	0	0	0	0.50	N/A
LEA (%R12,%R11,1),%R8	1	0	0	0	0	0	0	0	0	0	0	0	0	0	0.17	N/A
VMOVUPD (%R14,%R11,8),%YMM23	1	0	0	0.33	0.33	0	0	0	0	0	0	0	0.33	0-1	0.33	vect (50.0%)
VCMPPD $0x1,%YMM11,%YMM23,%K1	1	0	0	0	0	0	1	0	0	0	0	0	0	3	1	vect (50.0%)
VPBLENDMD %XMM21,%XMM22,%XMM24{%K1}	1	0.33	0.33	0	0	0	0.33	0	0	0	0	0	0	1	0.33	vect (25.0%)
VPMOVSXDQ %XMM24,%YMM24	1	0	0	0	0	0	1	0	0	0	0	0	0	3	1	vect (25.0%)
VPSUBQ %YMM1,%YMM24,%YMM26	1	0.33	0.33	0	0	0	0.33	0	0	0	0	0	0	0-1	0.33	vect (50.0%)
VPBROADCASTQ %R8,%YMM24	1	0	0	0	0	0	1	0	0	0	0	0	0	3	1	scal (12.5%)
VPSUBQ %YMM0,%YMM24,%YMM24	1	0.33	0.33	0	0	0	0.33	0	0	0	0	0	0	0-1	0.33	vect (50.0%)
VPSLLQ $0x3,%YMM24,%YMM24	1	0.50	0.50	0	0	0	0	0	0	0	0	0	0	2-4	0.50	vect (50.0%)
VPADDQ %YMM14,%YMM24,%YMM27	1	0.33	0.33	0	0	0	0.33	0	0	0	0	0	0	1	0.33	vect (50.0%)
VPMULLQ %YMM26,%YMM3,%YMM24	5	1.50	1.50	0	0	0	0	0	0	0	0	0	0	15	1.50	vect (50.0%)
VPADDQ %YMM27,%YMM17,%YMM25	1	0.33	0.33	0	0	0	0.33	0	0	0	0	0	0	1	0.33	vect (50.0%)
VPADDQ %YMM24,%YMM25,%YMM28	1	0.33	0.33	0	0	0	0.33	0	0	0	0	0	0	1	0.33	vect (50.0%)
VPXORD %XMM24,%XMM24,%XMM24	1	0.33	0.33	0	0	0	0.33	0	0	0	0	0	0	0-1	0.33	vect (25.0%)
VPCMPEQB %XMM0,%XMM0,%K2	1	0	0	0	0	0	1	0	0	0	0	0	0	3	1	vect (25.0%)
VGATHERQPD (,%YMM28,1),%YMM24{%K2}	5	1	1	1.33	1.33	0	1	0	0	0	0	0	1.33	28	2	vect (50.0%)
VPBLENDMD %XMM19,%XMM20,%XMM28{%K1}	1	0.33	0.33	0	0	0	0.33	0	0	0	0	0	0	1	0.33	vect (25.0%)
VPBLENDMD %XMM22,%XMM21,%XMM29{%K1}	1	0.33	0.33	0	0	0	0.33	0	0	0	0	0	0	1	0.33	vect (25.0%)
VPMOVSXDQ %XMM28,%YMM28	1	0	0	0	0	0	1	0	0	0	0	0	0	3	1	vect (25.0%)
VPSUBQ %YMM1,%YMM28,%YMM28	1	0.33	0.33	0	0	0	0.33	0	0	0	0	0	0	0-1	0.33	vect (50.0%)
VPMULLQ %YMM28,%YMM3,%YMM28	5	1.50	1.50	0	0	0	0	0	0	0	0	0	0	15	1.50	vect (50.0%)
VPADDQ %YMM28,%YMM25,%YMM28	1	0.33	0.33	0	0	0	0.33	0	0	0	0	0	0	1	0.33	vect (50.0%)
VXORPD %XMM30,%XMM30,%XMM30	1	0	0	0	0	0	0	0	0	0	0	0	0	0	0.17	vect (25.0%)
VPCMPEQB %XMM0,%XMM0,%K2	1	0	0	0	0	0	1	0	0	0	0	0	0	3	1	vect (25.0%)
VGATHERQPD (,%YMM28,1),%YMM30{%K2}	5	1	1	1.33	1.33	0	1	0	0	0	0	0	1.33	28	2	vect (50.0%)
VPMOVSXDQ %XMM29,%YMM28	1	0	0	0	0	0	1	0	0	0	0	0	0	3	1	vect (25.0%)
VPSUBQ %YMM1,%YMM28,%YMM28	1	0.33	0.33	0	0	0	0.33	0	0	0	0	0	0	0-1	0.33	vect (50.0%)
VPMULLQ %YMM28,%YMM3,%YMM28	5	1.50	1.50	0	0	0	0	0	0	0	0	0	0	15	1.50	vect (50.0%)
VPADDQ %YMM28,%YMM25,%YMM25	1	0.33	0.33	0	0	0	0.33	0	0	0	0	0	0	1	0.33	vect (50.0%)
VPXORD %XMM28,%XMM28,%XMM28	1	0.33	0.33	0	0	0	0.33	0	0	0	0	0	0	0-1	0.33	vect (25.0%)
VPCMPEQB %XMM0,%XMM0,%K2	1	0	0	0	0	0	1	0	0	0	0	0	0	3	1	vect (25.0%)
VGATHERQPD (,%YMM25,1),%YMM28{%K2}	5	1	1	1.33	1.33	0	1	0	0	0	0	0	1.33	28	2	vect (50.0%)
VSUBPD %YMM30,%YMM24,%YMM29	1	0	0.50	0	0	0	0.50	0	0	0	0	0	0	3	0.50	vect (50.0%)
VSUBPD %YMM24,%YMM28,%YMM25	1	0	0.50	0	0	0	0.50	0	0	0	0	0	0	3	0.50	vect (50.0%)
VMULPD %YMM29,%YMM25,%YMM28	1	0.50	0.50	0	0	0	0	0	0	0	0	0	0	4	0.50	vect (50.0%)
VCMPPD $0x1,%YMM28,%YMM11,%K2	1	0	0	0	0	0	1	0	0	0	0	0	0	3	1	vect (50.0%)
KORTESTB %K2,%K2	1	1	0	0	0	0	0	0	0	0	0	0	0	0	1	N/A
JE 41dd90 <advec_mom_kernel_mod_mp_advec_mom_kernel_.DIR.OMP.PARALLEL.2+0x23e0>	1	0.50	0	0	0	0	0	0.50	0	0	0	0	0	0	0.50	N/A

Report Configuration

Vectorization

Details

Workaround

512-bits vectorization

Workaround

Masked instructions

Details

Workaround

FMA

Workaround

Complex instructions

Details

Slow data structures access

Details

Workaround

Gather/scatter instructions

Details

Workaround

Type of elements and instruction set

Matching between your loop (in the source code) and the binary loop

Arithmetic intensity

General properties

Front-end

Back-end

Cycles summary

Vectorization ratios

Vector efficiency ratios

Cycles and memory resources usage

Vectorization

Details

Workaround

Execution units bottlenecks

Workaround

512-bits vectorization

Workaround

Masked instructions

Details

Workaround

FMA

Workaround

Complex instructions

Details

Slow data structures access

Details

Workaround

Vector unaligned load/store instructions

Details

Workaround

Gather/scatter instructions

Details

Workaround

Type of elements and instruction set

Matching between your loop (in the source code) and the binary loop

Arithmetic intensity

General properties

Front-end

Back-end

Cycles summary

Vectorization ratios

Vector efficiency ratios

Cycles and memory resources usage

Front-end bottlenecks

ASM code

Vectorization

Details

Workaround

Execution units bottlenecks

Workaround

512-bits vectorization

Workaround

Masked instructions

Details

Workaround

FMA

Workaround

Complex instructions

Details

Slow data structures access

Details