OV - exec - Loop 108

	Loop Id: 108	Module: exec	Source: advec_mom_kernel.f90:81-177 [...]	Coverage: 3.61%

0x422840 VCMPPD	$0x1,%ZMM21,%ZMM5,%K1

0x422847 VMOVAPD	%ZMM25,%ZMM20{%K1}{z}

0x42284d VSUBPD	%ZMM19,%ZMM10,%ZMM19

0x422853 VFMADD213PD	%ZMM18,%ZMM20,%ZMM19

0x422859 VMULPD	%ZMM17,%ZMM19,%ZMM17

0x42285f VMOVUPD	%ZMM17,(%RDI,%RBX,8)    [1]

0x422866 ADD	$0x8,%RBX

0x42286a CMP	%R11,%RBX

0x42286d JA	422a00

0x422873 VMOVUPD	(%RSI,%RBX,8),%ZMM17    [5]

0x42287a VFPCLASSPD	$0x50,%ZMM17,%K1

0x422881 LEA	(%R13,%RBX,1),%EAX

0x422886 VPBROADCASTD	%EAX,%YMM19

0x42288c VPADDD	%YMM7,%YMM19,%YMM20

0x422892 VPADDD	%YMM15,%YMM19,%YMM18

0x422898 VPBLENDMD	%YMM18,%YMM20,%YMM21{%K1}

0x42289e VMOVDQA32	%YMM20,%YMM18{%K1}

0x4228a4 VPMOVSXDQ	%YMM18,%ZMM18

0x4228aa VPSUBQ	%ZMM4,%ZMM18,%ZMM22

0x4228b0 KXNORW	%K0,%K0,%K2

0x4228b4 VXORPD	%XMM23,%XMM23,%XMM23

0x4228ba VGATHERQPD	(%R8,%ZMM22,8),%ZMM23{%K2}    [4]

0x4228c1 KXNORW	%K0,%K0,%K2

0x4228c5 VPXORD	%XMM18,%XMM18,%XMM18

0x4228cb VGATHERQPD	(%R10,%ZMM22,8),%ZMM18{%K2}    [6]

0x4228d2 VPADDD	%YMM14,%YMM19,%YMM22

0x4228d8 VMOVDQA64	%YMM22,%YMM24

0x4228de VPADDD	%YMM16,%YMM19,%YMM24{%K1}

0x4228e4 VANDPD	%ZMM9,%ZMM17,%ZMM19

0x4228ea VPMOVSXDQ	%YMM24,%ZMM24

0x4228f0 VPSUBQ	%ZMM4,%ZMM24,%ZMM24

0x4228f6 KXNORW	%K0,%K0,%K2

0x4228fa VXORPD	%XMM26,%XMM26,%XMM26

0x422900 VGATHERQPD	(%R10,%ZMM24,8),%ZMM26{%K2}    [6]

0x422907 VDIVPD	%ZMM23,%ZMM19,%ZMM19

0x42290d VPMOVSXDQ	%YMM21,%ZMM21

0x422913 VPSUBQ	%ZMM4,%ZMM21,%ZMM21

0x422919 KXNORW	%K0,%K0,%K2

0x42291d VXORPD	%XMM23,%XMM23,%XMM23

0x422923 VGATHERQPD	(%R10,%ZMM21,8),%ZMM23{%K2}    [6]

0x42292a VXORPD	%XMM25,%XMM25,%XMM25

0x422930 VSUBPD	%ZMM26,%ZMM18,%ZMM24

0x422936 VSUBPD	%ZMM18,%ZMM23,%ZMM23

0x42293c VMULPD	%ZMM24,%ZMM23,%ZMM21

0x422942 VCMPPD	$0x1,%ZMM21,%ZMM25,%K0

0x422949 KORTESTB	%K0,%K0

0x42294d JE	422840

0x422953 VCMPPD	$0x1,%ZMM21,%ZMM5,%K2

0x42295a VMOVDQA32	%YMM20,%YMM22{%K1}

0x422960 VMOVUPD	(%R9,%RBX,8),%ZMM20{%K2}{z}    [2]

0x422967 VANDPD	%ZMM9,%ZMM23,%ZMM25

0x42296d VSUBPD	%ZMM19,%ZMM11,%ZMM26

0x422973 VMULPD	%ZMM26,%ZMM25,%ZMM26

0x422979 VDIVPD	%ZMM20,%ZMM26,%ZMM26

0x42297f VPMOVSXDQ	%YMM22,%ZMM22

0x422985 VPSUBQ	%ZMM4,%ZMM22,%ZMM22

0x42298b VXORPD	%XMM27,%XMM27,%XMM27

0x422991 MOV	0x68(%RSP),%RAX    [3]

0x422996 VGATHERQPD	(%RAX,%ZMM22,8),%ZMM27{%K2}    [7]

0x42299d VANDPD	%ZMM9,%ZMM24,%ZMM22

0x4229a3 VMINPD	%ZMM25,%ZMM22,%ZMM24

0x4229a9 VFMADD213PD	%ZMM22,%ZMM19,%ZMM22

0x4229af VDIVPD	%ZMM27,%ZMM22,%ZMM22

0x4229b5 VADDPD	%ZMM26,%ZMM22,%ZMM22

0x4229bb VMULPD	%ZMM12,%ZMM20,%ZMM20

0x4229c1 VMULPD	%ZMM22,%ZMM20,%ZMM20

0x4229c7 VMINPD	%ZMM24,%ZMM20,%ZMM25

0x4229cd VFPCLASSPD	$0x56,%ZMM23,%K1

0x4229d4 VXORPD	%ZMM13,%ZMM25,%ZMM25{%K1}

0x4229da JMP	422840

/home/eoseret/qaas_runs_GNR/173-814-3702/intel/CloverLeaf1.3-FC/build/CloverLeaf1.3-FC/CloverLeaf_ref/kernels/advec_mom_kernel.f90: 81 - 177

--------------------------------------------------------------------------------

81:     IF(mom_sweep.EQ.1)THEN ! x 1

[...]

152:           IF(node_flux(j,k).LT.0.0)THEN

[...]

158:             upwind=j-1

159:             donor=j

160:             downwind=j+1

161:             dif=upwind

162:           ENDIF

163:           sigma=ABS(node_flux(j,k))/(node_mass_pre(donor,k))

164:           width=celldx(j)

165:           vdiffuw=vel1(donor,k)-vel1(upwind,k)

166:           vdiffdw=vel1(downwind,k)-vel1(donor,k)

167:           limiter=0.0

168:           IF(vdiffuw*vdiffdw.GT.0.0)THEN

169:             auw=ABS(vdiffuw)

170:             adw=ABS(vdiffdw)

171:             wind=1.0_8

172:             IF(vdiffdw.LE.0.0) wind=-1.0_8

173:             limiter=wind*MIN(width*((2.0_8-sigma)*adw/width+(1.0_8+sigma)*auw/celldx(dif))/6.0_8,auw,adw)

174:           ENDIF

175:           advec_vel_s=vel1(donor,k)+(1.0-sigma)*limiter

176:           mom_flux(j,k)=advec_vel_s*node_flux(j,k)

177:         ENDDO

Coverage (%)	Name	Source Location	Module
►100.00+	__kmp_invoke_microtask		libiomp5.so
○	__kmp_invoke_task_func		libiomp5.so

Path /

Metric	Value
CQA speedup if no scalar integer	1.04
CQA speedup if FP arith vectorized	1.00
CQA speedup if fully vectorized	1.01
CQA speedup if no inter-iteration dependency	NA
CQA speedup if next bottleneck killed	1.09
Bottlenecks
Function	advec_mom_kernel_mod_mp_advec_mom_kernel_.DIR.OMP.PARALLEL.2
Source	advec_mom_kernel.f90:81-81,advec_mom_kernel.f90:152-152,advec_mom_kernel.f90:158-177
Source loop unroll info	not unrolled or unrolled with no peel/tail loop
Source loop unroll confidence level	max
Unroll/vectorization loop type	NA
Unroll factor	NA
CQA cycles	33.25
CQA cycles if no scalar integer	32.00
CQA cycles if FP arith vectorized	33.25
CQA cycles if fully vectorized	33.03
Front-end cycles	13.42
P0 cycles	24.50
P1 cycles	6.00
P2 cycles	12.67
P3 cycles	12.67
P4 cycles	0.50
P5 cycles	24.50
P6 cycles	2.00
P7 cycles	0.50
P8 cycles	0.50
P9 cycles	0.50
P10 cycles	2.00
P11 cycles	12.67
DIV/SQRT cycles	32.00
Inter-iter dependencies cycles	0
FE+BE cycles (UFS)	NA
Stall cycles (UFS)	NA
Nb insns	58.50
Nb uops	80.50
Nb loads	6.50
Nb stores	1.00
Nb stack references	0.50
FLOP/cycle	3.01
Nb FLOP add-sub	32.00
Nb FLOP mul	28.00
Nb FLOP fma	12.00
Nb FLOP div	16.00
Nb FLOP rcp	0.00
Nb FLOP sqrt	0.00
Nb FLOP rsqrt	0.00
Bytes/cycle	15.80
Bytes prefetched	0.00
Bytes loaded	388.00
Bytes stored	64.00
Stride 0	0.50
Stride 1	2.50
Stride n	0.00
Stride unknown	0.00
Stride indirect	2.50
Vectorization ratio all	97.79
Vectorization ratio load	100.00
Vectorization ratio store	100.00
Vectorization ratio mul	100.00
Vectorization ratio add_sub	100.00
Vectorization ratio fma	100.00
Vectorization ratio div_sqrt	100.00
Vectorization ratio other	96.06
Vector-efficiency ratio all	77.05
Vector-efficiency ratio load	100.00
Vector-efficiency ratio store	100.00
Vector-efficiency ratio mul	100.00
Vector-efficiency ratio add_sub	82.31
Vector-efficiency ratio fma	100.00
Vector-efficiency ratio div_sqrt	100.00
Vector-efficiency ratio other	66.95

Metric	Value
CQA speedup if no scalar integer	1.00
CQA speedup if FP arith vectorized	1.00
CQA speedup if fully vectorized	1.00
CQA speedup if no inter-iteration dependency	NA
CQA speedup if next bottleneck killed	1.57
Bottlenecks	P0,
Function	advec_mom_kernel_mod_mp_advec_mom_kernel_.DIR.OMP.PARALLEL.2
Source	advec_mom_kernel.f90:81-81,advec_mom_kernel.f90:152-152,advec_mom_kernel.f90:158-177
Source loop unroll info	not unrolled or unrolled with no peel/tail loop
Source loop unroll confidence level	max
Unroll/vectorization loop type	NA
Unroll factor	NA
CQA cycles	48.00
CQA cycles if no scalar integer	48.00
CQA cycles if FP arith vectorized	48.00
CQA cycles if fully vectorized	48.00
Front-end cycles	16.00
P0 cycles	30.50
P1 cycles	6.00
P2 cycles	14.33
P3 cycles	14.33
P4 cycles	0.50
P5 cycles	30.50
P6 cycles	2.00
P7 cycles	0.50
P8 cycles	0.50
P9 cycles	0.50
P10 cycles	2.00
P11 cycles	14.33
DIV/SQRT cycles	48.00
Inter-iter dependencies cycles	0
FE+BE cycles (UFS)	NA
Stall cycles (UFS)	NA
Nb insns	70.00
Nb uops	96.00
Nb loads	8.00
Nb stores	1.00
Nb stack references	1.00
FLOP/cycle	2.83
Nb FLOP add-sub	40.00
Nb FLOP mul	40.00
Nb FLOP fma	16.00
Nb FLOP div	24.00
Nb FLOP rcp	0.00
Nb FLOP sqrt	0.00
Nb FLOP rsqrt	0.00
Bytes/cycle	10.83
Bytes prefetched	0.00
Bytes loaded	456.00
Bytes stored	64.00
Stride 0	1.00
Stride 1	3.00
Stride n	0.00
Stride unknown	0.00
Stride indirect	3.00
Vectorization ratio all	98.28
Vectorization ratio load	100.00
Vectorization ratio store	100.00
Vectorization ratio mul	100.00
Vectorization ratio add_sub	100.00
Vectorization ratio fma	100.00
Vectorization ratio div_sqrt	100.00
Vectorization ratio other	96.88
Vector-efficiency ratio all	80.28
Vector-efficiency ratio load	100.00
Vector-efficiency ratio store	100.00
Vector-efficiency ratio mul	100.00
Vector-efficiency ratio add_sub	84.62
Vector-efficiency ratio fma	100.00
Vector-efficiency ratio div_sqrt	100.00
Vector-efficiency ratio other	70.51

Metric	Value
CQA speedup if no scalar integer	1.16
CQA speedup if FP arith vectorized	1.00
CQA speedup if fully vectorized	1.02
CQA speedup if no inter-iteration dependency	NA
CQA speedup if next bottleneck killed	NA
Bottlenecks	P0, P5,
Function	advec_mom_kernel_mod_mp_advec_mom_kernel_.DIR.OMP.PARALLEL.2
Source	advec_mom_kernel.f90:81-81,advec_mom_kernel.f90:152-152,advec_mom_kernel.f90:158-177
Source loop unroll info	not unrolled or unrolled with no peel/tail loop
Source loop unroll confidence level	max
Unroll/vectorization loop type	NA
Unroll factor	NA
CQA cycles	18.50
CQA cycles if no scalar integer	16.00
CQA cycles if FP arith vectorized	18.50
CQA cycles if fully vectorized	18.06
Front-end cycles	10.83
P0 cycles	18.50
P1 cycles	6.00
P2 cycles	11.00
P3 cycles	11.00
P4 cycles	0.50
P5 cycles	18.50
P6 cycles	2.00
P7 cycles	0.50
P8 cycles	0.50
P9 cycles	0.50
P10 cycles	2.00
P11 cycles	11.00
DIV/SQRT cycles	16.00
Inter-iter dependencies cycles	0
FE+BE cycles (UFS)	NA
Stall cycles (UFS)	NA
Nb insns	47.00
Nb uops	65.00
Nb loads	5.00
Nb stores	1.00
Nb stack references	0.00
FLOP/cycle	3.46
Nb FLOP add-sub	24.00
Nb FLOP mul	16.00
Nb FLOP fma	8.00
Nb FLOP div	8.00
Nb FLOP rcp	0.00
Nb FLOP sqrt	0.00
Nb FLOP rsqrt	0.00
Bytes/cycle	20.76
Bytes prefetched	0.00
Bytes loaded	320.00
Bytes stored	64.00
Stride 0	0.00
Stride 1	2.00
Stride n	0.00
Stride unknown	0.00
Stride indirect	2.00
Vectorization ratio all	97.30
Vectorization ratio load	100.00
Vectorization ratio store	100.00
Vectorization ratio mul	100.00
Vectorization ratio add_sub	100.00
Vectorization ratio fma	100.00
Vectorization ratio div_sqrt	100.00
Vectorization ratio other	95.24
Vector-efficiency ratio all	73.82
Vector-efficiency ratio load	100.00
Vector-efficiency ratio store	100.00
Vector-efficiency ratio mul	100.00
Vector-efficiency ratio add_sub	80.00
Vector-efficiency ratio fma	100.00
Vector-efficiency ratio div_sqrt	100.00
Vector-efficiency ratio other	63.39

Path /

Average path: Display a virtual path defined by average values of all real paths

Function	advec_mom_kernel_mod_mp_advec_mom_kernel_.DIR.OMP.PARALLEL.2
Source file and lines	advec_mom_kernel.f90:81-177
Module	exec

The loop is defined in /home/eoseret/qaas_runs_GNR/173-814-3702/intel/CloverLeaf1.3-FC/build/CloverLeaf1.3-FC/CloverLeaf_ref/kernels/advec_mom_kernel.f90:81,152,158-177.

The related source loop is not unrolled or unrolled with no peel/tail loop.
The structure of this loop is probably <if then [else] end>.

The presence of multiple execution paths is typically the main/first bottleneck.
Try to simplify control inside loop: ideally, try to remove all conditional expressions, for example by (if applicable):

hoisting them (moving them outside the loop)
turning them into conditional moves, MIN or MAX

Ex: if (x<0) x=0 => x = max(0,x) (Fortran instrinsic procedure)

gain
potential
hint
expert

Vectorization

Your loop is highly vectorized. Only 77% of vector register length is used (average across all SSE/AVX instructions). By fully vectorizing your loop, you can lower the cost of an iteration from 33.25 to 33.03 cycles (1.01x speedup).

Details

97% of SSE/AVX instructions are used in vector version (process two or more data elements in vector registers):

96% of SSE/AVX instructions that are not load, store, addition, subtraction nor multiply instructions are used in vector version.

Since your execution units are vector units, only a fully vectorized loop can use their full power.

Workaround

Pass to your compiler a micro-architecture specialization option:
- use axHost or xHost
Use vector aligned instructions:
1. align your arrays on 64 bytes boundaries: compile with -align array64byte (remark: not affects arrays in COMMON blocks).
2. inform your compiler that your arrays are vector aligned: Append !DIR$ VECTOR ALIGNED to the loop if all accessed arrays are aligned, or !DIR$ ASSUME_ALIGNED FOO: 64 if only FOO is aligned.
Use the LOOP COUNT directive

Expensive FP math instructions/calls

Detected performance impact from expensive FP math instructions/calls. By removing/reexpressing them, you can lower the cost of an iteration from 33.25 to 21.50 cycles (1.55x speedup).

Masked instructions

Detected masked instructions.

Details

Vector registers are partially exploited, which is expected if your loop is irregular or mixes elements of different sizes.

Workaround

If your loop is irregular, try to remove or hoist conditional structures out of your loop. If it mixes elements of different sizes, try to uniformize them.

FMA

Detected 12 FMA (fused multiply-add) operations. Presence of both ADD/SUB and MUL operations.

Workaround

Try to change order in which elements are evaluated (using parentheses) in arithmetic expressions containing both ADD/SUB and MUL operations to enable your compiler to generate FMA instructions wherever possible. For instance a + b*c is a valid FMA (MUL then ADD). However (a+b)* c cannot be translated into an FMA (ADD then MUL).

Complex instructions

Detected COMPLEX INSTRUCTIONS.

Details

These instructions generate more than one micro-operation and only one of them can be decoded during a cycle and the extra micro-operations increase pressure on execution units.

VDIVPD: 0 occurrences
VGATHERQPD: 0 occurrences

Slow data structures access

Detected data structures (typically arrays) that cannot be efficiently read/written

Details

Irregular (variable stride) or indirect: 2 occurrence(s)

Non-unit stride (uncontiguous) accesses are not efficiently using data caches

Workaround

Try to remove indirect accesses. If applicable, precompute elements out of the innermost loop.

Gather/scatter instructions

Detected gather/scatter instructions (typically caused by indirect accesses).

Details

VGATHERQPD: 0 occurrences

Workaround

Try to simplify your code and/or replace indirect accesses with unit-stride ones.

Type of elements and instruction set

16 AVX-512 instructions are processing arithmetic or math operations on double precision FP elements in vector mode (eight at a time).

Matching between your loop (in the source code) and the binary loop

The binary loop is composed of 100 FP arithmetical operations:

44: addition or subtraction (12 inside FMA instructions)
40: multiply (12 inside FMA instructions)
16: divide

The binary loop is loading 388 bytes (48 double precision FP elements). The binary loop is storing 64 bytes (8 double precision FP elements).

Arithmetic intensity

Arithmetic intensity is 0.22 FP operations per loaded or stored byte.

General properties

nb instructions	58.50
nb uops	80.50
loop length	345
used x86 registers	9
used mmx registers	0
used xmm registers	4.50
used ymm registers	10
used zmm registers	16
nb stack references	0.50
ADD-SUB / MUL ratio	1.25

Front-end

MACRO FUSION NOT POSSIBLE

micro-operation queue	13.42 cycles
front end	13.42 cycles

Back-end

	P0	P1	P2	P3	P4	P5	P6	P7	P8	P9	P10	P11
uops	24.50	6.00	12.67	12.67	0.50	24.50	2.00	0.50	0.50	0.50	2.00	12.67
cycles	24.50	6.00	12.67	12.67	0.50	24.50	2.00	0.50	0.50	0.50	2.00	12.67

Execution ports to units layout:

P0 (256 bits): VPU, BRU, ALU, DIV/SQRT
P1 (256 bits): ALU, VPU
P2 (512 bits): load
P3 (512 bits): load
P4 (512 bits): store data
P5 (512 bits): ALU, VPU
P6: ALU, BRU
P7: store address
P8: store address
P9 (512 bits): store data
P10: ALU
P11 (512 bits): load

Cycles executing div or sqrt instructions	32.00
Longest recurrence chain latency (RecMII)	0.00

Cycles summary

Front-end	13.42
Dispatch	24.50
DIV/SQRT	32.00
Data deps.	0.00
Overall L1	33.25

Vectorization ratios

INT

all	93%
load	NA (no load vectorizable/vectorized instructions)
store	NA (no store vectorizable/vectorized instructions)
mul	NA (no mul vectorizable/vectorized instructions)
add-sub	100%
fma	NA (no fma vectorizable/vectorized instructions)
other	88%

all	100%
load	100%
store	100%
mul	100%
add-sub	100%
fma	100%
div/sqrt	100%
other	100%

INT+FP

all	97%
load	100%
store	100%
mul	100%
add-sub	100%
fma	100%
div/sqrt	100%
other	96%

Vector efficiency ratios

INT

all	56%
load	NA (no load vectorizable/vectorized instructions)
store	NA (no store vectorizable/vectorized instructions)
mul	NA (no mul vectorizable/vectorized instructions)
add-sub	73%
fma	NA (no fma vectorizable/vectorized instructions)
other	42%

all	88%
load	100%
store	100%
mul	100%
add-sub	100%
fma	100%
div/sqrt	100%
other	79%

INT+FP

all	77%
load	100%
store	100%
mul	100%
add-sub	82%
fma	100%
div/sqrt	100%
other	66%

Cycles and memory resources usage

Detected masked instructions: assuming all mask elements are active. Assuming all data fit into the L1 cache, each iteration of the binary loop takes 33.25 cycles. At this rate:

6% of peak load performance is reached (13.40 out of 192.00 bytes loaded per cycle (GB/s @ 1GHz))
1% of peak store performance is reached (2.40 out of 128.00 bytes stored per cycle (GB/s @ 1GHz))

Function	advec_mom_kernel_mod_mp_advec_mom_kernel_.DIR.OMP.PARALLEL.2
Source file and lines	advec_mom_kernel.f90:81-177
Module	exec

hoisting them (moving them outside the loop)
turning them into conditional moves, MIN or MAX

Ex: if (x<0) x=0 => x = max(0,x) (Fortran instrinsic procedure)

gain
potential
hint
expert

Vectorization

Your loop is highly vectorized. 80% of vector register length is used (average across all SSE/AVX instructions).

Details

98% of SSE/AVX instructions are used in vector version (process two or more data elements in vector registers):

96% of SSE/AVX instructions that are not load, store, addition, subtraction nor multiply instructions are used in vector version.

Execution units bottlenecks

Performance is limited by execution of divide and square root operations (the divide/square root unit is a bottleneck). By removing all these bottlenecks, you can lower the cost of an iteration from 48.00 to 30.50 cycles (1.57x speedup).

Workaround

Reduce the number of division or square root instructions:
- If denominator is constant over iterations, use reciprocal (replace x/y with x*(1/y)). Check precision impact. This will be done by your compiler with no-prec-div or Ofast
Check whether you really need double precision. If not, switch to single precision to speedup execution

Expensive FP math instructions/calls

Detected performance impact from expensive FP math instructions/calls. By removing/reexpressing them, you can lower the cost of an iteration from 48.00 to 26.00 cycles (1.85x speedup).

Masked instructions

Detected masked instructions.

Details

Vector registers are partially exploited, which is expected if your loop is irregular or mixes elements of different sizes.

Workaround

If your loop is irregular, try to remove or hoist conditional structures out of your loop. If it mixes elements of different sizes, try to uniformize them.

FMA

Detected 16 FMA (fused multiply-add) operations. Presence of both ADD/SUB and MUL operations.

Workaround

Complex instructions

Detected COMPLEX INSTRUCTIONS.

Details

These instructions generate more than one micro-operation and only one of them can be decoded during a cycle and the extra micro-operations increase pressure on execution units.

VDIVPD: 3 occurrences►
- /home/eoseret/qaas_runs_GNR/173-814-3702/intel/CloverLeaf1.3-FC/build/CloverLeaf1.3-FC/CloverLeaf_ref/kernels/advec_mom_kernel.f90:163
- /home/eoseret/qaas_runs_GNR/173-814-3702/intel/CloverLeaf1.3-FC/build/CloverLeaf1.3-FC/CloverLeaf_ref/kernels/advec_mom_kernel.f90:173
- /home/eoseret/qaas_runs_GNR/173-814-3702/intel/CloverLeaf1.3-FC/build/CloverLeaf1.3-FC/CloverLeaf_ref/kernels/advec_mom_kernel.f90:173
VGATHERQPD: 5 occurrences►
- /home/eoseret/qaas_runs_GNR/173-814-3702/intel/CloverLeaf1.3-FC/build/CloverLeaf1.3-FC/CloverLeaf_ref/kernels/advec_mom_kernel.f90:163
- /home/eoseret/qaas_runs_GNR/173-814-3702/intel/CloverLeaf1.3-FC/build/CloverLeaf1.3-FC/CloverLeaf_ref/kernels/advec_mom_kernel.f90:165
- /home/eoseret/qaas_runs_GNR/173-814-3702/intel/CloverLeaf1.3-FC/build/CloverLeaf1.3-FC/CloverLeaf_ref/kernels/advec_mom_kernel.f90:165
- /home/eoseret/qaas_runs_GNR/173-814-3702/intel/CloverLeaf1.3-FC/build/CloverLeaf1.3-FC/CloverLeaf_ref/kernels/advec_mom_kernel.f90:166
- /home/eoseret/qaas_runs_GNR/173-814-3702/intel/CloverLeaf1.3-FC/build/CloverLeaf1.3-FC/CloverLeaf_ref/kernels/advec_mom_kernel.f90:173

Slow data structures access

Detected data structures (typically arrays) that cannot be efficiently read/written

Details

Irregular (variable stride) or indirect: 3 occurrence(s)

Non-unit stride (uncontiguous) accesses are not efficiently using data caches

Workaround

Try to remove indirect accesses. If applicable, precompute elements out of the innermost loop.

Gather/scatter instructions

Detected gather/scatter instructions (typically caused by indirect accesses).

Details

VGATHERQPD: 5 occurrences►
- /home/eoseret/qaas_runs_GNR/173-814-3702/intel/CloverLeaf1.3-FC/build/CloverLeaf1.3-FC/CloverLeaf_ref/kernels/advec_mom_kernel.f90:163
- /home/eoseret/qaas_runs_GNR/173-814-3702/intel/CloverLeaf1.3-FC/build/CloverLeaf1.3-FC/CloverLeaf_ref/kernels/advec_mom_kernel.f90:165
- /home/eoseret/qaas_runs_GNR/173-814-3702/intel/CloverLeaf1.3-FC/build/CloverLeaf1.3-FC/CloverLeaf_ref/kernels/advec_mom_kernel.f90:165
- /home/eoseret/qaas_runs_GNR/173-814-3702/intel/CloverLeaf1.3-FC/build/CloverLeaf1.3-FC/CloverLeaf_ref/kernels/advec_mom_kernel.f90:166
- /home/eoseret/qaas_runs_GNR/173-814-3702/intel/CloverLeaf1.3-FC/build/CloverLeaf1.3-FC/CloverLeaf_ref/kernels/advec_mom_kernel.f90:173

Workaround

Try to simplify your code and/or replace indirect accesses with unit-stride ones.

Type of elements and instruction set

22 AVX-512 instructions are processing arithmetic or math operations on double precision FP elements in vector mode (eight at a time).

Matching between your loop (in the source code) and the binary loop

The binary loop is composed of 136 FP arithmetical operations:

56: addition or subtraction (16 inside FMA instructions)
56: multiply (16 inside FMA instructions)
24: divide

The binary loop is loading 456 bytes (57 double precision FP elements). The binary loop is storing 64 bytes (8 double precision FP elements).

Arithmetic intensity

Arithmetic intensity is 0.26 FP operations per loaded or stored byte.

General properties

nb instructions	70
nb uops	96
loop length	415
used x86 registers	10
used mmx registers	0
used xmm registers	5
used ymm registers	10
used zmm registers	18
nb stack references	1
ADD-SUB / MUL ratio	1.00

Front-end

MACRO FUSION NOT POSSIBLE FIT IN UOP CACHE

micro-operation queue	16.00 cycles
front end	16.00 cycles

Back-end

	P0	P1	P2	P3	P4	P5	P6	P7	P8	P9	P10	P11
uops	30.50	6.00	14.33	14.33	0.50	30.50	2.00	0.50	0.50	0.50	2.00	14.33
cycles	30.50	6.00	14.33	14.33	0.50	30.50	2.00	0.50	0.50	0.50	2.00	14.33

Execution ports to units layout:

P0 (256 bits): VPU, BRU, ALU, DIV/SQRT
P1 (256 bits): ALU, VPU
P2 (512 bits): load
P3 (512 bits): load
P4 (512 bits): store data
P5 (512 bits): ALU, VPU
P6: ALU, BRU
P7: store address
P8: store address
P9 (512 bits): store data
P10: ALU
P11 (512 bits): load

Cycles executing div or sqrt instructions	48.00
Longest recurrence chain latency (RecMII)	0.00

Cycles summary

Front-end	16.00
Dispatch	30.50
DIV/SQRT	48.00
Data deps.	0.00
Overall L1	48.00

Vectorization ratios

INT

all	94%
load	NA (no load vectorizable/vectorized instructions)
store	NA (no store vectorizable/vectorized instructions)
mul	NA (no mul vectorizable/vectorized instructions)
add-sub	100%
fma	NA (no fma vectorizable/vectorized instructions)
other	90%

all	100%
load	100%
store	100%
mul	100%
add-sub	100%
fma	100%
div/sqrt	100%
other	100%

INT+FP

all	98%
load	100%
store	100%
mul	100%
add-sub	100%
fma	100%
div/sqrt	100%
other	96%

Vector efficiency ratios

INT

all	57%
load	NA (no load vectorizable/vectorized instructions)
store	NA (no store vectorizable/vectorized instructions)
mul	NA (no mul vectorizable/vectorized instructions)
add-sub	75%
fma	NA (no fma vectorizable/vectorized instructions)
other	43%

all	90%
load	100%
store	100%
mul	100%
add-sub	100%
fma	100%
div/sqrt	100%
other	82%

INT+FP

all	80%
load	100%
store	100%
mul	100%
add-sub	84%
fma	100%
div/sqrt	100%
other	70%

Cycles and memory resources usage

Detected masked instructions: assuming all mask elements are active. Assuming all data fit into the L1 cache, each iteration of the binary loop takes 48.00 cycles. At this rate:

4% of peak load performance is reached (9.50 out of 192.00 bytes loaded per cycle (GB/s @ 1GHz))
1% of peak store performance is reached (1.33 out of 128.00 bytes stored per cycle (GB/s @ 1GHz))

Front-end bottlenecks

Found no such bottlenecks.

ASM code

In the binary file, the address of the loop is: 422873

Instruction	Nb FU	P0	P1	P2	P3	P4	P5	P6	P7	P8	P9	P10	P11	Latency	Recip. throughput	Vectorization
VCMPPD $0x1,%ZMM21,%ZMM5,%K1	1	0	0	0	0	0	1	0	0	0	0	0	0	3	1	vect (100.0%)
VMOVAPD %ZMM25,%ZMM20{%K1}{z}	1	0	0	0	0	0	0	0	0	0	0	0	0	0-1	0.17	vect (100.0%)
VSUBPD %ZMM19,%ZMM10,%ZMM19	1	0.50	0	0	0	0	0.50	0	0	0	0	0	0	3	0.50	vect (100.0%)
VFMADD213PD %ZMM18,%ZMM20,%ZMM19	1	0.50	0	0	0	0	0.50	0	0	0	0	0	0	4	0.50	vect (100.0%)
VMULPD %ZMM17,%ZMM19,%ZMM17	1	0.50	0	0	0	0	0.50	0	0	0	0	0	0	4	0.50	vect (100.0%)
VMOVUPD %ZMM17,(%RDI,%RBX,8)	1	0	0	0	0	0.50	0	0	0.50	0.50	0.50	0	0	0-1	1	vect (100.0%)
ADD $0x8,%RBX	1	0	0	0	0	0	0	0	0	0	0	0	0	1	0.17	N/A
CMP %R11,%RBX	1	0.20	0.20	0	0	0	0.20	0.20	0	0	0	0.20	0	1	0.20	N/A
JA 422a00 <advec_mom_kernel_mod_mp_advec_mom_kernel_.DIR.OMP.PARALLEL.2+0x3ba0>	1	0.50	0	0	0	0	0	0.50	0	0	0	0	0	0	0.50	N/A
VMOVUPD (%RSI,%RBX,8),%ZMM17	1	0	0	0.33	0.33	0	0	0	0	0	0	0	0.33	0-1	0.50	vect (100.0%)
VFPCLASSPD $0x50,%ZMM17,%K1	1	0	0	0	0	0	1	0	0	0	0	0	0	3	1	vect (100.0%)
LEA (%R13,%RBX,1),%EAX	1	0.20	0.20	0	0	0	0.20	0.20	0	0	0	0.20	0	1-2	0.20	N/A
VPBROADCASTD %EAX,%YMM19	1	0	0	0	0	0	1	0	0	0	0	0	0	3	1	scal (6.3%)
VPADDD %YMM7,%YMM19,%YMM20	1	0.33	0.33	0	0	0	0.33	0	0	0	0	0	0	1	0.33	vect (50.0%)
VPADDD %YMM15,%YMM19,%YMM18	1	0.33	0.33	0	0	0	0.33	0	0	0	0	0	0	1	0.33	vect (50.0%)
VPBLENDMD %YMM18,%YMM20,%YMM21{%K1}	1	0.33	0.33	0	0	0	0.33	0	0	0	0	0	0	1	0.33	vect (50.0%)
VMOVDQA32 %YMM20,%YMM18{%K1}	1	0	0	0	0	0	0	0	0	0	0	0	0	0-1	0.17	vect (50.0%)
VPMOVSXDQ %YMM18,%ZMM18	1	0	0	0	0	0	1	0	0	0	0	0	0	3	1	vect (50.0%)
VPSUBQ %ZMM4,%ZMM18,%ZMM22	1	0.50	0	0	0	0	0.50	0	0	0	0	0	0	0-1	0.50	vect (100.0%)
KXNORW %K0,%K0,%K2	1	1	0	0	0	0	0	0	0	0	0	0	0	1	1	N/A
VXORPD %XMM23,%XMM23,%XMM23	1	0	0	0	0	0	0	0	0	0	0	0	0	0	0.17	vect (25.0%)
VGATHERQPD (%R8,%ZMM22,8),%ZMM23{%K2}	5	1	0	2.67	2.67	0	2	0	0	0	0	0	2.67	28	3	vect (100.0%)
KXNORW %K0,%K0,%K2	1	1	0	0	0	0	0	0	0	0	0	0	0	1	1	N/A
VPXORD %XMM18,%XMM18,%XMM18	1	0.33	0.33	0	0	0	0.33	0	0	0	0	0	0	0-1	0.33	vect (25.0%)
VGATHERQPD (%R10,%ZMM22,8),%ZMM18{%K2}	5	1	0	2.67	2.67	0	2	0	0	0	0	0	2.67	28	3	vect (100.0%)
VPADDD %YMM14,%YMM19,%YMM22	1	0.33	0.33	0	0	0	0.33	0	0	0	0	0	0	1	0.33	vect (50.0%)
VMOVDQA64 %YMM22,%YMM24	1	0	0	0	0	0	0	0	0	0	0	0	0	0-1	0.17	vect (50.0%)
VPADDD %YMM16,%YMM19,%YMM24{%K1}	1	0.33	0.33	0	0	0	0.33	0	0	0	0	0	0	1	0.33	vect (50.0%)
VANDPD %ZMM9,%ZMM17,%ZMM19	1	0.50	0	0	0	0	0.50	0	0	0	0	0	0	1	0.50	vect (100.0%)
VPMOVSXDQ %YMM24,%ZMM24	1	0	0	0	0	0	1	0	0	0	0	0	0	3	1	vect (50.0%)
VPSUBQ %ZMM4,%ZMM24,%ZMM24	1	0.50	0	0	0	0	0.50	0	0	0	0	0	0	0-1	0.50	vect (100.0%)
KXNORW %K0,%K0,%K2	1	1	0	0	0	0	0	0	0	0	0	0	0	1	1	N/A
VXORPD %XMM26,%XMM26,%XMM26	1	0	0	0	0	0	0	0	0	0	0	0	0	0	0.17	vect (25.0%)
VGATHERQPD (%R10,%ZMM24,8),%ZMM26{%K2}	5	1	0	2.67	2.67	0	2	0	0	0	0	0	2.67	28	3	vect (100.0%)
VDIVPD %ZMM23,%ZMM19,%ZMM19	3	2.50	0	0	0	0	0.50	0	0	0	0	0	0	22-24	16	vect (100.0%)
VPMOVSXDQ %YMM21,%ZMM21	1	0	0	0	0	0	1	0	0	0	0	0	0	3	1	vect (50.0%)
VPSUBQ %ZMM4,%ZMM21,%ZMM21	1	0.50	0	0	0	0	0.50	0	0	0	0	0	0	0-1	0.50	vect (100.0%)
KXNORW %K0,%K0,%K2	1	1	0	0	0	0	0	0	0	0	0	0	0	1	1	N/A
VXORPD %XMM23,%XMM23,%XMM23	1	0	0	0	0	0	0	0	0	0	0	0	0	0	0.17	vect (25.0%)
VGATHERQPD (%R10,%ZMM21,8),%ZMM23{%K2}	5	1	0	2.67	2.67	0	2	0	0	0	0	0	2.67	28	3	vect (100.0%)
VXORPD %XMM25,%XMM25,%XMM25	1	0	0	0	0	0	0	0	0	0	0	0	0	0	0.17	vect (25.0%)
VSUBPD %ZMM26,%ZMM18,%ZMM24	1	0.50	0	0	0	0	0.50	0	0	0	0	0	0	3	0.50	vect (100.0%)
VSUBPD %ZMM18,%ZMM23,%ZMM23	1	0.50	0	0	0	0	0.50	0	0	0	0	0	0	3	0.50	vect (100.0%)
VMULPD %ZMM24,%ZMM23,%ZMM21	1	0.50	0	0	0	0	0.50	0	0	0	0	0	0	4	0.50	vect (100.0%)
VCMPPD $0x1,%ZMM21,%ZMM25,%K0	1	0	0	0	0	0	1	0	0	0	0	0	0	3	1	vect (100.0%)
KORTESTB %K0,%K0	1	1	0	0	0	0	0	0	0	0	0	0	0	0	1	N/A
JE 422840 <advec_mom_kernel_mod_mp_advec_mom_kernel_.DIR.OMP.PARALLEL.2+0x39e0>	1	0.50	0	0	0	0	0	0.50	0	0	0	0	0	0	0.50	N/A
VCMPPD $0x1,%ZMM21,%ZMM5,%K2	1	0	0	0	0	0	1	0	0	0	0	0	0	3	1	vect (100.0%)
VMOVDQA32 %YMM20,%YMM22{%K1}	1	0	0	0	0	0	0	0	0	0	0	0	0	0-1	0.17	vect (50.0%)
VMOVUPD (%R9,%RBX,8),%ZMM20{%K2}{z}	1	0	0	0.33	0.33	0	0	0	0	0	0	0	0.33	0-1	0.50	vect (100.0%)
VANDPD %ZMM9,%ZMM23,%ZMM25	1	0.50	0	0	0	0	0.50	0	0	0	0	0	0	1	0.50	vect (100.0%)
VSUBPD %ZMM19,%ZMM11,%ZMM26	1	0.50	0	0	0	0	0.50	0	0	0	0	0	0	3	0.50	vect (100.0%)
VMULPD %ZMM26,%ZMM25,%ZMM26	1	0.50	0	0	0	0	0.50	0	0	0	0	0	0	4	0.50	vect (100.0%)
VDIVPD %ZMM20,%ZMM26,%ZMM26	3	2.50	0	0	0	0	0.50	0	0	0	0	0	0	22-24	16	vect (100.0%)
VPMOVSXDQ %YMM22,%ZMM22	1	0	0	0	0	0	1	0	0	0	0	0	0	3	1	vect (50.0%)
VPSUBQ %ZMM4,%ZMM22,%ZMM22	1	0.50	0	0	0	0	0.50	0	0	0	0	0	0	0-1	0.50	vect (100.0%)
VXORPD %XMM27,%XMM27,%XMM27	1	0	0	0	0	0	0	0	0	0	0	0	0	0	0.17	vect (25.0%)
MOV 0x68(%RSP),%RAX	1	0	0	0.33	0.33	0	0	0	0	0	0	0	0.33	1	0.33	N/A
VGATHERQPD (%RAX,%ZMM22,8),%ZMM27{%K2}	5	1	0	2.67	2.67	0	2	0	0	0	0	0	2.67	28	3	vect (100.0%)
VANDPD %ZMM9,%ZMM24,%ZMM22	1	0.50	0	0	0	0	0.50	0	0	0	0	0	0	1	0.50	vect (100.0%)
VMINPD %ZMM25,%ZMM22,%ZMM24	1	0.50	0	0	0	0	0.50	0	0	0	0	0	0	4	0.50	vect (100.0%)
VFMADD213PD %ZMM22,%ZMM19,%ZMM22	1	0.50	0	0	0	0	0.50	0	0	0	0	0	0	4	0.50	vect (100.0%)
VDIVPD %ZMM27,%ZMM22,%ZMM22	3	2.50	0	0	0	0	0.50	0	0	0	0	0	0	22-24	16	vect (100.0%)
VADDPD %ZMM26,%ZMM22,%ZMM22	1	0.50	0	0	0	0	0.50	0	0	0	0	0	0	3	0.50	vect (100.0%)
VMULPD %ZMM12,%ZMM20,%ZMM20	1	0.50	0	0	0	0	0.50	0	0	0	0	0	0	4	0.50	vect (100.0%)
VMULPD %ZMM22,%ZMM20,%ZMM20	1	0.50	0	0	0	0	0.50	0	0	0	0	0	0	4	0.50	vect (100.0%)
VMINPD %ZMM24,%ZMM20,%ZMM25	1	0.50	0	0	0	0	0.50	0	0	0	0	0	0	4	0.50	vect (100.0%)
VFPCLASSPD $0x56,%ZMM23,%K1	1	0	0	0	0	0	1	0	0	0	0	0	0	3	1	vect (100.0%)
VXORPD %ZMM13,%ZMM25,%ZMM25{%K1}	1	0.50	0	0	0	0	0.50	0	0	0	0	0	0	0-1	0.50	vect (100.0%)
JMP 422840 <advec_mom_kernel_mod_mp_advec_mom_kernel_.DIR.OMP.PARALLEL.2+0x39e0>	1	0	0	0	0	0	0	0	0	0	0	0	0	0	2.08	N/A

Function	advec_mom_kernel_mod_mp_advec_mom_kernel_.DIR.OMP.PARALLEL.2
Source file and lines	advec_mom_kernel.f90:81-177
Module	exec

hoisting them (moving them outside the loop)
turning them into conditional moves, MIN or MAX

Ex: if (x<0) x=0 => x = max(0,x) (Fortran instrinsic procedure)

gain
potential
hint
expert

Code clean check

Detected a slowdown caused by scalar integer instructions (typically used for address computation). By removing them, you can lower the cost of an iteration from 18.50 to 16.00 cycles (1.16x speedup).

Workaround

Try to reorganize arrays of structures to structures of arrays
Consider to permute loops (see vectorization gain report)
To reference allocatable arrays, use "allocatable" instead of "pointer" pointers or qualify them with the "contiguous" attribute (Fortran 2008)
For structures, limit to one indirection. For example, use a_b%c instead of a%b%c with a_b set to a%b before this loop

Vectorization

Your loop is highly vectorized. Only 73% of vector register length is used (average across all SSE/AVX instructions). By fully vectorizing your loop, you can lower the cost of an iteration from 18.50 to 18.06 cycles (1.02x speedup).

Details

97% of SSE/AVX instructions are used in vector version (process two or more data elements in vector registers):

95% of SSE/AVX instructions that are not load, store, addition, subtraction nor multiply instructions are used in vector version.

Since your execution units are vector units, only a fully vectorized loop can use their full power.

Workaround

Pass to your compiler a micro-architecture specialization option:
- use axHost or xHost
Use vector aligned instructions:
1. align your arrays on 64 bytes boundaries: compile with -align array64byte (remark: not affects arrays in COMMON blocks).
2. inform your compiler that your arrays are vector aligned: Append !DIR$ VECTOR ALIGNED to the loop if all accessed arrays are aligned, or !DIR$ ASSUME_ALIGNED FOO: 64 if only FOO is aligned.
Use the LOOP COUNT directive

Execution units bottlenecks

Performance is limited by execution of FP multiply or FMA (fused multiply-add) operations (the FP multiply/FMA unit is a bottleneck).

Workaround

Reduce the number of FP multiply/FMA instructions

Expensive FP math instructions/calls

Detected performance impact from expensive FP math instructions/calls. By removing/reexpressing them, you can lower the cost of an iteration from 18.50 to 17.00 cycles (1.09x speedup).

Special instructions executing on a single port

1.16x slowdown from special instructions executing on a single port.

Details

KORTESTB: 1 occurrences►
- /home/eoseret/qaas_runs_GNR/173-814-3702/intel/CloverLeaf1.3-FC/build/CloverLeaf1.3-FC/CloverLeaf_ref/kernels/advec_mom_kernel.f90:81
KXNORW: 4 occurrences►
- /home/eoseret/qaas_runs_GNR/173-814-3702/intel/CloverLeaf1.3-FC/build/CloverLeaf1.3-FC/CloverLeaf_ref/kernels/advec_mom_kernel.f90:163
- /home/eoseret/qaas_runs_GNR/173-814-3702/intel/CloverLeaf1.3-FC/build/CloverLeaf1.3-FC/CloverLeaf_ref/kernels/advec_mom_kernel.f90:165
- /home/eoseret/qaas_runs_GNR/173-814-3702/intel/CloverLeaf1.3-FC/build/CloverLeaf1.3-FC/CloverLeaf_ref/kernels/advec_mom_kernel.f90:165
- /home/eoseret/qaas_runs_GNR/173-814-3702/intel/CloverLeaf1.3-FC/build/CloverLeaf1.3-FC/CloverLeaf_ref/kernels/advec_mom_kernel.f90:166
VCMPPD: 2 occurrences►
- /home/eoseret/qaas_runs_GNR/173-814-3702/intel/CloverLeaf1.3-FC/build/CloverLeaf1.3-FC/CloverLeaf_ref/kernels/advec_mom_kernel.f90:168
- /home/eoseret/qaas_runs_GNR/173-814-3702/intel/CloverLeaf1.3-FC/build/CloverLeaf1.3-FC/CloverLeaf_ref/kernels/advec_mom_kernel.f90:168
VFPCLASSPD: 1 occurrences►
- /home/eoseret/qaas_runs_GNR/173-814-3702/intel/CloverLeaf1.3-FC/build/CloverLeaf1.3-FC/CloverLeaf_ref/kernels/advec_mom_kernel.f90:152
VGATHERQPD: 4 occurrences►
- /home/eoseret/qaas_runs_GNR/173-814-3702/intel/CloverLeaf1.3-FC/build/CloverLeaf1.3-FC/CloverLeaf_ref/kernels/advec_mom_kernel.f90:163
- /home/eoseret/qaas_runs_GNR/173-814-3702/intel/CloverLeaf1.3-FC/build/CloverLeaf1.3-FC/CloverLeaf_ref/kernels/advec_mom_kernel.f90:165
- /home/eoseret/qaas_runs_GNR/173-814-3702/intel/CloverLeaf1.3-FC/build/CloverLeaf1.3-FC/CloverLeaf_ref/kernels/advec_mom_kernel.f90:165
- /home/eoseret/qaas_runs_GNR/173-814-3702/intel/CloverLeaf1.3-FC/build/CloverLeaf1.3-FC/CloverLeaf_ref/kernels/advec_mom_kernel.f90:166
VPBROADCASTD: 1 occurrences►
- /home/eoseret/qaas_runs_GNR/173-814-3702/intel/CloverLeaf1.3-FC/build/CloverLeaf1.3-FC/CloverLeaf_ref/kernels/advec_mom_kernel.f90:158
VPMOVSXDQ: 3 occurrences►
- /home/eoseret/qaas_runs_GNR/173-814-3702/intel/CloverLeaf1.3-FC/build/CloverLeaf1.3-FC/CloverLeaf_ref/kernels/advec_mom_kernel.f90:163
- /home/eoseret/qaas_runs_GNR/173-814-3702/intel/CloverLeaf1.3-FC/build/CloverLeaf1.3-FC/CloverLeaf_ref/kernels/advec_mom_kernel.f90:165
- /home/eoseret/qaas_runs_GNR/173-814-3702/intel/CloverLeaf1.3-FC/build/CloverLeaf1.3-FC/CloverLeaf_ref/kernels/advec_mom_kernel.f90:166

Masked instructions

Detected masked instructions.

Details

Vector registers are partially exploited, which is expected if your loop is irregular or mixes elements of different sizes.

Workaround

If your loop is irregular, try to remove or hoist conditional structures out of your loop. If it mixes elements of different sizes, try to uniformize them.

FMA

Detected 8 FMA (fused multiply-add) operations. Presence of both ADD/SUB and MUL operations.

Workaround

Complex instructions

Detected COMPLEX INSTRUCTIONS.

Details

These instructions generate more than one micro-operation and only one of them can be decoded during a cycle and the extra micro-operations increase pressure on execution units.

VDIVPD: 1 occurrences►
- /home/eoseret/qaas_runs_GNR/173-814-3702/intel/CloverLeaf1.3-FC/build/CloverLeaf1.3-FC/CloverLeaf_ref/kernels/advec_mom_kernel.f90:163
VGATHERQPD: 4 occurrences►
- /home/eoseret/qaas_runs_GNR/173-814-3702/intel/CloverLeaf1.3-FC/build/CloverLeaf1.3-FC/CloverLeaf_ref/kernels/advec_mom_kernel.f90:163
- /home/eoseret/qaas_runs_GNR/173-814-3702/intel/CloverLeaf1.3-FC/build/CloverLeaf1.3-FC/CloverLeaf_ref/kernels/advec_mom_kernel.f90:165
- /home/eoseret/qaas_runs_GNR/173-814-3702/intel/CloverLeaf1.3-FC/build/CloverLeaf1.3-FC/CloverLeaf_ref/kernels/advec_mom_kernel.f90:165
- /home/eoseret/qaas_runs_GNR/173-814-3702/intel/CloverLeaf1.3-FC/build/CloverLeaf1.3-FC/CloverLeaf_ref/kernels/advec_mom_kernel.f90:166

Slow data structures access

Detected data structures (typically arrays) that cannot be efficiently read/written

Details

Irregular (variable stride) or indirect: 2 occurrence(s)

Non-unit stride (uncontiguous) accesses are not efficiently using data caches

Workaround

Try to remove indirect accesses. If applicable, precompute elements out of the innermost loop.

Gather/scatter instructions

Detected gather/scatter instructions (typically caused by indirect accesses).

Details

VGATHERQPD: 4 occurrences►
- /home/eoseret/qaas_runs_GNR/173-814-3702/intel/CloverLeaf1.3-FC/build/CloverLeaf1.3-FC/CloverLeaf_ref/kernels/advec_mom_kernel.f90:163
- /home/eoseret/qaas_runs_GNR/173-814-3702/intel/CloverLeaf1.3-FC/build/CloverLeaf1.3-FC/CloverLeaf_ref/kernels/advec_mom_kernel.f90:165
- /home/eoseret/qaas_runs_GNR/173-814-3702/intel/CloverLeaf1.3-FC/build/CloverLeaf1.3-FC/CloverLeaf_ref/kernels/advec_mom_kernel.f90:165
- /home/eoseret/qaas_runs_GNR/173-814-3702/intel/CloverLeaf1.3-FC/build/CloverLeaf1.3-FC/CloverLeaf_ref/kernels/advec_mom_kernel.f90:166

Workaround

Try to simplify your code and/or replace indirect accesses with unit-stride ones.

Type of elements and instruction set

10 AVX-512 instructions are processing arithmetic or math operations on double precision FP elements in vector mode (eight at a time).

Matching between your loop (in the source code) and the binary loop

The binary loop is composed of 64 FP arithmetical operations:

32: addition or subtraction (8 inside FMA instructions)
24: multiply (8 inside FMA instructions)
8: divide

The binary loop is loading 320 bytes (40 double precision FP elements). The binary loop is storing 64 bytes (8 double precision FP elements).

Arithmetic intensity

Arithmetic intensity is 0.17 FP operations per loaded or stored byte.

Unroll opportunity

Loop is potentially data access bound.

Workaround

Unroll your loop if trip count is significantly higher than target unroll factor and if some data references are common to consecutive iterations. This can be done manually. Or with the unroll (resp. unroll_and_jam) directive on top of the inner (resp. surrounding) loop. You can enforce an unroll factor: !DIR$ UNROLL_AND_JAM(N), UNROLL_AND_JAM=N, UNROLL(N) or UNROLL=N

General properties

nb instructions	47
nb uops	65
loop length	275
used x86 registers	8
used mmx registers	0
used xmm registers	4
used ymm registers	10
used zmm registers	14
nb stack references	0
ADD-SUB / MUL ratio	1.50

Front-end

MACRO FUSION NOT POSSIBLE FIT IN UOP CACHE

micro-operation queue	10.83 cycles
front end	10.83 cycles

Back-end

	P0	P1	P2	P3	P4	P5	P6	P7	P8	P9	P10	P11
uops	18.50	6.00	11.00	11.00	0.50	18.50	2.00	0.50	0.50	0.50	2.00	11.00
cycles	18.50	6.00	11.00	11.00	0.50	18.50	2.00	0.50	0.50	0.50	2.00	11.00

Execution ports to units layout:

P0 (256 bits): VPU, BRU, ALU, DIV/SQRT
P1 (256 bits): ALU, VPU
P2 (512 bits): load
P3 (512 bits): load
P4 (512 bits): store data
P5 (512 bits): ALU, VPU
P6: ALU, BRU
P7: store address
P8: store address
P9 (512 bits): store data
P10: ALU
P11 (512 bits): load

Cycles executing div or sqrt instructions	16.00
Longest recurrence chain latency (RecMII)	0.00

Cycles summary

Front-end	10.83
Dispatch	18.50
DIV/SQRT	16.00
Data deps.	0.00
Overall L1	18.50

Vectorization ratios

INT

all	93%
load	NA (no load vectorizable/vectorized instructions)
store	NA (no store vectorizable/vectorized instructions)
mul	NA (no mul vectorizable/vectorized instructions)
add-sub	100%
fma	NA (no fma vectorizable/vectorized instructions)
other	87%

all	100%
load	100%
store	100%
mul	100%
add-sub	100%
fma	100%
div/sqrt	100%
other	100%

INT+FP

all	97%
load	100%
store	100%
mul	100%
add-sub	100%
fma	100%
div/sqrt	100%
other	95%

Vector efficiency ratios

INT

all	55%
load	NA (no load vectorizable/vectorized instructions)
store	NA (no store vectorizable/vectorized instructions)
mul	NA (no mul vectorizable/vectorized instructions)
add-sub	71%
fma	NA (no fma vectorizable/vectorized instructions)
other	41%

all	86%
load	100%
store	100%
mul	100%
add-sub	100%
fma	100%
div/sqrt	100%
other	76%

INT+FP

all	73%
load	100%
store	100%
mul	100%
add-sub	80%
fma	100%
div/sqrt	100%
other	63%

Cycles and memory resources usage

Detected masked instructions: assuming all mask elements are active. Assuming all data fit into the L1 cache, each iteration of the binary loop takes 18.50 cycles. At this rate:

9% of peak load performance is reached (17.30 out of 192.00 bytes loaded per cycle (GB/s @ 1GHz))
2% of peak store performance is reached (3.46 out of 128.00 bytes stored per cycle (GB/s @ 1GHz))

Front-end bottlenecks

Found no such bottlenecks.

ASM code

In the binary file, the address of the loop is: 422873

Instruction	Nb FU	P0	P1	P2	P3	P4	P5	P6	P7	P8	P9	P10	P11	Latency	Recip. throughput	Vectorization
VCMPPD $0x1,%ZMM21,%ZMM5,%K1	1	0	0	0	0	0	1	0	0	0	0	0	0	3	1	vect (100.0%)
VMOVAPD %ZMM25,%ZMM20{%K1}{z}	1	0	0	0	0	0	0	0	0	0	0	0	0	0-1	0.17	vect (100.0%)
VSUBPD %ZMM19,%ZMM10,%ZMM19	1	0.50	0	0	0	0	0.50	0	0	0	0	0	0	3	0.50	vect (100.0%)
VFMADD213PD %ZMM18,%ZMM20,%ZMM19	1	0.50	0	0	0	0	0.50	0	0	0	0	0	0	4	0.50	vect (100.0%)
VMULPD %ZMM17,%ZMM19,%ZMM17	1	0.50	0	0	0	0	0.50	0	0	0	0	0	0	4	0.50	vect (100.0%)
VMOVUPD %ZMM17,(%RDI,%RBX,8)	1	0	0	0	0	0.50	0	0	0.50	0.50	0.50	0	0	0-1	1	vect (100.0%)
ADD $0x8,%RBX	1	0	0	0	0	0	0	0	0	0	0	0	0	1	0.17	N/A
CMP %R11,%RBX	1	0.20	0.20	0	0	0	0.20	0.20	0	0	0	0.20	0	1	0.20	N/A
JA 422a00 <advec_mom_kernel_mod_mp_advec_mom_kernel_.DIR.OMP.PARALLEL.2+0x3ba0>	1	0.50	0	0	0	0	0	0.50	0	0	0	0	0	0	0.50	N/A
VMOVUPD (%RSI,%RBX,8),%ZMM17	1	0	0	0.33	0.33	0	0	0	0	0	0	0	0.33	0-1	0.50	vect (100.0%)
VFPCLASSPD $0x50,%ZMM17,%K1	1	0	0	0	0	0	1	0	0	0	0	0	0	3	1	vect (100.0%)
LEA (%R13,%RBX,1),%EAX	1	0.20	0.20	0	0	0	0.20	0.20	0	0	0	0.20	0	1-2	0.20	N/A
VPBROADCASTD %EAX,%YMM19	1	0	0	0	0	0	1	0	0	0	0	0	0	3	1	scal (6.3%)
VPADDD %YMM7,%YMM19,%YMM20	1	0.33	0.33	0	0	0	0.33	0	0	0	0	0	0	1	0.33	vect (50.0%)
VPADDD %YMM15,%YMM19,%YMM18	1	0.33	0.33	0	0	0	0.33	0	0	0	0	0	0	1	0.33	vect (50.0%)
VPBLENDMD %YMM18,%YMM20,%YMM21{%K1}	1	0.33	0.33	0	0	0	0.33	0	0	0	0	0	0	1	0.33	vect (50.0%)
VMOVDQA32 %YMM20,%YMM18{%K1}	1	0	0	0	0	0	0	0	0	0	0	0	0	0-1	0.17	vect (50.0%)
VPMOVSXDQ %YMM18,%ZMM18	1	0	0	0	0	0	1	0	0	0	0	0	0	3	1	vect (50.0%)
VPSUBQ %ZMM4,%ZMM18,%ZMM22	1	0.50	0	0	0	0	0.50	0	0	0	0	0	0	0-1	0.50	vect (100.0%)
KXNORW %K0,%K0,%K2	1	1	0	0	0	0	0	0	0	0	0	0	0	1	1	N/A
VXORPD %XMM23,%XMM23,%XMM23	1	0	0	0	0	0	0	0	0	0	0	0	0	0	0.17	vect (25.0%)
VGATHERQPD (%R8,%ZMM22,8),%ZMM23{%K2}	5	1	0	2.67	2.67	0	2	0	0	0	0	0	2.67	28	3	vect (100.0%)
KXNORW %K0,%K0,%K2	1	1	0	0	0	0	0	0	0	0	0	0	0	1	1	N/A
VPXORD %XMM18,%XMM18,%XMM18	1	0.33	0.33	0	0	0	0.33	0	0	0	0	0	0	0-1	0.33	vect (25.0%)
VGATHERQPD (%R10,%ZMM22,8),%ZMM18{%K2}	5	1	0	2.67	2.67	0	2	0	0	0	0	0	2.67	28	3	vect (100.0%)
VPADDD %YMM14,%YMM19,%YMM22	1	0.33	0.33	0	0	0	0.33	0	0	0	0	0	0	1	0.33	vect (50.0%)
VMOVDQA64 %YMM22,%YMM24	1	0	0	0	0	0	0	0	0	0	0	0	0	0-1	0.17	vect (50.0%)
VPADDD %YMM16,%YMM19,%YMM24{%K1}	1	0.33	0.33	0	0	0	0.33	0	0	0	0	0	0	1	0.33	vect (50.0%)
VANDPD %ZMM9,%ZMM17,%ZMM19	1	0.50	0	0	0	0	0.50	0	0	0	0	0	0	1	0.50	vect (100.0%)
VPMOVSXDQ %YMM24,%ZMM24	1	0	0	0	0	0	1	0	0	0	0	0	0	3	1	vect (50.0%)
VPSUBQ %ZMM4,%ZMM24,%ZMM24	1	0.50	0	0	0	0	0.50	0	0	0	0	0	0	0-1	0.50	vect (100.0%)
KXNORW %K0,%K0,%K2	1	1	0	0	0	0	0	0	0	0	0	0	0	1	1	N/A
VXORPD %XMM26,%XMM26,%XMM26	1	0	0	0	0	0	0	0	0	0	0	0	0	0	0.17	vect (25.0%)
VGATHERQPD (%R10,%ZMM24,8),%ZMM26{%K2}	5	1	0	2.67	2.67	0	2	0	0	0	0	0	2.67	28	3	vect (100.0%)
VDIVPD %ZMM23,%ZMM19,%ZMM19	3	2.50	0	0	0	0	0.50	0	0	0	0	0	0	22-24	16	vect (100.0%)
VPMOVSXDQ %YMM21,%ZMM21	1	0	0	0	0	0	1	0	0	0	0	0	0	3	1	vect (50.0%)
VPSUBQ %ZMM4,%ZMM21,%ZMM21	1	0.50	0	0	0	0	0.50	0	0	0	0	0	0	0-1	0.50	vect (100.0%)
KXNORW %K0,%K0,%K2	1	1	0	0	0	0	0	0	0	0	0	0	0	1	1	N/A
VXORPD %XMM23,%XMM23,%XMM23	1	0	0	0	0	0	0	0	0	0	0	0	0	0	0.17	vect (25.0%)
VGATHERQPD (%R10,%ZMM21,8),%ZMM23{%K2}	5	1	0	2.67	2.67	0	2	0	0	0	0	0	2.67	28	3	vect (100.0%)
VXORPD %XMM25,%XMM25,%XMM25	1	0	0	0	0	0	0	0	0	0	0	0	0	0	0.17	vect (25.0%)
VSUBPD %ZMM26,%ZMM18,%ZMM24	1	0.50	0	0	0	0	0.50	0	0	0	0	0	0	3	0.50	vect (100.0%)
VSUBPD %ZMM18,%ZMM23,%ZMM23	1	0.50	0	0	0	0	0.50	0	0	0	0	0	0	3	0.50	vect (100.0%)
VMULPD %ZMM24,%ZMM23,%ZMM21	1	0.50	0	0	0	0	0.50	0	0	0	0	0	0	4	0.50	vect (100.0%)
VCMPPD $0x1,%ZMM21,%ZMM25,%K0	1	0	0	0	0	0	1	0	0	0	0	0	0	3	1	vect (100.0%)
KORTESTB %K0,%K0	1	1	0	0	0	0	0	0	0	0	0	0	0	0	1	N/A
JE 422840 <advec_mom_kernel_mod_mp_advec_mom_kernel_.DIR.OMP.PARALLEL.2+0x39e0>	1	0.50	0	0	0	0	0	0.50	0	0	0	0	0	0	0.50	N/A

Report Configuration

Vectorization

Details

Workaround

Expensive FP math instructions/calls

Masked instructions

Details

Workaround

FMA

Workaround

Complex instructions

Details

Slow data structures access

Details

Workaround

Gather/scatter instructions

Details

Workaround

Type of elements and instruction set

Matching between your loop (in the source code) and the binary loop

Arithmetic intensity

General properties

Front-end

Back-end

Cycles summary

Vectorization ratios

Vector efficiency ratios

Cycles and memory resources usage

Vectorization

Details

Execution units bottlenecks

Workaround

Expensive FP math instructions/calls

Masked instructions

Details

Workaround

FMA

Workaround

Complex instructions

Details

Slow data structures access

Details

Workaround

Gather/scatter instructions

Details

Workaround

Type of elements and instruction set

Matching between your loop (in the source code) and the binary loop

Arithmetic intensity

General properties

Front-end

Back-end

Cycles summary

Vectorization ratios

Vector efficiency ratios

Cycles and memory resources usage

Front-end bottlenecks

ASM code

Code clean check

Workaround

Vectorization

Details

Workaround

Execution units bottlenecks

Workaround

Expensive FP math instructions/calls

Special instructions executing on a single port

Details

Masked instructions

Details

Workaround

FMA

Workaround

Complex instructions

Details

Slow data structures access

Details

Workaround

Gather/scatter instructions

Details