OV - exec - Loop 144

	Loop Id: 144	Module: exec	Source: advec_mom_kernel.f90:215-241 [...]	Coverage: 4.36%

0x424c40 MOVAPD	%XMM4,%XMM11

0x424c45 SUBSD	%XMM8,%XMM11

0x424c4a MULSD	%XMM10,%XMM11

0x424c4f ADDSD	%XMM9,%XMM11

0x424c54 MULSD	%XMM7,%XMM11

0x424c59 MOV	-0xe8(%RBP),%R13    [9]

0x424c60 MOV	(%R13),%RCX    [11]

0x424c64 IMUL	%R8,%RCX

0x424c68 ADD	%R10,%RCX

0x424c6b MOVSD	%XMM11,0x10(%RCX,%RAX,8)    [1]

0x424c72 INC	%RAX

0x424c75 CMP	%RAX,-0x30(%RBP)    [9]

0x424c79 JE	424bf0

0x424c7f MOV	-0x120(%RBP),%RCX    [9]

0x424c86 MOV	(%RCX),%R13    [7]

0x424c89 IMUL	%R8,%R13

0x424c8d ADD	-0x68(%RBP),%R13    [9]

0x424c91 MOVSD	0x10(%R13,%RAX,8),%XMM7    [12]

0x424c98 XORPD	%XMM10,%XMM10

0x424c9d UCOMISD	%XMM7,%XMM10

0x424ca2 MOV	%R15D,%R14D

0x424ca5 CMOVA	%R9D,%R14D

0x424ca9 MOV	%R12D,%ECX

0x424cac CMOVA	%EDX,%ECX

0x424caf MOV	%R9D,%R13D

0x424cb2 CMOVA	%R15D,%R13D

0x424cb6 MOVAPD	%XMM7,%XMM8

0x424cbb ANDPD	%XMM0,%XMM8

0x424cc0 MOVSXD	%R14D,%R14

0x424cc3 SUB	%RDI,%R14

0x424cc6 MOV	-0x150(%RBP),%RSI    [9]

0x424ccd MOV	(%RSI),%RSI    [10]

0x424cd0 IMUL	%R14,%RSI

0x424cd4 ADD	%RBX,%RSI

0x424cd7 DIVSD	0x10(%RSI,%RAX,8),%XMM8    [5]

0x424cde MOV	-0x1c8(%RBP),%RSI    [9]

0x424ce5 MOV	(%RSI),%RSI    [4]

0x424ce8 IMUL	%RSI,%R14

0x424cec ADD	%R11,%R14

0x424cef MOVSD	0x10(%R14,%RAX,8),%XMM9    [8]

0x424cf6 MOVSXD	%ECX,%RCX

0x424cf9 SUB	%RDI,%RCX

0x424cfc IMUL	%RSI,%RCX

0x424d00 ADD	%R11,%RCX

0x424d03 MOVAPD	%XMM9,%XMM12

0x424d08 SUBSD	0x10(%RCX,%RAX,8),%XMM12    [6]

0x424d0f MOVSXD	%R13D,%RCX

0x424d12 SUB	%RDI,%RCX

0x424d15 IMUL	%RSI,%RCX

0x424d19 ADD	%R11,%RCX

0x424d1c MOVSD	0x10(%RCX,%RAX,8),%XMM11    [2]

0x424d23 SUBSD	%XMM9,%XMM11

0x424d28 MOVAPD	%XMM11,%XMM13

0x424d2d MULSD	%XMM12,%XMM13

0x424d32 UCOMISD	%XMM10,%XMM13

0x424d37 JBE	424c40

0x424d3d UCOMISD	%XMM7,%XMM1

0x424d41 MOV	%R12D,%ECX

0x424d44 CMOVA	%R9D,%ECX

0x424d48 MOV	-0x100(%RBP),%RSI    [9]

0x424d4f MOV	-0x38(%RBP),%R14    [9]

0x424d53 MOVSD	(%RSI,%R14,8),%XMM13    [3]

0x424d59 ANDPD	%XMM2,%XMM12

0x424d5e MOVAPD	%XMM11,%XMM10

0x424d63 ANDPD	%XMM2,%XMM10

0x424d68 MOVAPD	%XMM3,%XMM14

0x424d6d SUBSD	%XMM8,%XMM14

0x424d72 MULSD	%XMM10,%XMM14

0x424d77 MOVAPD	%XMM8,%XMM15

0x424d7c ADDSD	%XMM4,%XMM15

0x424d81 MULSD	%XMM12,%XMM15

0x424d86 MOVSXD	%ECX,%RCX

0x424d89 SUB	%RDI,%RCX

0x424d8c DIVSD	(%RSI,%RCX,8),%XMM15    [3]

0x424d92 DIVSD	%XMM13,%XMM14

0x424d97 ADDSD	%XMM14,%XMM15

0x424d9c MULSD	%XMM5,%XMM13

0x424da1 MULSD	%XMM15,%XMM13

0x424da6 MOVAPD	%XMM12,%XMM14

0x424dab CMPSD	$0x2,%XMM10,%XMM14

0x424db1 ANDPD	%XMM14,%XMM12

0x424db6 ANDNPD	%XMM10,%XMM14

0x424dbb ORPD	%XMM12,%XMM14

0x424dc0 MOVAPD	%XMM13,%XMM10

0x424dc5 CMPSD	$0x2,%XMM14,%XMM10

0x424dcb ANDPD	%XMM10,%XMM13

0x424dd0 ANDNPD	%XMM14,%XMM10

0x424dd5 ORPD	%XMM13,%XMM10

0x424dda CMPSD	$0x2,%XMM1,%XMM11

0x424de0 MOVAPD	%XMM11,%XMM12

0x424de5 ANDNPD	%XMM10,%XMM12

0x424dea XORPD	%XMM6,%XMM10

0x424def ANDPD	%XMM11,%XMM10

0x424df4 ORPD	%XMM12,%XMM10

0x424df9 JMP	424c40

/home/eoseret/qaas_runs_ZEN5/174-049-0948/intel/CloverLeaf1.3-FC/build/CloverLeaf1.3-FC/CloverLeaf_ref/kernels/advec_mom_kernel.f90: 215 - 241

--------------------------------------------------------------------------------

215:           IF(node_flux(j,k).LT.0.0)THEN

[...]

227:           sigma=ABS(node_flux(j,k))/(node_mass_pre(j,donor))

228:           width=celldy(k)

229:           vdiffuw=vel1(j,donor)-vel1(j,upwind)

230:           vdiffdw=vel1(j,downwind)-vel1(j,donor)

231:           limiter=0.0

232:           IF(vdiffuw*vdiffdw.GT.0.0)THEN

233:             auw=ABS(vdiffuw)

234:             adw=ABS(vdiffdw)

235:             wind=1.0_8

236:             IF(vdiffdw.LE.0.0) wind=-1.0_8

237:             limiter=wind*MIN(width*((2.0_8-sigma)*adw/width+(1.0_8+sigma)*auw/celldy(dif))/6.0_8,auw,adw)

238:           ENDIF

239:           advec_vel_s=vel1(j,donor)+(1.0_8-sigma)*limiter

240:           mom_flux(j,k)=advec_vel_s*node_flux(j,k)

241:         ENDDO

Coverage (%)	Name	Source Location	Module
►53.55+	__kmp_invoke_microtask		libiomp5.so
○	__kmp_invoke_task_func		libiomp5.so
○	__kmp_launch_thread		libiomp5.so
○	_INTERNALea951ae6::__kmp_launc[...]		libiomp5.so
○	start_thread		libc.so.6
○	__GI___clone3		libc.so.6
►35.31+	__kmp_invoke_microtask		libiomp5.so
○	__kmp_invoke_task_func		libiomp5.so
○	__kmp_launch_thread		libiomp5.so
○	_INTERNALea951ae6::__kmp_launc[...]		libiomp5.so
►4.30+	__kmp_invoke_microtask		libiomp5.so
○	__kmp_invoke_task_func		libiomp5.so
○	__kmp_launch_thread		libiomp5.so
○	_INTERNALea951ae6::__kmp_launc[...]		libiomp5.so
○	start_thread		libc.so.6
►2.48+	__kmp_invoke_microtask		libiomp5.so
○	__kmp_invoke_task_func		libiomp5.so
○	__kmp_launch_thread		libiomp5.so
►1.07+	__kmp_invoke_microtask		libiomp5.so
○	__kmp_invoke_task_func		libiomp5.so
○	__kmp_fork_call		libiomp5.so
○	__kmpc_fork_call		libiomp5.so
○	advec_mom_kernel	advec_mom_kernel.f90:256	exec
○	advec_mom_driver	advec_mom_driver.f90:60	exec
○	advection	advection.f90:108	exec
○	hydro	hydro.f90:64	exec
○	main		exec
○	__libc_start_call_main		libc.so.6
○	__libc_start_main		libc.so.6
○	_start		exec
►1.06+	__kmp_invoke_microtask		libiomp5.so
○	__kmp_invoke_task_func		libiomp5.so
○	__kmp_fork_call		libiomp5.so
○	__kmpc_fork_call		libiomp5.so
○	advec_mom_kernel	advec_mom_kernel.f90:256	exec
○	advec_mom_driver	advec_mom_driver.f90:60	exec
○	advection	advection.f90:107	exec
○	hydro	hydro.f90:64	exec
○	main		exec
○	__libc_start_call_main		libc.so.6
○	__libc_start_main		libc.so.6
○	_start		exec
►1.01+	__kmp_invoke_microtask		libiomp5.so
○	__kmp_invoke_task_func		libiomp5.so
○	__kmp_fork_call		libiomp5.so
○	__kmpc_fork_call		libiomp5.so
○	advec_mom_kernel	advec_mom_kernel.f90:256	exec
○	advec_mom_driver	advec_mom_driver.f90:60	exec
○	advection	advection.f90:78	exec
○	hydro	hydro.f90:64	exec
○	main		exec
○	__libc_start_call_main		libc.so.6
○	__libc_start_main		libc.so.6
○	_start		exec

min	med	avg	max

Percentile Index	10	20	30	40	50	60	70	80	90	100
Value

min	med	avg	max

Percentile Index	10	20	30	40	50	60	70	80	90	100
Value

Path /

Metric	Value
CQA speedup if no scalar integer	1.23
CQA speedup if FP arith vectorized	3.44
CQA speedup if fully vectorized	7.70
CQA speedup if no inter-iteration dependency	NA
CQA speedup if next bottleneck killed	NA
Bottlenecks
Function	advec_mom_kernel_mod_mp_advec_mom_kernel_.DIR.OMP.PARALLEL.2
Source	advec_mom_kernel.f90:215-215,advec_mom_kernel.f90:227-241
Source loop unroll info	not unrolled or unrolled with no peel/tail loop
Source loop unroll confidence level	max
Unroll/vectorization loop type	NA
Unroll factor	NA
CQA cycles	10.19
CQA cycles if no scalar integer	8.31
CQA cycles if FP arith vectorized	2.96
CQA cycles if fully vectorized	1.32
Front-end cycles	10.19
P0 cycles	5.08
P1 cycles	5.08
P2 cycles	5.08
P3 cycles	5.08
P4 cycles	5.08
P5 cycles	5.08
P6 cycles	4.50
P7 cycles	4.50
P8 cycles	4.50
P9 cycles	4.50
P10 cycles	5.75
P11 cycles	5.75
P12 cycles	6.00
P13 cycles	6.00
P14 cycles	1.75
P15 cycles	1.75
DIV/SQRT cycles	8.00
Inter-iter dependencies cycles	0
FE+BE cycles (UFS)	NA
Stall cycles (UFS)	NA
Nb insns	75.50
Nb uops	81.50
Nb loads	17.00
Nb stores	1.00
Nb stack references	7.00
FLOP/cycle	1.23
Nb FLOP add-sub	5.50
Nb FLOP mul	5.00
Nb FLOP fma	0.00
Nb FLOP div	2.00
Nb FLOP rcp	0.00
Nb FLOP sqrt	0.00
Nb FLOP rsqrt	0.00
Bytes/cycle	14.67
Bytes prefetched	0.00
Bytes loaded	136.00
Bytes stored	8.00
Stride 0	1.00
Stride 1	6.00
Stride n	0.00
Stride unknown	4.00
Stride indirect	0.50
Vectorization ratio all	36.93
Vectorization ratio load	0.00
Vectorization ratio store	0.00
Vectorization ratio mul	0.00
Vectorization ratio add_sub	0.00
Vectorization ratio fma	NA
Vectorization ratio div_sqrt	0.00
Vectorization ratio other	72.04
Vector-efficiency ratio all	17.12
Vector-efficiency ratio load	12.50
Vector-efficiency ratio store	12.50
Vector-efficiency ratio mul	12.50
Vector-efficiency ratio add_sub	12.50
Vector-efficiency ratio fma	NA
Vector-efficiency ratio div_sqrt	12.50
Vector-efficiency ratio other	21.51

Metric	Value
CQA speedup if no scalar integer	1.06
CQA speedup if FP arith vectorized	3.28
CQA speedup if fully vectorized	7.52
CQA speedup if no inter-iteration dependency	NA
CQA speedup if next bottleneck killed	NA
Bottlenecks	micro-operation queue,
Function	advec_mom_kernel_mod_mp_advec_mom_kernel_.DIR.OMP.PARALLEL.2
Source	advec_mom_kernel.f90:215-215,advec_mom_kernel.f90:227-241
Source loop unroll info	not unrolled or unrolled with no peel/tail loop
Source loop unroll confidence level	max
Unroll/vectorization loop type	NA
Unroll factor	NA
CQA cycles	12.75
CQA cycles if no scalar integer	12.00
CQA cycles if FP arith vectorized	3.88
CQA cycles if fully vectorized	1.69
Front-end cycles	12.75
P0 cycles	5.50
P1 cycles	5.50
P2 cycles	5.50
P3 cycles	5.50
P4 cycles	5.50
P5 cycles	5.50
P6 cycles	5.00
P7 cycles	5.00
P8 cycles	5.00
P9 cycles	5.00
P10 cycles	9.00
P11 cycles	9.00
P12 cycles	9.00
P13 cycles	9.00
P14 cycles	2.00
P15 cycles	2.00
DIV/SQRT cycles	12.00
Inter-iter dependencies cycles	0
FE+BE cycles (UFS)	NA
Stall cycles (UFS)	NA
Nb insns	95.00
Nb uops	102.00
Nb loads	19.00
Nb stores	1.00
Nb stack references	8.00
FLOP/cycle	1.33
Nb FLOP add-sub	7.00
Nb FLOP mul	7.00
Nb FLOP fma	0.00
Nb FLOP div	3.00
Nb FLOP rcp	0.00
Nb FLOP sqrt	0.00
Nb FLOP rsqrt	0.00
Bytes/cycle	12.55
Bytes prefetched	0.00
Bytes loaded	152.00
Bytes stored	8.00
Stride 0	1.00
Stride 1	6.00
Stride n	0.00
Stride unknown	4.00
Stride indirect	1.00
Vectorization ratio all	45.28
Vectorization ratio load	0.00
Vectorization ratio store	0.00
Vectorization ratio mul	0.00
Vectorization ratio add_sub	0.00
Vectorization ratio fma	NA
Vectorization ratio div_sqrt	0.00
Vectorization ratio other	77.42
Vector-efficiency ratio all	18.16
Vector-efficiency ratio load	12.50
Vector-efficiency ratio store	12.50
Vector-efficiency ratio mul	12.50
Vector-efficiency ratio add_sub	12.50
Vector-efficiency ratio fma	NA
Vector-efficiency ratio div_sqrt	12.50
Vector-efficiency ratio other	22.18

Metric	Value
CQA speedup if no scalar integer	1.65
CQA speedup if FP arith vectorized	3.75
CQA speedup if fully vectorized	8.00
CQA speedup if no inter-iteration dependency	NA
CQA speedup if next bottleneck killed	1.63
Bottlenecks	micro-operation queue,
Function	advec_mom_kernel_mod_mp_advec_mom_kernel_.DIR.OMP.PARALLEL.2
Source	advec_mom_kernel.f90:215-215,advec_mom_kernel.f90:227-241
Source loop unroll info	not unrolled or unrolled with no peel/tail loop
Source loop unroll confidence level	max
Unroll/vectorization loop type	NA
Unroll factor	NA
CQA cycles	7.63
CQA cycles if no scalar integer	4.63
CQA cycles if FP arith vectorized	2.03
CQA cycles if fully vectorized	0.95
Front-end cycles	7.63
P0 cycles	4.67
P1 cycles	4.67
P2 cycles	4.67
P3 cycles	4.67
P4 cycles	4.67
P5 cycles	4.67
P6 cycles	4.00
P7 cycles	4.00
P8 cycles	4.00
P9 cycles	4.00
P10 cycles	2.50
P11 cycles	2.50
P12 cycles	3.00
P13 cycles	3.00
P14 cycles	1.50
P15 cycles	1.50
DIV/SQRT cycles	4.00
Inter-iter dependencies cycles	0
FE+BE cycles (UFS)	NA
Stall cycles (UFS)	NA
Nb insns	56.00
Nb uops	61.00
Nb loads	15.00
Nb stores	1.00
Nb stack references	6.00
FLOP/cycle	1.05
Nb FLOP add-sub	4.00
Nb FLOP mul	3.00
Nb FLOP fma	0.00
Nb FLOP div	1.00
Nb FLOP rcp	0.00
Nb FLOP sqrt	0.00
Nb FLOP rsqrt	0.00
Bytes/cycle	16.79
Bytes prefetched	0.00
Bytes loaded	120.00
Bytes stored	8.00
Stride 0	1.00
Stride 1	6.00
Stride n	0.00
Stride unknown	4.00
Stride indirect	0.00
Vectorization ratio all	28.57
Vectorization ratio load	0.00
Vectorization ratio store	0.00
Vectorization ratio mul	0.00
Vectorization ratio add_sub	0.00
Vectorization ratio fma	NA
Vectorization ratio div_sqrt	0.00
Vectorization ratio other	66.67
Vector-efficiency ratio all	16.07
Vector-efficiency ratio load	12.50
Vector-efficiency ratio store	12.50
Vector-efficiency ratio mul	12.50
Vector-efficiency ratio add_sub	12.50
Vector-efficiency ratio fma	NA
Vector-efficiency ratio div_sqrt	12.50
Vector-efficiency ratio other	20.83

Path /

Average path: Display a virtual path defined by average values of all real paths

Function	advec_mom_kernel_mod_mp_advec_mom_kernel_.DIR.OMP.PARALLEL.2
Source file and lines	advec_mom_kernel.f90:215-241
Module	exec

The loop is defined in /home/eoseret/qaas_runs_ZEN5/174-049-0948/intel/CloverLeaf1.3-FC/build/CloverLeaf1.3-FC/CloverLeaf_ref/kernels/advec_mom_kernel.f90:215,227-241.

The related source loop is not unrolled or unrolled with no peel/tail loop.
The structure of this loop is probably <if then [else] end>.

The presence of multiple execution paths is typically the main/first bottleneck.
Try to simplify control inside loop: ideally, try to remove all conditional expressions, for example by (if applicable):

hoisting them (moving them outside the loop)
turning them into conditional moves, MIN or MAX

Ex: if (x<0) x=0 => x = max(0,x) (Fortran instrinsic procedure)

gain
potential
hint
expert

Vectorization

Your loop is probably not vectorized. Only 17% of vector register length is used (average across all SSE/AVX instructions). By vectorizing your loop, you can lower the cost of an iteration from 10.19 to 1.32 cycles (7.70x speedup).

Details

Store and arithmetical SSE/AVX instructions are used in scalar version (process only one data element in vector registers). Since your execution units are vector units, only a vectorized loop can use their full power.

Workaround

Try another compiler or update/tune your current one:
- use the vec-report option to understand why your loop was not vectorized. If "existence of vector dependences", try the IVDEP directive. If, using IVDEP, "vectorization possible but seems inefficient", try the VECTOR ALWAYS directive.
Remove inter-iterations dependences from your loop and make it unit-stride:
- If your arrays have 2 or more dimensions, check whether elements are accessed contiguously and, otherwise, try to permute loops accordingly: Fortran storage order is column-major: do i do j a(i,j) = b(i,j) (slow, non stride 1) => do i do j a(j,i) = b(i,j) (fast, stride 1)
- If your loop streams arrays of structures (AoS), try to use structures of arrays instead (SoA): do i a(i)%x = b(i)%x (slow, non stride 1) => do i a%x(i) = b%x(i) (fast, stride 1)

FMA

Presence of both ADD/SUB and MUL operations.

Workaround

Pass to your compiler a micro-architecture specialization option:
- use axHost or xHost
Try to change order in which elements are evaluated (using parentheses) in arithmetic expressions containing both ADD/SUB and MUL operations to enable your compiler to generate FMA instructions wherever possible. For instance a + b*c is a valid FMA (MUL then ADD). However (a+b)* c cannot be translated into an FMA (ADD then MUL).

Complex instructions

Detected COMPLEX INSTRUCTIONS.

Details

These instructions generate more than one micro-operation and only one of them can be decoded during a cycle and the extra micro-operations increase pressure on execution units.

CMOVA: 0 occurrences
UCOMISD: 0 occurrences

Slow data structures access

Detected data structures (typically arrays) that cannot be efficiently read/written

Details

Constant unknown stride: 4 occurrence(s)
Irregular (variable stride) or indirect: 0 occurrence(s)

Non-unit stride (uncontiguous) accesses are not efficiently using data caches

Workaround

Try to reorganize arrays of structures to structures of arrays
Consider to permute loops (see vectorization gain report)
Try to remove indirect accesses. If applicable, precompute elements out of the innermost loop.

Type of elements and instruction set

16 SSE or AVX instructions are processing arithmetic or math operations on double precision FP elements in scalar mode (one at a time).

Matching between your loop (in the source code) and the binary loop

The binary loop is composed of 12.5 FP arithmetical operations:

5.50: addition or subtraction
5: multiply
2: divide

The binary loop is loading 136 bytes (17 double precision FP elements). The binary loop is storing 8 bytes (1 double precision FP elements).

Arithmetic intensity

Arithmetic intensity is 0.09 FP operations per loaded or stored byte.

Unroll opportunity

Loop is potentially data access bound.

Workaround

Unroll your loop if trip count is significantly higher than target unroll factor and if some data references are common to consecutive iterations. This can be done manually. Or with the unroll (resp. unroll_and_jam) directive on top of the inner (resp. surrounding) loop. You can enforce an unroll factor: !DIR$ UNROLL_AND_JAM(N), UNROLL_AND_JAM=N, UNROLL(N) or UNROLL=N

General properties

nb instructions	75.50
nb uops	81.50
loop length	349.50
used x86 registers	15
used mmx registers	0
used xmm registers	12.50
used ymm registers	0
used zmm registers	0
nb stack references	7
ADD-SUB / MUL ratio	1.17

Front-end

MACRO FUSION NOT POSSIBLE

micro-operation queue	10.19 cycles
front end	10.19 cycles

Back-end

	P0	P1	P2	P3	P4	P5	P6	P7	P8	P9	P10	P11	P12	P13	P14	P15
uops	5.08	5.08	5.08	5.08	5.08	5.08	4.50	4.50	4.50	4.50	5.75	5.75	6.00	6.00	1.75	1.75
cycles	5.08	5.08	5.08	5.08	5.08	5.08	4.50	4.50	4.50	4.50	5.75	5.75	6.00	6.00	1.75	1.75

Execution ports to units layout:

P0: ALU
P1: ALU
P2: ALU
P3: ALU, BRU
P4: ALU, BRU
P5: ALU, BRU
P6 (512 bits): store address, load
P7 (512 bits): store address, load
P8 (512 bits): store address, load
P9 (512 bits): store address, load
P10 (512 bits): VPU, DIV/SQRT
P11 (512 bits): VPU, DIV/SQRT
P12 (512 bits): VPU
P13 (512 bits): VPU
P14 (512 bits): FP store data
P15 (512 bits): FP store data

Cycles executing div or sqrt instructions	8.00
Longest recurrence chain latency (RecMII)	0.00

Cycles summary

Front-end	10.19
Dispatch	6.83
DIV/SQRT	8.00
Data deps.	0.00
Overall L1	10.19

Vectorization ratios

INT

all	0%
load	0%
store	NA (no store vectorizable/vectorized instructions)
mul	NA (no mul vectorizable/vectorized instructions)
add-sub	NA (no add-sub vectorizable/vectorized instructions)
fma	NA (no fma vectorizable/vectorized instructions)
other	0%

all	38%
load	0%
store	0%
mul	0%
add-sub	0%
fma	NA (no fma vectorizable/vectorized instructions)
div/sqrt	0%
other	77%

INT+FP

all	36%
load	0%
store	0%
mul	0%
add-sub	0%
fma	NA (no fma vectorizable/vectorized instructions)
div/sqrt	0%
other	72%

Vector efficiency ratios

INT

all	12%
load	12%
store	NA (no store vectorizable/vectorized instructions)
mul	NA (no mul vectorizable/vectorized instructions)
add-sub	NA (no add-sub vectorizable/vectorized instructions)
fma	NA (no fma vectorizable/vectorized instructions)
other	12%

all	17%
load	12%
store	12%
mul	12%
add-sub	12%
fma	NA (no fma vectorizable/vectorized instructions)
div/sqrt	12%
other	22%

INT+FP

all	17%
load	12%
store	12%
mul	12%
add-sub	12%
fma	NA (no fma vectorizable/vectorized instructions)
div/sqrt	12%
other	21%

Cycles and memory resources usage

Assuming all data fit into the L1 cache, each iteration of the binary loop takes 10.19 cycles. At this rate:

5% of peak load performance is reached (13.83 out of 256.00 bytes loaded per cycle (GB/s @ 1GHz))
0% of peak store performance is reached (0.84 out of 128.00 bytes stored per cycle (GB/s @ 1GHz))

Function	advec_mom_kernel_mod_mp_advec_mom_kernel_.DIR.OMP.PARALLEL.2
Source file and lines	advec_mom_kernel.f90:215-241
Module	exec

hoisting them (moving them outside the loop)
turning them into conditional moves, MIN or MAX

Ex: if (x<0) x=0 => x = max(0,x) (Fortran instrinsic procedure)

gain
potential
hint
expert

Code clean check

Detected a slowdown caused by scalar integer instructions (typically used for address computation). By removing them, you can lower the cost of an iteration from 12.75 to 12.00 cycles (1.06x speedup).

Workaround

Try to reorganize arrays of structures to structures of arrays
Consider to permute loops (see vectorization gain report)
To reference allocatable arrays, use "allocatable" instead of "pointer" pointers or qualify them with the "contiguous" attribute (Fortran 2008)
For structures, limit to one indirection. For example, use a_b%c instead of a%b%c with a_b set to a%b before this loop

Vectorization

Your loop is probably not vectorized. Only 18% of vector register length is used (average across all SSE/AVX instructions). By vectorizing your loop, you can lower the cost of an iteration from 12.75 to 1.69 cycles (7.52x speedup).

Details

Workaround

Try another compiler or update/tune your current one:
- use the vec-report option to understand why your loop was not vectorized. If "existence of vector dependences", try the IVDEP directive. If, using IVDEP, "vectorization possible but seems inefficient", try the VECTOR ALWAYS directive.
Remove inter-iterations dependences from your loop and make it unit-stride:
- If your arrays have 2 or more dimensions, check whether elements are accessed contiguously and, otherwise, try to permute loops accordingly: Fortran storage order is column-major: do i do j a(i,j) = b(i,j) (slow, non stride 1) => do i do j a(j,i) = b(i,j) (fast, stride 1)
- If your loop streams arrays of structures (AoS), try to use structures of arrays instead (SoA): do i a(i)%x = b(i)%x (slow, non stride 1) => do i a%x(i) = b%x(i) (fast, stride 1)

Execution units bottlenecks

Found no such bottlenecks but see expert reports for more complex bottlenecks.

FMA

Presence of both ADD/SUB and MUL operations.

Workaround

Pass to your compiler a micro-architecture specialization option:
- use axHost or xHost
Try to change order in which elements are evaluated (using parentheses) in arithmetic expressions containing both ADD/SUB and MUL operations to enable your compiler to generate FMA instructions wherever possible. For instance a + b*c is a valid FMA (MUL then ADD). However (a+b)* c cannot be translated into an FMA (ADD then MUL).

Complex instructions

Detected COMPLEX INSTRUCTIONS.

Details

These instructions generate more than one micro-operation and only one of them can be decoded during a cycle and the extra micro-operations increase pressure on execution units.

CMOVA: 4 occurrences►
- /home/eoseret/qaas_runs_ZEN5/174-049-0948/intel/CloverLeaf1.3-FC/build/CloverLeaf1.3-FC/CloverLeaf_ref/kernels/advec_mom_kernel.f90:215
- /home/eoseret/qaas_runs_ZEN5/174-049-0948/intel/CloverLeaf1.3-FC/build/CloverLeaf1.3-FC/CloverLeaf_ref/kernels/advec_mom_kernel.f90:215
- /home/eoseret/qaas_runs_ZEN5/174-049-0948/intel/CloverLeaf1.3-FC/build/CloverLeaf1.3-FC/CloverLeaf_ref/kernels/advec_mom_kernel.f90:215
- /home/eoseret/qaas_runs_ZEN5/174-049-0948/intel/CloverLeaf1.3-FC/build/CloverLeaf1.3-FC/CloverLeaf_ref/kernels/advec_mom_kernel.f90:215
UCOMISD: 3 occurrences►
- /home/eoseret/qaas_runs_ZEN5/174-049-0948/intel/CloverLeaf1.3-FC/build/CloverLeaf1.3-FC/CloverLeaf_ref/kernels/advec_mom_kernel.f90:215
- /home/eoseret/qaas_runs_ZEN5/174-049-0948/intel/CloverLeaf1.3-FC/build/CloverLeaf1.3-FC/CloverLeaf_ref/kernels/advec_mom_kernel.f90:232
- /home/eoseret/qaas_runs_ZEN5/174-049-0948/intel/CloverLeaf1.3-FC/build/CloverLeaf1.3-FC/CloverLeaf_ref/kernels/advec_mom_kernel.f90:215

Slow data structures access

Detected data structures (typically arrays) that cannot be efficiently read/written

Details

Constant unknown stride: 4 occurrence(s)
Irregular (variable stride) or indirect: 1 occurrence(s)

Non-unit stride (uncontiguous) accesses are not efficiently using data caches

Workaround

Try to reorganize arrays of structures to structures of arrays
Consider to permute loops (see vectorization gain report)
Try to remove indirect accesses. If applicable, precompute elements out of the innermost loop.

Type of elements and instruction set

23 SSE or AVX instructions are processing arithmetic or math operations on double precision FP elements in scalar mode (one at a time).

Matching between your loop (in the source code) and the binary loop

The binary loop is composed of 17 FP arithmetical operations:

7: addition or subtraction
7: multiply
3: divide

The binary loop is loading 152 bytes (19 double precision FP elements). The binary loop is storing 8 bytes (1 double precision FP elements).

Arithmetic intensity

Arithmetic intensity is 0.11 FP operations per loaded or stored byte.

General properties

nb instructions	95
nb uops	102
loop length	446
used x86 registers	15
used mmx registers	0
used xmm registers	16
used ymm registers	0
used zmm registers	0
nb stack references	8
ADD-SUB / MUL ratio	1.00

Front-end

MACRO FUSION NOT POSSIBLE FIT IN UOP CACHE

micro-operation queue	12.75 cycles
front end	12.75 cycles

Back-end

	P0	P1	P2	P3	P4	P5	P6	P7	P8	P9	P10	P11	P12	P13	P14	P15
uops	5.50	5.50	5.50	5.50	5.50	5.50	5.00	5.00	5.00	5.00	9.00	9.00	9.00	9.00	2.00	2.00
cycles	5.50	5.50	5.50	5.50	5.50	5.50	5.00	5.00	5.00	5.00	9.00	9.00	9.00	9.00	2.00	2.00

Execution ports to units layout:

P0: ALU
P1: ALU
P2: ALU
P3: ALU, BRU
P4: ALU, BRU
P5: ALU, BRU
P6 (512 bits): store address, load
P7 (512 bits): store address, load
P8 (512 bits): store address, load
P9 (512 bits): store address, load
P10 (512 bits): VPU, DIV/SQRT
P11 (512 bits): VPU, DIV/SQRT
P12 (512 bits): VPU
P13 (512 bits): VPU
P14 (512 bits): FP store data
P15 (512 bits): FP store data

Cycles executing div or sqrt instructions	12.00
Longest recurrence chain latency (RecMII)	0.00

Cycles summary

Front-end	12.75
Dispatch	9.00
DIV/SQRT	12.00
Data deps.	0.00
Overall L1	12.75

Vectorization ratios

INT

all	0%
load	0%
store	NA (no store vectorizable/vectorized instructions)
mul	NA (no mul vectorizable/vectorized instructions)
add-sub	NA (no add-sub vectorizable/vectorized instructions)
fma	NA (no fma vectorizable/vectorized instructions)
other	0%

all	46%
load	0%
store	0%
mul	0%
add-sub	0%
fma	NA (no fma vectorizable/vectorized instructions)
div/sqrt	0%
other	80%

INT+FP

all	45%
load	0%
store	0%
mul	0%
add-sub	0%
fma	NA (no fma vectorizable/vectorized instructions)
div/sqrt	0%
other	77%

Vector efficiency ratios

INT

all	12%
load	12%
store	NA (no store vectorizable/vectorized instructions)
mul	NA (no mul vectorizable/vectorized instructions)
add-sub	NA (no add-sub vectorizable/vectorized instructions)
fma	NA (no fma vectorizable/vectorized instructions)
other	12%

all	18%
load	12%
store	12%
mul	12%
add-sub	12%
fma	NA (no fma vectorizable/vectorized instructions)
div/sqrt	12%
other	22%

INT+FP

all	18%
load	12%
store	12%
mul	12%
add-sub	12%
fma	NA (no fma vectorizable/vectorized instructions)
div/sqrt	12%
other	22%

Cycles and memory resources usage

Assuming all data fit into the L1 cache, each iteration of the binary loop takes 12.75 cycles. At this rate:

4% of peak load performance is reached (11.92 out of 256.00 bytes loaded per cycle (GB/s @ 1GHz))
0% of peak store performance is reached (0.63 out of 128.00 bytes stored per cycle (GB/s @ 1GHz))

Front-end bottlenecks

Performance is limited by instruction throughput (loading/decoding program instructions to execution core) (front-end is a bottleneck).

ASM code

In the binary file, the address of the loop is: 424c7f

Instruction	Nb FU	P0	P1	P2	P3	P4	P5	P6	P7	P8	P9	P10	P11	P12	P13	P14	P15	Latency	Recip. throughput	Vectorization
MOVAPD %XMM4,%XMM11	1	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0.17	vect (25.0%)
SUBSD %XMM8,%XMM11	1	0	0	0	0	0	0	0	0	0	0	0	0	0.50	0.50	0	0	2	0.50	scal (12.5%)
MULSD %XMM10,%XMM11	1	0	0	0	0	0	0	0	0	0	0	0.50	0.50	0	0	0	0	3	0.50	scal (12.5%)
ADDSD %XMM9,%XMM11	1	0	0	0	0	0	0	0	0	0	0	0	0	0.50	0.50	0	0	2	0.50	scal (12.5%)
MULSD %XMM7,%XMM11	1	0	0	0	0	0	0	0	0	0	0	0.50	0.50	0	0	0	0	3	0.50	scal (12.5%)
MOV -0xe8(%RBP),%R13	1	0	0	0	0	0	0	0.25	0.25	0.25	0.25	0	0	0	0	0	0	3	0.25	N/A
MOV (%R13),%RCX	1	0	0	0	0	0	0	0.25	0.25	0.25	0.25	0	0	0	0	0	0	3	0.25	N/A
IMUL %R8,%RCX	1	0.33	0.33	0.33	0	0	0	0	0	0	0	0	0	0	0	0	0	3	0.33	N/A
ADD %R10,%RCX	1	0.17	0.17	0.17	0.17	0.17	0.17	0	0	0	0	0	0	0	0	0	0	1	0.17	N/A
MOVSD %XMM11,0x10(%RCX,%RAX,8)	1	0	0	0	0	0	0	0.25	0.25	0.25	0.25	0	0	0	0	0.50	0.50	1	0.50	scal (12.5%)
INC %RAX	1	0.17	0.17	0.17	0.17	0.17	0.17	0	0	0	0	0	0	0	0	0	0	1	0.17	N/A
CMP %RAX,-0x30(%RBP)	1	0.17	0.17	0.17	0.17	0.17	0.17	0.25	0.25	0.25	0.25	0	0	0	0	0	0	1	0.25	scal (12.5%)
JE 424bf0 <advec_mom_kernel_mod_mp_advec_mom_kernel_.DIR.OMP.PARALLEL.2+0x1820>	1	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	0	0	0	0	1	0.50-1	N/A
MOV -0x120(%RBP),%RCX	1	0	0	0	0	0	0	0.25	0.25	0.25	0.25	0	0	0	0	0	0	3	0.25	N/A
MOV (%RCX),%R13	1	0	0	0	0	0	0	0.25	0.25	0.25	0.25	0	0	0	0	0	0	3	0.25	N/A
IMUL %R8,%R13	1	0.33	0.33	0.33	0	0	0	0	0	0	0	0	0	0	0	0	0	3	0.33	N/A
ADD -0x68(%RBP),%R13	1	0.17	0.17	0.17	0.17	0.17	0.17	0.25	0.25	0.25	0.25	0	0	0	0	0	0	1	0.25	N/A
MOVSD 0x10(%R13,%RAX,8),%XMM7	1	0	0	0	0	0	0	0.25	0.25	0.25	0.25	0	0	0	0	0	0	1	0.50	scal (12.5%)
XORPD %XMM10,%XMM10	1	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0.17	vect (25.0%)
UCOMISD %XMM7,%XMM10	2	0	0	0	0	0	0	0	0	0	0	0	0	0.50	0.50	0.50	0.50	7	1	scal (12.5%)
MOV %R15D,%R14D	1	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0.17	N/A
CMOVA %R9D,%R14D	2	0.33	0.33	0.33	0.33	0.33	0.33	0	0	0	0	0	0	0	0	0	0	1	0.33	N/A
MOV %R12D,%ECX	1	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0.17	N/A
CMOVA %EDX,%ECX	2	0.33	0.33	0.33	0.33	0.33	0.33	0	0	0	0	0	0	0	0	0	0	1	0.33	N/A
MOV %R9D,%R13D	1	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0.17	N/A
CMOVA %R15D,%R13D	2	0.33	0.33	0.33	0.33	0.33	0.33	0	0	0	0	0	0	0	0	0	0	1	0.33	N/A
MOVAPD %XMM7,%XMM8	1	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0.17	vect (25.0%)
ANDPD %XMM0,%XMM8	1	0	0	0	0	0	0	0	0	0	0	0.25	0.25	0.25	0.25	0	0	1	0.25	vect (25.0%)
MOVSXD %R14D,%R14	1	0.17	0.17	0.17	0.17	0.17	0.17	0	0	0	0	0	0	0	0	0	0	1	0.17	N/A
SUB %RDI,%R14	1	0.17	0.17	0.17	0.17	0.17	0.17	0	0	0	0	0	0	0	0	0	0	1	0.17	N/A
MOV -0x150(%RBP),%RSI	1	0	0	0	0	0	0	0.25	0.25	0.25	0.25	0	0	0	0	0	0	3	0.25	N/A
MOV (%RSI),%RSI	1	0	0	0	0	0	0	0.25	0.25	0.25	0.25	0	0	0	0	0	0	3	0.25	N/A
IMUL %R14,%RSI	1	0.33	0.33	0.33	0	0	0	0	0	0	0	0	0	0	0	0	0	3	0.33	N/A
ADD %RBX,%RSI	1	0.17	0.17	0.17	0.17	0.17	0.17	0	0	0	0	0	0	0	0	0	0	1	0.17	N/A
DIVSD 0x10(%RSI,%RAX,8),%XMM8	1	0	0	0	0	0	0	0.25	0.25	0.25	0.25	0.50	0.50	0	0	0	0	13	4	scal (12.5%)
MOV -0x1c8(%RBP),%RSI	1	0	0	0	0	0	0	0.25	0.25	0.25	0.25	0	0	0	0	0	0	3	0.25	N/A
MOV (%RSI),%RSI	1	0	0	0	0	0	0	0.25	0.25	0.25	0.25	0	0	0	0	0	0	3	0.25	N/A
IMUL %RSI,%R14	1	0.33	0.33	0.33	0	0	0	0	0	0	0	0	0	0	0	0	0	3	0.33	N/A
ADD %R11,%R14	1	0.17	0.17	0.17	0.17	0.17	0.17	0	0	0	0	0	0	0	0	0	0	1	0.17	N/A
MOVSD 0x10(%R14,%RAX,8),%XMM9	1	0	0	0	0	0	0	0.25	0.25	0.25	0.25	0	0	0	0	0	0	1	0.50	scal (12.5%)
MOVSXD %ECX,%RCX	1	0.17	0.17	0.17	0.17	0.17	0.17	0	0	0	0	0	0	0	0	0	0	1	0.17	N/A
SUB %RDI,%RCX	1	0.17	0.17	0.17	0.17	0.17	0.17	0	0	0	0	0	0	0	0	0	0	1	0.17	N/A
IMUL %RSI,%RCX	1	0.33	0.33	0.33	0	0	0	0	0	0	0	0	0	0	0	0	0	3	0.33	N/A
ADD %R11,%RCX	1	0.17	0.17	0.17	0.17	0.17	0.17	0	0	0	0	0	0	0	0	0	0	1	0.17	N/A
MOVAPD %XMM9,%XMM12	1	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0.17	vect (25.0%)
SUBSD 0x10(%RCX,%RAX,8),%XMM12	1	0	0	0	0	0	0	0.25	0.25	0.25	0.25	0	0	0.50	0.50	0	0	2	0.50	scal (12.5%)
MOVSXD %R13D,%RCX	1	0.17	0.17	0.17	0.17	0.17	0.17	0	0	0	0	0	0	0	0	0	0	1	0.17	N/A
SUB %RDI,%RCX	1	0.17	0.17	0.17	0.17	0.17	0.17	0	0	0	0	0	0	0	0	0	0	1	0.17	N/A
IMUL %RSI,%RCX	1	0.33	0.33	0.33	0	0	0	0	0	0	0	0	0	0	0	0	0	3	0.33	N/A
ADD %R11,%RCX	1	0.17	0.17	0.17	0.17	0.17	0.17	0	0	0	0	0	0	0	0	0	0	1	0.17	N/A
MOVSD 0x10(%RCX,%RAX,8),%XMM11	1	0	0	0	0	0	0	0.25	0.25	0.25	0.25	0	0	0	0	0	0	1	0.50	scal (12.5%)
SUBSD %XMM9,%XMM11	1	0	0	0	0	0	0	0	0	0	0	0	0	0.50	0.50	0	0	2	0.50	scal (12.5%)
MOVAPD %XMM11,%XMM13	1	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0.17	vect (25.0%)
MULSD %XMM12,%XMM13	1	0	0	0	0	0	0	0	0	0	0	0.50	0.50	0	0	0	0	3	0.50	scal (12.5%)
UCOMISD %XMM10,%XMM13	2	0	0	0	0	0	0	0	0	0	0	0	0	0.50	0.50	0.50	0.50	7	1	scal (12.5%)
JBE 424c40 <advec_mom_kernel_mod_mp_advec_mom_kernel_.DIR.OMP.PARALLEL.2+0x1870>	1	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	0	0	0	0	1	0.50-1	N/A
UCOMISD %XMM7,%XMM1	2	0	0	0	0	0	0	0	0	0	0	0	0	0.50	0.50	0.50	0.50	7	1	scal (12.5%)
MOV %R12D,%ECX	1	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0.17	N/A
CMOVA %R9D,%ECX	2	0.33	0.33	0.33	0.33	0.33	0.33	0	0	0	0	0	0	0	0	0	0	1	0.33	N/A
MOV -0x100(%RBP),%RSI	1	0	0	0	0	0	0	0.25	0.25	0.25	0.25	0	0	0	0	0	0	3	0.25	N/A
MOV -0x38(%RBP),%R14	1	0	0	0	0	0	0	0.25	0.25	0.25	0.25	0	0	0	0	0	0	3	0.25	N/A
MOVSD (%RSI,%R14,8),%XMM13	1	0	0	0	0	0	0	0.25	0.25	0.25	0.25	0	0	0	0	0	0	1	0.50	scal (12.5%)
ANDPD %XMM2,%XMM12	1	0	0	0	0	0	0	0	0	0	0	0.25	0.25	0.25	0.25	0	0	1	0.25	vect (25.0%)
MOVAPD %XMM11,%XMM10	1	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0.17	vect (25.0%)
ANDPD %XMM2,%XMM10	1	0	0	0	0	0	0	0	0	0	0	0.25	0.25	0.25	0.25	0	0	1	0.25	vect (25.0%)
MOVAPD %XMM3,%XMM14	1	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0.17	vect (25.0%)
SUBSD %XMM8,%XMM14	1	0	0	0	0	0	0	0	0	0	0	0	0	0.50	0.50	0	0	2	0.50	scal (12.5%)
MULSD %XMM10,%XMM14	1	0	0	0	0	0	0	0	0	0	0	0.50	0.50	0	0	0	0	3	0.50	scal (12.5%)
MOVAPD %XMM8,%XMM15	1	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0.17	vect (25.0%)
ADDSD %XMM4,%XMM15	1	0	0	0	0	0	0	0	0	0	0	0	0	0.50	0.50	0	0	2	0.50	scal (12.5%)
MULSD %XMM12,%XMM15	1	0	0	0	0	0	0	0	0	0	0	0.50	0.50	0	0	0	0	3	0.50	scal (12.5%)
MOVSXD %ECX,%RCX	1	0.17	0.17	0.17	0.17	0.17	0.17	0	0	0	0	0	0	0	0	0	0	1	0.17	N/A
SUB %RDI,%RCX	1	0.17	0.17	0.17	0.17	0.17	0.17	0	0	0	0	0	0	0	0	0	0	1	0.17	N/A
DIVSD (%RSI,%RCX,8),%XMM15	1	0	0	0	0	0	0	0.25	0.25	0.25	0.25	0.50	0.50	0	0	0	0	13	4	scal (12.5%)
DIVSD %XMM13,%XMM14	1	0	0	0	0	0	0	0	0	0	0	0.50	0.50	0	0	0	0	13	4	scal (12.5%)
ADDSD %XMM14,%XMM15	1	0	0	0	0	0	0	0	0	0	0	0	0	0.50	0.50	0	0	2	0.50	scal (12.5%)
MULSD %XMM5,%XMM13	1	0	0	0	0	0	0	0	0	0	0	0.50	0.50	0	0	0	0	3	0.50	scal (12.5%)
MULSD %XMM15,%XMM13	1	0	0	0	0	0	0	0	0	0	0	0.50	0.50	0	0	0	0	3	0.50	scal (12.5%)
MOVAPD %XMM12,%XMM14	1	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0.17	vect (25.0%)
CMPSD $0x2,%XMM10,%XMM14	1	0	0	0	0	0	0	0	0	0	0	0	0	0.50	0.50	0	0	2	0.50	scal (12.5%)
ANDPD %XMM14,%XMM12	1	0	0	0	0	0	0	0	0	0	0	0.25	0.25	0.25	0.25	0	0	1	0.25	vect (25.0%)
ANDNPD %XMM10,%XMM14	1	0	0	0	0	0	0	0	0	0	0	0.25	0.25	0.25	0.25	0	0	1	0.25	vect (25.0%)
ORPD %XMM12,%XMM14	1	0	0	0	0	0	0	0	0	0	0	0.25	0.25	0.25	0.25	0	0	1	0.25	vect (25.0%)
MOVAPD %XMM13,%XMM10	1	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0.17	vect (25.0%)
CMPSD $0x2,%XMM14,%XMM10	1	0	0	0	0	0	0	0	0	0	0	0	0	0.50	0.50	0	0	2	0.50	scal (12.5%)
ANDPD %XMM10,%XMM13	1	0	0	0	0	0	0	0	0	0	0	0.25	0.25	0.25	0.25	0	0	1	0.25	vect (25.0%)
ANDNPD %XMM14,%XMM10	1	0	0	0	0	0	0	0	0	0	0	0.25	0.25	0.25	0.25	0	0	1	0.25	vect (25.0%)
ORPD %XMM13,%XMM10	1	0	0	0	0	0	0	0	0	0	0	0.25	0.25	0.25	0.25	0	0	1	0.25	vect (25.0%)
CMPSD $0x2,%XMM1,%XMM11	1	0	0	0	0	0	0	0	0	0	0	0	0	0.50	0.50	0	0	2	0.50	scal (12.5%)
MOVAPD %XMM11,%XMM12	1	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0.17	vect (25.0%)
ANDNPD %XMM10,%XMM12	1	0	0	0	0	0	0	0	0	0	0	0.25	0.25	0.25	0.25	0	0	1	0.25	vect (25.0%)
XORPD %XMM6,%XMM10	1	0	0	0	0	0	0	0	0	0	0	0.25	0.25	0.25	0.25	0	0	1	0.25	vect (25.0%)
ANDPD %XMM11,%XMM10	1	0	0	0	0	0	0	0	0	0	0	0.25	0.25	0.25	0.25	0	0	1	0.25	vect (25.0%)
ORPD %XMM12,%XMM10	1	0	0	0	0	0	0	0	0	0	0	0.25	0.25	0.25	0.25	0	0	1	0.25	vect (25.0%)
JMP 424c40 <advec_mom_kernel_mod_mp_advec_mom_kernel_.DIR.OMP.PARALLEL.2+0x1870>	1	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	0	0	0	0	0	1	N/A

Function	advec_mom_kernel_mod_mp_advec_mom_kernel_.DIR.OMP.PARALLEL.2
Source file and lines	advec_mom_kernel.f90:215-241
Module	exec

hoisting them (moving them outside the loop)
turning them into conditional moves, MIN or MAX

Ex: if (x<0) x=0 => x = max(0,x) (Fortran instrinsic procedure)

gain
potential
hint
expert

Code clean check

Detected a slowdown caused by scalar integer instructions (typically used for address computation). By removing them, you can lower the cost of an iteration from 7.63 to 4.63 cycles (1.65x speedup).

Workaround

Try to reorganize arrays of structures to structures of arrays
Consider to permute loops (see vectorization gain report)
To reference allocatable arrays, use "allocatable" instead of "pointer" pointers or qualify them with the "contiguous" attribute (Fortran 2008)
For structures, limit to one indirection. For example, use a_b%c instead of a%b%c with a_b set to a%b before this loop

Vectorization

Your loop is probably not vectorized. Only 16% of vector register length is used (average across all SSE/AVX instructions). By vectorizing your loop, you can lower the cost of an iteration from 7.63 to 0.95 cycles (8.00x speedup).

Details

Workaround

Try another compiler or update/tune your current one:
- use the vec-report option to understand why your loop was not vectorized. If "existence of vector dependences", try the IVDEP directive. If, using IVDEP, "vectorization possible but seems inefficient", try the VECTOR ALWAYS directive.
Remove inter-iterations dependences from your loop and make it unit-stride:
- If your arrays have 2 or more dimensions, check whether elements are accessed contiguously and, otherwise, try to permute loops accordingly: Fortran storage order is column-major: do i do j a(i,j) = b(i,j) (slow, non stride 1) => do i do j a(j,i) = b(i,j) (fast, stride 1)
- If your loop streams arrays of structures (AoS), try to use structures of arrays instead (SoA): do i a(i)%x = b(i)%x (slow, non stride 1) => do i a%x(i) = b%x(i) (fast, stride 1)

Execution units bottlenecks

Found no such bottlenecks but see expert reports for more complex bottlenecks.

FMA

Presence of both ADD/SUB and MUL operations.

Workaround

Pass to your compiler a micro-architecture specialization option:
- use axHost or xHost
Try to change order in which elements are evaluated (using parentheses) in arithmetic expressions containing both ADD/SUB and MUL operations to enable your compiler to generate FMA instructions wherever possible. For instance a + b*c is a valid FMA (MUL then ADD). However (a+b)* c cannot be translated into an FMA (ADD then MUL).

Complex instructions

Detected COMPLEX INSTRUCTIONS.

Details

These instructions generate more than one micro-operation and only one of them can be decoded during a cycle and the extra micro-operations increase pressure on execution units.

CMOVA: 3 occurrences►
- /home/eoseret/qaas_runs_ZEN5/174-049-0948/intel/CloverLeaf1.3-FC/build/CloverLeaf1.3-FC/CloverLeaf_ref/kernels/advec_mom_kernel.f90:215
- /home/eoseret/qaas_runs_ZEN5/174-049-0948/intel/CloverLeaf1.3-FC/build/CloverLeaf1.3-FC/CloverLeaf_ref/kernels/advec_mom_kernel.f90:215
- /home/eoseret/qaas_runs_ZEN5/174-049-0948/intel/CloverLeaf1.3-FC/build/CloverLeaf1.3-FC/CloverLeaf_ref/kernels/advec_mom_kernel.f90:215
UCOMISD: 2 occurrences►
- /home/eoseret/qaas_runs_ZEN5/174-049-0948/intel/CloverLeaf1.3-FC/build/CloverLeaf1.3-FC/CloverLeaf_ref/kernels/advec_mom_kernel.f90:215
- /home/eoseret/qaas_runs_ZEN5/174-049-0948/intel/CloverLeaf1.3-FC/build/CloverLeaf1.3-FC/CloverLeaf_ref/kernels/advec_mom_kernel.f90:232

Slow data structures access

Detected data structures (typically arrays) that cannot be efficiently read/written

Details

Constant unknown stride: 4 occurrence(s)

Non-unit stride (uncontiguous) accesses are not efficiently using data caches

Workaround

Try to reorganize arrays of structures to structures of arrays
Consider to permute loops (see vectorization gain report)

Type of elements and instruction set

10 SSE or AVX instructions are processing arithmetic or math operations on double precision FP elements in scalar mode (one at a time).

Matching between your loop (in the source code) and the binary loop

The binary loop is composed of 8 FP arithmetical operations:

4: addition or subtraction
3: multiply
1: divide

The binary loop is loading 120 bytes (15 double precision FP elements). The binary loop is storing 8 bytes (1 double precision FP elements).

Arithmetic intensity

Arithmetic intensity is 0.06 FP operations per loaded or stored byte.

Unroll opportunity

Loop is potentially data access bound.

Workaround

General properties

nb instructions	56
nb uops	61
loop length	253
used x86 registers	15
used mmx registers	0
used xmm registers	9
used ymm registers	0
used zmm registers	0
nb stack references	6
ADD-SUB / MUL ratio	1.33

Front-end

MACRO FUSION NOT POSSIBLE FIT IN UOP CACHE

micro-operation queue	7.63 cycles
front end	7.63 cycles

Back-end

	P0	P1	P2	P3	P4	P5	P6	P7	P8	P9	P10	P11	P12	P13	P14	P15
uops	4.67	4.67	4.67	4.67	4.67	4.67	4.00	4.00	4.00	4.00	2.50	2.50	3.00	3.00	1.50	1.50
cycles	4.67	4.67	4.67	4.67	4.67	4.67	4.00	4.00	4.00	4.00	2.50	2.50	3.00	3.00	1.50	1.50

Execution ports to units layout:

P0: ALU
P1: ALU
P2: ALU
P3: ALU, BRU
P4: ALU, BRU
P5: ALU, BRU
P6 (512 bits): store address, load
P7 (512 bits): store address, load
P8 (512 bits): store address, load
P9 (512 bits): store address, load
P10 (512 bits): VPU, DIV/SQRT
P11 (512 bits): VPU, DIV/SQRT
P12 (512 bits): VPU
P13 (512 bits): VPU
P14 (512 bits): FP store data
P15 (512 bits): FP store data

Cycles executing div or sqrt instructions	4.00
Longest recurrence chain latency (RecMII)	0.00

Cycles summary

Front-end	7.63
Dispatch	4.67
DIV/SQRT	4.00
Data deps.	0.00
Overall L1	7.63

Vectorization ratios

INT

all	0%
load	0%
store	NA (no store vectorizable/vectorized instructions)
mul	NA (no mul vectorizable/vectorized instructions)
add-sub	NA (no add-sub vectorizable/vectorized instructions)
fma	NA (no fma vectorizable/vectorized instructions)
other	0%

all	30%
load	0%
store	0%
mul	0%
add-sub	0%
fma	NA (no fma vectorizable/vectorized instructions)
div/sqrt	0%
other	75%

INT+FP

all	28%
load	0%
store	0%
mul	0%
add-sub	0%
fma	NA (no fma vectorizable/vectorized instructions)
div/sqrt	0%
other	66%

Vector efficiency ratios

INT

all	12%
load	12%
store	NA (no store vectorizable/vectorized instructions)
mul	NA (no mul vectorizable/vectorized instructions)
add-sub	NA (no add-sub vectorizable/vectorized instructions)
fma	NA (no fma vectorizable/vectorized instructions)
other	12%

all	16%
load	12%
store	12%
mul	12%
add-sub	12%
fma	NA (no fma vectorizable/vectorized instructions)
div/sqrt	12%
other	21%

INT+FP

all	16%
load	12%
store	12%
mul	12%
add-sub	12%
fma	NA (no fma vectorizable/vectorized instructions)
div/sqrt	12%
other	20%

Cycles and memory resources usage

Assuming all data fit into the L1 cache, each iteration of the binary loop takes 7.63 cycles. At this rate:

6% of peak load performance is reached (15.74 out of 256.00 bytes loaded per cycle (GB/s @ 1GHz))
0% of peak store performance is reached (1.05 out of 128.00 bytes stored per cycle (GB/s @ 1GHz))

Front-end bottlenecks

Performance is limited by instruction throughput (loading/decoding program instructions to execution core) (front-end is a bottleneck). By removing all these bottlenecks, you can lower the cost of an iteration from 7.63 to 4.67 cycles (1.63x speedup).

ASM code

In the binary file, the address of the loop is: 424c7f

Instruction	Nb FU	P0	P1	P2	P3	P4	P5	P6	P7	P8	P9	P10	P11	P12	P13	P14	P15	Latency	Recip. throughput	Vectorization
MOVAPD %XMM4,%XMM11	1	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0.17	vect (25.0%)
SUBSD %XMM8,%XMM11	1	0	0	0	0	0	0	0	0	0	0	0	0	0.50	0.50	0	0	2	0.50	scal (12.5%)
MULSD %XMM10,%XMM11	1	0	0	0	0	0	0	0	0	0	0	0.50	0.50	0	0	0	0	3	0.50	scal (12.5%)
ADDSD %XMM9,%XMM11	1	0	0	0	0	0	0	0	0	0	0	0	0	0.50	0.50	0	0	2	0.50	scal (12.5%)
MULSD %XMM7,%XMM11	1	0	0	0	0	0	0	0	0	0	0	0.50	0.50	0	0	0	0	3	0.50	scal (12.5%)
MOV -0xe8(%RBP),%R13	1	0	0	0	0	0	0	0.25	0.25	0.25	0.25	0	0	0	0	0	0	3	0.25	N/A
MOV (%R13),%RCX	1	0	0	0	0	0	0	0.25	0.25	0.25	0.25	0	0	0	0	0	0	3	0.25	N/A
IMUL %R8,%RCX	1	0.33	0.33	0.33	0	0	0	0	0	0	0	0	0	0	0	0	0	3	0.33	N/A
ADD %R10,%RCX	1	0.17	0.17	0.17	0.17	0.17	0.17	0	0	0	0	0	0	0	0	0	0	1	0.17	N/A
MOVSD %XMM11,0x10(%RCX,%RAX,8)	1	0	0	0	0	0	0	0.25	0.25	0.25	0.25	0	0	0	0	0.50	0.50	1	0.50	scal (12.5%)
INC %RAX	1	0.17	0.17	0.17	0.17	0.17	0.17	0	0	0	0	0	0	0	0	0	0	1	0.17	N/A
CMP %RAX,-0x30(%RBP)	1	0.17	0.17	0.17	0.17	0.17	0.17	0.25	0.25	0.25	0.25	0	0	0	0	0	0	1	0.25	scal (12.5%)
JE 424bf0 <advec_mom_kernel_mod_mp_advec_mom_kernel_.DIR.OMP.PARALLEL.2+0x1820>	1	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	0	0	0	0	1	0.50-1	N/A
MOV -0x120(%RBP),%RCX	1	0	0	0	0	0	0	0.25	0.25	0.25	0.25	0	0	0	0	0	0	3	0.25	N/A
MOV (%RCX),%R13	1	0	0	0	0	0	0	0.25	0.25	0.25	0.25	0	0	0	0	0	0	3	0.25	N/A
IMUL %R8,%R13	1	0.33	0.33	0.33	0	0	0	0	0	0	0	0	0	0	0	0	0	3	0.33	N/A
ADD -0x68(%RBP),%R13	1	0.17	0.17	0.17	0.17	0.17	0.17	0.25	0.25	0.25	0.25	0	0	0	0	0	0	1	0.25	N/A
MOVSD 0x10(%R13,%RAX,8),%XMM7	1	0	0	0	0	0	0	0.25	0.25	0.25	0.25	0	0	0	0	0	0	1	0.50	scal (12.5%)
XORPD %XMM10,%XMM10	1	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0.17	vect (25.0%)
UCOMISD %XMM7,%XMM10	2	0	0	0	0	0	0	0	0	0	0	0	0	0.50	0.50	0.50	0.50	7	1	scal (12.5%)
MOV %R15D,%R14D	1	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0.17	N/A
CMOVA %R9D,%R14D	2	0.33	0.33	0.33	0.33	0.33	0.33	0	0	0	0	0	0	0	0	0	0	1	0.33	N/A
MOV %R12D,%ECX	1	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0.17	N/A
CMOVA %EDX,%ECX	2	0.33	0.33	0.33	0.33	0.33	0.33	0	0	0	0	0	0	0	0	0	0	1	0.33	N/A
MOV %R9D,%R13D	1	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0.17	N/A
CMOVA %R15D,%R13D	2	0.33	0.33	0.33	0.33	0.33	0.33	0	0	0	0	0	0	0	0	0	0	1	0.33	N/A
MOVAPD %XMM7,%XMM8	1	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0.17	vect (25.0%)
ANDPD %XMM0,%XMM8	1	0	0	0	0	0	0	0	0	0	0	0.25	0.25	0.25	0.25	0	0	1	0.25	vect (25.0%)
MOVSXD %R14D,%R14	1	0.17	0.17	0.17	0.17	0.17	0.17	0	0	0	0	0	0	0	0	0	0	1	0.17	N/A
SUB %RDI,%R14	1	0.17	0.17	0.17	0.17	0.17	0.17	0	0	0	0	0	0	0	0	0	0	1	0.17	N/A
MOV -0x150(%RBP),%RSI	1	0	0	0	0	0	0	0.25	0.25	0.25	0.25	0	0	0	0	0	0	3	0.25	N/A
MOV (%RSI),%RSI	1	0	0	0	0	0	0	0.25	0.25	0.25	0.25	0	0	0	0	0	0	3	0.25	N/A
IMUL %R14,%RSI	1	0.33	0.33	0.33	0	0	0	0	0	0	0	0	0	0	0	0	0	3	0.33	N/A
ADD %RBX,%RSI	1	0.17	0.17	0.17	0.17	0.17	0.17	0	0	0	0	0	0	0	0	0	0	1	0.17	N/A
DIVSD 0x10(%RSI,%RAX,8),%XMM8	1	0	0	0	0	0	0	0.25	0.25	0.25	0.25	0.50	0.50	0	0	0	0	13	4	scal (12.5%)
MOV -0x1c8(%RBP),%RSI	1	0	0	0	0	0	0	0.25	0.25	0.25	0.25	0	0	0	0	0	0	3	0.25	N/A
MOV (%RSI),%RSI	1	0	0	0	0	0	0	0.25	0.25	0.25	0.25	0	0	0	0	0	0	3	0.25	N/A
IMUL %RSI,%R14	1	0.33	0.33	0.33	0	0	0	0	0	0	0	0	0	0	0	0	0	3	0.33	N/A
ADD %R11,%R14	1	0.17	0.17	0.17	0.17	0.17	0.17	0	0	0	0	0	0	0	0	0	0	1	0.17	N/A
MOVSD 0x10(%R14,%RAX,8),%XMM9	1	0	0	0	0	0	0	0.25	0.25	0.25	0.25	0	0	0	0	0	0	1	0.50	scal (12.5%)
MOVSXD %ECX,%RCX	1	0.17	0.17	0.17	0.17	0.17	0.17	0	0	0	0	0	0	0	0	0	0	1	0.17	N/A
SUB %RDI,%RCX	1	0.17	0.17	0.17	0.17	0.17	0.17	0	0	0	0	0	0	0	0	0	0	1	0.17	N/A
IMUL %RSI,%RCX	1	0.33	0.33	0.33	0	0	0	0	0	0	0	0	0	0	0	0	0	3	0.33	N/A
ADD %R11,%RCX	1	0.17	0.17	0.17	0.17	0.17	0.17	0	0	0	0	0	0	0	0	0	0	1	0.17	N/A
MOVAPD %XMM9,%XMM12	1	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0.17	vect (25.0%)
SUBSD 0x10(%RCX,%RAX,8),%XMM12	1	0	0	0	0	0	0	0.25	0.25	0.25	0.25	0	0	0.50	0.50	0	0	2	0.50	scal (12.5%)
MOVSXD %R13D,%RCX	1	0.17	0.17	0.17	0.17	0.17	0.17	0	0	0	0	0	0	0	0	0	0	1	0.17	N/A
SUB %RDI,%RCX	1	0.17	0.17	0.17	0.17	0.17	0.17	0	0	0	0	0	0	0	0	0	0	1	0.17	N/A
IMUL %RSI,%RCX	1	0.33	0.33	0.33	0	0	0	0	0	0	0	0	0	0	0	0	0	3	0.33	N/A
ADD %R11,%RCX	1	0.17	0.17	0.17	0.17	0.17	0.17	0	0	0	0	0	0	0	0	0	0	1	0.17	N/A
MOVSD 0x10(%RCX,%RAX,8),%XMM11	1	0	0	0	0	0	0	0.25	0.25	0.25	0.25	0	0	0	0	0	0	1	0.50	scal (12.5%)
SUBSD %XMM9,%XMM11	1	0	0	0	0	0	0	0	0	0	0	0	0	0.50	0.50	0	0	2	0.50	scal (12.5%)
MOVAPD %XMM11,%XMM13	1	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0.17	vect (25.0%)
MULSD %XMM12,%XMM13	1	0	0	0	0	0	0	0	0	0	0	0.50	0.50	0	0	0	0	3	0.50	scal (12.5%)
UCOMISD %XMM10,%XMM13	2	0	0	0	0	0	0	0	0	0	0	0	0	0.50	0.50	0.50	0.50	7	1	scal (12.5%)
JBE 424c40 <advec_mom_kernel_mod_mp_advec_mom_kernel_.DIR.OMP.PARALLEL.2+0x1870>	1	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	0	0	0	0	1	0.50-1	N/A

Report Configuration

Vectorization

Details

Workaround

FMA

Workaround

Complex instructions

Details

Slow data structures access

Details

Workaround

Type of elements and instruction set

Matching between your loop (in the source code) and the binary loop

Arithmetic intensity

Unroll opportunity

Workaround

General properties

Front-end

Back-end

Cycles summary

Vectorization ratios

Vector efficiency ratios

Cycles and memory resources usage

Code clean check

Workaround

Vectorization

Details

Workaround

Execution units bottlenecks

FMA

Workaround

Complex instructions

Details

Slow data structures access

Details

Workaround

Type of elements and instruction set

Matching between your loop (in the source code) and the binary loop

Arithmetic intensity

General properties

Front-end

Back-end

Cycles summary

Vectorization ratios

Vector efficiency ratios

Cycles and memory resources usage

Front-end bottlenecks

ASM code

Code clean check

Workaround

Vectorization

Details

Workaround

Execution units bottlenecks

FMA

Workaround

Complex instructions

Details

Slow data structures access

Details

Workaround

Type of elements and instruction set

Matching between your loop (in the source code) and the binary loop

Arithmetic intensity

Unroll opportunity

Workaround

General properties

Front-end

Back-end

Cycles summary

Vectorization ratios

Vector efficiency ratios

Cycles and memory resources usage

Front-end bottlenecks

ASM code