OV - exec - Loop 663 - libkripke.so

	Loop Id: 663	Module: libkripke.so	Source: forall.hpp:59-59 [...]	Coverage: 0.01%

0x52ef0 MOV	-0x110(%RBP),%RCX

0x52ef7 MOV	-0xe8(%RBP),%R12

0x52efe LEA	0x1(%RCX),%RDI

0x52f02 CMP	-0x108(%RBP),%RCX

0x52f09 JGE	53201

0x52f0f MOV	%RDI,%RAX

0x52f12 CQTO

0x52f14 MOV	%RDI,-0x110(%RBP)

0x52f1b IDIV	%R12

0x52f1e MOV	-0x38(%RBP),%RDI

0x52f22 ADD	-0x100(%RBP),%RDX

0x52f29 MOV	%RDX,%RSI

0x52f2c IMUL	-0xf0(%RBP),%RSI

0x52f34 MOV	%RDX,%R10

0x52f37 IMUL	-0x78(%RBP),%R10

0x52f3c IMUL	-0xb8(%RBP),%RDX

0x52f44 ADD	-0xc0(%RBP),%RDX

0x52f4b MOV	%RAX,%R9

0x52f4e MOV	-0x80(%RBP),%RAX

0x52f52 LEA	(%R9,%RAX,1),%RCX

0x52f56 MOV	-0x50(%RBP),%RAX

0x52f5a ADD	-0xe0(%RBP),%R9

0x52f61 MOV	%RCX,-0x148(%RBP)

0x52f68 IMUL	-0xf8(%RBP),%RCX

0x52f70 ADD	%RSI,%RAX

0x52f73 ADD	-0x48(%RBP),%RSI

0x52f77 ADD	%RCX,%RAX

0x52f7a LEA	(%RDI,%RAX,8),%R11

0x52f7e MOV	%RAX,-0x90(%RBP)

0x52f85 ADD	%RCX,%RSI

0x52f88 LEA	(%RDI,%RSI,8),%RCX

0x52f8c MOV	-0xd8(%RBP),%RDI

0x52f93 LEA	0x8(,%R9,8),%RSI

0x52f9b SAL	$0x3,%R9

0x52f9f MOV	%R11,-0x130(%RBP)

0x52fa6 XOR	%R11D,%R11D

0x52fa9 MOV	%R9,-0x140(%RBP)

0x52fb0 MOV	%RSI,-0x120(%RBP)

0x52fb7 MOV	-0xc8(%RBP),%RSI

0x52fbe MOV	%RCX,-0x128(%RBP)

0x52fc5 LEA	(%RDI,%R10,1),%R12

0x52fc9 ADD	-0xd0(%RBP),%R10

0x52fd0 LEA	(,%R12,8),%RCX

0x52fd8 MOV	%RCX,-0x118(%RBP)

0x52fdf SAL	$0x3,%R10

0x52fe3 MOV	%R10,-0x138(%RBP)

0x52fea LEA	(%RSI,%RAX,8),%R10

0x52fee JMP	5300b

  (664) 0x52ff0 ADD	-0xa8(%RBP),%RDX

  (664) 0x52ff7 ADD	-0x98(%RBP),%R12

  (664) 0x52ffe INC	%R11

  (664) 0x53001 CMP	-0x60(%RBP),%R11

  (664) 0x53005 JE	52ef0

  (664) 0x5300b MOV	-0x58(%RBP),%RCX

  (664) 0x5300f LEA	(%R11,%RCX,1),%R13

  (664) 0x53013 IMUL	-0xa0(%RBP),%R13

  (664) 0x5301b ADD	-0x148(%RBP),%R13

  (664) 0x53022 CMPB	$0,-0x29(%RBP)

  (664) 0x53026 JE	53070

  (664) 0x53028 XOR	%ESI,%ESI

  (664) 0x5302a MOV	-0x38(%RBP),%RDI

  (664) 0x5302e LEA	(%RAX,%RSI,1),%RCX

  (664) 0x53032 MOV	%R8,%R9

  (664) 0x53035 SUB	%RSI,%R9

  (664) 0x53038 ADD	%R12,%RSI

  (664) 0x5303b LEA	(%RDI,%RCX,8),%RCX

  (664) 0x5303f MOV	-0x40(%RBP),%RDI

  (664) 0x53043 LEA	(%RDI,%RSI,8),%RSI

  (664) 0x53047 XOR	%EDI,%EDI

  (664) 0x53049 NOPL	(%RAX)

    (662) 0x53050 VMOVSD	(%R14,%R13,8),%XMM0

    (662) 0x53056 VMOVSD	(%RSI,%RDI,8),%XMM1

    (662) 0x5305b VFMADD213SD	(%RCX,%RDI,8),%XMM0,%XMM1

    (662) 0x53061 VMOVSD	%XMM1,(%RCX,%RDI,8)

    (662) 0x53066 INC	%RDI

    (662) 0x53069 CMP	%RDI,%R9

    (662) 0x5306c JNE	53050

  (664) 0x5306e JMP	52ff0

  (664) 0x53070 MOV	-0xa8(%RBP),%RSI

  (664) 0x53077 MOV	-0x138(%RBP),%RCX

  (664) 0x5307e MOV	-0x40(%RBP),%RAX

  (664) 0x53082 MOV	-0x120(%RBP),%RDI

  (664) 0x53089 MOV	-0x140(%RBP),%R9

  (664) 0x53090 MOV	-0x128(%RBP),%RBX

  (664) 0x53097 IMUL	%R11,%RSI

  (664) 0x5309b LEA	(%RDI,%R11,8),%RDI

  (664) 0x5309f LEA	(%R9,%R11,8),%R9

  (664) 0x530a3 ADD	%R14,%RDI

  (664) 0x530a6 ADD	%R14,%R9

  (664) 0x530a9 ADD	%RSI,%RCX

  (664) 0x530ac ADD	-0x118(%RBP),%RSI

  (664) 0x530b3 ADD	%RAX,%RCX

  (664) 0x530b6 ADD	%RAX,%RSI

  (664) 0x530b9 MOV	-0x130(%RBP),%RAX

  (664) 0x530c0 CMP	%RDI,%RAX

  (664) 0x530c3 SETB	%DIL

  (664) 0x530c7 CMP	%RBX,%R9

  (664) 0x530ca SETB	%R9B

  (664) 0x530ce CMP	%RCX,%RAX

  (664) 0x530d1 SETB	%CL

  (664) 0x530d4 CMP	%RBX,%RSI

  (664) 0x530d7 SETB	%SIL

  (664) 0x530db TEST	%R9B,%DIL

  (664) 0x530de JNE	53102

  (664) 0x530e0 MOV	-0x90(%RBP),%RAX

  (664) 0x530e7 AND	%SIL,%CL

  (664) 0x530ea MOV	$0,%ESI

  (664) 0x530ef JNE	5302a

  (664) 0x530f5 CMP	$0x20,%R8

  (664) 0x530f9 JAE	5310e

  (664) 0x530fb XOR	%EDI,%EDI

  (664) 0x530fd JMP	5319f

  (664) 0x53102 MOV	-0x90(%RBP),%RAX

  (664) 0x53109 JMP	53028

  (664) 0x5310e VBROADCASTSD	(%R14,%R13,8),%ZMM0

  (664) 0x53115 XOR	%R9D,%R9D

  (664) 0x53118 NOPL	(%RAX,%RAX,1)

    (665) 0x53120 VMOVUPD	-0xc0(%RDX,%R9,8),%ZMM1

    (665) 0x53128 VMOVUPD	-0x80(%RDX,%R9,8),%ZMM2

    (665) 0x53130 VMOVUPD	-0x40(%RDX,%R9,8),%ZMM3

    (665) 0x53138 VMOVUPD	(%RDX,%R9,8),%ZMM4

    (665) 0x5313f VFMADD213PD	-0xc0(%R10,%R9,8),%ZMM0,%ZMM1

    (665) 0x53147 VFMADD213PD	-0x80(%R10,%R9,8),%ZMM0,%ZMM2

    (665) 0x5314f VFMADD213PD	-0x40(%R10,%R9,8),%ZMM0,%ZMM3

    (665) 0x53157 VFMADD213PD	(%R10,%R9,8),%ZMM0,%ZMM4

    (665) 0x5315e VMOVUPD	%ZMM1,-0xc0(%R10,%R9,8)

    (665) 0x53166 VMOVUPD	%ZMM2,-0x80(%R10,%R9,8)

    (665) 0x5316e VMOVUPD	%ZMM3,-0x40(%R10,%R9,8)

    (665) 0x53176 VMOVUPD	%ZMM4,(%R10,%R9,8)

    (665) 0x5317d ADD	$0x20,%R9

    (665) 0x53181 CMP	%R9,%R15

    (665) 0x53184 JNE	53120

  (664) 0x53186 CMP	%R15,%R8

  (664) 0x53189 JE	52ff0

  (664) 0x5318f MOV	%R15,%RDI

  (664) 0x53192 MOV	%R15,%RSI

  (664) 0x53195 TEST	$0x1c,%R8B

  (664) 0x53199 JE	5302a

  (664) 0x5319f VBROADCASTSD	(%R14,%R13,8),%YMM0

  (664) 0x531a5 MOV	-0x88(%RBP),%R9

  (664) 0x531ac MOV	-0x38(%RBP),%RSI

  (664) 0x531b0 MOV	-0x40(%RBP),%RBX

  (664) 0x531b4 LEA	(%RAX,%RDI,1),%RCX

  (664) 0x531b8 SUB	%RDI,%R9

  (664) 0x531bb ADD	%R12,%RDI

  (664) 0x531be LEA	(%RSI,%RCX,8),%RCX

  (664) 0x531c2 LEA	(%RBX,%RDI,8),%RSI

  (664) 0x531c6 XOR	%EDI,%EDI

  (664) 0x531c8 NOPL	(%RAX,%RAX,1)

    (666) 0x531d0 VMOVUPD	(%RSI,%RDI,8),%YMM1

    (666) 0x531d5 VFMADD213PD	(%RCX,%RDI,8),%YMM0,%YMM1

    (666) 0x531db VMOVUPD	%YMM1,(%RCX,%RDI,8)

    (666) 0x531e0 ADD	$0x4,%RDI

    (666) 0x531e4 CMP	%RDI,%R9

    (666) 0x531e7 JNE	531d0

  (664) 0x531e9 MOV	-0x88(%RBP),%RCX

  (664) 0x531f0 MOV	%RCX,%RSI

  (664) 0x531f3 CMP	%RCX,%R8

  (664) 0x531f6 JE	52ff0

  (664) 0x531fc JMP	5302a

/beegfs/hackathon/users/eoseret/qaas_runs/170-850-6313/intel/Kripke/build/Kripke/tpl/raja/include/RAJA/policy/openmp/kernel/Collapse.hpp: 83 - 83

--------------------------------------------------------------------------------

83:     for (i0 = 0; i0 < l0; ++i0) {

/beegfs/hackathon/users/eoseret/qaas_runs/170-850-6313/intel/Kripke/build/Kripke/src/Kripke/Kernel/LTimes.cpp: 62 - 62

--------------------------------------------------------------------------------

62:            phi(nm,g,z) += ell(nm, d) * psi(d, g, z);

/beegfs/hackathon/users/eoseret/qaas_runs/170-850-6313/intel/Kripke/build/Kripke/tpl/raja/include/RAJA/policy/loop/forall.hpp: 59 - 59

--------------------------------------------------------------------------------

59:   for (decltype(distance_it) i = 0; i < distance_it; ++i) {

Path /

Metric	Value
CQA speedup if no scalar integer	1.00
CQA speedup if FP arith vectorized	1.00
CQA speedup if fully vectorized	4.84 - 4.00
CQA speedup if no inter-iteration dependency	NA
CQA speedup if next bottleneck killed	1.08 - 1.38
Bottlenecks	P8, P9,
Function	.omp_outlined..152
Source	Collapse.hpp:83-83,forall.hpp:59-59
Source loop unroll info	NA
Source loop unroll confidence level	NA
Unroll/vectorization loop type	NA
Unroll factor	NA
CQA cycles	8.67 - 12.00
CQA cycles if no scalar integer	8.67 - 12.00
CQA cycles if FP arith vectorized	8.67 - 12.00
CQA cycles if fully vectorized	1.79 - 3.00
Front-end cycles	8.00
DIV/SQRT cycles	6.25
P0 cycles	6.25
P1 cycles	6.25
P2 cycles	6.25
P3 cycles	2.00
P4 cycles	8.67
P5 cycles	8.67
P6 cycles	8.67
P7 cycles	0.00
P8 cycles	0.00
P9 cycles	0.00
P10 cycles	0.00
P11 cycles	0.00
P12 cycles	0.00
P13 cycles	7.00 - 12.00
Inter-iter dependencies cycles	NA
FE+BE cycles (UFS)	NA
Stall cycles (UFS)	NA
Nb insns	48.00
Nb uops	48.00
Nb loads	17.00
Nb stores	9.00
Nb stack references	25.00
FLOP/cycle	0.00 - 0.00
Nb FLOP add-sub	0.00
Nb FLOP mul	0.00
Nb FLOP fma	0.00
Nb FLOP div	0.00
Nb FLOP rcp	0.00
Nb FLOP sqrt	0.00
Nb FLOP rsqrt	0.00
Bytes/cycle	17.33 - 24.00
Bytes prefetched	0.00
Bytes loaded	136.00
Bytes stored	72.00
Stride 0	NA
Stride 1	NA
Stride n	NA
Stride unknown	NA
Stride indirect	NA
Vectorization ratio all	0.00
Vectorization ratio load	0.00
Vectorization ratio store	0.00
Vectorization ratio mul	0.00
Vectorization ratio add_sub	0.00
Vectorization ratio fma	NA
Vectorization ratio div_sqrt	0.00
Vectorization ratio other	0.00
Vector-efficiency ratio all	12.11
Vector-efficiency ratio load	12.50
Vector-efficiency ratio store	12.50
Vector-efficiency ratio mul	12.50
Vector-efficiency ratio add_sub	12.50
Vector-efficiency ratio fma	NA
Vector-efficiency ratio div_sqrt	12.50
Vector-efficiency ratio other	10.42

Metric	Value
CQA speedup if no scalar integer	1.00
CQA speedup if FP arith vectorized	1.00
CQA speedup if fully vectorized	4.84 - 4.00
CQA speedup if no inter-iteration dependency	NA
CQA speedup if next bottleneck killed	1.08 - 1.38
Bottlenecks	P8, P9,
Function	.omp_outlined..152
Source	Collapse.hpp:83-83,forall.hpp:59-59
Source loop unroll info	NA
Source loop unroll confidence level	NA
Unroll/vectorization loop type	NA
Unroll factor	NA
CQA cycles	8.67 - 12.00
CQA cycles if no scalar integer	8.67 - 12.00
CQA cycles if FP arith vectorized	8.67 - 12.00
CQA cycles if fully vectorized	1.79 - 3.00
Front-end cycles	8.00
DIV/SQRT cycles	6.25
P0 cycles	6.25
P1 cycles	6.25
P2 cycles	6.25
P3 cycles	2.00
P4 cycles	8.67
P5 cycles	8.67
P6 cycles	8.67
P7 cycles	0.00
P8 cycles	0.00
P9 cycles	0.00
P10 cycles	0.00
P11 cycles	0.00
P12 cycles	0.00
P13 cycles	7.00 - 12.00
Inter-iter dependencies cycles	NA
FE+BE cycles (UFS)	NA
Stall cycles (UFS)	NA
Nb insns	48.00
Nb uops	48.00
Nb loads	17.00
Nb stores	9.00
Nb stack references	25.00
FLOP/cycle	0.00 - 0.00
Nb FLOP add-sub	0.00
Nb FLOP mul	0.00
Nb FLOP fma	0.00
Nb FLOP div	0.00
Nb FLOP rcp	0.00
Nb FLOP sqrt	0.00
Nb FLOP rsqrt	0.00
Bytes/cycle	17.33 - 24.00
Bytes prefetched	0.00
Bytes loaded	136.00
Bytes stored	72.00
Stride 0	NA
Stride 1	NA
Stride n	NA
Stride unknown	NA
Stride indirect	NA
Vectorization ratio all	0.00
Vectorization ratio load	0.00
Vectorization ratio store	0.00
Vectorization ratio mul	0.00
Vectorization ratio add_sub	0.00
Vectorization ratio fma	NA
Vectorization ratio div_sqrt	0.00
Vectorization ratio other	0.00
Vector-efficiency ratio all	12.11
Vector-efficiency ratio load	12.50
Vector-efficiency ratio store	12.50
Vector-efficiency ratio mul	12.50
Vector-efficiency ratio add_sub	12.50
Vector-efficiency ratio fma	NA
Vector-efficiency ratio div_sqrt	12.50
Vector-efficiency ratio other	10.42

Path /

Average path: Display a virtual path defined by average values of all real paths

Function	.omp_outlined..152
Source file and lines	forall.hpp:59-59
Module	libkripke.so

The loop is defined in:

/beegfs/hackathon/users/eoseret/qaas_runs/170-850-6313/intel/Kripke/build/Kripke/tpl/raja/include/RAJA/policy/openmp/kernel/Collapse.hpp:83
/beegfs/hackathon/users/eoseret/qaas_runs/170-850-6313/intel/Kripke/build/Kripke/src/Kripke/Kernel/LTimes.cpp:62
/beegfs/hackathon/users/eoseret/qaas_runs/170-850-6313/intel/Kripke/build/Kripke/tpl/raja/include/RAJA/policy/loop/forall.hpp:59

Warnings:

Non-innermost loop: analyzing only self part (ignoring child loops).
Ignoring paths for analysis
Too many paths. Rerun with max-paths=9
RecMII not computed since number of paths is unknown or > max_paths
Streams not analyzed since number of paths is unknown or > max_paths

Try to simplify control and/or increase the maximum number of paths per function/loop through the 'max-paths-nb' option.

This loop has 9 execution paths.

The presence of multiple execution paths is typically the main/first bottleneck.
Try to simplify control inside loop: ideally, try to remove all conditional expressions, for example by (if applicable):

hoisting them (moving them outside the loop)
turning them into conditional moves, MIN or MAX

Ex: if (x<0) x=0 => x = (x<0 ? 0 : x) (or MAX(0,x) after defining the corresponding macro)

gain
potential
hint
expert

Vectorization

Your loop is not vectorized. Only 12% of vector register length is used (average across all SSE/AVX instructions). By vectorizing your loop, you can lower the cost of an iteration from 12.00 to 3.00 cycles (4.00x speedup).

Details

All SSE/AVX instructions are used in scalar version (process only one data element in vector registers). Since your execution units are vector units, only a vectorized loop can use their full power.

Workaround

Try another compiler or update/tune your current one:
- recompile with ffast-math (included in Ofast) to extend loop vectorization to FP reductions.
Remove inter-iterations dependences from your loop and make it unit-stride:
- If your arrays have 2 or more dimensions, check whether elements are accessed contiguously and, otherwise, try to permute loops accordingly: C storage order is row-major: for(i) for(j) a[j][i] = b[j][i]; (slow, non stride 1) => for(i) for(j) a[i][j] = b[i][j]; (fast, stride 1)
- If your loop streams arrays of structures (AoS), try to use structures of arrays instead (SoA): for(i) a[i].x = b[i].x; (slow, non stride 1) => for(i) a.x[i] = b.x[i]; (fast, stride 1)

Execution units bottlenecks

Performance is limited by execution of divide and square root operations (the divide/square root unit is a bottleneck). By removing all these bottlenecks, you can lower the cost of an iteration from 12.00 to 8.67 cycles (1.38x speedup).

Workaround

Reduce the number of division or square root instructions:

If denominator is constant over iterations, use reciprocal (replace x/y with x*(1/y)). Check precision impact. This will be done by your compiler with ffast-math or Ofast

Expensive FP math instructions/calls

Detected performance impact from expensive FP math instructions/calls. By removing/reexpressing them, you can lower the cost of an iteration from 12.00 to 8.67 cycles (1.38x speedup).

Complex instructions

Detected COMPLEX INSTRUCTIONS.

Details

These instructions generate more than one micro-operation and only one of them can be decoded during a cycle and the extra micro-operations increase pressure on execution units.

IDIV: 1 occurrences

Conversion instructions

Detected expensive conversion instructions.

Details

CQTO: 1 occurrences

Workaround

Avoid mixing data with different types. In particular, check if the type of constants is the same as array elements.

Type of elements and instruction set

No instructions are processing arithmetic or math operations on FP elements. This loop is probably writing/copying data or processing integer elements.

Matching between your loop (in the source code) and the binary loop

The binary loop does not contain any FP arithmetical operations. The binary loop is loading 136 bytes. The binary loop is storing 72 bytes.

General properties

nb instructions	48
nb uops	48
loop length	256
used x86 registers	10
used mmx registers	0
used xmm registers	0
used ymm registers	0
used zmm registers	0
nb stack references	25

Front-end

MACRO FUSION NOT POSSIBLE FIT IN UOP CACHE

micro-operation queue	8.00 cycles
front end	8.00 cycles

Back-end

	ALU0/BRU0	ALU1	ALU2	ALU3	BRU1	AGU0	AGU1	AGU2	FP0	FP1	FP2	FP3	FP4	FP5
uops	6.25	6.25	6.25	6.25	2.00	8.67	8.67	8.67	0.00	0.00	0.00	0.00	0.00	0.00
cycles	6.25	6.25	6.25	6.25	2.00	8.67	8.67	8.67	0.00	0.00	0.00	0.00	0.00	0.00

Execution ports to units layout:

ALU0/BRU0: ALU
ALU1: ALU
ALU2: ALU
ALU3: ALU
BRU1:
AGU0 (256 bits): store address, load
AGU1 (256 bits): store address, load
AGU2 (256 bits): store address, load
FP0 (256 bits): VPU, DIV/SQRT
FP1 (256 bits): VPU, DIV/SQRT
FP2 (256 bits): VPU
FP3 (256 bits): VPU
FP4 (256 bits): FP store data
FP5 (256 bits): FP store data

Cycles executing div or sqrt instructions

7.00-12.00

Cycles summary

Front-end	8.00
Dispatch	8.67
DIV/SQRT	7.00-12.00
Overall L1	8.67-12.00

Vectorization ratios

all	0%
load	0%
store	0%
mul	0%
add-sub	0%
fma	NA (no fma vectorizable/vectorized instructions)
div/sqrt	0%
other	0%

Vector efficiency ratios

all	12%
load	12%
store	12%
mul	12%
add-sub	12%
fma	NA (no fma vectorizable/vectorized instructions)
div/sqrt	12%
other	10%

Cycles and memory resources usage

Assuming all data fit into the L1 cache, each iteration of the binary loop takes 12.00 cycles. At this rate:

11% of peak load performance is reached (11.33 out of 96.00 bytes loaded per cycle (GB/s @ 1GHz))
9% of peak store performance is reached (6.00 out of 64.00 bytes stored per cycle (GB/s @ 1GHz))

Front-end bottlenecks

Found no such bottlenecks.

ASM code

In the binary file, the address of the loop is: 52f0f

Instruction	Nb FU	ALU0/BRU0	ALU1	ALU2	ALU3	BRU1	AGU0	AGU1	AGU2	FP0	FP1	FP2	FP3	FP4	FP5	Latency	Recip. throughput
MOV -0x110(%RBP),%RCX	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	3	0.33
MOV -0xe8(%RBP),%R12	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	3	0.33
LEA 0x1(%RCX),%RDI	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
CMP -0x108(%RBP),%RCX	1	0.25	0.25	0.25	0.25	0	0.33	0.33	0.33	0	0	0	0	0	0	1	0.33
JGE 53201 <.omp_outlined..152+0x541>	1	0.50	0	0	0	0.50	0	0	0	0	0	0	0	0	0	1	0.50-1
MOV %RDI,%RAX	1	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0.17
CQTO
MOV %RDI,-0x110(%RBP)	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	4	0.50
IDIV %R12	2	2	0	0	0	0	0	0	0	0	0	0	0	0	0	9-19	7-12
MOV -0x38(%RBP),%RDI	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	3	0.33
ADD -0x100(%RBP),%RDX	1	0.25	0.25	0.25	0.25	0	0.33	0.33	0.33	0	0	0	0	0	0	1	0.33
MOV %RDX,%RSI	1	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0.17
IMUL -0xf0(%RBP),%RSI	1	0	1	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	3	1
MOV %RDX,%R10	1	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0.17
IMUL -0x78(%RBP),%R10	1	0	1	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	3	1
IMUL -0xb8(%RBP),%RDX	1	0	1	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	3	1
ADD -0xc0(%RBP),%RDX	1	0.25	0.25	0.25	0.25	0	0.33	0.33	0.33	0	0	0	0	0	0	1	0.33
MOV %RAX,%R9	1	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0.17
MOV -0x80(%RBP),%RAX	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	3	0.33
LEA (%R9,%RAX,1),%RCX	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
MOV -0x50(%RBP),%RAX	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	3	0.33
ADD -0xe0(%RBP),%R9	1	0.25	0.25	0.25	0.25	0	0.33	0.33	0.33	0	0	0	0	0	0	1	0.33
MOV %RCX,-0x148(%RBP)	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	4	0.50
IMUL -0xf8(%RBP),%RCX	1	0	1	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	3	1
ADD %RSI,%RAX	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
ADD -0x48(%RBP),%RSI	1	0.25	0.25	0.25	0.25	0	0.33	0.33	0.33	0	0	0	0	0	0	1	0.33
ADD %RCX,%RAX	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
LEA (%RDI,%RAX,8),%R11	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
MOV %RAX,-0x90(%RBP)	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	4	0.50
ADD %RCX,%RSI	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
LEA (%RDI,%RSI,8),%RCX	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
MOV -0xd8(%RBP),%RDI	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	3	0.33
LEA 0x8(,%R9,8),%RSI	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
SAL $0x3,%R9	1	0	0.50	0.50	0	0	0	0	0	0	0	0	0	0	0	1	0.50
MOV %R11,-0x130(%RBP)	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	4	0.50
XOR %R11D,%R11D	1	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0.25
MOV %R9,-0x140(%RBP)	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	4	0.50
MOV %RSI,-0x120(%RBP)	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	4	0.50
MOV -0xc8(%RBP),%RSI	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	3	0.33
MOV %RCX,-0x128(%RBP)	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	4	0.50
LEA (%RDI,%R10,1),%R12	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
ADD -0xd0(%RBP),%R10	1	0.25	0.25	0.25	0.25	0	0.33	0.33	0.33	0	0	0	0	0	0	1	0.33
LEA (,%R12,8),%RCX	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
MOV %RCX,-0x118(%RBP)	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	4	0.50
SAL $0x3,%R10	1	0	0.50	0.50	0	0	0	0	0	0	0	0	0	0	0	1	0.50
MOV %R10,-0x138(%RBP)	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	4	0.50
LEA (%RSI,%RAX,8),%R10	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
JMP 5300b <.omp_outlined..152+0x34b>	1	0.50	0	0	0	0.50	0	0	0	0	0	0	0	0	0	0	1

Function	.omp_outlined..152
Source file and lines	forall.hpp:59-59
Module	libkripke.so

The loop is defined in:

/beegfs/hackathon/users/eoseret/qaas_runs/170-850-6313/intel/Kripke/build/Kripke/tpl/raja/include/RAJA/policy/openmp/kernel/Collapse.hpp:83
/beegfs/hackathon/users/eoseret/qaas_runs/170-850-6313/intel/Kripke/build/Kripke/src/Kripke/Kernel/LTimes.cpp:62
/beegfs/hackathon/users/eoseret/qaas_runs/170-850-6313/intel/Kripke/build/Kripke/tpl/raja/include/RAJA/policy/loop/forall.hpp:59

Warnings:

Non-innermost loop: analyzing only self part (ignoring child loops).
Ignoring paths for analysis
Too many paths. Rerun with max-paths=9
RecMII not computed since number of paths is unknown or > max_paths
Streams not analyzed since number of paths is unknown or > max_paths

hoisting them (moving them outside the loop)
turning them into conditional moves, MIN or MAX

Ex: if (x<0) x=0 => x = (x<0 ? 0 : x) (or MAX(0,x) after defining the corresponding macro)

gain
potential
hint
expert

Vectorization

Details

All SSE/AVX instructions are used in scalar version (process only one data element in vector registers). Since your execution units are vector units, only a vectorized loop can use their full power.

Workaround

Try another compiler or update/tune your current one:
- recompile with ffast-math (included in Ofast) to extend loop vectorization to FP reductions.
Remove inter-iterations dependences from your loop and make it unit-stride:
- If your arrays have 2 or more dimensions, check whether elements are accessed contiguously and, otherwise, try to permute loops accordingly: C storage order is row-major: for(i) for(j) a[j][i] = b[j][i]; (slow, non stride 1) => for(i) for(j) a[i][j] = b[i][j]; (fast, stride 1)
- If your loop streams arrays of structures (AoS), try to use structures of arrays instead (SoA): for(i) a[i].x = b[i].x; (slow, non stride 1) => for(i) a.x[i] = b.x[i]; (fast, stride 1)

Execution units bottlenecks

Workaround

Reduce the number of division or square root instructions:

If denominator is constant over iterations, use reciprocal (replace x/y with x*(1/y)). Check precision impact. This will be done by your compiler with ffast-math or Ofast

Expensive FP math instructions/calls

Detected performance impact from expensive FP math instructions/calls. By removing/reexpressing them, you can lower the cost of an iteration from 12.00 to 8.67 cycles (1.38x speedup).

Complex instructions

Detected COMPLEX INSTRUCTIONS.

Details

These instructions generate more than one micro-operation and only one of them can be decoded during a cycle and the extra micro-operations increase pressure on execution units.

IDIV: 1 occurrences

Conversion instructions

Detected expensive conversion instructions.

Details

CQTO: 1 occurrences

Workaround

Avoid mixing data with different types. In particular, check if the type of constants is the same as array elements.

Type of elements and instruction set

No instructions are processing arithmetic or math operations on FP elements. This loop is probably writing/copying data or processing integer elements.

Matching between your loop (in the source code) and the binary loop

The binary loop does not contain any FP arithmetical operations. The binary loop is loading 136 bytes. The binary loop is storing 72 bytes.

General properties

nb instructions	48
nb uops	48
loop length	256
used x86 registers	10
used mmx registers	0
used xmm registers	0
used ymm registers	0
used zmm registers	0
nb stack references	25

Front-end

MACRO FUSION NOT POSSIBLE FIT IN UOP CACHE

micro-operation queue	8.00 cycles
front end	8.00 cycles

Back-end

	ALU0/BRU0	ALU1	ALU2	ALU3	BRU1	AGU0	AGU1	AGU2	FP0	FP1	FP2	FP3	FP4	FP5
uops	6.25	6.25	6.25	6.25	2.00	8.67	8.67	8.67	0.00	0.00	0.00	0.00	0.00	0.00
cycles	6.25	6.25	6.25	6.25	2.00	8.67	8.67	8.67	0.00	0.00	0.00	0.00	0.00	0.00

Execution ports to units layout:

ALU0/BRU0: ALU
ALU1: ALU
ALU2: ALU
ALU3: ALU
BRU1:
AGU0 (256 bits): store address, load
AGU1 (256 bits): store address, load
AGU2 (256 bits): store address, load
FP0 (256 bits): VPU, DIV/SQRT
FP1 (256 bits): VPU, DIV/SQRT
FP2 (256 bits): VPU
FP3 (256 bits): VPU
FP4 (256 bits): FP store data
FP5 (256 bits): FP store data

Cycles executing div or sqrt instructions

7.00-12.00

Cycles summary

Front-end	8.00
Dispatch	8.67
DIV/SQRT	7.00-12.00
Overall L1	8.67-12.00

Vectorization ratios

all	0%
load	0%
store	0%
mul	0%
add-sub	0%
fma	NA (no fma vectorizable/vectorized instructions)
div/sqrt	0%
other	0%

Vector efficiency ratios

all	12%
load	12%
store	12%
mul	12%
add-sub	12%
fma	NA (no fma vectorizable/vectorized instructions)
div/sqrt	12%
other	10%

Cycles and memory resources usage

Assuming all data fit into the L1 cache, each iteration of the binary loop takes 12.00 cycles. At this rate:

11% of peak load performance is reached (11.33 out of 96.00 bytes loaded per cycle (GB/s @ 1GHz))
9% of peak store performance is reached (6.00 out of 64.00 bytes stored per cycle (GB/s @ 1GHz))

Front-end bottlenecks

Found no such bottlenecks.

ASM code

In the binary file, the address of the loop is: 52f0f

Instruction	Nb FU	ALU0/BRU0	ALU1	ALU2	ALU3	BRU1	AGU0	AGU1	AGU2	FP0	FP1	FP2	FP3	FP4	FP5	Latency	Recip. throughput
MOV -0x110(%RBP),%RCX	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	3	0.33
MOV -0xe8(%RBP),%R12	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	3	0.33
LEA 0x1(%RCX),%RDI	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
CMP -0x108(%RBP),%RCX	1	0.25	0.25	0.25	0.25	0	0.33	0.33	0.33	0	0	0	0	0	0	1	0.33
JGE 53201 <.omp_outlined..152+0x541>	1	0.50	0	0	0	0.50	0	0	0	0	0	0	0	0	0	1	0.50-1
MOV %RDI,%RAX	1	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0.17
CQTO
MOV %RDI,-0x110(%RBP)	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	4	0.50
IDIV %R12	2	2	0	0	0	0	0	0	0	0	0	0	0	0	0	9-19	7-12
MOV -0x38(%RBP),%RDI	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	3	0.33
ADD -0x100(%RBP),%RDX	1	0.25	0.25	0.25	0.25	0	0.33	0.33	0.33	0	0	0	0	0	0	1	0.33
MOV %RDX,%RSI	1	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0.17
IMUL -0xf0(%RBP),%RSI	1	0	1	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	3	1
MOV %RDX,%R10	1	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0.17
IMUL -0x78(%RBP),%R10	1	0	1	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	3	1
IMUL -0xb8(%RBP),%RDX	1	0	1	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	3	1
ADD -0xc0(%RBP),%RDX	1	0.25	0.25	0.25	0.25	0	0.33	0.33	0.33	0	0	0	0	0	0	1	0.33
MOV %RAX,%R9	1	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0.17
MOV -0x80(%RBP),%RAX	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	3	0.33
LEA (%R9,%RAX,1),%RCX	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
MOV -0x50(%RBP),%RAX	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	3	0.33
ADD -0xe0(%RBP),%R9	1	0.25	0.25	0.25	0.25	0	0.33	0.33	0.33	0	0	0	0	0	0	1	0.33
MOV %RCX,-0x148(%RBP)	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	4	0.50
IMUL -0xf8(%RBP),%RCX	1	0	1	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	3	1
ADD %RSI,%RAX	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
ADD -0x48(%RBP),%RSI	1	0.25	0.25	0.25	0.25	0	0.33	0.33	0.33	0	0	0	0	0	0	1	0.33
ADD %RCX,%RAX	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
LEA (%RDI,%RAX,8),%R11	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
MOV %RAX,-0x90(%RBP)	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	4	0.50
ADD %RCX,%RSI	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
LEA (%RDI,%RSI,8),%RCX	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
MOV -0xd8(%RBP),%RDI	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	3	0.33
LEA 0x8(,%R9,8),%RSI	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
SAL $0x3,%R9	1	0	0.50	0.50	0	0	0	0	0	0	0	0	0	0	0	1	0.50
MOV %R11,-0x130(%RBP)	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	4	0.50
XOR %R11D,%R11D	1	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0.25
MOV %R9,-0x140(%RBP)	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	4	0.50
MOV %RSI,-0x120(%RBP)	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	4	0.50
MOV -0xc8(%RBP),%RSI	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	3	0.33
MOV %RCX,-0x128(%RBP)	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	4	0.50
LEA (%RDI,%R10,1),%R12	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
ADD -0xd0(%RBP),%R10	1	0.25	0.25	0.25	0.25	0	0.33	0.33	0.33	0	0	0	0	0	0	1	0.33
LEA (,%R12,8),%RCX	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
MOV %RCX,-0x118(%RBP)	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	4	0.50
SAL $0x3,%R10	1	0	0.50	0.50	0	0	0	0	0	0	0	0	0	0	0	1	0.50
MOV %R10,-0x138(%RBP)	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	4	0.50
LEA (%RSI,%RAX,8),%R10	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
JMP 5300b <.omp_outlined..152+0x34b>	1	0.50	0	0	0	0.50	0	0	0	0	0	0	0	0	0	0	1

Report Configuration

Vectorization

Details

Workaround

Execution units bottlenecks

Workaround

Expensive FP math instructions/calls

Complex instructions

Details

Conversion instructions

Details

Workaround

Type of elements and instruction set

Matching between your loop (in the source code) and the binary loop

General properties

Front-end

Back-end

Cycles summary

Vectorization ratios

Vector efficiency ratios

Cycles and memory resources usage

Front-end bottlenecks

ASM code

Vectorization

Details

Workaround

Execution units bottlenecks

Workaround

Expensive FP math instructions/calls

Complex instructions

Details

Conversion instructions

Details

Workaround

Type of elements and instruction set

Matching between your loop (in the source code) and the binary loop

General properties

Front-end

Back-end

Cycles summary

Vectorization ratios

Vector efficiency ratios

Cycles and memory resources usage

Front-end bottlenecks

ASM code