OV - attention-native - Loop 36

	Loop Id: 36	Module: attention-native	Source: attention.cpp:43-289 [...]	Coverage: 0.05%

  (38) 0x12b24 LDP	X9, X8, [SP, #88]

  (38) 0x12b28 ADD	X9, X9, #1

  (38) 0x12b2c CMP	X9, X8

  (38) 0x12b30 STR	X9, [SP, #88]

  (38) 0x12b34 B.EQ	12fbc

  (38) 0x12b38 LDR	X8, [SP, #256]

  (38) 0x12b3c MOVN	X14, #0

  (38) 0x12b40 MOVZ	W10, #1

  (38) 0x12b44 STR	XZR, [SP, #192]

  (38) 0x12b48 ORR	X9, XZR, X27

  (38) 0x12b4c STR	X8, [SP, #248]

  (38) 0x12b50 B	12bac

  (38) 0x12b60 LDR	X8, [SP, #152]

  (38) 0x12b64 LDP	X14, X10, [SP, #216]

  (38) 0x12b68 ADD	X9, X26, X24

  (38) 0x12b6c CNTH	X12, ALL

  (38) 0x12b70 PTRUE	P4.S, ALL

  (38) 0x12b74 STR	X22, [SP, #192]

  (38) 0x12b78 ORR	X23, XZR, X24

  (38) 0x12b7c STR	S20, [X8, X28,LSL #2]

  (38) 0x12b80 LDR	X8, [SP, #248]

  (38) 0x12b84 LDP	X28, X26, [SP, #136]

  (38) 0x12b88 RDVL	X13, #2

  (38) 0x12b8c ADD	X10, X10, #1

  (38) 0x12b90 SUB	X14, X14, #1

  (38) 0x12b94 ADD	X8, X8, X24

  (38) 0x12b98 STR	X8, [SP, #248]

  (38) 0x12b9c LDR	X8, [SP, #280]

  (38) 0x12ba0 CMP	X22, X8

  (38) 0x12ba4 LDR	X22, [SP, #160]

  (38) 0x12ba8 B.EQ	12b24

  (38) 0x12bac AND	X15, X10, #0x0

  (38) 0x12bb0 CMP	X10, #4

  (38) 0x12bb4 ORR	X28, XZR, X10

  (38) 0x12bb8 STR	X9, [SP, #184]

  (38) 0x12bbc B.CS	12be0

  (38) 0x12bc0 MOVN	W9, #128

  (38) 0x12bc4 LDR	X26, [SP, #248]

  (38) 0x12bc8 ORR	X8, XZR, XZR

  (38) 0x12bcc FMOV	S21, W9

  (38) 0x12bd0 B	12c80

  (38) 0x12be0 LDR	X26, [SP, #248]

  (38) 0x12be4 CMP	X28, X12

  (38) 0x12be8 B.CS	12c00

  (38) 0x12bec MOVN	W9, #128

  (38) 0x12bf0 ORR	X8, XZR, XZR

  (38) 0x12bf4 FMOV	S21, W9

  (38) 0x12bf8 B	12c50

  (38) 0x12c00 LDR	X8, [SP, #128]

  (38) 0x12c04 DUPM	Z0.S, #0xff7fffff

  (38) 0x12c08 DUPM	Z1.S, #0xff7fffff

  (38) 0x12c0c ORR	X11, XZR, X26

  (38) 0x12c10 AND	X9, X28, X8

  (38) 0x12c14 SUB	X8, X28, X9

  (38) 0x12c18 ADD	X10, X9, X14

  (38) 0x12c1c HINT	#0

    (34) 0x12c20 LDR	Z2, [X11, MUL VL]

    (34) 0x12c24 LDR	Z3, [X11, #1, MUL VL]

    (34) 0x12c28 ADDS	X10, X10, X12

    (34) 0x12c2c ADD	X11, X11, X13

    (34) 0x12c30 FMAXNM	Z0.S, P4/M, Z0.S, Z2.S

    (34) 0x12c34 FMAXNM	Z1.S, P4/M, Z1.S, Z3.S

    (34) 0x12c38 B.NE	12c20

  (38) 0x12c3c FMAXNM	Z0.S, P4/M, Z0.S, Z1.S

  (38) 0x12c40 FMAXNMV	S21, P4, Z0.S

  (38) 0x12c44 CBZ	X9, 12c94

  (38) 0x12c48 CMP	X9, #4

  (38) 0x12c4c B.CC	12c80

  (38) 0x12c50 DUP	V0.4S, V21.S[0]

  (38) 0x12c54 SUB	X9, X8, X15

  (38) 0x12c58 UBFM	X10, X8, #62, #61

  (38) 0x12c5c AND	X8, X28, #0x0

    (42) 0x12c60 LDR	Q1, [X26, X10]

    (42) 0x12c64 ADDS	X9, X9, #4

    (42) 0x12c68 ADD	X10, X10, #16

    (42) 0x12c6c FMAXNM	V0.4S, V1.4S, V0.4S

    (42) 0x12c70 B.NE	12c60

  (38) 0x12c74 FMAXNMV	S21, V0.4S

  (38) 0x12c78 B	12c8c

    (41) 0x12c80 LDR	S0, [X26, X8,LSL #2]

    (41) 0x12c84 ADD	X8, X8, #1

    (41) 0x12c88 FMAXNM	S21, S0, S21

    (41) 0x12c8c CMP	X28, X8

    (41) 0x12c90 B.NE	12c80

  (38) 0x12c94 CMP	X28, #4

  (38) 0x12c98 SUB	X8, X29, #16

  (38) 0x12c9c STR	X14, [SP, #216]

  (38) 0x12ca0 STR	X15, [SP, #176]

  (38) 0x12ca4 STR	Z21, [X8, #508, MUL VL]

  (38) 0x12ca8 B.CS	12cc0

  (38) 0x12cac MOVI	D20, #0

  (38) 0x12cb0 ORR	X22, XZR, XZR

  (38) 0x12cb4 B	12dc0

  (38) 0x12cc0 CMP	X28, X12

  (38) 0x12cc4 B.CS	12ce0

  (38) 0x12cc8 ORR	X22, XZR, XZR

  (38) 0x12ccc MOVI	D20, #0

  (38) 0x12cd0 B	12d64

  (38) 0x12ce0 LDR	X8, [SP, #128]

  (38) 0x12ce4 DUP	Z18.S, Z21.S[0]

  (38) 0x12ce8 MOVI	V19.2D, #0

  (38) 0x12cec MOVI	V20.2D, #0

  (38) 0x12cf0 ORR	X24, XZR, X26

  (38) 0x12cf4 AND	X8, X28, X8

  (38) 0x12cf8 SUB	X22, X28, X8

  (38) 0x12cfc ADD	X23, X8, X14

  (38) 0x12d00 STR	X8, [SP, #224]

    (35) 0x12d04 LDR	Z0, [X24, MUL VL]

    (35) 0x12d08 LDR	Z1, [X24, #1, MUL VL]

    (35) 0x12d0c ORR	P0.B, P4/Z, P4.B, P4.B

    (35) 0x12d10 FSUB	Z0.S, Z0.S, Z18.S

    (35) 0x12d14 FSUB	Z16.S, Z1.S, Z18.S

    (35) 0x12d18 BL	10190

    (35) 0x12d1c ORR	Z17.D, Z0.D, Z0.D

    (35) 0x12d20 ORR	Z0.D, Z16.D, Z16.D

    (35) 0x12d24 ORR	P0.B, P4/Z, P4.B, P4.B

    (35) 0x12d28 BL	10190

    (35) 0x12d2c CNTH	X8, ALL

    (35) 0x12d30 FADD	Z19.S, Z17.S, Z19.S

    (35) 0x12d34 FADD	Z20.S, Z0.S, Z20.S

    (35) 0x12d38 ADDS	X23, X23, X8

    (35) 0x12d3c RDVL	X8, #2

    (35) 0x12d40 ADD	X24, X24, X8

    (35) 0x12d44 B.NE	12d04

  (38) 0x12d48 FADD	Z0.S, Z20.S, Z19.S

  (38) 0x12d4c LDR	X23, [SP, #168]

  (38) 0x12d50 LDR	X8, [SP, #224]

  (38) 0x12d54 FADDV	S20, P4, Z0.S

  (38) 0x12d58 CBZ	X8, 12df0

  (38) 0x12d5c CMP	X8, #4

  (38) 0x12d60 B.CC	12dc0

  (38) 0x12d64 LDR	X8, [SP, #176]

  (38) 0x12d68 MOVI	V16.2D, #0

  (38) 0x12d6c DUP	V17.4S, V21.S[0]

  (38) 0x12d70 UBFM	X24, X22, #62, #61

  (38) 0x12d74 MOV	V16.S[0], V20.S[0]

  (38) 0x12d78 SUB	X23, X22, X8

  (38) 0x12d7c AND	X22, X28, #0x0

    (40) 0x12d80 LDR	Q0, [X26, X24]

    (40) 0x12d84 FSUB	V0.4S, V0.4S, V17.4S

    (40) 0x12d88 BL	10040

    (40) 0x12d8c FADD	V16.4S, V0.4S, V16.4S

    (40) 0x12d90 ADDS	X23, X23, #4

    (40) 0x12d94 ADD	X24, X24, #16

    (40) 0x12d98 B.NE	12d80

  (38) 0x12d9c SUB	X8, X29, #16

  (38) 0x12da0 FADDP	V0.4S, V16.4S, V16.4S

  (38) 0x12da4 LDR	X23, [SP, #168]

  (38) 0x12da8 CMP	X28, X22

  (38) 0x12dac LDR	Z21, [X8, #508, MUL VL]

  (38) 0x12db0 FADDP	S20, V0.2S

  (38) 0x12db4 B.EQ	12df0

  (38) 0x12db8 HINT	#0

  (38) 0x12dbc HINT	#0

    (39) 0x12dc0 LDR	S0, [X26, X22,LSL #2]

    (39) 0x12dc4 SUB	X8, X29, #16

    (39) 0x12dc8 STR	Z20, [X8, #510, MUL VL]

    (39) 0x12dcc FSUB	S0, S0, S21

    (39) 0x12dd0 BL	101d0

    (39) 0x12dd4 SUB	X8, X29, #16

    (39) 0x12dd8 ADD	X22, X22, #1

    (39) 0x12ddc LDR	Z20, [X8, #510, MUL VL]

    (39) 0x12de0 LDR	Z21, [X8, #508, MUL VL]

    (39) 0x12de4 CMP	X28, X22

    (39) 0x12de8 FADD	S20, S0, S20

    (39) 0x12dec B.NE	12dc0

  (38) 0x12df0 CMP	X28, #4

  (38) 0x12df4 SUB	X8, X29, #16

  (38) 0x12df8 STR	X28, [SP, #224]

  (38) 0x12dfc STR	Z20, [X8, #510, MUL VL]

  (38) 0x12e00 B.CS	12e20

  (38) 0x12e04 ORR	X9, XZR, X28

  (38) 0x12e08 ORR	X28, XZR, X26

  (38) 0x12e0c LDR	X26, [SP, #184]

  (38) 0x12e10 ORR	X24, XZR, X23

  (38) 0x12e14 ORR	X22, XZR, XZR

  (38) 0x12e18 B	12f40

0x12e20 CNTH	X8, ALL

0x12e24 CMP	X28, X8

0x12e28 B.CS	12e40

0x12e2c LDR	X26, [SP, #184]

0x12e30 ORR	X9, XZR, X28

0x12e34 LDR	X28, [SP, #248]

0x12e38 ORR	X22, XZR, XZR

0x12e3c B	12eec

0x12e40 LDR	X9, [SP, #128]

0x12e44 LDR	X26, [SP, #184]

0x12e48 DUP	Z18.S, Z20.S[0]

0x12e4c DUP	Z19.S, Z21.S[0]

0x12e50 PTRUE	P4.B, ALL

0x12e54 PTRUE	P5.S, ALL

0x12e58 ORR	X23, XZR, XZR

0x12e5c AND	X10, X28, X9

0x12e60 LDR	X9, [SP, #216]

0x12e64 SUB	X22, X28, X10

0x12e68 LDR	X28, [SP, #248]

0x12e6c STR	X10, [SP, #104]

0x12e70 ADD	X24, X10, X9

0x12e74 HINT	#0

0x12e78 HINT	#0

0x12e7c HINT	#0

  (33) 0x12e80 LD1B	{Z0.B}, P4/Z, [X28, X23]

  (33) 0x12e84 ADD	X8, X28, X23

  (33) 0x12e88 ORR	P0.B, P5/Z, P5.B, P5.B

  (33) 0x12e8c LDR	Z1, [X8, #1, MUL VL]

  (33) 0x12e90 FSUB	Z0.S, Z0.S, Z19.S

  (33) 0x12e94 FSUB	Z16.S, Z1.S, Z19.S

  (33) 0x12e98 BL	10190

  (33) 0x12e9c ORR	Z17.D, Z0.D, Z0.D

  (33) 0x12ea0 ORR	Z0.D, Z16.D, Z16.D

  (33) 0x12ea4 ORR	P0.B, P5/Z, P5.B, P5.B

  (33) 0x12ea8 BL	10190

  (33) 0x12eac FDIV	Z0.S, P5/M, Z0.S, Z18.S

  (33) 0x12eb0 ADD	X8, X26, X23

  (33) 0x12eb4 STR	Z0, [X8, #1, MUL VL]

  (33) 0x12eb8 FDIV	Z17.S, P5/M, Z17.S, Z18.S

  (33) 0x12ebc CNTH	X8, ALL

  (33) 0x12ec0 ST1B	{Z17.B}, P4, [X26, X23]

  (33) 0x12ec4 ADDS	X24, X24, X8

  (33) 0x12ec8 RDVL	X8, #2

  (33) 0x12ecc ADD	X23, X23, X8

  (33) 0x12ed0 B.NE	12e80

0x12ed4 LDR	X24, [SP, #168]

0x12ed8 LDR	X9, [SP, #224]

0x12edc LDR	X8, [SP, #104]

0x12ee0 CBZ	X8, 12f74

0x12ee4 CMP	X8, #4

0x12ee8 B.CC	12f40

0x12eec LDR	X8, [SP, #176]

0x12ef0 DUP	V16.4S, V20.S[0]

0x12ef4 DUP	V17.4S, V21.S[0]

0x12ef8 UBFM	X24, X22, #62, #61

0x12efc SUB	X23, X22, X8

0x12f00 AND	X22, X9, #0x0

  (43) 0x12f04 LDR	Q0, [X28, X24]

  (43) 0x12f08 FSUB	V0.4S, V0.4S, V17.4S

  (43) 0x12f0c BL	10040

  (43) 0x12f10 FDIV	V0.4S, V0.4S, V16.4S

  (43) 0x12f14 ADDS	X23, X23, #4

  (43) 0x12f18 STR	Q0, [X26, X24]

  (43) 0x12f1c ADD	X24, X24, #16

  (43) 0x12f20 B.NE	12f04

0x12f24 SUB	X8, X29, #16

0x12f28 LDR	X9, [SP, #224]

0x12f2c LDR	X24, [SP, #168]

0x12f30 LDR	Z21, [X8, #508, MUL VL]

0x12f34 LDR	Z20, [X8, #510, MUL VL]

0x12f38 CMP	X9, X22

0x12f3c B.EQ	12f74

    (37) 0x12f40 LDR	S0, [X28, X22,LSL #2]

    (37) 0x12f44 ORR	X23, XZR, X9

    (37) 0x12f48 FSUB	S0, S0, S21

    (37) 0x12f4c BL	101d0

    (37) 0x12f50 SUB	X8, X29, #16

    (37) 0x12f54 ORR	X9, XZR, X23

    (37) 0x12f58 LDR	Z20, [X8, #510, MUL VL]

    (37) 0x12f5c LDR	Z21, [X8, #508, MUL VL]

    (37) 0x12f60 FDIV	S0, S0, S20

    (37) 0x12f64 STR	S0, [X26, X22,LSL #2]

    (37) 0x12f68 ADD	X22, X22, #1

    (37) 0x12f6c CMP	X23, X22

    (37) 0x12f70 B.NE	12f40

  (38) 0x12f74 LDR	X28, [SP, #192]

  (38) 0x12f78 LDR	X8, [SP, #280]

  (38) 0x12f7c ADD	X22, X28, #1

  (38) 0x12f80 CMP	X22, X8

  (38) 0x12f84 B.CS	12b60

  (38) 0x12f88 LDP	X9, X8, [SP, #112]

  (38) 0x12f8c MOVZ	X10, #4

  (38) 0x12f90 ORR	W1, WZR, WZR

  (38) 0x12f94 MADD	X9, X9, X28, X10

  (38) 0x12f98 SUB	W8, W8, W28

  (38) 0x12f9c UBFM	X8, X8, #62, #31

  (38) 0x12fa0 AND	X9, X9, #0x0

  (38) 0x12fa4 ADD	X2, X8, #4

  (38) 0x12fa8 ADD	X0, X27, X9

  (38) 0x12fac BL	100c0

  (38) 0x12fb0 SUB	X8, X29, #16

  (38) 0x12fb4 LDR	Z20, [X8, #510, MUL VL]

  (38) 0x12fb8 B	12b60

/home/eoseret/llm-attention/attention.cpp: 43 - 289

--------------------------------------------------------------------------------

43:    for (int row = 0; row < N; ++row) {

44:       const float *S_row = &S[row * N];

45:

46:       float max_val = -FLT_MAX;

47:       for (int idx = 0; idx <= row; ++idx) // vectorised

48:          if (S_row[idx] > max_val) max_val = S_row[idx];

49:

50:       float sum = 0.0f;

51: #pragma clang loop vectorize(enable)

52:       for (int idx = 0; idx <= row; ++idx) // vectorised

53:          sum += expf(S_row[idx] - max_val);

54:

55:       for (int idx = 0; idx <= row; ++idx)  //vectorised

56:          P[row * N + idx] = expf(S_row[idx] - max_val) / sum;

57:

58:       for (int idx = row + 1; idx < N; ++idx)

59:          P[row * N + idx] = 0.0f;

60:

61:       D[row] = sum;

[...]

289:    for (size_t r = 0; r < rept; r++)

Coverage (%)	Name	Module
►100.00+	__libc_start_call_main	libc.so.6
○	__libc_start_main	libc.so.6
○	_start	attention-native

min	med	avg	max

Percentile Index	10	20	30	40	50	60	70	80	90	100
Value

min	med	avg	max

Percentile Index	10	20	30	40	50	60	70	80	90	100
Value

Path /

Metric	Value
CQA speedup if no scalar integer	2.30
CQA speedup if FP arith vectorized	1.00
CQA speedup if fully vectorized	1.41
CQA speedup if no inter-iteration dependency	NA
CQA speedup if next bottleneck killed	1.03
Bottlenecks	P10,
Function	main
Source	attention.cpp:55-55
Source loop unroll info	NA
Source loop unroll confidence level	NA
Unroll/vectorization loop type	NA
Unroll factor	NA
CQA cycles	5.17
CQA cycles if no scalar integer	2.25
CQA cycles if FP arith vectorized	5.17
CQA cycles if fully vectorized	3.66
Front-end cycles	5.00
P0 cycles	2.50
P1 cycles	2.50
P2 cycles	4.00
P3 cycles	4.00
P4 cycles	4.00
P5 cycles	4.00
P6 cycles	1.00
P7 cycles	1.00
P8 cycles	1.00
P9 cycles	1.00
P10 cycles	5.17
P11 cycles	4.83
P12 cycles	5.00
P13 cycles	0.50
P14 cycles	0.50
DIV/SQRT cycles	0.00
Inter-iter dependencies cycles	NA
FE+BE cycles (UFS)	NA
Stall cycles (UFS)	NA
Nb insns	43.00
Nb uops	40.00
Nb loads	NA
Nb stores	1.00
Nb stack references	13.00
FLOP/cycle	0.00
Nb FLOP add-sub	0.00
Nb FLOP mul	0.00
Nb FLOP fma	0.00
Nb FLOP div	0.00
Nb FLOP rcp	0.00
Nb FLOP sqrt	0.00
Nb FLOP rsqrt	0.00
Bytes/cycle	12.39
Bytes prefetched	0.00
Bytes loaded	64.00
Bytes stored	0.00
Stride 0	NA
Stride 1	NA
Stride n	NA
Stride unknown	NA
Stride indirect	NA
Vectorization ratio all	27.27
Vectorization ratio load	25.00
Vectorization ratio store	0.00
Vectorization ratio mul	NA
Vectorization ratio add_sub	0.00
Vectorization ratio fma	NA
Vectorization ratio div_sqrt	NA
Vectorization ratio other	36.36
Vector-efficiency ratio all	47.73
Vector-efficiency ratio load	43.75
Vector-efficiency ratio store	25.00
Vector-efficiency ratio mul	NA
Vector-efficiency ratio add_sub	25.00
Vector-efficiency ratio fma	NA
Vector-efficiency ratio div_sqrt	NA
Vector-efficiency ratio other	56.82

Metric	Value
CQA speedup if no scalar integer	2.30
CQA speedup if FP arith vectorized	1.00
CQA speedup if fully vectorized	1.41
CQA speedup if no inter-iteration dependency	NA
CQA speedup if next bottleneck killed	1.03
Bottlenecks	P10,
Function	main
Source	attention.cpp:55-55
Source loop unroll info	NA
Source loop unroll confidence level	NA
Unroll/vectorization loop type	NA
Unroll factor	NA
CQA cycles	5.17
CQA cycles if no scalar integer	2.25
CQA cycles if FP arith vectorized	5.17
CQA cycles if fully vectorized	3.66
Front-end cycles	5.00
P0 cycles	2.50
P1 cycles	2.50
P2 cycles	4.00
P3 cycles	4.00
P4 cycles	4.00
P5 cycles	4.00
P6 cycles	1.00
P7 cycles	1.00
P8 cycles	1.00
P9 cycles	1.00
P10 cycles	5.17
P11 cycles	4.83
P12 cycles	5.00
P13 cycles	0.50
P14 cycles	0.50
DIV/SQRT cycles	0.00
Inter-iter dependencies cycles	NA
FE+BE cycles (UFS)	NA
Stall cycles (UFS)	NA
Nb insns	43.00
Nb uops	40.00
Nb loads	NA
Nb stores	1.00
Nb stack references	13.00
FLOP/cycle	0.00
Nb FLOP add-sub	0.00
Nb FLOP mul	0.00
Nb FLOP fma	0.00
Nb FLOP div	0.00
Nb FLOP rcp	0.00
Nb FLOP sqrt	0.00
Nb FLOP rsqrt	0.00
Bytes/cycle	12.39
Bytes prefetched	0.00
Bytes loaded	64.00
Bytes stored	0.00
Stride 0	NA
Stride 1	NA
Stride n	NA
Stride unknown	NA
Stride indirect	NA
Vectorization ratio all	27.27
Vectorization ratio load	25.00
Vectorization ratio store	0.00
Vectorization ratio mul	NA
Vectorization ratio add_sub	0.00
Vectorization ratio fma	NA
Vectorization ratio div_sqrt	NA
Vectorization ratio other	36.36
Vector-efficiency ratio all	47.73
Vector-efficiency ratio load	43.75
Vector-efficiency ratio store	25.00
Vector-efficiency ratio mul	NA
Vector-efficiency ratio add_sub	25.00
Vector-efficiency ratio fma	NA
Vector-efficiency ratio div_sqrt	NA
Vector-efficiency ratio other	56.82

Path /

Average path: Display a virtual path defined by average values of all real paths

Function	main
Source file and lines	attention.cpp:43-289
Module	attention-native

The loop is defined in /home/eoseret/llm-attention/attention.cpp:43-61,289.

Analyzed code is defined in /home/eoseret/llm-attention/attention.cpp:55.

Warnings:

Non-innermost loop: analyzing only self part (ignoring child loops).
Ignoring paths for analysis
Failed to get the number of paths
RecMII not computed since number of paths is unknown or > max_paths
Streams not analyzed since number of paths is unknown or > max_paths

gain
potential
hint
expert

Code clean check

Detected a slowdown caused by scalar integer instructions (typically used for address computation). By removing them, you can lower the cost of an iteration from 5.17 to 2.25 cycles (2.30x speedup).

Workaround

Try to reorganize arrays of structures to structures of arrays
Consider to permute loops (see vectorization gain report)

Vectorization

Your loop is probably not vectorized. Only 47% of vector register length is used (average across all VPU instructions). By vectorizing your loop, you can lower the cost of an iteration from 5.17 to 3.66 cycles (1.41x speedup).

Details

Store and arithmetical VPU instructions are used in scalar version (process only one data element in vector registers). Since your execution units are vector units, only a vectorized loop can use their full power.

Workaround

Try another compiler or update/tune your current one:
- recompile with ffast-math (included in Ofast) to extend loop vectorization to FP reductions.
Remove inter-iterations dependences from your loop and make it unit-stride:
- If your arrays have 2 or more dimensions, check whether elements are accessed contiguously and, otherwise, try to permute loops accordingly: C storage order is row-major: for(i) for(j) a[j][i] = b[j][i]; (slow, non stride 1) => for(i) for(j) a[i][j] = b[i][j]; (fast, stride 1)
- If your loop streams arrays of structures (AoS), try to use structures of arrays instead (SoA): for(i) a[i].x = b[i].x; (slow, non stride 1) => for(i) a.x[i] = b.x[i]; (fast, stride 1)

No data for this section

Matching between your loop (in the source code) and the binary loop

The binary loop does not contain any FP arithmetical operations. The binary loop is loading 64 bytes.

General properties

nb instructions	43
nb uops	40
loop length	172
used w registers	0
used x registers	10
used b registers	0
used h registers	0
used s registers	1
used d registers	0
used q registers	0
used v registers	4
used z registers	4
nb stack references	13

Front-end

MACRO FUSION NOT POSSIBLE FIT IN UOP CACHE

micro-operation queue	5.00 cycles
front end	5.00 cycles

Back-end

	P0	P1	P2	P3	P4	P5	P6	P7	P8	P9	P10	P11	P12	P13	P14
uops	2.50	2.50	4.00	4.00	4.00	4.00	1.00	1.00	1.00	1.00	5.17	4.83	5.00	0.50	0.50
cycles	2.50	2.50	4.00	4.00	4.00	4.00	1.00	1.00	1.00	1.00	5.17	4.83	5.00	0.50	0.50

Execution ports to units layout:

P0: BRU
P1: BRU
P2: ALU
P3: ALU
P4: ALU
P5: ALU
P6 (128 bits): VPU, FP store data, ALU, DIV/SQRT
P7 (128 bits): VPU, ALU, FP store data
P8 (128 bits): VPU, ALU, DIV/SQRT
P9 (128 bits): ALU, VPU
P10 (256 bits): store address, load
P11 (256 bits): store address, load
P12 (256 bits): load
P13 (64 bits): store data
P14 (64 bits): store data

Cycles executing div or sqrt instructions

Cycles summary

Front-end	5.00
Dispatch	5.17
Overall L1	5.17

Vectorization ratios

all	27%
load	25%
store	0%
mul	NA (no mul vectorizable/vectorized instructions)
add-sub	0%
fma	NA (no fma vectorizable/vectorized instructions)
div/sqrt	NA (no div/sqrt vectorizable/vectorized instructions)
other	36%

Vector efficiency ratios

all	47%
load	43%
store	25%
mul	NA (no mul vectorizable/vectorized instructions)
add-sub	25%
fma	NA (no fma vectorizable/vectorized instructions)
div/sqrt	NA (no div/sqrt vectorizable/vectorized instructions)
other	56%

Cycles and memory resources usage

Assuming all data fit into the L1 cache, each iteration of the binary loop takes 5.17 cycles. At this rate:

12% of peak load performance is reached (12.39 out of 96.00 bytes loaded per cycle (GB/s @ 1GHz))

Front-end bottlenecks

Found no such bottlenecks.

ASM code

In the binary file, the address of the loop is: 12e20

Instruction	Nb FU	P0	P1	P2	P3	P4	P5	P6	P7	P8	P9	P10	P11	P12	P13	P14	Latency	Recip. throughput	Vectorization
CNTH X8, ALL	1	0	0	0	0	1	0	0	0	0	0	0	0	0	0	0	2	1	N/A
CMP X28, X8	1	0	0	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	1	0.33	scal (25.0%)
B.CS 12e40 <main+0x1e40>	1	0.50	0.50	0	0	0	0	0	0	0	0	0	0	0	0	0	1	0.50	N/A
LDR X26, [SP, #184]	1	0	0	0	0	0	0	0	0	0	0	0.33	0.33	0.33	0	0	4	0.33	scal (25.0%)
ORR X9, XZR, X28	1	0	0	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	1	0.25	N/A
LDR X28, [SP, #248]	1	0	0	0	0	0	0	0	0	0	0	0.33	0.33	0.33	0	0	4	0.33	scal (25.0%)
ORR X22, XZR, XZR	1	0	0	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	1	0.25	N/A
B 12eec <main+0x1eec>	1	0.50	0.50	0	0	0	0	0	0	0	0	0	0	0	0	0	1	0.50	N/A
LDR X9, [SP, #128]	1	0	0	0	0	0	0	0	0	0	0	0.33	0.33	0.33	0	0	4	0.33	N/A
LDR X26, [SP, #184]	1	0	0	0	0	0	0	0	0	0	0	0.33	0.33	0.33	0	0	4	0.33	scal (25.0%)
DUP Z18.S, Z20.S[0]	1	0	0	0	0	0	0	0.50	0.50	0	0	0	0	0	0	0	2	0.50	vect (100.0%)
DUP Z19.S, Z21.S[0]	1	0	0	0	0	0	0	0.50	0.50	0	0	0	0	0	0	0	2	0.50	vect (100.0%)
PTRUE P4.B, ALL	1	0	0	0	0	1	0	0	0	0	0	0	0	0	0	0	2	1	scal (100.0%)
PTRUE P5.S, ALL	1	0	0	0	0	1	0	0	0	0	0	0	0	0	0	0	2	1	scal (100.0%)
ORR X23, XZR, XZR	1	0	0	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	1	0.25	scal (25.0%)
AND X10, X28, X9	1	0	0	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	1	0.25	scal (25.0%)
LDR X9, [SP, #216]	1	0	0	0	0	0	0	0	0	0	0	0.33	0.33	0.33	0	0	4	0.33	N/A
SUB X22, X28, X10	1	0	0	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	1	0.25	N/A
LDR X28, [SP, #248]	1	0	0	0	0	0	0	0	0	0	0	0.33	0.33	0.33	0	0	4	0.33	scal (25.0%)
STR X10, [SP, #104]	1	0	0	0	0	0	0	0	0	0	0	0.50	0.50	0	0.50	0.50	1	0.50	scal (25.0%)
ADD X24, X10, X9	1	0	0	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	1	0.25	scal (25.0%)
HINT #0																			N/A
HINT #0																			N/A
HINT #0																			N/A
LDR X24, [SP, #168]	1	0	0	0	0	0	0	0	0	0	0	0.33	0.33	0.33	0	0	4	0.33	scal (25.0%)
LDR X9, [SP, #224]	1	0	0	0	0	0	0	0	0	0	0	0.33	0.33	0.33	0	0	4	0.33	N/A
LDR X8, [SP, #104]	1	0	0	0	0	0	0	0	0	0	0	0.33	0.33	0.33	0	0	4	0.33	N/A
CBZ X8, 12f74 <main+0x1f74>	1	0.50	0.50	0	0	0	0	0	0	0	0	0	0	0	0	0	1	0.50	N/A
CMP X8, #4	1	0	0	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	1	0.33	scal (25.0%)
B.CC 12f40 <main+0x1f40>	1	0.50	0.50	0	0	0	0	0	0	0	0	0	0	0	0	0	1	0.50	N/A
LDR X8, [SP, #176]	1	0	0	0	0	0	0	0	0	0	0	0.33	0.33	0.33	0	0	4	0.33	N/A
DUP V16.4S, V20.S[0]	1	0	0	0	0	0	0	0.25	0.25	0.25	0.25	0	0	0	0	0	2	0.25	vect (50.0%)
DUP V17.4S, V21.S[0]	1	0	0	0	0	0	0	0.25	0.25	0.25	0.25	0	0	0	0	0	2	0.25	vect (50.0%)
UBFM X24, X22, #62, #61	1	0	0	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	1	0.25	scal (25.0%)
SUB X23, X22, X8	1	0	0	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	1	0.25	scal (25.0%)
AND X22, X9, #0x0	1	0	0	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	1	0.25	N/A
SUB X8, X29, #16	1	0	0	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	1	0.25	N/A
LDR X9, [SP, #224]	1	0	0	0	0	0	0	0	0	0	0	0.33	0.33	0.33	0	0	4	0.33	N/A
LDR X24, [SP, #168]	1	0	0	0	0	0	0	0	0	0	0	0.33	0.33	0.33	0	0	4	0.33	scal (25.0%)
LDR Z21, [X8, #508, MUL VL]	1	0	0	0	0	0	0	0	0	0	0	0.50	0.50	0	0	0	6	0.50	vect (100.0%)
LDR Z20, [X8, #510, MUL VL]	1	0	0	0	0	0	0	0	0	0	0	0.50	0.50	0	0	0	6	0.50	vect (100.0%)
CMP X9, X22	1	0	0	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	1	0.33	N/A
B.EQ 12f74 <main+0x1f74>	1	0.50	0.50	0	0	0	0	0	0	0	0	0	0	0	0	0	1	0.50	N/A

Function	main
Source file and lines	attention.cpp:43-289
Module	attention-native

The loop is defined in /home/eoseret/llm-attention/attention.cpp:43-61,289.

Analyzed code is defined in /home/eoseret/llm-attention/attention.cpp:55.

Warnings:

Non-innermost loop: analyzing only self part (ignoring child loops).
Ignoring paths for analysis
Failed to get the number of paths
RecMII not computed since number of paths is unknown or > max_paths
Streams not analyzed since number of paths is unknown or > max_paths

gain
potential
hint
expert

Code clean check

Detected a slowdown caused by scalar integer instructions (typically used for address computation). By removing them, you can lower the cost of an iteration from 5.17 to 2.25 cycles (2.30x speedup).

Workaround

Try to reorganize arrays of structures to structures of arrays
Consider to permute loops (see vectorization gain report)

Vectorization

Details

Workaround

Try another compiler or update/tune your current one:
- recompile with ffast-math (included in Ofast) to extend loop vectorization to FP reductions.
Remove inter-iterations dependences from your loop and make it unit-stride:
- If your arrays have 2 or more dimensions, check whether elements are accessed contiguously and, otherwise, try to permute loops accordingly: C storage order is row-major: for(i) for(j) a[j][i] = b[j][i]; (slow, non stride 1) => for(i) for(j) a[i][j] = b[i][j]; (fast, stride 1)
- If your loop streams arrays of structures (AoS), try to use structures of arrays instead (SoA): for(i) a[i].x = b[i].x; (slow, non stride 1) => for(i) a.x[i] = b.x[i]; (fast, stride 1)

No data for this section

Matching between your loop (in the source code) and the binary loop

The binary loop does not contain any FP arithmetical operations. The binary loop is loading 64 bytes.

General properties

nb instructions	43
nb uops	40
loop length	172
used w registers	0
used x registers	10
used b registers	0
used h registers	0
used s registers	1
used d registers	0
used q registers	0
used v registers	4
used z registers	4
nb stack references	13

Front-end

MACRO FUSION NOT POSSIBLE FIT IN UOP CACHE

micro-operation queue	5.00 cycles
front end	5.00 cycles

Back-end

	P0	P1	P2	P3	P4	P5	P6	P7	P8	P9	P10	P11	P12	P13	P14
uops	2.50	2.50	4.00	4.00	4.00	4.00	1.00	1.00	1.00	1.00	5.17	4.83	5.00	0.50	0.50
cycles	2.50	2.50	4.00	4.00	4.00	4.00	1.00	1.00	1.00	1.00	5.17	4.83	5.00	0.50	0.50

Execution ports to units layout:

P0: BRU
P1: BRU
P2: ALU
P3: ALU
P4: ALU
P5: ALU
P6 (128 bits): VPU, FP store data, ALU, DIV/SQRT
P7 (128 bits): VPU, ALU, FP store data
P8 (128 bits): VPU, ALU, DIV/SQRT
P9 (128 bits): ALU, VPU
P10 (256 bits): store address, load
P11 (256 bits): store address, load
P12 (256 bits): load
P13 (64 bits): store data
P14 (64 bits): store data

Cycles executing div or sqrt instructions

Cycles summary

Front-end	5.00
Dispatch	5.17
Overall L1	5.17

Vectorization ratios

all	27%
load	25%
store	0%
mul	NA (no mul vectorizable/vectorized instructions)
add-sub	0%
fma	NA (no fma vectorizable/vectorized instructions)
div/sqrt	NA (no div/sqrt vectorizable/vectorized instructions)
other	36%

Vector efficiency ratios

all	47%
load	43%
store	25%
mul	NA (no mul vectorizable/vectorized instructions)
add-sub	25%
fma	NA (no fma vectorizable/vectorized instructions)
div/sqrt	NA (no div/sqrt vectorizable/vectorized instructions)
other	56%

Cycles and memory resources usage

Assuming all data fit into the L1 cache, each iteration of the binary loop takes 5.17 cycles. At this rate:

12% of peak load performance is reached (12.39 out of 96.00 bytes loaded per cycle (GB/s @ 1GHz))

Front-end bottlenecks

Found no such bottlenecks.

ASM code

In the binary file, the address of the loop is: 12e20

Instruction	Nb FU	P0	P1	P2	P3	P4	P5	P6	P7	P8	P9	P10	P11	P12	P13	P14	Latency	Recip. throughput	Vectorization
CNTH X8, ALL	1	0	0	0	0	1	0	0	0	0	0	0	0	0	0	0	2	1	N/A
CMP X28, X8	1	0	0	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	1	0.33	scal (25.0%)
B.CS 12e40 <main+0x1e40>	1	0.50	0.50	0	0	0	0	0	0	0	0	0	0	0	0	0	1	0.50	N/A
LDR X26, [SP, #184]	1	0	0	0	0	0	0	0	0	0	0	0.33	0.33	0.33	0	0	4	0.33	scal (25.0%)
ORR X9, XZR, X28	1	0	0	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	1	0.25	N/A
LDR X28, [SP, #248]	1	0	0	0	0	0	0	0	0	0	0	0.33	0.33	0.33	0	0	4	0.33	scal (25.0%)
ORR X22, XZR, XZR	1	0	0	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	1	0.25	N/A
B 12eec <main+0x1eec>	1	0.50	0.50	0	0	0	0	0	0	0	0	0	0	0	0	0	1	0.50	N/A
LDR X9, [SP, #128]	1	0	0	0	0	0	0	0	0	0	0	0.33	0.33	0.33	0	0	4	0.33	N/A
LDR X26, [SP, #184]	1	0	0	0	0	0	0	0	0	0	0	0.33	0.33	0.33	0	0	4	0.33	scal (25.0%)
DUP Z18.S, Z20.S[0]	1	0	0	0	0	0	0	0.50	0.50	0	0	0	0	0	0	0	2	0.50	vect (100.0%)
DUP Z19.S, Z21.S[0]	1	0	0	0	0	0	0	0.50	0.50	0	0	0	0	0	0	0	2	0.50	vect (100.0%)
PTRUE P4.B, ALL	1	0	0	0	0	1	0	0	0	0	0	0	0	0	0	0	2	1	scal (100.0%)
PTRUE P5.S, ALL	1	0	0	0	0	1	0	0	0	0	0	0	0	0	0	0	2	1	scal (100.0%)
ORR X23, XZR, XZR	1	0	0	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	1	0.25	scal (25.0%)
AND X10, X28, X9	1	0	0	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	1	0.25	scal (25.0%)
LDR X9, [SP, #216]	1	0	0	0	0	0	0	0	0	0	0	0.33	0.33	0.33	0	0	4	0.33	N/A
SUB X22, X28, X10	1	0	0	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	1	0.25	N/A
LDR X28, [SP, #248]	1	0	0	0	0	0	0	0	0	0	0	0.33	0.33	0.33	0	0	4	0.33	scal (25.0%)
STR X10, [SP, #104]	1	0	0	0	0	0	0	0	0	0	0	0.50	0.50	0	0.50	0.50	1	0.50	scal (25.0%)
ADD X24, X10, X9	1	0	0	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	1	0.25	scal (25.0%)
HINT #0																			N/A
HINT #0																			N/A
HINT #0																			N/A
LDR X24, [SP, #168]	1	0	0	0	0	0	0	0	0	0	0	0.33	0.33	0.33	0	0	4	0.33	scal (25.0%)
LDR X9, [SP, #224]	1	0	0	0	0	0	0	0	0	0	0	0.33	0.33	0.33	0	0	4	0.33	N/A
LDR X8, [SP, #104]	1	0	0	0	0	0	0	0	0	0	0	0.33	0.33	0.33	0	0	4	0.33	N/A
CBZ X8, 12f74 <main+0x1f74>	1	0.50	0.50	0	0	0	0	0	0	0	0	0	0	0	0	0	1	0.50	N/A
CMP X8, #4	1	0	0	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	1	0.33	scal (25.0%)
B.CC 12f40 <main+0x1f40>	1	0.50	0.50	0	0	0	0	0	0	0	0	0	0	0	0	0	1	0.50	N/A
LDR X8, [SP, #176]	1	0	0	0	0	0	0	0	0	0	0	0.33	0.33	0.33	0	0	4	0.33	N/A
DUP V16.4S, V20.S[0]	1	0	0	0	0	0	0	0.25	0.25	0.25	0.25	0	0	0	0	0	2	0.25	vect (50.0%)
DUP V17.4S, V21.S[0]	1	0	0	0	0	0	0	0.25	0.25	0.25	0.25	0	0	0	0	0	2	0.25	vect (50.0%)
UBFM X24, X22, #62, #61	1	0	0	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	1	0.25	scal (25.0%)
SUB X23, X22, X8	1	0	0	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	1	0.25	scal (25.0%)
AND X22, X9, #0x0	1	0	0	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	1	0.25	N/A
SUB X8, X29, #16	1	0	0	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	1	0.25	N/A
LDR X9, [SP, #224]	1	0	0	0	0	0	0	0	0	0	0	0.33	0.33	0.33	0	0	4	0.33	N/A
LDR X24, [SP, #168]	1	0	0	0	0	0	0	0	0	0	0	0.33	0.33	0.33	0	0	4	0.33	scal (25.0%)
LDR Z21, [X8, #508, MUL VL]	1	0	0	0	0	0	0	0	0	0	0	0.50	0.50	0	0	0	6	0.50	vect (100.0%)
LDR Z20, [X8, #510, MUL VL]	1	0	0	0	0	0	0	0	0	0	0	0.50	0.50	0	0	0	6	0.50	vect (100.0%)
CMP X9, X22	1	0	0	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	1	0.33	N/A
B.EQ 12f74 <main+0x1f74>	1	0.50	0.50	0	0	0	0	0	0	0	0	0	0	0	0	0	1	0.50	N/A

Report Configuration

Code clean check

Workaround

Vectorization

Details

Workaround

Matching between your loop (in the source code) and the binary loop

General properties

Front-end

Back-end

Cycles summary

Vectorization ratios

Vector efficiency ratios

Cycles and memory resources usage

Front-end bottlenecks

ASM code

Code clean check

Workaround

Vectorization

Details

Workaround

Matching between your loop (in the source code) and the binary loop

General properties

Front-end

Back-end

Cycles summary

Vectorization ratios

Vector efficiency ratios

Cycles and memory resources usage

Front-end bottlenecks

ASM code