OV - attention-gcc-native - Loop 33

	Loop Id: 33	Module: attention-gcc-native	Source: attention_v2.cpp:44-61	Coverage: 0.10%

0x411a84 ORR	X0, XZR, X20

0x411a88 DUPM	Z30.S, #0xff7fffff

0x411a8c ORR	Z31.D, Z30.D, Z30.D

0x411a90 ORR	Z28.D, Z30.D, Z30.D

0x411a94 ORR	Z29.D, Z30.D, Z30.D

0x411a98 HINT	#0

0x411a9c HINT	#0

  (32) 0x411aa0 LD1W	{Z26.S}, P7/Z, [X0, MUL VL]

  (32) 0x411aa4 LD1W	{Z27.S}, P7/Z, [X0, #1, MUL VL]

  (32) 0x411aa8 ADD	W1, W1, W25

  (32) 0x411aac FMAXNM	Z29.S, P7/M, Z29.S, Z26.S

  (32) 0x411ab0 FMAXNM	Z28.S, P7/M, Z28.S, Z27.S

  (32) 0x411ab4 LD1W	{Z26.S}, P7/Z, [X0, #2, MUL VL]

  (32) 0x411ab8 LD1W	{Z27.S}, P7/Z, [X0, #3, MUL VL]

  (32) 0x411abc FMAXNM	Z31.S, P7/M, Z31.S, Z26.S

  (32) 0x411ac0 ADD	X0, X0, X26

  (32) 0x411ac4 FMAXNM	Z30.S, P7/M, Z30.S, Z27.S

  (32) 0x411ac8 CMP	W23, W1

  (32) 0x411acc B.CS	411aa0

0x411ad0 FMAXNM	Z31.S, P7/M, Z31.S, Z30.S

0x411ad4 FMAXNM	Z29.S, P7/M, Z29.S, Z28.S

0x411ad8 FMAXNM	Z31.S, P7/M, Z31.S, Z29.S

0x411adc CMP	W2, W1

0x411ae0 B.EQ	411b14

0x411b14 MOVI	V14.2S, #0

0x411b18 ORR	X28, XZR, X20

0x411b1c PTRUE	P7.B, ALL

0x411b20 FMAXNMV	S15, P7, Z31.S

  (30) 0x411b24 LDR	S0, [X28], #4

  (30) 0x411b28 FSUB	S0, S0, S15

  (30) 0x411b2c BL	410140

  (30) 0x411b30 FADD	S14, S14, S0

  (30) 0x411b34 CMP	X28, X22

  (30) 0x411b38 B.NE	411b24

0x411b3c MOVZ	X28, #0

  (31) 0x411b40 LDR	S0, [X20, X28,LSL #2]

  (31) 0x411b44 FSUB	S0, S0, S15

  (31) 0x411b48 BL	410140

  (31) 0x411b4c FDIV	S0, S0, S14

  (31) 0x411b50 STR	S0, [X21, X28,LSL #2]

  (31) 0x411b54 ADD	X28, X28, #1

  (31) 0x411b58 CMP	X19, X28

  (31) 0x411b5c B.NE	411b40

0x411b60 LDR	X0, [SP, #128]

0x411b64 ADD	W23, W23, #1

0x411b68 ADD	X3, X19, #1

0x411b6c LDR	X28, [SP, #144]

0x411b70 ADD	X20, X20, X0

0x411b74 ADD	X21, X21, X0

0x411b78 LDR	X0, [SP, #136]

0x411b7c ADD	X22, X22, X28

0x411b80 CMP	X19, X0

0x411b84 ADD	X27, X27, X0

0x411b88 LDR	W0, [SP, #164]

0x411b8c B.EQ	411bf4

0x411b90 LDR	X19, [SP, #112]

0x411b94 SUB	W2, W0, W3

0x411b98 MOVZ	W1, #0

0x411b9c ADD	X2, X2, #1

0x411ba0 STR	X3, [SP, #120]

0x411ba4 UBFM	X2, X2, #62, #61

0x411ba8 ORR	X0, XZR, X19

0x411bac BL	410060

0x411bb0 LDR	X0, [SP, #152]

0x411bb4 ADD	X1, X19, X28

0x411bb8 PTRUE	P7.B, ALL

0x411bbc LDR	X3, [SP, #120]

0x411bc0 STR	X1, [SP, #112]

0x411bc4 CNTB	X1, ALL

0x411bc8 ADD	W1, W1, #15

0x411bcc SUB	X0, X0, #8

0x411bd0 ORR	X19, XZR, X3

0x411bd4 STR	S14, [X0, X3,LSL #2]

0x411bd8 SUB	W0, W19, #1

0x411bdc ORR	W2, WZR, W19

0x411be0 CMP	W0, W1

0x411be4 MOVZ	W1, #0

0x411be8 B.CS	411a84

/home/eoseret/llm-attention/attention_v2.cpp: 44 - 61

--------------------------------------------------------------------------------

44:       const float *S_row = &S[row * N];

45:

46:       float max_val = -FLT_MAX;

47:       for (int idx = 0; idx <= row; ++idx) // vectorised

48:          if (S_row[idx] > max_val) max_val = S_row[idx];

49:

50:       float sum = 0.0f;

51: #pragma clang loop vectorize(enable)

52:       for (int idx = 0; idx <= row; ++idx) // vectorised

53:          sum += expf(S_row[idx] - max_val);

54:

55:       for (int idx = 0; idx <= row; ++idx)  //vectorised

56:          P[row * N + idx] = expf(S_row[idx] - max_val) / sum;

57:

58:       for (int idx = row + 1; idx < N; ++idx)

59:          P[row * N + idx] = 0.0f;

60:

61:       D[row] = sum;

Coverage (%)	Name	Source Location	Module
►100.00+	main	attention_v2.cpp:283	attention-gcc-native
○	__libc_start_call_main		libc.so.6
○	__libc_start_main		libc.so.6
○	_start	stl_vector.h:1920	attention-gcc-native

min	med	avg	max

Percentile Index	10	20	30	40	50	60	70	80	90	100
Value

min	med	avg	max

Percentile Index	10	20	30	40	50	60	70	80	90	100
Value

Path /

Metric	Value
CQA speedup if no scalar integer	1.56
CQA speedup if FP arith vectorized	1.00
CQA speedup if fully vectorized	1.54
CQA speedup if no inter-iteration dependency	NA
CQA speedup if next bottleneck killed	1.12
Bottlenecks	P2, P3, P4, P5,
Function	softmax(float const, float, float*, int)
Source	attention_v2.cpp:44-44,attention_v2.cpp:47-47,attention_v2.cpp:50-52,attention_v2.cpp:58-61
Source loop unroll info	NA
Source loop unroll confidence level	NA
Unroll/vectorization loop type	NA
Unroll factor	NA
CQA cycles	7.00
CQA cycles if no scalar integer	4.50
CQA cycles if FP arith vectorized	7.00
CQA cycles if fully vectorized	4.55
Front-end cycles	6.25
P0 cycles	2.00
P1 cycles	2.00
P2 cycles	7.00
P3 cycles	7.00
P4 cycles	7.00
P5 cycles	7.00
P6 cycles	4.50
P7 cycles	4.50
P8 cycles	1.00
P9 cycles	1.00
P10 cycles	3.50
P11 cycles	3.17
P12 cycles	3.33
P13 cycles	1.00
P14 cycles	1.00
DIV/SQRT cycles	0.00
Inter-iter dependencies cycles	NA
FE+BE cycles (UFS)	NA
Stall cycles (UFS)	NA
Nb insns	52.00
Nb uops	50.00
Nb loads	NA
Nb stores	3.00
Nb stack references	9.00
FLOP/cycle	0.00
Nb FLOP add-sub	0.00
Nb FLOP mul	0.00
Nb FLOP fma	0.00
Nb FLOP div	0.00
Nb FLOP rcp	0.00
Nb FLOP sqrt	0.00
Nb FLOP rsqrt	0.00
Bytes/cycle	0.86
Bytes prefetched	0.00
Bytes loaded	3.00
Bytes stored	3.00
Stride 0	NA
Stride 1	NA
Stride n	NA
Stride unknown	NA
Stride indirect	NA
Vectorization ratio all	26.67
Vectorization ratio load	0.00
Vectorization ratio store	0.00
Vectorization ratio mul	NA
Vectorization ratio add_sub	0.00
Vectorization ratio fma	NA
Vectorization ratio div_sqrt	NA
Vectorization ratio other	44.44
Vector-efficiency ratio all	47.50
Vector-efficiency ratio load	25.00
Vector-efficiency ratio store	20.83
Vector-efficiency ratio mul	NA
Vector-efficiency ratio add_sub	21.43
Vector-efficiency ratio fma	NA
Vector-efficiency ratio div_sqrt	NA
Vector-efficiency ratio other	64.58

Metric	Value
CQA speedup if no scalar integer	1.56
CQA speedup if FP arith vectorized	1.00
CQA speedup if fully vectorized	1.54
CQA speedup if no inter-iteration dependency	NA
CQA speedup if next bottleneck killed	1.12
Bottlenecks	P2, P3, P4, P5,
Function	softmax(float const, float, float*, int)
Source	attention_v2.cpp:44-44,attention_v2.cpp:47-47,attention_v2.cpp:50-52,attention_v2.cpp:58-61
Source loop unroll info	NA
Source loop unroll confidence level	NA
Unroll/vectorization loop type	NA
Unroll factor	NA
CQA cycles	7.00
CQA cycles if no scalar integer	4.50
CQA cycles if FP arith vectorized	7.00
CQA cycles if fully vectorized	4.55
Front-end cycles	6.25
P0 cycles	2.00
P1 cycles	2.00
P2 cycles	7.00
P3 cycles	7.00
P4 cycles	7.00
P5 cycles	7.00
P6 cycles	4.50
P7 cycles	4.50
P8 cycles	1.00
P9 cycles	1.00
P10 cycles	3.50
P11 cycles	3.17
P12 cycles	3.33
P13 cycles	1.00
P14 cycles	1.00
DIV/SQRT cycles	0.00
Inter-iter dependencies cycles	NA
FE+BE cycles (UFS)	NA
Stall cycles (UFS)	NA
Nb insns	52.00
Nb uops	50.00
Nb loads	NA
Nb stores	3.00
Nb stack references	9.00
FLOP/cycle	0.00
Nb FLOP add-sub	0.00
Nb FLOP mul	0.00
Nb FLOP fma	0.00
Nb FLOP div	0.00
Nb FLOP rcp	0.00
Nb FLOP sqrt	0.00
Nb FLOP rsqrt	0.00
Bytes/cycle	0.86
Bytes prefetched	0.00
Bytes loaded	3.00
Bytes stored	3.00
Stride 0	NA
Stride 1	NA
Stride n	NA
Stride unknown	NA
Stride indirect	NA
Vectorization ratio all	26.67
Vectorization ratio load	0.00
Vectorization ratio store	0.00
Vectorization ratio mul	NA
Vectorization ratio add_sub	0.00
Vectorization ratio fma	NA
Vectorization ratio div_sqrt	NA
Vectorization ratio other	44.44
Vector-efficiency ratio all	47.50
Vector-efficiency ratio load	25.00
Vector-efficiency ratio store	20.83
Vector-efficiency ratio mul	NA
Vector-efficiency ratio add_sub	21.43
Vector-efficiency ratio fma	NA
Vector-efficiency ratio div_sqrt	NA
Vector-efficiency ratio other	64.58

Path /

Average path: Display a virtual path defined by average values of all real paths

Function	softmax(float const, float, float*, int)
Source file and lines	attention_v2.cpp:44-61
Module	attention-gcc-native

The loop is defined in /home/eoseret/llm-attention/attention_v2.cpp:44-61.

Analyzed code is defined in /home/eoseret/llm-attention/attention_v2.cpp:44,47,50-52,58-61.

Warnings:

Non-innermost loop: analyzing only self part (ignoring child loops).
Ignoring paths for analysis
Failed to get the number of paths
RecMII not computed since number of paths is unknown or > max_paths
Streams not analyzed since number of paths is unknown or > max_paths

gain
potential
hint
expert

Code clean check

Detected a slowdown caused by scalar integer instructions (typically used for address computation). By removing them, you can lower the cost of an iteration from 7.00 to 4.50 cycles (1.56x speedup).

Workaround

Try to reorganize arrays of structures to structures of arrays
Consider to permute loops (see vectorization gain report)

Vectorization

Your loop is probably not vectorized. Only 47% of vector register length is used (average across all VPU instructions). By vectorizing your loop, you can lower the cost of an iteration from 7.00 to 4.55 cycles (1.54x speedup).

Details

Store and arithmetical VPU instructions are used in scalar version (process only one data element in vector registers). Since your execution units are vector units, only a vectorized loop can use their full power.

Workaround

Try another compiler or update/tune your current one
Remove inter-iterations dependences from your loop and make it unit-stride:
- If your arrays have 2 or more dimensions, check whether elements are accessed contiguously and, otherwise, try to permute loops accordingly: C storage order is row-major: for(i) for(j) a[j][i] = b[j][i]; (slow, non stride 1) => for(i) for(j) a[i][j] = b[i][j]; (fast, stride 1)
- If your loop streams arrays of structures (AoS), try to use structures of arrays instead (SoA): for(i) a[i].x = b[i].x; (slow, non stride 1) => for(i) a.x[i] = b.x[i]; (fast, stride 1)

No data for this section

CALL instructions

Detected function call instructions.

Details

Calling (and then returning from) a function prevents many compiler optimizations (like vectorization), breaks control flow (which reduces pipeline performance) and executes extra instructions to save/restore the registers used inside it, which is very expensive (dozens of cycles). Consider to inline small functions.

unknown: 1 occurrences►
- /home/eoseret/llm-attention/attention_v2.cpp:59

Matching between your loop (in the source code) and the binary loop

The binary loop does not contain any FP arithmetical operations. The binary loop is loading 3 bytes. The binary loop is storing 3 bytes.

General properties

nb instructions	52
nb uops	50
loop length	208
used w registers	7
used x registers	11
used b registers	0
used h registers	0
used s registers	3
used d registers	0
used q registers	0
used v registers	1
used z registers	4
nb stack references	9

Front-end

MACRO FUSION NOT POSSIBLE FIT IN UOP CACHE

micro-operation queue	6.25 cycles
front end	6.25 cycles

Back-end

	P0	P1	P2	P3	P4	P5	P6	P7	P8	P9	P10	P11	P12	P13	P14
uops	2.00	2.00	7.00	7.00	7.00	7.00	4.50	4.50	1.00	1.00	3.50	3.17	3.33	1.00	1.00
cycles	2.00	2.00	7.00	7.00	7.00	7.00	4.50	4.50	1.00	1.00	3.50	3.17	3.33	1.00	1.00

Execution ports to units layout:

P0: BRU
P1: BRU
P2: ALU
P3: ALU
P4: ALU
P5: ALU
P6 (128 bits): VPU, FP store data, ALU, DIV/SQRT
P7 (128 bits): VPU, ALU, FP store data
P8 (128 bits): VPU, ALU, DIV/SQRT
P9 (128 bits): ALU, VPU
P10 (256 bits): store address, load
P11 (256 bits): store address, load
P12 (256 bits): load
P13 (64 bits): store data
P14 (64 bits): store data

Cycles executing div or sqrt instructions

Cycles summary

Front-end	6.25
Dispatch	7.00
Overall L1	7.00

Vectorization ratios

INT

all	15%
load	0%
store	0%
mul	NA (no mul vectorizable/vectorized instructions)
add-sub	0%
fma	NA (no fma vectorizable/vectorized instructions)
other	28%

all	100%
load	NA (no load vectorizable/vectorized instructions)
store	NA (no store vectorizable/vectorized instructions)
mul	NA (no mul vectorizable/vectorized instructions)
add-sub	NA (no add-sub vectorizable/vectorized instructions)
fma	NA (no fma vectorizable/vectorized instructions)
div/sqrt	NA (no div/sqrt vectorizable/vectorized instructions)
other	100%

INT+FP

all	26%
load	0%
store	0%
mul	NA (no mul vectorizable/vectorized instructions)
add-sub	0%
fma	NA (no fma vectorizable/vectorized instructions)
div/sqrt	NA (no div/sqrt vectorizable/vectorized instructions)
other	44%

Vector efficiency ratios

INT

all	39%
load	25%
store	20%
mul	NA (no mul vectorizable/vectorized instructions)
add-sub	21%
fma	NA (no fma vectorizable/vectorized instructions)
other	54%

all	100%
load	NA (no load vectorizable/vectorized instructions)
store	NA (no store vectorizable/vectorized instructions)
mul	NA (no mul vectorizable/vectorized instructions)
add-sub	NA (no add-sub vectorizable/vectorized instructions)
fma	NA (no fma vectorizable/vectorized instructions)
div/sqrt	NA (no div/sqrt vectorizable/vectorized instructions)
other	100%

INT+FP

all	47%
load	25%
store	20%
mul	NA (no mul vectorizable/vectorized instructions)
add-sub	21%
fma	NA (no fma vectorizable/vectorized instructions)
div/sqrt	NA (no div/sqrt vectorizable/vectorized instructions)
other	64%

Cycles and memory resources usage

Assuming all data fit into the L1 cache, each iteration of the binary loop takes 7.00 cycles. At this rate:

0% of peak load performance is reached (0.43 out of 96.00 bytes loaded per cycle (GB/s @ 1GHz))
0% of peak store performance is reached (0.43 out of 48.00 bytes stored per cycle (GB/s @ 1GHz))

Front-end bottlenecks

Found no such bottlenecks.

ASM code

In the binary file, the address of the loop is: 411b14

Instruction	Nb FU	P0	P1	P2	P3	P4	P5	P6	P7	P8	P9	P10	P11	P12	P13	P14	Latency	Recip. throughput	Vectorization
ORR X0, XZR, X20	1	0	0	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	1	0.25	N/A
DUPM Z30.S, #0xff7fffff	1	0	0	0	0	0	0	0.50	0.50	0	0	0	0	0	0	0	2	0.50	vect (100.0%)
ORR Z31.D, Z30.D, Z30.D	1	0	0	0	0	0	0	0.50	0.50	0	0	0	0	0	0	0	2	0.50	vect (100.0%)
ORR Z28.D, Z30.D, Z30.D	1	0	0	0	0	0	0	0.50	0.50	0	0	0	0	0	0	0	2	0.50	vect (100.0%)
ORR Z29.D, Z30.D, Z30.D	1	0	0	0	0	0	0	0.50	0.50	0	0	0	0	0	0	0	2	0.50	vect (100.0%)
HINT #0																			N/A
HINT #0																			N/A
FMAXNM Z31.S, P7/M, Z31.S, Z30.S	1	0	0	0	0	0	0	0.50	0.50	0	0	0	0	0	0	0	2	0.50	vect (100.0%)
FMAXNM Z29.S, P7/M, Z29.S, Z28.S	1	0	0	0	0	0	0	0.50	0.50	0	0	0	0	0	0	0	2	0.50	vect (100.0%)
FMAXNM Z31.S, P7/M, Z31.S, Z29.S	1	0	0	0	0	0	0	0.50	0.50	0	0	0	0	0	0	0	2	0.50	vect (100.0%)
CMP W2, W1	1	0	0	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	1	0.33	scal (12.5%)
B.EQ 411b14 <_Z7softmaxPKfPfS1_i+0x134>	1	0.50	0.50	0	0	0	0	0	0	0	0	0	0	0	0	0	1	0.50	N/A
MOVI V14.2S, #0	1	0	0	0	0	0	0	0.25	0.25	0.25	0.25	0	0	0	0	0	2	0.25	scal (12.5%)
ORR X28, XZR, X20	1	0	0	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	1	0.25	scal (25.0%)
PTRUE P7.B, ALL	1	0	0	0	0	1	0	0	0	0	0	0	0	0	0	0	2	1	scal (100.0%)
FMAXNMV S15, P7, Z31.S	1	0	0	0	0	0	0	0.75	0.75	0.25	0.25	0	0	0	0	0	11	2.50	vect (100.0%)
MOVZ X28, #0	1	0	0	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	1	0.25	scal (25.0%)
LDR X0, [SP, #128]	1	0	0	0	0	0	0	0	0	0	0	0.33	0.33	0.33	0	0	4	0.33	N/A
ADD W23, W23, #1	1	0	0	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	1	0.25	scal (12.5%)
ADD X3, X19, #1	1	0	0	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	1	0.25	N/A
LDR X28, [SP, #144]	1	0	0	0	0	0	0	0	0	0	0	0.33	0.33	0.33	0	0	4	0.33	scal (25.0%)
ADD X20, X20, X0	1	0	0	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	1	0.25	scal (25.0%)
ADD X21, X21, X0	1	0	0	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	1	0.25	scal (25.0%)
LDR X0, [SP, #136]	1	0	0	0	0	0	0	0	0	0	0	0.33	0.33	0.33	0	0	4	0.33	N/A
ADD X22, X22, X28	1	0	0	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	1	0.25	scal (25.0%)
CMP X19, X0	1	0	0	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	1	0.33	scal (25.0%)
ADD X27, X27, X0	1	0	0	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	1	0.25	scal (25.0%)
LDR W0, [SP, #164]	1	0	0	0	0	0	0	0	0	0	0	0.33	0.33	0.33	0	0	4	0.33	N/A
B.EQ 411bf4 <_Z7softmaxPKfPfS1_i+0x214>	1	0.50	0.50	0	0	0	0	0	0	0	0	0	0	0	0	0	1	0.50	N/A
LDR X19, [SP, #112]	1	0	0	0	0	0	0	0	0	0	0	0.33	0.33	0.33	0	0	4	0.33	scal (25.0%)
SUB W2, W0, W3	1	0	0	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	1	0.25	scal (12.5%)
MOVZ W1, #0	1	0	0	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	1	0.25	N/A
ADD X2, X2, #1	1	0	0	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	1	0.25	scal (25.0%)
STR X3, [SP, #120]	1	0	0	0	0	0	0	0	0	0	0	0.50	0.50	0	0.50	0.50	1	0.50	scal (25.0%)
UBFM X2, X2, #62, #61	1	0	0	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	1	0.25	scal (25.0%)
ORR X0, XZR, X19	1	0	0	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	1	0.25	N/A
BL 410060 <@plt_start@+0x40>	1	0.50	0.50	0.50	0.50	0	0	0	0	0	0	0	0	0	0	0	1	0.50	N/A
LDR X0, [SP, #152]	1	0	0	0	0	0	0	0	0	0	0	0.33	0.33	0.33	0	0	4	0.33	N/A
ADD X1, X19, X28	1	0	0	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	1	0.25	N/A
PTRUE P7.B, ALL	1	0	0	0	0	1	0	0	0	0	0	0	0	0	0	0	2	1	scal (100.0%)
LDR X3, [SP, #120]	1	0	0	0	0	0	0	0	0	0	0	0.33	0.33	0.33	0	0	4	0.33	N/A
STR X1, [SP, #112]	1	0	0	0	0	0	0	0	0	0	0	0.50	0.50	0	0.50	0.50	1	0.50	scal (25.0%)
CNTB X1, ALL	1	0	0	0	0	1	0	0	0	0	0	0	0	0	0	0	2	1	N/A
ADD W1, W1, #15	1	0	0	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	1	0.25	N/A
SUB X0, X0, #8	1	0	0	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	1	0.25	N/A
ORR X19, XZR, X3	1	0	0	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	1	0.25	scal (25.0%)
STR S14, [X0, X3,LSL #2]	1	0	0	0	0	0	0	0.50	0.50	0	0	0.50	0.50	0	0	0	2	0.50	scal (12.5%)
SUB W0, W19, #1	1	0	0	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	1	0.25	N/A
ORR W2, WZR, W19	1	0	0	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	1	0.25	scal (12.5%)
CMP W0, W1	1	0	0	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	1	0.33	N/A
MOVZ W1, #0	1	0	0	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	1	0.25	N/A
B.CS 411a84 <_Z7softmaxPKfPfS1_i+0xa4>	1	0.50	0.50	0	0	0	0	0	0	0	0	0	0	0	0	0	1	0.50	N/A

Function	softmax(float const, float, float*, int)
Source file and lines	attention_v2.cpp:44-61
Module	attention-gcc-native

The loop is defined in /home/eoseret/llm-attention/attention_v2.cpp:44-61.

Analyzed code is defined in /home/eoseret/llm-attention/attention_v2.cpp:44,47,50-52,58-61.

Warnings:

Non-innermost loop: analyzing only self part (ignoring child loops).
Ignoring paths for analysis
Failed to get the number of paths
RecMII not computed since number of paths is unknown or > max_paths
Streams not analyzed since number of paths is unknown or > max_paths

gain
potential
hint
expert

Code clean check

Detected a slowdown caused by scalar integer instructions (typically used for address computation). By removing them, you can lower the cost of an iteration from 7.00 to 4.50 cycles (1.56x speedup).

Workaround

Try to reorganize arrays of structures to structures of arrays
Consider to permute loops (see vectorization gain report)

Vectorization

Details

Workaround

Try another compiler or update/tune your current one
Remove inter-iterations dependences from your loop and make it unit-stride:
- If your arrays have 2 or more dimensions, check whether elements are accessed contiguously and, otherwise, try to permute loops accordingly: C storage order is row-major: for(i) for(j) a[j][i] = b[j][i]; (slow, non stride 1) => for(i) for(j) a[i][j] = b[i][j]; (fast, stride 1)
- If your loop streams arrays of structures (AoS), try to use structures of arrays instead (SoA): for(i) a[i].x = b[i].x; (slow, non stride 1) => for(i) a.x[i] = b.x[i]; (fast, stride 1)

No data for this section

CALL instructions

Detected function call instructions.

Details

unknown: 1 occurrences►
- /home/eoseret/llm-attention/attention_v2.cpp:59

Matching between your loop (in the source code) and the binary loop

The binary loop does not contain any FP arithmetical operations. The binary loop is loading 3 bytes. The binary loop is storing 3 bytes.

General properties

nb instructions	52
nb uops	50
loop length	208
used w registers	7
used x registers	11
used b registers	0
used h registers	0
used s registers	3
used d registers	0
used q registers	0
used v registers	1
used z registers	4
nb stack references	9

Front-end

MACRO FUSION NOT POSSIBLE FIT IN UOP CACHE

micro-operation queue	6.25 cycles
front end	6.25 cycles

Back-end

	P0	P1	P2	P3	P4	P5	P6	P7	P8	P9	P10	P11	P12	P13	P14
uops	2.00	2.00	7.00	7.00	7.00	7.00	4.50	4.50	1.00	1.00	3.50	3.17	3.33	1.00	1.00
cycles	2.00	2.00	7.00	7.00	7.00	7.00	4.50	4.50	1.00	1.00	3.50	3.17	3.33	1.00	1.00

Execution ports to units layout:

P0: BRU
P1: BRU
P2: ALU
P3: ALU
P4: ALU
P5: ALU
P6 (128 bits): VPU, FP store data, ALU, DIV/SQRT
P7 (128 bits): VPU, ALU, FP store data
P8 (128 bits): VPU, ALU, DIV/SQRT
P9 (128 bits): ALU, VPU
P10 (256 bits): store address, load
P11 (256 bits): store address, load
P12 (256 bits): load
P13 (64 bits): store data
P14 (64 bits): store data

Cycles executing div or sqrt instructions

Cycles summary

Front-end	6.25
Dispatch	7.00
Overall L1	7.00

Vectorization ratios

INT

all	15%
load	0%
store	0%
mul	NA (no mul vectorizable/vectorized instructions)
add-sub	0%
fma	NA (no fma vectorizable/vectorized instructions)
other	28%

all	100%
load	NA (no load vectorizable/vectorized instructions)
store	NA (no store vectorizable/vectorized instructions)
mul	NA (no mul vectorizable/vectorized instructions)
add-sub	NA (no add-sub vectorizable/vectorized instructions)
fma	NA (no fma vectorizable/vectorized instructions)
div/sqrt	NA (no div/sqrt vectorizable/vectorized instructions)
other	100%

INT+FP

all	26%
load	0%
store	0%
mul	NA (no mul vectorizable/vectorized instructions)
add-sub	0%
fma	NA (no fma vectorizable/vectorized instructions)
div/sqrt	NA (no div/sqrt vectorizable/vectorized instructions)
other	44%

Vector efficiency ratios

INT

all	39%
load	25%
store	20%
mul	NA (no mul vectorizable/vectorized instructions)
add-sub	21%
fma	NA (no fma vectorizable/vectorized instructions)
other	54%

all	100%
load	NA (no load vectorizable/vectorized instructions)
store	NA (no store vectorizable/vectorized instructions)
mul	NA (no mul vectorizable/vectorized instructions)
add-sub	NA (no add-sub vectorizable/vectorized instructions)
fma	NA (no fma vectorizable/vectorized instructions)
div/sqrt	NA (no div/sqrt vectorizable/vectorized instructions)
other	100%

INT+FP

all	47%
load	25%
store	20%
mul	NA (no mul vectorizable/vectorized instructions)
add-sub	21%
fma	NA (no fma vectorizable/vectorized instructions)
div/sqrt	NA (no div/sqrt vectorizable/vectorized instructions)
other	64%

Cycles and memory resources usage

Assuming all data fit into the L1 cache, each iteration of the binary loop takes 7.00 cycles. At this rate:

0% of peak load performance is reached (0.43 out of 96.00 bytes loaded per cycle (GB/s @ 1GHz))
0% of peak store performance is reached (0.43 out of 48.00 bytes stored per cycle (GB/s @ 1GHz))

Front-end bottlenecks

Found no such bottlenecks.

ASM code

In the binary file, the address of the loop is: 411b14

Instruction	Nb FU	P0	P1	P2	P3	P4	P5	P6	P7	P8	P9	P10	P11	P12	P13	P14	Latency	Recip. throughput	Vectorization
ORR X0, XZR, X20	1	0	0	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	1	0.25	N/A
DUPM Z30.S, #0xff7fffff	1	0	0	0	0	0	0	0.50	0.50	0	0	0	0	0	0	0	2	0.50	vect (100.0%)
ORR Z31.D, Z30.D, Z30.D	1	0	0	0	0	0	0	0.50	0.50	0	0	0	0	0	0	0	2	0.50	vect (100.0%)
ORR Z28.D, Z30.D, Z30.D	1	0	0	0	0	0	0	0.50	0.50	0	0	0	0	0	0	0	2	0.50	vect (100.0%)
ORR Z29.D, Z30.D, Z30.D	1	0	0	0	0	0	0	0.50	0.50	0	0	0	0	0	0	0	2	0.50	vect (100.0%)
HINT #0																			N/A
HINT #0																			N/A
FMAXNM Z31.S, P7/M, Z31.S, Z30.S	1	0	0	0	0	0	0	0.50	0.50	0	0	0	0	0	0	0	2	0.50	vect (100.0%)
FMAXNM Z29.S, P7/M, Z29.S, Z28.S	1	0	0	0	0	0	0	0.50	0.50	0	0	0	0	0	0	0	2	0.50	vect (100.0%)
FMAXNM Z31.S, P7/M, Z31.S, Z29.S	1	0	0	0	0	0	0	0.50	0.50	0	0	0	0	0	0	0	2	0.50	vect (100.0%)
CMP W2, W1	1	0	0	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	1	0.33	scal (12.5%)
B.EQ 411b14 <_Z7softmaxPKfPfS1_i+0x134>	1	0.50	0.50	0	0	0	0	0	0	0	0	0	0	0	0	0	1	0.50	N/A
MOVI V14.2S, #0	1	0	0	0	0	0	0	0.25	0.25	0.25	0.25	0	0	0	0	0	2	0.25	scal (12.5%)
ORR X28, XZR, X20	1	0	0	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	1	0.25	scal (25.0%)
PTRUE P7.B, ALL	1	0	0	0	0	1	0	0	0	0	0	0	0	0	0	0	2	1	scal (100.0%)
FMAXNMV S15, P7, Z31.S	1	0	0	0	0	0	0	0.75	0.75	0.25	0.25	0	0	0	0	0	11	2.50	vect (100.0%)
MOVZ X28, #0	1	0	0	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	1	0.25	scal (25.0%)
LDR X0, [SP, #128]	1	0	0	0	0	0	0	0	0	0	0	0.33	0.33	0.33	0	0	4	0.33	N/A
ADD W23, W23, #1	1	0	0	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	1	0.25	scal (12.5%)
ADD X3, X19, #1	1	0	0	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	1	0.25	N/A
LDR X28, [SP, #144]	1	0	0	0	0	0	0	0	0	0	0	0.33	0.33	0.33	0	0	4	0.33	scal (25.0%)
ADD X20, X20, X0	1	0	0	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	1	0.25	scal (25.0%)
ADD X21, X21, X0	1	0	0	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	1	0.25	scal (25.0%)
LDR X0, [SP, #136]	1	0	0	0	0	0	0	0	0	0	0	0.33	0.33	0.33	0	0	4	0.33	N/A
ADD X22, X22, X28	1	0	0	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	1	0.25	scal (25.0%)
CMP X19, X0	1	0	0	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	1	0.33	scal (25.0%)
ADD X27, X27, X0	1	0	0	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	1	0.25	scal (25.0%)
LDR W0, [SP, #164]	1	0	0	0	0	0	0	0	0	0	0	0.33	0.33	0.33	0	0	4	0.33	N/A
B.EQ 411bf4 <_Z7softmaxPKfPfS1_i+0x214>	1	0.50	0.50	0	0	0	0	0	0	0	0	0	0	0	0	0	1	0.50	N/A
LDR X19, [SP, #112]	1	0	0	0	0	0	0	0	0	0	0	0.33	0.33	0.33	0	0	4	0.33	scal (25.0%)
SUB W2, W0, W3	1	0	0	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	1	0.25	scal (12.5%)
MOVZ W1, #0	1	0	0	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	1	0.25	N/A
ADD X2, X2, #1	1	0	0	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	1	0.25	scal (25.0%)
STR X3, [SP, #120]	1	0	0	0	0	0	0	0	0	0	0	0.50	0.50	0	0.50	0.50	1	0.50	scal (25.0%)
UBFM X2, X2, #62, #61	1	0	0	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	1	0.25	scal (25.0%)
ORR X0, XZR, X19	1	0	0	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	1	0.25	N/A
BL 410060 <@plt_start@+0x40>	1	0.50	0.50	0.50	0.50	0	0	0	0	0	0	0	0	0	0	0	1	0.50	N/A
LDR X0, [SP, #152]	1	0	0	0	0	0	0	0	0	0	0	0.33	0.33	0.33	0	0	4	0.33	N/A
ADD X1, X19, X28	1	0	0	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	1	0.25	N/A
PTRUE P7.B, ALL	1	0	0	0	0	1	0	0	0	0	0	0	0	0	0	0	2	1	scal (100.0%)
LDR X3, [SP, #120]	1	0	0	0	0	0	0	0	0	0	0	0.33	0.33	0.33	0	0	4	0.33	N/A
STR X1, [SP, #112]	1	0	0	0	0	0	0	0	0	0	0	0.50	0.50	0	0.50	0.50	1	0.50	scal (25.0%)
CNTB X1, ALL	1	0	0	0	0	1	0	0	0	0	0	0	0	0	0	0	2	1	N/A
ADD W1, W1, #15	1	0	0	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	1	0.25	N/A
SUB X0, X0, #8	1	0	0	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	1	0.25	N/A
ORR X19, XZR, X3	1	0	0	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	1	0.25	scal (25.0%)
STR S14, [X0, X3,LSL #2]	1	0	0	0	0	0	0	0.50	0.50	0	0	0.50	0.50	0	0	0	2	0.50	scal (12.5%)
SUB W0, W19, #1	1	0	0	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	1	0.25	N/A
ORR W2, WZR, W19	1	0	0	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	1	0.25	scal (12.5%)
CMP W0, W1	1	0	0	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	1	0.33	N/A
MOVZ W1, #0	1	0	0	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	1	0.25	N/A
B.CS 411a84 <_Z7softmaxPKfPfS1_i+0xa4>	1	0.50	0.50	0	0	0	0	0	0	0	0	0	0	0	0	0	1	0.50	N/A

Report Configuration

Code clean check

Workaround

Vectorization

Details

Workaround

CALL instructions

Details

Matching between your loop (in the source code) and the binary loop

General properties

Front-end

Back-end

Cycles summary

Vectorization ratios

Vector efficiency ratios

Cycles and memory resources usage

Front-end bottlenecks

ASM code

Code clean check

Workaround

Vectorization

Details

Workaround

CALL instructions

Details

Matching between your loop (in the source code) and the binary loop

General properties

Front-end

Back-end

Cycles summary

Vectorization ratios

Vector efficiency ratios

Cycles and memory resources usage

Front-end bottlenecks

ASM code