DPDK 23.11.2
Loading...
Searching...
No Matches
rte_swx_pipeline_internal.h
1/* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2021 Intel Corporation
3 */
4#ifndef __INCLUDE_RTE_SWX_PIPELINE_INTERNAL_H__
5#define __INCLUDE_RTE_SWX_PIPELINE_INTERNAL_H__
6
7#include <inttypes.h>
8#include <string.h>
9#include <sys/queue.h>
10
11#include <rte_bitops.h>
12#include <rte_byteorder.h>
13#include <rte_common.h>
14#include <rte_cycles.h>
15#include <rte_prefetch.h>
16#include <rte_meter.h>
17
20#include <rte_swx_pipeline.h>
21#include <rte_swx_ctl.h>
22
23#ifndef TRACE_LEVEL
24#define TRACE_LEVEL 0
25#endif
26
27#if TRACE_LEVEL
28#define TRACE(...) printf(__VA_ARGS__)
29#else
30#define TRACE(...)
31#endif
32
33/*
34 * Environment.
35 */
36#define ntoh64(x) rte_be_to_cpu_64(x)
37#define hton64(x) rte_cpu_to_be_64(x)
38
39/*
40 * Struct.
41 */
42struct field {
43 char name[RTE_SWX_NAME_SIZE];
44 uint32_t n_bits;
45 uint32_t offset;
46 int var_size;
47};
48
49struct struct_type {
50 TAILQ_ENTRY(struct_type) node;
51 char name[RTE_SWX_NAME_SIZE];
52 struct field *fields;
53 uint32_t n_fields;
54 uint32_t n_bits;
55 uint32_t n_bits_min;
56 int var_size;
57};
58
59TAILQ_HEAD(struct_type_tailq, struct_type);
60
61/*
62 * Input port.
63 */
64struct port_in_type {
65 TAILQ_ENTRY(port_in_type) node;
66 char name[RTE_SWX_NAME_SIZE];
67 struct rte_swx_port_in_ops ops;
68};
69
70TAILQ_HEAD(port_in_type_tailq, port_in_type);
71
72struct port_in {
73 TAILQ_ENTRY(port_in) node;
74 struct port_in_type *type;
75 void *obj;
76 uint32_t id;
77};
78
79TAILQ_HEAD(port_in_tailq, port_in);
80
81struct port_in_runtime {
83 void *obj;
84};
85
86/*
87 * Output port.
88 */
89struct port_out_type {
90 TAILQ_ENTRY(port_out_type) node;
91 char name[RTE_SWX_NAME_SIZE];
92 struct rte_swx_port_out_ops ops;
93};
94
95TAILQ_HEAD(port_out_type_tailq, port_out_type);
96
97struct port_out {
98 TAILQ_ENTRY(port_out) node;
99 struct port_out_type *type;
100 void *obj;
101 uint32_t id;
102};
103
104TAILQ_HEAD(port_out_tailq, port_out);
105
106struct port_out_runtime {
108 rte_swx_port_out_pkt_fast_clone_tx_t pkt_fast_clone_tx;
111 void *obj;
112};
113
114/*
115 * Packet mirroring.
116 */
117struct mirroring_session {
118 uint32_t port_id;
119 int fast_clone;
120 uint32_t truncation_length;
121};
122
123/*
124 * Extern object.
125 */
126struct extern_type_member_func {
127 TAILQ_ENTRY(extern_type_member_func) node;
128 char name[RTE_SWX_NAME_SIZE];
130 uint32_t id;
131};
132
133TAILQ_HEAD(extern_type_member_func_tailq, extern_type_member_func);
134
135struct extern_type {
136 TAILQ_ENTRY(extern_type) node;
137 char name[RTE_SWX_NAME_SIZE];
138 struct struct_type *mailbox_struct_type;
141 struct extern_type_member_func_tailq funcs;
142 uint32_t n_funcs;
143};
144
145TAILQ_HEAD(extern_type_tailq, extern_type);
146
147struct extern_obj {
148 TAILQ_ENTRY(extern_obj) node;
149 char name[RTE_SWX_NAME_SIZE];
150 struct extern_type *type;
151 void *obj;
152 uint32_t struct_id;
153 uint32_t id;
154};
155
156TAILQ_HEAD(extern_obj_tailq, extern_obj);
157
158#ifndef RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX
159#define RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX 8
160#endif
161
162struct extern_obj_runtime {
163 void *obj;
164 uint8_t *mailbox;
165 rte_swx_extern_type_member_func_t funcs[RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX];
166};
167
168/*
169 * Extern function.
170 */
171struct extern_func {
172 TAILQ_ENTRY(extern_func) node;
173 char name[RTE_SWX_NAME_SIZE];
174 struct struct_type *mailbox_struct_type;
176 uint32_t struct_id;
177 uint32_t id;
178};
179
180TAILQ_HEAD(extern_func_tailq, extern_func);
181
182struct extern_func_runtime {
183 uint8_t *mailbox;
185};
186
187/*
188 * Hash function.
189 */
190struct hash_func {
191 TAILQ_ENTRY(hash_func) node;
192 char name[RTE_SWX_NAME_SIZE];
194 uint32_t id;
195};
196
197TAILQ_HEAD(hash_func_tailq, hash_func);
198
199struct hash_func_runtime {
201};
202
203/*
204 * RSS.
205 */
206struct rss {
207 TAILQ_ENTRY(rss) node;
208 char name[RTE_SWX_NAME_SIZE];
209 uint32_t id;
210};
211
212TAILQ_HEAD(rss_tailq, rss);
213
214struct rss_runtime {
215 uint32_t key_size; /* key size in bytes. */
216 uint8_t key[0]; /* key. */
217};
218
219/*
220 * Header.
221 */
222struct header {
223 TAILQ_ENTRY(header) node;
224 char name[RTE_SWX_NAME_SIZE];
225 struct struct_type *st;
226 uint32_t struct_id;
227 uint32_t id;
228};
229
230TAILQ_HEAD(header_tailq, header);
231
232struct header_runtime {
233 uint8_t *ptr0;
234 uint32_t n_bytes;
235};
236
237struct header_out_runtime {
238 uint8_t *ptr0;
239 uint8_t *ptr;
240 uint32_t n_bytes;
241};
242
243/*
244 * Instruction.
245 */
246
247/* Packet headers are always in Network Byte Order (NBO), i.e. big endian.
248 * Packet meta-data fields are always assumed to be in Host Byte Order (HBO).
249 * Table entry fields can be in either NBO or HBO; they are assumed to be in HBO
250 * when transferred to packet meta-data and in NBO when transferred to packet
251 * headers.
252 */
253
254/* Notation conventions:
255 * -Header field: H = h.header.field (dst/src)
256 * -Meta-data field: M = m.field (dst/src)
257 * -Extern object mailbox field: E = e.field (dst/src)
258 * -Extern function mailbox field: F = f.field (dst/src)
259 * -Table action data field: T = t.field (src only)
260 * -Immediate value: I = 32-bit unsigned value (src only)
261 */
262
263enum instruction_type {
264 /* rx m.port_in */
265 INSTR_RX,
266
267 /* tx port_out
268 * port_out = MI
269 */
270 INSTR_TX, /* port_out = M */
271 INSTR_TX_I, /* port_out = I */
272 INSTR_DROP,
273
274 /*
275 * mirror slot_id session_id
276 * slot_id = MEFT
277 * session_id = MEFT
278 */
279 INSTR_MIRROR,
280
281 /* recirculate
282 */
283 INSTR_RECIRCULATE,
284
285 /* recircid m.recirc_pass_id
286 * Read the internal recirculation pass ID into the specified meta-data field.
287 */
288 INSTR_RECIRCID,
289
290 /* extract h.header */
291 INSTR_HDR_EXTRACT,
292 INSTR_HDR_EXTRACT2,
293 INSTR_HDR_EXTRACT3,
294 INSTR_HDR_EXTRACT4,
295 INSTR_HDR_EXTRACT5,
296 INSTR_HDR_EXTRACT6,
297 INSTR_HDR_EXTRACT7,
298 INSTR_HDR_EXTRACT8,
299
300 /* extract h.header m.last_field_size */
301 INSTR_HDR_EXTRACT_M,
302
303 /* lookahead h.header */
304 INSTR_HDR_LOOKAHEAD,
305
306 /* emit h.header */
307 INSTR_HDR_EMIT,
308 INSTR_HDR_EMIT_TX,
309 INSTR_HDR_EMIT2_TX,
310 INSTR_HDR_EMIT3_TX,
311 INSTR_HDR_EMIT4_TX,
312 INSTR_HDR_EMIT5_TX,
313 INSTR_HDR_EMIT6_TX,
314 INSTR_HDR_EMIT7_TX,
315 INSTR_HDR_EMIT8_TX,
316
317 /* validate h.header */
318 INSTR_HDR_VALIDATE,
319
320 /* invalidate h.header */
321 INSTR_HDR_INVALIDATE,
322
323 /* mov dst src
324 * dst = src
325 * dst = HMEF, src = HMEFTI
326 */
327 INSTR_MOV, /* dst = MEF, src = MEFT; size(dst) <= 64 bits, size(src) <= 64 bits. */
328 INSTR_MOV_MH, /* dst = MEF, src = H; size(dst) <= 64 bits, size(src) <= 64 bits. */
329 INSTR_MOV_HM, /* dst = H, src = MEFT; size(dst) <= 64 bits, size(src) <= 64 bits. */
330 INSTR_MOV_HH, /* dst = H, src = H; size(dst) <= 64 bits, size(src) <= 64 bits. */
331 INSTR_MOV_DMA, /* dst and src in NBO format. */
332 INSTR_MOV_128, /* dst and src in NBO format, size(dst) = size(src) = 128 bits. */
333 INSTR_MOV_128_32, /* dst and src in NBO format, size(dst) = 128 bits, size(src) = 32 b. */
334 INSTR_MOV_I, /* dst = HMEF, src = I; size(dst) <= 64 bits. */
335
336 /* dma h.header t.field
337 * memcpy(h.header, t.field, sizeof(h.header))
338 */
339 INSTR_DMA_HT,
340 INSTR_DMA_HT2,
341 INSTR_DMA_HT3,
342 INSTR_DMA_HT4,
343 INSTR_DMA_HT5,
344 INSTR_DMA_HT6,
345 INSTR_DMA_HT7,
346 INSTR_DMA_HT8,
347
348 /* add dst src
349 * dst += src
350 * dst = HMEF, src = HMEFTI
351 */
352 INSTR_ALU_ADD, /* dst = MEF, src = MEF */
353 INSTR_ALU_ADD_MH, /* dst = MEF, src = H */
354 INSTR_ALU_ADD_HM, /* dst = H, src = MEF */
355 INSTR_ALU_ADD_HH, /* dst = H, src = H */
356 INSTR_ALU_ADD_MI, /* dst = MEF, src = I */
357 INSTR_ALU_ADD_HI, /* dst = H, src = I */
358
359 /* sub dst src
360 * dst -= src
361 * dst = HMEF, src = HMEFTI
362 */
363 INSTR_ALU_SUB, /* dst = MEF, src = MEF */
364 INSTR_ALU_SUB_MH, /* dst = MEF, src = H */
365 INSTR_ALU_SUB_HM, /* dst = H, src = MEF */
366 INSTR_ALU_SUB_HH, /* dst = H, src = H */
367 INSTR_ALU_SUB_MI, /* dst = MEF, src = I */
368 INSTR_ALU_SUB_HI, /* dst = H, src = I */
369
370 /* ckadd dst src
371 * dst = dst '+ src[0:1] '+ src[2:3] '+ ...
372 * dst = H, src = {H, h.header}, '+ = 1's complement addition operator
373 */
374 INSTR_ALU_CKADD_FIELD, /* src = H */
375 INSTR_ALU_CKADD_STRUCT20, /* src = h.header, with sizeof(header) = 20 bytes. */
376 INSTR_ALU_CKADD_STRUCT, /* src = h.header, with sizeof(header) any 4-byte multiple. */
377
378 /* cksub dst src
379 * dst = dst '- src
380 * dst = H, src = H, '- = 1's complement subtraction operator
381 */
382 INSTR_ALU_CKSUB_FIELD,
383
384 /* and dst src
385 * dst &= src
386 * dst = HMEF, src = HMEFTI
387 */
388 INSTR_ALU_AND, /* dst = MEF, src = MEFT */
389 INSTR_ALU_AND_MH, /* dst = MEF, src = H */
390 INSTR_ALU_AND_HM, /* dst = H, src = MEFT */
391 INSTR_ALU_AND_HH, /* dst = H, src = H */
392 INSTR_ALU_AND_I, /* dst = HMEF, src = I */
393
394 /* or dst src
395 * dst |= src
396 * dst = HMEF, src = HMEFTI
397 */
398 INSTR_ALU_OR, /* dst = MEF, src = MEFT */
399 INSTR_ALU_OR_MH, /* dst = MEF, src = H */
400 INSTR_ALU_OR_HM, /* dst = H, src = MEFT */
401 INSTR_ALU_OR_HH, /* dst = H, src = H */
402 INSTR_ALU_OR_I, /* dst = HMEF, src = I */
403
404 /* xor dst src
405 * dst ^= src
406 * dst = HMEF, src = HMEFTI
407 */
408 INSTR_ALU_XOR, /* dst = MEF, src = MEFT */
409 INSTR_ALU_XOR_MH, /* dst = MEF, src = H */
410 INSTR_ALU_XOR_HM, /* dst = H, src = MEFT */
411 INSTR_ALU_XOR_HH, /* dst = H, src = H */
412 INSTR_ALU_XOR_I, /* dst = HMEF, src = I */
413
414 /* shl dst src
415 * dst <<= src
416 * dst = HMEF, src = HMEFTI
417 */
418 INSTR_ALU_SHL, /* dst = MEF, src = MEF */
419 INSTR_ALU_SHL_MH, /* dst = MEF, src = H */
420 INSTR_ALU_SHL_HM, /* dst = H, src = MEF */
421 INSTR_ALU_SHL_HH, /* dst = H, src = H */
422 INSTR_ALU_SHL_MI, /* dst = MEF, src = I */
423 INSTR_ALU_SHL_HI, /* dst = H, src = I */
424
425 /* shr dst src
426 * dst >>= src
427 * dst = HMEF, src = HMEFTI
428 */
429 INSTR_ALU_SHR, /* dst = MEF, src = MEF */
430 INSTR_ALU_SHR_MH, /* dst = MEF, src = H */
431 INSTR_ALU_SHR_HM, /* dst = H, src = MEF */
432 INSTR_ALU_SHR_HH, /* dst = H, src = H */
433 INSTR_ALU_SHR_MI, /* dst = MEF, src = I */
434 INSTR_ALU_SHR_HI, /* dst = H, src = I */
435
436 /* regprefetch REGARRAY index
437 * prefetch REGARRAY[index]
438 * index = HMEFTI
439 */
440 INSTR_REGPREFETCH_RH, /* index = H */
441 INSTR_REGPREFETCH_RM, /* index = MEFT */
442 INSTR_REGPREFETCH_RI, /* index = I */
443
444 /* regrd dst REGARRAY index
445 * dst = REGARRAY[index]
446 * dst = HMEF, index = HMEFTI
447 */
448 INSTR_REGRD_HRH, /* dst = H, index = H */
449 INSTR_REGRD_HRM, /* dst = H, index = MEFT */
450 INSTR_REGRD_HRI, /* dst = H, index = I */
451 INSTR_REGRD_MRH, /* dst = MEF, index = H */
452 INSTR_REGRD_MRM, /* dst = MEF, index = MEFT */
453 INSTR_REGRD_MRI, /* dst = MEF, index = I */
454
455 /* regwr REGARRAY index src
456 * REGARRAY[index] = src
457 * index = HMEFTI, src = HMEFTI
458 */
459 INSTR_REGWR_RHH, /* index = H, src = H */
460 INSTR_REGWR_RHM, /* index = H, src = MEFT */
461 INSTR_REGWR_RHI, /* index = H, src = I */
462 INSTR_REGWR_RMH, /* index = MEFT, src = H */
463 INSTR_REGWR_RMM, /* index = MEFT, src = MEFT */
464 INSTR_REGWR_RMI, /* index = MEFT, src = I */
465 INSTR_REGWR_RIH, /* index = I, src = H */
466 INSTR_REGWR_RIM, /* index = I, src = MEFT */
467 INSTR_REGWR_RII, /* index = I, src = I */
468
469 /* regadd REGARRAY index src
470 * REGARRAY[index] += src
471 * index = HMEFTI, src = HMEFTI
472 */
473 INSTR_REGADD_RHH, /* index = H, src = H */
474 INSTR_REGADD_RHM, /* index = H, src = MEFT */
475 INSTR_REGADD_RHI, /* index = H, src = I */
476 INSTR_REGADD_RMH, /* index = MEFT, src = H */
477 INSTR_REGADD_RMM, /* index = MEFT, src = MEFT */
478 INSTR_REGADD_RMI, /* index = MEFT, src = I */
479 INSTR_REGADD_RIH, /* index = I, src = H */
480 INSTR_REGADD_RIM, /* index = I, src = MEFT */
481 INSTR_REGADD_RII, /* index = I, src = I */
482
483 /* metprefetch METARRAY index
484 * prefetch METARRAY[index]
485 * index = HMEFTI
486 */
487 INSTR_METPREFETCH_H, /* index = H */
488 INSTR_METPREFETCH_M, /* index = MEFT */
489 INSTR_METPREFETCH_I, /* index = I */
490
491 /* meter METARRAY index length color_in color_out
492 * color_out = meter(METARRAY[index], length, color_in)
493 * index = HMEFTI, length = HMEFT, color_in = MEFTI, color_out = MEF
494 */
495 INSTR_METER_HHM, /* index = H, length = H, color_in = MEFT */
496 INSTR_METER_HHI, /* index = H, length = H, color_in = I */
497 INSTR_METER_HMM, /* index = H, length = MEFT, color_in = MEFT */
498 INSTR_METER_HMI, /* index = H, length = MEFT, color_in = I */
499 INSTR_METER_MHM, /* index = MEFT, length = H, color_in = MEFT */
500 INSTR_METER_MHI, /* index = MEFT, length = H, color_in = I */
501 INSTR_METER_MMM, /* index = MEFT, length = MEFT, color_in = MEFT */
502 INSTR_METER_MMI, /* index = MEFT, length = MEFT, color_in = I */
503 INSTR_METER_IHM, /* index = I, length = H, color_in = MEFT */
504 INSTR_METER_IHI, /* index = I, length = H, color_in = I */
505 INSTR_METER_IMM, /* index = I, length = MEFT, color_in = MEFT */
506 INSTR_METER_IMI, /* index = I, length = MEFT, color_in = I */
507
508 /* table TABLE */
509 INSTR_TABLE,
510 INSTR_TABLE_AF,
511 INSTR_SELECTOR,
512 INSTR_LEARNER,
513 INSTR_LEARNER_AF,
514
515 /* learn ACTION_NAME [ m.action_first_arg ] m.timeout_id */
516 INSTR_LEARNER_LEARN,
517
518 /* rearm [ m.timeout_id ] */
519 INSTR_LEARNER_REARM,
520 INSTR_LEARNER_REARM_NEW,
521
522 /* forget */
523 INSTR_LEARNER_FORGET,
524
525 /* entryid m.table_entry_id
526 * Read the internal table entry ID into the specified meta-data field.
527 */
528 INSTR_ENTRYID,
529
530 /* extern e.obj.func */
531 INSTR_EXTERN_OBJ,
532
533 /* extern f.func */
534 INSTR_EXTERN_FUNC,
535
536 /* hash HASH_FUNC_NAME dst src_first src_last
537 * Compute hash value over range of struct fields.
538 * dst = M
539 * src_first = HMEFT
540 * src_last = HMEFT
541 * src_first and src_last must be fields within the same struct
542 */
543 INSTR_HASH_FUNC,
544
545 /* rss RSS_OBJ_NAME dst src_first src_last
546 * Compute the RSS hash value over range of struct fields.
547 * dst = M
548 * src_first = HMEFT
549 * src_last = HMEFT
550 * src_first and src_last must be fields within the same struct
551 */
552 INSTR_RSS,
553
554 /* jmp LABEL
555 * Unconditional jump
556 */
557 INSTR_JMP,
558
559 /* jmpv LABEL h.header
560 * Jump if header is valid
561 */
562 INSTR_JMP_VALID,
563
564 /* jmpnv LABEL h.header
565 * Jump if header is invalid
566 */
567 INSTR_JMP_INVALID,
568
569 /* jmph LABEL
570 * Jump if table lookup hit
571 */
572 INSTR_JMP_HIT,
573
574 /* jmpnh LABEL
575 * Jump if table lookup miss
576 */
577 INSTR_JMP_MISS,
578
579 /* jmpa LABEL ACTION
580 * Jump if action run
581 */
582 INSTR_JMP_ACTION_HIT,
583
584 /* jmpna LABEL ACTION
585 * Jump if action not run
586 */
587 INSTR_JMP_ACTION_MISS,
588
589 /* jmpeq LABEL a b
590 * Jump if a is equal to b
591 * a = HMEFT, b = HMEFTI
592 */
593 INSTR_JMP_EQ, /* a = MEFT, b = MEFT */
594 INSTR_JMP_EQ_MH, /* a = MEFT, b = H */
595 INSTR_JMP_EQ_HM, /* a = H, b = MEFT */
596 INSTR_JMP_EQ_HH, /* a = H, b = H */
597 INSTR_JMP_EQ_I, /* (a, b) = (MEFT, I) or (a, b) = (H, I) */
598
599 /* jmpneq LABEL a b
600 * Jump if a is not equal to b
601 * a = HMEFT, b = HMEFTI
602 */
603 INSTR_JMP_NEQ, /* a = MEFT, b = MEFT */
604 INSTR_JMP_NEQ_MH, /* a = MEFT, b = H */
605 INSTR_JMP_NEQ_HM, /* a = H, b = MEFT */
606 INSTR_JMP_NEQ_HH, /* a = H, b = H */
607 INSTR_JMP_NEQ_I, /* (a, b) = (MEFT, I) or (a, b) = (H, I) */
608
609 /* jmplt LABEL a b
610 * Jump if a is less than b
611 * a = HMEFT, b = HMEFTI
612 */
613 INSTR_JMP_LT, /* a = MEFT, b = MEFT */
614 INSTR_JMP_LT_MH, /* a = MEFT, b = H */
615 INSTR_JMP_LT_HM, /* a = H, b = MEFT */
616 INSTR_JMP_LT_HH, /* a = H, b = H */
617 INSTR_JMP_LT_MI, /* a = MEFT, b = I */
618 INSTR_JMP_LT_HI, /* a = H, b = I */
619
620 /* jmpgt LABEL a b
621 * Jump if a is greater than b
622 * a = HMEFT, b = HMEFTI
623 */
624 INSTR_JMP_GT, /* a = MEFT, b = MEFT */
625 INSTR_JMP_GT_MH, /* a = MEFT, b = H */
626 INSTR_JMP_GT_HM, /* a = H, b = MEFT */
627 INSTR_JMP_GT_HH, /* a = H, b = H */
628 INSTR_JMP_GT_MI, /* a = MEFT, b = I */
629 INSTR_JMP_GT_HI, /* a = H, b = I */
630
631 /* return
632 * Return from action
633 */
634 INSTR_RETURN,
635
636 /* Start of custom instructions. */
637 INSTR_CUSTOM_0,
638};
639
640struct instr_operand {
641 uint8_t struct_id;
642 uint8_t n_bits;
643 uint8_t offset;
644 uint8_t pad;
645};
646
647struct instr_io {
648 struct {
649 union {
650 struct {
651 uint8_t offset;
652 uint8_t n_bits;
653 uint8_t pad[2];
654 };
655
656 uint32_t val;
657 };
658 } io;
659
660 struct {
661 uint8_t header_id[8];
662 uint8_t struct_id[8];
663 uint8_t n_bytes[8];
664 } hdr;
665};
666
667struct instr_hdr_validity {
668 uint8_t header_id;
669 uint8_t struct_id;
670};
671
672struct instr_table {
673 uint8_t table_id;
674};
675
676struct instr_learn {
677 uint8_t action_id;
678 uint8_t mf_first_arg_offset;
679 uint8_t mf_timeout_id_offset;
680 uint8_t mf_timeout_id_n_bits;
681};
682
683struct instr_extern_obj {
684 uint8_t ext_obj_id;
685 uint8_t func_id;
686};
687
688struct instr_extern_func {
689 uint8_t ext_func_id;
690};
691
692struct instr_hash_func {
693 uint8_t hash_func_id;
694
695 struct {
696 uint8_t offset;
697 uint8_t n_bits;
698 } dst;
699
700 struct {
701 uint8_t struct_id;
702 uint16_t offset;
703 uint16_t n_bytes;
704 } src;
705};
706
707struct instr_rss {
708 uint8_t rss_obj_id;
709
710 struct {
711 uint8_t offset;
712 uint8_t n_bits;
713 } dst;
714
715 struct {
716 uint8_t struct_id;
717 uint16_t offset;
718 uint16_t n_bytes;
719 } src;
720};
721
722struct instr_dst_src {
723 struct instr_operand dst;
724 union {
725 struct instr_operand src;
726 uint64_t src_val;
727 };
728};
729
730struct instr_regarray {
731 uint8_t regarray_id;
732 uint8_t pad[3];
733
734 union {
735 struct instr_operand idx;
736 uint32_t idx_val;
737 };
738
739 union {
740 struct instr_operand dstsrc;
741 uint64_t dstsrc_val;
742 };
743};
744
745struct instr_meter {
746 uint8_t metarray_id;
747 uint8_t pad[3];
748
749 union {
750 struct instr_operand idx;
751 uint32_t idx_val;
752 };
753
754 struct instr_operand length;
755
756 union {
757 struct instr_operand color_in;
758 uint32_t color_in_val;
759 };
760
761 struct instr_operand color_out;
762};
763
764struct instr_dma {
765 struct {
766 uint8_t header_id[8];
767 uint8_t struct_id[8];
768 } dst;
769
770 struct {
771 uint8_t offset[8];
772 } src;
773
774 uint16_t n_bytes[8];
775};
776
777struct instr_jmp {
778 struct instruction *ip;
779
780 union {
781 struct instr_operand a;
782 uint8_t header_id;
783 uint8_t action_id;
784 };
785
786 union {
787 struct instr_operand b;
788 uint64_t b_val;
789 };
790};
791
792struct instruction {
793 enum instruction_type type;
794 union {
795 struct instr_io io;
796 struct instr_dst_src mirror;
797 struct instr_hdr_validity valid;
798 struct instr_dst_src mov;
799 struct instr_regarray regarray;
800 struct instr_meter meter;
801 struct instr_dma dma;
802 struct instr_dst_src alu;
803 struct instr_table table;
804 struct instr_learn learn;
805 struct instr_extern_obj ext_obj;
806 struct instr_extern_func ext_func;
807 struct instr_hash_func hash_func;
808 struct instr_rss rss;
809 struct instr_jmp jmp;
810 };
811};
812
813struct instruction_data {
814 char label[RTE_SWX_NAME_SIZE];
815 char jmp_label[RTE_SWX_NAME_SIZE];
816 uint32_t n_users; /* user = jmp instruction to this instruction. */
817 int invalid;
818};
819
820typedef void (*instr_exec_t)(struct rte_swx_pipeline *);
821
822/*
823 * Action.
824 */
825typedef void
826(*action_func_t)(struct rte_swx_pipeline *p);
827
828struct action {
829 TAILQ_ENTRY(action) node;
830 char name[RTE_SWX_NAME_SIZE];
831 struct struct_type *st;
832 int *args_endianness; /* 0 = Host Byte Order (HBO); 1 = Network Byte Order (NBO). */
833 struct instruction *instructions;
834 struct instruction_data *instruction_data;
835 uint32_t n_instructions;
836 uint32_t id;
837};
838
839TAILQ_HEAD(action_tailq, action);
840
841/*
842 * Table.
843 */
844struct table_type {
845 TAILQ_ENTRY(table_type) node;
846 char name[RTE_SWX_NAME_SIZE];
847 enum rte_swx_table_match_type match_type;
848 struct rte_swx_table_ops ops;
849};
850
851TAILQ_HEAD(table_type_tailq, table_type);
852
853struct match_field {
854 enum rte_swx_table_match_type match_type;
855 struct field *field;
856};
857
858struct table {
859 TAILQ_ENTRY(table) node;
860 char name[RTE_SWX_NAME_SIZE];
861 char args[RTE_SWX_NAME_SIZE];
862 struct table_type *type; /* NULL when n_fields == 0. */
863
864 /* Match. */
865 struct match_field *fields;
866 uint32_t n_fields;
867 struct header *header; /* Only valid when n_fields > 0. */
868
869 /* Action. */
870 struct action **actions;
871 struct action *default_action;
872 uint8_t *default_action_data;
873 uint32_t n_actions;
874 int default_action_is_const;
875 uint32_t action_data_size_max;
876 int *action_is_for_table_entries;
877 int *action_is_for_default_entry;
878
879 struct hash_func *hf;
880 uint32_t size;
881 uint32_t id;
882};
883
884TAILQ_HEAD(table_tailq, table);
885
886struct table_runtime {
888 void *mailbox;
889 uint8_t **key;
890};
891
892struct table_statistics {
893 uint64_t n_pkts_hit[2]; /* 0 = Miss, 1 = Hit. */
894 uint64_t *n_pkts_action;
895};
896
897/*
898 * Selector.
899 */
900struct selector {
901 TAILQ_ENTRY(selector) node;
902 char name[RTE_SWX_NAME_SIZE];
903
904 struct field *group_id_field;
905 struct field **selector_fields;
906 uint32_t n_selector_fields;
907 struct header *selector_header;
908 struct field *member_id_field;
909
910 uint32_t n_groups_max;
911 uint32_t n_members_per_group_max;
912
913 uint32_t id;
914};
915
916TAILQ_HEAD(selector_tailq, selector);
917
918struct selector_runtime {
919 void *mailbox;
920 uint8_t **group_id_buffer;
921 uint8_t **selector_buffer;
922 uint8_t **member_id_buffer;
923};
924
925struct selector_statistics {
926 uint64_t n_pkts;
927};
928
929/*
930 * Learner table.
931 */
932struct learner {
933 TAILQ_ENTRY(learner) node;
934 char name[RTE_SWX_NAME_SIZE];
935
936 /* Match. */
937 struct field **fields;
938 uint32_t n_fields;
939 struct header *header;
940
941 /* Action. */
942 struct action **actions;
943 struct action *default_action;
944 uint8_t *default_action_data;
945 uint32_t n_actions;
946 int default_action_is_const;
947 uint32_t action_data_size_max;
948 int *action_is_for_table_entries;
949 int *action_is_for_default_entry;
950
951 struct hash_func *hf;
952 uint32_t size;
954 uint32_t n_timeouts;
955 uint32_t id;
956};
957
958TAILQ_HEAD(learner_tailq, learner);
959
960struct learner_runtime {
961 void *mailbox;
962 uint8_t **key;
963};
964
965struct learner_statistics {
966 uint64_t n_pkts_hit[2]; /* 0 = Miss, 1 = Hit. */
967 uint64_t n_pkts_learn[2]; /* 0 = Learn OK, 1 = Learn error. */
968 uint64_t n_pkts_rearm;
969 uint64_t n_pkts_forget;
970 uint64_t *n_pkts_action;
971};
972
973/*
974 * Register array.
975 */
976struct regarray {
977 TAILQ_ENTRY(regarray) node;
978 char name[RTE_SWX_NAME_SIZE];
979 uint64_t init_val;
980 uint32_t size;
981 uint32_t id;
982};
983
984TAILQ_HEAD(regarray_tailq, regarray);
985
986struct regarray_runtime {
987 uint64_t *regarray;
988 uint32_t size_mask;
989};
990
991/*
992 * Meter array.
993 */
994struct meter_profile {
995 TAILQ_ENTRY(meter_profile) node;
996 char name[RTE_SWX_NAME_SIZE];
997 struct rte_meter_trtcm_params params;
998 struct rte_meter_trtcm_profile profile;
999 uint32_t n_users;
1000};
1001
1002TAILQ_HEAD(meter_profile_tailq, meter_profile);
1003
1004struct metarray {
1005 TAILQ_ENTRY(metarray) node;
1006 char name[RTE_SWX_NAME_SIZE];
1007 uint32_t size;
1008 uint32_t id;
1009};
1010
1011TAILQ_HEAD(metarray_tailq, metarray);
1012
1013struct meter {
1014 struct rte_meter_trtcm m;
1015 struct meter_profile *profile;
1016 enum rte_color color_mask;
1017 uint8_t pad[20];
1018
1019 uint64_t n_pkts[RTE_COLORS];
1020 uint64_t n_bytes[RTE_COLORS];
1021};
1022
1023struct metarray_runtime {
1024 struct meter *metarray;
1025 uint32_t size_mask;
1026};
1027
1028/*
1029 * Pipeline.
1030 */
1031struct thread {
1032 /* Packet. */
1033 struct rte_swx_pkt pkt;
1034 uint8_t *ptr;
1035 uint32_t *mirroring_slots;
1036 uint64_t mirroring_slots_mask;
1037 int recirculate;
1038 uint32_t recirc_pass_id;
1039
1040 /* Structures. */
1041 uint8_t **structs;
1042
1043 /* Packet headers. */
1044 struct header_runtime *headers; /* Extracted or generated headers. */
1045 struct header_out_runtime *headers_out; /* Emitted headers. */
1046 uint8_t *header_storage;
1047 uint8_t *header_out_storage;
1048 uint64_t valid_headers;
1049 uint32_t n_headers_out;
1050
1051 /* Packet meta-data. */
1052 uint8_t *metadata;
1053
1054 /* Tables. */
1055 struct table_runtime *tables;
1056 struct selector_runtime *selectors;
1057 struct learner_runtime *learners;
1058 struct rte_swx_table_state *table_state;
1059 uint64_t action_id;
1060 size_t entry_id;
1061 int hit; /* 0 = Miss, 1 = Hit. */
1062 uint32_t learner_id;
1063 uint64_t time;
1064
1065 /* Extern objects and functions. */
1066 struct extern_obj_runtime *extern_objs;
1067 struct extern_func_runtime *extern_funcs;
1068
1069 /* Instructions. */
1070 struct instruction *ip;
1071 struct instruction *ret;
1072};
1073
1074#define MASK64_BIT_GET(mask, pos) ((mask) & (1LLU << (pos)))
1075#define MASK64_BIT_SET(mask, pos) ((mask) | (1LLU << (pos)))
1076#define MASK64_BIT_CLR(mask, pos) ((mask) & ~(1LLU << (pos)))
1077
1078#define HEADER_VALID(thread, header_id) \
1079 MASK64_BIT_GET((thread)->valid_headers, header_id)
1080
1081static inline uint64_t
1082instr_operand_hbo(struct thread *t, const struct instr_operand *x)
1083{
1084 uint8_t *x_struct = t->structs[x->struct_id];
1085 uint64_t *x64_ptr = (uint64_t *)&x_struct[x->offset];
1086 uint64_t x64 = *x64_ptr;
1087 uint64_t x64_mask = UINT64_MAX >> (64 - x->n_bits);
1088
1089 return x64 & x64_mask;
1090}
1091
1092#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
1093
1094static inline uint64_t
1095instr_operand_nbo(struct thread *t, const struct instr_operand *x)
1096{
1097 uint8_t *x_struct = t->structs[x->struct_id];
1098 uint64_t *x64_ptr = (uint64_t *)&x_struct[x->offset];
1099 uint64_t x64 = *x64_ptr;
1100
1101 return ntoh64(x64) >> (64 - x->n_bits);
1102}
1103
1104#else
1105
1106#define instr_operand_nbo instr_operand_hbo
1107
1108#endif
1109
1110#define ALU(thread, ip, operator) \
1111{ \
1112 uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id]; \
1113 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset]; \
1114 uint64_t dst64 = *dst64_ptr; \
1115 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits); \
1116 uint64_t dst = dst64 & dst64_mask; \
1117 \
1118 uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id]; \
1119 uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset]; \
1120 uint64_t src64 = *src64_ptr; \
1121 uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->alu.src.n_bits); \
1122 uint64_t src = src64 & src64_mask; \
1123 \
1124 uint64_t result = dst operator src; \
1125 \
1126 *dst64_ptr = (dst64 & ~dst64_mask) | (result & dst64_mask); \
1127}
1128
1129#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
1130
1131#define ALU_MH(thread, ip, operator) \
1132{ \
1133 uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id]; \
1134 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset]; \
1135 uint64_t dst64 = *dst64_ptr; \
1136 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits); \
1137 uint64_t dst = dst64 & dst64_mask; \
1138 \
1139 uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id]; \
1140 uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset]; \
1141 uint64_t src64 = *src64_ptr; \
1142 uint64_t src = ntoh64(src64) >> (64 - (ip)->alu.src.n_bits); \
1143 \
1144 uint64_t result = dst operator src; \
1145 \
1146 *dst64_ptr = (dst64 & ~dst64_mask) | (result & dst64_mask); \
1147}
1148
1149#define ALU_HM(thread, ip, operator) \
1150{ \
1151 uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id]; \
1152 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset]; \
1153 uint64_t dst64 = *dst64_ptr; \
1154 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits); \
1155 uint64_t dst = ntoh64(dst64) >> (64 - (ip)->alu.dst.n_bits); \
1156 \
1157 uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id]; \
1158 uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset]; \
1159 uint64_t src64 = *src64_ptr; \
1160 uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->alu.src.n_bits); \
1161 uint64_t src = src64 & src64_mask; \
1162 \
1163 uint64_t result = dst operator src; \
1164 result = hton64(result << (64 - (ip)->alu.dst.n_bits)); \
1165 \
1166 *dst64_ptr = (dst64 & ~dst64_mask) | result; \
1167}
1168
1169#define ALU_HM_FAST(thread, ip, operator) \
1170{ \
1171 uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id]; \
1172 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset]; \
1173 uint64_t dst64 = *dst64_ptr; \
1174 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits); \
1175 uint64_t dst = dst64 & dst64_mask; \
1176 \
1177 uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id]; \
1178 uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset]; \
1179 uint64_t src64 = *src64_ptr; \
1180 uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->alu.src.n_bits); \
1181 uint64_t src = hton64(src64 & src64_mask) >> (64 - (ip)->alu.dst.n_bits); \
1182 \
1183 uint64_t result = dst operator src; \
1184 \
1185 *dst64_ptr = (dst64 & ~dst64_mask) | result; \
1186}
1187
1188#define ALU_HH(thread, ip, operator) \
1189{ \
1190 uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id]; \
1191 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset]; \
1192 uint64_t dst64 = *dst64_ptr; \
1193 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits); \
1194 uint64_t dst = ntoh64(dst64) >> (64 - (ip)->alu.dst.n_bits); \
1195 \
1196 uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id]; \
1197 uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset]; \
1198 uint64_t src64 = *src64_ptr; \
1199 uint64_t src = ntoh64(src64) >> (64 - (ip)->alu.src.n_bits); \
1200 \
1201 uint64_t result = dst operator src; \
1202 result = hton64(result << (64 - (ip)->alu.dst.n_bits)); \
1203 \
1204 *dst64_ptr = (dst64 & ~dst64_mask) | result; \
1205}
1206
1207#define ALU_HH_FAST(thread, ip, operator) \
1208{ \
1209 uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id]; \
1210 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset]; \
1211 uint64_t dst64 = *dst64_ptr; \
1212 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits); \
1213 uint64_t dst = dst64 & dst64_mask; \
1214 \
1215 uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id]; \
1216 uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset]; \
1217 uint64_t src64 = *src64_ptr; \
1218 uint64_t src = (src64 << (64 - (ip)->alu.src.n_bits)) >> (64 - (ip)->alu.dst.n_bits); \
1219 \
1220 uint64_t result = dst operator src; \
1221 \
1222 *dst64_ptr = (dst64 & ~dst64_mask) | result; \
1223}
1224
1225#else
1226
1227#define ALU_MH ALU
1228#define ALU_HM ALU
1229#define ALU_HM_FAST ALU
1230#define ALU_HH ALU
1231#define ALU_HH_FAST ALU
1232
1233#endif
1234
1235#define ALU_I(thread, ip, operator) \
1236{ \
1237 uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id]; \
1238 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset]; \
1239 uint64_t dst64 = *dst64_ptr; \
1240 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits); \
1241 uint64_t dst = dst64 & dst64_mask; \
1242 \
1243 uint64_t src = (ip)->alu.src_val; \
1244 \
1245 uint64_t result = dst operator src; \
1246 \
1247 *dst64_ptr = (dst64 & ~dst64_mask) | (result & dst64_mask); \
1248}
1249
1250#define ALU_MI ALU_I
1251
1252#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
1253
1254#define ALU_HI(thread, ip, operator) \
1255{ \
1256 uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id]; \
1257 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset]; \
1258 uint64_t dst64 = *dst64_ptr; \
1259 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits); \
1260 uint64_t dst = ntoh64(dst64) >> (64 - (ip)->alu.dst.n_bits); \
1261 \
1262 uint64_t src = (ip)->alu.src_val; \
1263 \
1264 uint64_t result = dst operator src; \
1265 result = hton64(result << (64 - (ip)->alu.dst.n_bits)); \
1266 \
1267 *dst64_ptr = (dst64 & ~dst64_mask) | result; \
1268}
1269
1270#else
1271
1272#define ALU_HI ALU_I
1273
1274#endif
1275
1276#define MOV(thread, ip) \
1277{ \
1278 uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id]; \
1279 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset]; \
1280 uint64_t dst64 = *dst64_ptr; \
1281 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits); \
1282 \
1283 uint8_t *src_struct = (thread)->structs[(ip)->mov.src.struct_id]; \
1284 uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->mov.src.offset]; \
1285 uint64_t src64 = *src64_ptr; \
1286 uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->mov.src.n_bits); \
1287 uint64_t src = src64 & src64_mask; \
1288 \
1289 *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask); \
1290}
1291
1292#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
1293
1294#define MOV_MH(thread, ip) \
1295{ \
1296 uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id]; \
1297 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset]; \
1298 uint64_t dst64 = *dst64_ptr; \
1299 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits); \
1300 \
1301 uint8_t *src_struct = (thread)->structs[(ip)->mov.src.struct_id]; \
1302 uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->mov.src.offset]; \
1303 uint64_t src64 = *src64_ptr; \
1304 uint64_t src = ntoh64(src64) >> (64 - (ip)->mov.src.n_bits); \
1305 \
1306 *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask); \
1307}
1308
1309#define MOV_HM(thread, ip) \
1310{ \
1311 uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id]; \
1312 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset]; \
1313 uint64_t dst64 = *dst64_ptr; \
1314 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits); \
1315 \
1316 uint8_t *src_struct = (thread)->structs[(ip)->mov.src.struct_id]; \
1317 uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->mov.src.offset]; \
1318 uint64_t src64 = *src64_ptr; \
1319 uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->mov.src.n_bits); \
1320 uint64_t src = src64 & src64_mask; \
1321 \
1322 src = hton64(src) >> (64 - (ip)->mov.dst.n_bits); \
1323 *dst64_ptr = (dst64 & ~dst64_mask) | src; \
1324}
1325
1326#define MOV_HH(thread, ip) \
1327{ \
1328 uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id]; \
1329 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset]; \
1330 uint64_t dst64 = *dst64_ptr; \
1331 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits); \
1332 \
1333 uint8_t *src_struct = (thread)->structs[(ip)->mov.src.struct_id]; \
1334 uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->mov.src.offset]; \
1335 uint64_t src64 = *src64_ptr; \
1336 \
1337 uint64_t src = src64 << (64 - (ip)->mov.src.n_bits); \
1338 src = src >> (64 - (ip)->mov.dst.n_bits); \
1339 *dst64_ptr = (dst64 & ~dst64_mask) | src; \
1340}
1341
1342#else
1343
1344#define MOV_MH MOV
1345#define MOV_HM MOV
1346#define MOV_HH MOV
1347
1348#endif
1349
1350#define MOV_I(thread, ip) \
1351{ \
1352 uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id]; \
1353 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset]; \
1354 uint64_t dst64 = *dst64_ptr; \
1355 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits); \
1356 \
1357 uint64_t src = (ip)->mov.src_val; \
1358 \
1359 *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask); \
1360}
1361
1362#define JMP_CMP(thread, ip, operator) \
1363{ \
1364 uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id]; \
1365 uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset]; \
1366 uint64_t a64 = *a64_ptr; \
1367 uint64_t a64_mask = UINT64_MAX >> (64 - (ip)->jmp.a.n_bits); \
1368 uint64_t a = a64 & a64_mask; \
1369 \
1370 uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id]; \
1371 uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset]; \
1372 uint64_t b64 = *b64_ptr; \
1373 uint64_t b64_mask = UINT64_MAX >> (64 - (ip)->jmp.b.n_bits); \
1374 uint64_t b = b64 & b64_mask; \
1375 \
1376 (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1); \
1377}
1378
1379#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
1380
1381#define JMP_CMP_MH(thread, ip, operator) \
1382{ \
1383 uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id]; \
1384 uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset]; \
1385 uint64_t a64 = *a64_ptr; \
1386 uint64_t a64_mask = UINT64_MAX >> (64 - (ip)->jmp.a.n_bits); \
1387 uint64_t a = a64 & a64_mask; \
1388 \
1389 uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id]; \
1390 uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset]; \
1391 uint64_t b64 = *b64_ptr; \
1392 uint64_t b = ntoh64(b64) >> (64 - (ip)->jmp.b.n_bits); \
1393 \
1394 (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1); \
1395}
1396
1397#define JMP_CMP_HM(thread, ip, operator) \
1398{ \
1399 uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id]; \
1400 uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset]; \
1401 uint64_t a64 = *a64_ptr; \
1402 uint64_t a = ntoh64(a64) >> (64 - (ip)->jmp.a.n_bits); \
1403 \
1404 uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id]; \
1405 uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset]; \
1406 uint64_t b64 = *b64_ptr; \
1407 uint64_t b64_mask = UINT64_MAX >> (64 - (ip)->jmp.b.n_bits); \
1408 uint64_t b = b64 & b64_mask; \
1409 \
1410 (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1); \
1411}
1412
1413#define JMP_CMP_HH(thread, ip, operator) \
1414{ \
1415 uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id]; \
1416 uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset]; \
1417 uint64_t a64 = *a64_ptr; \
1418 uint64_t a = ntoh64(a64) >> (64 - (ip)->jmp.a.n_bits); \
1419 \
1420 uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id]; \
1421 uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset]; \
1422 uint64_t b64 = *b64_ptr; \
1423 uint64_t b = ntoh64(b64) >> (64 - (ip)->jmp.b.n_bits); \
1424 \
1425 (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1); \
1426}
1427
1428#define JMP_CMP_HH_FAST(thread, ip, operator) \
1429{ \
1430 uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id]; \
1431 uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset]; \
1432 uint64_t a64 = *a64_ptr; \
1433 uint64_t a = a64 << (64 - (ip)->jmp.a.n_bits); \
1434 \
1435 uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id]; \
1436 uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset]; \
1437 uint64_t b64 = *b64_ptr; \
1438 uint64_t b = b64 << (64 - (ip)->jmp.b.n_bits); \
1439 \
1440 (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1); \
1441}
1442
1443#else
1444
1445#define JMP_CMP_MH JMP_CMP
1446#define JMP_CMP_HM JMP_CMP
1447#define JMP_CMP_HH JMP_CMP
1448#define JMP_CMP_HH_FAST JMP_CMP
1449
1450#endif
1451
1452#define JMP_CMP_I(thread, ip, operator) \
1453{ \
1454 uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id]; \
1455 uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset]; \
1456 uint64_t a64 = *a64_ptr; \
1457 uint64_t a64_mask = UINT64_MAX >> (64 - (ip)->jmp.a.n_bits); \
1458 uint64_t a = a64 & a64_mask; \
1459 \
1460 uint64_t b = (ip)->jmp.b_val; \
1461 \
1462 (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1); \
1463}
1464
1465#define JMP_CMP_MI JMP_CMP_I
1466
1467#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
1468
1469#define JMP_CMP_HI(thread, ip, operator) \
1470{ \
1471 uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id]; \
1472 uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset]; \
1473 uint64_t a64 = *a64_ptr; \
1474 uint64_t a = ntoh64(a64) >> (64 - (ip)->jmp.a.n_bits); \
1475 \
1476 uint64_t b = (ip)->jmp.b_val; \
1477 \
1478 (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1); \
1479}
1480
1481#else
1482
1483#define JMP_CMP_HI JMP_CMP_I
1484
1485#endif
1486
1487#define METADATA_READ(thread, offset, n_bits) \
1488({ \
1489 uint64_t *m64_ptr = (uint64_t *)&(thread)->metadata[offset]; \
1490 uint64_t m64 = *m64_ptr; \
1491 uint64_t m64_mask = UINT64_MAX >> (64 - (n_bits)); \
1492 (m64 & m64_mask); \
1493})
1494
1495#define METADATA_WRITE(thread, offset, n_bits, value) \
1496{ \
1497 uint64_t *m64_ptr = (uint64_t *)&(thread)->metadata[offset]; \
1498 uint64_t m64 = *m64_ptr; \
1499 uint64_t m64_mask = UINT64_MAX >> (64 - (n_bits)); \
1500 \
1501 uint64_t m_new = value; \
1502 \
1503 *m64_ptr = (m64 & ~m64_mask) | (m_new & m64_mask); \
1504}
1505
1506#ifndef RTE_SWX_PIPELINE_THREADS_MAX
1507#define RTE_SWX_PIPELINE_THREADS_MAX 16
1508#endif
1509
1510#ifndef RTE_SWX_PIPELINE_INSTRUCTION_TABLE_SIZE_MAX
1511#define RTE_SWX_PIPELINE_INSTRUCTION_TABLE_SIZE_MAX 1024
1512#endif
1513
1514struct rte_swx_pipeline {
1515 char name[RTE_SWX_NAME_SIZE];
1516
1517 struct struct_type_tailq struct_types;
1518 struct port_in_type_tailq port_in_types;
1519 struct port_in_tailq ports_in;
1520 struct port_out_type_tailq port_out_types;
1521 struct port_out_tailq ports_out;
1522 struct extern_type_tailq extern_types;
1523 struct extern_obj_tailq extern_objs;
1524 struct extern_func_tailq extern_funcs;
1525 struct hash_func_tailq hash_funcs;
1526 struct rss_tailq rss;
1527 struct header_tailq headers;
1528 struct struct_type *metadata_st;
1529 uint32_t metadata_struct_id;
1530 struct action_tailq actions;
1531 struct table_type_tailq table_types;
1532 struct table_tailq tables;
1533 struct selector_tailq selectors;
1534 struct learner_tailq learners;
1535 struct regarray_tailq regarrays;
1536 struct meter_profile_tailq meter_profiles;
1537 struct metarray_tailq metarrays;
1538
1539 struct port_in_runtime *in;
1540 struct port_out_runtime *out;
1541 struct mirroring_session *mirroring_sessions;
1542 struct instruction **action_instructions;
1543 action_func_t *action_funcs;
1544 struct rte_swx_table_state *table_state;
1545 struct table_statistics *table_stats;
1546 struct selector_statistics *selector_stats;
1547 struct learner_statistics *learner_stats;
1548 struct hash_func_runtime *hash_func_runtime;
1549 struct rss_runtime **rss_runtime;
1550 struct regarray_runtime *regarray_runtime;
1551 struct metarray_runtime *metarray_runtime;
1552 struct instruction *instructions;
1553 struct instruction_data *instruction_data;
1554 instr_exec_t *instruction_table;
1555 struct thread threads[RTE_SWX_PIPELINE_THREADS_MAX];
1556 void *lib;
1557
1558 uint32_t n_structs;
1559 uint32_t n_ports_in;
1560 uint32_t n_ports_out;
1561 uint32_t n_mirroring_slots;
1562 uint32_t n_mirroring_sessions;
1563 uint32_t n_extern_objs;
1564 uint32_t n_extern_funcs;
1565 uint32_t n_hash_funcs;
1566 uint32_t n_rss;
1567 uint32_t n_actions;
1568 uint32_t n_tables;
1569 uint32_t n_selectors;
1570 uint32_t n_learners;
1571 uint32_t n_regarrays;
1572 uint32_t n_metarrays;
1573 uint32_t n_headers;
1574 uint32_t thread_id;
1575 uint32_t port_id;
1576 uint32_t n_instructions;
1577 int build_done;
1578 int numa_node;
1579};
1580
1581/*
1582 * Instruction.
1583 */
1584static inline void
1585pipeline_port_inc(struct rte_swx_pipeline *p)
1586{
1587 p->port_id = (p->port_id + 1) & (p->n_ports_in - 1);
1588}
1589
1590static inline void
1591thread_ip_reset(struct rte_swx_pipeline *p, struct thread *t)
1592{
1593 t->ip = p->instructions;
1594}
1595
1596static inline void
1597thread_ip_set(struct thread *t, struct instruction *ip)
1598{
1599 t->ip = ip;
1600}
1601
1602static inline void
1603thread_ip_action_call(struct rte_swx_pipeline *p,
1604 struct thread *t,
1605 uint32_t action_id)
1606{
1607 t->ret = t->ip + 1;
1608 t->ip = p->action_instructions[action_id];
1609}
1610
1611static inline void
1612thread_ip_inc(struct rte_swx_pipeline *p);
1613
1614static inline void
1615thread_ip_inc(struct rte_swx_pipeline *p)
1616{
1617 struct thread *t = &p->threads[p->thread_id];
1618
1619 t->ip++;
1620}
1621
1622static inline void
1623thread_ip_inc_cond(struct thread *t, int cond)
1624{
1625 t->ip += cond;
1626}
1627
1628static inline void
1629thread_yield(struct rte_swx_pipeline *p)
1630{
1631 p->thread_id = (p->thread_id + 1) & (RTE_SWX_PIPELINE_THREADS_MAX - 1);
1632}
1633
1634static inline void
1635thread_yield_cond(struct rte_swx_pipeline *p, int cond)
1636{
1637 p->thread_id = (p->thread_id + cond) & (RTE_SWX_PIPELINE_THREADS_MAX - 1);
1638}
1639
1640/*
1641 * rx.
1642 */
1643static inline int
1644__instr_rx_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
1645{
1646 struct port_in_runtime *port = &p->in[p->port_id];
1647 struct rte_swx_pkt *pkt = &t->pkt;
1648 int pkt_received;
1649
1650 /* Recirculation: keep the current packet. */
1651 if (t->recirculate) {
1652 TRACE("[Thread %2u] rx - recirculate (pass %u)\n",
1653 p->thread_id,
1654 t->recirc_pass_id + 1);
1655
1656 /* Packet. */
1657 t->ptr = &pkt->pkt[pkt->offset];
1658 t->mirroring_slots_mask = 0;
1659 t->recirculate = 0;
1660 t->recirc_pass_id++;
1661
1662 /* Headers. */
1663 t->valid_headers = 0;
1664 t->n_headers_out = 0;
1665
1666 /* Tables. */
1667 t->table_state = p->table_state;
1668
1669 return 1;
1670 }
1671
1672 /* Packet. */
1673 pkt_received = port->pkt_rx(port->obj, pkt);
1674 t->ptr = &pkt->pkt[pkt->offset];
1675 rte_prefetch0(t->ptr);
1676
1677 TRACE("[Thread %2u] rx %s from port %u\n",
1678 p->thread_id,
1679 pkt_received ? "1 pkt" : "0 pkts",
1680 p->port_id);
1681
1682 t->mirroring_slots_mask = 0;
1683 t->recirc_pass_id = 0;
1684
1685 /* Headers. */
1686 t->valid_headers = 0;
1687 t->n_headers_out = 0;
1688
1689 /* Meta-data. */
1690 METADATA_WRITE(t, ip->io.io.offset, ip->io.io.n_bits, p->port_id);
1691
1692 /* Tables. */
1693 t->table_state = p->table_state;
1694
1695 /* Thread. */
1696 pipeline_port_inc(p);
1697
1698 return pkt_received;
1699}
1700
1701static inline void
1702instr_rx_exec(struct rte_swx_pipeline *p)
1703{
1704 struct thread *t = &p->threads[p->thread_id];
1705 struct instruction *ip = t->ip;
1706 int pkt_received;
1707
1708 /* Packet. */
1709 pkt_received = __instr_rx_exec(p, t, ip);
1710
1711 /* Thread. */
1712 thread_ip_inc_cond(t, pkt_received);
1713 thread_yield(p);
1714}
1715
1716/*
1717 * tx.
1718 */
1719static inline void
1720emit_handler(struct thread *t)
1721{
1722 struct header_out_runtime *h0 = &t->headers_out[0];
1723 struct header_out_runtime *h1 = &t->headers_out[1];
1724 uint32_t offset = 0, i;
1725
1726 /* No header change or header decapsulation. */
1727 if ((t->n_headers_out == 1) &&
1728 (h0->ptr + h0->n_bytes == t->ptr)) {
1729 TRACE("Emit handler: no header change or header decap.\n");
1730
1731 t->pkt.offset -= h0->n_bytes;
1732 t->pkt.length += h0->n_bytes;
1733
1734 return;
1735 }
1736
1737 /* Header encapsulation (optionally, with prior header decapsulation). */
1738 if ((t->n_headers_out == 2) &&
1739 (h1->ptr + h1->n_bytes == t->ptr) &&
1740 (h0->ptr == h0->ptr0)) {
1741 uint32_t offset;
1742
1743 TRACE("Emit handler: header encapsulation.\n");
1744
1745 offset = h0->n_bytes + h1->n_bytes;
1746 memcpy(t->ptr - offset, h0->ptr, h0->n_bytes);
1747 t->pkt.offset -= offset;
1748 t->pkt.length += offset;
1749
1750 return;
1751 }
1752
1753 /* For any other case. */
1754 TRACE("Emit handler: complex case.\n");
1755
1756 for (i = 0; i < t->n_headers_out; i++) {
1757 struct header_out_runtime *h = &t->headers_out[i];
1758
1759 memcpy(&t->header_out_storage[offset], h->ptr, h->n_bytes);
1760 offset += h->n_bytes;
1761 }
1762
1763 if (offset) {
1764 memcpy(t->ptr - offset, t->header_out_storage, offset);
1765 t->pkt.offset -= offset;
1766 t->pkt.length += offset;
1767 }
1768}
1769
1770static inline void
1771mirroring_handler(struct rte_swx_pipeline *p, struct thread *t, struct rte_swx_pkt *pkt)
1772{
1773 uint64_t slots_mask = t->mirroring_slots_mask, slot_mask;
1774 uint32_t slot_id;
1775
1776 for (slot_id = 0, slot_mask = 1LLU ; slots_mask; slot_id++, slot_mask <<= 1)
1777 if (slot_mask & slots_mask) {
1778 struct port_out_runtime *port;
1779 struct mirroring_session *session;
1780 uint32_t port_id, session_id;
1781
1782 session_id = t->mirroring_slots[slot_id];
1783 session = &p->mirroring_sessions[session_id];
1784
1785 port_id = session->port_id;
1786 port = &p->out[port_id];
1787
1788 if (session->fast_clone)
1789 port->pkt_fast_clone_tx(port->obj, pkt);
1790 else
1791 port->pkt_clone_tx(port->obj, pkt, session->truncation_length);
1792
1793 slots_mask &= ~slot_mask;
1794 }
1795}
1796
1797static inline void
1798__instr_tx_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
1799{
1800 uint64_t port_id = METADATA_READ(t, ip->io.io.offset, ip->io.io.n_bits);
1801 struct port_out_runtime *port = &p->out[port_id];
1802 struct rte_swx_pkt *pkt = &t->pkt;
1803
1804 /* Recirculation: keep the current packet. */
1805 if (t->recirculate) {
1806 TRACE("[Thread %2u]: tx 1 pkt - recirculate\n",
1807 p->thread_id);
1808
1809 /* Headers. */
1810 emit_handler(t);
1811
1812 /* Packet. */
1813 mirroring_handler(p, t, pkt);
1814
1815 return;
1816 }
1817
1818 TRACE("[Thread %2u]: tx 1 pkt to port %u\n",
1819 p->thread_id,
1820 (uint32_t)port_id);
1821
1822 /* Headers. */
1823 emit_handler(t);
1824
1825 /* Packet. */
1826 mirroring_handler(p, t, pkt);
1827 port->pkt_tx(port->obj, pkt);
1828}
1829
1830static inline void
1831__instr_tx_i_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
1832{
1833 uint64_t port_id = ip->io.io.val;
1834 struct port_out_runtime *port = &p->out[port_id];
1835 struct rte_swx_pkt *pkt = &t->pkt;
1836
1837 /* Recirculation: keep the current packet. */
1838 if (t->recirculate) {
1839 TRACE("[Thread %2u]: tx (i) 1 pkt - recirculate\n",
1840 p->thread_id);
1841
1842 /* Headers. */
1843 emit_handler(t);
1844
1845 /* Packet. */
1846 mirroring_handler(p, t, pkt);
1847
1848 return;
1849 }
1850
1851 TRACE("[Thread %2u]: tx (i) 1 pkt to port %u\n",
1852 p->thread_id,
1853 (uint32_t)port_id);
1854
1855 /* Headers. */
1856 emit_handler(t);
1857
1858 /* Packet. */
1859 mirroring_handler(p, t, pkt);
1860 port->pkt_tx(port->obj, pkt);
1861}
1862
1863static inline void
1864__instr_drop_exec(struct rte_swx_pipeline *p,
1865 struct thread *t,
1866 const struct instruction *ip __rte_unused)
1867{
1868 uint64_t port_id = p->n_ports_out - 1;
1869 struct port_out_runtime *port = &p->out[port_id];
1870 struct rte_swx_pkt *pkt = &t->pkt;
1871
1872 TRACE("[Thread %2u]: drop 1 pkt\n",
1873 p->thread_id);
1874
1875 /* Headers. */
1876 emit_handler(t);
1877
1878 /* Packet. */
1879 mirroring_handler(p, t, pkt);
1880 port->pkt_tx(port->obj, pkt);
1881}
1882
1883static inline void
1884__instr_mirror_exec(struct rte_swx_pipeline *p,
1885 struct thread *t,
1886 const struct instruction *ip)
1887{
1888 uint64_t slot_id = instr_operand_hbo(t, &ip->mirror.dst);
1889 uint64_t session_id = instr_operand_hbo(t, &ip->mirror.src);
1890
1891 slot_id &= p->n_mirroring_slots - 1;
1892 session_id &= p->n_mirroring_sessions - 1;
1893
1894 TRACE("[Thread %2u]: mirror pkt (slot = %u, session = %u)\n",
1895 p->thread_id,
1896 (uint32_t)slot_id,
1897 (uint32_t)session_id);
1898
1899 t->mirroring_slots[slot_id] = session_id;
1900 t->mirroring_slots_mask |= 1LLU << slot_id;
1901}
1902
1903static inline void
1904__instr_recirculate_exec(struct rte_swx_pipeline *p __rte_unused,
1905 struct thread *t,
1906 const struct instruction *ip __rte_unused)
1907{
1908 TRACE("[Thread %2u]: recirculate\n",
1909 p->thread_id);
1910
1911 t->recirculate = 1;
1912}
1913
1914static inline void
1915__instr_recircid_exec(struct rte_swx_pipeline *p __rte_unused,
1916 struct thread *t,
1917 const struct instruction *ip)
1918{
1919 TRACE("[Thread %2u]: recircid (pass %u)\n",
1920 p->thread_id,
1921 t->recirc_pass_id);
1922
1923 /* Meta-data. */
1924 METADATA_WRITE(t, ip->io.io.offset, ip->io.io.n_bits, t->recirc_pass_id);
1925}
1926
1927/*
1928 * extract.
1929 */
1930static inline void
1931__instr_hdr_extract_many_exec(struct rte_swx_pipeline *p __rte_unused,
1932 struct thread *t,
1933 const struct instruction *ip,
1934 uint32_t n_extract)
1935{
1936 uint64_t valid_headers = t->valid_headers;
1937 uint8_t *ptr = t->ptr;
1938 uint32_t offset = t->pkt.offset;
1939 uint32_t length = t->pkt.length;
1940 uint32_t i;
1941
1942 for (i = 0; i < n_extract; i++) {
1943 uint32_t header_id = ip->io.hdr.header_id[i];
1944 uint32_t struct_id = ip->io.hdr.struct_id[i];
1945 uint32_t n_bytes = ip->io.hdr.n_bytes[i];
1946
1947 TRACE("[Thread %2u]: extract header %u (%u bytes)\n",
1948 p->thread_id,
1949 header_id,
1950 n_bytes);
1951
1952 /* Headers. */
1953 t->structs[struct_id] = ptr;
1954 valid_headers = MASK64_BIT_SET(valid_headers, header_id);
1955
1956 /* Packet. */
1957 offset += n_bytes;
1958 length -= n_bytes;
1959 ptr += n_bytes;
1960 }
1961
1962 /* Headers. */
1963 t->valid_headers = valid_headers;
1964
1965 /* Packet. */
1966 t->pkt.offset = offset;
1967 t->pkt.length = length;
1968 t->ptr = ptr;
1969}
1970
1971static inline void
1972__instr_hdr_extract_exec(struct rte_swx_pipeline *p,
1973 struct thread *t,
1974 const struct instruction *ip)
1975{
1976 __instr_hdr_extract_many_exec(p, t, ip, 1);
1977}
1978
1979static inline void
1980__instr_hdr_extract2_exec(struct rte_swx_pipeline *p,
1981 struct thread *t,
1982 const struct instruction *ip)
1983{
1984 TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n", p->thread_id);
1985
1986 __instr_hdr_extract_many_exec(p, t, ip, 2);
1987}
1988
1989static inline void
1990__instr_hdr_extract3_exec(struct rte_swx_pipeline *p,
1991 struct thread *t,
1992 const struct instruction *ip)
1993{
1994 TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n", p->thread_id);
1995
1996 __instr_hdr_extract_many_exec(p, t, ip, 3);
1997}
1998
1999static inline void
2000__instr_hdr_extract4_exec(struct rte_swx_pipeline *p,
2001 struct thread *t,
2002 const struct instruction *ip)
2003{
2004 TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n", p->thread_id);
2005
2006 __instr_hdr_extract_many_exec(p, t, ip, 4);
2007}
2008
2009static inline void
2010__instr_hdr_extract5_exec(struct rte_swx_pipeline *p,
2011 struct thread *t,
2012 const struct instruction *ip)
2013{
2014 TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n", p->thread_id);
2015
2016 __instr_hdr_extract_many_exec(p, t, ip, 5);
2017}
2018
2019static inline void
2020__instr_hdr_extract6_exec(struct rte_swx_pipeline *p,
2021 struct thread *t,
2022 const struct instruction *ip)
2023{
2024 TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n", p->thread_id);
2025
2026 __instr_hdr_extract_many_exec(p, t, ip, 6);
2027}
2028
2029static inline void
2030__instr_hdr_extract7_exec(struct rte_swx_pipeline *p,
2031 struct thread *t,
2032 const struct instruction *ip)
2033{
2034 TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n", p->thread_id);
2035
2036 __instr_hdr_extract_many_exec(p, t, ip, 7);
2037}
2038
2039static inline void
2040__instr_hdr_extract8_exec(struct rte_swx_pipeline *p,
2041 struct thread *t,
2042 const struct instruction *ip)
2043{
2044 TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n", p->thread_id);
2045
2046 __instr_hdr_extract_many_exec(p, t, ip, 8);
2047}
2048
2049static inline void
2050__instr_hdr_extract_m_exec(struct rte_swx_pipeline *p __rte_unused,
2051 struct thread *t,
2052 const struct instruction *ip)
2053{
2054 uint64_t valid_headers = t->valid_headers;
2055 uint8_t *ptr = t->ptr;
2056 uint32_t offset = t->pkt.offset;
2057 uint32_t length = t->pkt.length;
2058
2059 uint32_t n_bytes_last = METADATA_READ(t, ip->io.io.offset, ip->io.io.n_bits);
2060 uint32_t header_id = ip->io.hdr.header_id[0];
2061 uint32_t struct_id = ip->io.hdr.struct_id[0];
2062 uint32_t n_bytes = ip->io.hdr.n_bytes[0];
2063
2064 struct header_runtime *h = &t->headers[header_id];
2065
2066 TRACE("[Thread %2u]: extract header %u (%u + %u bytes)\n",
2067 p->thread_id,
2068 header_id,
2069 n_bytes,
2070 n_bytes_last);
2071
2072 n_bytes += n_bytes_last;
2073
2074 /* Headers. */
2075 t->structs[struct_id] = ptr;
2076 t->valid_headers = MASK64_BIT_SET(valid_headers, header_id);
2077 h->n_bytes = n_bytes;
2078
2079 /* Packet. */
2080 t->pkt.offset = offset + n_bytes;
2081 t->pkt.length = length - n_bytes;
2082 t->ptr = ptr + n_bytes;
2083}
2084
2085static inline void
2086__instr_hdr_lookahead_exec(struct rte_swx_pipeline *p __rte_unused,
2087 struct thread *t,
2088 const struct instruction *ip)
2089{
2090 uint64_t valid_headers = t->valid_headers;
2091 uint8_t *ptr = t->ptr;
2092
2093 uint32_t header_id = ip->io.hdr.header_id[0];
2094 uint32_t struct_id = ip->io.hdr.struct_id[0];
2095
2096 TRACE("[Thread %2u]: lookahead header %u\n",
2097 p->thread_id,
2098 header_id);
2099
2100 /* Headers. */
2101 t->structs[struct_id] = ptr;
2102 t->valid_headers = MASK64_BIT_SET(valid_headers, header_id);
2103}
2104
2105/*
2106 * emit.
2107 */
2108static inline void
2109__instr_hdr_emit_many_exec(struct rte_swx_pipeline *p __rte_unused,
2110 struct thread *t,
2111 const struct instruction *ip,
2112 uint32_t n_emit)
2113{
2114 uint64_t valid_headers = t->valid_headers;
2115 uint32_t n_headers_out = t->n_headers_out;
2116 struct header_out_runtime *ho = NULL;
2117 uint8_t *ho_ptr = NULL;
2118 uint32_t ho_nbytes = 0, i;
2119
2120 for (i = 0; i < n_emit; i++) {
2121 uint32_t header_id = ip->io.hdr.header_id[i];
2122 uint32_t struct_id = ip->io.hdr.struct_id[i];
2123
2124 struct header_runtime *hi = &t->headers[header_id];
2125 uint8_t *hi_ptr0 = hi->ptr0;
2126 uint32_t n_bytes = hi->n_bytes;
2127
2128 uint8_t *hi_ptr = t->structs[struct_id];
2129
2130 if (!MASK64_BIT_GET(valid_headers, header_id)) {
2131 TRACE("[Thread %2u]: emit header %u (invalid)\n",
2132 p->thread_id,
2133 header_id);
2134
2135 continue;
2136 }
2137
2138 TRACE("[Thread %2u]: emit header %u (valid)\n",
2139 p->thread_id,
2140 header_id);
2141
2142 /* Headers. */
2143 if (!ho) {
2144 if (!n_headers_out) {
2145 ho = &t->headers_out[0];
2146
2147 ho->ptr0 = hi_ptr0;
2148 ho->ptr = hi_ptr;
2149
2150 ho_ptr = hi_ptr;
2151 ho_nbytes = n_bytes;
2152
2153 n_headers_out = 1;
2154
2155 continue;
2156 } else {
2157 ho = &t->headers_out[n_headers_out - 1];
2158
2159 ho_ptr = ho->ptr;
2160 ho_nbytes = ho->n_bytes;
2161 }
2162 }
2163
2164 if (ho_ptr + ho_nbytes == hi_ptr) {
2165 ho_nbytes += n_bytes;
2166 } else {
2167 ho->n_bytes = ho_nbytes;
2168
2169 ho++;
2170 ho->ptr0 = hi_ptr0;
2171 ho->ptr = hi_ptr;
2172
2173 ho_ptr = hi_ptr;
2174 ho_nbytes = n_bytes;
2175
2176 n_headers_out++;
2177 }
2178 }
2179
2180 if (ho)
2181 ho->n_bytes = ho_nbytes;
2182 t->n_headers_out = n_headers_out;
2183}
2184
2185static inline void
2186__instr_hdr_emit_exec(struct rte_swx_pipeline *p,
2187 struct thread *t,
2188 const struct instruction *ip)
2189{
2190 __instr_hdr_emit_many_exec(p, t, ip, 1);
2191}
2192
2193static inline void
2194__instr_hdr_emit_tx_exec(struct rte_swx_pipeline *p,
2195 struct thread *t,
2196 const struct instruction *ip)
2197{
2198 TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n", p->thread_id);
2199
2200 __instr_hdr_emit_many_exec(p, t, ip, 1);
2201 __instr_tx_exec(p, t, ip);
2202}
2203
2204static inline void
2205__instr_hdr_emit2_tx_exec(struct rte_swx_pipeline *p,
2206 struct thread *t,
2207 const struct instruction *ip)
2208{
2209 TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n", p->thread_id);
2210
2211 __instr_hdr_emit_many_exec(p, t, ip, 2);
2212 __instr_tx_exec(p, t, ip);
2213}
2214
2215static inline void
2216__instr_hdr_emit3_tx_exec(struct rte_swx_pipeline *p,
2217 struct thread *t,
2218 const struct instruction *ip)
2219{
2220 TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n", p->thread_id);
2221
2222 __instr_hdr_emit_many_exec(p, t, ip, 3);
2223 __instr_tx_exec(p, t, ip);
2224}
2225
2226static inline void
2227__instr_hdr_emit4_tx_exec(struct rte_swx_pipeline *p,
2228 struct thread *t,
2229 const struct instruction *ip)
2230{
2231 TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n", p->thread_id);
2232
2233 __instr_hdr_emit_many_exec(p, t, ip, 4);
2234 __instr_tx_exec(p, t, ip);
2235}
2236
2237static inline void
2238__instr_hdr_emit5_tx_exec(struct rte_swx_pipeline *p,
2239 struct thread *t,
2240 const struct instruction *ip)
2241{
2242 TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n", p->thread_id);
2243
2244 __instr_hdr_emit_many_exec(p, t, ip, 5);
2245 __instr_tx_exec(p, t, ip);
2246}
2247
2248static inline void
2249__instr_hdr_emit6_tx_exec(struct rte_swx_pipeline *p,
2250 struct thread *t,
2251 const struct instruction *ip)
2252{
2253 TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n", p->thread_id);
2254
2255 __instr_hdr_emit_many_exec(p, t, ip, 6);
2256 __instr_tx_exec(p, t, ip);
2257}
2258
2259static inline void
2260__instr_hdr_emit7_tx_exec(struct rte_swx_pipeline *p,
2261 struct thread *t,
2262 const struct instruction *ip)
2263{
2264 TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n", p->thread_id);
2265
2266 __instr_hdr_emit_many_exec(p, t, ip, 7);
2267 __instr_tx_exec(p, t, ip);
2268}
2269
2270static inline void
2271__instr_hdr_emit8_tx_exec(struct rte_swx_pipeline *p,
2272 struct thread *t,
2273 const struct instruction *ip)
2274{
2275 TRACE("[Thread %2u] *** The next 9 instructions are fused. ***\n", p->thread_id);
2276
2277 __instr_hdr_emit_many_exec(p, t, ip, 8);
2278 __instr_tx_exec(p, t, ip);
2279}
2280
2281/*
2282 * validate.
2283 */
2284static inline void
2285__instr_hdr_validate_exec(struct rte_swx_pipeline *p __rte_unused,
2286 struct thread *t,
2287 const struct instruction *ip)
2288{
2289 uint32_t header_id = ip->valid.header_id;
2290 uint32_t struct_id = ip->valid.struct_id;
2291 uint64_t valid_headers = t->valid_headers;
2292 struct header_runtime *h = &t->headers[header_id];
2293
2294 TRACE("[Thread %2u] validate header %u\n", p->thread_id, header_id);
2295
2296 /* If this header is already valid, then its associated t->structs[] element is also valid
2297 * and therefore it should not be modified. It could point to the packet buffer (in case of
2298 * extracted header) and setting it to the default location (h->ptr0) would be incorrect.
2299 */
2300 if (MASK64_BIT_GET(valid_headers, header_id))
2301 return;
2302
2303 /* Headers. */
2304 t->structs[struct_id] = h->ptr0;
2305 t->valid_headers = MASK64_BIT_SET(valid_headers, header_id);
2306}
2307
2308/*
2309 * invalidate.
2310 */
2311static inline void
2312__instr_hdr_invalidate_exec(struct rte_swx_pipeline *p __rte_unused,
2313 struct thread *t,
2314 const struct instruction *ip)
2315{
2316 uint32_t header_id = ip->valid.header_id;
2317
2318 TRACE("[Thread %2u] invalidate header %u\n", p->thread_id, header_id);
2319
2320 /* Headers. */
2321 t->valid_headers = MASK64_BIT_CLR(t->valid_headers, header_id);
2322}
2323
2324/*
2325 * learn.
2326 */
2327static inline void
2328__instr_learn_exec(struct rte_swx_pipeline *p,
2329 struct thread *t,
2330 const struct instruction *ip)
2331{
2332 uint64_t action_id = ip->learn.action_id;
2333 uint32_t mf_first_arg_offset = ip->learn.mf_first_arg_offset;
2334 uint32_t timeout_id = METADATA_READ(t, ip->learn.mf_timeout_id_offset,
2335 ip->learn.mf_timeout_id_n_bits);
2336 uint32_t learner_id = t->learner_id;
2337 struct rte_swx_table_state *ts = &t->table_state[p->n_tables +
2338 p->n_selectors + learner_id];
2339 struct learner_runtime *l = &t->learners[learner_id];
2340 struct learner_statistics *stats = &p->learner_stats[learner_id];
2341 uint32_t status;
2342
2343 /* Table. */
2344 status = rte_swx_table_learner_add(ts->obj,
2345 l->mailbox,
2346 t->time,
2347 action_id,
2348 &t->metadata[mf_first_arg_offset],
2349 timeout_id);
2350
2351 TRACE("[Thread %2u] learner %u learn %s\n",
2352 p->thread_id,
2353 learner_id,
2354 status ? "ok" : "error");
2355
2356 stats->n_pkts_learn[status] += 1;
2357}
2358
2359/*
2360 * rearm.
2361 */
2362static inline void
2363__instr_rearm_exec(struct rte_swx_pipeline *p,
2364 struct thread *t,
2365 const struct instruction *ip __rte_unused)
2366{
2367 uint32_t learner_id = t->learner_id;
2368 struct rte_swx_table_state *ts = &t->table_state[p->n_tables +
2369 p->n_selectors + learner_id];
2370 struct learner_runtime *l = &t->learners[learner_id];
2371 struct learner_statistics *stats = &p->learner_stats[learner_id];
2372
2373 /* Table. */
2374 rte_swx_table_learner_rearm(ts->obj, l->mailbox, t->time);
2375
2376 TRACE("[Thread %2u] learner %u rearm\n",
2377 p->thread_id,
2378 learner_id);
2379
2380 stats->n_pkts_rearm += 1;
2381}
2382
2383static inline void
2384__instr_rearm_new_exec(struct rte_swx_pipeline *p,
2385 struct thread *t,
2386 const struct instruction *ip)
2387{
2388 uint32_t timeout_id = METADATA_READ(t, ip->learn.mf_timeout_id_offset,
2389 ip->learn.mf_timeout_id_n_bits);
2390 uint32_t learner_id = t->learner_id;
2391 struct rte_swx_table_state *ts = &t->table_state[p->n_tables +
2392 p->n_selectors + learner_id];
2393 struct learner_runtime *l = &t->learners[learner_id];
2394 struct learner_statistics *stats = &p->learner_stats[learner_id];
2395
2396 /* Table. */
2397 rte_swx_table_learner_rearm_new(ts->obj, l->mailbox, t->time, timeout_id);
2398
2399 TRACE("[Thread %2u] learner %u rearm with timeout ID %u\n",
2400 p->thread_id,
2401 learner_id,
2402 timeout_id);
2403
2404 stats->n_pkts_rearm += 1;
2405}
2406
2407/*
2408 * forget.
2409 */
2410static inline void
2411__instr_forget_exec(struct rte_swx_pipeline *p,
2412 struct thread *t,
2413 const struct instruction *ip __rte_unused)
2414{
2415 uint32_t learner_id = t->learner_id;
2416 struct rte_swx_table_state *ts = &t->table_state[p->n_tables +
2417 p->n_selectors + learner_id];
2418 struct learner_runtime *l = &t->learners[learner_id];
2419 struct learner_statistics *stats = &p->learner_stats[learner_id];
2420
2421 /* Table. */
2422 rte_swx_table_learner_delete(ts->obj, l->mailbox);
2423
2424 TRACE("[Thread %2u] learner %u forget\n",
2425 p->thread_id,
2426 learner_id);
2427
2428 stats->n_pkts_forget += 1;
2429}
2430
2431/*
2432 * entryid.
2433 */
2434static inline void
2435__instr_entryid_exec(struct rte_swx_pipeline *p __rte_unused,
2436 struct thread *t,
2437 const struct instruction *ip)
2438{
2439 TRACE("[Thread %2u]: entryid\n",
2440 p->thread_id);
2441
2442 /* Meta-data. */
2443 METADATA_WRITE(t, ip->mov.dst.offset, ip->mov.dst.n_bits, t->entry_id);
2444}
2445
2446/*
2447 * extern.
2448 */
2449static inline uint32_t
2450__instr_extern_obj_exec(struct rte_swx_pipeline *p __rte_unused,
2451 struct thread *t,
2452 const struct instruction *ip)
2453{
2454 uint32_t obj_id = ip->ext_obj.ext_obj_id;
2455 uint32_t func_id = ip->ext_obj.func_id;
2456 struct extern_obj_runtime *obj = &t->extern_objs[obj_id];
2457 rte_swx_extern_type_member_func_t func = obj->funcs[func_id];
2458 uint32_t done;
2459
2460 TRACE("[Thread %2u] extern obj %u member func %u\n",
2461 p->thread_id,
2462 obj_id,
2463 func_id);
2464
2465 done = func(obj->obj, obj->mailbox);
2466
2467 return done;
2468}
2469
2470static inline uint32_t
2471__instr_extern_func_exec(struct rte_swx_pipeline *p __rte_unused,
2472 struct thread *t,
2473 const struct instruction *ip)
2474{
2475 uint32_t ext_func_id = ip->ext_func.ext_func_id;
2476 struct extern_func_runtime *ext_func = &t->extern_funcs[ext_func_id];
2477 rte_swx_extern_func_t func = ext_func->func;
2478 uint32_t done;
2479
2480 TRACE("[Thread %2u] extern func %u\n",
2481 p->thread_id,
2482 ext_func_id);
2483
2484 done = func(ext_func->mailbox);
2485
2486 return done;
2487}
2488
2489/*
2490 * hash.
2491 */
2492static inline void
2493__instr_hash_func_exec(struct rte_swx_pipeline *p,
2494 struct thread *t,
2495 const struct instruction *ip)
2496{
2497 uint32_t hash_func_id = ip->hash_func.hash_func_id;
2498 uint32_t dst_offset = ip->hash_func.dst.offset;
2499 uint32_t n_dst_bits = ip->hash_func.dst.n_bits;
2500 uint32_t src_struct_id = ip->hash_func.src.struct_id;
2501 uint32_t src_offset = ip->hash_func.src.offset;
2502 uint32_t n_src_bytes = ip->hash_func.src.n_bytes;
2503
2504 struct hash_func_runtime *func = &p->hash_func_runtime[hash_func_id];
2505 uint8_t *src_ptr = t->structs[src_struct_id];
2506 uint32_t result;
2507
2508 TRACE("[Thread %2u] hash %u\n",
2509 p->thread_id,
2510 hash_func_id);
2511
2512 result = func->func(&src_ptr[src_offset], n_src_bytes, 0);
2513 METADATA_WRITE(t, dst_offset, n_dst_bits, result);
2514}
2515
2516/*
2517 * rss.
2518 */
2519static inline uint32_t
2520rss_func(void *rss_key, uint32_t rss_key_size, void *input_data, uint32_t input_data_size)
2521{
2522 uint32_t *key = (uint32_t *)rss_key;
2523 uint32_t *data = (uint32_t *)input_data;
2524 uint32_t key_size = rss_key_size >> 2;
2525 uint32_t data_size = input_data_size >> 2;
2526 uint32_t hash_val = 0, i;
2527
2528 for (i = 0; i < data_size; i++) {
2529 uint32_t d;
2530
2531 for (d = data[i]; d; d &= (d - 1)) {
2532 uint32_t key0, key1, pos;
2533
2534 pos = rte_bsf32(d);
2535 key0 = key[i % key_size] << (31 - pos);
2536 key1 = key[(i + 1) % key_size] >> (pos + 1);
2537 hash_val ^= key0 | key1;
2538 }
2539 }
2540
2541 return hash_val;
2542}
2543
2544static inline void
2545__instr_rss_exec(struct rte_swx_pipeline *p,
2546 struct thread *t,
2547 const struct instruction *ip)
2548{
2549 uint32_t rss_obj_id = ip->rss.rss_obj_id;
2550 uint32_t dst_offset = ip->rss.dst.offset;
2551 uint32_t n_dst_bits = ip->rss.dst.n_bits;
2552 uint32_t src_struct_id = ip->rss.src.struct_id;
2553 uint32_t src_offset = ip->rss.src.offset;
2554 uint32_t n_src_bytes = ip->rss.src.n_bytes;
2555
2556 struct rss_runtime *r = p->rss_runtime[rss_obj_id];
2557 uint8_t *src_ptr = t->structs[src_struct_id];
2558 uint32_t result;
2559
2560 TRACE("[Thread %2u] rss %u\n",
2561 p->thread_id,
2562 rss_obj_id);
2563
2564 result = rss_func(r->key, r->key_size, &src_ptr[src_offset], n_src_bytes);
2565 METADATA_WRITE(t, dst_offset, n_dst_bits, result);
2566}
2567
2568/*
2569 * mov.
2570 */
2571static inline void
2572__instr_mov_exec(struct rte_swx_pipeline *p __rte_unused,
2573 struct thread *t,
2574 const struct instruction *ip)
2575{
2576 TRACE("[Thread %2u] mov\n", p->thread_id);
2577
2578 MOV(t, ip);
2579}
2580
2581static inline void
2582__instr_mov_mh_exec(struct rte_swx_pipeline *p __rte_unused,
2583 struct thread *t,
2584 const struct instruction *ip)
2585{
2586 TRACE("[Thread %2u] mov (mh)\n", p->thread_id);
2587
2588 MOV_MH(t, ip);
2589}
2590
2591static inline void
2592__instr_mov_hm_exec(struct rte_swx_pipeline *p __rte_unused,
2593 struct thread *t,
2594 const struct instruction *ip)
2595{
2596 TRACE("[Thread %2u] mov (hm)\n", p->thread_id);
2597
2598 MOV_HM(t, ip);
2599}
2600
2601static inline void
2602__instr_mov_hh_exec(struct rte_swx_pipeline *p __rte_unused,
2603 struct thread *t,
2604 const struct instruction *ip)
2605{
2606 TRACE("[Thread %2u] mov (hh)\n", p->thread_id);
2607
2608 MOV_HH(t, ip);
2609}
2610
2611static inline void
2612__instr_mov_dma_exec(struct rte_swx_pipeline *p __rte_unused,
2613 struct thread *t,
2614 const struct instruction *ip)
2615{
2616 uint8_t *dst = t->structs[ip->mov.dst.struct_id] + ip->mov.dst.offset;
2617 uint8_t *src = t->structs[ip->mov.src.struct_id] + ip->mov.src.offset;
2618
2619 uint32_t n_dst = ip->mov.dst.n_bits >> 3;
2620 uint32_t n_src = ip->mov.src.n_bits >> 3;
2621
2622 TRACE("[Thread %2u] mov (dma) %u bytes\n", p->thread_id, n);
2623
2624 /* Both dst and src are in NBO format. */
2625 if (n_dst > n_src) {
2626 uint32_t n_dst_zero = n_dst - n_src;
2627
2628 /* Zero padding the most significant bytes in dst. */
2629 memset(dst, 0, n_dst_zero);
2630 dst += n_dst_zero;
2631
2632 /* Copy src to dst. */
2633 memcpy(dst, src, n_src);
2634 } else {
2635 uint32_t n_src_skipped = n_src - n_dst;
2636
2637 /* Copy src to dst. */
2638 src += n_src_skipped;
2639 memcpy(dst, src, n_dst);
2640 }
2641}
2642
2643static inline void
2644__instr_mov_128_exec(struct rte_swx_pipeline *p __rte_unused,
2645 struct thread *t,
2646 const struct instruction *ip)
2647{
2648 uint8_t *dst_struct = t->structs[ip->mov.dst.struct_id];
2649 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->mov.dst.offset];
2650
2651 uint8_t *src_struct = t->structs[ip->mov.src.struct_id];
2652 uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->mov.src.offset];
2653
2654 TRACE("[Thread %2u] mov (128)\n", p->thread_id);
2655
2656 dst64_ptr[0] = src64_ptr[0];
2657 dst64_ptr[1] = src64_ptr[1];
2658}
2659
2660static inline void
2661__instr_mov_128_32_exec(struct rte_swx_pipeline *p __rte_unused,
2662 struct thread *t,
2663 const struct instruction *ip)
2664{
2665 uint8_t *dst = t->structs[ip->mov.dst.struct_id] + ip->mov.dst.offset;
2666 uint8_t *src = t->structs[ip->mov.src.struct_id] + ip->mov.src.offset;
2667
2668 uint32_t *dst32 = (uint32_t *)dst;
2669 uint32_t *src32 = (uint32_t *)src;
2670
2671 TRACE("[Thread %2u] mov (128 <- 32)\n", p->thread_id);
2672
2673 dst32[0] = 0;
2674 dst32[1] = 0;
2675 dst32[2] = 0;
2676 dst32[3] = src32[0];
2677}
2678
2679static inline void
2680__instr_mov_i_exec(struct rte_swx_pipeline *p __rte_unused,
2681 struct thread *t,
2682 const struct instruction *ip)
2683{
2684 TRACE("[Thread %2u] mov m.f %" PRIx64 "\n", p->thread_id, ip->mov.src_val);
2685
2686 MOV_I(t, ip);
2687}
2688
2689/*
2690 * dma.
2691 */
2692static inline void
2693__instr_dma_ht_many_exec(struct rte_swx_pipeline *p __rte_unused,
2694 struct thread *t,
2695 const struct instruction *ip,
2696 uint32_t n_dma)
2697{
2698 uint8_t *action_data = t->structs[0];
2699 uint64_t valid_headers = t->valid_headers;
2700 uint32_t i;
2701
2702 for (i = 0; i < n_dma; i++) {
2703 uint32_t header_id = ip->dma.dst.header_id[i];
2704 uint32_t struct_id = ip->dma.dst.struct_id[i];
2705 uint32_t offset = ip->dma.src.offset[i];
2706 uint32_t n_bytes = ip->dma.n_bytes[i];
2707
2708 struct header_runtime *h = &t->headers[header_id];
2709 uint8_t *h_ptr0 = h->ptr0;
2710 uint8_t *h_ptr = t->structs[struct_id];
2711
2712 void *dst = MASK64_BIT_GET(valid_headers, header_id) ?
2713 h_ptr : h_ptr0;
2714 void *src = &action_data[offset];
2715
2716 TRACE("[Thread %2u] dma h.s t.f\n", p->thread_id);
2717
2718 /* Headers. */
2719 memcpy(dst, src, n_bytes);
2720 t->structs[struct_id] = dst;
2721 valid_headers = MASK64_BIT_SET(valid_headers, header_id);
2722 }
2723
2724 t->valid_headers = valid_headers;
2725}
2726
2727static inline void
2728__instr_dma_ht_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
2729{
2730 __instr_dma_ht_many_exec(p, t, ip, 1);
2731}
2732
2733static inline void
2734__instr_dma_ht2_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
2735{
2736 TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n", p->thread_id);
2737
2738 __instr_dma_ht_many_exec(p, t, ip, 2);
2739}
2740
2741static inline void
2742__instr_dma_ht3_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
2743{
2744 TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n", p->thread_id);
2745
2746 __instr_dma_ht_many_exec(p, t, ip, 3);
2747}
2748
2749static inline void
2750__instr_dma_ht4_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
2751{
2752 TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n", p->thread_id);
2753
2754 __instr_dma_ht_many_exec(p, t, ip, 4);
2755}
2756
2757static inline void
2758__instr_dma_ht5_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
2759{
2760 TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n", p->thread_id);
2761
2762 __instr_dma_ht_many_exec(p, t, ip, 5);
2763}
2764
2765static inline void
2766__instr_dma_ht6_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
2767{
2768 TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n", p->thread_id);
2769
2770 __instr_dma_ht_many_exec(p, t, ip, 6);
2771}
2772
2773static inline void
2774__instr_dma_ht7_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
2775{
2776 TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n", p->thread_id);
2777
2778 __instr_dma_ht_many_exec(p, t, ip, 7);
2779}
2780
2781static inline void
2782__instr_dma_ht8_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
2783{
2784 TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n", p->thread_id);
2785
2786 __instr_dma_ht_many_exec(p, t, ip, 8);
2787}
2788
2789/*
2790 * alu.
2791 */
2792static inline void
2793__instr_alu_add_exec(struct rte_swx_pipeline *p __rte_unused,
2794 struct thread *t,
2795 const struct instruction *ip)
2796{
2797 TRACE("[Thread %2u] add\n", p->thread_id);
2798
2799 ALU(t, ip, +);
2800}
2801
2802static inline void
2803__instr_alu_add_mh_exec(struct rte_swx_pipeline *p __rte_unused,
2804 struct thread *t,
2805 const struct instruction *ip)
2806{
2807 TRACE("[Thread %2u] add (mh)\n", p->thread_id);
2808
2809 ALU_MH(t, ip, +);
2810}
2811
2812static inline void
2813__instr_alu_add_hm_exec(struct rte_swx_pipeline *p __rte_unused,
2814 struct thread *t,
2815 const struct instruction *ip)
2816{
2817 TRACE("[Thread %2u] add (hm)\n", p->thread_id);
2818
2819 ALU_HM(t, ip, +);
2820}
2821
2822static inline void
2823__instr_alu_add_hh_exec(struct rte_swx_pipeline *p __rte_unused,
2824 struct thread *t,
2825 const struct instruction *ip)
2826{
2827 TRACE("[Thread %2u] add (hh)\n", p->thread_id);
2828
2829 ALU_HH(t, ip, +);
2830}
2831
2832static inline void
2833__instr_alu_add_mi_exec(struct rte_swx_pipeline *p __rte_unused,
2834 struct thread *t,
2835 const struct instruction *ip)
2836{
2837 TRACE("[Thread %2u] add (mi)\n", p->thread_id);
2838
2839 ALU_MI(t, ip, +);
2840}
2841
2842static inline void
2843__instr_alu_add_hi_exec(struct rte_swx_pipeline *p __rte_unused,
2844 struct thread *t,
2845 const struct instruction *ip)
2846{
2847 TRACE("[Thread %2u] add (hi)\n", p->thread_id);
2848
2849 ALU_HI(t, ip, +);
2850}
2851
2852static inline void
2853__instr_alu_sub_exec(struct rte_swx_pipeline *p __rte_unused,
2854 struct thread *t,
2855 const struct instruction *ip)
2856{
2857 TRACE("[Thread %2u] sub\n", p->thread_id);
2858
2859 ALU(t, ip, -);
2860}
2861
2862static inline void
2863__instr_alu_sub_mh_exec(struct rte_swx_pipeline *p __rte_unused,
2864 struct thread *t,
2865 const struct instruction *ip)
2866{
2867 TRACE("[Thread %2u] sub (mh)\n", p->thread_id);
2868
2869 ALU_MH(t, ip, -);
2870}
2871
2872static inline void
2873__instr_alu_sub_hm_exec(struct rte_swx_pipeline *p __rte_unused,
2874 struct thread *t,
2875 const struct instruction *ip)
2876{
2877 TRACE("[Thread %2u] sub (hm)\n", p->thread_id);
2878
2879 ALU_HM(t, ip, -);
2880}
2881
2882static inline void
2883__instr_alu_sub_hh_exec(struct rte_swx_pipeline *p __rte_unused,
2884 struct thread *t,
2885 const struct instruction *ip)
2886{
2887 TRACE("[Thread %2u] sub (hh)\n", p->thread_id);
2888
2889 ALU_HH(t, ip, -);
2890}
2891
2892static inline void
2893__instr_alu_sub_mi_exec(struct rte_swx_pipeline *p __rte_unused,
2894 struct thread *t,
2895 const struct instruction *ip)
2896{
2897 TRACE("[Thread %2u] sub (mi)\n", p->thread_id);
2898
2899 ALU_MI(t, ip, -);
2900}
2901
2902static inline void
2903__instr_alu_sub_hi_exec(struct rte_swx_pipeline *p __rte_unused,
2904 struct thread *t,
2905 const struct instruction *ip)
2906{
2907 TRACE("[Thread %2u] sub (hi)\n", p->thread_id);
2908
2909 ALU_HI(t, ip, -);
2910}
2911
2912static inline void
2913__instr_alu_shl_exec(struct rte_swx_pipeline *p __rte_unused,
2914 struct thread *t,
2915 const struct instruction *ip)
2916{
2917 TRACE("[Thread %2u] shl\n", p->thread_id);
2918
2919 ALU(t, ip, <<);
2920}
2921
2922static inline void
2923__instr_alu_shl_mh_exec(struct rte_swx_pipeline *p __rte_unused,
2924 struct thread *t,
2925 const struct instruction *ip)
2926{
2927 TRACE("[Thread %2u] shl (mh)\n", p->thread_id);
2928
2929 ALU_MH(t, ip, <<);
2930}
2931
2932static inline void
2933__instr_alu_shl_hm_exec(struct rte_swx_pipeline *p __rte_unused,
2934 struct thread *t,
2935 const struct instruction *ip)
2936{
2937 TRACE("[Thread %2u] shl (hm)\n", p->thread_id);
2938
2939 ALU_HM(t, ip, <<);
2940}
2941
2942static inline void
2943__instr_alu_shl_hh_exec(struct rte_swx_pipeline *p __rte_unused,
2944 struct thread *t,
2945 const struct instruction *ip)
2946{
2947 TRACE("[Thread %2u] shl (hh)\n", p->thread_id);
2948
2949 ALU_HH(t, ip, <<);
2950}
2951
2952static inline void
2953__instr_alu_shl_mi_exec(struct rte_swx_pipeline *p __rte_unused,
2954 struct thread *t,
2955 const struct instruction *ip)
2956{
2957 TRACE("[Thread %2u] shl (mi)\n", p->thread_id);
2958
2959 ALU_MI(t, ip, <<);
2960}
2961
2962static inline void
2963__instr_alu_shl_hi_exec(struct rte_swx_pipeline *p __rte_unused,
2964 struct thread *t,
2965 const struct instruction *ip)
2966{
2967 TRACE("[Thread %2u] shl (hi)\n", p->thread_id);
2968
2969 ALU_HI(t, ip, <<);
2970}
2971
2972static inline void
2973__instr_alu_shr_exec(struct rte_swx_pipeline *p __rte_unused,
2974 struct thread *t,
2975 const struct instruction *ip)
2976{
2977 TRACE("[Thread %2u] shr\n", p->thread_id);
2978
2979 ALU(t, ip, >>);
2980}
2981
2982static inline void
2983__instr_alu_shr_mh_exec(struct rte_swx_pipeline *p __rte_unused,
2984 struct thread *t,
2985 const struct instruction *ip)
2986{
2987 TRACE("[Thread %2u] shr (mh)\n", p->thread_id);
2988
2989 ALU_MH(t, ip, >>);
2990}
2991
2992static inline void
2993__instr_alu_shr_hm_exec(struct rte_swx_pipeline *p __rte_unused,
2994 struct thread *t,
2995 const struct instruction *ip)
2996{
2997 TRACE("[Thread %2u] shr (hm)\n", p->thread_id);
2998
2999 ALU_HM(t, ip, >>);
3000}
3001
3002static inline void
3003__instr_alu_shr_hh_exec(struct rte_swx_pipeline *p __rte_unused,
3004 struct thread *t,
3005 const struct instruction *ip)
3006{
3007 TRACE("[Thread %2u] shr (hh)\n", p->thread_id);
3008
3009 ALU_HH(t, ip, >>);
3010}
3011
3012static inline void
3013__instr_alu_shr_mi_exec(struct rte_swx_pipeline *p __rte_unused,
3014 struct thread *t,
3015 const struct instruction *ip)
3016{
3017 TRACE("[Thread %2u] shr (mi)\n", p->thread_id);
3018
3019 /* Structs. */
3020 ALU_MI(t, ip, >>);
3021}
3022
3023static inline void
3024__instr_alu_shr_hi_exec(struct rte_swx_pipeline *p __rte_unused,
3025 struct thread *t,
3026 const struct instruction *ip)
3027{
3028 TRACE("[Thread %2u] shr (hi)\n", p->thread_id);
3029
3030 ALU_HI(t, ip, >>);
3031}
3032
3033static inline void
3034__instr_alu_and_exec(struct rte_swx_pipeline *p __rte_unused,
3035 struct thread *t,
3036 const struct instruction *ip)
3037{
3038 TRACE("[Thread %2u] and\n", p->thread_id);
3039
3040 ALU(t, ip, &);
3041}
3042
3043static inline void
3044__instr_alu_and_mh_exec(struct rte_swx_pipeline *p __rte_unused,
3045 struct thread *t,
3046 const struct instruction *ip)
3047{
3048 TRACE("[Thread %2u] and (mh)\n", p->thread_id);
3049
3050 ALU_MH(t, ip, &);
3051}
3052
3053static inline void
3054__instr_alu_and_hm_exec(struct rte_swx_pipeline *p __rte_unused,
3055 struct thread *t,
3056 const struct instruction *ip)
3057{
3058 TRACE("[Thread %2u] and (hm)\n", p->thread_id);
3059
3060 ALU_HM_FAST(t, ip, &);
3061}
3062
3063static inline void
3064__instr_alu_and_hh_exec(struct rte_swx_pipeline *p __rte_unused,
3065 struct thread *t,
3066 const struct instruction *ip)
3067{
3068 TRACE("[Thread %2u] and (hh)\n", p->thread_id);
3069
3070 ALU_HH_FAST(t, ip, &);
3071}
3072
3073static inline void
3074__instr_alu_and_i_exec(struct rte_swx_pipeline *p __rte_unused,
3075 struct thread *t,
3076 const struct instruction *ip)
3077{
3078 TRACE("[Thread %2u] and (i)\n", p->thread_id);
3079
3080 ALU_I(t, ip, &);
3081}
3082
3083static inline void
3084__instr_alu_or_exec(struct rte_swx_pipeline *p __rte_unused,
3085 struct thread *t,
3086 const struct instruction *ip)
3087{
3088 TRACE("[Thread %2u] or\n", p->thread_id);
3089
3090 ALU(t, ip, |);
3091}
3092
3093static inline void
3094__instr_alu_or_mh_exec(struct rte_swx_pipeline *p __rte_unused,
3095 struct thread *t,
3096 const struct instruction *ip)
3097{
3098 TRACE("[Thread %2u] or (mh)\n", p->thread_id);
3099
3100 ALU_MH(t, ip, |);
3101}
3102
3103static inline void
3104__instr_alu_or_hm_exec(struct rte_swx_pipeline *p __rte_unused,
3105 struct thread *t,
3106 const struct instruction *ip)
3107{
3108 TRACE("[Thread %2u] or (hm)\n", p->thread_id);
3109
3110 ALU_HM_FAST(t, ip, |);
3111}
3112
3113static inline void
3114__instr_alu_or_hh_exec(struct rte_swx_pipeline *p __rte_unused,
3115 struct thread *t,
3116 const struct instruction *ip)
3117{
3118 TRACE("[Thread %2u] or (hh)\n", p->thread_id);
3119
3120 ALU_HH_FAST(t, ip, |);
3121}
3122
3123static inline void
3124__instr_alu_or_i_exec(struct rte_swx_pipeline *p __rte_unused,
3125 struct thread *t,
3126 const struct instruction *ip)
3127{
3128 TRACE("[Thread %2u] or (i)\n", p->thread_id);
3129
3130 ALU_I(t, ip, |);
3131}
3132
3133static inline void
3134__instr_alu_xor_exec(struct rte_swx_pipeline *p __rte_unused,
3135 struct thread *t,
3136 const struct instruction *ip)
3137{
3138 TRACE("[Thread %2u] xor\n", p->thread_id);
3139
3140 ALU(t, ip, ^);
3141}
3142
3143static inline void
3144__instr_alu_xor_mh_exec(struct rte_swx_pipeline *p __rte_unused,
3145 struct thread *t,
3146 const struct instruction *ip)
3147{
3148 TRACE("[Thread %2u] xor (mh)\n", p->thread_id);
3149
3150 ALU_MH(t, ip, ^);
3151}
3152
3153static inline void
3154__instr_alu_xor_hm_exec(struct rte_swx_pipeline *p __rte_unused,
3155 struct thread *t,
3156 const struct instruction *ip)
3157{
3158 TRACE("[Thread %2u] xor (hm)\n", p->thread_id);
3159
3160 ALU_HM_FAST(t, ip, ^);
3161}
3162
3163static inline void
3164__instr_alu_xor_hh_exec(struct rte_swx_pipeline *p __rte_unused,
3165 struct thread *t,
3166 const struct instruction *ip)
3167{
3168 TRACE("[Thread %2u] xor (hh)\n", p->thread_id);
3169
3170 ALU_HH_FAST(t, ip, ^);
3171}
3172
3173static inline void
3174__instr_alu_xor_i_exec(struct rte_swx_pipeline *p __rte_unused,
3175 struct thread *t,
3176 const struct instruction *ip)
3177{
3178 TRACE("[Thread %2u] xor (i)\n", p->thread_id);
3179
3180 ALU_I(t, ip, ^);
3181}
3182
3183static inline void
3184__instr_alu_ckadd_field_exec(struct rte_swx_pipeline *p __rte_unused,
3185 struct thread *t,
3186 const struct instruction *ip)
3187{
3188 uint8_t *dst_struct, *src_struct;
3189 uint16_t *dst16_ptr, dst;
3190 uint64_t *src64_ptr, src64, src64_mask, src;
3191 uint64_t r;
3192
3193 TRACE("[Thread %2u] ckadd (field)\n", p->thread_id);
3194
3195 /* Structs. */
3196 dst_struct = t->structs[ip->alu.dst.struct_id];
3197 dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
3198 dst = *dst16_ptr;
3199
3200 src_struct = t->structs[ip->alu.src.struct_id];
3201 src64_ptr = (uint64_t *)&src_struct[ip->alu.src.offset];
3202 src64 = *src64_ptr;
3203 src64_mask = UINT64_MAX >> (64 - ip->alu.src.n_bits);
3204 src = src64 & src64_mask;
3205
3206 /* Initialize the result with destination 1's complement. */
3207 r = dst;
3208 r = ~r & 0xFFFF;
3209
3210 /* The first input (r) is a 16-bit number. The second and the third
3211 * inputs are 32-bit numbers. In the worst case scenario, the sum of the
3212 * three numbers (output r) is a 34-bit number.
3213 */
3214 r += (src >> 32) + (src & 0xFFFFFFFF);
3215
3216 /* The first input is a 16-bit number. The second input is an 18-bit
3217 * number. In the worst case scenario, the sum of the two numbers is a
3218 * 19-bit number.
3219 */
3220 r = (r & 0xFFFF) + (r >> 16);
3221
3222 /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
3223 * a 3-bit number (0 .. 7). Their sum is a 17-bit number (0 .. 0x10006).
3224 */
3225 r = (r & 0xFFFF) + (r >> 16);
3226
3227 /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
3228 * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
3229 * 0x10006), the output r is (0 .. 7). So no carry bit can be generated,
3230 * therefore the output r is always a 16-bit number.
3231 */
3232 r = (r & 0xFFFF) + (r >> 16);
3233
3234 /* Apply 1's complement to the result. */
3235 r = ~r & 0xFFFF;
3236 r = r ? r : 0xFFFF;
3237
3238 *dst16_ptr = (uint16_t)r;
3239}
3240
3241static inline void
3242__instr_alu_cksub_field_exec(struct rte_swx_pipeline *p __rte_unused,
3243 struct thread *t,
3244 const struct instruction *ip)
3245{
3246 uint8_t *dst_struct, *src_struct;
3247 uint16_t *dst16_ptr, dst;
3248 uint64_t *src64_ptr, src64, src64_mask, src;
3249 uint64_t r;
3250
3251 TRACE("[Thread %2u] cksub (field)\n", p->thread_id);
3252
3253 /* Structs. */
3254 dst_struct = t->structs[ip->alu.dst.struct_id];
3255 dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
3256 dst = *dst16_ptr;
3257
3258 src_struct = t->structs[ip->alu.src.struct_id];
3259 src64_ptr = (uint64_t *)&src_struct[ip->alu.src.offset];
3260 src64 = *src64_ptr;
3261 src64_mask = UINT64_MAX >> (64 - ip->alu.src.n_bits);
3262 src = src64 & src64_mask;
3263
3264 /* Initialize the result with destination 1's complement. */
3265 r = dst;
3266 r = ~r & 0xFFFF;
3267
3268 /* Subtraction in 1's complement arithmetic (i.e. a '- b) is the same as
3269 * the following sequence of operations in 2's complement arithmetic:
3270 * a '- b = (a - b) % 0xFFFF.
3271 *
3272 * In order to prevent an underflow for the below subtraction, in which
3273 * a 33-bit number (the subtrahend) is taken out of a 16-bit number (the
3274 * minuend), we first add a multiple of the 0xFFFF modulus to the
3275 * minuend. The number we add to the minuend needs to be a 34-bit number
3276 * or higher, so for readability reasons we picked the 36-bit multiple.
3277 * We are effectively turning the 16-bit minuend into a 36-bit number:
3278 * (a - b) % 0xFFFF = (a + 0xFFFF00000 - b) % 0xFFFF.
3279 */
3280 r += 0xFFFF00000ULL; /* The output r is a 36-bit number. */
3281
3282 /* A 33-bit number is subtracted from a 36-bit number (the input r). The
3283 * result (the output r) is a 36-bit number.
3284 */
3285 r -= (src >> 32) + (src & 0xFFFFFFFF);
3286
3287 /* The first input is a 16-bit number. The second input is a 20-bit
3288 * number. Their sum is a 21-bit number.
3289 */
3290 r = (r & 0xFFFF) + (r >> 16);
3291
3292 /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
3293 * a 5-bit number (0 .. 31). The sum is a 17-bit number (0 .. 0x1001E).
3294 */
3295 r = (r & 0xFFFF) + (r >> 16);
3296
3297 /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
3298 * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
3299 * 0x1001E), the output r is (0 .. 31). So no carry bit can be
3300 * generated, therefore the output r is always a 16-bit number.
3301 */
3302 r = (r & 0xFFFF) + (r >> 16);
3303
3304 /* Apply 1's complement to the result. */
3305 r = ~r & 0xFFFF;
3306 r = r ? r : 0xFFFF;
3307
3308 *dst16_ptr = (uint16_t)r;
3309}
3310
3311static inline void
3312__instr_alu_ckadd_struct20_exec(struct rte_swx_pipeline *p __rte_unused,
3313 struct thread *t,
3314 const struct instruction *ip)
3315{
3316 uint8_t *dst_struct, *src_struct;
3317 uint16_t *dst16_ptr, dst;
3318 uint32_t *src32_ptr;
3319 uint64_t r0, r1;
3320
3321 TRACE("[Thread %2u] ckadd (struct of 20 bytes)\n", p->thread_id);
3322
3323 /* Structs. */
3324 dst_struct = t->structs[ip->alu.dst.struct_id];
3325 dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
3326 dst = *dst16_ptr;
3327
3328 src_struct = t->structs[ip->alu.src.struct_id];
3329 src32_ptr = (uint32_t *)&src_struct[0];
3330
3331 /* Initialize the result with destination 1's complement. */
3332 r0 = dst;
3333 r0 = ~r0 & 0xFFFF;
3334
3335 r0 += src32_ptr[0]; /* The output r0 is a 33-bit number. */
3336 r1 = src32_ptr[1]; /* r1 is a 32-bit number. */
3337 r0 += src32_ptr[2]; /* The output r0 is a 34-bit number. */
3338 r1 += src32_ptr[3]; /* The output r1 is a 33-bit number. */
3339 r0 += r1 + src32_ptr[4]; /* The output r0 is a 35-bit number. */
3340
3341 /* The first input is a 16-bit number. The second input is a 19-bit
3342 * number. Their sum is a 20-bit number.
3343 */
3344 r0 = (r0 & 0xFFFF) + (r0 >> 16);
3345
3346 /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
3347 * a 4-bit number (0 .. 15). The sum is a 17-bit number (0 .. 0x1000E).
3348 */
3349 r0 = (r0 & 0xFFFF) + (r0 >> 16);
3350
3351 /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
3352 * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
3353 * 0x1000E), the output r is (0 .. 15). So no carry bit can be
3354 * generated, therefore the output r is always a 16-bit number.
3355 */
3356 r0 = (r0 & 0xFFFF) + (r0 >> 16);
3357
3358 /* Apply 1's complement to the result. */
3359 r0 = ~r0 & 0xFFFF;
3360 r0 = r0 ? r0 : 0xFFFF;
3361
3362 *dst16_ptr = (uint16_t)r0;
3363}
3364
3365static inline void
3366__instr_alu_ckadd_struct_exec(struct rte_swx_pipeline *p __rte_unused,
3367 struct thread *t,
3368 const struct instruction *ip)
3369{
3370 uint32_t src_header_id = ip->alu.src.n_bits; /* The src header ID is stored here. */
3371 uint32_t n_src_header_bytes = t->headers[src_header_id].n_bytes;
3372 uint8_t *dst_struct, *src_struct;
3373 uint16_t *dst16_ptr, dst;
3374 uint32_t *src32_ptr;
3375 uint64_t r;
3376 uint32_t i;
3377
3378 if (n_src_header_bytes == 20) {
3379 __instr_alu_ckadd_struct20_exec(p, t, ip);
3380 return;
3381 }
3382
3383 TRACE("[Thread %2u] ckadd (struct)\n", p->thread_id);
3384
3385 /* Structs. */
3386 dst_struct = t->structs[ip->alu.dst.struct_id];
3387 dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
3388 dst = *dst16_ptr;
3389
3390 src_struct = t->structs[ip->alu.src.struct_id];
3391 src32_ptr = (uint32_t *)&src_struct[0];
3392
3393 /* Initialize the result with destination 1's complement. */
3394 r = dst;
3395 r = ~r & 0xFFFF;
3396
3397 /* The max number of 32-bit words in a 32K-byte header is 2^13.
3398 * Therefore, in the worst case scenario, a 45-bit number is added to a
3399 * 16-bit number (the input r), so the output r is 46-bit number.
3400 */
3401 for (i = 0; i < n_src_header_bytes / 4; i++, src32_ptr++)
3402 r += *src32_ptr;
3403
3404 /* The first input is a 16-bit number. The second input is a 30-bit
3405 * number. Their sum is a 31-bit number.
3406 */
3407 r = (r & 0xFFFF) + (r >> 16);
3408
3409 /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
3410 * a 15-bit number (0 .. 0x7FFF). The sum is a 17-bit number (0 .. 0x17FFE).
3411 */
3412 r = (r & 0xFFFF) + (r >> 16);
3413
3414 /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
3415 * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
3416 * 0x17FFE), the output r is (0 .. 0x7FFF). So no carry bit can be
3417 * generated, therefore the output r is always a 16-bit number.
3418 */
3419 r = (r & 0xFFFF) + (r >> 16);
3420
3421 /* Apply 1's complement to the result. */
3422 r = ~r & 0xFFFF;
3423 r = r ? r : 0xFFFF;
3424
3425 *dst16_ptr = (uint16_t)r;
3426}
3427
3428/*
3429 * Register array.
3430 */
3431static inline uint64_t *
3432instr_regarray_regarray(struct rte_swx_pipeline *p, const struct instruction *ip)
3433{
3434 struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
3435 return r->regarray;
3436}
3437
3438static inline uint64_t
3439instr_regarray_idx_hbo(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3440{
3441 struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
3442
3443 uint8_t *idx_struct = t->structs[ip->regarray.idx.struct_id];
3444 uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->regarray.idx.offset];
3445 uint64_t idx64 = *idx64_ptr;
3446 uint64_t idx64_mask = UINT64_MAX >> (64 - ip->regarray.idx.n_bits);
3447 uint64_t idx = idx64 & idx64_mask & r->size_mask;
3448
3449 return idx;
3450}
3451
3452#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
3453
3454static inline uint64_t
3455instr_regarray_idx_nbo(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3456{
3457 struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
3458
3459 uint8_t *idx_struct = t->structs[ip->regarray.idx.struct_id];
3460 uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->regarray.idx.offset];
3461 uint64_t idx64 = *idx64_ptr;
3462 uint64_t idx = (ntoh64(idx64) >> (64 - ip->regarray.idx.n_bits)) & r->size_mask;
3463
3464 return idx;
3465}
3466
3467#else
3468
3469#define instr_regarray_idx_nbo instr_regarray_idx_hbo
3470
3471#endif
3472
3473static inline uint64_t
3474instr_regarray_idx_imm(struct rte_swx_pipeline *p, const struct instruction *ip)
3475{
3476 struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
3477
3478 uint64_t idx = ip->regarray.idx_val & r->size_mask;
3479
3480 return idx;
3481}
3482
3483static inline uint64_t
3484instr_regarray_src_hbo(struct thread *t, const struct instruction *ip)
3485{
3486 uint8_t *src_struct = t->structs[ip->regarray.dstsrc.struct_id];
3487 uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->regarray.dstsrc.offset];
3488 uint64_t src64 = *src64_ptr;
3489 uint64_t src64_mask = UINT64_MAX >> (64 - ip->regarray.dstsrc.n_bits);
3490 uint64_t src = src64 & src64_mask;
3491
3492 return src;
3493}
3494
3495#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
3496
3497static inline uint64_t
3498instr_regarray_src_nbo(struct thread *t, const struct instruction *ip)
3499{
3500 uint8_t *src_struct = t->structs[ip->regarray.dstsrc.struct_id];
3501 uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->regarray.dstsrc.offset];
3502 uint64_t src64 = *src64_ptr;
3503 uint64_t src = ntoh64(src64) >> (64 - ip->regarray.dstsrc.n_bits);
3504
3505 return src;
3506}
3507
3508#else
3509
3510#define instr_regarray_src_nbo instr_regarray_src_hbo
3511
3512#endif
3513
3514static inline void
3515instr_regarray_dst_hbo_src_hbo_set(struct thread *t, const struct instruction *ip, uint64_t src)
3516{
3517 uint8_t *dst_struct = t->structs[ip->regarray.dstsrc.struct_id];
3518 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->regarray.dstsrc.offset];
3519 uint64_t dst64 = *dst64_ptr;
3520 uint64_t dst64_mask = UINT64_MAX >> (64 - ip->regarray.dstsrc.n_bits);
3521
3522 *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);
3523
3524}
3525
3526#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
3527
3528static inline void
3529instr_regarray_dst_nbo_src_hbo_set(struct thread *t, const struct instruction *ip, uint64_t src)
3530{
3531 uint8_t *dst_struct = t->structs[ip->regarray.dstsrc.struct_id];
3532 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->regarray.dstsrc.offset];
3533 uint64_t dst64 = *dst64_ptr;
3534 uint64_t dst64_mask = UINT64_MAX >> (64 - ip->regarray.dstsrc.n_bits);
3535
3536 src = hton64(src) >> (64 - ip->regarray.dstsrc.n_bits);
3537 *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);
3538}
3539
3540#else
3541
3542#define instr_regarray_dst_nbo_src_hbo_set instr_regarray_dst_hbo_src_hbo_set
3543
3544#endif
3545
3546static inline void
3547__instr_regprefetch_rh_exec(struct rte_swx_pipeline *p,
3548 struct thread *t,
3549 const struct instruction *ip)
3550{
3551 uint64_t *regarray, idx;
3552
3553 TRACE("[Thread %2u] regprefetch (r[h])\n", p->thread_id);
3554
3555 regarray = instr_regarray_regarray(p, ip);
3556 idx = instr_regarray_idx_nbo(p, t, ip);
3557 rte_prefetch0(&regarray[idx]);
3558}
3559
3560static inline void
3561__instr_regprefetch_rm_exec(struct rte_swx_pipeline *p,
3562 struct thread *t,
3563 const struct instruction *ip)
3564{
3565 uint64_t *regarray, idx;
3566
3567 TRACE("[Thread %2u] regprefetch (r[m])\n", p->thread_id);
3568
3569 regarray = instr_regarray_regarray(p, ip);
3570 idx = instr_regarray_idx_hbo(p, t, ip);
3571 rte_prefetch0(&regarray[idx]);
3572}
3573
3574static inline void
3575__instr_regprefetch_ri_exec(struct rte_swx_pipeline *p,
3576 struct thread *t __rte_unused,
3577 const struct instruction *ip)
3578{
3579 uint64_t *regarray, idx;
3580
3581 TRACE("[Thread %2u] regprefetch (r[i])\n", p->thread_id);
3582
3583 regarray = instr_regarray_regarray(p, ip);
3584 idx = instr_regarray_idx_imm(p, ip);
3585 rte_prefetch0(&regarray[idx]);
3586}
3587
3588static inline void
3589__instr_regrd_hrh_exec(struct rte_swx_pipeline *p,
3590 struct thread *t,
3591 const struct instruction *ip)
3592{
3593 uint64_t *regarray, idx;
3594
3595 TRACE("[Thread %2u] regrd (h = r[h])\n", p->thread_id);
3596
3597 regarray = instr_regarray_regarray(p, ip);
3598 idx = instr_regarray_idx_nbo(p, t, ip);
3599 instr_regarray_dst_nbo_src_hbo_set(t, ip, regarray[idx]);
3600}
3601
3602static inline void
3603__instr_regrd_hrm_exec(struct rte_swx_pipeline *p,
3604 struct thread *t,
3605 const struct instruction *ip)
3606{
3607 uint64_t *regarray, idx;
3608
3609 TRACE("[Thread %2u] regrd (h = r[m])\n", p->thread_id);
3610
3611 /* Structs. */
3612 regarray = instr_regarray_regarray(p, ip);
3613 idx = instr_regarray_idx_hbo(p, t, ip);
3614 instr_regarray_dst_nbo_src_hbo_set(t, ip, regarray[idx]);
3615}
3616
3617static inline void
3618__instr_regrd_mrh_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3619{
3620 uint64_t *regarray, idx;
3621
3622 TRACE("[Thread %2u] regrd (m = r[h])\n", p->thread_id);
3623
3624 regarray = instr_regarray_regarray(p, ip);
3625 idx = instr_regarray_idx_nbo(p, t, ip);
3626 instr_regarray_dst_hbo_src_hbo_set(t, ip, regarray[idx]);
3627}
3628
3629static inline void
3630__instr_regrd_mrm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3631{
3632 uint64_t *regarray, idx;
3633
3634 TRACE("[Thread %2u] regrd (m = r[m])\n", p->thread_id);
3635
3636 regarray = instr_regarray_regarray(p, ip);
3637 idx = instr_regarray_idx_hbo(p, t, ip);
3638 instr_regarray_dst_hbo_src_hbo_set(t, ip, regarray[idx]);
3639}
3640
3641static inline void
3642__instr_regrd_hri_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3643{
3644 uint64_t *regarray, idx;
3645
3646 TRACE("[Thread %2u] regrd (h = r[i])\n", p->thread_id);
3647
3648 regarray = instr_regarray_regarray(p, ip);
3649 idx = instr_regarray_idx_imm(p, ip);
3650 instr_regarray_dst_nbo_src_hbo_set(t, ip, regarray[idx]);
3651}
3652
3653static inline void
3654__instr_regrd_mri_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3655{
3656 uint64_t *regarray, idx;
3657
3658 TRACE("[Thread %2u] regrd (m = r[i])\n", p->thread_id);
3659
3660 regarray = instr_regarray_regarray(p, ip);
3661 idx = instr_regarray_idx_imm(p, ip);
3662 instr_regarray_dst_hbo_src_hbo_set(t, ip, regarray[idx]);
3663}
3664
3665static inline void
3666__instr_regwr_rhh_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3667{
3668 uint64_t *regarray, idx, src;
3669
3670 TRACE("[Thread %2u] regwr (r[h] = h)\n", p->thread_id);
3671
3672 regarray = instr_regarray_regarray(p, ip);
3673 idx = instr_regarray_idx_nbo(p, t, ip);
3674 src = instr_regarray_src_nbo(t, ip);
3675 regarray[idx] = src;
3676}
3677
3678static inline void
3679__instr_regwr_rhm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3680{
3681 uint64_t *regarray, idx, src;
3682
3683 TRACE("[Thread %2u] regwr (r[h] = m)\n", p->thread_id);
3684
3685 regarray = instr_regarray_regarray(p, ip);
3686 idx = instr_regarray_idx_nbo(p, t, ip);
3687 src = instr_regarray_src_hbo(t, ip);
3688 regarray[idx] = src;
3689}
3690
3691static inline void
3692__instr_regwr_rmh_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3693{
3694 uint64_t *regarray, idx, src;
3695
3696 TRACE("[Thread %2u] regwr (r[m] = h)\n", p->thread_id);
3697
3698 regarray = instr_regarray_regarray(p, ip);
3699 idx = instr_regarray_idx_hbo(p, t, ip);
3700 src = instr_regarray_src_nbo(t, ip);
3701 regarray[idx] = src;
3702}
3703
3704static inline void
3705__instr_regwr_rmm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3706{
3707 uint64_t *regarray, idx, src;
3708
3709 TRACE("[Thread %2u] regwr (r[m] = m)\n", p->thread_id);
3710
3711 regarray = instr_regarray_regarray(p, ip);
3712 idx = instr_regarray_idx_hbo(p, t, ip);
3713 src = instr_regarray_src_hbo(t, ip);
3714 regarray[idx] = src;
3715}
3716
3717static inline void
3718__instr_regwr_rhi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3719{
3720 uint64_t *regarray, idx, src;
3721
3722 TRACE("[Thread %2u] regwr (r[h] = i)\n", p->thread_id);
3723
3724 regarray = instr_regarray_regarray(p, ip);
3725 idx = instr_regarray_idx_nbo(p, t, ip);
3726 src = ip->regarray.dstsrc_val;
3727 regarray[idx] = src;
3728}
3729
3730static inline void
3731__instr_regwr_rmi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3732{
3733 uint64_t *regarray, idx, src;
3734
3735 TRACE("[Thread %2u] regwr (r[m] = i)\n", p->thread_id);
3736
3737 regarray = instr_regarray_regarray(p, ip);
3738 idx = instr_regarray_idx_hbo(p, t, ip);
3739 src = ip->regarray.dstsrc_val;
3740 regarray[idx] = src;
3741}
3742
3743static inline void
3744__instr_regwr_rih_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3745{
3746 uint64_t *regarray, idx, src;
3747
3748 TRACE("[Thread %2u] regwr (r[i] = h)\n", p->thread_id);
3749
3750 regarray = instr_regarray_regarray(p, ip);
3751 idx = instr_regarray_idx_imm(p, ip);
3752 src = instr_regarray_src_nbo(t, ip);
3753 regarray[idx] = src;
3754}
3755
3756static inline void
3757__instr_regwr_rim_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3758{
3759 uint64_t *regarray, idx, src;
3760
3761 TRACE("[Thread %2u] regwr (r[i] = m)\n", p->thread_id);
3762
3763 regarray = instr_regarray_regarray(p, ip);
3764 idx = instr_regarray_idx_imm(p, ip);
3765 src = instr_regarray_src_hbo(t, ip);
3766 regarray[idx] = src;
3767}
3768
3769static inline void
3770__instr_regwr_rii_exec(struct rte_swx_pipeline *p,
3771 struct thread *t __rte_unused,
3772 const struct instruction *ip)
3773{
3774 uint64_t *regarray, idx, src;
3775
3776 TRACE("[Thread %2u] regwr (r[i] = i)\n", p->thread_id);
3777
3778 regarray = instr_regarray_regarray(p, ip);
3779 idx = instr_regarray_idx_imm(p, ip);
3780 src = ip->regarray.dstsrc_val;
3781 regarray[idx] = src;
3782}
3783
3784static inline void
3785__instr_regadd_rhh_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3786{
3787 uint64_t *regarray, idx, src;
3788
3789 TRACE("[Thread %2u] regadd (r[h] += h)\n", p->thread_id);
3790
3791 regarray = instr_regarray_regarray(p, ip);
3792 idx = instr_regarray_idx_nbo(p, t, ip);
3793 src = instr_regarray_src_nbo(t, ip);
3794 regarray[idx] += src;
3795}
3796
3797static inline void
3798__instr_regadd_rhm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3799{
3800 uint64_t *regarray, idx, src;
3801
3802 TRACE("[Thread %2u] regadd (r[h] += m)\n", p->thread_id);
3803
3804 regarray = instr_regarray_regarray(p, ip);
3805 idx = instr_regarray_idx_nbo(p, t, ip);
3806 src = instr_regarray_src_hbo(t, ip);
3807 regarray[idx] += src;
3808}
3809
3810static inline void
3811__instr_regadd_rmh_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3812{
3813 uint64_t *regarray, idx, src;
3814
3815 TRACE("[Thread %2u] regadd (r[m] += h)\n", p->thread_id);
3816
3817 regarray = instr_regarray_regarray(p, ip);
3818 idx = instr_regarray_idx_hbo(p, t, ip);
3819 src = instr_regarray_src_nbo(t, ip);
3820 regarray[idx] += src;
3821}
3822
3823static inline void
3824__instr_regadd_rmm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3825{
3826 uint64_t *regarray, idx, src;
3827
3828 TRACE("[Thread %2u] regadd (r[m] += m)\n", p->thread_id);
3829
3830 regarray = instr_regarray_regarray(p, ip);
3831 idx = instr_regarray_idx_hbo(p, t, ip);
3832 src = instr_regarray_src_hbo(t, ip);
3833 regarray[idx] += src;
3834}
3835
3836static inline void
3837__instr_regadd_rhi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3838{
3839 uint64_t *regarray, idx, src;
3840
3841 TRACE("[Thread %2u] regadd (r[h] += i)\n", p->thread_id);
3842
3843 regarray = instr_regarray_regarray(p, ip);
3844 idx = instr_regarray_idx_nbo(p, t, ip);
3845 src = ip->regarray.dstsrc_val;
3846 regarray[idx] += src;
3847}
3848
3849static inline void
3850__instr_regadd_rmi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3851{
3852 uint64_t *regarray, idx, src;
3853
3854 TRACE("[Thread %2u] regadd (r[m] += i)\n", p->thread_id);
3855
3856 regarray = instr_regarray_regarray(p, ip);
3857 idx = instr_regarray_idx_hbo(p, t, ip);
3858 src = ip->regarray.dstsrc_val;
3859 regarray[idx] += src;
3860}
3861
3862static inline void
3863__instr_regadd_rih_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3864{
3865 uint64_t *regarray, idx, src;
3866
3867 TRACE("[Thread %2u] regadd (r[i] += h)\n", p->thread_id);
3868
3869 regarray = instr_regarray_regarray(p, ip);
3870 idx = instr_regarray_idx_imm(p, ip);
3871 src = instr_regarray_src_nbo(t, ip);
3872 regarray[idx] += src;
3873}
3874
3875static inline void
3876__instr_regadd_rim_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3877{
3878 uint64_t *regarray, idx, src;
3879
3880 TRACE("[Thread %2u] regadd (r[i] += m)\n", p->thread_id);
3881
3882 regarray = instr_regarray_regarray(p, ip);
3883 idx = instr_regarray_idx_imm(p, ip);
3884 src = instr_regarray_src_hbo(t, ip);
3885 regarray[idx] += src;
3886}
3887
3888static inline void
3889__instr_regadd_rii_exec(struct rte_swx_pipeline *p,
3890 struct thread *t __rte_unused,
3891 const struct instruction *ip)
3892{
3893 uint64_t *regarray, idx, src;
3894
3895 TRACE("[Thread %2u] regadd (r[i] += i)\n", p->thread_id);
3896
3897 regarray = instr_regarray_regarray(p, ip);
3898 idx = instr_regarray_idx_imm(p, ip);
3899 src = ip->regarray.dstsrc_val;
3900 regarray[idx] += src;
3901}
3902
3903/*
3904 * metarray.
3905 */
3906static inline struct meter *
3907instr_meter_idx_hbo(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3908{
3909 struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id];
3910
3911 uint8_t *idx_struct = t->structs[ip->meter.idx.struct_id];
3912 uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->meter.idx.offset];
3913 uint64_t idx64 = *idx64_ptr;
3914 uint64_t idx64_mask = UINT64_MAX >> (64 - (ip)->meter.idx.n_bits);
3915 uint64_t idx = idx64 & idx64_mask & r->size_mask;
3916
3917 return &r->metarray[idx];
3918}
3919
3920#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
3921
3922static inline struct meter *
3923instr_meter_idx_nbo(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3924{
3925 struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id];
3926
3927 uint8_t *idx_struct = t->structs[ip->meter.idx.struct_id];
3928 uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->meter.idx.offset];
3929 uint64_t idx64 = *idx64_ptr;
3930 uint64_t idx = (ntoh64(idx64) >> (64 - ip->meter.idx.n_bits)) & r->size_mask;
3931
3932 return &r->metarray[idx];
3933}
3934
3935#else
3936
3937#define instr_meter_idx_nbo instr_meter_idx_hbo
3938
3939#endif
3940
3941static inline struct meter *
3942instr_meter_idx_imm(struct rte_swx_pipeline *p, const struct instruction *ip)
3943{
3944 struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id];
3945
3946 uint64_t idx = ip->meter.idx_val & r->size_mask;
3947
3948 return &r->metarray[idx];
3949}
3950
3951static inline uint32_t
3952instr_meter_length_hbo(struct thread *t, const struct instruction *ip)
3953{
3954 uint8_t *src_struct = t->structs[ip->meter.length.struct_id];
3955 uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.length.offset];
3956 uint64_t src64 = *src64_ptr;
3957 uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->meter.length.n_bits);
3958 uint64_t src = src64 & src64_mask;
3959
3960 return (uint32_t)src;
3961}
3962
3963#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
3964
3965static inline uint32_t
3966instr_meter_length_nbo(struct thread *t, const struct instruction *ip)
3967{
3968 uint8_t *src_struct = t->structs[ip->meter.length.struct_id];
3969 uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.length.offset];
3970 uint64_t src64 = *src64_ptr;
3971 uint64_t src = ntoh64(src64) >> (64 - ip->meter.length.n_bits);
3972
3973 return (uint32_t)src;
3974}
3975
3976#else
3977
3978#define instr_meter_length_nbo instr_meter_length_hbo
3979
3980#endif
3981
3982static inline enum rte_color
3983instr_meter_color_in_hbo(struct thread *t, const struct instruction *ip)
3984{
3985 uint8_t *src_struct = t->structs[ip->meter.color_in.struct_id];
3986 uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.color_in.offset];
3987 uint64_t src64 = *src64_ptr;
3988 uint64_t src64_mask = UINT64_MAX >> (64 - ip->meter.color_in.n_bits);
3989 uint64_t src = src64 & src64_mask;
3990
3991 return (enum rte_color)src;
3992}
3993
3994static inline void
3995instr_meter_color_out_hbo_set(struct thread *t,
3996 const struct instruction *ip,
3997 enum rte_color color_out)
3998{
3999 uint8_t *dst_struct = t->structs[ip->meter.color_out.struct_id];
4000 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->meter.color_out.offset];
4001 uint64_t dst64 = *dst64_ptr;
4002 uint64_t dst64_mask = UINT64_MAX >> (64 - ip->meter.color_out.n_bits);
4003
4004 uint64_t src = (uint64_t)color_out;
4005
4006 *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);
4007}
4008
4009static inline void
4010__instr_metprefetch_h_exec(struct rte_swx_pipeline *p,
4011 struct thread *t,
4012 const struct instruction *ip)
4013{
4014 struct meter *m;
4015
4016 TRACE("[Thread %2u] metprefetch (h)\n", p->thread_id);
4017
4018 m = instr_meter_idx_nbo(p, t, ip);
4019 rte_prefetch0(m);
4020}
4021
4022static inline void
4023__instr_metprefetch_m_exec(struct rte_swx_pipeline *p,
4024 struct thread *t,
4025 const struct instruction *ip)
4026{
4027 struct meter *m;
4028
4029 TRACE("[Thread %2u] metprefetch (m)\n", p->thread_id);
4030
4031 m = instr_meter_idx_hbo(p, t, ip);
4032 rte_prefetch0(m);
4033}
4034
4035static inline void
4036__instr_metprefetch_i_exec(struct rte_swx_pipeline *p,
4037 struct thread *t __rte_unused,
4038 const struct instruction *ip)
4039{
4040 struct meter *m;
4041
4042 TRACE("[Thread %2u] metprefetch (i)\n", p->thread_id);
4043
4044 m = instr_meter_idx_imm(p, ip);
4045 rte_prefetch0(m);
4046}
4047
4048static inline void
4049__instr_meter_hhm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
4050{
4051 struct meter *m;
4052 uint64_t time, n_pkts, n_bytes;
4053 uint32_t length;
4054 enum rte_color color_in, color_out;
4055
4056 TRACE("[Thread %2u] meter (hhm)\n", p->thread_id);
4057
4058 m = instr_meter_idx_nbo(p, t, ip);
4059 rte_prefetch0(m->n_pkts);
4060 time = rte_get_tsc_cycles();
4061 length = instr_meter_length_nbo(t, ip);
4062 color_in = instr_meter_color_in_hbo(t, ip);
4063
4064 color_out = rte_meter_trtcm_color_aware_check(&m->m,
4065 &m->profile->profile,
4066 time,
4067 length,
4068 color_in);
4069
4070 color_out &= m->color_mask;
4071
4072 n_pkts = m->n_pkts[color_out];
4073 n_bytes = m->n_bytes[color_out];
4074
4075 instr_meter_color_out_hbo_set(t, ip, color_out);
4076
4077 m->n_pkts[color_out] = n_pkts + 1;
4078 m->n_bytes[color_out] = n_bytes + length;
4079}
4080
4081static inline void
4082__instr_meter_hhi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
4083{
4084 struct meter *m;
4085 uint64_t time, n_pkts, n_bytes;
4086 uint32_t length;
4087 enum rte_color color_in, color_out;
4088
4089 TRACE("[Thread %2u] meter (hhi)\n", p->thread_id);
4090
4091 m = instr_meter_idx_nbo(p, t, ip);
4092 rte_prefetch0(m->n_pkts);
4093 time = rte_get_tsc_cycles();
4094 length = instr_meter_length_nbo(t, ip);
4095 color_in = (enum rte_color)ip->meter.color_in_val;
4096
4097 color_out = rte_meter_trtcm_color_aware_check(&m->m,
4098 &m->profile->profile,
4099 time,
4100 length,
4101 color_in);
4102
4103 color_out &= m->color_mask;
4104
4105 n_pkts = m->n_pkts[color_out];
4106 n_bytes = m->n_bytes[color_out];
4107
4108 instr_meter_color_out_hbo_set(t, ip, color_out);
4109
4110 m->n_pkts[color_out] = n_pkts + 1;
4111 m->n_bytes[color_out] = n_bytes + length;
4112}
4113
4114static inline void
4115__instr_meter_hmm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
4116{
4117 struct meter *m;
4118 uint64_t time, n_pkts, n_bytes;
4119 uint32_t length;
4120 enum rte_color color_in, color_out;
4121
4122 TRACE("[Thread %2u] meter (hmm)\n", p->thread_id);
4123
4124 m = instr_meter_idx_nbo(p, t, ip);
4125 rte_prefetch0(m->n_pkts);
4126 time = rte_get_tsc_cycles();
4127 length = instr_meter_length_hbo(t, ip);
4128 color_in = instr_meter_color_in_hbo(t, ip);
4129
4130 color_out = rte_meter_trtcm_color_aware_check(&m->m,
4131 &m->profile->profile,
4132 time,
4133 length,
4134 color_in);
4135
4136 color_out &= m->color_mask;
4137
4138 n_pkts = m->n_pkts[color_out];
4139 n_bytes = m->n_bytes[color_out];
4140
4141 instr_meter_color_out_hbo_set(t, ip, color_out);
4142
4143 m->n_pkts[color_out] = n_pkts + 1;
4144 m->n_bytes[color_out] = n_bytes + length;
4145}
4146
4147static inline void
4148__instr_meter_hmi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
4149{
4150 struct meter *m;
4151 uint64_t time, n_pkts, n_bytes;
4152 uint32_t length;
4153 enum rte_color color_in, color_out;
4154
4155 TRACE("[Thread %2u] meter (hmi)\n", p->thread_id);
4156
4157 m = instr_meter_idx_nbo(p, t, ip);
4158 rte_prefetch0(m->n_pkts);
4159 time = rte_get_tsc_cycles();
4160 length = instr_meter_length_hbo(t, ip);
4161 color_in = (enum rte_color)ip->meter.color_in_val;
4162
4163 color_out = rte_meter_trtcm_color_aware_check(&m->m,
4164 &m->profile->profile,
4165 time,
4166 length,
4167 color_in);
4168
4169 color_out &= m->color_mask;
4170
4171 n_pkts = m->n_pkts[color_out];
4172 n_bytes = m->n_bytes[color_out];
4173
4174 instr_meter_color_out_hbo_set(t, ip, color_out);
4175
4176 m->n_pkts[color_out] = n_pkts + 1;
4177 m->n_bytes[color_out] = n_bytes + length;
4178}
4179
4180static inline void
4181__instr_meter_mhm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
4182{
4183 struct meter *m;
4184 uint64_t time, n_pkts, n_bytes;
4185 uint32_t length;
4186 enum rte_color color_in, color_out;
4187
4188 TRACE("[Thread %2u] meter (mhm)\n", p->thread_id);
4189
4190 m = instr_meter_idx_hbo(p, t, ip);
4191 rte_prefetch0(m->n_pkts);
4192 time = rte_get_tsc_cycles();
4193 length = instr_meter_length_nbo(t, ip);
4194 color_in = instr_meter_color_in_hbo(t, ip);
4195
4196 color_out = rte_meter_trtcm_color_aware_check(&m->m,
4197 &m->profile->profile,
4198 time,
4199 length,
4200 color_in);
4201
4202 color_out &= m->color_mask;
4203
4204 n_pkts = m->n_pkts[color_out];
4205 n_bytes = m->n_bytes[color_out];
4206
4207 instr_meter_color_out_hbo_set(t, ip, color_out);
4208
4209 m->n_pkts[color_out] = n_pkts + 1;
4210 m->n_bytes[color_out] = n_bytes + length;
4211}
4212
4213static inline void
4214__instr_meter_mhi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
4215{
4216 struct meter *m;
4217 uint64_t time, n_pkts, n_bytes;
4218 uint32_t length;
4219 enum rte_color color_in, color_out;
4220
4221 TRACE("[Thread %2u] meter (mhi)\n", p->thread_id);
4222
4223 m = instr_meter_idx_hbo(p, t, ip);
4224 rte_prefetch0(m->n_pkts);
4225 time = rte_get_tsc_cycles();
4226 length = instr_meter_length_nbo(t, ip);
4227 color_in = (enum rte_color)ip->meter.color_in_val;
4228
4229 color_out = rte_meter_trtcm_color_aware_check(&m->m,
4230 &m->profile->profile,
4231 time,
4232 length,
4233 color_in);
4234
4235 color_out &= m->color_mask;
4236
4237 n_pkts = m->n_pkts[color_out];
4238 n_bytes = m->n_bytes[color_out];
4239
4240 instr_meter_color_out_hbo_set(t, ip, color_out);
4241
4242 m->n_pkts[color_out] = n_pkts + 1;
4243 m->n_bytes[color_out] = n_bytes + length;
4244}
4245
4246static inline void
4247__instr_meter_mmm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
4248{
4249 struct meter *m;
4250 uint64_t time, n_pkts, n_bytes;
4251 uint32_t length;
4252 enum rte_color color_in, color_out;
4253
4254 TRACE("[Thread %2u] meter (mmm)\n", p->thread_id);
4255
4256 m = instr_meter_idx_hbo(p, t, ip);
4257 rte_prefetch0(m->n_pkts);
4258 time = rte_get_tsc_cycles();
4259 length = instr_meter_length_hbo(t, ip);
4260 color_in = instr_meter_color_in_hbo(t, ip);
4261
4262 color_out = rte_meter_trtcm_color_aware_check(&m->m,
4263 &m->profile->profile,
4264 time,
4265 length,
4266 color_in);
4267
4268 color_out &= m->color_mask;
4269
4270 n_pkts = m->n_pkts[color_out];
4271 n_bytes = m->n_bytes[color_out];
4272
4273 instr_meter_color_out_hbo_set(t, ip, color_out);
4274
4275 m->n_pkts[color_out] = n_pkts + 1;
4276 m->n_bytes[color_out] = n_bytes + length;
4277}
4278
4279static inline void
4280__instr_meter_mmi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
4281{
4282 struct meter *m;
4283 uint64_t time, n_pkts, n_bytes;
4284 uint32_t length;
4285 enum rte_color color_in, color_out;
4286
4287 TRACE("[Thread %2u] meter (mmi)\n", p->thread_id);
4288
4289 m = instr_meter_idx_hbo(p, t, ip);
4290 rte_prefetch0(m->n_pkts);
4291 time = rte_get_tsc_cycles();
4292 length = instr_meter_length_hbo(t, ip);
4293 color_in = (enum rte_color)ip->meter.color_in_val;
4294
4295 color_out = rte_meter_trtcm_color_aware_check(&m->m,
4296 &m->profile->profile,
4297 time,
4298 length,
4299 color_in);
4300
4301 color_out &= m->color_mask;
4302
4303 n_pkts = m->n_pkts[color_out];
4304 n_bytes = m->n_bytes[color_out];
4305
4306 instr_meter_color_out_hbo_set(t, ip, color_out);
4307
4308 m->n_pkts[color_out] = n_pkts + 1;
4309 m->n_bytes[color_out] = n_bytes + length;
4310}
4311
4312static inline void
4313__instr_meter_ihm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
4314{
4315 struct meter *m;
4316 uint64_t time, n_pkts, n_bytes;
4317 uint32_t length;
4318 enum rte_color color_in, color_out;
4319
4320 TRACE("[Thread %2u] meter (ihm)\n", p->thread_id);
4321
4322 m = instr_meter_idx_imm(p, ip);
4323 rte_prefetch0(m->n_pkts);
4324 time = rte_get_tsc_cycles();
4325 length = instr_meter_length_nbo(t, ip);
4326 color_in = instr_meter_color_in_hbo(t, ip);
4327
4328 color_out = rte_meter_trtcm_color_aware_check(&m->m,
4329 &m->profile->profile,
4330 time,
4331 length,
4332 color_in);
4333
4334 color_out &= m->color_mask;
4335
4336 n_pkts = m->n_pkts[color_out];
4337 n_bytes = m->n_bytes[color_out];
4338
4339 instr_meter_color_out_hbo_set(t, ip, color_out);
4340
4341 m->n_pkts[color_out] = n_pkts + 1;
4342 m->n_bytes[color_out] = n_bytes + length;
4343}
4344
4345static inline void
4346__instr_meter_ihi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
4347{
4348 struct meter *m;
4349 uint64_t time, n_pkts, n_bytes;
4350 uint32_t length;
4351 enum rte_color color_in, color_out;
4352
4353 TRACE("[Thread %2u] meter (ihi)\n", p->thread_id);
4354
4355 m = instr_meter_idx_imm(p, ip);
4356 rte_prefetch0(m->n_pkts);
4357 time = rte_get_tsc_cycles();
4358 length = instr_meter_length_nbo(t, ip);
4359 color_in = (enum rte_color)ip->meter.color_in_val;
4360
4361 color_out = rte_meter_trtcm_color_aware_check(&m->m,
4362 &m->profile->profile,
4363 time,
4364 length,
4365 color_in);
4366
4367 color_out &= m->color_mask;
4368
4369 n_pkts = m->n_pkts[color_out];
4370 n_bytes = m->n_bytes[color_out];
4371
4372 instr_meter_color_out_hbo_set(t, ip, color_out);
4373
4374 m->n_pkts[color_out] = n_pkts + 1;
4375 m->n_bytes[color_out] = n_bytes + length;
4376}
4377
4378static inline void
4379__instr_meter_imm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
4380{
4381 struct meter *m;
4382 uint64_t time, n_pkts, n_bytes;
4383 uint32_t length;
4384 enum rte_color color_in, color_out;
4385
4386 TRACE("[Thread %2u] meter (imm)\n", p->thread_id);
4387
4388 m = instr_meter_idx_imm(p, ip);
4389 rte_prefetch0(m->n_pkts);
4390 time = rte_get_tsc_cycles();
4391 length = instr_meter_length_hbo(t, ip);
4392 color_in = instr_meter_color_in_hbo(t, ip);
4393
4394 color_out = rte_meter_trtcm_color_aware_check(&m->m,
4395 &m->profile->profile,
4396 time,
4397 length,
4398 color_in);
4399
4400 color_out &= m->color_mask;
4401
4402 n_pkts = m->n_pkts[color_out];
4403 n_bytes = m->n_bytes[color_out];
4404
4405 instr_meter_color_out_hbo_set(t, ip, color_out);
4406
4407 m->n_pkts[color_out] = n_pkts + 1;
4408 m->n_bytes[color_out] = n_bytes + length;
4409}
4410
4411static inline void
4412__instr_meter_imi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
4413{
4414 struct meter *m;
4415 uint64_t time, n_pkts, n_bytes;
4416 uint32_t length;
4417 enum rte_color color_in, color_out;
4418
4419 TRACE("[Thread %2u] meter (imi)\n", p->thread_id);
4420
4421 m = instr_meter_idx_imm(p, ip);
4422 rte_prefetch0(m->n_pkts);
4423 time = rte_get_tsc_cycles();
4424 length = instr_meter_length_hbo(t, ip);
4425 color_in = (enum rte_color)ip->meter.color_in_val;
4426
4427 color_out = rte_meter_trtcm_color_aware_check(&m->m,
4428 &m->profile->profile,
4429 time,
4430 length,
4431 color_in);
4432
4433 color_out &= m->color_mask;
4434
4435 n_pkts = m->n_pkts[color_out];
4436 n_bytes = m->n_bytes[color_out];
4437
4438 instr_meter_color_out_hbo_set(t, ip, color_out);
4439
4440 m->n_pkts[color_out] = n_pkts + 1;
4441 m->n_bytes[color_out] = n_bytes + length;
4442}
4443
4444#endif
static uint32_t rte_bsf32(uint32_t v)
Definition rte_bitops.h:487
#define __rte_unused
Definition rte_common.h:143
static uint64_t rte_get_tsc_cycles(void)
static enum rte_color rte_meter_trtcm_color_aware_check(struct rte_meter_trtcm *m, struct rte_meter_trtcm_profile *p, uint64_t time, uint32_t pkt_len, enum rte_color pkt_color)
Definition rte_meter.h:522
rte_color
Definition rte_meter.h:32
@ RTE_COLORS
Definition rte_meter.h:36
static void rte_prefetch0(const volatile void *p)
void(* rte_swx_extern_type_destructor_t)(void *object)
int(* rte_swx_extern_func_t)(void *mailbox)
int(* rte_swx_extern_type_member_func_t)(void *object, void *mailbox)
void *(* rte_swx_extern_type_constructor_t)(const char *args)
uint32_t(* rte_swx_hash_func_t)(const void *key, uint32_t length, uint32_t seed)
#define RTE_SWX_NAME_SIZE
int(* rte_swx_port_in_pkt_rx_t)(void *port, struct rte_swx_pkt *pkt)
void(* rte_swx_port_out_flush_t)(void *port)
void(* rte_swx_port_out_pkt_clone_tx_t)(void *port, struct rte_swx_pkt *pkt, uint32_t truncation_length)
void(* rte_swx_port_out_pkt_tx_t)(void *port, struct rte_swx_pkt *pkt)
void(* rte_swx_port_out_pkt_fast_clone_tx_t)(void *port, struct rte_swx_pkt *pkt)
rte_swx_table_match_type
int(* rte_swx_table_lookup_t)(void *table, void *mailbox, uint8_t **key, uint64_t *action_id, uint8_t **action_data, size_t *entry_id, int *hit)
__rte_experimental uint32_t rte_swx_table_learner_add(void *table, void *mailbox, uint64_t time, uint64_t action_id, uint8_t *action_data, uint32_t key_timeout_id)
__rte_experimental void rte_swx_table_learner_rearm_new(void *table, void *mailbox, uint64_t time, uint32_t key_timeout_id)
__rte_experimental void rte_swx_table_learner_delete(void *table, void *mailbox)
__rte_experimental void rte_swx_table_learner_rearm(void *table, void *mailbox, uint64_t time)
#define RTE_SWX_TABLE_LEARNER_N_KEY_TIMEOUTS_MAX
uint8_t * pkt
uint32_t offset
uint32_t length