Wiredancer F2 Support (#5916)

* Add double-buffered terminal rendering to wd frank mon

* Add double-buffered terminal rendering to fd frank mon

* Make colorized output column-safe in monitor formatting functions

* Add missing sha512_pre module from 902fb24

* Renaming top_f1 to top_wd

* Add auto-generated aws-fpga hdk project files for cl_wiredancer

Source: https://github.com/aws/aws-fpga
Version: v2.1.1
License: Amazon Software License

* Add wiredancer design for f2 platform

* Update README with f2 instance support

Author: monological

src/app/frank/fd_frank_mon.bin.c

@@ -330,6 +330,9 @@ int pretty_num(char* st, uint64_t cnt, char* suffix)
     return sel;
 }
 
+#define MAX_LINES 128
+#define MAX_LINE_WIDTH 512
+
 void* mon_thread(void* arg)
 {
     uint64_t cnts[2][64];
@@ -338,6 +341,9 @@ void* mon_thread(void* arg)
 
     memset(cnts, 0, sizeof(cnts));
 
+    char buffer_prev[MAX_LINES][MAX_LINE_WIDTH] = {0};
+    char buffer_curr[MAX_LINES][MAX_LINE_WIDTH] = {0};
+
     int first = 1;
     uint32_t from[2] = {0, 0};
     uint32_t to[2] = {0, 0};
@@ -356,6 +362,11 @@ void* mon_thread(void* arg)
 
         if (!first)
             ascii_move_to(from, to);
+
+        /* switch to alternate buffer, clear, hide cursor */
+        if (first) {
+            printf("\033[?1049h\033[2J\033[?25l");
+        }
 
         from[0] = 0;
 
@@ -491,11 +502,11 @@ void* mon_thread(void* arg)
         }
 
         // draw ascii art
-        for (uint32_t li = 0; li < 1024; li ++)
+        for (uint32_t li = 0; li < MAX_LINES; li ++)
         {
             if (ascii_chart[li][0] == 0)
                 break;
-            char out_st[512*64];
+            char* out_st = buffer_curr[li];
             int out_pos = 0;
             for (uint32_t ci = 0;; ci ++)
             {
@@ -529,9 +540,9 @@ void* mon_thread(void* arg)
                         if (anm_data[anm_i][0] == 0)
                             break;
                         if (1
-                            & (anm_data[anm_i][0] + (anm_data[anm_i][2] * anm_data[anm_i][7]) == (int)li)
-                            & (anm_data[anm_i][1] + (anm_data[anm_i][3] * anm_data[anm_i][7]) == (int)ci)
-                            & (cnts[1][anm_data[anm_i][6]] != 0)
+                            && (anm_data[anm_i][0] + (anm_data[anm_i][2] * anm_data[anm_i][7]) == (int)li)
+                            && (anm_data[anm_i][1] + (anm_data[anm_i][3] * anm_data[anm_i][7]) == (int)ci)
+                            && (cnts[1][anm_data[anm_i][6]] != 0)
                         )
                         {
                             out_pos += ascii_color(out_st+out_pos, (uint32_t)anm_data[anm_i][5]);
@@ -547,8 +558,13 @@ void* mon_thread(void* arg)
                 else
                     out_pos += ascii_color(out_st+out_pos, 0);
             }
-            printf ("%s\n", out_st);
-            from[0] ++;
+            out_st[out_pos] = '\0';
+            /* only print changed lines – clear to EOL first */
+            if (strcmp(buffer_prev[li], buffer_curr[li]) != 0)
+            {
+                printf("\033[%d;1H\033[K%s", li + 1, buffer_curr[li]);
+                strcpy(buffer_prev[li], buffer_curr[li]);
+            }
         }
 
         // update animation states
@@ -574,5 +590,9 @@ void* mon_thread(void* arg)
         fflush(stdout);
     }
     state->stopped = 1;
+
+    /* show cursor, restore normal screen buffer */
+    printf("\033[?25h\033[?1049l");
+
     return 0;
 }

src/app/frank/wd_frank_f1_mon.c

@@ -3,6 +3,7 @@ This directory includes all resources to build and use Wiredancer functionalitie
 
 ## Supported Platforms ##
 * AWS-F1 series
+* AWS-F2 series
 
 ## Available Functions ##
 * SigVerify
@@ -18,51 +19,55 @@ WD adopts an asynchronous API.  In fact WD uses the same Tango mcache mechanism
 - `wd_free_pci(wd_wksp_t*)`
   - Frees PCIe resources.
 
+# Suggestion from our friends at ABK: Running WD Demo using Pulumi #
 
+To quickly run the Wiredancer Demo without manually building and running everything from scratch, use ABK Lab's Pulumi project. It will provision an AWS F2 instance, install all required dependencies, and launch the demo application.
 
+You can either provide your own AFGI ID or use the one included in the project.
 
+[svmkit-examples-wiredancer](https://github.com/abklabs/svmkit-examples-wiredancer)
 
-
-
-
-
-
-
-
-
-
-#  #
 # Building WD #
 
-## AWS-F1 Series ##
-
-+ To build for AWS-F1 series EC2 instances, you need an EC2 build machine.  Detailes to provision such instance are provided in [AWS-FPGA github page](https://github.com/aws/aws-fpga).
++ To build for AWS-F1 or AWS-F2 series EC2 instances, you need an EC2 build machine.  Details to provision such instance are provided in [AWS-FPGA github page](https://github.com/aws/aws-fpga).
 
 + Inside the build machine, clone AWS-FPGA git repo
   - `git clone https://github.com/aws/aws-fpga`
+  -  for AWS-F2 series, you also need to checkout the `f2` branch
+
+## AWS-F1 Series ##
 
 - Follow the repo's instructions to build `<AWS-FPGA>/hdk/cl/examples/cl_dram_dma`
++ Copy all files from `<FD>/src/wiredancer/platforms/f1/design` directory into `<AWS-FPGA>/hdk/cl/examples/cl_dram_dma/design` replacing existing files.
++ Copy all files from `<FD>/src/wiredancer/rtl` directory into `<AWS-FPGA>/hdk/cl/examples/cl_dram_dma/design` replacing existing files.
++ Rebuild `<AWS-FPGA>/hdk/cl/examples/cl_dram_dma` with the same instructions from AWS repo as before.
 
-+ Copy all files from `<FD>/wiredancer/platforms/f1/build` directory into `<AWS-FPGA>/hdk/cl/examples/cl_dram_dma` replacing existing files.
+## AWS-F2 Series ##
 
-+ Rebuild `<AWS-FPGA>/hdk/cl/examples/cl_dram_dma` with the same instructions from AWS repo as before.
+- Follow the repo's instructions to create a new cl example`<AWS-FPGA>/hdk/cl/examples/cl_wiredancer`
++ Copy all files from `<FD>/src/wiredancer/platforms/f2/design` directory into `<AWS-FPGA>/hdk/cl/examples/cl_wiredancer/design` replacing existing files.
++ Copy all files from `<FD>/src/wiredancer/rtl` directory into `<AWS-FPGA>/hdk/cl/examples/cl_wiredancer/design` replacing existing files.
++ Rebuild `<AWS-FPGA>/hdk/cl/examples/cl_wiredancer` with the same instructions from AWS repo as before.
 
+## Generate Bitstream ##
+
++ Follow the repo's instructions to generate the bitstream from the dcp file and get an AGFI for the cl example you just built.
 
-#  #
 # Running WD #
 
-## AWS-F1 Series ##
+## AWS-F1 and AWS-F2 Series ##
 
-+ To run FD with WD support, you need an EC2 F1 machine.
++ To run FD with WD support, you need an EC2 F1 or F2 machine.
 
 + Inside the machine, clone AWS-FPGA git repo
   - `git clone https://github.com/aws/aws-fpga`
+  -  for AWS-F2 series, you also need to checkout the `f2` branch
 
 + Install the SDK inside the repo:
   - `source $AWS-FPGA/sdk_setup.sh`
 
 + Load WD image on FPGA slot-0
-  - `sudo fpga-load-local-image -S 0 -I agfi-01051ff14d1bba4e0`
+  - `sudo fpga-load-local-image -S 0 -I <AGFI-ID>`
 
 + Make FD with WD support
   - `./deps.sh`
@@ -77,8 +82,12 @@ WD adopts an asynchronous API.  In fact WD uses the same Tango mcache mechanism
   - `sudo build/linux/gcc/x86_64/bin/fd_shmem_cfg query`
 
 + Configure app-frank
+  - copy over the ./misc/solana_pcap file to the machine (e.g. `/tmp/solana.pcap`)
   - `sudo ./build/linux/gcc/x86_64/bin/fd_frank_init_demo frank 1-6 ./build/linux/gcc/x86_64 /tmp/solana.pcap 0 0 1 0`
 
++ Enable pcie bus mastering
+ - `sudo setpci -s 34:00.0 command=06`
+
 + Run app-frank
   - `sudo ./build/linux/gcc/x86_64/bin/fd_frank_run frank "1-6"`
 
@@ -88,12 +97,6 @@ WD adopts an asynchronous API.  In fact WD uses the same Tango mcache mechanism
 + Run fd-monitor
   - `sudo taskset -c 7 build/linux/gcc/x86_64/bin/fd_frank_mon frank --duration 10e12 --dt-min 1e7 --dt-max 1e7`
 
-
-
-
-
-
-#  #
 # WD-SigVerify #
 
 SigVerify is the verification process of [ED25519](https://en.wikipedia.org/wiki/EdDSA).  This is a computationally intesive operation.  In order to match SigVerify's throughput with the rest of the FD system, many high performance cores are required.  However WD.SigVerify uses hardware acceleration to achieve 1Mps throughput using only one FPGA.  Table below shows the throughput of a single core running FD.SigVerify on various architectures, and the number of cores required to reach a throughput of one million per second.

src/wiredancer/README.md

@@ -491,7 +491,7 @@ showahead_fifo #(
 // AAAAAAA                   AAAAAAAPPPPPPPPPP          PPPPPPPPPP          
 
 `ifndef TOP_NAME
-`define TOP_NAME top_f1
+`define TOP_NAME top_wd
 `endif
 
 `TOP_NAME #(

src/wiredancer/platform/f1/design/cl_dram_dma.sv

@@ -103,7 +103,7 @@ read_verilog -sv $CL_DIR/design/ed25519_sigverify_ecc.sv
 read_verilog -sv $CL_DIR/design/ed25519_sigverify_0.sv
 read_verilog -sv $CL_DIR/design/ed25519_sigverify_1.sv
 read_verilog -sv $CL_DIR/design/ed25519_sigverify_2.sv
-read_verilog -sv $CL_DIR/design/top_f1.sv
+read_verilog -sv $CL_DIR/design/top_wd.sv
 
 
 puts "AWS FPGA: Reading IP blocks";

src/wiredancer/platform/f1/scripts/synth_cl_dram_dma.tcl

@@ -0,0 +1,18 @@
+# =============================================================================
+# Amazon FPGA Hardware Development Kit
+#
+# Copyright 2024 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Amazon Software License (the "License"). You may not use
+# this file except in compliance with the License. A copy of the License is
+# located at
+#
+#    http://aws.amazon.com/asl/
+#
+# or in the "license" file accompanying this file. This file is distributed on
+# an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, express or
+# implied. See the License for the specific language governing permissions and
+# limitations under the License.
+# =============================================================================
+
+set_property MAX_FANOUT 50 [get_nets -of_objects [get_pins CL_PCIM_MSTR/CL_TST_PCI/sync_rst_n_reg/Q]]

src/wiredancer/platform/f2/constraints/cl_synth_user.xdc

@@ -0,0 +1,32 @@
+# =============================================================================
+# Amazon FPGA Hardware Development Kit
+#
+# Copyright 2024 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Amazon Software License (the "License"). You may not use
+# this file except in compliance with the License. A copy of the License is
+# located at
+#
+#    http://aws.amazon.com/asl/
+#
+# or in the "license" file accompanying this file. This file is distributed on
+# an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, express or
+# implied. See the License for the specific language governing permissions and
+# limitations under the License.
+# =============================================================================
+
+
+#################################################################################
+### Generated Clocks
+#################################################################################
+# Alias of Shell interface clock
+set clk_main_a0 [get_clocks -of_objects [get_ports clk_main_a0]]
+
+
+#################################################################################
+### Clock Groups
+#################################################################################
+# false path inside sh_ddr
+set_false_path -from [get_pins -of_objects \
+                          [get_cells -hierarchical -filter { NAME =~ *ram_reg*}] -filter {REF_PIN_NAME == CLK}] \
+               -to   [get_cells -hierarchical -filter { NAME =~ *rd_do_reg[*]}]

src/wiredancer/platform/f2/constraints/cl_timing_user.xdc

@@ -0,0 +1,132 @@
+# =============================================================================
+# Amazon FPGA Hardware Development Kit
+#
+# Copyright 2024 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Amazon Software License (the "License"). You may not use
+# this file except in compliance with the License. A copy of the License is
+# located at
+#
+#    http://aws.amazon.com/asl/
+#
+# or in the "license" file accompanying this file. This file is distributed on
+# an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, express or
+# implied. See the License for the specific language governing permissions and
+# limitations under the License.
+# =============================================================================
+
+# Level 1 Wiredancer floorplan for Small Shell
+
+
+###############################################################################
+# Child Pblock in SLR2
+###############################################################################
+########################################
+# Pblock
+########################################
+create_pblock pblock_CL_SLR2
+
+# Complete CRs in SLR2
+# Match the parent pblock width to avoid DRC errors
+resize_pblock pblock_CL_SLR2 -add {CLOCKREGION_X0Y8:CLOCKREGION_X5Y11}
+
+set_property parent pblock_CL [get_pblocks pblock_CL_SLR2]
+
+###############################################################################
+# Child Pblock in SLR1
+###############################################################################
+########################################
+# Pblock
+########################################
+create_pblock pblock_CL_SLR1
+
+# Complete CRs in SLR1
+resize_pblock pblock_CL_SLR1 -add {CLOCKREGION_X0Y4:CLOCKREGION_X3Y7}
+resize_pblock pblock_CL_SLR1 -add {CLOCKREGION_X5Y4:CLOCKREGION_X5Y7}
+resize_pblock pblock_CL_SLR1 -add {CLOCKREGION_X4Y6:CLOCKREGION_X4Y7}
+resize_pblock pblock_CL_SLR1 -add {CLOCKREGION_X4Y4:CLOCKREGION_X4Y4}
+
+# Partial CRs
+resize_pblock pblock_CL_SLR1 -add {SLICE_X122Y300:SLICE_X145Y359 \
+                                   DSP48E2_X16Y114:DSP48E2_X19Y137 \
+                                   RAMB18_X9Y120:RAMB18_X9Y143 \
+                                   RAMB36_X9Y60:RAMB36_X9Y71 \
+                                   URAM288_X2Y80:URAM288_X2Y95}
+resize_pblock pblock_CL_SLR1 -add {RAMB18_X8Y120:RAMB18_X8Y143 \
+                                   RAMB36_X8Y60:RAMB36_X8Y71}
+
+set_property SNAPPING_MODE ON [get_pblocks pblock_CL_SLR1]
+
+set_property parent pblock_CL [get_pblocks pblock_CL_SLR1]
+
+###############################################################################
+# Child Pblock in SLR0
+###############################################################################
+########################################
+# Pblock
+########################################
+create_pblock pblock_CL_SLR0
+
+# Complete CRs
+resize_pblock pblock_CL_SLR0 -add {CLOCKREGION_X0Y0:CLOCKREGION_X3Y3}
+resize_pblock pblock_CL_SLR0 -add {CLOCKREGION_X4Y0:CLOCKREGION_X7Y0}
+resize_pblock pblock_CL_SLR0 -add {CLOCKREGION_X4Y3:CLOCKREGION_X4Y3}
+resize_pblock pblock_CL_SLR0 -add {CLOCKREGION_X6Y1:CLOCKREGION_X6Y1}
+resize_pblock pblock_CL_SLR0 -add {CLOCKREGION_X5Y1:CLOCKREGION_X5Y3}
+
+# Partial CRs
+resize_pblock pblock_CL_SLR0 -add {SLICE_X120Y120:SLICE_X145Y179 \
+                                   DSP48E2_X16Y42:DSP48E2_X19Y65 \
+                                   RAMB18_X8Y48:RAMB18_X9Y71 \
+                                   RAMB36_X8Y24:RAMB36_X9Y35 \
+                                   URAM288_X2Y32:URAM288_X2Y47}
+
+resize_pblock pblock_CL_SLR0 -add {SLICE_X118Y60:SLICE_X145Y119 \
+                                   DSP48E2_X16Y18:DSP48E2_X19Y41 \
+                                   RAMB18_X8Y24:RAMB18_X9Y47 \
+                                   RAMB36_X8Y12:RAMB36_X9Y23 \
+                                   URAM288_X2Y16:URAM288_X2Y31}
+
+resize_pblock pblock_CL_SLR0 -add {SLICE_X206Y60:SLICE_X219Y119 \
+                                   DSP48E2_X30Y18:DSP48E2_X30Y41 \
+                                   RAMB18_X12Y24:RAMB18_X12Y47 \
+                                   RAMB36_X12Y12:RAMB36_X12Y23}
+
+resize_pblock pblock_CL_SLR0 -add {SLICE_X221Y60:SLICE_X232Y119 \
+                                   DSP48E2_X31Y18:DSP48E2_X31Y41 \
+                                   RAMB18_X13Y24:RAMB18_X13Y47 \
+                                   RAMB36_X13Y12:RAMB36_X13Y23 \
+                                   BUFG_GT_X1Y24:BUFG_GT_X1Y47 \
+                                   BUFG_GT_SYNC_X1Y15:BUFG_GT_SYNC_X1Y29 \
+                                   GTYE4_COMMON_X1Y1:GTYE4_COMMON_X1Y1 \
+                                   GTYE4_CHANNEL_X1Y4:GTYE4_CHANNEL_X1Y7}
+
+
+set_property parent pblock_CL [get_pblocks pblock_CL_SLR0]
+
+
+########################################
+# Wiredancer
+########################################
+
+# PCIS logic: appears in DMA/PCIM bridging in cl_wiredancer
+add_cells_to_pblock [get_pblocks pblock_CL_SLR2] [get_cells -hierarchical -filter {NAME =~ WRAPPER/CL/cl_wiredancer/st_in_*}]
+add_cells_to_pblock [get_pblocks pblock_CL_SLR2] [get_cells -hierarchical -filter {NAME =~ WRAPPER/CL/cl_wiredancer/dma_*}]
+add_cells_to_pblock [get_pblocks pblock_CL_SLR2] [get_cells -hierarchical -filter {NAME =~ WRAPPER/CL/cl_wiredancer/pcim_*}]
+
+# PCIM logic: PCIM bridge master interface
+add_cells_to_pblock [get_pblocks pblock_CL_SLR1] [get_cells -hierarchical -filter {NAME =~ WRAPPER/CL/cl_wiredancer/cl_sh_pcim*}]
+
+# OCL AXI-lite logic: control path state machine
+add_cells_to_pblock [get_pblocks pblock_CL_SLR1] [get_cells -hierarchical -filter {NAME =~ WRAPPER/CL/cl_wiredancer/st_ocl*}]
+add_cells_to_pblock [get_pblocks pblock_CL_SLR1] [get_cells -hierarchical -filter {NAME =~ WRAPPER/CL/cl_wiredancer/avmm_fh_*}]
+
+
+# Monitoring/status (vdip, vled, bresp_status)
+add_cells_to_pblock [get_pblocks pblock_CL_SLR0] [get_cells -hierarchical -filter {NAME =~ WRAPPER/CL/cl_wiredancer/vdip_*}]
+add_cells_to_pblock [get_pblocks pblock_CL_SLR0] [get_cells -hierarchical -filter {NAME =~ WRAPPER/CL/cl_wiredancer/bresp_status}]
+
+# Previously all `top_inst` modules were forced into SLR1. This can lead to
+# placement failures due to resource pressure. Allow Vivado to distribute the
+# logic across SLRs by not constraining these modules to SLR1.
+#add_cells_to_pblock [get_pblocks pblock_CL_SLR1] [get_cells -hierarchical -filter {NAME =~ WRAPPER/CL/top_inst*}]