diff --git a/.gitmodules b/.gitmodules index 9efd5cc..9fb8e08 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,6 @@ [submodule "framework/host/json"] path = framework/host/json url = https://github.com/nlohmann/json.git +[submodule "framework/fpga/f4pga-examples"] + path = framework/fpga/f4pga-examples + url = https://github.com/chipsalliance/f4pga-examples diff --git a/README.md b/README.md index 258a40e..f6d6eb6 100644 --- a/README.md +++ b/README.md @@ -112,7 +112,7 @@ Looking specifically at the Amazon F1 platform, F1 provides powerful Xilinx FPGA - Documentation is often misleading as APIs and infrastructure are evolving. - External dependencies are poorly managed, so tutorials break at random. - - Xilinx tools, Vivado and SDAccel, while powerful, are difficult to learn and use, slow, and arcane. + - Xilinx tools, Vivado and Vitis, while powerful, are difficult to learn and use, slow, and arcane. - OpenCL is a whole other beast, built for folks who want to design hardware like it's software... which it obviously isn't. - Developers must understand AXI protocols and manage AXI controllers. - The AWS platform can be intimidating to a newcomer. diff --git a/apps/mandelbrot/prebuilt/hw/xilinx_aws-vu9p-f1_4ddr-xpr-2pr_4.0/host b/apps/mandelbrot/prebuilt/hw/xilinx_aws-vu9p-f1_4ddr-xpr-2pr_4.0/host index 084d4ff..c23dad5 100755 Binary files a/apps/mandelbrot/prebuilt/hw/xilinx_aws-vu9p-f1_4ddr-xpr-2pr_4.0/host and b/apps/mandelbrot/prebuilt/hw/xilinx_aws-vu9p-f1_4ddr-xpr-2pr_4.0/host differ diff --git a/apps/mandelbrot/prebuilt/hw/xilinx_aws-vu9p-f1_4ddr-xpr-2pr_4.0/mandelbrot.awsxclbin b/apps/mandelbrot/prebuilt/hw/xilinx_aws-vu9p-f1_4ddr-xpr-2pr_4.0/mandelbrot.awsxclbin index 431370d..f7ded9e 100644 Binary files a/apps/mandelbrot/prebuilt/hw/xilinx_aws-vu9p-f1_4ddr-xpr-2pr_4.0/mandelbrot.awsxclbin and b/apps/mandelbrot/prebuilt/hw/xilinx_aws-vu9p-f1_4ddr-xpr-2pr_4.0/mandelbrot.awsxclbin differ diff --git a/bin/install_verilator b/bin/install_verilator index 844aaaa..2e27ec0 100755 --- a/bin/install_verilator +++ b/bin/install_verilator @@ -32,9 +32,10 @@ then echo "$(pwd)/verilator exists." echo "To reinstall, first 'rm -rf $(pwd)/verilator'." else - curl https://www.veripool.org/ftp/verilator-4.018.tgz | tar -zx + wget -qO- https://github.com/verilator/verilator/archive/refs/tags/v5.016.tar.gz | tar xvz mv verilator* verilator # So path is not version-dependent. cd verilator + autoconf ./configure make -j$(nproc) fi diff --git a/doc/DevelopersGuide.md b/doc/DevelopersGuide.md index 27ace41..7e5ec47 100644 --- a/doc/DevelopersGuide.md +++ b/doc/DevelopersGuide.md @@ -52,7 +52,7 @@ The top-level file structure is: - `regress`: a regression script that launches framework and app regression scripts - `doc`: documentation (including this document) - `init`: a script to set up the repository for development - - `sdaccel_setup`: a script to set up for development using Xilinx tools on AWS + - `vitis_setup`: a script to set up for development using Xilinx tools on AWS An application _`foo`_ is constructed from code in `framework` and `apps/foo`. `framework` and `apps/foo` contain files/directories using nearly identical structure: @@ -90,7 +90,7 @@ An actual F1, and even AWS as a whole, are needed very little during development - Software (`sw`): An optional mode where the RTL kernel is not utilized. Custom C++ code is required to provide emulated kernel behavior. - Simulation (`sim`): Verilator is used for 2-state simulation of the custom RTL kernel. Verilator creates a C++ model of the kernel which is directly compiled in with the host executable. The Host Application C++ code controls the kernel clock and decides when to send/receive data to/from the kernel. - - Hardware Emulation (`hw_emu`): This mode is supported by Xilinx SDAccel on AWS. All FPGA logic is simulated, including the custom kernel and surrounding shell logic. This runs much slower than Simulation. + - Hardware Emulation (`hw_emu`): This mode is supported by Xilinx Vitis on AWS. All FPGA logic is simulated, including the custom kernel and surrounding shell logic. This runs much slower than Simulation. - Hardware (`hw`): Uses a real F1 FPGA. Generally, most kernel development is done in Simulation. Hardware Emulation is used to refine the implementation of the design and may catch a few new bugs because of 4-state modeling and test bench differences. Hardware compilation is primarily for testing the application at speed and for deployment. diff --git a/doc/GettingStartedF1.md b/doc/GettingStartedF1.md index a6d6406..7864503 100644 --- a/doc/GettingStartedF1.md +++ b/doc/GettingStartedF1.md @@ -113,7 +113,7 @@ The command below uses **IMPORTANT:** Be sure not to accidentally leave instances running!!! You should configure monitoring of your resources, but the options, though plentiful, seem very limited for catching instances you fail to stop. Also be warned that stopping an instance can fail. We have found it important to always refresh the page before changing machine state. And, be sure your instance transitions to "stopped" state (or, according to AWS support, charging stops at "stopping" state). -You must choose a Linux password for your new instance (which must not contain single/double quotes nor backslash). Obviously, since you are typing it here in plain text, be sure it is not visible to wandering eyes, and perhaps run `clear` after command completion. +You must choose a Linux password for your new instance (which must not contain single/double quotes nor backslash). **Make Sure to use a strong password probably with more than 8 characters and having alphabets + numbers + special characters** .Obviously, since you are typing it here in plain text, be sure it is not visible to wandering eyes, and perhaps run `clear` after command completion. ```sh make development_instance LINUX_PASSWORD= @@ -145,6 +145,13 @@ make desktop Enter your Linux password. (We do not register your password with Remmina because Remmina must be carefully configure to keep your password secure.) +If you have Windows installed, you can use RDC (Remote Desktop Connection). If you want to access the remote server manually, type the Public IP Address in the field. + +![RDC sample](/doc/img/RDC.png "RDC Window") + +If you face any color depth issue when using Remote Desktop especially with Xilinx Tools (they get blurry and test is none to visible), you can set High Colour to **16 bit** + +![RDC color](/doc/img/RDC_color.png "RDC Colour") ### SSH Access @@ -183,7 +190,7 @@ Open a new terminal in your remote desktop. Each time you do so, you must: ```sh cd ~/1st-CLaaS # (~/1st-CLaaS is a symbolic link.) -source sdaccel_setup +source vitis_setup ``` @@ -225,7 +232,7 @@ git commit ... git push # If not to master, you would pull from corresponding branch on F1 instance. ``` -> Note: Sourcing `sdaccel_setup` currently breaks `git gui` and `gitk`, so use these in a separate shell without `sdaccel_setup`. +> Note: Sourcing `vitis_setup` currently breaks `git gui` and `gitk`, so use these in a separate shell without `vitis_setup`. @@ -239,7 +246,7 @@ make f1_instance ``` ```sh -make ssh SSH_CMD='source 1st-CLaaS/sdaccel_setup && cd 1st-CLaaS/app/mandelbrot/build && make launch PREBUILT=true' # TARGET=hw is the default on F1. +make ssh SSH_CMD='source 1st-CLaaS/vitis_setup && cd 1st-CLaaS/app/mandelbrot/build && make launch PREBUILT=true' # TARGET=hw is the default on F1. ``` As before, open `http://:8888` in your browser (using the new IP). Now you can select renderer "FPGA", and navigate at FPGA speed. (Try "velocity" nagivation mode.) diff --git a/doc/img/RDC.png b/doc/img/RDC.png new file mode 100755 index 0000000..72295e6 Binary files /dev/null and b/doc/img/RDC.png differ diff --git a/doc/img/RDC_color.png b/doc/img/RDC_color.png new file mode 100755 index 0000000..6ead76e Binary files /dev/null and b/doc/img/RDC_color.png differ diff --git a/framework/build/Makefile b/framework/build/Makefile index 9b9edc0..ffb8d17 100644 --- a/framework/build/Makefile +++ b/framework/build/Makefile @@ -265,6 +265,7 @@ SUPPORTED_TARGETS :=$(SUPPORTED_TARGETS)sw |$(SPACE) endif + # # Determine BUILD_TARGET. [hw (default) | hw_emu | sim | sw] # @@ -364,7 +365,7 @@ S3_LOGS_KEY=$(KERNEL_NAME)_log S3_DCP_KEY=$(KERNEL_NAME)_dcp S3_TRANSFER_KEY=$(KERNEL_NAME)_xfer -XOCC=xocc +VPP=v++ CC=g++ @@ -395,7 +396,7 @@ HOST_HDRS=$(SW_HDRS) $(FRAMEWORK_HOST_DIR)/kernel.h HOST_HDRS=$(SW_HDRS) $(FRAMEWORK_HOST_DIR)/hw_kernel.h # TODO: It seems SDX_PLATFORM should be set to a value. For hw_emu, I see one device: "xilinx:pcie-hw-em:7v3:1.0" # What's the emconfigutil command (for configuring the platform for hw_emu?) -HOST_CFLAGS=$(SW_CFLAGS) -D KERNEL_AVAIL -D FPGA_DEVICE -D OPENCL -I$(XILINX_XRT)/runtime/include/1_2 -D C_KERNEL -D SDX_PLATFORM=$(AWS_PLATFORM) -D KERNEL=$(KERNEL_NAME) +HOST_CFLAGS=$(SW_CFLAGS) -D KERNEL_AVAIL -D FPGA_DEVICE -D OPENCL -I$(XILINX_XRT)/runtime/include/1_2 -D C_KERNEL -D VITIS_PLATFORM=xilinx_aws-vu9p-f1_shell-v04261818_201920_3 -D KERNEL=$(KERNEL_NAME) HOST_LFLAGS=$(SW_LFLAGS) -lxilinxopencl #Simulation flags @@ -410,15 +411,15 @@ HOST_EXE=host #Kernel KERNEL_SRC= -# Flags for xocc (include -I args). +# Flags for v++ (include -I args). KERNEL_FLAGS= KERNEL_EXE=$(KERNEL_NAME) -#Custom flag to give to xocc -KERNEL_LDCLFLAGS=--nk $(KERNEL_NAME):1 \ - --xp param:compiler.preserveHlsOutput=1 \ - --max_memory_ports $(KERNEL_NAME) \ - --memory_port_data_width $(KERNEL_NAME):512 \ +#Custom flag to give to v++ +KERNEL_LDCLFLAGS= --connectivity.nk $(KERNEL_NAME):1 \ + --xp param:compiler.preserveHlsOutput=1 \ + --hls.max_memory_ports $(KERNEL_NAME) \ + --hls.memory_port_data_width $(KERNEL_NAME):512 \ KERNEL_ADDITIONAL_FLAGS= @@ -549,7 +550,7 @@ HOST_CMD=$(VALGRIND_PREFIX) $(HOST_EXE_PATH) $(HOST_ARGS) endif ifeq ($(BUILD_TARGET),hw_emu) BUILD_TARGETS=$(BUILD_DIR)/$(HOST_EXE) $(HOST_XCLBIN) -HOST_CMD=export XCL_EMULATION_MODE=$(BUILD_TARGET) && $(XILINX_SDX)/bin/emconfigutil --od $(DEST_DIR) --nd 1 --platform $(AWS_PLATFORM) && $(VALGRIND_PREFIX) $(HOST_EXE_PATH) $(HOST_ARGS) $(HOST_XCLBIN) +HOST_CMD=export XCL_EMULATION_MODE=$(BUILD_TARGET) && $(XILINX_VITIS)/bin/emconfigutil --od $(DEST_DIR) --nd 1 --platform $(AWS_PLATFORM) && $(VALGRIND_PREFIX) $(HOST_EXE_PATH) $(HOST_ARGS) $(HOST_XCLBIN) endif ifeq ($(BUILD_TARGET),hw) BUILD_TARGETS=$(BUILD_DIR)/$(HOST_EXE) $(HOST_XCLBIN) @@ -794,7 +795,7 @@ project: $(USER_KERNEL_ADDED_FILE) $(DEST_DIR)/$(KERNEL_EXE).xclbin: $(XO_FILE) - cd $(DEST_DIR); $(XOCC) -g --platform $(AWS_PLATFORM) --target $(BUILD_TARGET) --link -O quick --save-temps $(REPORT) --kernel $(KERNEL_NAME) ../../$(XO_FILE) $(KERNEL_LDCLFLAGS) $(KERNEL_FLAGS) $(KERNEL_ADDITIONAL_FLAGS) --output $(KERNEL_EXE).xclbin + cd $(DEST_DIR); $(VPP) -g --platform $(AWS_PLATFORM) --target $(BUILD_TARGET) --link -O quick --save-temps $(REPORT) --kernel $(KERNEL_NAME) --input_files ../../$(XO_FILE) $(KERNEL_LDCLFLAGS) $(KERNEL_FLAGS) $(KERNEL_ADDITIONAL_FLAGS) --output $(KERNEL_EXE).xclbin # Create the AFI. # The steps are: @@ -812,7 +813,7 @@ $(BUILD_DIR)/$(KERNEL_EXE).awsxclbin: $(DEST_DIR)/$(KERNEL_EXE).xclbin @# Create bucket if it doesn't exist. aws s3api create-bucket --bucket '$(S3_BUCKET)' --acl private > /dev/null && aws s3api wait bucket-exists --bucket '$(S3_BUCKET)' # Create AFI and Wait for creation to complete. afi- and _afi_id.txt are extracted into files for use. - cd $(DEST_DIR) && $(SDACCEL_DIR)/tools/create_sdaccel_afi.sh -xclbin=$(KERNEL_EXE).xclbin -o=$(KERNEL_NAME) -s3_bucket=$(S3_BUCKET) -s3_dcp_key=$(S3_DCP_KEY) -s3_logs_key=$(S3_LOGS_KEY) -aws_profile_name=$(AWS_PROFILE) \ + cd $(DEST_DIR) && $(VITIS_DIR)/tools/create_vitis_afi.sh -xclbin=$(KERNEL_EXE).xclbin -o=$(KERNEL_NAME) -s3_bucket=$(S3_BUCKET) -s3_dcp_key=$(S3_DCP_KEY) -s3_logs_key=$(S3_LOGS_KEY) -aws_profile_name=$(AWS_PROFILE) \ && grep '"afi-' *_afi_id.txt | sed 's/^.*"\(afi-[0-9a-zA-Z]*\)".*$$/\1/' > $(KERNEL_NAME)_afi_id.txt \ && ls *_afi_id.txt | sed 's/^\(.*\)_afi_id.txt$$/\1/' > $(KERNEL_NAME)_timestamp.txt \ && wait_for_afi.py --afi "$$(cat $(KERNEL_NAME)_afi_id.txt)" diff --git a/framework/build/launch b/framework/build/launch index 03fad15..faf4dc5 100755 --- a/framework/build/launch +++ b/framework/build/launch @@ -67,7 +67,6 @@ # TODO: Make a function to wrap commands in sudo. - usage () { echo "Usage: launch [-p #] [-h host-pid] [-c compile-command] [-w web-server-args] (sw|hw_emu|hw) host-command" exit 1 @@ -129,6 +128,12 @@ else USE_XILINX=true fi +if [[ $TARGET = "hw" ]] +then + USE_FPGA=true +else + USE_FPGA=false +fi if [[ $USE_XILINX = "false" ]] && [[ $HOST = "" ]]; then @@ -156,8 +161,15 @@ launch () { if [[ $USE_XILINX = "false" ]]; then $HOST & - else - sudo -- sh -c "source /opt/xilinx/xrt/setup.sh ; $HOST" & + fi + else if [[ $USE_XILINX = "false" ]] || [[ $USE_FPGA = "false" ]]; + then + sudo -- sh -c "source $XILINX_VITIS/settings64.sh; source /opt/xilinx/xrt/setup.sh; source $AWS_FPGA_REPO_DIR/vitis_setup.sh; emconfigutil --nd 1 --platform $AWS_PLATFORM; export XCL_EMULATION_MODE=hw_emu ;$HOST" & + fi + + if [[ $USE_XILINX = "true" ]] && [[ $USE_FPGA = "true" ]]; + then + sudo -- sh -c "source $XILINX_VITIS/settings64.sh; source /opt/xilinx/xrt/setup.sh; source $AWS_FPGA_REPO_DIR/vitis_setup.sh; source $AWS_FPGA_REPO_DIR/vitis_runtime_setup.sh; export LANG=C; $HOST" & fi export HOST_PID=$! fi diff --git a/framework/fpga/f4pga-examples b/framework/fpga/f4pga-examples new file mode 160000 index 0000000..a5a44fa --- /dev/null +++ b/framework/fpga/f4pga-examples @@ -0,0 +1 @@ +Subproject commit a5a44fa0c585783dcc1c6be32e88613be800287b diff --git a/framework/fpga/scripts/produce_tcl_file.py b/framework/fpga/scripts/produce_tcl_file.py index 5de0771..7ce8ff5 100644 --- a/framework/fpga/scripts/produce_tcl_file.py +++ b/framework/fpga/scripts/produce_tcl_file.py @@ -106,7 +106,7 @@ def json_to_tcl_config (string): 'create_project kernel_wizard $wizardDir -force\n' '\n' '# Instantiate the SDx kernel wizard IP\n' - 'create_ip -name sdx_kernel_wizard -vendor xilinx.com -library ip -module_name $kernelName\n\n') + 'create_ip -name rtl_kernel_wizard -vendor xilinx.com -library ip -module_name $kernelName\n\n') string += json_to_tcl_config(config_file) diff --git a/framework/fpga/scripts/tcl/rtl_kernel_wiz.tcl b/framework/fpga/scripts/tcl/rtl_kernel_wiz.tcl index ae21dbb..f31039a 100644 --- a/framework/fpga/scripts/tcl/rtl_kernel_wiz.tcl +++ b/framework/fpga/scripts/tcl/rtl_kernel_wiz.tcl @@ -19,7 +19,7 @@ set kernelDir $workspace/$kernelPrjName create_project kernel_wizard $wizardDir -force # Instantiate the SDx kernel wizard IP -create_ip -name sdx_kernel_wizard -vendor xilinx.com -library ip -module_name $kernelName +create_ip -name rtl_kernel_wizard -vendor xilinx.com -library ip -module_name $kernelName set cmd "set_property -dict \[list CONFIG.NUM_CLOCKS {2} CONFIG.NUM_INPUT_ARGS {1} CONFIG.ARG00_NAME {ctrl_length} CONFIG.NUM_M_AXI {1} CONFIG.M00_AXI_NUM_ARGS {1} CONFIG.M00_AXI_ARG00_NAME {a} CONFIG.KERNEL_NAME {$kernelName} CONFIG.KERNEL_VENDOR {$kernelVendor}] \[get_ips $kernelName]" eval $cmd diff --git a/framework/host/hw_kernel.c b/framework/host/hw_kernel.c index 8b653f6..1696368 100644 --- a/framework/host/hw_kernel.c +++ b/framework/host/hw_kernel.c @@ -51,10 +51,17 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #include #include "kernel.h" +#define CL_USE_DEPRECATED_OPENCL_1_2_APIS #include - +#include #include "server_main.h" +#if defined(VITIS_PLATFORM) +#define STR_VALUE(arg) #arg +#define GET_STRING(name) STR_VALUE(name) +#define TARGET_DEVICE GET_STRING(VITIS_PLATFORM) +#endif + HW_Kernel::HW_Kernel() { } @@ -64,8 +71,8 @@ void HW_Kernel::perror(const char * msg) { status = EXIT_FAILURE; } -int HW_Kernel::load_file_to_memory(const char *filename, char **result) { - uint size = 0; +cl_uint HW_Kernel::load_file_to_memory(const char *filename, char **result) { + cl_uint size = 0; FILE *f = fopen(filename, "rb"); if (f == NULL) { *result = NULL; @@ -81,6 +88,8 @@ int HW_Kernel::load_file_to_memory(const char *filename, char **result) { } fclose(f); (*result)[size] = 0; + // For Debugging + printf("File loaded to memory\n"); return size; } @@ -90,6 +99,12 @@ void HW_Kernel::initialize_platform() { cl_uint platform_count; char cl_platform_vendor[1001]; + cl_uint num_devices; // Number of cl_devices if more than one + cl_uint device_found = 0; + cl_device_id devices[16]; // compute device id + char cl_device_name[1001]; + char target_device_name[1001] = TARGET_DEVICE; + int err; int platform_found = 0; @@ -129,11 +144,40 @@ void HW_Kernel::initialize_platform() { #endif printf("get device, fpga is %d \n", fpga); err = clGetDeviceIDs(platform_id, fpga ? CL_DEVICE_TYPE_ACCELERATOR : CL_DEVICE_TYPE_CPU, - 1, &device_id, NULL); + 16, devices, &num_devices); if (err != CL_SUCCESS) { perror("Error: Failed to create a device group!\nTest failed\n"); return; } + + //iterate all devices to select the target device. + for (cl_uint i=0; i /dev/null)" ]] then # Ubuntu sudo apt-get update - sudo apt-get -y install curl g++ python3 python3-pip python3-pil $COMMON_PACKAGES + sudo apt-get -y install curl g++ python3 python3-pip python3-pil xz-utils $COMMON_PACKAGES sudo apt -y install $DEBUG_PACKAGES #sudo python3 -m pip install Pillow elif [[ -n "$(which yum 2> /dev/null)" ]] @@ -41,13 +40,15 @@ then sudo yum -y install python3 # this will install python3.6 version # To look like Ubuntu, we need python3 and pip3. - sudo yum -y install $COMMON_PACKAGES # python-tornado python-imaging redundant w/ pip install? + sudo yum -y install which xz $COMMON_PACKAGES # python-tornado python-imaging redundant w/ pip install? sudo yum -y install $DEBUG_PACKAGES ## bash lives at /usr/bin/bash in CentOS, but in /bin/bash for Ubuntu. Create /bin/bash. #sudo ln -s /bin/bash /usr/bin/bash # TODO: So far, untested. Test and delete this comment. fi +# Install submodules. +git submodule update --init --recursive # Install python libraries for user only so as not to affect system installs. echo "Installing Python packages for user $USER." @@ -115,11 +116,18 @@ then echo "Downloading Terraform" mkdir terraform ( cd terraform && \ - curl https://releases.hashicorp.com/terraform/0.12.4/terraform_0.12.4_linux_amd64.zip > terraform.zip \ + curl https://releases.hashicorp.com/terraform/1.5.2/terraform_1.5.2_linux_amd64.zip > terraform.zip \ && unzip terraform.zip \ && rm terraform.zip \ && chmod +x terraform ) fi + +if [[ ! -f "$(ls "framework/fpga/f4pga-examples/conda_installer.sh" 2> /dev/null)" ]] +then + echo "Downloading Conda Installer" + ( cd framework/fpga/f4pga-examples && \ + wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O conda_installer.sh ) +fi # Note that framework/Makefile and bin/regress use the existence of terraform/terraform to indicate that this script was run. diff --git a/sdaccel_setup b/sdaccel_setup deleted file mode 100755 index d94f692..0000000 --- a/sdaccel_setup +++ /dev/null @@ -1 +0,0 @@ -pushd $AWS_FPGA_REPO_DIR; source ./sdaccel_setup.sh; popd diff --git a/vitis_setup b/vitis_setup new file mode 100755 index 0000000..259ad03 --- /dev/null +++ b/vitis_setup @@ -0,0 +1 @@ +pushd $AWS_FPGA_REPO_DIR; source ./vitis_setup.sh; popd