From 6da5e63e2c25ba6c53d8e16438fcf595f81ac44d Mon Sep 17 00:00:00 2001 From: Marco van Zwetselaar Date: Fri, 10 May 2024 10:57:39 +0300 Subject: [PATCH] Fix detection of cudnn-frontend in '.' on Linux Plus minor legibility fixes in the cudnn-frontend explanation. --- Makefile | 10 ++++------ README.md | 2 +- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/Makefile b/Makefile index eabb5646d..46abdc9a5 100644 --- a/Makefile +++ b/Makefile @@ -19,8 +19,7 @@ NVCC_INCLUDES = NVCC_LDLIBS = NCLL_INCUDES = NVCC_CUDNN = -# overridable flag for multi-GPU training. by default we won't build with cudnn -# because it bloats up the compile time from a few seconds to ~minute +# By default we don't build with cudnn because it blows up compile time from a few seconds to ~minute USE_CUDNN ?= 0 # Function to check if a file exists in the PATH @@ -86,16 +85,15 @@ else endif # Check and include cudnn if available -# You can override the path to cudnn frontend by setting CUDNN_FRONTEND_PATH=your_path on the make command line +# You can override the path to cudnn frontend by setting CUDNN_FRONTEND_PATH on the make command line +# By default, we look for it in HOME/cudnn-frontend/include and ./cudnn-frontend/include # Refer to the README for cuDNN install instructions ifeq ($(USE_CUDNN), 1) ifeq ($(SHELL_UNAME), Linux) - # hard-coded path for now in either . or ($HOME) directory - # this can be overridden by setting CUDNN_FRONTEND_PATH on the command line ifeq ($(shell [ -d $(HOME)/cudnn-frontend/include ] && echo "exists"), exists) $(info ✓ cuDNN found, will run with flash-attention) CUDNN_FRONTEND_PATH ?= $(HOME)/cudnn-frontend/include - else ifeq ($(shell [ -d cudnn-frontend/include ] && echo "exists"),) + else ifeq ($(shell [ -d cudnn-frontend/include ] && echo "exists"), exists) $(info ✓ cuDNN found, will run with flash-attention) CUDNN_FRONTEND_PATH ?= cudnn-frontend/include else diff --git a/README.md b/README.md index f87161261..b4298acff 100644 --- a/README.md +++ b/README.md @@ -265,7 +265,7 @@ sudo apt-get update sudo apt-get -y install libcudnn9-dev-cuda-12 ``` -On top of this you need the [cuDNN frontend](https://github.com/NVIDIA/cudnn-frontend/tree/main), but this is just header files. So simply download the repo to your disk, currently assumed to be in your home directory (i.e. the Makefile looks for `~/cudnn-frontend/include`). +On top of this you need the [cuDNN frontend](https://github.com/NVIDIA/cudnn-frontend/tree/main), but this is just header files. Simply clone the repo to your disk. The Makefile currently looks for it in either your home directory or the current directory. If you have put it elsewhere, add `CUDNN_FRONTEND_PATH=/path/to/your/cudnn-frontend/include` to the `make` command-line. **Multi-GPU training**. As of April 26, 2024 there is now also support for multi-GPU training using MPI and NCCL. Make sure you install MPI, e.g. on Linux: