Add requirements.

Dockerfile

@@ -0,0 +1,26 @@
+FROM pytorch/pytorch:1.10.0-cuda11.3-cudnn8-devel
+
+ENV LC_ALL=C.UTF-8
+ENV LANG=C.UTF-8
+ENV CUDA_HOME=/usr/local/cuda/
+
+RUN apt-get -y update
+RUN apt-get -y install git vim 
+
+RUN curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | bash
+RUN apt-get -y install git-lfs
+
+WORKDIR /stage/
+
+COPY requirements.txt .
+RUN pip install -r requirements.txt
+RUN python -c "import nltk; nltk.download('punkt', quiet=True)"
+
+COPY ds_configs ds_configs
+
+COPY src src
+COPY scripts scripts
+RUN chmod +x scripts/*
+
+# for interactive session
+RUN chmod -R 777 /stage/

README.md

@@ -1,7 +1,26 @@
 # Tk-Instruct
-This repo releases our code for training the Tk-Instruct model in the [Natural Instructions V2 paper](https://arxiv.org/abs/2204.07705). You can play with  model via our online [demo](https://instructions.apps.allenai.org/demo)!
 
-## Requiments
+- This repo releases our implementation for the Tk-Instruct model in the [Natural Instructions V2 paper](https://arxiv.org/abs/2204.07705).
+- Tk-Instruct is a preliminary attempt towards general-purpose AI that can solve new tasks by following instructions.
+- You can play with this model via our online [demo](https://instructions.apps.allenai.org/demo)!
+
+## Requirements
+
+Our experiments are conducted on the following environment:
+
+- CUDA (11.3)
+- cuDNN (8.2.0.53)
+- Pytorch (1.10.0)
+- Transformers (4.17.0)
+- DeepSpeed
+
+You can refer to the [Dockerfile](Dockerfile) for setting up the environment and install the required python libraries by running
+
+```script
+pip install -r requirements.txt
+```
+
+## Data
 
 ## Training
 
@@ -11,4 +30,4 @@ This repo releases our code for training the Tk-Instruct model in the [Natural I
 
 ## Checkpoints
 
-You can download our 3B and 11B model checkpoints that are trained to follow different types of encodings are released in [Huggingface Hub](https://huggingface.co/models?search=tk-instruct-).
+Our 3B and 11B model checkpoints are accessible via [Huggingface Hub](https://huggingface.co/models?search=tk-instruct-).

requirements.txt

@@ -0,0 +1,14 @@
+# git+https://github.com/huggingface/transformers.git@52d2e6f6e904ef9b75c78716ce77b98196ed837a
+# git+https://github.com/stas00/transformers.git@ds-bf16
+transformers==4.17.0
+datasets==1.17.0
+# git+https://github.com/microsoft/DeepSpeed.git@2151c787a27166f795eb4516f1e191e6730e823d
+git+https://github.com/microsoft/DeepSpeed.git@9f7126fc10a1f231c695db11632cbcb84f1af4d3
+fairscale==0.4.5
+ipython
+nltk
+tensorboard
+tqdm
+rouge_score
+wandb==0.12.10
+sentencepiece==0.1.96