bump nemo and remove manual tensorstore install (#619)

With NVIDIA/NeMo#11777, NVIDIA/NeMo#11858, and NVIDIA/Megatron-LM@a26b93d, we're now ready to remove the manual tensorstore install from bionemo. Signed-off-by: Peter St. John <[email protected]>
NVIDIA · Jan 21, 2025 · 28450df · 28450df
1 parent 29f650e
commit 28450df
Show file tree

Hide file tree

Showing 2 changed files with 3 additions and 13 deletions.
diff --git a/3rdparty/NeMo b/3rdparty/NeMo
diff --git a/Dockerfile b/Dockerfile
@@ -105,18 +105,6 @@ COPY --from=rust-env /usr/local/rustup /usr/local/rustup
 ENV PATH="/usr/local/cargo/bin:/usr/local/rustup/bin:${PATH}"
 ENV RUSTUP_HOME="/usr/local/rustup"
 
-RUN --mount=type=cache,target=/root/.cache <<EOF
-set -eo pipefail
-uv pip install maturin --no-build-isolation
-
-pip install --use-deprecated=legacy-resolver  --no-build-isolation \
-  tensorstore==0.1.45
-sed -i 's/^Version: 0\.0\.0$/Version: 0.1.45/' \
-  /usr/local/lib/python3.12/dist-packages/tensorstore-0.0.0.dist-info/METADATA
-mv /usr/local/lib/python3.12/dist-packages/tensorstore-0.0.0.dist-info \
-/usr/local/lib/python3.12/dist-packages/tensorstore-0.1.45.dist-info
-EOF
-
 WORKDIR /workspace/bionemo2
 
 # Install 3rd-party deps and bionemo submodules.
@@ -133,6 +121,8 @@ RUN --mount=type=bind,source=./.git,target=./.git \
     --mount=type=cache,target=/root/.cache <<EOF
 set -eo pipefail
 
+uv pip install maturin --no-build-isolation
+
 uv pip install --no-build-isolation \
   ./3rdparty/* \
   ./sub-packages/bionemo-* \
+1 −1		.github/workflows/_test_template.yml
+19 −4		.github/workflows/cicd-main.yml
+29 −50		.github/workflows/import-test.yml
+11 −19		Dockerfile.ci
+34 −4		README.md
+1 −2		docs/source/nlp/information_retrieval.rst
+36 −2		nemo/collections/common/tokenizers/sentencepiece_tokenizer.py
+0 −1		nemo/collections/diffusion/scripts/train.sh
+3 −0		nemo/collections/llm/gpt/model/gemma.py
+1 −1		nemo/collections/llm/gpt/model/ssm.py
+2 −2		nemo/collections/llm/inference/base.py
+2 −2		nemo/collections/llm/peft/api.py
+0 −2		nemo/collections/llm/recipes/gemma_2b.py
+0 −4		nemo/collections/llm/recipes/gemma_7b.py
+5 −0		nemo/collections/multimodal/data/energon/base.py
+22 −2		nemo/collections/multimodal/data/energon/config.py
+1 −1		nemo/collections/multimodal/data/energon/conversation.py
+143 −27		nemo/collections/multimodal/data/energon/task_encoder.py
+1 −1		nemo/collections/multimodal/models/text_to_image/controlnet/controlnet.py
+1 −1		nemo/collections/multimodal/models/text_to_image/imagen/imagen_pipeline.py
+13 −4		nemo/collections/multimodal/models/text_to_image/instruct_pix2pix/ldm/ddpm_edit.py
+3 −3		nemo/collections/multimodal/models/text_to_image/stable_diffusion/ldm/autoencoder.py
+2 −2		nemo/collections/multimodal/models/text_to_image/stable_diffusion/ldm/ddpm.py
+1 −1		nemo/collections/multimodal/modules/stable_diffusion/diffusionmodules/openaimodel.py
+2 −2		nemo/collections/multimodal/speech_llm/models/modular_models.py
+1 −1		nemo/collections/multimodal/speech_llm/parts/mixins/adapter_mixin.py
+10 −6		nemo/collections/nlp/data/language_modeling/megatron/gpt_sft_chat_dataset.py
+3 −36		nemo/collections/nlp/models/language_modeling/megatron/gpt_layer_modelopt_spec.py
+6 −0		nemo/collections/nlp/models/language_modeling/megatron_base_model.py
+1 −0		nemo/collections/nlp/models/language_modeling/megatron_gpt_model.py
+3 −0		nemo/collections/nlp/models/language_modeling/megatron_retro_model.py
+2 −0		nemo/collections/nlp/modules/common/megatron/megatron_init.py
+1 −1		nemo/collections/nlp/modules/common/tokenizer_utils.py
+1 −1		nemo/collections/nlp/parts/mixins/multimodal_adapter_mixins.py
+1 −1		nemo/collections/nlp/parts/mixins/nlp_adapter_mixins.py
+1 −0		nemo/collections/nlp/parts/nlp_overrides.py
+1 −1		nemo/collections/vlm/inference/base.py
+2 −2		nemo/collections/vlm/mllama/model/language.py
+3 −3		nemo/collections/vlm/neva/data/config.py
+63 −52		nemo/collections/vlm/neva/data/lazy.py
+64 −5		nemo/collections/vlm/neva/data/mock.py
+157 −0		nemo/collections/vlm/neva/data/sequence_packing.py
+34 −4		nemo/collections/vlm/neva/model/base.py
+1 −1		nemo/collections/vlm/recipes/llava15_13b.py
+1 −1		nemo/collections/vlm/recipes/llava15_7b.py
+1 −1		nemo/collections/vlm/recipes/llava_next_7b.py
+3 −0		nemo/export/trt_llm/nemo_ckpt_loader/nemo_file.py
+4 −1		nemo/lightning/megatron_parallel.py
+1 −1		nemo/lightning/pytorch/callbacks/peft.py
+17 −0		nemo/utils/app_state.py
+0 −3		pyproject.toml
+1 −1		requirements/requirements_deploy.txt
+1 −1		requirements/requirements_infer.txt
+1 −1		requirements/requirements_multimodal.txt
+1 −2		requirements/requirements_nlp.txt
+2 −1		scripts/checkpoint_converters/convert_bert_hf_to_nemo.py
+44 −4		scripts/checkpoint_converters/convert_mistral_7b_hf_to_nemo.py
+1 −1		scripts/vlm/llava_next_finetune.py
+1 −1		scripts/vlm/llava_next_pretrain.py
+1 −1		scripts/vlm/mllama_finetune.py
+80 −29		scripts/vlm/neva_finetune.py
+2 −0		tests/collections/llm/bitexact/mixtral/pretrain_mini_mixtral.py
+2 −2		tests/collections/llm/bitexact/mixtral/run.sh
+5 −0		tests/collections/llm/gpt/model/test_model_import.py
+0 −1		tests/collections/llm/hf/peft_nemorun.py
+0 −1		tests/collections/llm/hf/sft_nemorun.py
+2 −0		tests/collections/llm/megatron_mixtral_pretraining.py
+1 −0		tests/collections/llm/test_mnist_model_nemo2.py
+1 −0		tests/collections/llm/test_mnist_model_nemo2_fsdp.py
+2 −2		tests/collections/multimodal/data/energon/test_data_module.py
+73 −0		tests/collections/nlp/test_chat_sft_dataset.py
+63 −15		tests/collections/nlp/test_tokenizer_with_special_tokens.py
+0 −0		tests/collections/vlm/test_mllama_train.py
+7 −0		tests/collections/vlm/test_neva_train.py
+14 −0		tests/conftest.py
+2 −2		tests/core/test_exp_manager.py
+5 −5		tests/lightning/test_nemo_resume_from_ckpt.py
+2 −2		tutorials/llm/llama-3/nemo2-sft-peft/README.rst
+1 −3		tutorials/llm/llama-3/nemo2-sft-peft/nemo2-peft.ipynb
+0 −2		tutorials/llm/llama-3/nemo2-sft-peft/nemo2-sft.ipynb
+2 −7		tutorials/llm/mamba/mamba.rst