From d1d6aea8ab0a7877ae064f0435fdca8e11696c55 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E9=9B=AA=E5=B3=B0?= Date: Mon, 27 Jan 2025 00:29:49 +0800 Subject: [PATCH] optimize model path for auto-download --- README.md | 30 ++++++++++++++---- README_CN.md | 24 +++++++++++--- eva_clip/factory.py | 5 ++- eva_clip/pretrained.py | 12 ++++--- pulidflux.py | 72 +++++++++++++++++++++++++++++++++++------- pyproject.toml | 2 +- 6 files changed, 118 insertions(+), 27 deletions(-) diff --git a/README.md b/README.md index 380ed5a..344a510 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,12 @@ Must uninstall or disable `ComfyUI-PuLID-Flux` and other PuLID-Flux nodes before Need upgrade ComfyUI Version>=0.3.7 +## Update logs +### 2025.01.27 +- Changed the model path of facexlib to `ComfyUI/models/facexlib/`. +- When automatically downloading, modify the path of Antelope v2 model to `ComfyUI/models/insightface/models/antelopev2/`. +- Changed the model path of EVA_CLIP_L_14_336 to `ComfyUI/models/clip/`. + ## Preview (Image with WorkFlow) ![save api extended](examples/PuLID_with_speedup.png) ![save api extended](examples/PuLID_with_attn_mask.png) @@ -24,17 +30,29 @@ Need upgrade ComfyUI Version>=0.3.7 # restart ComfyUI ``` -## Model -Please see [ComfyUI-PuLID-Flux](https://github.com/balazik/ComfyUI-PuLID-Flux) +## Models +### Available Flux models +- 32bit/16bit (~22GB VRAM): [model](https://huggingface.co/black-forest-labs/FLUX.1-dev/blob/main/flux1-dev.safetensors), [encoder](https://huggingface.co/comfyanonymous/flux_text_encoders/blob/main/t5xxl_fp16.safetensors) +- 8bit gguf (~12GB VRAM): [model](https://huggingface.co/city96/FLUX.1-dev-gguf/blob/main/flux1-dev-Q8_0.gguf), [encoder](https://huggingface.co/city96/t5-v1_1-xxl-encoder-gguf/blob/main/t5-v1_1-xxl-encoder-Q8_0.gguf) +- 8 bit FP8 e5m2 (~12GB VRAM): [model](https://huggingface.co/Kijai/flux-fp8/blob/main/flux1-dev-fp8-e5m2.safetensors), [encoder](https://huggingface.co/comfyanonymous/flux_text_encoders/blob/main/t5xxl_fp8_e4m3fn.safetensors) +- 8 bit FP8 e4m3fn (~12GB VRAM): [model](https://huggingface.co/Kijai/flux-fp8/blob/main/flux1-dev-fp8-e4m3fn.safetensors), [encoder](https://huggingface.co/comfyanonymous/flux_text_encoders/blob/main/t5xxl_fp8_e4m3fn.safetensors) +- Clip and VAE (for all models): [clip](https://huggingface.co/comfyanonymous/flux_text_encoders/blob/main/clip_l.safetensors), [vae](https://huggingface.co/black-forest-labs/FLUX.1-schnell/blob/main/ae.safetensors) + +#### For GGUF models you will need to install [ComfyUI-GGUF](https://github.com/city96/ComfyUI-GGUF) +### PuLID models +- Download [PuLID-Flux](https://huggingface.co/guozinan/PuLID/resolve/main/pulid_flux_v0.9.1.safetensors?download=true) => `ComfyUI/models/pulid/`. +- (Support auto-download) Download [EVA02-CLIP-L-14-336](https://huggingface.co/QuanSun/EVA-CLIP/blob/main/EVA02_CLIP_L_336_psz14_s6B.pt?download=true) => `ComfyUI/models/clip/`. +- (Support auto-download) Download all models like `*.onnx` from [AntelopeV2](https://huggingface.co/MonsterMMORPG/tools/tree/main) => `ComfyUI/models/insightface/models/antelopev2/`. +- (Support auto-download) Download [parsing_bisenet](https://github.com/xinntao/facexlib/releases/download/v0.2.0/parsing_bisenet.pth), [parsing_parsenet](https://github.com/xinntao/facexlib/releases/download/v0.2.0/parsing_parsenet.pth) and [Resnet50](https://github.com/xinntao/facexlib/releases/download/v0.1.0/detection_Resnet50_Final.pth) => `ComfyUI/models/facexlib/`. ## Nodes - PulidFluxModelLoader - - See [ComfyUI-PuLID-Flux](https://github.com/balazik/ComfyUI-PuLID-Flux) + - See chapter [PuLID models](#pulid-models) - PulidFluxInsightFaceLoader - - See [ComfyUI-PuLID-Flux](https://github.com/balazik/ComfyUI-PuLID-Flux) + - See chapter [PuLID models](#pulid-models) - PulidFluxEvaClipLoader - - See [ComfyUI-PuLID-Flux](https://github.com/balazik/ComfyUI-PuLID-Flux) + - See chapter [PuLID models](#pulid-models) - ApplyPulidFlux - Solved the model pollution problem of the original plugin ComfyUI-PuLID-Flux - `attn_mask` ~~may not work correctly (I have no idea how to apply it, I have tried multiple methods and the results have been not satisfactory)~~ works now. @@ -51,4 +69,4 @@ Please see [ComfyUI-PuLID-Flux](https://github.com/balazik/ComfyUI-PuLID-Flux) [TeaCache](https://github.com/ali-vilab/TeaCache) -[Comfy-WaveSpeed](https://github.com/chengzeyi/Comfy-WaveSpeed) \ No newline at end of file +[Comfy-WaveSpeed](https://github.com/chengzeyi/Comfy-WaveSpeed) diff --git a/README_CN.md b/README_CN.md index 2389f6e..643b771 100644 --- a/README_CN.md +++ b/README_CN.md @@ -9,6 +9,12 @@ ComfyUI主体版本需要>=0.3.7 +## 更新日志 +### 2025.01.27 +- 修改 facexlib 的模型路径为 `ComfyUI/models/facexlib/`. +- 自动下载时 修改 Antelopev2 模型的路径为 `ComfyUI/models/insightface/models/antelopev2/`. +- 修改 EVA_CLIP_L_14_336 的模型路径为 `ComfyUI/models/clip/`. + ## 预览 (图片含工作流) ![save api extended](examples/PuLID_with_speedup.png) ![save api extended](examples/PuLID_with_attn_mask.png) @@ -24,15 +30,25 @@ ComfyUI主体版本需要>=0.3.7 # 重启 ComfyUI ``` ## 模型 -查看[ComfyUI-PuLID-Flux](https://github.com/balazik/ComfyUI-PuLID-Flux) +### 可用的 Flux 模型 +- 32bit/16bit (~22GB VRAM): [model](https://huggingface.co/black-forest-labs/FLUX.1-dev/blob/main/flux1-dev.safetensors), [encoder](https://huggingface.co/comfyanonymous/flux_text_encoders/blob/main/t5xxl_fp16.safetensors) +- 8bit gguf (~12GB VRAM): [model](https://huggingface.co/city96/FLUX.1-dev-gguf/blob/main/flux1-dev-Q8_0.gguf), [encoder](https://huggingface.co/city96/t5-v1_1-xxl-encoder-gguf/blob/main/t5-v1_1-xxl-encoder-Q8_0.gguf) +- 8 bit FP8 e5m2 (~12GB VRAM): [model](https://huggingface.co/Kijai/flux-fp8/blob/main/flux1-dev-fp8-e5m2.safetensors), [encoder](https://huggingface.co/comfyanonymous/flux_text_encoders/blob/main/t5xxl_fp8_e4m3fn.safetensors) +- 8 bit FP8 e4m3fn (~12GB VRAM): [model](https://huggingface.co/Kijai/flux-fp8/blob/main/flux1-dev-fp8-e4m3fn.safetensors), [encoder](https://huggingface.co/comfyanonymous/flux_text_encoders/blob/main/t5xxl_fp8_e4m3fn.safetensors) +- Clip and VAE (for all models): [clip](https://huggingface.co/comfyanonymous/flux_text_encoders/blob/main/clip_l.safetensors), [vae](https://huggingface.co/black-forest-labs/FLUX.1-schnell/blob/main/ae.safetensors) + +#### 若使用 GGUF 需要安装 [ComfyUI-GGUF](https://github.com/city96/ComfyUI-GGUF) + +### PuLID 模型 +- 下载 [PuLID-Flux](https://huggingface.co/guozinan/PuLID/resolve/main/pulid_flux_v0.9.1.safetensors?download=true) 到目录 `ComfyUI/models/pulid/` +- （支持自动下载）下载 [EVA02-CLIP-L-14-336](https://huggingface.co/QuanSun/EVA-CLIP/blob/main/EVA02_CLIP_L_336_psz14_s6B.pt?download=true) 到目录 `ComfyUI/models/clip/` +- （支持自动下载）从 [AntelopeV2](https://huggingface.co/MonsterMMORPG/tools/tree/main) 下载所有`*.onnx`模型文件到目录 `ComfyUI/models/insightface/models/antelopev2/`. +- （支持自动下载）下载 [parsing_bisenet](https://github.com/xinntao/facexlib/releases/download/v0.2.0/parsing_bisenet.pth), [parsing_parsenet](https://github.com/xinntao/facexlib/releases/download/v0.2.0/parsing_parsenet.pth) and [Resnet50](https://github.com/xinntao/facexlib/releases/download/v0.1.0/detection_Resnet50_Final.pth) 到目录 `ComfyUI/models/facexlib/`. ## 节点 - PulidFluxModelLoader - - 同 [ComfyUI-PuLID-Flux](https://github.com/balazik/ComfyUI-PuLID-Flux) - PulidFluxInsightFaceLoader - - 同 [ComfyUI-PuLID-Flux](https://github.com/balazik/ComfyUI-PuLID-Flux) - PulidFluxEvaClipLoader - - 同 [ComfyUI-PuLID-Flux](https://github.com/balazik/ComfyUI-PuLID-Flux) - ApplyPulidFlux - 解决了原插件中模型污染的问题 - `attn_mask`~~可能不能正确工作， 因为我不知道如何实现它， 尝试了多种方式效果都未能达到预期~~，可以正常工作了。 diff --git a/eva_clip/factory.py b/eva_clip/factory.py index ced8999..5596078 100644 --- a/eva_clip/factory.py +++ b/eva_clip/factory.py @@ -223,6 +223,7 @@ def create_model( pretrained_visual_model: str = None, pretrained_text_model: str = None, cache_dir: Optional[str] = None, + local_dir: Optional[str] = None, skip_list: list = [], ): model_name = model_name.replace('/', '-') # for callers using old naming with / in ViT names @@ -276,7 +277,7 @@ def create_model( checkpoint_path = '' pretrained_cfg = get_pretrained_cfg(model_name, pretrained) if pretrained_cfg: - checkpoint_path = download_pretrained(pretrained_cfg, cache_dir=cache_dir) + checkpoint_path = download_pretrained(pretrained_cfg, cache_dir=cache_dir, local_dir=local_dir) elif os.path.exists(pretrained): checkpoint_path = pretrained @@ -372,6 +373,7 @@ def create_model_and_transforms( image_mean: Optional[Tuple[float, ...]] = None, image_std: Optional[Tuple[float, ...]] = None, cache_dir: Optional[str] = None, + local_dir: Optional[str] = None, skip_list: list = [], ): model = create_model( @@ -389,6 +391,7 @@ def create_model_and_transforms( pretrained_visual_model=pretrained_visual_model, pretrained_text_model=pretrained_text_model, cache_dir=cache_dir, + local_dir=local_dir, skip_list=skip_list, ) diff --git a/eva_clip/pretrained.py b/eva_clip/pretrained.py index a1e55dc..eda0723 100644 --- a/eva_clip/pretrained.py +++ b/eva_clip/pretrained.py @@ -239,7 +239,9 @@ def get_pretrained_url(model: str, tag: str): def download_pretrained_from_url( url: str, cache_dir: Union[str, None] = None, + local_dir: Union[str, None] = None, ): + cache_dir = local_dir if not local_dir else cache_dir if not cache_dir: cache_dir = os.path.expanduser("~/.cache/clip") os.makedirs(cache_dir, exist_ok=True) @@ -295,9 +297,10 @@ def download_pretrained_from_hf( filename: str = 'open_clip_pytorch_model.bin', revision=None, cache_dir: Union[str, None] = None, + local_dir: Union[str, None] = None, ): has_hf_hub(True) - cached_file = hf_hub_download(model_id, filename, revision=revision, cache_dir=cache_dir) + cached_file = hf_hub_download(model_id, filename, revision=revision, cache_dir=cache_dir, local_dir=local_dir) return cached_file @@ -305,6 +308,7 @@ def download_pretrained( cfg: Dict, force_hf_hub: bool = False, cache_dir: Union[str, None] = None, + local_dir: Union[str, None] = None, ): target = '' if not cfg: @@ -317,7 +321,7 @@ def download_pretrained( download_url = '' if download_url: - target = download_pretrained_from_url(download_url, cache_dir=cache_dir) + target = download_pretrained_from_url(download_url, cache_dir=cache_dir, local_dir=local_dir) elif download_hf_hub: has_hf_hub(True) # we assume the hf_hub entries in pretrained config combine model_id + filename in @@ -325,8 +329,8 @@ def download_pretrained( # use 'open_clip_pytorch_model.bin' default, there must be a trailing slash 'org/model_name/'. model_id, filename = os.path.split(download_hf_hub) if filename: - target = download_pretrained_from_hf(model_id, filename=filename, cache_dir=cache_dir) + target = download_pretrained_from_hf(model_id, filename=filename, cache_dir=cache_dir, local_dir=local_dir) else: - target = download_pretrained_from_hf(model_id, cache_dir=cache_dir) + target = download_pretrained_from_hf(model_id, cache_dir=cache_dir, local_dir=local_dir) return target diff --git a/pulidflux.py b/pulidflux.py index 89f3810..b41b0db 100644 --- a/pulidflux.py +++ b/pulidflux.py @@ -1,7 +1,10 @@ import types +import zipfile import torch -from torch import nn, Tensor +from insightface.utils.download import download_file +from insightface.utils.storage import BASE_REPO_URL +from torch import nn from torchvision import transforms from torchvision.transforms import functional import os @@ -19,14 +22,30 @@ from .PulidFluxHook import pulid_forward_orig, set_model_dit_patch_replace, pulid_enter, pulid_patch_double_blocks_after from .patch_util import PatchKeys, add_model_patch_option, set_model_patch -INSIGHTFACE_DIR = os.path.join(folder_paths.models_dir, "insightface") -MODELS_DIR = os.path.join(folder_paths.models_dir, "pulid") -if "pulid" not in folder_paths.folder_names_and_paths: - current_paths = [MODELS_DIR] -else: - current_paths, _ = folder_paths.folder_names_and_paths["pulid"] -folder_paths.folder_names_and_paths["pulid"] = (current_paths, folder_paths.supported_pt_extensions) +def set_extra_config_model_path(extra_config_models_dir_key, models_dir_name:str): + models_dir_default = os.path.join(folder_paths.models_dir, models_dir_name) + if extra_config_models_dir_key not in folder_paths.folder_names_and_paths: + folder_paths.folder_names_and_paths[extra_config_models_dir_key] = ( + [os.path.join(folder_paths.models_dir, models_dir_name)], folder_paths.supported_pt_extensions) + else: + if not os.path.exists(models_dir_default): + os.makedirs(models_dir_default, exist_ok=True) + folder_paths.add_model_folder_path(extra_config_models_dir_key, models_dir_default, is_default=True) + +set_extra_config_model_path("pulid", "pulid") +set_extra_config_model_path("insightface", "insightface") +set_extra_config_model_path("facexlib", "facexlib") + +INSIGHTFACE_DIR = folder_paths.get_folder_paths("insightface")[0] +FACEXLIB_DIR = folder_paths.get_folder_paths("facexlib")[0] + +# MODELS_DIR = os.path.join(folder_paths.models_dir, "pulid") +# if "pulid" not in folder_paths.folder_names_and_paths: +# current_paths = [MODELS_DIR] +# else: +# current_paths, _ = folder_paths.folder_names_and_paths["pulid"] +# folder_paths.folder_names_and_paths["pulid"] = (current_paths, folder_paths.supported_pt_extensions) class PulidFluxModel(nn.Module): def __init__(self): @@ -115,6 +134,29 @@ def load_model(self, pulid_file): return (model_patcher,) +def download_insightface_model(sub_dir, name, force=False, root='~/.insightface'): + # Copied and modified from insightface.utils.storage.download + # Solve https://github.com/deepinsight/insightface/issues/2711 + _root = os.path.expanduser(root) + dir_path = os.path.join(_root, sub_dir, name) + if os.path.exists(dir_path) and not force: + return dir_path + print('download_path:', dir_path) + zip_file_path = os.path.join(_root, sub_dir, name + '.zip') + model_url = "%s/%s.zip"%(BASE_REPO_URL, name) + download_file(model_url, + path=zip_file_path, + overwrite=True) + if not os.path.exists(dir_path): + os.makedirs(dir_path) + + # zip file has contains ${name} + real_dir_path = os.path.join(_root, sub_dir) + with zipfile.ZipFile(zip_file_path) as zf: + zf.extractall(real_dir_path) + #os.remove(zip_file_path) + return dir_path + class PulidFluxInsightFaceLoader: @classmethod def INPUT_TYPES(s): @@ -129,7 +171,9 @@ def INPUT_TYPES(s): CATEGORY = "pulid" def load_insightface(self, provider): - model = FaceAnalysis(name="antelopev2", root=INSIGHTFACE_DIR, providers=[provider + 'ExecutionProvider',]) # alternative to buffalo_l + name = "antelopev2" + download_insightface_model("models", name, root=INSIGHTFACE_DIR) + model = FaceAnalysis(name=name, root=INSIGHTFACE_DIR, providers=[provider + 'ExecutionProvider', ]) # alternative to buffalo_l model.prepare(ctx_id=0, det_size=(640, 640)) return (model,) @@ -148,7 +192,12 @@ def INPUT_TYPES(s): def load_eva_clip(self): from .eva_clip.factory import create_model_and_transforms - model, _, _ = create_model_and_transforms('EVA02-CLIP-L-14-336', 'eva_clip', force_custom_clip=True) + clip_file_path = folder_paths.get_full_path("text_encoders", 'EVA02_CLIP_L_336_psz14_s6B.pt') + if clip_file_path is None: + clip_dir = os.path.join(folder_paths.models_dir, "clip") + else: + clip_dir = os.path.dirname(clip_file_path) + model, _, _ = create_model_and_transforms('EVA02-CLIP-L-14-336', 'eva_clip', force_custom_clip=True, local_dir=clip_dir) model = model.visual @@ -221,10 +270,11 @@ def apply_pulid_flux(self, model, pulid_flux, eva_clip, face_analysis, image, we det_model='retinaface_resnet50', save_ext='png', device=device, + model_rootpath=FACEXLIB_DIR ) face_helper.face_parse = None - face_helper.face_parse = init_parsing_model(model_name='bisenet', device=device) + face_helper.face_parse = init_parsing_model(model_name='bisenet', device=device, model_rootpath=FACEXLIB_DIR) bg_label = [0, 16, 18, 7, 8, 9, 14, 15] cond = [] diff --git a/pyproject.toml b/pyproject.toml index 31405bb..d6141ab 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "comfyui_pulid_flux_ll" description = "The implementation for PuLID-Flux, support use with TeaCache and WaveSpeed, no model pollution." -version = "1.0.9" +version = "1.1.0" license = {file = "LICENSE"} dependencies = ['facexlib', 'insightface', 'onnxruntime', 'onnxruntime-gpu; sys_platform != "darwin" and platform_machine == "x86_64"', 'ftfy', 'timm']