diff --git a/scripts/deforum_helpers/deforum_controlnet.py b/scripts/deforum_helpers/deforum_controlnet.py index cef20cbc8..a19e1618e 100644 --- a/scripts/deforum_helpers/deforum_controlnet.py +++ b/scripts/deforum_helpers/deforum_controlnet.py @@ -27,7 +27,7 @@ from modules import scripts, shared from .deforum_controlnet_gradio import hide_ui_by_cn_status, hide_file_textboxes, ToolButton from .general_utils import count_files_in_folder, clean_gradio_path_strings # TODO: do it another way -from .video_audio_utilities import vid2frames, convert_image +from .video_audio_utilities import SUPPORTED_IMAGE_EXTENSIONS, SUPPORTED_VIDEO_EXTENSIONS, get_extension_if_valid, vid2frames, convert_image from .animation_key_frames import ControlNetKeys from .load_images import load_image from .general_utils import debug_print @@ -323,23 +323,19 @@ def find_controlnet_script(p): raise Exception("ControlNet script not found.") return controlnet_script -def process_controlnet_input_frames(args, anim_args, controlnet_args, video_path, mask_path, outdir_suffix, id): - if (video_path or mask_path) and getattr(controlnet_args, f'cn_{id}_enabled'): +def process_controlnet_input_frames(args, anim_args, controlnet_args, input_path, is_mask, outdir_suffix, id): + if (input_path) and getattr(controlnet_args, f'cn_{id}_enabled'): frame_path = os.path.join(args.outdir, f'controlnet_{id}_{outdir_suffix}') os.makedirs(frame_path, exist_ok=True) - accepted_image_extensions = ('.jpg', '.jpeg', '.png', '.bmp') - if video_path and video_path.lower().endswith(accepted_image_extensions): - convert_image(video_path, os.path.join(frame_path, '000000000.jpg')) - print(f"Copied CN Model {id}'s single input image to inputframes folder!") - elif mask_path and mask_path.lower().endswith(accepted_image_extensions): - convert_image(mask_path, os.path.join(frame_path, '000000000.jpg')) - print(f"Copied CN Model {id}'s single input image to inputframes *mask* folder!") - else: - print(f'Unpacking ControlNet {id} {"video mask" if mask_path else "base video"}') + input_extension = get_extension_if_valid(input_path, SUPPORTED_IMAGE_EXTENSIONS + SUPPORTED_VIDEO_EXTENSIONS) + input_is_video = input_extension in SUPPORTED_VIDEO_EXTENSIONS + + if input_is_video: + print(f'Unpacking ControlNet {id} {"video mask" if is_mask else "base video"}') print(f"Exporting Video Frames to {frame_path}...") vid2frames( - video_path=video_path or mask_path, + video_path=input_path, video_in_frame_path=frame_path, n=1 if anim_args.animation_mode != 'Video Input' else anim_args.extract_nth_frame, overwrite=getattr(controlnet_args, f'cn_{id}_overwrite_frames'), @@ -348,7 +344,11 @@ def process_controlnet_input_frames(args, anim_args, controlnet_args, video_path numeric_files_output=True ) print(f"Loading {anim_args.max_frames} input frames from {frame_path} and saving video frames to {args.outdir}") - print(f'ControlNet {id} {"video mask" if mask_path else "base video"} unpacked!') + print(f'ControlNet {id} {"video mask" if is_mask else "base video"} unpacked!') + else: + convert_image(input_path, os.path.join(frame_path, '000000000.jpg')) + print(f"Copied CN Model {id}'s single input image to {frame_path} folder!") + def unpack_controlnet_vids(args, anim_args, controlnet_args): # this func gets called from render.py once for an entire animation run --> @@ -362,7 +362,7 @@ def unpack_controlnet_vids(args, anim_args, controlnet_args): mask_path = clean_gradio_path_strings(getattr(controlnet_args, f'cn_{i}_mask_vid_path', None)) if vid_path: # Process base video, if available - process_controlnet_input_frames(args, anim_args, controlnet_args, vid_path, None, 'inputframes', i) + process_controlnet_input_frames(args, anim_args, controlnet_args, vid_path, False, 'inputframes', i) if mask_path: # Process mask video, if available - process_controlnet_input_frames(args, anim_args, controlnet_args, None, mask_path, 'maskframes', i) + process_controlnet_input_frames(args, anim_args, controlnet_args, mask_path, True, 'maskframes', i) diff --git a/scripts/deforum_helpers/http_client.py b/scripts/deforum_helpers/http_client.py new file mode 100644 index 000000000..8bfcf5c61 --- /dev/null +++ b/scripts/deforum_helpers/http_client.py @@ -0,0 +1,25 @@ +import requests +from requests.adapters import HTTPAdapter +from urllib3.util.retry import Retry + + +# Returns a retrying http client. This is low overhead +# so fine to retrieve on every request. +def get_http_client( + retries=5, + backoff_factor=0.3, + status_forcelist=[429, 500, 502, 503, 504], + session=None, +) -> requests.Session : + session = session or requests.Session() + retry = Retry( + total=retries, + read=retries, + connect=retries, + backoff_factor=backoff_factor, + status_forcelist=status_forcelist, + ) + adapter = HTTPAdapter(max_retries=retry) + session.mount('http://', adapter) + session.mount('https://', adapter) + return session \ No newline at end of file diff --git a/scripts/deforum_helpers/hybrid_video.py b/scripts/deforum_helpers/hybrid_video.py index 03e742a0c..fdff95d71 100644 --- a/scripts/deforum_helpers/hybrid_video.py +++ b/scripts/deforum_helpers/hybrid_video.py @@ -350,7 +350,7 @@ def get_flow_from_images(i1, i2, method, raft_model, prev_flow=None): elif method == "PCAFlow": # Unused - requires running opencv-contrib-python (full opencv) INSTEAD of opencv-python return get_flow_from_images_PCAFlow(i1, i2, prev_flow) elif method == "Farneback": # Farneback Normal: - return get_flow_from_images_Farneback(i1, i2, prev_flow) + return get_flow_from_images_Farneback(i1, i2, last_flow=prev_flow) # if we reached this point, something went wrong. raise an error: raise RuntimeError(f"Invald flow method name: '{method}'") diff --git a/scripts/deforum_helpers/parseq_adapter.py b/scripts/deforum_helpers/parseq_adapter.py index d217a71b2..19023483d 100644 --- a/scripts/deforum_helpers/parseq_adapter.py +++ b/scripts/deforum_helpers/parseq_adapter.py @@ -64,7 +64,7 @@ def __init__(self, parseq_args, anim_args, video_args, controlnet_args, loop_arg # Wrap the original schedules with Parseq decorators, so that Parseq values will override the original values IFF appropriate. self.anim_keys = ParseqAnimKeysDecorator(self, DeformAnimKeys(anim_args)) - self.cn_keys = ParseqControlNetKeysDecorator(self, ControlNetKeys(anim_args, controlnet_args)) if controlnet_args else None + self.cn_keys = ParseqControlNetKeysDecorator(self, ControlNetKeys(anim_args, controlnet_args)) if (controlnet_args and len(controlnet_args.__dict__)>0) else None # -1 because seed seems to be unused in LooperAnimKeys self.looper_keys = ParseqLooperKeysDecorator(self, LooperAnimKeys(loop_args, anim_args, -1)) if loop_args else None diff --git a/scripts/deforum_helpers/run_deforum.py b/scripts/deforum_helpers/run_deforum.py index ca25d152b..34e4d43f1 100644 --- a/scripts/deforum_helpers/run_deforum.py +++ b/scripts/deforum_helpers/run_deforum.py @@ -163,7 +163,7 @@ def run_deforum(*args): mp4 = open(mp4_path, 'rb').read() data_url = f"data:video/mp4;base64, {b64encode(mp4).decode()}" global last_vid_data - last_vid_data = f'

Deforum extension for auto1111 — version 2.4b

' + last_vid_data = f'

Deforum extension for auto1111-0v1.9 — version 3.1

' except Exception as e: if need_to_frame_interpolate: print(f"FFMPEG DID NOT STITCH ANY VIDEO. However, you requested to frame interpolate - so we will continue to frame interpolation, but you'll be left only with the interpolated frames and not a video, since ffmpeg couldn't run. Original ffmpeg error: {e}") diff --git a/scripts/deforum_helpers/video_audio_utilities.py b/scripts/deforum_helpers/video_audio_utilities.py index 3935d518f..344d22a21 100644 --- a/scripts/deforum_helpers/video_audio_utilities.py +++ b/scripts/deforum_helpers/video_audio_utilities.py @@ -18,40 +18,48 @@ import cv2 import shutil import math -import requests import subprocess import time import tempfile import re import glob +import numpy as np import concurrent.futures from pathlib import Path from pkg_resources import resource_filename from modules.shared import state, opts from .general_utils import checksum, clean_gradio_path_strings, debug_print +from basicsr.utils.download_util import load_file_from_url from .rich import console import shutil from threading import Thread -try: - from modules.modelloader import load_file_from_url -except: - print("Try to fallback to basicsr with older modules") - from basicsr.utils.download_util import load_file_from_url +from .http_client import get_http_client + +SUPPORTED_IMAGE_EXTENSIONS = ["png", "jpg", "jpeg", "bmp", "webp"] +SUPPORTED_VIDEO_EXTENSIONS = ["mov", "mpeg", "mp4", "m4v", "avi", "mpg", "webm"] def convert_image(input_path, output_path): + extension = get_extension_if_valid(input_path, SUPPORTED_IMAGE_EXTENSIONS) + + if not extension: + return + # Read the input image - img = cv2.imread(input_path) - # Get the file extension of the output path - out_ext = os.path.splitext(output_path)[1].lower() + if input_path.startswith('http://') or input_path.startswith('https://'): + resp = get_http_client().get(input_path, allow_redirects=True) + arr = np.asarray(bytearray(resp.content), dtype=np.uint8) + img = cv2.imdecode(arr, -1) + else: + img = cv2.imread(input_path) + # Convert the image to the specified output format - if out_ext == ".png": + if extension == "png": cv2.imwrite(output_path, img, [cv2.IMWRITE_PNG_COMPRESSION, 9]) - elif out_ext == ".jpg" or out_ext == ".jpeg": + elif extension == "jpg" or extension == "jpeg": cv2.imwrite(output_path, img, [cv2.IMWRITE_JPEG_QUALITY, 99]) - elif out_ext == ".bmp": - cv2.imwrite(output_path, img) else: - print(f"Unsupported output format: {out_ext}") + cv2.imwrite(output_path, img) + def get_ffmpeg_params(): # get ffmpeg params from webui's settings -> deforum tab. actual opts are set in deforum.py f_location = opts.data.get("deforum_ffmpeg_location", find_ffmpeg_binary()) @@ -90,7 +98,7 @@ def vid2frames(video_path, video_in_frame_path, n=1, overwrite=True, extract_fro video_path = clean_gradio_path_strings(video_path) # check vid path using a function and only enter if we get True - if is_vid_path_valid(video_path): + if get_extension_if_valid(video_path, SUPPORTED_VIDEO_EXTENSIONS): name = get_frame_name(video_path) @@ -153,36 +161,50 @@ def vid2frames(video_path, video_in_frame_path, n=1, overwrite=True, extract_fro vidcap.release() return video_fps -# make sure the video_path provided is an existing local file or a web URL with a supported file extension -def is_vid_path_valid(video_path): - # make sure file format is supported! - file_formats = ["mov", "mpeg", "mp4", "m4v", "avi", "mpg", "webm"] - extension = video_path.rsplit('.', 1)[-1].lower() - # vid path is actually a URL, check it - if video_path.startswith('http://') or video_path.startswith('https://'): - response = requests.head(video_path, allow_redirects=True) - extension = extension.rsplit('?', 1)[0] # remove query string before checking file format extension. +# Make sure path_to_check provided is an existing local file or a web URL with a supported file extension +# Check Content-Disposition if necessary. +# If so, return the extension. If not, raise an error. +def get_extension_if_valid(path_to_check, acceptable_extensions: list[str] ) -> str: + + if path_to_check.startswith('http://') or path_to_check.startswith('https://'): + extension = path_to_check.rsplit('?', 1)[0].rsplit('.', 1)[-1] # remove query string before checking file format extension. + if extension in acceptable_extensions: + return extension + + # Path is actually a URL. Make sure it resolves and has a valid file extension. + response = get_http_client().head(path_to_check, allow_redirects=True) + if response.status_code != 200: + # Pre-signed URLs for GET requests might not like HEAD requests. Try a 0 range GET request. + debug_print("Failed HEAD request, trying 0 range GET. Status code: " + str(response.status_code)) + response = get_http_client().get(path_to_check, headers={'Range': 'bytes=0-0'}, allow_redirects=True) + if response.status_code != 200: + debug_print("Also failed 0 range GET. Status code: " + str(response.status_code)) + raise ConnectionError(f"URL {path_to_check} is not valid. Response status code: {response.status_code}") + + content_disposition_extension = None content_disposition = response.headers.get('Content-Disposition') - if content_disposition and extension not in file_formats: - # Filename doesn't look valid, but perhaps the content disposition will say otherwise? + if content_disposition: match = re.search(r'filename="?(?P[^"]+)"?', content_disposition) if match: - extension = match.group('filename').rsplit('.', 1)[-1].lower() - if response.status_code == 404: - raise ConnectionError(f"Video URL {video_path} is not valid. Response status code: {response.status_code}") - elif response.status_code == 302: - response = requests.head(response.headers['location'], allow_redirects=True) - if response.status_code != 200: - raise ConnectionError(f"Video URL {video_path} is not valid. Response status code: {response.status_code}") - if extension not in file_formats: - raise ValueError(f"Video file {video_path} has format '{extension}', which is not supported. Supported formats are: {file_formats}") + content_disposition_extension = match.group('filename').rsplit('.', 1)[-1].lower() + + if content_disposition_extension in acceptable_extensions: + return content_disposition_extension + + raise ValueError(f"File {path_to_check} has format '{extension}' (from URL) or '{content_disposition_extension}' (from content disposition), which are not supported. Supported formats are: {acceptable_extensions}") + else: - video_path = os.path.realpath(video_path) - if not os.path.exists(video_path): - raise RuntimeError(f"Video path does not exist: {video_path}") - if extension not in file_formats: - raise ValueError(f"Video file {video_path} has format '{extension}', which is not supported. Supported formats are: {file_formats}") - return True + path_to_check = os.path.realpath(path_to_check) + extension = path_to_check.rsplit('.', 1)[-1].lower() + + if not os.path.exists(path_to_check): + raise RuntimeError(f"Path does not exist: {path_to_check}") + if extension in acceptable_extensions: + return extension + + raise ValueError(f"File {path_to_check} has format '{extension}', which is not supported. Supported formats are: {acceptable_extensions}") + + # quick-retreive frame count, FPS and H/W dimensions of a video (local or URL-based) def get_quick_vid_info(vid_path): @@ -246,7 +268,7 @@ def ffmpeg_stitch_video(ffmpeg_location=None, fps=None, outmp4_path=None, stitch if (audio_path.startswith('http://') or audio_path.startswith('https://')): url = audio_path print(f"Downloading audio file from: {url}") - response = requests.get(url, stream=True) + response = get_http_client().get(url, stream=True) response.raise_for_status() temp_file = tempfile.NamedTemporaryFile(delete=False) # Write the content of the downloaded file into the temporary file @@ -281,7 +303,7 @@ def ffmpeg_stitch_video(ffmpeg_location=None, fps=None, outmp4_path=None, stitch add_soundtrack_status = f"\rError adding audio to video: {e}" add_soundtrack_success = False finally: - if temp_file: + if temp_file is not None: file_path = Path(temp_file.name) file_path.unlink(missing_ok=True)