From 4c5d890ced9f7a969b281e204dc3eb70ebe7d39a Mon Sep 17 00:00:00 2001 From: Jason Kneen Date: Tue, 13 Aug 2024 12:33:49 +0100 Subject: [PATCH 01/12] Update .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index a6088af..76f143a 100644 --- a/.gitignore +++ b/.gitignore @@ -21,3 +21,4 @@ models/inswapper_128.onnx models/GFPGANv1.4.pth *.onnx models/DMDNet.pth +.venv/ From 89912f7929eb8bc4889e0be97505df736717fa03 Mon Sep 17 00:00:00 2001 From: Jason Kneen Date: Tue, 13 Aug 2024 13:06:14 +0100 Subject: [PATCH 02/12] Update .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 76f143a..bb9984b 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,7 @@ __pycache__/ .todo *.log *.backup +.tfenv/ *.png *.mp4 From 2935187a4fedc9b51e369748c47bb7fb780b893a Mon Sep 17 00:00:00 2001 From: Jason Kneen Date: Tue, 13 Aug 2024 13:06:48 +0100 Subject: [PATCH 03/12] Update .gitignore --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index bb9984b..657a113 100644 --- a/.gitignore +++ b/.gitignore @@ -6,7 +6,7 @@ __pycache__/ .todo *.log *.backup -.tfenv/ +.tf_env/ *.png *.mp4 From fb695b0208b330a1eb44c276a2b454e8a5522a2a Mon Sep 17 00:00:00 2001 From: Jason Kneen Date: Tue, 13 Aug 2024 13:07:49 +0100 Subject: [PATCH 04/12] Update .gitignore --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 657a113..c457104 100644 --- a/.gitignore +++ b/.gitignore @@ -6,7 +6,6 @@ __pycache__/ .todo *.log *.backup -.tf_env/ *.png *.mp4 @@ -23,3 +22,5 @@ models/GFPGANv1.4.pth *.onnx models/DMDNet.pth .venv/ +tf_env/ +.tf_env/ From 359848a54ebe02ec0a0a3d5a511b5bd301e29db0 Mon Sep 17 00:00:00 2001 From: Jason Kneen Date: Tue, 13 Aug 2024 13:08:06 +0100 Subject: [PATCH 05/12] Updates for macOS and coreML / Metal --- modules/core.py | 180 ++++++++++++----------- modules/processors/frame/face_swapper.py | 4 +- modules/ui.py | 10 +- modules/utilities.py | 15 +- requirements.txt | 41 +++--- 5 files changed, 139 insertions(+), 111 deletions(-) diff --git a/modules/core.py b/modules/core.py index 3ec1617..25dd272 100644 --- a/modules/core.py +++ b/modules/core.py @@ -5,6 +5,8 @@ if any(arg.startswith('--execution-provider') for arg in sys.argv): os.environ['OMP_NUM_THREADS'] = '1' # reduce tensorflow log level os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2' +# Force TensorFlow to use Metal +os.environ['TENSORFLOW_METAL'] = '1' import warnings from typing import List import platform @@ -35,23 +37,17 @@ def parse_args() -> None: program.add_argument('-t', '--target', help='select an target image or video', dest='target_path') program.add_argument('-o', '--output', help='select output file or directory', dest='output_path') program.add_argument('--frame-processor', help='pipeline of frame processors', dest='frame_processor', default=['face_swapper'], choices=['face_swapper', 'face_enhancer'], nargs='+') - program.add_argument('--keep-fps', help='keep original fps', dest='keep_fps', action='store_true', default=False) + program.add_argument('--keep-fps', help='keep original fps', dest='keep_fps', action='store_true', default=True) program.add_argument('--keep-audio', help='keep original audio', dest='keep_audio', action='store_true', default=True) - program.add_argument('--keep-frames', help='keep temporary frames', dest='keep_frames', action='store_true', default=False) + program.add_argument('--keep-frames', help='keep temporary frames', dest='keep_frames', action='store_true', default=True) program.add_argument('--many-faces', help='process every face', dest='many_faces', action='store_true', default=False) program.add_argument('--video-encoder', help='adjust output video encoder', dest='video_encoder', default='libx264', choices=['libx264', 'libx265', 'libvpx-vp9']) - program.add_argument('--video-quality', help='adjust output video quality', dest='video_quality', type=int, default=18, choices=range(52), metavar='[0-51]') + program.add_argument('--video-quality', help='adjust output video quality', dest='video_quality', type=int, default=50, choices=range(52), metavar='[0-51]') program.add_argument('--max-memory', help='maximum amount of RAM in GB', dest='max_memory', type=int, default=suggest_max_memory()) - program.add_argument('--execution-provider', help='execution provider', dest='execution_provider', default=['cpu'], choices=suggest_execution_providers(), nargs='+') + program.add_argument('--execution-provider', help='execution provider', dest='execution_provider', default=['coreml'], choices=suggest_execution_providers(), nargs='+') program.add_argument('--execution-threads', help='number of execution threads', dest='execution_threads', type=int, default=suggest_execution_threads()) program.add_argument('-v', '--version', action='version', version=f'{modules.metadata.name} {modules.metadata.version}') - # register deprecated args - program.add_argument('-f', '--face', help=argparse.SUPPRESS, dest='source_path_deprecated') - program.add_argument('--cpu-cores', help=argparse.SUPPRESS, dest='cpu_cores_deprecated', type=int) - program.add_argument('--gpu-vendor', help=argparse.SUPPRESS, dest='gpu_vendor_deprecated') - program.add_argument('--gpu-threads', help=argparse.SUPPRESS, dest='gpu_threads_deprecated', type=int) - args = program.parse_args() modules.globals.source_path = args.source_path @@ -66,10 +62,9 @@ def parse_args() -> None: modules.globals.video_encoder = args.video_encoder modules.globals.video_quality = args.video_quality modules.globals.max_memory = args.max_memory - modules.globals.execution_providers = decode_execution_providers(args.execution_provider) + modules.globals.execution_providers = ['CoreMLExecutionProvider'] # Force CoreML modules.globals.execution_threads = args.execution_threads - #for ENHANCER tumbler: if 'face_enhancer' in args.frame_processor: modules.globals.fp_ui['face_enhancer'] = True else: @@ -77,36 +72,6 @@ def parse_args() -> None: modules.globals.nsfw = False - # translate deprecated args - if args.source_path_deprecated: - print('\033[33mArgument -f and --face are deprecated. Use -s and --source instead.\033[0m') - modules.globals.source_path = args.source_path_deprecated - modules.globals.output_path = normalize_output_path(args.source_path_deprecated, modules.globals.target_path, args.output_path) - if args.cpu_cores_deprecated: - print('\033[33mArgument --cpu-cores is deprecated. Use --execution-threads instead.\033[0m') - modules.globals.execution_threads = args.cpu_cores_deprecated - if args.gpu_vendor_deprecated == 'apple': - print('\033[33mArgument --gpu-vendor apple is deprecated. Use --execution-provider coreml instead.\033[0m') - modules.globals.execution_providers = decode_execution_providers(['coreml']) - if args.gpu_vendor_deprecated == 'nvidia': - print('\033[33mArgument --gpu-vendor nvidia is deprecated. Use --execution-provider cuda instead.\033[0m') - modules.globals.execution_providers = decode_execution_providers(['cuda']) - if args.gpu_vendor_deprecated == 'amd': - print('\033[33mArgument --gpu-vendor amd is deprecated. Use --execution-provider cuda instead.\033[0m') - modules.globals.execution_providers = decode_execution_providers(['rocm']) - if args.gpu_threads_deprecated: - print('\033[33mArgument --gpu-threads is deprecated. Use --execution-threads instead.\033[0m') - modules.globals.execution_threads = args.gpu_threads_deprecated - - -def encode_execution_providers(execution_providers: List[str]) -> List[str]: - return [execution_provider.replace('ExecutionProvider', '').lower() for execution_provider in execution_providers] - - -def decode_execution_providers(execution_providers: List[str]) -> List[str]: - return [provider for provider, encoded_execution_provider in zip(onnxruntime.get_available_providers(), encode_execution_providers(onnxruntime.get_available_providers())) - if any(execution_provider in encoded_execution_provider for execution_provider in execution_providers)] - def suggest_max_memory() -> int: if platform.system().lower() == 'darwin': @@ -115,39 +80,22 @@ def suggest_max_memory() -> int: def suggest_execution_providers() -> List[str]: - return encode_execution_providers(onnxruntime.get_available_providers()) + return ['coreml'] # Only suggest CoreML def suggest_execution_threads() -> int: - if 'DmlExecutionProvider' in modules.globals.execution_providers: - return 1 - if 'ROCMExecutionProvider' in modules.globals.execution_providers: - return 1 return 8 def limit_resources() -> None: - # prevent tensorflow memory leak - gpus = tensorflow.config.experimental.list_physical_devices('GPU') - for gpu in gpus: - tensorflow.config.experimental.set_memory_growth(gpu, True) - # limit memory usage if modules.globals.max_memory: - memory = modules.globals.max_memory * 1024 ** 3 - if platform.system().lower() == 'darwin': - memory = modules.globals.max_memory * 1024 ** 6 - if platform.system().lower() == 'windows': - import ctypes - kernel32 = ctypes.windll.kernel32 - kernel32.SetProcessWorkingSetSize(-1, ctypes.c_size_t(memory), ctypes.c_size_t(memory)) - else: - import resource - resource.setrlimit(resource.RLIMIT_DATA, (memory, memory)) + memory = modules.globals.max_memory * 1024 ** 6 + import resource + resource.setrlimit(resource.RLIMIT_DATA, (memory, memory)) def release_resources() -> None: - if 'CUDAExecutionProvider' in modules.globals.execution_providers: - torch.cuda.empty_cache() + pass # No need to release CUDA resources def pre_check() -> bool: @@ -170,23 +118,28 @@ def start() -> None: for frame_processor in get_frame_processors_modules(modules.globals.frame_processors): if not frame_processor.pre_start(): return - # process image to image if has_image_extension(modules.globals.target_path): - if modules.globals.nsfw == False: - from modules.predicter import predict_image - if predict_image(modules.globals.target_path): - destroy() - shutil.copy2(modules.globals.target_path, modules.globals.output_path) - for frame_processor in get_frame_processors_modules(modules.globals.frame_processors): - update_status('Progressing...', frame_processor.NAME) - frame_processor.process_image(modules.globals.source_path, modules.globals.output_path, modules.globals.output_path) - release_resources() - if is_image(modules.globals.target_path): - update_status('Processing to image succeed!') - else: - update_status('Processing to image failed!') - return - # process image to videos + process_image() + else: + process_video() + + +def process_image(): + if modules.globals.nsfw == False: + from modules.predicter import predict_image + if predict_image(modules.globals.target_path): + destroy() + shutil.copy2(modules.globals.target_path, modules.globals.output_path) + for frame_processor in get_frame_processors_modules(modules.globals.frame_processors): + update_status('Progressing...', frame_processor.NAME) + frame_processor.process_image(modules.globals.source_path, modules.globals.output_path, modules.globals.output_path) + if is_image(modules.globals.target_path): + update_status('Processing to image succeed!') + else: + update_status('Processing to image failed!') + + +def process_video(): if modules.globals.nsfw == False: from modules.predicter import predict_video if predict_video(modules.globals.target_path): @@ -199,8 +152,6 @@ def start() -> None: for frame_processor in get_frame_processors_modules(modules.globals.frame_processors): update_status('Progressing...', frame_processor.NAME) frame_processor.process_video(modules.globals.source_path, temp_frame_paths) - release_resources() - # handles fps if modules.globals.keep_fps: update_status('Detecting fps...') fps = detect_fps(modules.globals.target_path) @@ -209,7 +160,6 @@ def start() -> None: else: update_status('Creating video with 30.0 fps...') create_video(modules.globals.target_path) - # handle audio if modules.globals.keep_audio: if modules.globals.keep_fps: update_status('Restoring audio...') @@ -218,7 +168,6 @@ def start() -> None: restore_audio(modules.globals.target_path, modules.globals.output_path) else: move_temp(modules.globals.target_path, modules.globals.output_path) - # clean and validate clean_temp(modules.globals.target_path) if is_video(modules.globals.target_path): update_status('Processing to video succeed!') @@ -240,6 +189,69 @@ def run() -> None: if not frame_processor.pre_check(): return limit_resources() + print(f"ONNX Runtime version: {onnxruntime.__version__}") + print(f"Available execution providers: {onnxruntime.get_available_providers()}") + print(f"Selected execution provider: CoreMLExecutionProvider") + + # Configure ONNX Runtime to use only CoreML + onnxruntime.set_default_logger_severity(3) # Set to WARNING level + options = onnxruntime.SessionOptions() + options.graph_optimization_level = onnxruntime.GraphOptimizationLevel.ORT_ENABLE_ALL + + # Test CoreML with a dummy model + try: + import numpy as np + from onnx import helper, TensorProto + + # Create a simple ONNX model + X = helper.make_tensor_value_info('input', TensorProto.FLOAT, [1, 3, 224, 224]) + Y = helper.make_tensor_value_info('output', TensorProto.FLOAT, [1, 3, 224, 224]) + node = helper.make_node('Identity', ['input'], ['output']) + graph = helper.make_graph([node], 'test_model', [X], [Y]) + model = helper.make_model(graph) + + # Save the model + model_path = 'test_model.onnx' + with open(model_path, 'wb') as f: + f.write(model.SerializeToString()) + + # Create a CoreML session + session = onnxruntime.InferenceSession(model_path, options, providers=['CoreMLExecutionProvider']) + + # Run inference + input_data = np.random.rand(1, 3, 224, 224).astype(np.float32) + output = session.run(None, {'input': input_data}) + + print("CoreML init successful and being used") + print(f"Input shape: {input_data.shape}, Output shape: {output[0].shape}") + + # Clean up + os.remove(model_path) + except Exception as e: + print(f"Error testing CoreML: {str(e)}") + print("The application may not be able to use GPU acceleration") + + # Configure TensorFlow to use Metal + try: + tf_devices = tensorflow.config.list_physical_devices() + print("TensorFlow devices:", tf_devices) + if any('GPU' in device.name for device in tf_devices): + print("TensorFlow is using GPU (Metal)") + else: + print("TensorFlow is not using GPU") + except Exception as e: + print(f"Error configuring TensorFlow: {str(e)}") + + # Configure PyTorch to use MPS (Metal Performance Shaders) + try: + if torch.backends.mps.is_available(): + print("PyTorch is using MPS (Metal Performance Shaders)") + torch.set_default_device('mps') + else: + print("PyTorch MPS is not available") + except Exception as e: + print(f"Error configuring PyTorch: {str(e)}") + if modules.globals.headless: start() else: diff --git a/modules/processors/frame/face_swapper.py b/modules/processors/frame/face_swapper.py index 4b4a222..1e39ffd 100644 --- a/modules/processors/frame/face_swapper.py +++ b/modules/processors/frame/face_swapper.py @@ -17,7 +17,7 @@ NAME = 'DLC.FACE-SWAPPER' def pre_check() -> bool: download_directory_path = resolve_relative_path('../models') - conditional_download(download_directory_path, ['https://huggingface.co/hacksider/deep-live-cam/blob/main/inswapper_128_fp16.onnx']) + conditional_download(download_directory_path, ['https://huggingface.co/hacksider/deep-live-cam/blob/main/inswapper_128.onnx']) return True @@ -39,7 +39,7 @@ def get_face_swapper() -> Any: with THREAD_LOCK: if FACE_SWAPPER is None: - model_path = resolve_relative_path('../models/inswapper_128_fp16.onnx') + model_path = resolve_relative_path('../models/inswapper_128.onnx') FACE_SWAPPER = insightface.model_zoo.get_model(model_path, providers=modules.globals.execution_providers) return FACE_SWAPPER diff --git a/modules/ui.py b/modules/ui.py index 1d0bb69..220eec4 100644 --- a/modules/ui.py +++ b/modules/ui.py @@ -257,11 +257,11 @@ def webcam_preview(): global preview_label, PREVIEW cap = cv2.VideoCapture(0) # Use index for the webcam (adjust the index accordingly if necessary) - cap.set(cv2.CAP_PROP_FRAME_WIDTH, 960) # Set the width of the resolution - cap.set(cv2.CAP_PROP_FRAME_HEIGHT, 540) # Set the height of the resolution + cap.set(cv2.CAP_PROP_FRAME_WIDTH, 1024) # Set the width of the resolution + cap.set(cv2.CAP_PROP_FRAME_HEIGHT, 768) # Set the height of the resolution cap.set(cv2.CAP_PROP_FPS, 60) # Set the frame rate of the webcam - PREVIEW_MAX_WIDTH = 960 - PREVIEW_MAX_HEIGHT = 540 + PREVIEW_MAX_WIDTH = 1024 + PREVIEW_MAX_HEIGHT = 768 preview_label.configure(image=None) # Reset the preview image before startup @@ -285,7 +285,7 @@ def webcam_preview(): for frame_processor in frame_processors: temp_frame = frame_processor.process_frame(source_image, temp_frame) - image = cv2.cvtColor(temp_frame, cv2.COLOR_BGR2RGB) # Convert the image to RGB format to display it with Tkinter + image = cv2.cvtColor(temp_frame, cv2.COLOR_BGR2RGB) # Convert the image to RGB format to display it with Tkinter image = Image.fromarray(image) image = ImageOps.contain(image, (PREVIEW_MAX_WIDTH, PREVIEW_MAX_HEIGHT), Image.LANCZOS) image = ctk.CTkImage(image, size=image.size) diff --git a/modules/utilities.py b/modules/utilities.py index 782395f..e3f5930 100644 --- a/modules/utilities.py +++ b/modules/utilities.py @@ -9,6 +9,7 @@ import urllib from pathlib import Path from typing import List, Any from tqdm import tqdm +import cv2 import modules.globals @@ -44,7 +45,19 @@ def detect_fps(target_path: str) -> float: def extract_frames(target_path: str) -> None: temp_directory_path = get_temp_directory_path(target_path) - run_ffmpeg(['-i', target_path, '-pix_fmt', 'rgb24', os.path.join(temp_directory_path, '%04d.png')]) + cap = cv2.VideoCapture(target_path) + + frame_count = 0 + while True: + ret, frame = cap.read() + if not ret: + break + + # Save the frame + cv2.imwrite(os.path.join(temp_directory_path, f'{frame_count:04d}.png'), frame) + frame_count += 1 + + cap.release() def create_video(target_path: str, fps: float = 30.0) -> None: diff --git a/requirements.txt b/requirements.txt index f65195e..9396c93 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,23 +1,26 @@ ---extra-index-url https://download.pytorch.org/whl/cu118 +# Deep Live Cam requirements -numpy==1.23.5 +# Core dependencies +numpy==1.26.4 +onnxruntime-silicon==1.16.3 opencv-python==4.8.1.78 -onnx==1.16.0 -insightface==0.7.3 -psutil==5.9.8 -tk==0.1.0 -customtkinter==5.2.2 pillow==9.5.0 -torch==2.0.1+cu118; sys_platform != 'darwin' -torch==2.0.1; sys_platform == 'darwin' -torchvision==0.15.2+cu118; sys_platform != 'darwin' -torchvision==0.15.2; sys_platform == 'darwin' -onnxruntime==1.18.0; sys_platform == 'darwin' and platform_machine != 'arm64' -onnxruntime-silicon==1.16.3; sys_platform == 'darwin' and platform_machine == 'arm64' -onnxruntime-gpu==1.18.0; sys_platform != 'darwin' -tensorflow==2.13.0rc1; sys_platform == 'darwin' -tensorflow==2.12.1; sys_platform != 'darwin' -opennsfw2==0.10.2 -protobuf==4.23.2 +insightface==0.7.3 +torch==2.1.0 # Add the specific version you're using +tensorflow==2.16.1 # Add the specific version you're using + +# Image processing +scikit-image==0.24.0 +matplotlib==3.9.1.post1 + +# Machine learning +scikit-learn==1.5.1 + +# Utilities tqdm==4.66.4 -gfpgan==1.3.8 +requests==2.32.3 +prettytable==3.11.0 + +# Optional dependencies (comment out if not needed) +# albumentations==1.4.13 +# coloredlogs==15.0.1 From 68fc4579a94b3fa5a2afbb9fe16e11f9c4e97c0d Mon Sep 17 00:00:00 2001 From: Jason Kneen Date: Tue, 13 Aug 2024 13:11:51 +0100 Subject: [PATCH 06/12] Update README.md --- README.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index d8e004f..aa1a4b4 100644 --- a/README.md +++ b/README.md @@ -74,19 +74,20 @@ python run.py --execution-provider coreml ``` ### [](https://github.com/s0md3v/roop/wiki/2.-Acceleration#coreml-execution-provider-apple-legacy)CoreML Execution Provider (Apple Legacy) +Metal support has been added for improved performance on macOS devices. 1. Install dependencies: ``` -pip uninstall onnxruntime onnxruntime-coreml -pip install onnxruntime-coreml==1.13.1 +pip uninstall onnxruntime onnxruntime-silicon +pip install onnxruntime-silicon==1.13.1 ``` 2. Usage in case the provider is available: ``` -python run.py --execution-provider coreml +python run.py --execution-provider metal ``` From e55bdd7aee13b7423b61c34667d68ac431ce82de Mon Sep 17 00:00:00 2001 From: Jason Kneen Date: Tue, 13 Aug 2024 13:21:52 +0100 Subject: [PATCH 07/12] Enable GPU on tensorflow --- requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 9396c93..d459449 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,7 +7,8 @@ opencv-python==4.8.1.78 pillow==9.5.0 insightface==0.7.3 torch==2.1.0 # Add the specific version you're using -tensorflow==2.16.1 # Add the specific version you're using +tensorflow-macos==2.16.2 # Add the specific version you're using +tensorflow-metal==1.1.0 # Add the specific version you're using # Image processing scikit-image==0.24.0 From 060f7c1f058ef0528d7fe7cc8e5fda8810e6eebd Mon Sep 17 00:00:00 2001 From: Jason Kneen Date: Tue, 13 Aug 2024 13:57:58 +0100 Subject: [PATCH 08/12] Model change to use GPU properly --- modules/processors/frame/face_swapper.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/processors/frame/face_swapper.py b/modules/processors/frame/face_swapper.py index 4b4a222..1e39ffd 100644 --- a/modules/processors/frame/face_swapper.py +++ b/modules/processors/frame/face_swapper.py @@ -17,7 +17,7 @@ NAME = 'DLC.FACE-SWAPPER' def pre_check() -> bool: download_directory_path = resolve_relative_path('../models') - conditional_download(download_directory_path, ['https://huggingface.co/hacksider/deep-live-cam/blob/main/inswapper_128_fp16.onnx']) + conditional_download(download_directory_path, ['https://huggingface.co/hacksider/deep-live-cam/blob/main/inswapper_128.onnx']) return True @@ -39,7 +39,7 @@ def get_face_swapper() -> Any: with THREAD_LOCK: if FACE_SWAPPER is None: - model_path = resolve_relative_path('../models/inswapper_128_fp16.onnx') + model_path = resolve_relative_path('../models/inswapper_128.onnx') FACE_SWAPPER = insightface.model_zoo.get_model(model_path, providers=modules.globals.execution_providers) return FACE_SWAPPER From 51d4984fef34bd269a2e2321cf5f59e06ac760ac Mon Sep 17 00:00:00 2001 From: Jason Kneen Date: Tue, 13 Aug 2024 19:03:22 +0100 Subject: [PATCH 09/12] Performance tweaks --- modules/core.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/modules/core.py b/modules/core.py index 25dd272..4a1eb5a 100644 --- a/modules/core.py +++ b/modules/core.py @@ -41,8 +41,8 @@ def parse_args() -> None: program.add_argument('--keep-audio', help='keep original audio', dest='keep_audio', action='store_true', default=True) program.add_argument('--keep-frames', help='keep temporary frames', dest='keep_frames', action='store_true', default=True) program.add_argument('--many-faces', help='process every face', dest='many_faces', action='store_true', default=False) - program.add_argument('--video-encoder', help='adjust output video encoder', dest='video_encoder', default='libx264', choices=['libx264', 'libx265', 'libvpx-vp9']) - program.add_argument('--video-quality', help='adjust output video quality', dest='video_quality', type=int, default=50, choices=range(52), metavar='[0-51]') + program.add_argument('--video-encoder', help='adjust output video encoder', dest='video_encoder', default='libx265', choices=['libx264', 'libx265', 'libvpx-vp9']) + program.add_argument('--video-quality', help='adjust output video quality', dest='video_quality', type=int, default=1, choices=range(52), metavar='[0-51]') program.add_argument('--max-memory', help='maximum amount of RAM in GB', dest='max_memory', type=int, default=suggest_max_memory()) program.add_argument('--execution-provider', help='execution provider', dest='execution_provider', default=['coreml'], choices=suggest_execution_providers(), nargs='+') program.add_argument('--execution-threads', help='number of execution threads', dest='execution_threads', type=int, default=suggest_execution_threads()) @@ -75,8 +75,8 @@ def parse_args() -> None: def suggest_max_memory() -> int: if platform.system().lower() == 'darwin': - return 4 - return 16 + return 6 + return 4 def suggest_execution_providers() -> List[str]: @@ -84,7 +84,10 @@ def suggest_execution_providers() -> List[str]: def suggest_execution_threads() -> int: - return 8 + if platform.system().lower() == 'darwin': + return 12 + return 4 + def limit_resources() -> None: From 1570d91a120aecc6f126d3f0e0914abafc1fe200 Mon Sep 17 00:00:00 2001 From: Jason Kneen Date: Tue, 13 Aug 2024 19:06:25 +0100 Subject: [PATCH 10/12] Update face_swapper.py --- modules/processors/frame/face_swapper.py | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/processors/frame/face_swapper.py b/modules/processors/frame/face_swapper.py index 1e39ffd..ad36979 100644 --- a/modules/processors/frame/face_swapper.py +++ b/modules/processors/frame/face_swapper.py @@ -15,6 +15,7 @@ THREAD_LOCK = threading.Lock() NAME = 'DLC.FACE-SWAPPER' +# model update to use GPU / Metal on MacOS def pre_check() -> bool: download_directory_path = resolve_relative_path('../models') conditional_download(download_directory_path, ['https://huggingface.co/hacksider/deep-live-cam/blob/main/inswapper_128.onnx']) From 6cda41de6c33fb14604a459bf9dc365a9190b21d Mon Sep 17 00:00:00 2001 From: Jason Kneen Date: Tue, 13 Aug 2024 20:54:50 +0100 Subject: [PATCH 11/12] WIP fixes and performance --- modules/core.py | 68 +++++++++- modules/processors/frame/face_swapper.py | 15 ++- modules/ui.py | 162 +++++++++++++++++------ requirements.txt | 11 +- 4 files changed, 205 insertions(+), 51 deletions(-) diff --git a/modules/core.py b/modules/core.py index 4a1eb5a..83eb751 100644 --- a/modules/core.py +++ b/modules/core.py @@ -29,6 +29,40 @@ if 'ROCMExecutionProvider' in modules.globals.execution_providers: warnings.filterwarnings('ignore', category=FutureWarning, module='insightface') warnings.filterwarnings('ignore', category=UserWarning, module='torchvision') +def get_system_memory() -> int: + """ + Get the total system memory in GB. + + Returns: + int: Total system memory in GB. + """ + if platform.system().lower() == 'darwin': + try: + import psutil + return psutil.virtual_memory().total // (1024 ** 3) + except ImportError: + # If psutil is not available, return a default value + return 16 # Assuming 16GB as a default for macOS + else: + # For other systems, we can use psutil if available, or implement system-specific methods + try: + import psutil + return psutil.virtual_memory().total // (1024 ** 3) + except ImportError: + # If psutil is not available, return a default value + return 8 # Assuming 8GB as a default for other systems + +def suggest_max_memory() -> int: + """ + Suggest the maximum memory to use based on the system's total memory. + + Returns: + int: Suggested maximum memory in GB. + """ + total_memory = get_system_memory() + # Suggest using 70% of total memory, but not more than 64GB + suggested_memory = min(int(total_memory * 0.7), 64) + return max(suggested_memory, 4) # Ensure at least 4GB is suggested def parse_args() -> None: signal.signal(signal.SIGINT, lambda signal_number, frame: destroy()) @@ -46,6 +80,8 @@ def parse_args() -> None: program.add_argument('--max-memory', help='maximum amount of RAM in GB', dest='max_memory', type=int, default=suggest_max_memory()) program.add_argument('--execution-provider', help='execution provider', dest='execution_provider', default=['coreml'], choices=suggest_execution_providers(), nargs='+') program.add_argument('--execution-threads', help='number of execution threads', dest='execution_threads', type=int, default=suggest_execution_threads()) + program.add_argument('--video-processor', help='video processor to use', dest='video_processor', default='cv2', choices=['cv2', 'ffmpeg']) + program.add_argument('--model', help='model to use for face swapping', dest='model', default='inswapper_128v2.fp16.onnx') program.add_argument('-v', '--version', action='version', version=f'{modules.metadata.name} {modules.metadata.version}') args = program.parse_args() @@ -64,6 +100,8 @@ def parse_args() -> None: modules.globals.max_memory = args.max_memory modules.globals.execution_providers = ['CoreMLExecutionProvider'] # Force CoreML modules.globals.execution_threads = args.execution_threads + modules.globals.video_processor = args.video_processor + modules.globals.model = args.model if 'face_enhancer' in args.frame_processor: modules.globals.fp_ui['face_enhancer'] = True @@ -89,7 +127,6 @@ def suggest_execution_threads() -> int: return 4 - def limit_resources() -> None: if modules.globals.max_memory: memory = modules.globals.max_memory * 1024 ** 6 @@ -150,7 +187,10 @@ def process_video(): update_status('Creating temp resources...') create_temp(modules.globals.target_path) update_status('Extracting frames...') - extract_frames(modules.globals.target_path) + if modules.globals.video_processor == 'cv2': + extract_frames_cv2(modules.globals.target_path) + else: + extract_frames_ffmpeg(modules.globals.target_path) temp_frame_paths = get_temp_frame_paths(modules.globals.target_path) for frame_processor in get_frame_processors_modules(modules.globals.frame_processors): update_status('Progressing...', frame_processor.NAME) @@ -178,6 +218,30 @@ def process_video(): update_status('Processing to video failed!') +def extract_frames_cv2(target_path: str) -> None: + import cv2 + capture = cv2.VideoCapture(target_path) + frame_num = 0 + while True: + success, frame = capture.read() + if not success: + break + cv2.imwrite(f'{get_temp_frame_paths(target_path)}/%04d.png' % frame_num, frame) + frame_num += 1 + capture.release() + + +def extract_frames_ffmpeg(target_path: str) -> None: + import ffmpeg + ( + ffmpeg + .input(target_path) + .output(f'{get_temp_frame_paths(target_path)}/%04d.png', start_number=0) + .overwrite_output() + .run(capture_stdout=True, capture_stderr=True) + ) + + def destroy() -> None: if modules.globals.target_path: clean_temp(modules.globals.target_path) diff --git a/modules/processors/frame/face_swapper.py b/modules/processors/frame/face_swapper.py index ad36979..f833093 100644 --- a/modules/processors/frame/face_swapper.py +++ b/modules/processors/frame/face_swapper.py @@ -2,6 +2,7 @@ from typing import Any, List import cv2 import insightface import threading +import numpy as np import modules.globals import modules.processors.frame.core @@ -15,7 +16,6 @@ THREAD_LOCK = threading.Lock() NAME = 'DLC.FACE-SWAPPER' -# model update to use GPU / Metal on MacOS def pre_check() -> bool: download_directory_path = resolve_relative_path('../models') conditional_download(download_directory_path, ['https://huggingface.co/hacksider/deep-live-cam/blob/main/inswapper_128.onnx']) @@ -46,7 +46,18 @@ def get_face_swapper() -> Any: def swap_face(source_face: Face, target_face: Face, temp_frame: Frame) -> Frame: - return get_face_swapper().get(temp_frame, target_face, source_face, paste_back=True) + try: + print("Debug: Starting face swap") + print(f"Debug: temp_frame shape: {temp_frame.shape}, dtype: {temp_frame.dtype}") + print(f"Debug: target_face keys: {target_face.keys()}") + print(f"Debug: source_face keys: {source_face.keys()}") + + result = get_face_swapper().get(temp_frame, target_face, source_face, paste_back=True) + print("Debug: Face swap completed successfully") + return result + except Exception as e: + print(f"Error in swap_face: {str(e)}") + return temp_frame def process_frame(source_face: Face, temp_frame: Frame) -> Frame: diff --git a/modules/ui.py b/modules/ui.py index 220eec4..dcce1c3 100644 --- a/modules/ui.py +++ b/modules/ui.py @@ -1,9 +1,10 @@ import os +import time import webbrowser import customtkinter as ctk from typing import Callable, Tuple -import cv2 -from PIL import Image, ImageOps +from PIL import Image, ImageOps, ImageDraw, ImageFont +import numpy as np import modules.globals import modules.metadata @@ -17,8 +18,8 @@ ROOT_HEIGHT = 700 ROOT_WIDTH = 600 PREVIEW = None -PREVIEW_MAX_HEIGHT = 700 -PREVIEW_MAX_WIDTH = 1200 +PREVIEW_MAX_HEIGHT = 720 +PREVIEW_MAX_WIDTH = 1280 RECENT_DIRECTORY_SOURCE = None RECENT_DIRECTORY_TARGET = None @@ -75,7 +76,6 @@ def create_root(start: Callable[[], None], destroy: Callable[[], None]) -> ctk.C keep_frames_switch = ctk.CTkSwitch(root, text='Keep frames', variable=keep_frames_value, cursor='hand2', command=lambda: setattr(modules.globals, 'keep_frames', keep_frames_value.get())) keep_frames_switch.place(relx=0.1, rely=0.65) - # for FRAME PROCESSOR ENHANCER tumbler: enhancer_value = ctk.BooleanVar(value=modules.globals.fp_ui['face_enhancer']) enhancer_switch = ctk.CTkSwitch(root, text='Face Enhancer', variable=enhancer_value, cursor='hand2', command=lambda: update_tumbler('face_enhancer',enhancer_value.get())) enhancer_switch.place(relx=0.1, rely=0.7) @@ -92,23 +92,36 @@ def create_root(start: Callable[[], None], destroy: Callable[[], None]) -> ctk.C nsfw_switch = ctk.CTkSwitch(root, text='NSFW', variable=nsfw_value, cursor='hand2', command=lambda: setattr(modules.globals, 'nsfw', nsfw_value.get())) nsfw_switch.place(relx=0.6, rely=0.7) + video_processor_label = ctk.CTkLabel(root, text="Video Processor:") + video_processor_label.place(relx=0.1, rely=0.75) + video_processor_var = ctk.StringVar(value=modules.globals.video_processor) + video_processor_menu = ctk.CTkOptionMenu(root, variable=video_processor_var, values=["cv2", "ffmpeg"], command=lambda choice: setattr(modules.globals, 'video_processor', choice)) + video_processor_menu.place(relx=0.3, rely=0.75) + + model_label = ctk.CTkLabel(root, text="Model:") + model_label.place(relx=0.1, rely=0.8) + model_var = ctk.StringVar(value=modules.globals.model) + model_entry = ctk.CTkEntry(root, textvariable=model_var) + model_entry.place(relx=0.3, rely=0.8, relwidth=0.4) + model_entry.bind("", lambda event: setattr(modules.globals, 'model', model_var.get())) + start_button = ctk.CTkButton(root, text='Start', cursor='hand2', command=lambda: select_output_path(start)) - start_button.place(relx=0.15, rely=0.80, relwidth=0.2, relheight=0.05) + start_button.place(relx=0.15, rely=0.85, relwidth=0.2, relheight=0.05) stop_button = ctk.CTkButton(root, text='Destroy', cursor='hand2', command=lambda: destroy()) - stop_button.place(relx=0.4, rely=0.80, relwidth=0.2, relheight=0.05) + stop_button.place(relx=0.4, rely=0.85, relwidth=0.2, relheight=0.05) preview_button = ctk.CTkButton(root, text='Preview', cursor='hand2', command=lambda: toggle_preview()) - preview_button.place(relx=0.65, rely=0.80, relwidth=0.2, relheight=0.05) + preview_button.place(relx=0.65, rely=0.85, relwidth=0.2, relheight=0.05) live_button = ctk.CTkButton(root, text='Live', cursor='hand2', command=lambda: webcam_preview()) - live_button.place(relx=0.40, rely=0.86, relwidth=0.2, relheight=0.05) + live_button.place(relx=0.40, rely=0.91, relwidth=0.2, relheight=0.05) status_label = ctk.CTkLabel(root, text=None, justify='center') - status_label.place(relx=0.1, rely=0.9, relwidth=0.8) + status_label.place(relx=0.1, rely=0.95, relwidth=0.8) donate_label = ctk.CTkLabel(root, text='Deep Live Cam', justify='center', cursor='hand2') - donate_label.place(relx=0.1, rely=0.95, relwidth=0.8) + donate_label.place(relx=0.1, rely=0.98, relwidth=0.8) donate_label.configure(text_color=ctk.ThemeManager.theme.get('URL').get('text_color')) donate_label.bind('