From f6814777704c90ea84dd5dacd22adb7bab926059 Mon Sep 17 00:00:00 2001 From: Spencer Perlmutter <64663042+SMPearly@users.noreply.github.com> Date: Tue, 30 May 2023 23:48:01 +0000 Subject: [PATCH 1/2] Initial commit From 7c310e622b9066ab3640b6934f41b04400adb927 Mon Sep 17 00:00:00 2001 From: Spencer Perlmutter <64663042+SMPearly@users.noreply.github.com> Date: Tue, 30 May 2023 23:48:04 +0000 Subject: [PATCH 2/2] Pending changes exported from your codespace --- quick_demo.ipynb | 394 +++++++++++++++++++++++++++++++++++++---------- video-retalking | 1 + 2 files changed, 315 insertions(+), 80 deletions(-) create mode 160000 video-retalking diff --git a/quick_demo.ipynb b/quick_demo.ipynb index 87ef8332..6f5b6661 100644 --- a/quick_demo.ipynb +++ b/quick_demo.ipynb @@ -1,28 +1,10 @@ { - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "provenance": [], - "authorship_tag": "ABX9TyMPin07iIA2oewCCP9ZTz6w", - "include_colab_link": true - }, - "kernelspec": { - "name": "python3", - "display_name": "Python 3" - }, - "language_info": { - "name": "python" - }, - "accelerator": "GPU", - "gpuClass": "standard" - }, "cells": [ { "cell_type": "markdown", "metadata": { - "id": "view-in-github", - "colab_type": "text" + "colab_type": "text", + "id": "view-in-github" }, "source": [ "\"Open" @@ -30,6 +12,9 @@ }, { "cell_type": "markdown", + "metadata": { + "id": "NVfkv2BXSpr3" + }, "source": [ "## VideoReTalking：Audio-based Lip Synchronization for Talking Head Video Editing In the Wild\n", "\n", @@ -41,27 +26,141 @@ "\n", "*SIGGRAPH Asia 2022 Conferenence Track*\n", "\n" - ], - "metadata": { - "id": "NVfkv2BXSpr3" - } + ] }, { "cell_type": "markdown", - "source": [ - "**Installation** (30s)" - ], "metadata": { "id": "u9hdPaH6UL_F" - } + }, + "source": [ + "**Installation** (30s)" + ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "metadata": { "id": "PnKT9goiQ3Hk" }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "/bin/bash: nvidia-smi: command not found\n", + "Python 3.10.4\n", + "Reading package lists... Done\n", + "E: List directory /var/lib/apt/lists/partial is missing. - Acquire (13: Permission denied)\n", + "Git clone project and install requirements...\n", + "/workspaces/video-retalking/video-retalking\n", + "Requirement already satisfied: basicsr==1.4.2 in /home/codespace/.local/lib/python3.10/site-packages (from -r requirements.txt (line 1)) (1.4.2)\n", + "Requirement already satisfied: kornia==0.5.1 in /home/codespace/.local/lib/python3.10/site-packages (from -r requirements.txt (line 2)) (0.5.1)\n", + "Requirement already satisfied: face-alignment==1.3.5 in /home/codespace/.local/lib/python3.10/site-packages (from -r requirements.txt (line 3)) (1.3.5)\n", + "Requirement already satisfied: ninja==1.10.2.3 in /home/codespace/.local/lib/python3.10/site-packages (from -r requirements.txt (line 4)) (1.10.2.3)\n", + "Requirement already satisfied: einops==0.4.1 in /home/codespace/.local/lib/python3.10/site-packages (from -r requirements.txt (line 5)) (0.4.1)\n", + "Requirement already satisfied: facexlib==0.2.5 in /home/codespace/.local/lib/python3.10/site-packages (from -r requirements.txt (line 6)) (0.2.5)\n", + "Requirement already satisfied: librosa==0.9.2 in /home/codespace/.local/lib/python3.10/site-packages (from -r requirements.txt (line 7)) (0.9.2)\n", + "Requirement already satisfied: dlib==19.24.0 in /home/codespace/.local/lib/python3.10/site-packages (from -r requirements.txt (line 8)) (19.24.0)\n", + "Requirement already satisfied: tqdm in /home/codespace/.local/lib/python3.10/site-packages (from basicsr==1.4.2->-r requirements.txt (line 1)) (4.65.0)\n", + "Requirement already satisfied: lmdb in /home/codespace/.local/lib/python3.10/site-packages (from basicsr==1.4.2->-r requirements.txt (line 1)) (1.4.1)\n", + "Requirement already satisfied: numpy>=1.17 in /home/codespace/.local/lib/python3.10/site-packages (from basicsr==1.4.2->-r requirements.txt (line 1)) (1.24.2)\n", + "Requirement already satisfied: future in /home/codespace/.local/lib/python3.10/site-packages (from basicsr==1.4.2->-r requirements.txt (line 1)) (0.18.3)\n", + "Requirement already satisfied: scipy in /home/codespace/.local/lib/python3.10/site-packages (from basicsr==1.4.2->-r requirements.txt (line 1)) (1.10.1)\n", + "Requirement already satisfied: requests in /home/codespace/.local/lib/python3.10/site-packages (from basicsr==1.4.2->-r requirements.txt (line 1)) (2.29.0)\n", + "Requirement already satisfied: scikit-image in /home/codespace/.local/lib/python3.10/site-packages (from basicsr==1.4.2->-r requirements.txt (line 1)) (0.20.0)\n", + "Requirement already satisfied: tb-nightly in /home/codespace/.local/lib/python3.10/site-packages (from basicsr==1.4.2->-r requirements.txt (line 1)) (2.13.0a20230426)\n", + "Requirement already satisfied: torchvision in /home/codespace/.local/lib/python3.10/site-packages (from basicsr==1.4.2->-r requirements.txt (line 1)) (0.15.1)\n", + "Requirement already satisfied: pyyaml in /home/codespace/.local/lib/python3.10/site-packages (from basicsr==1.4.2->-r requirements.txt (line 1)) (6.0)\n", + "Requirement already satisfied: torch>=1.7 in /home/codespace/.local/lib/python3.10/site-packages (from basicsr==1.4.2->-r requirements.txt (line 1)) (2.0.0)\n", + "Requirement already satisfied: yapf in /home/codespace/.local/lib/python3.10/site-packages (from basicsr==1.4.2->-r requirements.txt (line 1)) (0.33.0)\n", + "Requirement already satisfied: opencv-python in /home/codespace/.local/lib/python3.10/site-packages (from basicsr==1.4.2->-r requirements.txt (line 1)) (4.7.0.72)\n", + "Requirement already satisfied: Pillow in /home/codespace/.local/lib/python3.10/site-packages (from basicsr==1.4.2->-r requirements.txt (line 1)) (9.5.0)\n", + "Requirement already satisfied: addict in /home/codespace/.local/lib/python3.10/site-packages (from basicsr==1.4.2->-r requirements.txt (line 1)) (2.4.0)\n", + "Requirement already satisfied: numba in /home/codespace/.local/lib/python3.10/site-packages (from face-alignment==1.3.5->-r requirements.txt (line 3)) (0.56.4)\n", + "Requirement already satisfied: filterpy in /home/codespace/.local/lib/python3.10/site-packages (from facexlib==0.2.5->-r requirements.txt (line 6)) (1.4.5)\n", + "Requirement already satisfied: pooch>=1.0 in /home/codespace/.local/lib/python3.10/site-packages (from librosa==0.9.2->-r requirements.txt (line 7)) (1.7.0)\n", + "Requirement already satisfied: decorator>=4.0.10 in /home/codespace/.local/lib/python3.10/site-packages (from librosa==0.9.2->-r requirements.txt (line 7)) (5.1.1)\n", + "Requirement already satisfied: scikit-learn>=0.19.1 in /home/codespace/.local/lib/python3.10/site-packages (from librosa==0.9.2->-r requirements.txt (line 7)) (1.2.2)\n", + "Requirement already satisfied: soundfile>=0.10.2 in /home/codespace/.local/lib/python3.10/site-packages (from librosa==0.9.2->-r requirements.txt (line 7)) (0.12.1)\n", + "Requirement already satisfied: packaging>=20.0 in /home/codespace/.local/lib/python3.10/site-packages (from librosa==0.9.2->-r requirements.txt (line 7)) (23.1)\n", + "Requirement already satisfied: joblib>=0.14 in /home/codespace/.local/lib/python3.10/site-packages (from librosa==0.9.2->-r requirements.txt (line 7)) (1.2.0)\n", + "Requirement already satisfied: resampy>=0.2.2 in /home/codespace/.local/lib/python3.10/site-packages (from librosa==0.9.2->-r requirements.txt (line 7)) (0.4.2)\n", + "Requirement already satisfied: audioread>=2.1.9 in /home/codespace/.local/lib/python3.10/site-packages (from librosa==0.9.2->-r requirements.txt (line 7)) (3.0.0)\n", + "Requirement already satisfied: setuptools in /home/codespace/.local/lib/python3.10/site-packages (from numba->face-alignment==1.3.5->-r requirements.txt (line 3)) (67.7.2)\n", + "Collecting numpy>=1.17\n", + " Downloading numpy-1.23.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (17.1 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m17.1/17.1 MB\u001b[0m \u001b[31m47.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: llvmlite<0.40,>=0.39.0dev0 in /home/codespace/.local/lib/python3.10/site-packages (from numba->face-alignment==1.3.5->-r requirements.txt (line 3)) (0.39.1)\n", + "Requirement already satisfied: platformdirs>=2.5.0 in /home/codespace/.local/lib/python3.10/site-packages (from pooch>=1.0->librosa==0.9.2->-r requirements.txt (line 7)) (3.5.0)\n", + "Requirement already satisfied: charset-normalizer<4,>=2 in /home/codespace/.local/lib/python3.10/site-packages (from requests->basicsr==1.4.2->-r requirements.txt (line 1)) (3.1.0)\n", + "Requirement already satisfied: certifi>=2017.4.17 in /home/codespace/.local/lib/python3.10/site-packages (from requests->basicsr==1.4.2->-r requirements.txt (line 1)) (2022.12.7)\n", + "Requirement already satisfied: idna<4,>=2.5 in /home/codespace/.local/lib/python3.10/site-packages (from requests->basicsr==1.4.2->-r requirements.txt (line 1)) (3.4)\n", + "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /home/codespace/.local/lib/python3.10/site-packages (from requests->basicsr==1.4.2->-r requirements.txt (line 1)) (1.26.15)\n", + "Requirement already satisfied: threadpoolctl>=2.0.0 in /home/codespace/.local/lib/python3.10/site-packages (from scikit-learn>=0.19.1->librosa==0.9.2->-r requirements.txt (line 7)) (3.1.0)\n", + "Requirement already satisfied: cffi>=1.0 in /home/codespace/.local/lib/python3.10/site-packages (from soundfile>=0.10.2->librosa==0.9.2->-r requirements.txt (line 7)) (1.15.1)\n", + "Requirement already satisfied: nvidia-curand-cu11==10.2.10.91 in /home/codespace/.local/lib/python3.10/site-packages (from torch>=1.7->basicsr==1.4.2->-r requirements.txt (line 1)) (10.2.10.91)\n", + "Requirement already satisfied: nvidia-nvtx-cu11==11.7.91 in /home/codespace/.local/lib/python3.10/site-packages (from torch>=1.7->basicsr==1.4.2->-r requirements.txt (line 1)) (11.7.91)\n", + "Requirement already satisfied: nvidia-cudnn-cu11==8.5.0.96 in /home/codespace/.local/lib/python3.10/site-packages (from torch>=1.7->basicsr==1.4.2->-r requirements.txt (line 1)) (8.5.0.96)\n", + "Requirement already satisfied: nvidia-cublas-cu11==11.10.3.66 in /home/codespace/.local/lib/python3.10/site-packages (from torch>=1.7->basicsr==1.4.2->-r requirements.txt (line 1)) (11.10.3.66)\n", + "Requirement already satisfied: typing-extensions in /home/codespace/.local/lib/python3.10/site-packages (from torch>=1.7->basicsr==1.4.2->-r requirements.txt (line 1)) (4.5.0)\n", + "Requirement already satisfied: nvidia-cuda-runtime-cu11==11.7.99 in /home/codespace/.local/lib/python3.10/site-packages (from torch>=1.7->basicsr==1.4.2->-r requirements.txt (line 1)) (11.7.99)\n", + "Requirement already satisfied: nvidia-nccl-cu11==2.14.3 in /home/codespace/.local/lib/python3.10/site-packages (from torch>=1.7->basicsr==1.4.2->-r requirements.txt (line 1)) (2.14.3)\n", + "Requirement already satisfied: sympy in /home/codespace/.local/lib/python3.10/site-packages (from torch>=1.7->basicsr==1.4.2->-r requirements.txt (line 1)) (1.11.1)\n", + "Requirement already satisfied: nvidia-cuda-nvrtc-cu11==11.7.99 in /home/codespace/.local/lib/python3.10/site-packages (from torch>=1.7->basicsr==1.4.2->-r requirements.txt (line 1)) (11.7.99)\n", + "Requirement already satisfied: jinja2 in /home/codespace/.local/lib/python3.10/site-packages (from torch>=1.7->basicsr==1.4.2->-r requirements.txt (line 1)) (3.1.2)\n", + "Requirement already satisfied: nvidia-cusparse-cu11==11.7.4.91 in /home/codespace/.local/lib/python3.10/site-packages (from torch>=1.7->basicsr==1.4.2->-r requirements.txt (line 1)) (11.7.4.91)\n", + "Requirement already satisfied: triton==2.0.0 in /home/codespace/.local/lib/python3.10/site-packages (from torch>=1.7->basicsr==1.4.2->-r requirements.txt (line 1)) (2.0.0)\n", + "Requirement already satisfied: nvidia-cuda-cupti-cu11==11.7.101 in /home/codespace/.local/lib/python3.10/site-packages (from torch>=1.7->basicsr==1.4.2->-r requirements.txt (line 1)) (11.7.101)\n", + "Requirement already satisfied: networkx in /home/codespace/.local/lib/python3.10/site-packages (from torch>=1.7->basicsr==1.4.2->-r requirements.txt (line 1)) (3.1)\n", + "Requirement already satisfied: nvidia-cusolver-cu11==11.4.0.1 in /home/codespace/.local/lib/python3.10/site-packages (from torch>=1.7->basicsr==1.4.2->-r requirements.txt (line 1)) (11.4.0.1)\n", + "Requirement already satisfied: filelock in /home/codespace/.local/lib/python3.10/site-packages (from torch>=1.7->basicsr==1.4.2->-r requirements.txt (line 1)) (3.12.0)\n", + "Requirement already satisfied: nvidia-cufft-cu11==10.9.0.58 in /home/codespace/.local/lib/python3.10/site-packages (from torch>=1.7->basicsr==1.4.2->-r requirements.txt (line 1)) (10.9.0.58)\n", + "Requirement already satisfied: wheel in /home/codespace/.local/lib/python3.10/site-packages (from nvidia-cublas-cu11==11.10.3.66->torch>=1.7->basicsr==1.4.2->-r requirements.txt (line 1)) (0.40.0)\n", + "Requirement already satisfied: lit in /home/codespace/.local/lib/python3.10/site-packages (from triton==2.0.0->torch>=1.7->basicsr==1.4.2->-r requirements.txt (line 1)) (16.0.0)\n", + "Requirement already satisfied: cmake in /home/codespace/.local/lib/python3.10/site-packages (from triton==2.0.0->torch>=1.7->basicsr==1.4.2->-r requirements.txt (line 1)) (3.26.3)\n", + "Requirement already satisfied: matplotlib in /home/codespace/.local/lib/python3.10/site-packages (from filterpy->facexlib==0.2.5->-r requirements.txt (line 6)) (3.7.1)\n", + "Requirement already satisfied: imageio>=2.4.1 in /home/codespace/.local/lib/python3.10/site-packages (from scikit-image->basicsr==1.4.2->-r requirements.txt (line 1)) (2.28.0)\n", + "Requirement already satisfied: tifffile>=2019.7.26 in /home/codespace/.local/lib/python3.10/site-packages (from scikit-image->basicsr==1.4.2->-r requirements.txt (line 1)) (2023.4.12)\n", + "Requirement already satisfied: lazy_loader>=0.1 in /home/codespace/.local/lib/python3.10/site-packages (from scikit-image->basicsr==1.4.2->-r requirements.txt (line 1)) (0.2)\n", + "Requirement already satisfied: PyWavelets>=1.1.1 in /home/codespace/.local/lib/python3.10/site-packages (from scikit-image->basicsr==1.4.2->-r requirements.txt (line 1)) (1.4.1)\n", + "Requirement already satisfied: werkzeug>=1.0.1 in /home/codespace/.local/lib/python3.10/site-packages (from tb-nightly->basicsr==1.4.2->-r requirements.txt (line 1)) (2.3.1)\n", + "Requirement already satisfied: grpcio>=1.48.2 in /home/codespace/.local/lib/python3.10/site-packages (from tb-nightly->basicsr==1.4.2->-r requirements.txt (line 1)) (1.54.0)\n", + "Requirement already satisfied: tensorboard-data-server<0.8.0,>=0.7.0 in /home/codespace/.local/lib/python3.10/site-packages (from tb-nightly->basicsr==1.4.2->-r requirements.txt (line 1)) (0.7.0)\n", + "Requirement already satisfied: google-auth-oauthlib<1.1,>=0.5 in /home/codespace/.local/lib/python3.10/site-packages (from tb-nightly->basicsr==1.4.2->-r requirements.txt (line 1)) (1.0.0)\n", + "Requirement already satisfied: google-auth<3,>=1.6.3 in /home/codespace/.local/lib/python3.10/site-packages (from tb-nightly->basicsr==1.4.2->-r requirements.txt (line 1)) (2.17.3)\n", + "Requirement already satisfied: protobuf>=3.19.6 in /home/codespace/.local/lib/python3.10/site-packages (from tb-nightly->basicsr==1.4.2->-r requirements.txt (line 1)) (4.22.3)\n", + "Requirement already satisfied: absl-py>=0.4 in /home/codespace/.local/lib/python3.10/site-packages (from tb-nightly->basicsr==1.4.2->-r requirements.txt (line 1)) (1.4.0)\n", + "Requirement already satisfied: markdown>=2.6.8 in /home/codespace/.local/lib/python3.10/site-packages (from tb-nightly->basicsr==1.4.2->-r requirements.txt (line 1)) (3.4.3)\n", + "Requirement already satisfied: tomli>=2.0.1 in /home/codespace/.local/lib/python3.10/site-packages (from yapf->basicsr==1.4.2->-r requirements.txt (line 1)) (2.0.1)\n", + "Requirement already satisfied: pycparser in /home/codespace/.local/lib/python3.10/site-packages (from cffi>=1.0->soundfile>=0.10.2->librosa==0.9.2->-r requirements.txt (line 7)) (2.21)\n", + "Requirement already satisfied: pyasn1-modules>=0.2.1 in /home/codespace/.local/lib/python3.10/site-packages (from google-auth<3,>=1.6.3->tb-nightly->basicsr==1.4.2->-r requirements.txt (line 1)) (0.3.0)\n", + "Requirement already satisfied: cachetools<6.0,>=2.0.0 in /home/codespace/.local/lib/python3.10/site-packages (from google-auth<3,>=1.6.3->tb-nightly->basicsr==1.4.2->-r requirements.txt (line 1)) (5.3.0)\n", + "Requirement already satisfied: six>=1.9.0 in /home/codespace/.local/lib/python3.10/site-packages (from google-auth<3,>=1.6.3->tb-nightly->basicsr==1.4.2->-r requirements.txt (line 1)) (1.16.0)\n", + "Requirement already satisfied: rsa<5,>=3.1.4 in /home/codespace/.local/lib/python3.10/site-packages (from google-auth<3,>=1.6.3->tb-nightly->basicsr==1.4.2->-r requirements.txt (line 1)) (4.9)\n", + "Requirement already satisfied: requests-oauthlib>=0.7.0 in /home/codespace/.local/lib/python3.10/site-packages (from google-auth-oauthlib<1.1,>=0.5->tb-nightly->basicsr==1.4.2->-r requirements.txt (line 1)) (1.3.1)\n", + "Requirement already satisfied: MarkupSafe>=2.1.1 in /home/codespace/.local/lib/python3.10/site-packages (from werkzeug>=1.0.1->tb-nightly->basicsr==1.4.2->-r requirements.txt (line 1)) (2.1.2)\n", + "Requirement already satisfied: pyparsing>=2.3.1 in /home/codespace/.local/lib/python3.10/site-packages (from matplotlib->filterpy->facexlib==0.2.5->-r requirements.txt (line 6)) (3.0.9)\n", + "Requirement already satisfied: cycler>=0.10 in /home/codespace/.local/lib/python3.10/site-packages (from matplotlib->filterpy->facexlib==0.2.5->-r requirements.txt (line 6)) (0.11.0)\n", + "Requirement already satisfied: fonttools>=4.22.0 in /home/codespace/.local/lib/python3.10/site-packages (from matplotlib->filterpy->facexlib==0.2.5->-r requirements.txt (line 6)) (4.39.3)\n", + "Requirement already satisfied: contourpy>=1.0.1 in /home/codespace/.local/lib/python3.10/site-packages (from matplotlib->filterpy->facexlib==0.2.5->-r requirements.txt (line 6)) (1.0.7)\n", + "Requirement already satisfied: kiwisolver>=1.0.1 in /home/codespace/.local/lib/python3.10/site-packages (from matplotlib->filterpy->facexlib==0.2.5->-r requirements.txt (line 6)) (1.4.4)\n", + "Requirement already satisfied: python-dateutil>=2.7 in /home/codespace/.local/lib/python3.10/site-packages (from matplotlib->filterpy->facexlib==0.2.5->-r requirements.txt (line 6)) (2.8.2)\n", + "Requirement already satisfied: mpmath>=0.19 in /home/codespace/.local/lib/python3.10/site-packages (from sympy->torch>=1.7->basicsr==1.4.2->-r requirements.txt (line 1)) (1.3.0)\n", + "Requirement already satisfied: pyasn1<0.6.0,>=0.4.6 in /home/codespace/.local/lib/python3.10/site-packages (from pyasn1-modules>=0.2.1->google-auth<3,>=1.6.3->tb-nightly->basicsr==1.4.2->-r requirements.txt (line 1)) (0.5.0)\n", + "Requirement already satisfied: oauthlib>=3.0.0 in /home/codespace/.local/lib/python3.10/site-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<1.1,>=0.5->tb-nightly->basicsr==1.4.2->-r requirements.txt (line 1)) (3.2.2)\n", + "Installing collected packages: numpy\n", + " Attempting uninstall: numpy\n", + " Found existing installation: numpy 1.24.2\n", + " Uninstalling numpy-1.24.2:\n", + " Successfully uninstalled numpy-1.24.2\n", + "Successfully installed numpy-1.23.5\n", + "\n", + "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.0.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m23.1.2\u001b[0m\n", + "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpython -m pip install --upgrade pip\u001b[0m\n" + ] + } + ], "source": [ "#@title\n", "### make sure that CUDA is available in Edit -> Nootbook settings -> GPU\n", @@ -80,15 +179,128 @@ }, { "cell_type": "markdown", - "source": [ - "**Download Pretrained Models**" - ], "metadata": { "id": "uwJS0eaM61Cq" - } + }, + "source": [ + "**Download Pretrained Models**" + ] }, { "cell_type": "code", + "execution_count": 2, + "metadata": { + "id": "x18qYuQY678E" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Download pre-trained models...\n", + "--2023-04-28 01:50:30-- https://github.com/vinthony/video-retalking/releases/download/v0.0.1/30_net_gen.pth\n", + "Resolving github.com (github.com)... 140.82.114.3\n", + "Connecting to github.com (github.com)|140.82.114.3|:443... connected.\n", + "HTTP request sent, awaiting response... 404 Not Found\n", + "2023-04-28 01:50:30 ERROR 404: Not Found.\n", + "\n", + "--2023-04-28 01:50:31-- https://github.com/vinthony/video-retalking/releases/download/v0.0.1/BFM.zip\n", + "Resolving github.com (github.com)... 140.82.114.3\n", + "Connecting to github.com (github.com)|140.82.114.3|:443... connected.\n", + "HTTP request sent, awaiting response... 404 Not Found\n", + "2023-04-28 01:50:31 ERROR 404: Not Found.\n", + "\n", + "--2023-04-28 01:50:32-- https://github.com/vinthony/video-retalking/releases/download/v0.0.1/DNet.pt\n", + "Resolving github.com (github.com)... 140.82.114.3\n", + "Connecting to github.com (github.com)|140.82.114.3|:443... connected.\n", + "HTTP request sent, awaiting response... 404 Not Found\n", + "2023-04-28 01:50:32 ERROR 404: Not Found.\n", + "\n", + "--2023-04-28 01:50:33-- https://github.com/vinthony/video-retalking/releases/download/v0.0.1/ENet.pth\n", + "Resolving github.com (github.com)... 140.82.114.3\n", + "Connecting to github.com (github.com)|140.82.114.3|:443... connected.\n", + "HTTP request sent, awaiting response... 404 Not Found\n", + "2023-04-28 01:50:33 ERROR 404: Not Found.\n", + "\n", + "--2023-04-28 01:50:33-- https://github.com/vinthony/video-retalking/releases/download/v0.0.1/expression.mat\n", + "Resolving github.com (github.com)... 140.82.114.3\n", + "Connecting to github.com (github.com)|140.82.114.3|:443... connected.\n", + "HTTP request sent, awaiting response... 404 Not Found\n", + "2023-04-28 01:50:33 ERROR 404: Not Found.\n", + "\n", + "--2023-04-28 01:50:34-- https://github.com/vinthony/video-retalking/releases/download/v0.0.1/face3d_pretrain_epoch_20.pth\n", + "Resolving github.com (github.com)... 140.82.114.3\n", + "Connecting to github.com (github.com)|140.82.114.3|:443... connected.\n", + "HTTP request sent, awaiting response... 404 Not Found\n", + "2023-04-28 01:50:34 ERROR 404: Not Found.\n", + "\n", + "--2023-04-28 01:50:35-- https://github.com/vinthony/video-retalking/releases/download/v0.0.1/GFPGANv1.3.pth\n", + "Resolving github.com (github.com)... 140.82.114.3\n", + "Connecting to github.com (github.com)|140.82.114.3|:443... connected.\n", + "HTTP request sent, awaiting response... 404 Not Found\n", + "2023-04-28 01:50:35 ERROR 404: Not Found.\n", + "\n", + "--2023-04-28 01:50:36-- https://github.com/vinthony/video-retalking/releases/download/v0.0.1/GPEN-BFR-512.pth\n", + "Resolving github.com (github.com)... 140.82.114.3\n", + "Connecting to github.com (github.com)|140.82.114.3|:443... connected.\n", + "HTTP request sent, awaiting response... 404 Not Found\n", + "2023-04-28 01:50:36 ERROR 404: Not Found.\n", + "\n", + "--2023-04-28 01:50:37-- https://github.com/vinthony/video-retalking/releases/download/v0.0.1/LNet.pth\n", + "Resolving github.com (github.com)... 140.82.114.3\n", + "Connecting to github.com (github.com)|140.82.114.3|:443... connected.\n", + "HTTP request sent, awaiting response... 404 Not Found\n", + "2023-04-28 01:50:37 ERROR 404: Not Found.\n", + "\n", + "--2023-04-28 01:50:37-- https://github.com/vinthony/video-retalking/releases/download/v0.0.1/ParseNet-latest.pth\n", + "Resolving github.com (github.com)... 140.82.114.3\n", + "Connecting to github.com (github.com)|140.82.114.3|:443... connected.\n", + "HTTP request sent, awaiting response... 404 Not Found\n", + "2023-04-28 01:50:37 ERROR 404: Not Found.\n", + "\n", + "--2023-04-28 01:50:38-- https://github.com/vinthony/video-retalking/releases/download/v0.0.1/RetinaFace-R50.pth\n", + "Resolving github.com (github.com)... 140.82.114.3\n", + "Connecting to github.com (github.com)|140.82.114.3|:443... connected.\n", + "HTTP request sent, awaiting response... 404 Not Found\n", + "2023-04-28 01:50:38 ERROR 404: Not Found.\n", + "\n", + "--2023-04-28 01:50:39-- https://github.com/vinthony/video-retalking/releases/download/v0.0.1/shape_predictor_68_face_landmarks.dat\n", + "Resolving github.com (github.com)... 140.82.114.3\n", + "Connecting to github.com (github.com)|140.82.114.3|:443... connected.\n", + "HTTP request sent, awaiting response... 404 Not Found\n", + "2023-04-28 01:50:39 ERROR 404: Not Found.\n", + "\n", + "Archive: ./checkpoints/BFM.zip\n", + " End-of-central-directory signature not found. Either this file is not\n", + " a zipfile, or it constitutes one disk of a multi-part archive. In the\n", + " latter case the central directory and zipfile comment will be found on\n", + " the last disk(s) of this archive.\n", + "unzip: cannot find zipfile directory in one of ./checkpoints/BFM.zip or\n", + " ./checkpoints/BFM.zip.zip, and cannot find ./checkpoints/BFM.zip.ZIP, period.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Bad pipe message: %s [b'LIFa\\xa5\\xfdW;H4\\xf2\\xd9>9\\xfe(\\x0b\\x88 \\xf4\\xf6@\\xd9u?\\xd3\\x07{A\\xc4\\xd7\\xb6\\xdb\\x8f\\xe5\\xcb\\xc3\\xf72\\xf3ep\\xcc\\xe6.Z\\xdd\\xe6)\\xa5\\xc1\\x00\\x08\\x13\\x02\\x13\\x03\\x13\\x01\\x00\\xff\\x01\\x00\\x00\\x8f\\x00\\x00\\x00\\x0e\\x00\\x0c\\x00\\x00\\t127.0.0.1\\x00\\x0b\\x00\\x04\\x03\\x00\\x01\\x02\\x00\\n\\x00\\x0c\\x00\\n\\x00\\x1d\\x00\\x17\\x00\\x1e\\x00\\x19\\x00\\x18\\x00#\\x00\\x00\\x00\\x16\\x00\\x00\\x00\\x17\\x00\\x00\\x00\\r\\x00\\x1e\\x00\\x1c\\x04\\x03\\x05\\x03\\x06\\x03\\x08\\x07\\x08\\x08\\x08\\t\\x08\\n\\x08\\x0b\\x08\\x04\\x08\\x05\\x08\\x06\\x04\\x01\\x05\\x01\\x06\\x01\\x00+\\x00\\x03\\x02\\x03\\x04\\x00-\\x00\\x02\\x01\\x01\\x003\\x00&\\x00$\\x00\\x1d\\x00 Pd\\x04\\xe6_!\\xd8\\x07\\xc3ebrn\\x90\\xeb\\x00Y3bS\\xbeWb\\x1cQ\\x17']\n", + "Bad pipe message: %s [b\"\\xa7\\xdd\\x91b\\x9e\\x9f\\xa6.\\x96\\xb69\\x1f6CN\\xc7\\xa8\\xa0\\x00\\x00|\\xc0,\\xc00\\x00\\xa3\\x00\\x9f\\xcc\\xa9\\xcc\\xa8\\xcc\\xaa\\xc0\\xaf\\xc0\\xad\\xc0\\xa3\\xc0\\x9f\\xc0]\\xc0a\\xc0W\\xc0S\\xc0+\\xc0/\\x00\\xa2\\x00\\x9e\\xc0\\xae\\xc0\\xac\\xc0\\xa2\\xc0\\x9e\\xc0\\\\\\xc0`\\xc0V\\xc0R\\xc0$\\xc0(\\x00k\\x00j\\xc0#\\xc0'\\x00g\\x00@\\xc0\\n\\xc0\\x14\\x009\\x008\\xc0\\t\\xc0\\x13\\x003\\x002\\x00\\x9d\\xc0\\xa1\\xc0\\x9d\\xc0Q\\x00\\x9c\\xc0\\xa0\\xc0\\x9c\\xc0P\\x00=\\x00<\\x005\\x00\"]\n", + "Bad pipe message: %s [b'\\x9a\\x00\\x99\\xc0\\x07\\xc0\\x11\\x00\\x96\\x00\\x05\\x00\\xff\\x01\\x00\\x00j\\x00\\x00\\x00\\x0e\\x00\\x0c\\x00\\x00\\t127.0.0.1\\x00\\x0b\\x00\\x04\\x03\\x00\\x01\\x02\\x00\\n\\x00']\n", + "Bad pipe message: %s [b'\\x02u\\x04ja\\xf7Y|^hu\\xf5\\xa6\\\\jTj?\\x00\\x00\\xa6\\xc0,\\xc00\\x00\\xa3\\x00\\x9f\\xcc\\xa9\\xcc\\xa8\\xcc\\xaa\\xc0\\xaf\\xc0\\xad\\xc0\\xa3\\xc0\\x9f']\n", + "Bad pipe message: %s [b'\\n\\x00\\x1d\\x00\\x17\\x00\\x1e\\x00\\x19\\x00\\x18']\n", + "Bad pipe message: %s [b'\\xe8F\\xcbB\\xa5V', b'\\x9aQ4\\xdf9,\\xd9\\xa0\\x00\\xe4\\x00\\x00>\\xc0\\x14\\xc0\\n\\x009\\x008\\x007\\x006\\xc0\\x0f\\xc0\\x05\\x005\\xc0\\x13\\xc0\\t\\x003\\x002\\x001\\x000\\xc0\\x0e\\xc0\\x04\\x00/\\x00\\x9a\\x00\\x99\\x00\\x98\\x00\\x97\\x00\\x96\\x00\\x07\\xc0\\x11\\xc0\\x07\\xc0\\x0c\\xc0\\x02\\x00\\x05\\x00\\x04\\x00\\xff\\x02\\x01\\x00\\x00C\\x00\\x00\\x00', b'\\x0c\\x00\\x00\\t127.0.0.1']\n", + "Bad pipe message: %s [b'l\\n\\xc8ZBU)\\xb4\\xd6c;\\xc4\\xaf\\\\\\xa2\\xd8S_\\x00\\x00>\\xc0', b'\\n\\x009\\x008\\x007\\x006\\xc0\\x0f\\xc0\\x05\\x005\\xc0\\x13\\xc0\\t']\n", + "Bad pipe message: %s [b'FR\\x9a)\\xa2\\x0e\\xe0\\xfa\\xe7\\xe2BR\\x88\\xf4\\x06\\x9d\\x1f<\\x00']\n", + "Bad pipe message: %s [b'6\\xd5&\\x97\\xbd\\xd84,\\x1d\\x003\\x002\\x001\\x000\\xc01\\xc0-\\xc0)\\xc0%\\xc0\\x0e\\xc0\\x04\\x00\\x9c\\x00<\\x00/\\x00\\x9a\\x00\\x99\\x00\\x98\\x00\\x97\\x00\\x96\\x00\\x07\\xc0\\x11\\xc0\\x07\\xc0\\x0c\\xc0\\x02\\x00\\x05\\x00\\x04\\x00\\xff\\x02\\x01\\x00\\x00g\\x00\\x00\\x00\\x0e\\x00\\x0c\\x00\\x00\\t127.0.0.1\\x00\\x0b\\x00\\x04\\x03\\x00\\x01\\x02\\x00\\n\\x00\\x1c\\x00\\x1a\\x00\\x17\\x00\\x19\\x00\\x1c\\x00\\x1b\\x00\\x18\\x00\\x1a\\x00\\x16\\x00\\x0e\\x00\\r\\x00\\x0b\\x00\\x0c\\x00\\t\\x00\\n\\x00#\\x00\\x00\\x00\\r\\x00 \\x00\\x1e\\x06\\x01\\x06\\x02\\x06\\x03\\x05\\x01\\x05\\x02\\x05\\x03\\x04\"]\n", + "Bad pipe message: %s [b'']\n", + "Bad pipe message: %s [b'\\x03']\n", + "Bad pipe message: %s [b'\\x03\\x02', b'\\x02\\x01', b'\\x02']\n", + "Bad pipe message: %s [b'\\x0f\\x00']\n", + "Bad pipe message: %s [b\"\\x99\\x07\\xc8\\x0e\\xe7\\xd6\\x81\\x97\\x9e\\x93\\x83\\xc4v\\x8fX\\xe8Z\\x80\\x00\\x00\\xf4\\xc00\\xc0,\\xc0(\\xc0$\\xc0\\x14\\xc0\\n\\x00\\xa5\\x00\\xa3\\x00\\xa1\\x00\\x9f\\x00k\\x00j\\x00i\\x00h\\x009\\x008\\x007\\x006\\x00\\x88\\x00\\x87\\x00\\x86\\x00\\x85\\xc0\\x19\\x00\\xa7\\x00m\\x00:\\x00\\x89\\xc02\\xc0.\\xc0*\\xc0&\\xc0\\x0f\\xc0\\x05\\x00\\x9d\\x00=\\x005\\x00\\x84\\xc0/\\xc0+\\xc0'\\xc0#\\xc0\\x13\\xc0\\t\\x00\\xa4\\x00\\xa2\\x00\\xa0\\x00\\x9e\\x00g\\x00@\\x00?\\x00>\\x003\\x002\"]\n" + ] + } + ], "source": [ "#@title\n", "print('Download pre-trained models...')\n", @@ -106,15 +318,13 @@ "!wget https://github.com/vinthony/video-retalking/releases/download/v0.0.1/RetinaFace-R50.pth -O ./checkpoints/RetinaFace-R50.pth\n", "!wget https://github.com/vinthony/video-retalking/releases/download/v0.0.1/shape_predictor_68_face_landmarks.dat -O ./checkpoints/shape_predictor_68_face_landmarks.dat\n", "!unzip -d ./checkpoints/BFM ./checkpoints/BFM.zip" - ], - "metadata": { - "id": "x18qYuQY678E" - }, - "execution_count": null, - "outputs": [] + ] }, { "cell_type": "markdown", + "metadata": { + "id": "QJRTF4U8UOjv" + }, "source": [ "**Inference**\n", "\n", @@ -127,13 +337,16 @@ "\n", "\n", "\n" - ], - "metadata": { - "id": "QJRTF4U8UOjv" - } + ] }, { "cell_type": "code", + "execution_count": null, + "metadata": { + "cellView": "form", + "id": "U-IY-cBSporP" + }, + "outputs": [], "source": [ "#@title\n", "import glob, os, sys\n", @@ -152,25 +365,25 @@ "audio_list.sort()\n", "default_audio_name = widgets.Dropdown(options=audio_list, value='1.wav')\n", "display(default_audio_name)\n" - ], - "metadata": { - "id": "U-IY-cBSporP", - "cellView": "form" - }, - "execution_count": null, - "outputs": [] + ] }, { "cell_type": "markdown", - "source": [ - "Visualize the input video and audio:" - ], "metadata": { "id": "-MtI_R1bLJ-f" - } + }, + "source": [ + "Visualize the input video and audio:" + ] }, { "cell_type": "code", + "execution_count": null, + "metadata": { + "cellView": "form", + "id": "ljbScdofJyGO" + }, + "outputs": [], "source": [ "#@title\n", "input_video_name = './examples/face/{}'.format(default_vid_name.value)\n", @@ -192,16 +405,15 @@ " \n", " \n", " \"\"\" % input_audio_data_url))\n" - ], - "metadata": { - "id": "ljbScdofJyGO", - "cellView": "form" - }, - "execution_count": null, - "outputs": [] + ] }, { "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "D7hUwRCyUYEA" + }, + "outputs": [], "source": [ "input_video_path = 'examples/face/{}'.format(default_vid_name.value)\n", "input_audio_path = 'examples/audio/{}'.format(default_audio_name.value)\n", @@ -210,24 +422,25 @@ " --face {input_video_path} \\\n", " --audio {input_audio_path} \\\n", " --outfile results/output.mp4" - ], - "metadata": { - "id": "D7hUwRCyUYEA" - }, - "execution_count": null, - "outputs": [] + ] }, { "cell_type": "markdown", - "source": [ - "Visualize the output video:" - ], "metadata": { "id": "JB5RbKc-njkB" - } + }, + "source": [ + "Visualize the output video:" + ] }, { "cell_type": "code", + "execution_count": null, + "metadata": { + "cellView": "form", + "id": "ravs9UDucMfy" + }, + "outputs": [], "source": [ "#@title\n", "# visualize code from makeittalk\n", @@ -281,13 +494,34 @@ " \n", " \n", " \"\"\" % output_concat_video_data_url))\n" - ], - "metadata": { - "id": "ravs9UDucMfy", - "cellView": "form" + ] + } + ], + "metadata": { + "accelerator": "GPU", + "colab": { + "authorship_tag": "ABX9TyMPin07iIA2oewCCP9ZTz6w", + "include_colab_link": true, + "provenance": [] + }, + "gpuClass": "standard", + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 }, - "execution_count": null, - "outputs": [] + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.4" } - ] -} \ No newline at end of file + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/video-retalking b/video-retalking new file mode 160000 index 00000000..1c13c4cb --- /dev/null +++ b/video-retalking @@ -0,0 +1 @@ +Subproject commit 1c13c4cb4bb73c1a85b8b6cad86aeff91b21b094