pix2video.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "private_outputs": true,
      "provenance": [],
      "authorship_tag": "ABX9TyO3+YCCbMgNlH3iCCUz50aY",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU",
    "gpuClass": "standard"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/albertotrunk/depth2video/blob/main/pix2video.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install opencv-python\n",
        "!pip install ffmpeg-python\n",
        "!pip install torch --extra-index-url https://download.pytorch.org/whl/cu113 torchvision==0.13.1+cu113\n",
        "!pip install diffusers\n",
        "!pip install transformers\n",
        "!pip install scipy\n",
        "!pip install ftfy\n",
        "!pip install psutil\n",
        "!pip install accelerate\n",
        "!pip install !pip install safetensors\n",
        "!pip install transformers\n",
        "!pip install safetensors\n",
        "!pip install xformers --pre\n",
        "!pip install moviepy\n",
        "!pip install imageio==2.4.1\n",
        "!pip install gradio"
      ],
      "metadata": {
        "id": "5oVoTu2ITKRk"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "6ATclF0vU715"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!git clone https://github.com/huggingface/diffusers.git\n",
        "!git clone https://github.com/huggingface/transformers"
      ],
      "metadata": {
        "id": "05r5vhOLUuM8"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "ViuvCJ8SVEZ3"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ZupXs0YTS8Py"
      },
      "outputs": [],
      "source": [
        "loading_icon_html = \"\"\"<svg id=\"share-btn-loading-icon\" style=\"display:none;\" class=\"animate-spin\"\n",
        "   style=\"color: #ffffff; \n",
        "\"\n",
        "   xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\" aria-hidden=\"true\" fill=\"none\" focusable=\"false\" role=\"img\" width=\"1em\" height=\"1em\" preserveAspectRatio=\"xMidYMid meet\" viewBox=\"0 0 24 24\"><circle style=\"opacity: 0.25;\" cx=\"12\" cy=\"12\" r=\"10\" stroke=\"white\" stroke-width=\"4\"></circle><path style=\"opacity: 0.75;\" fill=\"white\" d=\"M4 12a8 8 0 018-8V0C5.373 0 0 5.373 0 12h4zm2 5.291A7.962 7.962 0 014 12H0c0 3.042 1.135 5.824 3 7.938l3-2.647z\"></path></svg>\"\"\"\n",
        "\n",
        "\n",
        "\n",
        "import gradio as gr\n",
        "import os\n",
        "import cv2\n",
        "import numpy as np\n",
        "from moviepy.editor import *\n",
        "\n",
        "from diffusers import DiffusionPipeline, EulerAncestralDiscreteScheduler\n",
        "import torch\n",
        "from PIL import Image\n",
        "import time\n",
        "import psutil\n",
        "import random\n",
        "\n",
        "\n",
        "pipe = DiffusionPipeline.from_pretrained(\"timbrooks/instruct-pix2pix\", torch_dtype=torch.float16, safety_checker=None)\n",
        "pipe.scheduler = EulerAncestralDiscreteScheduler.from_config(pipe.scheduler.config)\n",
        "pipe.enable_xformers_memory_efficient_attention()\n",
        "pipe.unet.to(memory_format=torch.channels_last)\n",
        "\n",
        "device = \"GPU 🔥\" if torch.cuda.is_available() else \"CPU 🥶\"\n",
        "\n",
        "if torch.cuda.is_available():\n",
        "    pipe = pipe.to(\"cuda\")\n",
        "\n",
        "def pix2pix(\n",
        "    prompt,\n",
        "    text_guidance_scale,\n",
        "    image_guidance_scale,\n",
        "    image,\n",
        "    steps,\n",
        "    neg_prompt=\"\",\n",
        "    width=512,\n",
        "    height=512,\n",
        "    seed=0,\n",
        "):\n",
        "    print(psutil.virtual_memory())  # print memory usage\n",
        "\n",
        "    if seed == 0:\n",
        "        seed = random.randint(0, 2147483647)\n",
        "\n",
        "    generator = torch.Generator(\"cuda\").manual_seed(seed)\n",
        "\n",
        "    try:\n",
        "        image = Image.open(image)\n",
        "        ratio = min(height / image.height, width / image.width)\n",
        "        image = image.resize((int(image.width * ratio), int(image.height * ratio)), Image.LANCZOS)\n",
        "\n",
        "        result = pipe(\n",
        "            prompt,\n",
        "            negative_prompt=neg_prompt,\n",
        "            image=image,\n",
        "            num_inference_steps=int(steps),\n",
        "            image_guidance_scale=image_guidance_scale,\n",
        "            guidance_scale=text_guidance_scale,\n",
        "            generator=generator,\n",
        "        )\n",
        "\n",
        "        # return replace_nsfw_images(result)\n",
        "        return result.images, result.nsfw_content_detected, seed\n",
        "    except Exception as e:\n",
        "        return None, None, error_str(e)\n",
        "\n",
        "def error_str(error, title=\"Error\"):\n",
        "    return (\n",
        "        f\"\"\"#### {title}\n",
        "            {error}\"\"\"\n",
        "        if error\n",
        "        else \"\"\n",
        "    )\n",
        "\n",
        "def get_frames(video_in):\n",
        "    frames = []\n",
        "    #resize the video\n",
        "    clip = VideoFileClip(video_in)\n",
        "    \n",
        "    #check fps\n",
        "    if clip.fps > 30:\n",
        "        print(\"vide rate is over 30, resetting to 30\")\n",
        "        clip_resized = clip.resize(height=512)\n",
        "        clip_resized.write_videofile(\"video_resized.mp4\", fps=30)\n",
        "    else:\n",
        "        print(\"video rate is OK\")\n",
        "        clip_resized = clip.resize(height=512)\n",
        "        clip_resized.write_videofile(\"video_resized.mp4\", fps=clip.fps)\n",
        "    \n",
        "    print(\"video resized to 512 height\")\n",
        "    \n",
        "    # Opens the Video file with CV2\n",
        "    cap= cv2.VideoCapture(\"video_resized.mp4\")\n",
        "    \n",
        "    fps = cap.get(cv2.CAP_PROP_FPS)\n",
        "    print(\"video fps: \" + str(fps))\n",
        "    i=0\n",
        "    while(cap.isOpened()):\n",
        "        ret, frame = cap.read()\n",
        "        if ret == False:\n",
        "            break\n",
        "        cv2.imwrite('kang'+str(i)+'.jpg',frame)\n",
        "        frames.append('kang'+str(i)+'.jpg')\n",
        "        i+=1\n",
        "    \n",
        "    cap.release()\n",
        "    cv2.destroyAllWindows()\n",
        "    print(\"broke the video into frames\")\n",
        "    \n",
        "    return frames, fps\n",
        "\n",
        "\n",
        "def create_video(frames, fps):\n",
        "    print(\"building video result\")\n",
        "    clip = ImageSequenceClip(frames, fps=fps)\n",
        "    clip.write_videofile(\"movie.mp4\", fps=fps)\n",
        "    \n",
        "    return 'movie.mp4'\n",
        "\n",
        "\n",
        "def infer(prompt,video_in, seed_in, trim_value):\n",
        "    print(prompt)\n",
        "    break_vid = get_frames(video_in)\n",
        "    \n",
        "    frames_list= break_vid[0]\n",
        "    fps = break_vid[1]\n",
        "    n_frame = int(trim_value*fps)\n",
        "    \n",
        "    if n_frame >= len(frames_list):\n",
        "        print(\"video is shorter than the cut value\")\n",
        "        n_frame = len(frames_list)\n",
        "    \n",
        "    result_frames = []\n",
        "    print(\"set stop frames to: \" + str(n_frame))\n",
        "    \n",
        "    for i in frames_list[0:int(n_frame)]:\n",
        "        pix2pix_img = pix2pix(prompt,5.5,1.5,i,15,\"\",512,512,seed_in)\n",
        "        images = pix2pix_img[0]\n",
        "        rgb_im = images[0].convert(\"RGB\")\n",
        "  \n",
        "        # exporting the image\n",
        "        rgb_im.save(f\"result_img-{i}.jpg\")\n",
        "        result_frames.append(f\"result_img-{i}.jpg\")\n",
        "        print(\"frame \" + i + \"/\" + str(n_frame) + \": done;\")\n",
        "\n",
        "    final_vid = create_video(result_frames, fps)\n",
        "    print(\"finished !\")\n",
        "    \n",
        "    return final_vid, gr.Group.update(visible=True)\n",
        "\n",
        "title = \"\"\"\n",
        "    <div style=\"text-align: center; max-width: 700px; margin: 0 auto;\">\n",
        "        <div\n",
        "        style=\"\n",
        "            display: inline-flex;\n",
        "            align-items: center;\n",
        "            gap: 0.8rem;\n",
        "            font-size: 1.75rem;\n",
        "        \"\n",
        "        >\n",
        "        <h1 style=\"font-weight: 900; margin-bottom: 7px; margin-top: 5px;\">\n",
        "            Pix2Pix Video\n",
        "        </h1>\n",
        "        </div>\n",
        "        <p style=\"margin-bottom: 10px; font-size: 94%\">\n",
        "        Apply Instruct Pix2Pix Diffusion to a video \n",
        "        </p>\n",
        "    </div>\n",
        "\"\"\"\n",
        "\n",
        "article = \"\"\"\n",
        "    \n",
        "    <div class=\"footer\">\n",
        "        <p>\n",
        "        Examples by <a href=\"https://twitter.com/CitizenPlain\" target=\"_blank\">Nathan Shipley</a> •&nbsp;\n",
        "        Follow <a href=\"https://twitter.com/fffiloni\" target=\"_blank\">Sylvain Filoni</a> for future updates 🤗\n",
        "        </p>\n",
        "    </div>\n",
        "    <div id=\"may-like-container\" style=\"display: flex;justify-content: center;flex-direction: column;align-items: center;margin-bottom: 30px;\">\n",
        "        <p>You may also like: </p>\n",
        "        <div id=\"may-like-content\" style=\"display:flex;flex-wrap: wrap;align-items:center;height:20px;\">\n",
        "            \n",
        "            <svg height=\"20\" width=\"162\" style=\"margin-left:4px;margin-bottom: 6px;\">       \n",
        "                 <a href=\"https://huggingface.co/spaces/timbrooks/instruct-pix2pix\" target=\"_blank\">\n",
        "                    <image href=\"https://img.shields.io/badge/🤗 Spaces-Instruct_Pix2Pix-blue\" src=\"https://img.shields.io/badge/🤗 Spaces-Instruct_Pix2Pix-blue.png\" height=\"20\"/>\n",
        "                 </a>\n",
        "            </svg>\n",
        "            \n",
        "        </div>\n",
        "    \n",
        "    </div>\n",
        "    \n",
        "\"\"\"\n",
        "\n",
        "with gr.Blocks(css='style.css') as demo:\n",
        "    with gr.Column(elem_id=\"col-container\"):\n",
        "        gr.HTML(title)\n",
        "        with gr.Row():\n",
        "            with gr.Column():\n",
        "                video_inp = gr.Video(label=\"Video source\", source=\"upload\", type=\"filepath\", elem_id=\"input-vid\")\n",
        "                prompt = gr.Textbox(label=\"Prompt\", placeholder=\"enter prompt\", show_label=False, elem_id=\"prompt-in\")\n",
        "                with gr.Row():\n",
        "                    seed_inp = gr.Slider(label=\"Seed\", minimum=0, maximum=2147483647, step=1, value=123456)\n",
        "                    trim_in = gr.Slider(label=\"Cut video at (s)\", minimun=1, maximum=5, step=1, value=1)\n",
        "            with gr.Column():\n",
        "                video_out = gr.Video(label=\"Pix2pix video result\", elem_id=\"video-output\")\n",
        "                gr.HTML(\"\"\"\n",
        "                <a style=\"display:inline-block\" href=\"https://huggingface.co/spaces/fffiloni/Pix2Pix-Video?duplicate=true\"><img src=\"https://img.shields.io/badge/-Duplicate%20Space-blue?labelColor=white&style=flat&logo=data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAAAXNSR0IArs4c6QAAAP5JREFUOE+lk7FqAkEURY+ltunEgFXS2sZGIbXfEPdLlnxJyDdYB62sbbUKpLbVNhyYFzbrrA74YJlh9r079973psed0cvUD4A+4HoCjsA85X0Dfn/RBLBgBDxnQPfAEJgBY+A9gALA4tcbamSzS4xq4FOQAJgCDwV2CPKV8tZAJcAjMMkUe1vX+U+SMhfAJEHasQIWmXNN3abzDwHUrgcRGmYcgKe0bxrblHEB4E/pndMazNpSZGcsZdBlYJcEL9Afo75molJyM2FxmPgmgPqlWNLGfwZGG6UiyEvLzHYDmoPkDDiNm9JR9uboiONcBXrpY1qmgs21x1QwyZcpvxt9NS09PlsPAAAAAElFTkSuQmCC&logoWidth=14\" alt=\"Duplicate Space\"></a> \n",
        "                work with longer videos / skip the queue: \n",
        "                \"\"\", elem_id=\"duplicate-container\")\n",
        "                submit_btn = gr.Button(\"Generate Pix2Pix video\")\n",
        "\n",
        "                with gr.Group(elem_id=\"share-btn-container\", visible=False) as share_group:\n",
        "                    loading_icon = gr.HTML(loading_icon_html)\n",
        "                    \n",
        "        \n",
        "        inputs = [prompt,video_inp,seed_inp, trim_in]\n",
        "        outputs = [video_out, share_group]\n",
        "        \n",
        "        ex = gr.Examples(\n",
        "            [\n",
        "                [\"Make it a marble sculpture\", \"./examples/pexels-jill-burrow-7665249_512x512.mp4\", 422112651, 4],\n",
        "                [\"Make it molten lava\", \"./examples/Ocean_Pexels_ 8953474_512x512.mp4\", 43571876, 4]\n",
        "            ],\n",
        "            inputs=inputs,\n",
        "            outputs=outputs,\n",
        "            fn=infer,\n",
        "            cache_examples=True,\n",
        "        )\n",
        "        \n",
        "        gr.HTML(article)\n",
        "    \n",
        "    submit_btn.click(infer, inputs, outputs)\n",
        "    \n",
        "\n",
        "    \n",
        "    \n",
        "demo.launch()"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from IPython.display import HTML\n",
        "from base64 import b64encode\n",
        " \n",
        "def show_video(video_path, video_width = 600):\n",
        "   \n",
        "  video_file = open(video_path, \"r+b\").read()\n",
        " \n",
        "  video_url = f\"data:video/mp4;base64,{b64encode(video_file).decode()}\"\n",
        "  return HTML(f\"\"\"<video width={video_width} controls><source src=\"{video_url}\"></video>\"\"\")\n",
        " \n",
        "show_video(\"video_resized.mp4\")\n"
      ],
      "metadata": {
        "id": "cWlOkzWtjUdy"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "show_video(\"movie.mp4\")"
      ],
      "metadata": {
        "id": "JWdkvid6jixs"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}