diff --git a/Disco_Diffusion.ipynb b/Disco_Diffusion.ipynb
index 30d8caa2..3614620b 100644
--- a/Disco_Diffusion.ipynb
+++ b/Disco_Diffusion.ipynb
@@ -3,8 +3,8 @@
{
"cell_type": "markdown",
"metadata": {
- "id": "view-in-github",
- "colab_type": "text"
+ "colab_type": "text",
+ "id": "view-in-github"
},
"source": [
"![\"Open](\"https://colab.research.google.com/assets/colab-badge.svg\")
"
@@ -29,7 +29,7 @@
"id": "CreditsChTop"
},
"source": [
- "### Credits & Changelog \u2b07\ufe0f"
+ "### Credits & Changelog ⬇️"
]
},
{
@@ -166,10 +166,12 @@
},
{
"cell_type": "code",
+ "execution_count": null,
"metadata": {
"cellView": "form",
"id": "Changelog"
},
+ "outputs": [],
"source": [
"#@title <- View Changelog\n",
"skip_for_run_all = True #@param {type: 'boolean'}\n",
@@ -273,9 +275,7 @@
"\n",
" '''\n",
" )"
- ],
- "outputs": [],
- "execution_count": null
+ ]
},
{
"cell_type": "markdown",
@@ -355,10 +355,12 @@
},
{
"cell_type": "code",
+ "execution_count": null,
"metadata": {
"cellView": "form",
"id": "CheckGPU"
},
+ "outputs": [],
"source": [
"#@title 1.1 Check GPU Status\n",
"import subprocess\n",
@@ -373,16 +375,16 @@
" print(nvidiasmi_output)\n",
" nvidiasmi_ecc_note = subprocess.run(['nvidia-smi', '-i', '0', '-e', '0'], stdout=subprocess.PIPE).stdout.decode('utf-8')\n",
" print(nvidiasmi_ecc_note)"
- ],
- "outputs": [],
- "execution_count": null
+ ]
},
{
"cell_type": "code",
+ "execution_count": null,
"metadata": {
"cellView": "form",
"id": "PrepFolders"
},
+ "outputs": [],
"source": [
"#@title 1.2 Prepare Folders\n",
"import subprocess, os, sys, ipykernel\n",
@@ -448,16 +450,16 @@
"\n",
"# libraries = f'{root_path}/libraries'\n",
"# createPath(libraries)"
- ],
- "outputs": [],
- "execution_count": null
+ ]
},
{
"cell_type": "code",
+ "execution_count": null,
"metadata": {
"cellView": "form",
"id": "InstallDeps"
},
+ "outputs": [],
"source": [
"#@title ### 1.3 Install, import dependencies and set up runtime devices\n",
"\n",
@@ -612,16 +614,16 @@
" if torch.cuda.get_device_capability(DEVICE) == (8,0): ## A100 fix thanks to Emad\n",
" print('Disabling CUDNN for A100 gpu', file=sys.stderr)\n",
" torch.backends.cudnn.enabled = False"
- ],
- "outputs": [],
- "execution_count": null
+ ]
},
{
"cell_type": "code",
+ "execution_count": null,
"metadata": {
"cellView": "form",
"id": "DefMidasFns"
},
+ "outputs": [],
"source": [
"#@title ### 1.4 Define Midas functions\n",
"\n",
@@ -724,16 +726,16 @@
"\n",
" print(f\"MiDaS '{midas_model_type}' depth model initialized.\")\n",
" return midas_model, midas_transform, net_w, net_h, resize_mode, normalization"
- ],
- "outputs": [],
- "execution_count": null
+ ]
},
{
"cell_type": "code",
+ "execution_count": null,
"metadata": {
"cellView": "form",
"id": "DefFns"
},
+ "outputs": [],
"source": [
"#@title 1.5 Define necessary functions\n",
"\n",
@@ -1540,16 +1542,16 @@
" # print('Settings:', setting_list)\n",
" with open(f\"{batchFolder}/{batch_name}({batchNum})_settings.txt\", \"w+\") as f: #save settings\n",
" json.dump(setting_list, f, ensure_ascii=False, indent=4)"
- ],
- "outputs": [],
- "execution_count": null
+ ]
},
{
"cell_type": "code",
+ "execution_count": null,
"metadata": {
"cellView": "form",
"id": "DefSecModel"
},
+ "outputs": [],
"source": [
"#@title 1.6 Define the secondary diffusion model\n",
"\n",
@@ -1714,9 +1716,7 @@
" pred = input * alphas - v * sigmas\n",
" eps = input * sigmas + v * alphas\n",
" return DiffusionOutput(v, pred, eps)"
- ],
- "outputs": [],
- "execution_count": null
+ ]
},
{
"cell_type": "markdown",
@@ -1729,9 +1729,11 @@
},
{
"cell_type": "code",
+ "execution_count": null,
"metadata": {
"id": "ModelSettings"
},
+ "outputs": [],
"source": [
"#@markdown ####**Models Settings:**\n",
"diffusion_model = \"512x512_diffusion_uncond_finetune_008100\" #@param [\"256x256_diffusion_uncond\", \"512x512_diffusion_uncond_finetune_008100\"]\n",
@@ -1739,12 +1741,12 @@
"diffusion_sampling_mode = 'ddim' #@param ['plms','ddim'] \n",
"\n",
"\n",
- "use_checkpoint = True #@param {type: 'boolean'}\n",
- "ViTB32 = True #@param{type:\"boolean\"}\n",
- "ViTB16 = True #@param{type:\"boolean\"}\n",
+ "use_checkpoint = False #@param {type: 'boolean'}\n",
+ "ViTB32 = False #@param{type:\"boolean\"}\n",
+ "ViTB16 = False #@param{type:\"boolean\"}\n",
"ViTL14 = False #@param{type:\"boolean\"}\n",
"RN101 = False #@param{type:\"boolean\"}\n",
- "RN50 = True #@param{type:\"boolean\"}\n",
+ "RN50 = False #@param{type:\"boolean\"}\n",
"RN50x4 = False #@param{type:\"boolean\"}\n",
"RN50x16 = False #@param{type:\"boolean\"}\n",
"RN50x64 = False #@param{type:\"boolean\"}\n",
@@ -1919,9 +1921,52 @@
"\n",
"normalize = T.Normalize(mean=[0.48145466, 0.4578275, 0.40821073], std=[0.26862954, 0.26130258, 0.27577711])\n",
"lpips_model = lpips.LPIPS(net='vgg').to(device)"
- ],
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
"outputs": [],
- "execution_count": null
+ "source": [
+ "#@markdown ####**Additional CLIP models**\n",
+ "\n",
+ "#@markdown This cell loads perceptor models via https://github.com/dmarx/multi-modal-comparators. Some model comparisons [here](https://t.co/iShJpm5GjL)\n",
+ "\n",
+ "# install MMC if not already installed\n",
+ "\n",
+ "try:\n",
+ " import mmc\n",
+ "except ImportError:\n",
+ " !pip install --quiet poetry\n",
+ " !git clone --quiet https://github.com/dmarx/Multi-Modal-Comparators\n",
+ " !cd Multi-Modal-Comparators; poetry build; pip install dist/mmc*.whl\n",
+ " !python Multi-Modal-Comparators/src/mmc/napm_installs/__init__.py\n",
+ "\n",
+ "import mmc\n",
+ "from mmc.registry import REGISTRY\n",
+ "import mmc.loaders # force trigger model registrations\n",
+ "from mmc.mock.openai import MockOpenaiClip\n",
+ "\n",
+ "# @markdown Select up to three models\n",
+ "\n",
+ "model1 = \"\" # @param [\"[clip - openai - RN50]\",\"[clip - openai - RN101]\",\"[clip - openai - RN50x4]\",\"[clip - openai - RN50x16]\",\"[clip - openai - RN50x64]\",\"[clip - openai - ViT-B/32]\",\"[clip - openai - ViT-B/16]\",\"[clip - openai - ViT-L/14]\",\"[clip - openai - ViT-L/14@336px]\",\"[clip - mlfoundations - RN50--openai]\",\"[clip - mlfoundations - RN50--yfcc15m]\",\"[clip - mlfoundations - RN50--cc12m]\",\"[clip - mlfoundations - RN50-quickgelu--openai]\",\"[clip - mlfoundations - RN50-quickgelu--yfcc15m]\",\"[clip - mlfoundations - RN50-quickgelu--cc12m]\",\"[clip - mlfoundations - RN101--openai]\",\"[clip - mlfoundations - RN101--yfcc15m]\",\"[clip - mlfoundations - RN101-quickgelu--openai]\",\"[clip - mlfoundations - RN101-quickgelu--yfcc15m]\",\"[clip - mlfoundations - RN50x4--openai]\",\"[clip - mlfoundations - RN50x16--openai]\",\"[clip - mlfoundations - ViT-B-32--openai]\",\"[clip - mlfoundations - ViT-B-32--laion400m_e31]\",\"[clip - mlfoundations - ViT-B-32--laion400m_e32]\",\"[clip - mlfoundations - ViT-B-32--laion400m_avg]\",\"[clip - mlfoundations - ViT-B-32-quickgelu--openai]\",\"[clip - mlfoundations - ViT-B-32-quickgelu--laion400m_e31]\",\"[clip - mlfoundations - ViT-B-32-quickgelu--laion400m_e32]\",\"[clip - mlfoundations - ViT-B-32-quickgelu--laion400m_avg]\",\"[clip - mlfoundations - ViT-B-16--openai]\",\"[clip - mlfoundations - ViT-B-16--laion400m_e31]\",\"[clip - mlfoundations - ViT-B-16--laion400m_e32]\",\"[clip - mlfoundations - ViT-L-14--openai]\",\"[clip - mlfoundations - ViT-L-14-336--openai]\",\"[clip - sbert - ViT-B-32-multilingual-v1]\",\"[clip - sajjjadayobi - clipfa]\",\"[cloob - crowsonkb - cloob_laion_400m_vit_b_16_16_epochs]\",\"[cloob - crowsonkb - cloob_laion_400m_vit_b_16_32_epochs]\",\"[clip - navervision - kelip_ViT-B/32]\",\"[clip - facebookresearch - clip_small_25ep]\",\"[simclr - facebookresearch - simclr_small_25ep]\",\"[slip - facebookresearch - slip_small_25ep]\",\"[slip - facebookresearch - slip_small_50ep]\",\"[slip - facebookresearch - slip_small_100ep]\",\"[clip - facebookresearch - clip_base_25ep]\",\"[simclr - facebookresearch - simclr_base_25ep]\",\"[slip - facebookresearch - slip_base_25ep]\",\"[slip - facebookresearch - slip_base_50ep]\",\"[slip - facebookresearch - slip_base_100ep]\",\"[clip - facebookresearch - clip_large_25ep]\",\"[simclr - facebookresearch - simclr_large_25ep]\",\"[slip - facebookresearch - slip_large_25ep]\",\"[slip - facebookresearch - slip_large_50ep]\",\"[slip - facebookresearch - slip_large_100ep]\",\"[clip - facebookresearch - clip_base_cc3m_40ep]\",\"[slip - facebookresearch - slip_base_cc3m_40ep]\",\"[slip - facebookresearch - slip_base_cc12m_35ep]\",\"[clip - facebookresearch - clip_base_cc12m_35ep]\"] {allow-input: true}\n",
+ "model2 = \"\" # @param [\"[clip - openai - RN50]\",\"[clip - openai - RN101]\",\"[clip - openai - RN50x4]\",\"[clip - openai - RN50x16]\",\"[clip - openai - RN50x64]\",\"[clip - openai - ViT-B/32]\",\"[clip - openai - ViT-B/16]\",\"[clip - openai - ViT-L/14]\",\"[clip - openai - ViT-L/14@336px]\",\"[clip - mlfoundations - RN50--openai]\",\"[clip - mlfoundations - RN50--yfcc15m]\",\"[clip - mlfoundations - RN50--cc12m]\",\"[clip - mlfoundations - RN50-quickgelu--openai]\",\"[clip - mlfoundations - RN50-quickgelu--yfcc15m]\",\"[clip - mlfoundations - RN50-quickgelu--cc12m]\",\"[clip - mlfoundations - RN101--openai]\",\"[clip - mlfoundations - RN101--yfcc15m]\",\"[clip - mlfoundations - RN101-quickgelu--openai]\",\"[clip - mlfoundations - RN101-quickgelu--yfcc15m]\",\"[clip - mlfoundations - RN50x4--openai]\",\"[clip - mlfoundations - RN50x16--openai]\",\"[clip - mlfoundations - ViT-B-32--openai]\",\"[clip - mlfoundations - ViT-B-32--laion400m_e31]\",\"[clip - mlfoundations - ViT-B-32--laion400m_e32]\",\"[clip - mlfoundations - ViT-B-32--laion400m_avg]\",\"[clip - mlfoundations - ViT-B-32-quickgelu--openai]\",\"[clip - mlfoundations - ViT-B-32-quickgelu--laion400m_e31]\",\"[clip - mlfoundations - ViT-B-32-quickgelu--laion400m_e32]\",\"[clip - mlfoundations - ViT-B-32-quickgelu--laion400m_avg]\",\"[clip - mlfoundations - ViT-B-16--openai]\",\"[clip - mlfoundations - ViT-B-16--laion400m_e31]\",\"[clip - mlfoundations - ViT-B-16--laion400m_e32]\",\"[clip - mlfoundations - ViT-L-14--openai]\",\"[clip - mlfoundations - ViT-L-14-336--openai]\",\"[clip - sbert - ViT-B-32-multilingual-v1]\",\"[clip - sajjjadayobi - clipfa]\",\"[cloob - crowsonkb - cloob_laion_400m_vit_b_16_16_epochs]\",\"[cloob - crowsonkb - cloob_laion_400m_vit_b_16_32_epochs]\",\"[clip - navervision - kelip_ViT-B/32]\",\"[clip - facebookresearch - clip_small_25ep]\",\"[simclr - facebookresearch - simclr_small_25ep]\",\"[slip - facebookresearch - slip_small_25ep]\",\"[slip - facebookresearch - slip_small_50ep]\",\"[slip - facebookresearch - slip_small_100ep]\",\"[clip - facebookresearch - clip_base_25ep]\",\"[simclr - facebookresearch - simclr_base_25ep]\",\"[slip - facebookresearch - slip_base_25ep]\",\"[slip - facebookresearch - slip_base_50ep]\",\"[slip - facebookresearch - slip_base_100ep]\",\"[clip - facebookresearch - clip_large_25ep]\",\"[simclr - facebookresearch - simclr_large_25ep]\",\"[slip - facebookresearch - slip_large_25ep]\",\"[slip - facebookresearch - slip_large_50ep]\",\"[slip - facebookresearch - slip_large_100ep]\",\"[clip - facebookresearch - clip_base_cc3m_40ep]\",\"[slip - facebookresearch - slip_base_cc3m_40ep]\",\"[slip - facebookresearch - slip_base_cc12m_35ep]\",\"[clip - facebookresearch - clip_base_cc12m_35ep]\"] {allow-input: true}\n",
+ "model3 = \"\" # @param [\"[clip - openai - RN50]\",\"[clip - openai - RN101]\",\"[clip - openai - RN50x4]\",\"[clip - openai - RN50x16]\",\"[clip - openai - RN50x64]\",\"[clip - openai - ViT-B/32]\",\"[clip - openai - ViT-B/16]\",\"[clip - openai - ViT-L/14]\",\"[clip - openai - ViT-L/14@336px]\",\"[clip - mlfoundations - RN50--openai]\",\"[clip - mlfoundations - RN50--yfcc15m]\",\"[clip - mlfoundations - RN50--cc12m]\",\"[clip - mlfoundations - RN50-quickgelu--openai]\",\"[clip - mlfoundations - RN50-quickgelu--yfcc15m]\",\"[clip - mlfoundations - RN50-quickgelu--cc12m]\",\"[clip - mlfoundations - RN101--openai]\",\"[clip - mlfoundations - RN101--yfcc15m]\",\"[clip - mlfoundations - RN101-quickgelu--openai]\",\"[clip - mlfoundations - RN101-quickgelu--yfcc15m]\",\"[clip - mlfoundations - RN50x4--openai]\",\"[clip - mlfoundations - RN50x16--openai]\",\"[clip - mlfoundations - ViT-B-32--openai]\",\"[clip - mlfoundations - ViT-B-32--laion400m_e31]\",\"[clip - mlfoundations - ViT-B-32--laion400m_e32]\",\"[clip - mlfoundations - ViT-B-32--laion400m_avg]\",\"[clip - mlfoundations - ViT-B-32-quickgelu--openai]\",\"[clip - mlfoundations - ViT-B-32-quickgelu--laion400m_e31]\",\"[clip - mlfoundations - ViT-B-32-quickgelu--laion400m_e32]\",\"[clip - mlfoundations - ViT-B-32-quickgelu--laion400m_avg]\",\"[clip - mlfoundations - ViT-B-16--openai]\",\"[clip - mlfoundations - ViT-B-16--laion400m_e31]\",\"[clip - mlfoundations - ViT-B-16--laion400m_e32]\",\"[clip - mlfoundations - ViT-L-14--openai]\",\"[clip - mlfoundations - ViT-L-14-336--openai]\",\"[clip - sbert - ViT-B-32-multilingual-v1]\",\"[clip - sajjjadayobi - clipfa]\",\"[cloob - crowsonkb - cloob_laion_400m_vit_b_16_16_epochs]\",\"[cloob - crowsonkb - cloob_laion_400m_vit_b_16_32_epochs]\",\"[clip - navervision - kelip_ViT-B/32]\",\"[clip - facebookresearch - clip_small_25ep]\",\"[simclr - facebookresearch - simclr_small_25ep]\",\"[slip - facebookresearch - slip_small_25ep]\",\"[slip - facebookresearch - slip_small_50ep]\",\"[slip - facebookresearch - slip_small_100ep]\",\"[clip - facebookresearch - clip_base_25ep]\",\"[simclr - facebookresearch - simclr_base_25ep]\",\"[slip - facebookresearch - slip_base_25ep]\",\"[slip - facebookresearch - slip_base_50ep]\",\"[slip - facebookresearch - slip_base_100ep]\",\"[clip - facebookresearch - clip_large_25ep]\",\"[simclr - facebookresearch - simclr_large_25ep]\",\"[slip - facebookresearch - slip_large_25ep]\",\"[slip - facebookresearch - slip_large_50ep]\",\"[slip - facebookresearch - slip_large_100ep]\",\"[clip - facebookresearch - clip_base_cc3m_40ep]\",\"[slip - facebookresearch - slip_base_cc3m_40ep]\",\"[slip - facebookresearch - slip_base_cc12m_35ep]\",\"[clip - facebookresearch - clip_base_cc12m_35ep]\"] {allow-input: true}\n",
+ "\n",
+ "mmc_models = []\n",
+ "\n",
+ "for model_key in (model1, model2, model3):\n",
+ " if not model_key:\n",
+ " continue\n",
+ " arch, pub, m_id = model_key[1:-1].split(' - ')\n",
+ " ldr = REGISTRY.find(architecture=arch, publisher=pub, id=m_id)[0]\n",
+ " model = MockOpenaiClip(ldr.load(device=device))\n",
+ " mmc_models.append(model)\n",
+ "\n",
+ "#clip_models.extend(mmc_models)\n",
+ "clip_models = mmc_models"
+ ]
},
{
"cell_type": "markdown",
@@ -1934,9 +1979,11 @@
},
{
"cell_type": "code",
+ "execution_count": null,
"metadata": {
"id": "BasicSettings"
},
+ "outputs": [],
"source": [
"#@markdown ####**Basic Settings:**\n",
"batch_name = 'TimeToDisco' #@param{type: 'string'}\n",
@@ -1974,9 +2021,7 @@
"#Make folder for batch\n",
"batchFolder = f'{outDirPath}/{batch_name}'\n",
"createPath(batchFolder)"
- ],
- "outputs": [],
- "execution_count": null
+ ]
},
{
"cell_type": "markdown",
@@ -1989,9 +2034,11 @@
},
{
"cell_type": "code",
+ "execution_count": null,
"metadata": {
"id": "AnimSettings"
},
+ "outputs": [],
"source": [
"#@markdown ####**Animation Mode:**\n",
"animation_mode = 'None' #@param ['None', '2D', '3D', 'Video Input'] {type:'string'}\n",
@@ -2345,9 +2392,7 @@
" rotation_3d_x = float(rotation_3d_x)\n",
" rotation_3d_y = float(rotation_3d_y)\n",
" rotation_3d_z = float(rotation_3d_z)"
- ],
- "outputs": [],
- "execution_count": null
+ ]
},
{
"cell_type": "markdown",
@@ -2361,9 +2406,11 @@
},
{
"cell_type": "code",
+ "execution_count": null,
"metadata": {
"id": "ExtraSettings"
},
+ "outputs": [],
"source": [
"#@markdown ####**Saving:**\n",
"\n",
@@ -2423,9 +2470,7 @@
"cut_innercut =\"[4]*400+[12]*600\"#@param {type: 'string'} \n",
"cut_ic_pow = 1#@param {type: 'number'} \n",
"cut_icgray_p = \"[0.2]*400+[0]*600\"#@param {type: 'string'}"
- ],
- "outputs": [],
- "execution_count": null
+ ]
},
{
"cell_type": "markdown",
@@ -2439,9 +2484,11 @@
},
{
"cell_type": "code",
+ "execution_count": null,
"metadata": {
"id": "Prompts"
},
+ "outputs": [],
"source": [
"text_prompts = {\n",
" 0: [\"A beautiful painting of a singular lighthouse, shining its light across a tumultuous sea of blood by greg rutkowski and thomas kinkade, Trending on artstation.\", \"yellow color scheme\"],\n",
@@ -2451,9 +2498,7 @@
"image_prompts = {\n",
" # 0:['ImagePromptsWorkButArentVeryGood.png:2',],\n",
"}"
- ],
- "outputs": [],
- "execution_count": null
+ ]
},
{
"cell_type": "markdown",
@@ -2466,9 +2511,11 @@
},
{
"cell_type": "code",
+ "execution_count": null,
"metadata": {
"id": "DoTheRun"
},
+ "outputs": [],
"source": [
"#@title Do the Run!\n",
"#@markdown `n_batches` ignored with animation modes.\n",
@@ -2649,9 +2696,7 @@
" print('Seed used:', seed)\n",
" gc.collect()\n",
" torch.cuda.empty_cache()"
- ],
- "outputs": [],
- "execution_count": null
+ ]
},
{
"cell_type": "markdown",
@@ -2664,9 +2709,11 @@
},
{
"cell_type": "code",
+ "execution_count": null,
"metadata": {
"id": "CreateVid"
},
+ "outputs": [],
"source": [
"# @title ### **Create video**\n",
"#@markdown Video file will save in the same folder as your images.\n",
@@ -2742,14 +2789,12 @@
" # mp4 = open(filepath,'rb').read()\n",
" # data_url = \"data:video/mp4;base64,\" + b64encode(mp4).decode()\n",
" # display.HTML(f'')"
- ],
- "outputs": [],
- "execution_count": null
+ ]
}
],
"metadata": {
- "anaconda-cloud": {},
"accelerator": "GPU",
+ "anaconda-cloud": {},
"colab": {
"collapsed_sections": [
"CreditsChTop",
@@ -2763,11 +2808,11 @@
"AnimSetTop",
"ExtraSetTop"
],
+ "include_colab_link": true,
"machine_shape": "hm",
"name": "Disco Diffusion v5.2 [w/ VR Mode]",
"private_outputs": true,
- "provenance": [],
- "include_colab_link": true
+ "provenance": []
},
"kernelspec": {
"display_name": "Python 3",
@@ -2789,4 +2834,4 @@
},
"nbformat": 4,
"nbformat_minor": 4
-}
\ No newline at end of file
+}