|
| 1 | +{ |
| 2 | + "cells": [ |
| 3 | + { |
| 4 | + "cell_type": "code", |
| 5 | + "execution_count": 1, |
| 6 | + "id": "5d8abf15-5fe0-49cc-ad5c-c688cbde3304", |
| 7 | + "metadata": {}, |
| 8 | + "outputs": [], |
| 9 | + "source": [ |
| 10 | + "import matplotlib.pyplot as plt\n", |
| 11 | + "%matplotlib inline\n", |
| 12 | + "import numpy as np\n", |
| 13 | + "import scipy.stats" |
| 14 | + ] |
| 15 | + }, |
| 16 | + { |
| 17 | + "cell_type": "markdown", |
| 18 | + "id": "660d443e-26c2-4406-a035-d9dd599f79b0", |
| 19 | + "metadata": {}, |
| 20 | + "source": [ |
| 21 | + "# Kullback-Leibler divergence" |
| 22 | + ] |
| 23 | + }, |
| 24 | + { |
| 25 | + "cell_type": "markdown", |
| 26 | + "id": "6b1ad233-a4a9-4ef0-abcd-1d20be1feb34", |
| 27 | + "metadata": {}, |
| 28 | + "source": [ |
| 29 | + "Define the Kullback-Leibler divergence as:\n", |
| 30 | + "$$\n", |
| 31 | + "\\textrm{KL}(p, q) = \\sum_{i} p_i \\log \\frac{p_i}{q_i}\n", |
| 32 | + "$$" |
| 33 | + ] |
| 34 | + }, |
| 35 | + { |
| 36 | + "cell_type": "code", |
| 37 | + "execution_count": 2, |
| 38 | + "id": "fe17c58f-6154-41e7-a83b-87ecf7a8ed00", |
| 39 | + "metadata": {}, |
| 40 | + "outputs": [], |
| 41 | + "source": [ |
| 42 | + "def kullback_leibler_div(p, q):\n", |
| 43 | + " return np.sum(p*np.log(p/q))" |
| 44 | + ] |
| 45 | + }, |
| 46 | + { |
| 47 | + "cell_type": "markdown", |
| 48 | + "id": "f79fdff3-8608-4233-9021-cfa174c48398", |
| 49 | + "metadata": {}, |
| 50 | + "source": [ |
| 51 | + "# Example" |
| 52 | + ] |
| 53 | + }, |
| 54 | + { |
| 55 | + "cell_type": "markdown", |
| 56 | + "id": "2d361c22-645c-4a55-9fc5-f45edf5463cb", |
| 57 | + "metadata": {}, |
| 58 | + "source": [ |
| 59 | + "Consider a normal distribution with mean $\\mu = 0.0$ and standard deviation $\\sigma = 1.0$. The distribution is computed on the interval $[-3, 3]$." |
| 60 | + ] |
| 61 | + }, |
| 62 | + { |
| 63 | + "cell_type": "code", |
| 64 | + "execution_count": 3, |
| 65 | + "id": "782dee3c-5265-48ab-ab4c-1b92f334ade5", |
| 66 | + "metadata": {}, |
| 67 | + "outputs": [], |
| 68 | + "source": [ |
| 69 | + "μ, σ = 0.0, 1.0" |
| 70 | + ] |
| 71 | + }, |
| 72 | + { |
| 73 | + "cell_type": "code", |
| 74 | + "execution_count": 4, |
| 75 | + "id": "77c3a026-a09a-4837-8633-a7f6796d1a23", |
| 76 | + "metadata": {}, |
| 77 | + "outputs": [], |
| 78 | + "source": [ |
| 79 | + "x = np.linspace(-3.0, 3.0, 601)" |
| 80 | + ] |
| 81 | + }, |
| 82 | + { |
| 83 | + "cell_type": "code", |
| 84 | + "execution_count": 5, |
| 85 | + "id": "69f724b8-008c-40b9-9d54-7d10109a4030", |
| 86 | + "metadata": {}, |
| 87 | + "outputs": [], |
| 88 | + "source": [ |
| 89 | + "p = scipy.stats.norm(loc=μ, scale=σ).pdf(x)" |
| 90 | + ] |
| 91 | + }, |
| 92 | + { |
| 93 | + "cell_type": "markdown", |
| 94 | + "id": "98d280f3-b05c-41e5-b51d-2cbb79b3a7d9", |
| 95 | + "metadata": {}, |
| 96 | + "source": [ |
| 97 | + "Next, we consider distribution with the same mean value $\\mu = 0.0$, but various different values for the standard deviation." |
| 98 | + ] |
| 99 | + }, |
| 100 | + { |
| 101 | + "cell_type": "code", |
| 102 | + "execution_count": 6, |
| 103 | + "id": "a028fe7b-9657-437f-8675-4f2aa7118d13", |
| 104 | + "metadata": {}, |
| 105 | + "outputs": [], |
| 106 | + "source": [ |
| 107 | + "σs = np.arange(0.1, 2.1, 0.01)" |
| 108 | + ] |
| 109 | + }, |
| 110 | + { |
| 111 | + "cell_type": "code", |
| 112 | + "execution_count": 7, |
| 113 | + "id": "e6fc4509-a9a9-47b0-a7a0-9916edc3509b", |
| 114 | + "metadata": {}, |
| 115 | + "outputs": [], |
| 116 | + "source": [ |
| 117 | + "qs = [scipy.stats.norm(loc=μ, scale=σ).pdf(x) for σ in σs]" |
| 118 | + ] |
| 119 | + }, |
| 120 | + { |
| 121 | + "cell_type": "markdown", |
| 122 | + "id": "55fd805b-0265-4e7b-9396-36aabd0d2b84", |
| 123 | + "metadata": {}, |
| 124 | + "source": [ |
| 125 | + "Compute the divergence between the distributions $q$ and $p$." |
| 126 | + ] |
| 127 | + }, |
| 128 | + { |
| 129 | + "cell_type": "code", |
| 130 | + "execution_count": 8, |
| 131 | + "id": "0c374926-759f-4893-bbde-41a4830aabde", |
| 132 | + "metadata": {}, |
| 133 | + "outputs": [], |
| 134 | + "source": [ |
| 135 | + "divs = [kullback_leibler_div(p, q) for q in qs]" |
| 136 | + ] |
| 137 | + }, |
| 138 | + { |
| 139 | + "cell_type": "code", |
| 140 | + "execution_count": 9, |
| 141 | + "id": "9bcca801-a677-4ee3-a8ea-a4d89057032f", |
| 142 | + "metadata": {}, |
| 143 | + "outputs": [ |
| 144 | + { |
| 145 | + "data": { |
| 146 | + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY4AAAEGCAYAAABy53LJAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8/fFQqAAAACXBIWXMAAAsTAAALEwEAmpwYAAAqoUlEQVR4nO3deXxU9b3/8dcnG9nIRsKaxIABZBPBgIobttZiFbBaFZdqFUXtT2t7b2+v7b1tb7er7e21rYpatdatylWrVetat1pFVmWVLawJkEAIJAESIMn398cEjTSBGTIzZ+bk/Xw85mHmzJkzH8YD73y/33O+X3POISIiEqwErwsQEZH4ouAQEZGQKDhERCQkCg4REQmJgkNEREKS5HUB0ZCfn+9KSkq8LkNEJG4sXLiwxjlX0NFr3SI4SkpKWLBggddliIjEDTPb2Nlrvu6qMrPJZvZAXV2d16WIiPiGr4PDOfeSc25Gdna216WIiPiGr4NDLQ4RkfDzdXCoxSEiEn6+Dg61OEREws/XwaEWh4hI+Pk6OEREJPx8HRxd6apqaXXMmreJ15ZVRaAyEZH45evg6EpXVYLB43M28otXPuFAS2sEqhMRiU++Do6uMDO+e85QKmobeWZBpdfliIjEDF8HR1evqpo4tICxxTnc/fYamg60hLk6EZH45Ovg6OpVVQdbHVvrmnhq3qYwVyciEp98HRzhMKE0n5MH5THznbXs3d/sdTkiIp5TcAThX88ZSs3ufTw6u9PJIkVEug0FRxDGleRx1tAC7n2nnG0NTV6XIyLiKV8HRzinHPnh+cNpam7hjldWhqEyEZH45evgCOeUI4MKMplxxiCe+3gz8zfUhqE6EZH45OvgCLf/d1Yp/bNT+eFfltGsmwJFpJtScIQgPSWJH54/nJVVDTw+RwPlItI9KThCNGlkX84YUsCvX19FRe1er8sREYk6BUeIzIzbLxwVuDnwmcW0tjqvSxIRiSpfB0ekFnIakJPGj84fztz1tTz64YawHltEJNb5OjgiuZDTxWWFfOG43vzytZWs27477McXEYlVvg6OSDIz7rhwFD2SEvnO04vZ36yrrESke1BwdEHvrFTuuHAUiyt28avXdGOgiHQPCo4uOndUP64+5Rgeen89byzXaoEi4n8KjjD4wXnDGDUgm+8+s1iX6IqI7yk4wqBHUiIzLx+LA775p4+06JOI+FrcBYeZDTOz+83sWTO7yet6Dirulc6dl5zA0s11/Pufl+Cc7u8QEX+KieAws4fNbJuZLTtk+yQzW2Vm5WZ2G4BzboVz7kbgEqDMi3o786Xhffi3Lw/lhUVbuO/va70uR0QkImIiOIBHgEntN5hZIjATOBcYDlxmZsPbXpsCvA+8Fd0yj+ybE49lyuj+/M/rq3jzk2qvyxERCbuYCA7n3HvAoXOVjwfKnXPrnHP7gVnA1Lb9X3TOTQCuiG6lR2Zm/OprxzNqQDbfmvUxyzaH9651ERGvxURwdGIAUNHueSUwwMwmmtldZvZ74JXO3mxmM8xsgZkt2L59e6Rr/ZzU5EQeuqqM3PQUvvHH+brSSkR8JZaDwzrY5pxz7zrnvuWcu8E5N7OzNzvnHgB+AnyUkpISsSI70zsrlUevHU9zaytXPzyP2j37o16DiEgkxHJwVAJF7Z4XAltCOUAk56oKRmnvTB66qozNuxqZ/uh89uxr9qQOEZFwiuXgmA8MNrOBZpYCTANeDOUAkZodNxRlJXncddkYllTWcd2jC3SPh4jEvZgIDjN7CvgQGGpmlWY23TnXDNwMvA6sAJ52zi0P5bhetzgO+vKIvtx5yWjmrN/BjMcXsq9Z4SEi8cv8fKOamU0GJpeWll6/Zs0ar8vh6fkVfO/PSzh7WB/uu3IsyYkxkdsiIv/EzBY65zq8V87X/3LFSovjoEvGFfGzqSN4c0U13561iOYWTcUuIvEnyesCupuvn1LCvuZWfv7yChISjDsvGa2Wh4jEFV8HR7uuKq9L+ZzrTh9Ec6vjjldX0nSghXsuH0OPpESvyxIRCYqvf9WNta6q9m4881h+MmUEf/ukmuseXUDjfg2Yi0h88HVwxMLluIdz9YQSfnXR8bxfXsPVf5zHbt3nISJxwNfBEcstjoMuGVfE76aNYeHGnVzx0Fx26g5zEYlxvg6OeDFldH/uu2IsK7bWc9H9szW3lYjENAVHjDhnRF+emH4SNQ37uOi+2SzfEpvdayIivg6OWB/jONT4gXk8e9MEEhOMS38/h9nlNV6XJCLyT3wdHPEwxnGoIX168uebJtA/J5Wr/ziPFxZt9rokEZHP8XVwxKv+OWk8c+MExhTncuusRfz2zdVaw1xEYoaCI0ZlpyXz+PTxXDh2AL99cw23zlqkmXVFJCb4OjjibYzjUD2SEvnfi0fzvUlDeXHxFi57cA7bG/Z5XZaIdHO+Do54HOM4lJnxzYml3H9l4HLdC2Z+wMqqeq/LEpFuzNfB4SeTRvbjmRsm0NzaykX3zubtldVelyQi3ZSCI46MKszmhf93GgMLMpj+6AJmvlOuQXMRiToFR5zpm53KMzdMYPLx/fmf11dx0xMfaY4rEYkqBUccSktJ5HfTTuA/zxvGG59U8dWZH7C+Zo/XZYlIN+Hr4Ij3q6oOx8y47vRBgWlKdu9jyj3va9xDRKLC18Hhh6uqjmRCaT4v3XIaxXnpTH90AXe9tYbWVo17iEjk+Do4uovC3HT+fNMELjhhAHf+bTU3PLGQhqYDXpclIj6l4PCJ1ORE7rxkND+ePJy3V25jyj2630NEIkPB4SNmxjWnDuTJ605i975mLpj5Ac8urPS6LBHxGQWHD500qBcvf+s0TijK4bvPLOa2Py/RPFciEjYKDp/q3TOVJ6afxDcnHsus+RVcdN9sNu7QJbsi0nUKDh9LSkzge5OO4w9Xl1G5s5Hz736fN5ZXeV2WiMS5uAsOM7vAzB40sxfM7Byv64kHXxzWh7/echoD8zOY8fhCbn9lBQdaWr0uS0TiVEwEh5k9bGbbzGzZIdsnmdkqMys3s9sAnHN/cc5dD3wDuNSDcuNSUV46z9x4CleeXMzv31vHFQ/Opbq+yeuyRCQOxURwAI8Ak9pvMLNEYCZwLjAcuMzMhrfb5T/bXpcg9UhK5OcXjOK3l57A0s11nHfXP5i9Vuuai0hoYiI4nHPvAbWHbB4PlDvn1jnn9gOzgKkW8EvgVefcR50d08xmmNkCM1uwffv2yBUfhy4YM4AXbz6V7LRkrnxoLjPfKdfd5iIStJgIjk4MACraPa9s23YLcDbwNTO7sbM3O+cecM6VOefKCgoKIltpHBrcpycv3nwa57XNsjv90fns2rvf67JEJA7EcnBYB9ucc+4u59yJzrkbnXP3H/YAPp7kMBwyeiRx17QT+OnUEbxfXsN5d73P4opdXpclIjEuloOjEihq97wQ2BLKAbrDJIddZWZcdUoJz9w4AYCL7/+Qxz/coAWiRKRTsRwc84HBZjbQzFKAacCLoRxALY7gnVCUw19vOY1TS3vxwxeWc+usRezRAlEi0oGYCA4zewr4EBhqZpVmNt051wzcDLwOrACeds4tD+W4anGEJjcjhT9cPY5/+/JQ/rpkC1NnfsCa6gavyxKRGGN+7pIws8nA5NLS0uvXrFnjdTlxZXZ5Dd+a9TF79rVwx0WjmHrCAK9LEpEoMrOFzrmyjl6LiRZHpKjFcfQmlObz8rdOZ+SALG6dtYj//MtS9jVrokQR8XlwaIyja/pkpfLk9SdzwxmDeGLOJi6+/0Mqavd6XZaIeMzXwaEWR9clJybw/a8M4/dfP5H1NXs4/+73eWuF1jYX6c58HRwSPl8e0Ze/3nIaA3LSmP7oAn712kqaNVGiSLfk6+BQV1V4HdMrg+e+OYHLxhdx77trufIPc9nWoIkSRbobXweHuqrCLzU5kdsvPJ5fXzyaRRW7mHL3ByzS3eYi3Yqvg0Mi52snFvLcTaeSlGhccv+HPL2g4shvEhFf8HVwqKsqsob3z+Klm09j3MBcvvfsEn78wjItECUSQyJ1Cb2vbwA8qKyszC1YsMDrMnyruaWVX762kgf/sZ7xA/O494qx5Gf28LosEV/bs6+ZLbsa2byrkaq6JrbWNVFd30RVfRNVdYH/NjQ1s+pnk0hKDL2NcLgbAJO6XL10e0mJCfzHecMZOSCb7z27hCl3v8/vv17GqEKNLYkcjeaWVqob9rFlV+On4bB1V9NnP9c1Udd44HPvMYNeGT3ol51KYW46ZSW59M1KpbnVkZQY3voUHBI2U08YwLEFmdzw+EIuun82t391FBedWOh1WSIx50BLK1t2NbKpdi+bavdSURsIhINBUV3fxKFrq+WkJ9M/O43C3DTGD8yjf05a4JGdSr+cNHr37EHyUbQsjoaCQ8Jq5IBsXrz5VG5+8mP+9ZnFrN7WwL9/+TgSEjpaXkXEn5xz7Nx7oC0U9n7uv5tq97JlV+PngiE50RiQk0a/7DQmHJvPgJzUz4IhJ5V+2Wlk9Iidf659PcahSQ6909zSyn+9tJwn5mziyyP68JtLTyA9JXZOfJFwqNt7gLU1u1m3fQ/r2/67YUcgJHYfsixBfmYKRXnpFLc92v/cJyuVxBj75epwYxy+Do6DNDjuDeccf/xgAz9/+ROG98/iD1ePo09WqtdliYRkX3MLm3bsZV3NHtZt38O67btZX7OHdTV7qN3z2XLLiQlGcV46Jb3SOaZXxueCoTA3tloMwdDguHjCzLj2tIGU5Kdzy5MfM/WeD3jo6jJGDtCgucSe/c2trK/Zw6rqBlZXNbC6OvDYVLv3c91K+Zk9GJSfwTnD+zCoIIOB+ZkMKsigOC89amMMXlOLQ6JixdZ6pj8yn517D/C7aSdwzoi+Xpck3VRLq2Pjjj2srt7N6uqGT4Nifc0emtsSIjHBKOmVzpA+PSntHQiGQfmZlORnkJ2W7PGfIDrC2lVlZhlAk3MubhZnUHDEhm31TVz/2AKWbK7jR+cP55pTB3pdkvjcvuYWVlftZvmWOpZvqWf5ljpWbG2g8cBn/3wV56UzpE8mQ/r0ZGjfngzu3ZNBBRmkJof5GtY406WuKjNLILDe9xXAOGAf0MPMtgOvAA845zTyLEfUOyuVWTNO4dZZH/OTlz5hW8M+vvfloZjF1qCgxKemAy0s21zH0s0HQ6KeNdUNn7YiMnskMbxfFpeOK2J4/yyGtrUm4m3sIRYE8429A7wJfB9Y5pxrBTCzPOAs4A4ze94590Tkyjw67a6q8roUaZOWksh9V57ID19Yxn3vrqW6volfXnR8t+kblvBobXWs37GHRZt2sagi8Fixtf7TkMjP7MGI/lmcNbSAEf2zGdE/i+K8dF0WHiZH7Koys2Tn3IGu7uMldVXFHuccd79dzp1/W82ZQwq494qx+s1POtXQdICFG3fyUVtQLK7Y9emd05k9kji+MJsTinIYU5zL8YXZunovDLrUVXUwEMxsDbAMWAIsBpY458rb7yMSLDPjW18cTO+ePfjB80u5/KG5PHrNOHLSU7wuTWJA7Z79zN9Qy7z1gcfyLXW0OkgwGNKnJ+eO7MuY4hxOKMqltHdmzN0D4Xeh/Ir3HJAGVAHnAE+YWQ2wmUCI3BCB+sTnpo0vJi8jhZuf/JhpD8zhietO0gSJ3dCuvfuZvXYHs9fWMG99LaurdwOQkpTAmKIcbj6rlHED8xhTnEumWqaeC/qqKjOb55wb3+75ScBXgXuA0c65lyNTYtepqyr2/WPNdq5/bAH9c9J48rqT6ZutrgY/29/cysebdvKPNTX8o7yGJZW7cA4yUhI5sSSPkwbmMX5gHscXZtMj3DP0SVDCcjmumb0DfNs5t7jdtsXOudHhKTNyFBzxYd76Wq59ZD65Gck8ed3JFOWle12ShFFF7V7eWlHNP9bUMGfdDvbsbyExwTihKIfTSvM5Y0g+owtzjmoKcAm/cAVHKfAYsBxYBAwDypxzJ4epzohRcMSPRRW7uPrheaSnJPKn605iUEGm1yXJUWptdSyu3MWbK6p585NtrKpuAOCYXumcPjif0wcXcMqxvchK7R431MWbsN0A2HZPxwXAGGAH8Lhzbkc4igyhhkHAfwDZzrmvBfMeBUd8WbG1nisfmktyYgJP33AKxb3U8ogXB1pa+aC8hteWVfHmim3U7N5HYoIxriSXs4f14exhfSjJz/C6TAlCzE9yaGYPA+cD25xzI9ttnwT8DkgEHnLO3dHutWcVHP61Yms9lz04h4yUJJ6+8RQG5KR5XZJ0ormllTnranl56RZeW1bFzr0HyOyRxJlDC/jSsD5MHFqgq+XiUDxMcvgIgUH2xw5uMLNEYCbwJaASmG9mLzrnPvGkQomqYf2yePzak7j8oTlc/uAcnr7hFF2bH0Occ3y0aSfPf7yZ15ZVUbN7PxkpiZw9vA/nH9+f0wfnd/spO/zsqIPDzPoBtc65fV0twjn3npmVHLJ5PFDunFvX9nmzgKlAUMFhZjOAGQDFxcVdLVE8MKowm0evHc/XH5rL5Q/OYdaMUyjoqUt1vVRd38RzH23mmYUVrNu+h9TkBL44rA+Tj+/HxKG9FRbdRFdaHI8Dx5rZn51z3w1XQe0MACraPa8ETjKzXsAvgDFm9n3n3O0dvdk59wDwAAS6qiJQn0TB2OJc/njNeK5+eB5XPjSX/7vhZHV7RNn+5lbeWlHNMwsreXfVNlodjCvJ5cYzj+W8Uf10x383dNT/x51zZ1tgdrrhYaynvY5uBXVtg/E3BnUAzVXlC+MH5vHQ1WVc88f5XP/YAh6ffpJ+s42C6vom/jR3E0/O3UTN7n30zUrlponH8rUTixioAe5uLejgMLNzgf8CcghMOXKnc24OgctzI6ESKGr3vBDYEsoBnHMvAS+VlZVdH87CJPpOLc3nzktHc8tTH3PrrI+594oTNc1EhCzcWMsjszfy6tKttDjHxCEFXHVKCWcMKdB3LkBoLY57gSsJjDGcCPzazGY6556KSGUwHxhsZgMJTGsyDbg8lAOoxeEv5x/fn+0N+/jJS5/woxeW8fMLRmpK9jBxzvH2ym3c9+5aFmzcSc/UJK6eUMLXTz5Gl8/KPwklOKqdcx+0/fymmX0IzAW6HBxm9hQwEcg3s0rgx865P5jZzcDrBC7Hfdg5F1LrRi0O/7nm1IFU1Tfx+7+vo6RXBtefMcjrkuJac0srf12ylfveXcuq6gYG5KTxX5OHc3FZkcYupFOhnBkbzOznwE+dc/uBA0BDOIpwzl3WyfZXCCwWdVTU4vCnf//ycVTU7uW/X13BwPwMzh7ex+uS4k5rq+OlJVv4zd9Ws2HHXob0yeTOS0YzeXR/rY0iRxTKlCNPAaOBXsAaoBj4E4GWQEyvAKgbAP2ncX8Ll/z+Q9Zt382zN01gWL8sr0uKCwe7pP7n9VWsrGpgWL8svnP2YM4e1keLHMnnhHvN8VRgJIEQOfgY5JwrOuwbPdCuxXH9mjUxnW1yFKrqmpg6832SEhJ48eZT6aXp2A/ro007+cXLK1i4cSclvdL5l3OGcv6ofgoM6VCXgsPMzB1hJzNLOLikbCxSi8O/llbWcdH9sxlXkstj156kq346ULN7H798dSXPLKykd88efPvsIVxcVqguKTmswwVHMGfOO2Z2i5l97vZrM0sxsy+Y2aPAVeEoVCRUowqz+fnUkXxQvoPfvrna63JiSnNLK498sJ6zfv0uz3+8mRvOHMTb353I5ScVKzSkS4IZHJ8EXAs81XZp7C4glcCVTm8Av3HOLYpUgV2hwfHu4ZJxRSzcuJO73y5nTHEOXzhOg+XLNtfxb88uYcXWek4fnM+PJ4+gtLemqJfwCHVa9WQgH2h0zu2KVFHhpq4q/2s60MJF982monYvL3/r9G67CNS+5hbufquc+/6+ll4ZKfxkyggmjeyr+10kZF3tqvqUc+6Ac25rPIWGdA+pyYncd8WJtDr416cX09La/aYnW1XVwNR7PuCed8r56pgB/O07Z3LuqH4KDQm7LnV0mtkHR95LJDqKe6Xz06kjmLehlvv/vtbrcqLGOccTczYy5Z73qdm9n4e/UcavLx5NdrpW1pPI6Oqtof3DUkWEaIyj+/nqmAG8tXIbv/nbas4YXMCowmyvS4qohqYDfO/ZJby6rIozhhTwvxeP1tTzEnFHbHGY2T1mNsPMTjGznoe8HNP9Ac65l5xzM7Kz/f2Ph3zGzPjFBSPJz+zBrf/3MY37W7wuKWLW1+zhq/fO5o1PqvnBV47jkW+MU2hIVATTVbUEOB64g8C0I+vN7EUz+wVwaJCIeC4nPYU7LxnNuu17+I1PL9F9d9U2ptzzPjt27+Px6eOZccaxupFPoiaYrqpTnHPXHHxiZoUEgmQU8FqkChPpigml+Vw2vog/vL+eKaP7M3KAf1qdf5q7kR/+ZRlD+vTkwavKuu0VZOKdYFociWb204NPnHOVbZMP3kfgfo6YZWaTzeyBuro6r0sRD9w2aRi56Sl8/7mlvrjKyjnH795cw388v4wzhxTw3DcnKDTEE8EEx7XAKWZ23cENZjYamAesiFRh4aAxju4tOz2ZH08eztLNdTwye4PX5XRJa6vjRy8s5zdvruaisYU8cFUZ6Sma9ly8ccQzzznXbGYXAu+a2RYCa4H/B3Cdc+7NSBco0hXnH9+P5z6q5H/fWMWkkX0ZkJPmdUkha211/OD5pcyaX8ENZw7itknH6d4M8VQwV1XdC1wB/Ar4I3ApcJJCQ+KBmfGzC0bS6hy/fHWl1+WEzDnHj19czqz5FdzyhVK+f+4whYZ4LpiuqkUEplH/JpAMDAEeNLNfmNm0CNYmEhaFuelcd9ogXly8hUUVu7wuJ2jOOX761094fM5GbjhjEP/ypSFelyQCBBEczrkHnHM3O+fOdM7lAROA+4F64LxIFygSDjdOPJb8zBT+++UVhLoGjVfufXctf/xgA9ecWsJt56p7SmJHyFOOHLyqyjn3S+fc1yNRVLjoqio5KLNHEt/50hDmbajljU+qvS7niF5ZupX/eX0VU0b350fnD1doSEzx9aT8uqpK2ru0rIjS3pnc8epKDrTE7LpjLK7YxXf+bxFji3P41deOV2hIzPF1cIi0l5SYwA++chzra/Ywa94mr8vpUFVdE9c9toCCnj144KoyUpMTvS5J5J8oOKRbOWtob8qOyeXed9eyrzm25rFqaXXcOutj9uxr5uFvjCNfa6hLjFJwSLdiZtx69mC21jXxzIJKr8v5nHvfKWfu+lp+OnUkQ/poGjiJXQoO6XZOK81nbHEO9727NmbGOhZurOW3b61hyuj+XDR2gNfliByWgkO6HTPj5i+UsnlXIy8v2ep1OdQ1HuBbTy2if04qP//qSA2GS8xTcEi3NHFIb0p7Z/LAe+s8v6/j9ldWUFXfxO+mjSErVav2SeyLu+Awswwze9TMHjSzK7yuR+JTQoIx4/RBfLK1ng/Kd3hWx5x1O5g1v4LrThvI2OJcz+oQCUVMBIeZPWxm28xs2SHbJ5nZKjMrN7Pb2jZfCDzrnLsemBL1YsU3po7pT35mimcz5+5vbuUHzy+lKC+Nb5+t6UQkfsREcACPAJPabzCzRGAmcC4wHLjMzIYDhUBF226xdT2lxJUeSYlcOq6It1dWs3lXY9Q//7EPN7Bu+x5+MmUEaSm6X0PiR0wEh3PuPaD2kM3jgXLn3Drn3H5gFjAVqCQQHnCY+tvWSV9gZgu2b98eibLFBy4bXwzAU3Oje0Pgjt37+N1bazhzSAFnDe0d1c8W6aqYCI5ODOCzlgUEAmMA8BxwkZndB7zU2ZvbJmcsc86VFRQURLZSiVuFuel84bjezJq/KaqX5t79djl797fww/M1TbrEn1gOjo7+Njnn3B7n3DXOuZucc3867AE0yaEE4bLxxdTs3s/fV0WnZbplVyNPzt3ExScWUtpbN/pJ/Inl4KgEito9LwS2eFSL+NgZQwrolZHCcx9H507yu98uB+CWLw6OyueJhFssB8d8YLCZDTSzFGAa8GIoB9DsuBKM5MQEppzQnzc/2cauvfsj+llb6xp5ZkEF08YXxeUytiIQI8FhZk8BHwJDzazSzKY755qBm4HXgRXA08655SEeV11VEpSLxhayv6WVlyJ8J/nD76/HATPOGBTRzxGJpJgIDufcZc65fs65ZOdcoXPuD23bX3HODXHOHeuc+8VRHFctDgnKiP5ZlPbO5OUlkesNrWs8wFPzKjj/+H4U5qZH7HNEIi0mgkPEa2bGuSP7Mm99LTt274vIZ/zf/E3s3tes1obEPV8Hh7qqJBSTRval1RGRpWVbWx1Pzt3E+JI8RvRXC1jim6+DQ11VEorh/bI4plc6ry6rCvuxP1y3gw079nL5ScVhP7ZItPk6OERCYWZMGtmX2eU11DcdCOuxn5y3iZz0ZCaN7BvW44p4wdfBoa4qCdUXj+tDc6tjdnlN2I65a+9+3lhexYVjCrWGuPiCr4NDXVUSqjHFOWT2SOLvq8MXHK8tq+JAi+NCrewnPuHr4BAJVXJiAqeW9uK91dvDtsDTS0u2UNIrnRH9s8JyPBGv+To41FUlR+OMIQVs3tXI2u17unysbQ1NfLh2B5NH99dkhuIbvg4OdVXJ0ThjcGA25fdWd33Sw1eXVtHqYPLo/l0+lkis8HVwiByNorx0SnqlM3tt15eUfXNFNYPyMxjSR7Pgin8oOEQ6UFaSx0ebdnZpnKNxfwtz19cyUQs1ic/4Ojg0xiFHq+yYXGr37GddzdGPc3y4rob9za2cdZwWEhN/8XVwaIxDjlZZSS4ACzfsPOpjvLtqO2nJiYwfmBeuskRigq+DQ+RoDcrPJCc9mQUba4/q/c453l21nQnH9qJHkm76E39RcIh0ICHBOLE4lwUbj67FUbmzkU21ezljiLqpxH8UHCKdOLEkl3Xb91C7J/RVAQ+2VMaVqJtK/EfBIdKJ0YU5AHyypT7k9y7cuJPMHkkM7avLcMV/FBwinTiu7R/9lVVHExy7GFOcQ2KC7hYX//F1cOhyXOmKXpk9KOjZgxVbG0J6X0PTAVZV1TO2ODdClYl4y9fBoctxpauO69sz5BbHoopdtLrPLukV8RtfB4dIVw3rl8Wa6t00t7QG/Z4FG3aSYHBCUU7kChPxkIJD5DCG9evJ/pbWkO4gX76lnkEFmfRMTY5gZSLeUXCIHMZxfQNraKzYGnx31erqBl1NJb6m4BA5jGMLMklKMFZWBTdAvnd/MxU79zKkt4JD/EvBIXIYKUkJDCrIYE317qD2L9+2G+dgaN/MCFcm4p24Cw4zG2RmfzCzZ72uRbqH4rx0KnfuDWrf1W0BM1jrb4iPRTU4zOxhM9tmZssO2T7JzFaZWbmZ3Xa4Yzjn1jnnpke2UpHPFOamU7mzMai1OdZUN5CSlMAxeelRqEzEG9FucTwCTGq/wcwSgZnAucBw4DIzG25mo8zsr4c8tCKORF1hbhq79zVT13jgiPuuqm4IjIskxl1jXiRoSdH8MOfce2ZWcsjm8UC5c24dgJnNAqY6524Hzj/azzKzGcAMgOLi4qM9jAiFuYHWQ0VtIznpKYfdd031bsbpxj/xuVj4tWgAUNHueWXbtg6ZWS8zux8YY2bf72w/59wDzrky51xZQYGmtpajV5ibBnDEcY6GpgNs3tWo8Q3xvai2ODrR0SxwnXYmO+d2ADcGdWCzycDk0tLSoyxNBIraxisqjhAcFbWNAAzMz4h4TSJeioUWRyVQ1O55IbAlHAfWXFUSDtlpyfRMTaJyZ+Nh96uqD7zeLzs1GmWJeCYWgmM+MNjMBppZCjANeDEcB9bsuBIuRbnpVNQevsWxta4JgL4KDvG5aF+O+xTwITDUzCrNbLpzrhm4GXgdWAE87ZxbHo7PU4tDwqUwN+2ILY7quiYSDAoye0SpKhFvRPuqqss62f4K8Eq4P09jHBIuhbnp/GNNDc45zDpenGlrXRO9e6bqUlzxPV+f4WpxSLgU5aXReKCFHYdZf7yqvok+6qaSbsDXwaExDgmX/jmBS3K37mrqdJ+quib6ZSk4xP98HRxqcUi45KQF1taob+r87vGquiYNjEu34OvgEAmXrIPB0cm0Iw1NB2jY16zgkG7B18GhrioJl6wjtDiq6wNdWLqHQ7oDXweHuqokXLJSAxcg1jc2d/h6Vd0+APpqjEO6AV8Hh0i4ZKQkkWCdtzi21gXu8VBXlXQHvg4OdVVJuCQkGD1Tkzsd46hqu2u8j1oc0g34OjjUVSXhlJWWRH1TJ11V9U3kZaSQmpwY5apEos/XwSESTlmHaXHsajxATnpylCsS8YaCQyRIWanJh72Po+OJSET8R8EhEqSstKROr6oS6U58HRwaHJdwOlKLQ6S78HVwaHBcwikrrfMxDpHuxNfBIRJOWanJ7NnfQnNLq9eliHhKwSESpKy0wN3jDZ1ckivSXSg4RIKUlXrkGXJFugMFh0iQPpshVy0O6d58HRy6qkrC6dOJDtXikG7O18Ghq6oknA62OBoUHNLN+To4RMJJXVUiAQoOkSCpq0okQMEhEqSMlCTMOl8+VqS7UHCIBCkhwejZo/Op1UW6CwWHSAg07YiIgkMkJFmpydQpOKSbi7vgMLMLzOxBM3vBzM7xuh7pXnokJ7Bfc1VJNxfV4DCzh81sm5ktO2T7JDNbZWblZnbb4Y7hnPuLc+564BvApREsV0REOpAU5c97BLgHeOzgBjNLBGYCXwIqgflm9iKQCNx+yPuvdc5ta/v5P9veJyIiURTV4HDOvWdmJYdsHg+UO+fWAZjZLGCqc+524PxDj2FmBtwBvOqc+6izzzKzGcAMgOLi4vD8AUREJOotjo4MACraPa8ETjrM/rcAZwPZZlbqnLu/o52ccw+Y2VZgckpKyolhq1ZEpJuLhcFx62Cb62xn59xdzrkTnXM3dhYa7fbVXFUiImEWC8FRCRS1e14IbAnHgTU7rohI+MVCcMwHBpvZQDNLAaYBL4bjwGpxiIiEX7Qvx30K+BAYamaVZjbdOdcM3Ay8DqwAnnbOLQ/T56nFISISZtG+quqyTra/ArwSgc97CXiprKzs+nAfW0SkuzLnOh2H9g0z2w5s9LqODuQDNV4X0QHVFbpYrU11hSZW64Lo13aMc66goxe6RXDEKjNb4Jwr87qOQ6mu0MVqbaorNLFaF8RWbbEwOC4iInFEwSEiIiFRcHjrAa8L6ITqCl2s1qa6QhOrdUEM1aYxDhERCYlaHCIiEhIFh4iIhETBEQFHWpjKzK4wsyVtj9lmNrrdaxvMbKmZLTKzBR7UNtHM6to+f5GZ/SjY90a4rn9rV9MyM2sxs7y21yLynXW28Fi7183M7mqreYmZjQ32zxOF2jw5x4Koy6vz60h1Rf38ajt2kZm9Y2YrzGy5md3awT6enWedcs7pEcYHgQWo1gKDgBRgMTD8kH0mALltP58LzG332gYg38PaJgJ/PZr3RrKuQ/afDLwd6e8MOAMYCyzr5PWvAK8SmOH55IP/HyP5XYVQm1fn2JHqivr5FUxdXpxfbcfuB4xt+7knsLqDv5OenWedPdTiCL9PF6Zyzu0HZgFT2+/gnJvtnNvZ9nQOgRmBY6K2CL033Me+DHgqTJ/dKefce0DtYXaZCjzmAuYAOWbWj8h+V0HV5tU5FsR31pmIfmch1hWV8wvAObfVtS1I55xrIDBf34BDdvPsPOuMgiP8OlqY6tATob3pBH6bOMgBb5jZQgusYuhFbaeY2WIze9XMRoT43kjWhZmlA5OAP7fbHMnv7HA6qzuS39XRiOY5Foxon19B8/L8ssDqqGOAuYe8FHPnWSysAOg3QS9MZWZnEfhLfVq7zac657aYWW/gb2a2su23pWjV9hGBOWp2m9lXgL8Ag4N8byTrOmgy8IFzrv1vj5H8zg6ns7oj+V2FxINz7Ei8OL9C4cn5ZWaZBMLq2865+kNf7uAtnp5nanGEX1ALU5nZ8cBDBNZX33Fwu3NuS9t/twHPE2iORq0251y9c25328+vAMlmlh/MeyNZVzvTOKQbIcLf2eF0Vnckv6ugeXSOHZZH51coon5+mVkygdD4k3PuuQ52ib3zLBoDKd3pQaAVtw4YyGcDViMO2acYKAcmHLI9A+jZ7ufZwKQo19aXz24MHQ9sIvCbzRHfG8m62vbLJtBPnRHF76yEzgd6z+Pzg5bzQvnzRLg2T86xIOqK+vkVTF0enl8GPAb89jD7eHqedfRQV1WYOeeazezgwlSJwMPOueVmdmPb6/cDPwJ6AfeaGUCzC8x62Qd4vm1bEvCkc+61KNf2NeAmM2sGGoFpLnCWdvjeKNYF8FXgDefcnnZvj9h3ZoGFxyYC+WZWCfwYSG5X0ysErngpB/YC1xzuzxOOmkKozZNzLIi6on5+BVkXRPn8anMq8HVgqZktatv2AwLB7/l51hlNOSIiIiHRGIeIiIREwSEiIiFRcIiISEgUHCIiEhIFh4iIhETBISIiIVFwiIhISBQcIh4ws9Fm9p6ZfWJmrWbmzOwnXtclEgzdACgSZWaWCiwCrnLOzTOznwGpwPec/kJKHFCLQyT6zgY+cs7Na3u+BMhTaEi8UHCIRN9IYGm752MJTDcuEhc0yaFI9O0AvgBgZkOACwks9SoSFzTGIRJlbYv2PEVgOuwa4F9c2/KhIvFAwSEiIiHRGIeIiIREwSEiIiFRcIiISEgUHCIiEhIFh4iIhETBISIiIVFwiIhISP4/xh4mLGecw7wAAAAASUVORK5CYII=\n", |
| 147 | + "text/plain": [ |
| 148 | + "<Figure size 432x288 with 1 Axes>" |
| 149 | + ] |
| 150 | + }, |
| 151 | + "metadata": { |
| 152 | + "needs_background": "light" |
| 153 | + }, |
| 154 | + "output_type": "display_data" |
| 155 | + } |
| 156 | + ], |
| 157 | + "source": [ |
| 158 | + "plt.semilogy(σs, divs);\n", |
| 159 | + "plt.xlabel(r'$\\sigma$');\n", |
| 160 | + "plt.ylabel(r'$KL(p, q)$');" |
| 161 | + ] |
| 162 | + }, |
| 163 | + { |
| 164 | + "cell_type": "markdown", |
| 165 | + "id": "92d8aece-f1e3-447a-a387-52a4d2d1658e", |
| 166 | + "metadata": {}, |
| 167 | + "source": [ |
| 168 | + "For $\\sigma = 1.0$ the distributions $p$ and $q$ are identical, and the Kullback-Leibler divergence is 0." |
| 169 | + ] |
| 170 | + } |
| 171 | + ], |
| 172 | + "metadata": { |
| 173 | + "kernelspec": { |
| 174 | + "display_name": "Python 3 (ipykernel)", |
| 175 | + "language": "python", |
| 176 | + "name": "python3" |
| 177 | + }, |
| 178 | + "language_info": { |
| 179 | + "codemirror_mode": { |
| 180 | + "name": "ipython", |
| 181 | + "version": 3 |
| 182 | + }, |
| 183 | + "file_extension": ".py", |
| 184 | + "mimetype": "text/x-python", |
| 185 | + "name": "python", |
| 186 | + "nbconvert_exporter": "python", |
| 187 | + "pygments_lexer": "ipython3", |
| 188 | + "version": "3.9.7" |
| 189 | + } |
| 190 | + }, |
| 191 | + "nbformat": 4, |
| 192 | + "nbformat_minor": 5 |
| 193 | +} |
0 commit comments