diff --git a/Notebooks/torch_trainer.ipynb b/Notebooks/torch_trainer.ipynb new file mode 100644 index 00000000..96e2a185 --- /dev/null +++ b/Notebooks/torch_trainer.ipynb @@ -0,0 +1,1767 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 30, + "id": "ea344379", + "metadata": {}, + "outputs": [], + "source": [ + "import pandas as pd\n", + "import numpy as np\n", + "import os\n", + "import glob\n", + "import PIL.Image as Image\n", + "import PIL" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "132fe8c4", + "metadata": {}, + "outputs": [], + "source": [ + "import SciServer.CasJobs as CasJobs # query with CasJobs, the primary database for the SDSS\n", + "import SciServer.SkyServer as SkyServer # show individual objects through SkyServer\n", + "import SciServer.SciDrive" + ] + }, + { + "cell_type": "markdown", + "id": "e9ddf1e5", + "metadata": {}, + "source": [ + "# 1. Acquire data\n", + "\n", + "`PCC.cat` has the ra, dec, label" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "id": "8f2dc168", + "metadata": {}, + "outputs": [], + "source": [ + "df0 = pd.read_fwf('../PCC_cat.txt', header=None)\n" + ] + }, + { + "cell_type": "markdown", + "id": "ff2840b0", + "metadata": {}, + "source": [ + "# Bright objects\n", + "Unfortunately training the model on incredibly faint images had produced poor results. Our first attempt at rectifying this was choosing the brightest images. Since magnitudes go backwards, we choose a $r < 19.4$ magnitude" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "id": "f45a3e60", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
0123456789...1617181920label222324files
5NaNPCC-000649.238841.463119.030.000.690.0119.281.35...-0.11NaNNaNNaNNaNLikely merging systemNaNNaNhttp://dc.zah.uni-heidelberg.de/pcc/q/stamp/dl...sdss_ra=49.2388_dec=41.4631.png
6NaNPCC-000749.239241.421519.260.030.970.0520.763.79...0.680.880.66NaNNaNLikely background ETG or unresolved sourceNaNNaNhttp://dc.zah.uni-heidelberg.de/pcc/q/stamp/dl...sdss_ra=49.2392_dec=41.4215.png
7NaNPCC-000849.241141.499118.510.011.780.0220.952.50...0.820.930.66NaNNaNCluster or background LTGNaNNaNhttp://dc.zah.uni-heidelberg.de/pcc/q/stamp/dl...sdss_ra=49.2411_dec=41.4991.png
10NaNPCC-001149.242041.445419.190.011.510.0221.951.02...0.590.700.47NaNNaNCluster or background LTGNaNNaNhttp://dc.zah.uni-heidelberg.de/pcc/q/stamp/dl...sdss_ra=49.242_dec=41.4454.png
25NaNPCC-002649.246641.445119.340.012.040.0221.311.20...0.850.980.75NaNNaNLikely cluster or background edge-on disk galaxyNaNNaNhttp://dc.zah.uni-heidelberg.de/pcc/q/stamp/dl...sdss_ra=49.2466_dec=41.4451.png
..................................................................
5386NaNPCC-538749.998541.385618.600.012.840.0622.782.45...0.700.820.65NaNNaNCluster or background LTGNaNNaNhttp://dc.zah.uni-heidelberg.de/pcc/q/stamp/dl...sdss_ra=49.9985_dec=41.3856.png
5407NaNPCC-540850.001841.680618.160.000.160.0015.853.30...0.12NaNNaNNaNNaNLikely background ETG or unresolved sourceNaNNaNhttp://dc.zah.uni-heidelberg.de/pcc/q/stamp/dl...sdss_ra=50.0018_dec=41.6806.png
5416NaNPCC-541750.002841.338418.840.011.270.0220.742.42...0.750.780.71NaNNaNLikely background ETG or unresolved sourceNaNNaNhttp://dc.zah.uni-heidelberg.de/pcc/q/stamp/dl...sdss_ra=50.0028_dec=41.3384.png
5422NaNPCC-542350.004041.341017.500.011.860.0220.454.00...0.760.930.69NaNNaNLikely background ETG or unresolved sourceNaNNaNhttp://dc.zah.uni-heidelberg.de/pcc/q/stamp/dl...sdss_ra=50.004_dec=41.341.png
5427NaNPCC-542850.005141.655717.830.012.820.0421.882.69...0.660.730.56NaNNaNCluster or background LTGNaNNaNhttp://dc.zah.uni-heidelberg.de/pcc/q/stamp/dl...sdss_ra=50.0051_dec=41.6557.png
\n", + "

272 rows × 26 columns

\n", + "
" + ], + "text/plain": [ + " 0 1 2 3 4 5 6 7 8 9 \\\n", + "5 NaN PCC-0006 49.2388 41.4631 19.03 0.00 0.69 0.01 19.28 1.35 \n", + "6 NaN PCC-0007 49.2392 41.4215 19.26 0.03 0.97 0.05 20.76 3.79 \n", + "7 NaN PCC-0008 49.2411 41.4991 18.51 0.01 1.78 0.02 20.95 2.50 \n", + "10 NaN PCC-0011 49.2420 41.4454 19.19 0.01 1.51 0.02 21.95 1.02 \n", + "25 NaN PCC-0026 49.2466 41.4451 19.34 0.01 2.04 0.02 21.31 1.20 \n", + "... .. ... ... ... ... ... ... ... ... ... \n", + "5386 NaN PCC-5387 49.9985 41.3856 18.60 0.01 2.84 0.06 22.78 2.45 \n", + "5407 NaN PCC-5408 50.0018 41.6806 18.16 0.00 0.16 0.00 15.85 3.30 \n", + "5416 NaN PCC-5417 50.0028 41.3384 18.84 0.01 1.27 0.02 20.74 2.42 \n", + "5422 NaN PCC-5423 50.0040 41.3410 17.50 0.01 1.86 0.02 20.45 4.00 \n", + "5427 NaN PCC-5428 50.0051 41.6557 17.83 0.01 2.82 0.04 21.88 2.69 \n", + "\n", + " ... 16 17 18 19 20 \\\n", + "5 ... -0.11 NaN NaN NaN NaN \n", + "6 ... 0.68 0.88 0.66 NaN NaN \n", + "7 ... 0.82 0.93 0.66 NaN NaN \n", + "10 ... 0.59 0.70 0.47 NaN NaN \n", + "25 ... 0.85 0.98 0.75 NaN NaN \n", + "... ... ... ... ... ... ... \n", + "5386 ... 0.70 0.82 0.65 NaN NaN \n", + "5407 ... 0.12 NaN NaN NaN NaN \n", + "5416 ... 0.75 0.78 0.71 NaN NaN \n", + "5422 ... 0.76 0.93 0.69 NaN NaN \n", + "5427 ... 0.66 0.73 0.56 NaN NaN \n", + "\n", + " label 22 23 \\\n", + "5 Likely merging system NaN NaN \n", + "6 Likely background ETG or unresolved source NaN NaN \n", + "7 Cluster or background LTG NaN NaN \n", + "10 Cluster or background LTG NaN NaN \n", + "25 Likely cluster or background edge-on disk galaxy NaN NaN \n", + "... ... ... ... \n", + "5386 Cluster or background LTG NaN NaN \n", + "5407 Likely background ETG or unresolved source NaN NaN \n", + "5416 Likely background ETG or unresolved source NaN NaN \n", + "5422 Likely background ETG or unresolved source NaN NaN \n", + "5427 Cluster or background LTG NaN NaN \n", + "\n", + " 24 \\\n", + "5 http://dc.zah.uni-heidelberg.de/pcc/q/stamp/dl... \n", + "6 http://dc.zah.uni-heidelberg.de/pcc/q/stamp/dl... \n", + "7 http://dc.zah.uni-heidelberg.de/pcc/q/stamp/dl... \n", + "10 http://dc.zah.uni-heidelberg.de/pcc/q/stamp/dl... \n", + "25 http://dc.zah.uni-heidelberg.de/pcc/q/stamp/dl... \n", + "... ... \n", + "5386 http://dc.zah.uni-heidelberg.de/pcc/q/stamp/dl... \n", + "5407 http://dc.zah.uni-heidelberg.de/pcc/q/stamp/dl... \n", + "5416 http://dc.zah.uni-heidelberg.de/pcc/q/stamp/dl... \n", + "5422 http://dc.zah.uni-heidelberg.de/pcc/q/stamp/dl... \n", + "5427 http://dc.zah.uni-heidelberg.de/pcc/q/stamp/dl... \n", + "\n", + " files \n", + "5 sdss_ra=49.2388_dec=41.4631.png \n", + "6 sdss_ra=49.2392_dec=41.4215.png \n", + "7 sdss_ra=49.2411_dec=41.4991.png \n", + "10 sdss_ra=49.242_dec=41.4454.png \n", + "25 sdss_ra=49.2466_dec=41.4451.png \n", + "... ... \n", + "5386 sdss_ra=49.9985_dec=41.3856.png \n", + "5407 sdss_ra=50.0018_dec=41.6806.png \n", + "5416 sdss_ra=50.0028_dec=41.3384.png \n", + "5422 sdss_ra=50.004_dec=41.341.png \n", + "5427 sdss_ra=50.0051_dec=41.6557.png \n", + "\n", + "[272 rows x 26 columns]" + ] + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "brightDF = df0[df0[4] <= 19.4].copy()\n", + "\n", + "brightDF['files'] = (\n", + " \"sdss_ra=\" + brightDF[2].astype(str) +\n", + " \"_dec=\" + brightDF[3].astype(str) +\n", + " \".png\"\n", + ")\n", + "\n", + "brightDF = brightDF.rename(columns={21: \"label\"})\n", + "brightDF" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "id": "393bafa6", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
0123456789...17181920label222324filesbinary_label
0NaNPCC-292849.687641.226017.820.011.870.0220.843.10...0.270.25NaNNaNCluster or background LTGNaNNaNhttp://dc.zah.uni-heidelberg.de/pcc/q/stamp/dl...sdss_ra=49.6876_dec=41.226.png0
1NaNPCC-340849.746341.637719.340.001.400.0121.933.27...1.400.76NaNNaNLikely background ETG or unresolved sourceNaNNaNhttp://dc.zah.uni-heidelberg.de/pcc/q/stamp/dl...sdss_ra=49.7463_dec=41.6377.png0
2NaNPCC-496949.943741.378618.530.001.700.0021.370.52...0.320.32NaNNaNCluster or background LTGNaNNaNhttp://dc.zah.uni-heidelberg.de/pcc/q/stamp/dl...sdss_ra=49.9437_dec=41.3786.png0
3NaNPCC-158149.508241.406919.000.001.410.0121.271.63...0.890.71NaNNaNCluster or background LTGNaNNaNhttp://dc.zah.uni-heidelberg.de/pcc/q/stamp/dl...sdss_ra=49.5082_dec=41.4069.png0
4NaNPCC-057149.343141.598719.090.001.900.0121.501.13...1.040.78NaNNaNCluster or background LTGNaNNaNhttp://dc.zah.uni-heidelberg.de/pcc/q/stamp/dl...sdss_ra=49.3431_dec=41.5987.png0
..................................................................
247NaNPCC-504749.952341.557918.070.005.000.0323.171.27...NaN0.28NaNNaNLikely dE/ETGcluster candidateUnsureabout nucleationhttp://dc.zah.uni-heidelberg.de/pcc/q/stamp/dl...sdss_ra=49.9523_dec=41.5579.png1
248NaNPCC-099349.414941.561218.910.012.950.0223.201.26...0.630.46NaNNaNLikely dE/ETGcluster candidateUnsureabout nucleationhttp://dc.zah.uni-heidelberg.de/pcc/q/stamp/dl...sdss_ra=49.4149_dec=41.5612.png1
249NaNPCC-455149.890241.553617.050.001.090.0018.992.67...0.600.65NaNNaNLikely dE/ETGcluster candidater Brigt central sourcehttp://dc.zah.uni-heidelberg.de/pcc/q/stamp/dl...sdss_ra=49.8902_dec=41.5536.png1
250NaNPCC-295949.692541.404917.430.007.620.0423.071.78...0.640.52NaNNaNLikely dE/ETGcluster candidateUnsureabout nucleationhttp://dc.zah.uni-heidelberg.de/pcc/q/stamp/dl...sdss_ra=49.6925_dec=41.4049.png1
251NaNPCC-487649.932641.457015.070.002.950.0019.343.75...0.740.68NaNNaNLikely dE/ETGcluster candidater Brigt central sourcehttp://dc.zah.uni-heidelberg.de/pcc/q/stamp/dl...sdss_ra=49.9326_dec=41.457.png1
\n", + "

252 rows × 27 columns

\n", + "
" + ], + "text/plain": [ + " 0 1 2 3 4 5 6 7 8 9 \\\n", + "0 NaN PCC-2928 49.6876 41.2260 17.82 0.01 1.87 0.02 20.84 3.10 \n", + "1 NaN PCC-3408 49.7463 41.6377 19.34 0.00 1.40 0.01 21.93 3.27 \n", + "2 NaN PCC-4969 49.9437 41.3786 18.53 0.00 1.70 0.00 21.37 0.52 \n", + "3 NaN PCC-1581 49.5082 41.4069 19.00 0.00 1.41 0.01 21.27 1.63 \n", + "4 NaN PCC-0571 49.3431 41.5987 19.09 0.00 1.90 0.01 21.50 1.13 \n", + ".. .. ... ... ... ... ... ... ... ... ... \n", + "247 NaN PCC-5047 49.9523 41.5579 18.07 0.00 5.00 0.03 23.17 1.27 \n", + "248 NaN PCC-0993 49.4149 41.5612 18.91 0.01 2.95 0.02 23.20 1.26 \n", + "249 NaN PCC-4551 49.8902 41.5536 17.05 0.00 1.09 0.00 18.99 2.67 \n", + "250 NaN PCC-2959 49.6925 41.4049 17.43 0.00 7.62 0.04 23.07 1.78 \n", + "251 NaN PCC-4876 49.9326 41.4570 15.07 0.00 2.95 0.00 19.34 3.75 \n", + "\n", + " ... 17 18 19 20 label \\\n", + "0 ... 0.27 0.25 NaN NaN Cluster or background LTG \n", + "1 ... 1.40 0.76 NaN NaN Likely background ETG or unresolved source \n", + "2 ... 0.32 0.32 NaN NaN Cluster or background LTG \n", + "3 ... 0.89 0.71 NaN NaN Cluster or background LTG \n", + "4 ... 1.04 0.78 NaN NaN Cluster or background LTG \n", + ".. ... ... ... ... ... ... \n", + "247 ... NaN 0.28 NaN NaN Likely dE/ETGcluster candidate \n", + "248 ... 0.63 0.46 NaN NaN Likely dE/ETGcluster candidate \n", + "249 ... 0.60 0.65 NaN NaN Likely dE/ETGcluster candidate \n", + "250 ... 0.64 0.52 NaN NaN Likely dE/ETGcluster candidate \n", + "251 ... 0.74 0.68 NaN NaN Likely dE/ETGcluster candidate \n", + "\n", + " 22 23 \\\n", + "0 NaN NaN \n", + "1 NaN NaN \n", + "2 NaN NaN \n", + "3 NaN NaN \n", + "4 NaN NaN \n", + ".. ... ... \n", + "247 Unsure about nucleation \n", + "248 Unsure about nucleation \n", + "249 r Brig t central source \n", + "250 Unsure about nucleation \n", + "251 r Brig t central source \n", + "\n", + " 24 \\\n", + "0 http://dc.zah.uni-heidelberg.de/pcc/q/stamp/dl... \n", + "1 http://dc.zah.uni-heidelberg.de/pcc/q/stamp/dl... \n", + "2 http://dc.zah.uni-heidelberg.de/pcc/q/stamp/dl... \n", + "3 http://dc.zah.uni-heidelberg.de/pcc/q/stamp/dl... \n", + "4 http://dc.zah.uni-heidelberg.de/pcc/q/stamp/dl... \n", + ".. ... \n", + "247 http://dc.zah.uni-heidelberg.de/pcc/q/stamp/dl... \n", + "248 http://dc.zah.uni-heidelberg.de/pcc/q/stamp/dl... \n", + "249 http://dc.zah.uni-heidelberg.de/pcc/q/stamp/dl... \n", + "250 http://dc.zah.uni-heidelberg.de/pcc/q/stamp/dl... \n", + "251 http://dc.zah.uni-heidelberg.de/pcc/q/stamp/dl... \n", + "\n", + " files binary_label \n", + "0 sdss_ra=49.6876_dec=41.226.png 0 \n", + "1 sdss_ra=49.7463_dec=41.6377.png 0 \n", + "2 sdss_ra=49.9437_dec=41.3786.png 0 \n", + "3 sdss_ra=49.5082_dec=41.4069.png 0 \n", + "4 sdss_ra=49.3431_dec=41.5987.png 0 \n", + ".. ... ... \n", + "247 sdss_ra=49.9523_dec=41.5579.png 1 \n", + "248 sdss_ra=49.4149_dec=41.5612.png 1 \n", + "249 sdss_ra=49.8902_dec=41.5536.png 1 \n", + "250 sdss_ra=49.6925_dec=41.4049.png 1 \n", + "251 sdss_ra=49.9326_dec=41.457.png 1 \n", + "\n", + "[252 rows x 27 columns]" + ] + }, + "execution_count": 24, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "L_BG_LTG = \"Cluster or background LTG\"\n", + "BG_ETG = \"Likely background ETG or unresolved source\"\n", + "EDGE_DISK = \"Likely cluster or background edge-on disk galaxy\"\n", + "dE_ETG = \"Likely dE/ETGcluster candidate\"\n", + "MERGING = \"Likely merging system\"\n", + "POSS_dE = \"Possible dE/ETGcluster candidate\"\n", + "WEAK_BG = \"background galaxy with possibly weak substructure\"\n", + "desired = {\n", + " L_BG_LTG: 384,\n", + " BG_ETG: 400,\n", + " EDGE_DISK: 400,\n", + " dE_ETG: None,\n", + " MERGING: None,\n", + " POSS_dE: None,\n", + " WEAK_BG: None,\n", + "}\n", + "\n", + "groups = []\n", + "for lbl, group in brightDF.groupby(\"label\"):\n", + " n = desired.get(lbl) # type: ignore\n", + " if n is None:\n", + " groups.append(group)\n", + " else:\n", + " k = min(n, len(group))\n", + " groups.append(group.sample(n=k, random_state=42))\n", + "\n", + "downSampleDf0 = pd.concat(groups, ignore_index=True)\n", + "\n", + "# old labels 0,1,2,6 -> new label 0 : background, \n", + "# old labels 3 + 5 -> new label 1 : cluster galaxy\n", + "map_to_binary = {\n", + " L_BG_LTG: 0,\n", + " BG_ETG: 0,\n", + " EDGE_DISK: 0,\n", + " WEAK_BG: 0,\n", + " \n", + " dE_ETG: 1,\n", + " POSS_dE: 1,\n", + " MERGING: 1, # if this belongs with the positives\n", + "}\n", + "\n", + "downSampleDf1 = downSampleDf0.assign(\n", + " binary_label = downSampleDf0['label'].map(map_to_binary)\n", + ")\n", + "\n", + "# rebalance to equalize classes 0 vs. 1\n", + "df0_bin = downSampleDf1[downSampleDf1['binary_label'] == 0]\n", + "df1_bin = downSampleDf1[downSampleDf1['binary_label'] == 1]\n", + "\n", + "min_size = min(len(df0_bin), len(df1_bin))\n", + "\n", + "downSampleDf1_balanced = pd.concat([\n", + " df0_bin.sample(n=min_size, random_state=42),\n", + " df1_bin.sample(n=min_size, random_state=42)\n", + "]).reset_index(drop=True)\n", + "\n", + "downSampleDf1_balanced\n" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "id": "d57144a0", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
0123456789...1920label222324filesbinary_labelradec
0NaNPCC-260549.646941.450518.420.001.930.0121.053.24...NaNNaNCluster or background LTGNaNNaNhttp://dc.zah.uni-heidelberg.de/pcc/q/stamp/dl...sdss_ra=49.6469_dec=41.4505.png049.646941.4505
1NaNPCC-076649.375741.313215.980.000.260.0221.821.58...NaNNaNCluster or background LTGNaNNaNhttp://dc.zah.uni-heidelberg.de/pcc/q/stamp/dl...sdss_ra=49.3757_dec=41.3132.png049.375741.3132
2NaNPCC-016849.275841.541517.690.004.860.0221.931.66...NaNNaNCluster or background LTGNaNNaNhttp://dc.zah.uni-heidelberg.de/pcc/q/stamp/dl...sdss_ra=49.2758_dec=41.5415.png049.275841.5415
3NaNPCC-051449.332941.499918.530.001.950.0021.440.51...NaNNaNCluster or background LTGNaNNaNhttp://dc.zah.uni-heidelberg.de/pcc/q/stamp/dl...sdss_ra=49.3329_dec=41.4999.png049.332941.4999
4NaNPCC-133649.470941.346519.190.002.220.0122.070.80...NaNNaNCluster or background LTGNaNNaNhttp://dc.zah.uni-heidelberg.de/pcc/q/stamp/dl...sdss_ra=49.4709_dec=41.3465.png049.470941.3465
..................................................................
267NaNPCC-219549.602841.536718.460.002.240.0121.881.99...Clusterorbackground galaxy with possibly weak substructureNaNNaNhttp://dc.zah.uni-heidelberg.de/pcc/q/stamp/dl...sdss_ra=49.6028_dec=41.5367.png049.602841.5367
268NaNPCC-297149.694741.587619.330.000.840.0020.791.08...Clusterorbackground galaxy with possibly weak substructureNaNNaNhttp://dc.zah.uni-heidelberg.de/pcc/q/stamp/dl...sdss_ra=49.6947_dec=41.5876.png049.694741.5876
269NaNPCC-328649.731441.556318.950.003.700.0222.711.10...Clusterorbackground galaxy with possibly weak substructureNaNNaNhttp://dc.zah.uni-heidelberg.de/pcc/q/stamp/dl...sdss_ra=49.7314_dec=41.5563.png049.731441.5563
270NaNPCC-346249.754541.302718.940.000.700.0019.662.14...Clusterorbackground galaxy with possibly weak substructureNaNNaNhttp://dc.zah.uni-heidelberg.de/pcc/q/stamp/dl...sdss_ra=49.7545_dec=41.3027.png049.754541.3027
271NaNPCC-496749.943541.603319.310.010.900.0120.872.55...Clusterorbackground galaxy with possibly weak substructureNaNNaNhttp://dc.zah.uni-heidelberg.de/pcc/q/stamp/dl...sdss_ra=49.9435_dec=41.6033.png049.943541.6033
\n", + "

272 rows × 29 columns

\n", + "
" + ], + "text/plain": [ + " 0 1 2 3 4 5 6 7 8 9 \\\n", + "0 NaN PCC-2605 49.6469 41.4505 18.42 0.00 1.93 0.01 21.05 3.24 \n", + "1 NaN PCC-0766 49.3757 41.3132 15.98 0.00 0.26 0.02 21.82 1.58 \n", + "2 NaN PCC-0168 49.2758 41.5415 17.69 0.00 4.86 0.02 21.93 1.66 \n", + "3 NaN PCC-0514 49.3329 41.4999 18.53 0.00 1.95 0.00 21.44 0.51 \n", + "4 NaN PCC-1336 49.4709 41.3465 19.19 0.00 2.22 0.01 22.07 0.80 \n", + ".. .. ... ... ... ... ... ... ... ... ... \n", + "267 NaN PCC-2195 49.6028 41.5367 18.46 0.00 2.24 0.01 21.88 1.99 \n", + "268 NaN PCC-2971 49.6947 41.5876 19.33 0.00 0.84 0.00 20.79 1.08 \n", + "269 NaN PCC-3286 49.7314 41.5563 18.95 0.00 3.70 0.02 22.71 1.10 \n", + "270 NaN PCC-3462 49.7545 41.3027 18.94 0.00 0.70 0.00 19.66 2.14 \n", + "271 NaN PCC-4967 49.9435 41.6033 19.31 0.01 0.90 0.01 20.87 2.55 \n", + "\n", + " ... 19 20 label \\\n", + "0 ... NaN NaN Cluster or background LTG \n", + "1 ... NaN NaN Cluster or background LTG \n", + "2 ... NaN NaN Cluster or background LTG \n", + "3 ... NaN NaN Cluster or background LTG \n", + "4 ... NaN NaN Cluster or background LTG \n", + ".. ... ... ... ... \n", + "267 ... Cluster or background galaxy with possibly weak substructure \n", + "268 ... Cluster or background galaxy with possibly weak substructure \n", + "269 ... Cluster or background galaxy with possibly weak substructure \n", + "270 ... Cluster or background galaxy with possibly weak substructure \n", + "271 ... Cluster or background galaxy with possibly weak substructure \n", + "\n", + " 22 23 24 \\\n", + "0 NaN NaN http://dc.zah.uni-heidelberg.de/pcc/q/stamp/dl... \n", + "1 NaN NaN http://dc.zah.uni-heidelberg.de/pcc/q/stamp/dl... \n", + "2 NaN NaN http://dc.zah.uni-heidelberg.de/pcc/q/stamp/dl... \n", + "3 NaN NaN http://dc.zah.uni-heidelberg.de/pcc/q/stamp/dl... \n", + "4 NaN NaN http://dc.zah.uni-heidelberg.de/pcc/q/stamp/dl... \n", + ".. ... ... ... \n", + "267 NaN NaN http://dc.zah.uni-heidelberg.de/pcc/q/stamp/dl... \n", + "268 NaN NaN http://dc.zah.uni-heidelberg.de/pcc/q/stamp/dl... \n", + "269 NaN NaN http://dc.zah.uni-heidelberg.de/pcc/q/stamp/dl... \n", + "270 NaN NaN http://dc.zah.uni-heidelberg.de/pcc/q/stamp/dl... \n", + "271 NaN NaN http://dc.zah.uni-heidelberg.de/pcc/q/stamp/dl... \n", + "\n", + " files binary_label ra dec \n", + "0 sdss_ra=49.6469_dec=41.4505.png 0 49.6469 41.4505 \n", + "1 sdss_ra=49.3757_dec=41.3132.png 0 49.3757 41.3132 \n", + "2 sdss_ra=49.2758_dec=41.5415.png 0 49.2758 41.5415 \n", + "3 sdss_ra=49.3329_dec=41.4999.png 0 49.3329 41.4999 \n", + "4 sdss_ra=49.4709_dec=41.3465.png 0 49.4709 41.3465 \n", + ".. ... ... ... ... \n", + "267 sdss_ra=49.6028_dec=41.5367.png 0 49.6028 41.5367 \n", + "268 sdss_ra=49.6947_dec=41.5876.png 0 49.6947 41.5876 \n", + "269 sdss_ra=49.7314_dec=41.5563.png 0 49.7314 41.5563 \n", + "270 sdss_ra=49.7545_dec=41.3027.png 0 49.7545 41.3027 \n", + "271 sdss_ra=49.9435_dec=41.6033.png 0 49.9435 41.6033 \n", + "\n", + "[272 rows x 29 columns]" + ] + }, + "execution_count": 25, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "PCC_list = downSampleDf1['files'].to_list()\n", + "columnLength = len(PCC_list)\n", + "ra_list = [None] * columnLength\n", + "dec_list = [None] * columnLength\n", + "for i in range(columnLength):\n", + " splitted = PCC_list[i].split('=')\n", + " ra_list[i] = float(splitted[1][:-4])\n", + " dec_list[i] = float(splitted[2][:-4])\n", + "\n", + "downSampleDf1['ra'] = ra_list\n", + "downSampleDf1['dec'] = dec_list\n", + "downSampleDf1" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "id": "f82a0132", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "binary_label\n", + "0 146\n", + "1 126\n", + "Name: count, dtype: int64\n", + "binary_label\n", + "0 126\n", + "1 126\n", + "Name: count, dtype: int64\n" + ] + } + ], + "source": [ + "print(downSampleDf1['binary_label'].value_counts())\n", + "print(downSampleDf1_balanced['binary_label'].value_counts())\n" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "id": "14e81668", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "specbgs: 117\n", + "specmems: 116\n", + "total specs 233\n", + "True\n", + "pccbgs: 146\n", + "pccmems: 126\n", + "total pccs: 272\n", + "True\n", + "Total: 505 \n" + ] + } + ], + "source": [ + "searchDf = pd.read_csv('../Sheets/SpecSearchNoCuts.csv')\n", + "searchDf = searchDf.drop_duplicates(subset = 'objID') # sometimes SDSS has some objID duplicates\n", + "# pcc_crossDf = pcc_crossDf.drop_duplicates(subset = 'objID')\n", + "\n", + "## Members 0.01 < spec-z < 0.033, else are NonMembers \n", + "specMembers = searchDf['z'].between(0.01, 0.033)\n", + "specNonMembers = np.invert(specMembers)\n", + "\n", + "searchDf_specMembers = searchDf.loc[specMembers].copy()\n", + "searchDf_specMembers['binary_label'] = np.ones(int(searchDf_specMembers.shape[0]), dtype = int)\n", + "\n", + "searchDf_specNonMembers = searchDf.loc[specNonMembers].copy()\n", + "searchDf_specNonMembers['binary_label'] = np.zeros(int(searchDf_specNonMembers.shape[0]), dtype = int)\n", + "\n", + "pcc_nonMembers = downSampleDf1.loc[(downSampleDf1['binary_label'] == 0)]\n", + "pcc_Members = downSampleDf1.loc[(downSampleDf1['binary_label'] == 1)]\n", + "\n", + "## sanity checks\n", + "print('specbgs:', searchDf_specNonMembers.shape[0])\n", + "print('specmems:', searchDf_specMembers.shape[0])\n", + "print('total specs', searchDf_specNonMembers.shape[0] + searchDf_specMembers.shape[0])\n", + "print(searchDf_specNonMembers.shape[0] + searchDf_specMembers.shape[0] == searchDf.shape[0])\n", + "\n", + "print('pccbgs:', pcc_nonMembers.shape[0])\n", + "print('pccmems:', pcc_Members.shape[0])\n", + "print('total pccs:', pcc_nonMembers.shape[0] + pcc_Members.shape[0])\n", + "print(pcc_nonMembers.shape[0] + pcc_Members.shape[0] == downSampleDf1.shape[0])\n", + "\n", + "print(f'Total: {searchDf_specNonMembers.shape[0] + searchDf_specMembers.shape[0] + pcc_nonMembers.shape[0] + pcc_Members.shape[0]} ')\n" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "id": "4d2449ed", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
radecbinary_label
PCC_Bg049.64690041.4505000
149.37570041.3132000
249.27580041.5415000
349.33290041.4999000
449.47090041.3465000
...............
Spec_Mems20650.62274641.0504451
21050.69459341.9412301
21149.70584940.8275891
21249.11265841.1802161
21349.47027540.8972441
\n", + "

505 rows × 3 columns

\n", + "
" + ], + "text/plain": [ + " ra dec binary_label\n", + "PCC_Bg 0 49.646900 41.450500 0\n", + " 1 49.375700 41.313200 0\n", + " 2 49.275800 41.541500 0\n", + " 3 49.332900 41.499900 0\n", + " 4 49.470900 41.346500 0\n", + "... ... ... ...\n", + "Spec_Mems 206 50.622746 41.050445 1\n", + " 210 50.694593 41.941230 1\n", + " 211 49.705849 40.827589 1\n", + " 212 49.112658 41.180216 1\n", + " 213 49.470275 40.897244 1\n", + "\n", + "[505 rows x 3 columns]" + ] + }, + "execution_count": 28, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# pcc_crossDf[['objID', 'ra', 'dec']]\n", + "trainObjs = pd.concat([pcc_nonMembers[['ra', 'dec', 'binary_label']],\n", + " pcc_Members[['ra', 'dec', 'binary_label']],\n", + " searchDf_specNonMembers[['ra', 'dec', 'binary_label']],\n", + " searchDf_specMembers[['ra', 'dec', 'binary_label']]],\n", + " keys = ('PCC_Bg', 'PCC_Mems', 'Spec_Bg', 'Spec_Mems')) # want to preserve where these come from\n", + "\n", + "trainObjs" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "id": "c01a43c0", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "c:\\Users\\jsonp\\Documents\\Github\\JPAstro\\.venv\\Lib\\site-packages\\SciServer\\SkyServer.py:124: Warning: In Authentication.getToken: Authentication token is not defined: the user did not log in with the Authentication.login function, or the token has not been stored in the command line argument --ident.\n", + " token = Authentication.getToken()\n", + "c:\\Users\\jsonp\\Documents\\Github\\JPAstro\\.venv\\Lib\\site-packages\\SciServer\\SkyServer.py:124: Warning: In Authentication.getToken: Authentication token is not defined: the user did not log in with the Authentication.login function, or the token has not been stored in the command line argument --ident.\n", + " token = Authentication.getToken()\n", + "c:\\Users\\jsonp\\Documents\\Github\\JPAstro\\.venv\\Lib\\site-packages\\SciServer\\SkyServer.py:124: Warning: In Authentication.getToken: Authentication token is not defined: the user did not log in with the Authentication.login function, or the token has not been stored in the command line argument --ident.\n", + " token = Authentication.getToken()\n", + "c:\\Users\\jsonp\\Documents\\Github\\JPAstro\\.venv\\Lib\\site-packages\\SciServer\\SkyServer.py:124: Warning: In Authentication.getToken: Authentication token is not defined: the user did not log in with the Authentication.login function, or the token has not been stored in the command line argument --ident.\n", + " token = Authentication.getToken()\n", + "c:\\Users\\jsonp\\Documents\\Github\\JPAstro\\.venv\\Lib\\site-packages\\SciServer\\SkyServer.py:124: Warning: In Authentication.getToken: Authentication token is not defined: the user did not log in with the Authentication.login function, or the token has not been stored in the command line argument --ident.\n", + " token = Authentication.getToken()\n", + "c:\\Users\\jsonp\\Documents\\Github\\JPAstro\\.venv\\Lib\\site-packages\\SciServer\\SkyServer.py:124: Warning: In Authentication.getToken: Authentication token is not defined: the user did not log in with the Authentication.login function, or the token has not been stored in the command line argument --ident.\n", + " token = Authentication.getToken()\n", + "c:\\Users\\jsonp\\Documents\\Github\\JPAstro\\.venv\\Lib\\site-packages\\SciServer\\SkyServer.py:124: Warning: In Authentication.getToken: Authentication token is not defined: the user did not log in with the Authentication.login function, or the token has not been stored in the command line argument --ident.\n", + " token = Authentication.getToken()\n", + "c:\\Users\\jsonp\\Documents\\Github\\JPAstro\\.venv\\Lib\\site-packages\\SciServer\\SkyServer.py:124: Warning: In Authentication.getToken: Authentication token is not defined: the user did not log in with the Authentication.login function, or the token has not been stored in the command line argument --ident.\n", + " token = Authentication.getToken()\n", + "c:\\Users\\jsonp\\Documents\\Github\\JPAstro\\.venv\\Lib\\site-packages\\SciServer\\SkyServer.py:124: Warning: In Authentication.getToken: Authentication token is not defined: the user did not log in with the Authentication.login function, or the token has not been stored in the command line argument --ident.\n", + " token = Authentication.getToken()\n", + "c:\\Users\\jsonp\\Documents\\Github\\JPAstro\\.venv\\Lib\\site-packages\\SciServer\\SkyServer.py:124: Warning: In Authentication.getToken: Authentication token is not defined: the user did not log in with the Authentication.login function, or the token has not been stored in the command line argument --ident.\n", + " token = Authentication.getToken()\n", + "c:\\Users\\jsonp\\Documents\\Github\\JPAstro\\.venv\\Lib\\site-packages\\SciServer\\SkyServer.py:124: Warning: In Authentication.getToken: Authentication token is not defined: the user did not log in with the Authentication.login function, or the token has not been stored in the command line argument --ident.\n", + " token = Authentication.getToken()\n", + "c:\\Users\\jsonp\\Documents\\Github\\JPAstro\\.venv\\Lib\\site-packages\\SciServer\\SkyServer.py:124: Warning: In Authentication.getToken: Authentication token is not defined: the user did not log in with the Authentication.login function, or the token has not been stored in the command line argument --ident.\n", + " token = Authentication.getToken()\n", + "c:\\Users\\jsonp\\Documents\\Github\\JPAstro\\.venv\\Lib\\site-packages\\SciServer\\SkyServer.py:124: Warning: In Authentication.getToken: Authentication token is not defined: the user did not log in with the Authentication.login function, or the token has not been stored in the command line argument --ident.\n", + " token = Authentication.getToken()\n", + "c:\\Users\\jsonp\\Documents\\Github\\JPAstro\\.venv\\Lib\\site-packages\\SciServer\\SkyServer.py:124: Warning: In Authentication.getToken: Authentication token is not defined: the user did not log in with the Authentication.login function, or the token has not been stored in the command line argument --ident.\n", + " token = Authentication.getToken()\n", + "c:\\Users\\jsonp\\Documents\\Github\\JPAstro\\.venv\\Lib\\site-packages\\SciServer\\SkyServer.py:124: Warning: In Authentication.getToken: Authentication token is not defined: the user did not log in with the Authentication.login function, or the token has not been stored in the command line argument --ident.\n", + " token = Authentication.getToken()\n", + "c:\\Users\\jsonp\\Documents\\Github\\JPAstro\\.venv\\Lib\\site-packages\\SciServer\\SkyServer.py:124: Warning: In Authentication.getToken: Authentication token is not defined: the user did not log in with the Authentication.login function, or the token has not been stored in the command line argument --ident.\n", + " token = Authentication.getToken()\n", + "c:\\Users\\jsonp\\Documents\\Github\\JPAstro\\.venv\\Lib\\site-packages\\SciServer\\SkyServer.py:124: Warning: In Authentication.getToken: Authentication token is not defined: the user did not log in with the Authentication.login function, or the token has not been stored in the command line argument --ident.\n", + " token = Authentication.getToken()\n", + "c:\\Users\\jsonp\\Documents\\Github\\JPAstro\\.venv\\Lib\\site-packages\\SciServer\\SkyServer.py:124: Warning: In Authentication.getToken: Authentication token is not defined: the user did not log in with the Authentication.login function, or the token has not been stored in the command line argument --ident.\n", + " token = Authentication.getToken()\n", + "c:\\Users\\jsonp\\Documents\\Github\\JPAstro\\.venv\\Lib\\site-packages\\SciServer\\SkyServer.py:124: Warning: In Authentication.getToken: Authentication token is not defined: the user did not log in with the Authentication.login function, or the token has not been stored in the command line argument --ident.\n", + " token = Authentication.getToken()\n", + "c:\\Users\\jsonp\\Documents\\Github\\JPAstro\\.venv\\Lib\\site-packages\\SciServer\\SkyServer.py:124: Warning: In Authentication.getToken: Authentication token is not defined: the user did not log in with the Authentication.login function, or the token has not been stored in the command line argument --ident.\n", + " token = Authentication.getToken()\n", + "c:\\Users\\jsonp\\Documents\\Github\\JPAstro\\.venv\\Lib\\site-packages\\SciServer\\SkyServer.py:124: Warning: In Authentication.getToken: Authentication token is not defined: the user did not log in with the Authentication.login function, or the token has not been stored in the command line argument --ident.\n", + " token = Authentication.getToken()\n", + "c:\\Users\\jsonp\\Documents\\Github\\JPAstro\\.venv\\Lib\\site-packages\\SciServer\\SkyServer.py:124: Warning: In Authentication.getToken: Authentication token is not defined: the user did not log in with the Authentication.login function, or the token has not been stored in the command line argument --ident.\n", + " token = Authentication.getToken()\n", + "c:\\Users\\jsonp\\Documents\\Github\\JPAstro\\.venv\\Lib\\site-packages\\SciServer\\SkyServer.py:124: Warning: In Authentication.getToken: Authentication token is not defined: the user did not log in with the Authentication.login function, or the token has not been stored in the command line argument --ident.\n", + " token = Authentication.getToken()\n", + "c:\\Users\\jsonp\\Documents\\Github\\JPAstro\\.venv\\Lib\\site-packages\\SciServer\\SkyServer.py:124: Warning: In Authentication.getToken: Authentication token is not defined: the user did not log in with the Authentication.login function, or the token has not been stored in the command line argument --ident.\n", + " token = Authentication.getToken()\n", + "c:\\Users\\jsonp\\Documents\\Github\\JPAstro\\.venv\\Lib\\site-packages\\SciServer\\SkyServer.py:124: Warning: In Authentication.getToken: Authentication token is not defined: the user did not log in with the Authentication.login function, or the token has not been stored in the command line argument --ident.\n", + " token = Authentication.getToken()\n", + "c:\\Users\\jsonp\\Documents\\Github\\JPAstro\\.venv\\Lib\\site-packages\\SciServer\\SkyServer.py:124: Warning: In Authentication.getToken: Authentication token is not defined: the user did not log in with the Authentication.login function, or the token has not been stored in the command line argument --ident.\n", + " token = Authentication.getToken()\n", + "c:\\Users\\jsonp\\Documents\\Github\\JPAstro\\.venv\\Lib\\site-packages\\SciServer\\SkyServer.py:124: Warning: In Authentication.getToken: Authentication token is not defined: the user did not log in with the Authentication.login function, or the token has not been stored in the command line argument --ident.\n", + " token = Authentication.getToken()\n", + "c:\\Users\\jsonp\\Documents\\Github\\JPAstro\\.venv\\Lib\\site-packages\\SciServer\\SkyServer.py:124: Warning: In Authentication.getToken: Authentication token is not defined: the user did not log in with the Authentication.login function, or the token has not been stored in the command line argument --ident.\n", + " token = Authentication.getToken()\n", + "c:\\Users\\jsonp\\Documents\\Github\\JPAstro\\.venv\\Lib\\site-packages\\SciServer\\SkyServer.py:124: Warning: In Authentication.getToken: Authentication token is not defined: the user did not log in with the Authentication.login function, or the token has not been stored in the command line argument --ident.\n", + " token = Authentication.getToken()\n", + "c:\\Users\\jsonp\\Documents\\Github\\JPAstro\\.venv\\Lib\\site-packages\\SciServer\\SkyServer.py:124: Warning: In Authentication.getToken: Authentication token is not defined: the user did not log in with the Authentication.login function, or the token has not been stored in the command line argument --ident.\n", + " token = Authentication.getToken()\n", + "c:\\Users\\jsonp\\Documents\\Github\\JPAstro\\.venv\\Lib\\site-packages\\SciServer\\SkyServer.py:124: Warning: In Authentication.getToken: Authentication token is not defined: the user did not log in with the Authentication.login function, or the token has not been stored in the command line argument --ident.\n", + " token = Authentication.getToken()\n", + "c:\\Users\\jsonp\\Documents\\Github\\JPAstro\\.venv\\Lib\\site-packages\\SciServer\\SkyServer.py:124: Warning: In Authentication.getToken: Authentication token is not defined: the user did not log in with the Authentication.login function, or the token has not been stored in the command line argument --ident.\n", + " token = Authentication.getToken()\n", + "c:\\Users\\jsonp\\Documents\\Github\\JPAstro\\.venv\\Lib\\site-packages\\SciServer\\SkyServer.py:124: Warning: In Authentication.getToken: Authentication token is not defined: the user did not log in with the Authentication.login function, or the token has not been stored in the command line argument --ident.\n", + " token = Authentication.getToken()\n", + "c:\\Users\\jsonp\\Documents\\Github\\JPAstro\\.venv\\Lib\\site-packages\\SciServer\\SkyServer.py:124: Warning: In Authentication.getToken: Authentication token is not defined: the user did not log in with the Authentication.login function, or the token has not been stored in the command line argument --ident.\n", + " token = Authentication.getToken()\n", + "c:\\Users\\jsonp\\Documents\\Github\\JPAstro\\.venv\\Lib\\site-packages\\SciServer\\SkyServer.py:124: Warning: In Authentication.getToken: Authentication token is not defined: the user did not log in with the Authentication.login function, or the token has not been stored in the command line argument --ident.\n", + " token = Authentication.getToken()\n", + "c:\\Users\\jsonp\\Documents\\Github\\JPAstro\\.venv\\Lib\\site-packages\\SciServer\\SkyServer.py:124: Warning: In Authentication.getToken: Authentication token is not defined: the user did not log in with the Authentication.login function, or the token has not been stored in the command line argument --ident.\n", + " token = Authentication.getToken()\n", + "c:\\Users\\jsonp\\Documents\\Github\\JPAstro\\.venv\\Lib\\site-packages\\SciServer\\SkyServer.py:124: Warning: In Authentication.getToken: Authentication token is not defined: the user did not log in with the Authentication.login function, or the token has not been stored in the command line argument --ident.\n", + " token = Authentication.getToken()\n", + "c:\\Users\\jsonp\\Documents\\Github\\JPAstro\\.venv\\Lib\\site-packages\\SciServer\\SkyServer.py:124: Warning: In Authentication.getToken: Authentication token is not defined: the user did not log in with the Authentication.login function, or the token has not been stored in the command line argument --ident.\n", + " token = Authentication.getToken()\n", + "c:\\Users\\jsonp\\Documents\\Github\\JPAstro\\.venv\\Lib\\site-packages\\SciServer\\SkyServer.py:124: Warning: In Authentication.getToken: Authentication token is not defined: the user did not log in with the Authentication.login function, or the token has not been stored in the command line argument --ident.\n", + " token = Authentication.getToken()\n", + "c:\\Users\\jsonp\\Documents\\Github\\JPAstro\\.venv\\Lib\\site-packages\\SciServer\\SkyServer.py:124: Warning: In Authentication.getToken: Authentication token is not defined: the user did not log in with the Authentication.login function, or the token has not been stored in the command line argument --ident.\n", + " token = Authentication.getToken()\n", + "c:\\Users\\jsonp\\Documents\\Github\\JPAstro\\.venv\\Lib\\site-packages\\SciServer\\SkyServer.py:124: Warning: In Authentication.getToken: Authentication token is not defined: the user did not log in with the Authentication.login function, or the token has not been stored in the command line argument --ident.\n", + " token = Authentication.getToken()\n", + "c:\\Users\\jsonp\\Documents\\Github\\JPAstro\\.venv\\Lib\\site-packages\\SciServer\\SkyServer.py:124: Warning: In Authentication.getToken: Authentication token is not defined: the user did not log in with the Authentication.login function, or the token has not been stored in the command line argument --ident.\n", + " token = Authentication.getToken()\n", + "c:\\Users\\jsonp\\Documents\\Github\\JPAstro\\.venv\\Lib\\site-packages\\SciServer\\SkyServer.py:124: Warning: In Authentication.getToken: Authentication token is not defined: the user did not log in with the Authentication.login function, or the token has not been stored in the command line argument --ident.\n", + " token = Authentication.getToken()\n", + "c:\\Users\\jsonp\\Documents\\Github\\JPAstro\\.venv\\Lib\\site-packages\\SciServer\\SkyServer.py:124: Warning: In Authentication.getToken: Authentication token is not defined: the user did not log in with the Authentication.login function, or the token has not been stored in the command line argument --ident.\n", + " token = Authentication.getToken()\n", + "c:\\Users\\jsonp\\Documents\\Github\\JPAstro\\.venv\\Lib\\site-packages\\SciServer\\SkyServer.py:124: Warning: In Authentication.getToken: Authentication token is not defined: the user did not log in with the Authentication.login function, or the token has not been stored in the command line argument --ident.\n", + " token = Authentication.getToken()\n", + "c:\\Users\\jsonp\\Documents\\Github\\JPAstro\\.venv\\Lib\\site-packages\\SciServer\\SkyServer.py:124: Warning: In Authentication.getToken: Authentication token is not defined: the user did not log in with the Authentication.login function, or the token has not been stored in the command line argument --ident.\n", + " token = Authentication.getToken()\n", + "c:\\Users\\jsonp\\Documents\\Github\\JPAstro\\.venv\\Lib\\site-packages\\SciServer\\SkyServer.py:124: Warning: In Authentication.getToken: Authentication token is not defined: the user did not log in with the Authentication.login function, or the token has not been stored in the command line argument --ident.\n", + " token = Authentication.getToken()\n", + "c:\\Users\\jsonp\\Documents\\Github\\JPAstro\\.venv\\Lib\\site-packages\\SciServer\\SkyServer.py:124: Warning: In Authentication.getToken: Authentication token is not defined: the user did not log in with the Authentication.login function, or the token has not been stored in the command line argument --ident.\n", + " token = Authentication.getToken()\n", + "c:\\Users\\jsonp\\Documents\\Github\\JPAstro\\.venv\\Lib\\site-packages\\SciServer\\SkyServer.py:124: Warning: In Authentication.getToken: Authentication token is not defined: the user did not log in with the Authentication.login function, or the token has not been stored in the command line argument --ident.\n", + " token = Authentication.getToken()\n", + "c:\\Users\\jsonp\\Documents\\Github\\JPAstro\\.venv\\Lib\\site-packages\\SciServer\\SkyServer.py:124: Warning: In Authentication.getToken: Authentication token is not defined: the user did not log in with the Authentication.login function, or the token has not been stored in the command line argument --ident.\n", + " token = Authentication.getToken()\n", + "c:\\Users\\jsonp\\Documents\\Github\\JPAstro\\.venv\\Lib\\site-packages\\SciServer\\SkyServer.py:124: Warning: In Authentication.getToken: Authentication token is not defined: the user did not log in with the Authentication.login function, or the token has not been stored in the command line argument --ident.\n", + " token = Authentication.getToken()\n", + "c:\\Users\\jsonp\\Documents\\Github\\JPAstro\\.venv\\Lib\\site-packages\\SciServer\\SkyServer.py:124: Warning: In Authentication.getToken: Authentication token is not defined: the user did not log in with the Authentication.login function, or the token has not been stored in the command line argument --ident.\n", + " token = Authentication.getToken()\n", + "c:\\Users\\jsonp\\Documents\\Github\\JPAstro\\.venv\\Lib\\site-packages\\SciServer\\SkyServer.py:124: Warning: In Authentication.getToken: Authentication token is not defined: the user did not log in with the Authentication.login function, or the token has not been stored in the command line argument --ident.\n", + " token = Authentication.getToken()\n", + "c:\\Users\\jsonp\\Documents\\Github\\JPAstro\\.venv\\Lib\\site-packages\\SciServer\\SkyServer.py:124: Warning: In Authentication.getToken: Authentication token is not defined: the user did not log in with the Authentication.login function, or the token has not been stored in the command line argument --ident.\n", + " token = Authentication.getToken()\n", + "c:\\Users\\jsonp\\Documents\\Github\\JPAstro\\.venv\\Lib\\site-packages\\SciServer\\SkyServer.py:124: Warning: In Authentication.getToken: Authentication token is not defined: the user did not log in with the Authentication.login function, or the token has not been stored in the command line argument --ident.\n", + " token = Authentication.getToken()\n", + "c:\\Users\\jsonp\\Documents\\Github\\JPAstro\\.venv\\Lib\\site-packages\\SciServer\\SkyServer.py:124: Warning: In Authentication.getToken: Authentication token is not defined: the user did not log in with the Authentication.login function, or the token has not been stored in the command line argument --ident.\n", + " token = Authentication.getToken()\n" + ] + }, + { + "ename": "KeyboardInterrupt", + "evalue": "", + "output_type": "error", + "traceback": [ + "\u001b[31m---------------------------------------------------------------------------\u001b[39m", + "\u001b[31mKeyboardInterrupt\u001b[39m Traceback (most recent call last)", + "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[31]\u001b[39m\u001b[32m, line 19\u001b[39m\n\u001b[32m 16\u001b[39m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[32m 17\u001b[39m \u001b[38;5;66;03m# for id, r, d in zip(searchDf['objID'], trainObjs['ra'], trainObjs['dec']):\u001b[39;00m\n\u001b[32m 18\u001b[39m \u001b[38;5;28;01mfor\u001b[39;00m r, d, l \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mzip\u001b[39m(trainObjs[\u001b[33m'\u001b[39m\u001b[33mra\u001b[39m\u001b[33m'\u001b[39m], trainObjs[\u001b[33m'\u001b[39m\u001b[33mdec\u001b[39m\u001b[33m'\u001b[39m], trainObjs[\u001b[33m'\u001b[39m\u001b[33mbinary_label\u001b[39m\u001b[33m'\u001b[39m]):\n\u001b[32m---> \u001b[39m\u001b[32m19\u001b[39m img_array = \u001b[43mSkyServer\u001b[49m\u001b[43m.\u001b[49m\u001b[43mgetJpegImgCutout\u001b[49m\u001b[43m(\u001b[49m\u001b[43mra\u001b[49m\u001b[43m=\u001b[49m\u001b[43mr\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdec\u001b[49m\u001b[43m=\u001b[49m\u001b[43md\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mwidth\u001b[49m\u001b[43m=\u001b[49m\u001b[43mimg_width\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mheight\u001b[49m\u001b[43m=\u001b[49m\u001b[43mimg_height\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mscale\u001b[49m\u001b[43m=\u001b[49m\u001b[32;43m0.1\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\n\u001b[32m 20\u001b[39m \u001b[43m \u001b[49m\u001b[43mdataRelease\u001b[49m\u001b[43m=\u001b[49m\u001b[43mSkyServer_DataRelease\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 21\u001b[39m \u001b[38;5;66;03m# print(f'{id}-label={labeler(z)}')\u001b[39;00m\n\u001b[32m 22\u001b[39m \u001b[38;5;66;03m# outPicTemplate = f'{id}-label={labeler(z)}.png'\u001b[39;00m\n\u001b[32m 23\u001b[39m outPicTemplate = \u001b[33mf\u001b[39m\u001b[33m'\u001b[39m\u001b[33msdss_ra=\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mr\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m_dec=\u001b[39m\u001b[38;5;132;01m{\u001b[39;00md\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m-label=\u001b[39m\u001b[38;5;132;01m{\u001b[39;00ml\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m.png\u001b[39m\u001b[33m'\u001b[39m\n", + "\u001b[36mFile \u001b[39m\u001b[32mc:\\Users\\jsonp\\Documents\\Github\\JPAstro\\.venv\\Lib\\site-packages\\SciServer\\SkyServer.py:128\u001b[39m, in \u001b[36mgetJpegImgCutout\u001b[39m\u001b[34m(ra, dec, scale, width, height, opt, query, dataRelease)\u001b[39m\n\u001b[32m 125\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m token \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;129;01mand\u001b[39;00m token != \u001b[33m\"\u001b[39m\u001b[33m\"\u001b[39m:\n\u001b[32m 126\u001b[39m headers[\u001b[33m'\u001b[39m\u001b[33mX-Auth-Token\u001b[39m\u001b[33m'\u001b[39m] = token\n\u001b[32m--> \u001b[39m\u001b[32m128\u001b[39m response = \u001b[43mrequests\u001b[49m\u001b[43m.\u001b[49m\u001b[43mget\u001b[49m\u001b[43m(\u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\u001b[43mheaders\u001b[49m\u001b[43m=\u001b[49m\u001b[43mheaders\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mstream\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m)\u001b[49m\n\u001b[32m 129\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m response.status_code != \u001b[32m200\u001b[39m:\n\u001b[32m 130\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m response.status_code == \u001b[32m404\u001b[39m \u001b[38;5;129;01mor\u001b[39;00m response.status_code == \u001b[32m500\u001b[39m:\n", + "\u001b[36mFile \u001b[39m\u001b[32mc:\\Users\\jsonp\\Documents\\Github\\JPAstro\\.venv\\Lib\\site-packages\\requests\\api.py:73\u001b[39m, in \u001b[36mget\u001b[39m\u001b[34m(url, params, **kwargs)\u001b[39m\n\u001b[32m 62\u001b[39m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34mget\u001b[39m(url, params=\u001b[38;5;28;01mNone\u001b[39;00m, **kwargs):\n\u001b[32m 63\u001b[39m \u001b[38;5;250m \u001b[39m\u001b[33mr\u001b[39m\u001b[33;03m\"\"\"Sends a GET request.\u001b[39;00m\n\u001b[32m 64\u001b[39m \n\u001b[32m 65\u001b[39m \u001b[33;03m :param url: URL for the new :class:`Request` object.\u001b[39;00m\n\u001b[32m (...)\u001b[39m\u001b[32m 70\u001b[39m \u001b[33;03m :rtype: requests.Response\u001b[39;00m\n\u001b[32m 71\u001b[39m \u001b[33;03m \"\"\"\u001b[39;00m\n\u001b[32m---> \u001b[39m\u001b[32m73\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mrequest\u001b[49m\u001b[43m(\u001b[49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43mget\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mparams\u001b[49m\u001b[43m=\u001b[49m\u001b[43mparams\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n", + "\u001b[36mFile \u001b[39m\u001b[32mc:\\Users\\jsonp\\Documents\\Github\\JPAstro\\.venv\\Lib\\site-packages\\requests\\api.py:59\u001b[39m, in \u001b[36mrequest\u001b[39m\u001b[34m(method, url, **kwargs)\u001b[39m\n\u001b[32m 55\u001b[39m \u001b[38;5;66;03m# By using the 'with' statement we are sure the session is closed, thus we\u001b[39;00m\n\u001b[32m 56\u001b[39m \u001b[38;5;66;03m# avoid leaving sockets open which can trigger a ResourceWarning in some\u001b[39;00m\n\u001b[32m 57\u001b[39m \u001b[38;5;66;03m# cases, and look like a memory leak in others.\u001b[39;00m\n\u001b[32m 58\u001b[39m \u001b[38;5;28;01mwith\u001b[39;00m sessions.Session() \u001b[38;5;28;01mas\u001b[39;00m session:\n\u001b[32m---> \u001b[39m\u001b[32m59\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43msession\u001b[49m\u001b[43m.\u001b[49m\u001b[43mrequest\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m=\u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[43m=\u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n", + "\u001b[36mFile \u001b[39m\u001b[32mc:\\Users\\jsonp\\Documents\\Github\\JPAstro\\.venv\\Lib\\site-packages\\requests\\sessions.py:589\u001b[39m, in \u001b[36mSession.request\u001b[39m\u001b[34m(self, method, url, params, data, headers, cookies, files, auth, timeout, allow_redirects, proxies, hooks, stream, verify, cert, json)\u001b[39m\n\u001b[32m 584\u001b[39m send_kwargs = {\n\u001b[32m 585\u001b[39m \u001b[33m\"\u001b[39m\u001b[33mtimeout\u001b[39m\u001b[33m\"\u001b[39m: timeout,\n\u001b[32m 586\u001b[39m \u001b[33m\"\u001b[39m\u001b[33mallow_redirects\u001b[39m\u001b[33m\"\u001b[39m: allow_redirects,\n\u001b[32m 587\u001b[39m }\n\u001b[32m 588\u001b[39m send_kwargs.update(settings)\n\u001b[32m--> \u001b[39m\u001b[32m589\u001b[39m resp = \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43msend\u001b[49m\u001b[43m(\u001b[49m\u001b[43mprep\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43msend_kwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 591\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m resp\n", + "\u001b[36mFile \u001b[39m\u001b[32mc:\\Users\\jsonp\\Documents\\Github\\JPAstro\\.venv\\Lib\\site-packages\\requests\\sessions.py:703\u001b[39m, in \u001b[36mSession.send\u001b[39m\u001b[34m(self, request, **kwargs)\u001b[39m\n\u001b[32m 700\u001b[39m start = preferred_clock()\n\u001b[32m 702\u001b[39m \u001b[38;5;66;03m# Send the request\u001b[39;00m\n\u001b[32m--> \u001b[39m\u001b[32m703\u001b[39m r = \u001b[43madapter\u001b[49m\u001b[43m.\u001b[49m\u001b[43msend\u001b[49m\u001b[43m(\u001b[49m\u001b[43mrequest\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 705\u001b[39m \u001b[38;5;66;03m# Total elapsed time of the request (approximately)\u001b[39;00m\n\u001b[32m 706\u001b[39m elapsed = preferred_clock() - start\n", + "\u001b[36mFile \u001b[39m\u001b[32mc:\\Users\\jsonp\\Documents\\Github\\JPAstro\\.venv\\Lib\\site-packages\\requests\\adapters.py:644\u001b[39m, in \u001b[36mHTTPAdapter.send\u001b[39m\u001b[34m(self, request, stream, timeout, verify, cert, proxies)\u001b[39m\n\u001b[32m 641\u001b[39m timeout = TimeoutSauce(connect=timeout, read=timeout)\n\u001b[32m 643\u001b[39m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[32m--> \u001b[39m\u001b[32m644\u001b[39m resp = \u001b[43mconn\u001b[49m\u001b[43m.\u001b[49m\u001b[43murlopen\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 645\u001b[39m \u001b[43m \u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m=\u001b[49m\u001b[43mrequest\u001b[49m\u001b[43m.\u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 646\u001b[39m \u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[43m=\u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 647\u001b[39m \u001b[43m \u001b[49m\u001b[43mbody\u001b[49m\u001b[43m=\u001b[49m\u001b[43mrequest\u001b[49m\u001b[43m.\u001b[49m\u001b[43mbody\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 648\u001b[39m \u001b[43m \u001b[49m\u001b[43mheaders\u001b[49m\u001b[43m=\u001b[49m\u001b[43mrequest\u001b[49m\u001b[43m.\u001b[49m\u001b[43mheaders\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 649\u001b[39m \u001b[43m \u001b[49m\u001b[43mredirect\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[32m 650\u001b[39m \u001b[43m \u001b[49m\u001b[43massert_same_host\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[32m 651\u001b[39m \u001b[43m \u001b[49m\u001b[43mpreload_content\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[32m 652\u001b[39m \u001b[43m \u001b[49m\u001b[43mdecode_content\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[32m 653\u001b[39m \u001b[43m \u001b[49m\u001b[43mretries\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mmax_retries\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 654\u001b[39m \u001b[43m \u001b[49m\u001b[43mtimeout\u001b[49m\u001b[43m=\u001b[49m\u001b[43mtimeout\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 655\u001b[39m \u001b[43m \u001b[49m\u001b[43mchunked\u001b[49m\u001b[43m=\u001b[49m\u001b[43mchunked\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 656\u001b[39m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 658\u001b[39m \u001b[38;5;28;01mexcept\u001b[39;00m (ProtocolError, \u001b[38;5;167;01mOSError\u001b[39;00m) \u001b[38;5;28;01mas\u001b[39;00m err:\n\u001b[32m 659\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mConnectionError\u001b[39;00m(err, request=request)\n", + "\u001b[36mFile \u001b[39m\u001b[32mc:\\Users\\jsonp\\Documents\\Github\\JPAstro\\.venv\\Lib\\site-packages\\urllib3\\connectionpool.py:787\u001b[39m, in \u001b[36mHTTPConnectionPool.urlopen\u001b[39m\u001b[34m(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, preload_content, decode_content, **response_kw)\u001b[39m\n\u001b[32m 784\u001b[39m response_conn = conn \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m release_conn \u001b[38;5;28;01melse\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[32m 786\u001b[39m \u001b[38;5;66;03m# Make the request on the HTTPConnection object\u001b[39;00m\n\u001b[32m--> \u001b[39m\u001b[32m787\u001b[39m response = \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_make_request\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 788\u001b[39m \u001b[43m \u001b[49m\u001b[43mconn\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 789\u001b[39m \u001b[43m \u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 790\u001b[39m \u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 791\u001b[39m \u001b[43m \u001b[49m\u001b[43mtimeout\u001b[49m\u001b[43m=\u001b[49m\u001b[43mtimeout_obj\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 792\u001b[39m \u001b[43m \u001b[49m\u001b[43mbody\u001b[49m\u001b[43m=\u001b[49m\u001b[43mbody\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 793\u001b[39m \u001b[43m \u001b[49m\u001b[43mheaders\u001b[49m\u001b[43m=\u001b[49m\u001b[43mheaders\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 794\u001b[39m \u001b[43m \u001b[49m\u001b[43mchunked\u001b[49m\u001b[43m=\u001b[49m\u001b[43mchunked\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 795\u001b[39m \u001b[43m \u001b[49m\u001b[43mretries\u001b[49m\u001b[43m=\u001b[49m\u001b[43mretries\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 796\u001b[39m \u001b[43m \u001b[49m\u001b[43mresponse_conn\u001b[49m\u001b[43m=\u001b[49m\u001b[43mresponse_conn\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 797\u001b[39m \u001b[43m \u001b[49m\u001b[43mpreload_content\u001b[49m\u001b[43m=\u001b[49m\u001b[43mpreload_content\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 798\u001b[39m \u001b[43m \u001b[49m\u001b[43mdecode_content\u001b[49m\u001b[43m=\u001b[49m\u001b[43mdecode_content\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 799\u001b[39m \u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mresponse_kw\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 800\u001b[39m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 802\u001b[39m \u001b[38;5;66;03m# Everything went great!\u001b[39;00m\n\u001b[32m 803\u001b[39m clean_exit = \u001b[38;5;28;01mTrue\u001b[39;00m\n", + "\u001b[36mFile \u001b[39m\u001b[32mc:\\Users\\jsonp\\Documents\\Github\\JPAstro\\.venv\\Lib\\site-packages\\urllib3\\connectionpool.py:534\u001b[39m, in \u001b[36mHTTPConnectionPool._make_request\u001b[39m\u001b[34m(self, conn, method, url, body, headers, retries, timeout, chunked, response_conn, preload_content, decode_content, enforce_content_length)\u001b[39m\n\u001b[32m 532\u001b[39m \u001b[38;5;66;03m# Receive the response from the server\u001b[39;00m\n\u001b[32m 533\u001b[39m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[32m--> \u001b[39m\u001b[32m534\u001b[39m response = \u001b[43mconn\u001b[49m\u001b[43m.\u001b[49m\u001b[43mgetresponse\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 535\u001b[39m \u001b[38;5;28;01mexcept\u001b[39;00m (BaseSSLError, \u001b[38;5;167;01mOSError\u001b[39;00m) \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[32m 536\u001b[39m \u001b[38;5;28mself\u001b[39m._raise_timeout(err=e, url=url, timeout_value=read_timeout)\n", + "\u001b[36mFile \u001b[39m\u001b[32mc:\\Users\\jsonp\\Documents\\Github\\JPAstro\\.venv\\Lib\\site-packages\\urllib3\\connection.py:565\u001b[39m, in \u001b[36mHTTPConnection.getresponse\u001b[39m\u001b[34m(self)\u001b[39m\n\u001b[32m 562\u001b[39m _shutdown = \u001b[38;5;28mgetattr\u001b[39m(\u001b[38;5;28mself\u001b[39m.sock, \u001b[33m\"\u001b[39m\u001b[33mshutdown\u001b[39m\u001b[33m\"\u001b[39m, \u001b[38;5;28;01mNone\u001b[39;00m)\n\u001b[32m 564\u001b[39m \u001b[38;5;66;03m# Get the response from http.client.HTTPConnection\u001b[39;00m\n\u001b[32m--> \u001b[39m\u001b[32m565\u001b[39m httplib_response = \u001b[38;5;28;43msuper\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m.\u001b[49m\u001b[43mgetresponse\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 567\u001b[39m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[32m 568\u001b[39m assert_header_parsing(httplib_response.msg)\n", + "\u001b[36mFile \u001b[39m\u001b[32m~\\AppData\\Roaming\\uv\\python\\cpython-3.13.7-windows-x86_64-none\\Lib\\http\\client.py:1430\u001b[39m, in \u001b[36mHTTPConnection.getresponse\u001b[39m\u001b[34m(self)\u001b[39m\n\u001b[32m 1428\u001b[39m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[32m 1429\u001b[39m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[32m-> \u001b[39m\u001b[32m1430\u001b[39m \u001b[43mresponse\u001b[49m\u001b[43m.\u001b[49m\u001b[43mbegin\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 1431\u001b[39m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mConnectionError\u001b[39;00m:\n\u001b[32m 1432\u001b[39m \u001b[38;5;28mself\u001b[39m.close()\n", + "\u001b[36mFile \u001b[39m\u001b[32m~\\AppData\\Roaming\\uv\\python\\cpython-3.13.7-windows-x86_64-none\\Lib\\http\\client.py:331\u001b[39m, in \u001b[36mHTTPResponse.begin\u001b[39m\u001b[34m(self)\u001b[39m\n\u001b[32m 329\u001b[39m \u001b[38;5;66;03m# read until we get a non-100 response\u001b[39;00m\n\u001b[32m 330\u001b[39m \u001b[38;5;28;01mwhile\u001b[39;00m \u001b[38;5;28;01mTrue\u001b[39;00m:\n\u001b[32m--> \u001b[39m\u001b[32m331\u001b[39m version, status, reason = \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_read_status\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 332\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m status != CONTINUE:\n\u001b[32m 333\u001b[39m \u001b[38;5;28;01mbreak\u001b[39;00m\n", + "\u001b[36mFile \u001b[39m\u001b[32m~\\AppData\\Roaming\\uv\\python\\cpython-3.13.7-windows-x86_64-none\\Lib\\http\\client.py:292\u001b[39m, in \u001b[36mHTTPResponse._read_status\u001b[39m\u001b[34m(self)\u001b[39m\n\u001b[32m 291\u001b[39m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34m_read_status\u001b[39m(\u001b[38;5;28mself\u001b[39m):\n\u001b[32m--> \u001b[39m\u001b[32m292\u001b[39m line = \u001b[38;5;28mstr\u001b[39m(\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mfp\u001b[49m\u001b[43m.\u001b[49m\u001b[43mreadline\u001b[49m\u001b[43m(\u001b[49m\u001b[43m_MAXLINE\u001b[49m\u001b[43m \u001b[49m\u001b[43m+\u001b[49m\u001b[43m \u001b[49m\u001b[32;43m1\u001b[39;49m\u001b[43m)\u001b[49m, \u001b[33m\"\u001b[39m\u001b[33miso-8859-1\u001b[39m\u001b[33m\"\u001b[39m)\n\u001b[32m 293\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mlen\u001b[39m(line) > _MAXLINE:\n\u001b[32m 294\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m LineTooLong(\u001b[33m\"\u001b[39m\u001b[33mstatus line\u001b[39m\u001b[33m\"\u001b[39m)\n", + "\u001b[36mFile \u001b[39m\u001b[32m~\\AppData\\Roaming\\uv\\python\\cpython-3.13.7-windows-x86_64-none\\Lib\\socket.py:719\u001b[39m, in \u001b[36mSocketIO.readinto\u001b[39m\u001b[34m(self, b)\u001b[39m\n\u001b[32m 717\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mOSError\u001b[39;00m(\u001b[33m\"\u001b[39m\u001b[33mcannot read from timed out object\u001b[39m\u001b[33m\"\u001b[39m)\n\u001b[32m 718\u001b[39m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[32m--> \u001b[39m\u001b[32m719\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_sock\u001b[49m\u001b[43m.\u001b[49m\u001b[43mrecv_into\u001b[49m\u001b[43m(\u001b[49m\u001b[43mb\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 720\u001b[39m \u001b[38;5;28;01mexcept\u001b[39;00m timeout:\n\u001b[32m 721\u001b[39m \u001b[38;5;28mself\u001b[39m._timeout_occurred = \u001b[38;5;28;01mTrue\u001b[39;00m\n", + "\u001b[36mFile \u001b[39m\u001b[32m~\\AppData\\Roaming\\uv\\python\\cpython-3.13.7-windows-x86_64-none\\Lib\\ssl.py:1304\u001b[39m, in \u001b[36mSSLSocket.recv_into\u001b[39m\u001b[34m(self, buffer, nbytes, flags)\u001b[39m\n\u001b[32m 1300\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m flags != \u001b[32m0\u001b[39m:\n\u001b[32m 1301\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\n\u001b[32m 1302\u001b[39m \u001b[33m\"\u001b[39m\u001b[33mnon-zero flags not allowed in calls to recv_into() on \u001b[39m\u001b[38;5;132;01m%s\u001b[39;00m\u001b[33m\"\u001b[39m %\n\u001b[32m 1303\u001b[39m \u001b[38;5;28mself\u001b[39m.\u001b[34m__class__\u001b[39m)\n\u001b[32m-> \u001b[39m\u001b[32m1304\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mread\u001b[49m\u001b[43m(\u001b[49m\u001b[43mnbytes\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mbuffer\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 1305\u001b[39m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[32m 1306\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28msuper\u001b[39m().recv_into(buffer, nbytes, flags)\n", + "\u001b[36mFile \u001b[39m\u001b[32m~\\AppData\\Roaming\\uv\\python\\cpython-3.13.7-windows-x86_64-none\\Lib\\ssl.py:1138\u001b[39m, in \u001b[36mSSLSocket.read\u001b[39m\u001b[34m(self, len, buffer)\u001b[39m\n\u001b[32m 1136\u001b[39m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[32m 1137\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m buffer \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[32m-> \u001b[39m\u001b[32m1138\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_sslobj\u001b[49m\u001b[43m.\u001b[49m\u001b[43mread\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mlen\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mbuffer\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 1139\u001b[39m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[32m 1140\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m._sslobj.read(\u001b[38;5;28mlen\u001b[39m)\n", + "\u001b[31mKeyboardInterrupt\u001b[39m: " + ] + } + ], + "source": [ + "\n", + "# trainObjs = trainObjs.drop_duplicates(subset = 'objID')\n", + "trainObjs['binary_label'] = pd.to_numeric(trainObjs['binary_label'], downcast='integer')\n", + "\n", + "img_width, img_height = 200, 200\n", + "SkyServer_DataRelease = 'DR16'\n", + "\n", + "dirName = 'PCC-and-SpecSearch'\n", + "outDir = os.path.join('..', 'Images', dirName)\n", + "\n", + "fileList = list()\n", + "if not os.path.exists(outDir):\n", + " os.makedirs(outDir)\n", + " \n", + "if len(glob.glob(os.path.join(outDir, '*.png'))) == trainObjs.shape[0]:\n", + " print('Skipping Populate')\n", + "else:\n", + " # for id, r, d in zip(searchDf['objID'], trainObjs['ra'], trainObjs['dec']):\n", + " for r, d, l in zip(trainObjs['ra'], trainObjs['dec'], trainObjs['binary_label']):\n", + " img_array = SkyServer.getJpegImgCutout(ra=r, dec=d, width=img_width, height=img_height, scale=0.1, \n", + " dataRelease=SkyServer_DataRelease)\n", + " # print(f'{id}-label={labeler(z)}')\n", + " # outPicTemplate = f'{id}-label={labeler(z)}.png'\n", + " outPicTemplate = f'sdss_ra={r}_dec={d}-label={l}.png'\n", + " \n", + " img0 = PIL.Image.fromarray(img_array, 'RGB')\n", + " img0.save(f'{outDir}/{outPicTemplate}')\n", + " fileList.append(f'{outPicTemplate}')\n", + "\n", + "print(f'Finished populate with {len(fileList)} images')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bf582b23", + "metadata": {}, + "outputs": [], + "source": [ + "final_df = pd.concat([\n", + " searchDf_specMembers[['files','labels']],\n", + " searchDf_specNonMembers[['files','labels']],\n", + " pcc_Members.rename(columns={'binary_label':'labels'})[['files','labels']],\n", + " pcc_nonMembers.rename(columns={'binary_label':'labels'})[['files','labels']]\n", + "]).reset_index(drop=True)\n", + "final_df" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1c43301b", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "JPAstro", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.13.7" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/README.md b/README.md index 6cc51c25..6ea1d77a 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,4 @@ # Current working summary of project: -(Writeup/Masters Project Report will be included in the repo soon) -(Presentation will be added as well) * Take 5437 objects from [Wittmann et al.](https://iopscience.iop.org/article/10.3847/1538-4365/ab4998) * 7 distinct classes from paper - grouped into 2 / binary classification @@ -52,9 +50,13 @@ JPAstro/ (New name may be needed. Below are the most important files in each dir # Installation ## Method 1 uv install 1. Install uv -## Method 2 Tensorflow on GPU via conda +2. `uv sync` +3. `uv pip install SciScript-Python/py3` + 1. Follow the steps to authenticate SciServer (you'll have to change the relevant files) + +## Method 2 Tensorflow on GPU via conda (being deprecated) 1. Install anaconda 2. `conda update -n base -c defaults conda` -3. `conda env create -f tensorflow_gpu_modified.yaml` +3. `conda env create -f tf_gpu_astro.yaml` 4. `conda activate tf_gpu_astro` diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..8aacfa18 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,53 @@ +[build-system] +requires = ["setuptools>=61.0", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "JPAstro" +version = "0.1.0" +description = "Galaxy Classification via Convolutional Neural Network" +readme = "README.md" +requires-python = ">=3.12" +license = "MIT" +license-files = ["LICENSE.txt"] +authors = [ { name = "Jason Pruitt" } ] + +dependencies = [ + "opencv-python", + "pandas", + "matplotlib", + "seaborn", + "scikit-learn", + "scikit-image", + "ipykernel", + "ruff>=0.13.3", + "pillow>=11.3.0", + "numpy>=2.0.2", + "pyesasky>=2.0.4", + "requests>=2.32.5", +] + +[project.optional-dependencies] +pytorch = [ + "torch>=2.1.0", + "torchvision>=0.16.0", + "torchaudio>=2.1.0", + "torchmetrics", + "tensorboard>=2.14.1", +] +[tool.uv.sources] +torch = [ + { index = "pytorch-cu128", marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, +] +torchvision = [ + { index = "pytorch-cu128", marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, +] +[[tool.uv.index]] +name = "pytorch-cu128" +url = "https://download.pytorch.org/whl/cu128" +explicit = true +dynamic = [] + +[project.urls] +"Repository" = "https://github.com/Json-To-String/JPAstro" +"Homepage" = "https://github.com/Json-To-String/JPAstro" diff --git a/src/check_cuda.py b/src/check_cuda.py new file mode 100644 index 00000000..73c24648 --- /dev/null +++ b/src/check_cuda.py @@ -0,0 +1,3 @@ +import torch + +print(torch.cuda.is_available()) \ No newline at end of file diff --git a/src/populateDataset.py b/src/populateDataset.py index b532082a..7c9e9dbd 100644 --- a/src/populateDataset.py +++ b/src/populateDataset.py @@ -1,35 +1,35 @@ - import requests import pandas as pd -import sys import os -#from skimage import io, transform -import matplotlib.pyplot as plt import time -# testUrl = f'https://skyserver.sdss.org/dr16/SkyServerWS/ImgCutout/getjpeg?TaskName=Skyserver.Explore.Image&ra={raEx}&dec={dcEx}&scale=0.1&width=200&height=200' - - -df0 = pd.read_fwf('PCC_cat.txt', header=None) +# testUrl = f'https://skyserver.sdss.org/dr16/SkyServerWS/ImgCutout/getjpeg?TaskName=Skyserver.Explore.Image&ra={raEx}&dec={dcEx}&scale=0.1&width=200&height=200' +df0 = (pd.read_fwf("PCC_cat.txt", header=None),) ra = df0[2] dec = df0[3] -outDir = 'SDSS400' -height = 400 -width = 400 - -for i in range(len(ra)): -# for i in range(800): - urlVar = f'https://skyserver.sdss.org/dr16/SkyServerWS/ImgCutout/' f'getjpeg?TaskName=Skyserver.Explore.Image&ra={str(ra[i]).strip()}' f'&dec={str(dec[i]).strip()}&scale=0.1&width={width}&height={height}' - - # tell the loop to pause for a bit - every 100 images - if i%100==0: - time.sleep(5) - - img_data = requests.get(urlVar).content - with open(f'{outDir}/sdss_ra={ra[i]}_dec={dec[i]}.png', 'wb') as handler: - handler.write(img_data) - -#images = [f'https://skyserver.sdss.org/dr16/SkyServerWS/ImgCutout/' f'getjpeg?TaskName=Skyserver.Explore.Image&ra={str(ra[i]).strip()}' f'&dec={str(dec[i]).strip()}&scale=0.1&width=200&height=200' for i in range(len(ra))] -# os.path.join('SDSS', f'sdss_ra={str(df0[2][1])}_dec={str(df0[3][1])}.jpeg') +def populateDataset( + outDir="SDSS200", + height=200, + width=200, + scale=0.1, +): + for i in range(len(ra)): + urlVar = ( + f"https://skyserver.sdss.org/dr16/SkyServerWS/ImgCutout/" + f"getjpeg?TaskName=Skyserver.Explore.Image&ra={str(ra[i]).strip()}" + f"&dec={str(dec[i]).strip()}&scale={scale}&width={width}&height={height}" + ) + + # tell the loop to pause for a bit - every 100 images (to avoid timeout) + if i % 100 == 0: + time.sleep(5) + + img_data = requests.get(urlVar).content + out_path = os.path.join(outDir, f"sdss_ra={str(ra[i])}_dec={str(dec[i])}.png") + with open(out_path, "wb") as handler: + handler.write(img_data) + +if __name__ == "__main__": + populateDataset() \ No newline at end of file