Merge pull request #27 from sdss/dtn

Switching server to dtn and adding support for sdss-v product download

Author: havok2063

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 1.1.2dev
+current_version = 2.0.0dev
 commit = True
 tag = False
 tag_name = {new_version}

.github/workflows/pythonapp.yml .github/workflows/build.yml.github/workflows/pythonapp.yml renamed to .github/workflows/build.yml

@@ -1,13 +1,9 @@
 # This workflow will install Python dependencies, run tests and lint with a single version of Python
 # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
 
-name: Python application
+name: Build and Test
 
-on:
-  push:
-    branches: [ master, versions ]
-  pull_request:
-    branches: [ master ]
+on: [push, pull_request]
 
 jobs:
   build:
@@ -31,7 +27,20 @@ jobs:
         flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
         # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
         flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
-    # - name: Test with pytest
-    #   run: |
-    #     pip install pytest
-    #     pytest
+    - name: Setup SDSS-IV netrc
+      uses: extractions/netrc@v1
+      with:
+        machine: data.sdss.org
+        username: ${{ secrets.S4_USERNAME }}
+        password: ${{ secrets.S4_PASSWORD }}
+    - name: Setup SDSS-V netrc
+      uses: extractions/netrc@v1
+      with:
+        machine: data.sdss5.org
+        username: ${{ secrets.S5_USERNAME }}
+        password: ${{ secrets.S5_PASSWORD }}
+
+    - name: Test with pytest
+      run: |
+        pip install pytest
+        pytest

.github/workflows/pythonpublish.yml

@@ -0,0 +1,49 @@
+on:
+    push:
+        # Sequence of patterns matched against refs/tags
+        tags:
+            - "[0-9]+.[0-9]+.[0-9]+"  # Exclude pre-releases
+
+name: Create Release
+
+jobs:
+    build:
+
+        runs-on: ubuntu-latest
+
+        steps:
+
+            - name: Checkout code
+              uses: actions/checkout@v2
+
+            - name: Create Release
+              id: create_release
+              uses: actions/create-release@v1
+              env:
+                  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}  # This token is provided by Actions, you do not need to create your own token
+              with:
+                  tag_name: ${{ github.ref }}
+                  release_name: sdss_access ${{ github.ref }}
+                  body:
+                  draft: false
+                  prerelease: false
+
+            - name: Set up Python
+              uses: actions/setup-python@v2
+              with:
+                  python-version: 3.7
+
+            - name: Install dependencies
+              run: |
+                python -m pip install --upgrade pip
+                pip install setuptools wheel twine
+
+            - name: Build package
+              run: |
+                python setup.py sdist bdist_wheel --universal
+
+            - name: Publish to PyPI
+              uses: pypa/gh-action-pypi-publish@master
+              with:
+                  user: __token__
+                  password: ${{ secrets.pypi_password }}

.github/workflows/release.yml

@@ -4,11 +4,7 @@ name: Build Sphinx Documentation
 
 # Controls when the action will run. Triggers the workflow on push or pull request
 # events but only for the master branch
-on:
-  push:
-    branches: [ master, versions ]
-  pull_request:
-    branches: [ master ]
+on: [push, pull_request]
 
 # A workflow run is made up of one or more jobs that can run sequentially or in parallel
 jobs:

.github/workflows/sphinxbuild.yml

@@ -6,8 +6,15 @@ Change Log
 
 This document records the main changes to the sdss_access code.
 
+2.0.0 (unreleased)
+------------------
+- Breaking Change: switches `sdss_access` to use new SDSS dtn.sdss.org server
+- Feature: adds new "data.sdss5.org" server to support rsync, curl, and http downloads for sdss5 products
+- Adding new Sphinx docs for SDSS collaboration authentication
+- Updates the public release check to now check todays date against the tree release date, and "DR" in release name. 
+
 1.1.1 (2020-11-11)
------------------
+------------------
 - Feature - Adds new functions for mwm `healpixgrp`, `apginst`, and `apgprefix` to determine other keywords from existing `telescope` kwarg
 - Support :issue:`25` - Adds ``preserve_envvars`` option to preserve original os environment variables
 

CHANGELOG.rst

@@ -0,0 +1,35 @@
+
+.. _auth:
+
+SDSS Authentication
+===================
+
+For collaboration members, when downloading proprietary collaboration data products, 
+``sdss_access`` requires authentication using the ``.netrc`` file.  To set this up, create or 
+edit a file in your home called ``.netrc`` and copy the following lines, depending on which
+SDSS collaboration you are a member of.
+
+SDSS-IV members
+---------------
+::
+
+    machine data.sdss.org
+       login <username>
+       password <password>
+
+
+SDSS-V members
+--------------
+::
+
+    machine data.sdss5.org
+       login <username>
+       password <password>
+
+In the above, replace ``<username>`` and ``<password>`` with your login credentials. The default 
+SDSS username and password is also acceptable for anonymous access.  
+**Finally, run** ``chmod 600 ~/.netrc`` **to make the file only accessible to your user.**
+
+.. note::
+
+  The default SDSS username and password is different for SDSS-IV and SDSS-V members. 

docs/sphinx/auth.rst

@@ -18,6 +18,7 @@ Contents
   Installation <installation>
   Introduction to sdss_access <intro>
   Available sdss-access paths <paths>
+  Collaboration Authentication <auth>
 
 Reference
 ---------

docs/sphinx/index.rst

@@ -150,6 +150,9 @@ You can download files from the SAS and place them in your local SAS.  ``sdss_ac
 that mimics the real SAS at Utah.  If you do not already have a `SAS_BASE_DIR` set, one will be defined in your
 home directory, as a new ``sas`` directory.
 
+``sdss_access`` requires valid authentication to download proprietary data.  See :ref:`auth` 
+for more information.
+
 sdss_access has four classes designed to facilitate access to SAS data.
 
 - **Access** - class that automatically decides between `.RsyncAccess` and `.CurlAccess` based on the operating system.
@@ -235,6 +238,37 @@ In all all cases, successful ``sdss_access`` downloads will return a code of 0.
 occurred.  If no verbose message is displayed, you may need to check the ``sdss_access_XX.log`` and ``sdss_access_XX.err``
 files within the temporary directory.
 
+Accessing SDSS-V Products
+-------------------------
+
+As of version 2.0.0, ``sdss_access`` has been updated to support downloading of 
+SDSS-V products.  The usage of ``sdss_access`` remains the same.  The only difference is SDSS-V 
+products are now delivered by the "data.sdss5.org" server instead of "data.sdss.org".  
+When specifying ``release="sdss5"``, you may notice the new server location, e.g.
+::
+
+    >>> from sdss_access import Access
+    >>> access = Access(release='sdss5')
+    >>> access
+    <Access(access_mode="rsync", using="data.sdss5.org")>
+
+As with SDSS-IV, ``sdss_access`` requires valid authentication to download 
+proprietary data for SDSS-V.  See :ref:`auth` for more information.  Here is an example accessing
+the robostrategy completeness files for SDSS-V.
+
+.. warning::
+    The below example contains large data, ~8 GB, and may take a while to download.    
+
+::
+
+    from sdss_access import Access
+    access = Access(release='sdss5')
+    access.remote()
+    access.add('rsCompleteness', observatory='apo', plan='epsilon-2-core-*')
+    access.set_stream()
+    access.commit()
+
+
 Accessing Public Data Products
 ------------------------------
 

docs/sphinx/intro.rst

@@ -17,6 +17,14 @@ SDSSWORK
    :prog: sdsswork
    :templates:
 
+.. _dr17:
+
+DR17
+----
+
+.. datamodel:: sdss_access.path.path:Path
+   :prog: DR17
+   :templates:
 
 .. _dr16:
 
@@ -99,6 +107,23 @@ DR8
    :prog: DR8
    :templates:
 
+.. _mpl11:
+
+MPL11
+-----
+
+.. datamodel:: sdss_access.path.path:Path
+   :prog: MPL11
+   :templates:
+
+.. _mpl10:
+
+MPL10
+-----
+
+.. datamodel:: sdss_access.path.path:Path
+   :prog: MPL10
+   :templates:
 
 .. _mpl9: