Merge pull request #280 from PNNL-CompBio/panc_pdo

Addition of pancreaticPDO dataset.

Author: jjacobson95

.github/workflows/build.yml

@@ -177,6 +177,30 @@ jobs:
           push: true
           platforms: linux/amd64
 
+  build-pancpdo:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      - name: Login to DockerHub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_PASSWORD }}
+      - name: Build and push pancpdo
+        uses: docker/build-push-action@v3
+        with:
+          file: ./build/docker/Dockerfile.pancpdo
+          tags: |
+            sgosline/pancpdo:latest
+            sgosline/pancpdo:${{ github.ref_name }}
+          push: true
+          platforms: linux/amd64
+
   build-upload:
     runs-on: ubuntu-latest
     steps:

.gitignore

@@ -18,3 +18,4 @@ tests/__pycache__
 dist
 build/lib
 build/local
+local/

build/build_dataset.py

@@ -42,6 +42,7 @@ def process_docker(dataset,validate):
         'beataml': ['beataml'],
         'mpnst': ['mpnst'],
         'mpnstpdx': ['mpnstpdx'],
+        'pancpdo': ['pancpdo'],
         'cptac': ['cptac'],
         'genes': ['genes'],
         'upload': ['upload']
@@ -123,6 +124,7 @@ def process_omics(executor, dataset, should_continue):
         'broad_sanger': ['copy_number', 'mutations', 'proteomics', 'transcriptomics'],
         'cptac': ['copy_number', 'mutations', 'proteomics', 'transcriptomics'],
         'hcmi': ['mutations', 'transcriptomics'],
+        'pancpdo': ['transcriptomics'],
         'mpnstpdx':['copy_number', 'mutations', 'proteomics', 'transcriptomics']
     }
 

build/docker/Dockerfile.pancpdo

@@ -0,0 +1,21 @@
+FROM python:3.9
+
+WORKDIR /usr/src/app
+
+COPY build/pancpdo/01-createPancPDOSamplesFile.py .
+COPY build/pancpdo/02-getPancPDOData.py .
+COPY build/pancpdo/03-getPancPDODrugs.py .
+COPY build/pancpdo/04-getPancPDOExperiments.py .
+COPY build/pancpdo/05-addPrecalcAUC.py .
+COPY build/pancpdo/full_manifest.txt .
+COPY build/pancpdo/requirements.txt .
+COPY build/pancpdo/*sh ./
+COPY build/pancpdo/pancpdo_cancer_types.csv ./
+COPY build/utils/* ./
+
+# Set MPLCONFIGDIR to a writable directory
+ENV MPLCONFIGDIR=/app/tmp/matplotlib
+RUN mkdir -p /app/tmp/matplotlib
+
+RUN pip install --no-cache-dir -r requirements.txt
+VOLUME ['/tmp']

build/docker/docker-compose.yml

@@ -26,6 +26,14 @@ services:
         HTTPS_PROXY: ${HTTPS_PROXY}
     platform: linux/amd64
     image: hcmi:latest
+  pancpdo:
+    build:
+      context: ../../
+      dockerfile: build/docker/Dockerfile.pancpdo
+      args:
+        HTTPS_PROXY: ${HTTPS_PROXY}
+    platform: linux/amd64
+    image: pancpdo:latest
 
   beataml:
     build:

build/hcmi/02-getHCMIData.py

@@ -36,6 +36,30 @@ def download_tool(url):
     # Download the file
     print("Downloading tool...")
     filename = wget.download(url)
+##commented due to merge conflict
+#    files_before = os.listdir()
+#    # shutil.unpack_archive(filename)
+#    ##there are two files to unpack
+#    print('Unpacking platform-specific path')
+#    shutil.unpack_archive(os.path.basename(url))
+#    #This is just set for AWS to debug. This will have to be mapped to OS.  They changed their file structure. This should be updated.
+#    print('Unpacking secondary zip')
+#    fnames={
+#        'Darwin':"gdc-client_2.3_OSX_x64.zip",
+#        'Linux':"gdc-client_2.3_Ubuntu_x64.zip",
+#        'Windows':"gdc-client_2.3_Windows_x64.zip"
+#        }
+#    shutil.unpack_archive(fnames[platform.system()]) 
+#    #This is just set for AWS to debug. This will have to be mapped to OS.  They changed their file structure. This should be updated.
+#    shutil.unpack_archive("gdc-client_2.3_Ubuntu_x64.zip") 
+#    if not os.path.exists('gdc-client'):
+#        raise FileNotFoundError("gdc-client executable not found after extraction.")
+#    # Ensure 'gdc-client' is executable
+#    st = os.stat('gdc-client')
+#    os.chmod('gdc-client', st.st_mode | stat.S_IEXEC)
+#    # Return the path to the executable
+#    return './gdc-client'
+
 
     # First extraction
     print(f"\nExtracting {filename}...")
@@ -65,6 +89,7 @@ def download_tool(url):
 
     return gdc_client_path
 
+
 def is_tool(name):
     """
     Check if a specific tool is available on the system.