current state

Author: TejasMorbagal

deep_code/tests/tools/test_new.py

@@ -0,0 +1,59 @@
+import unittest
+import shutil
+from pathlib import Path
+from unittest.mock import patch, MagicMock
+
+from deep_code.tools.new import RepositoryInitializer
+
+
+class TestRepositoryInitializer(unittest.TestCase):
+    def setUp(self):
+        """Set up a temporary directory for testing."""
+        self.test_repo_name = "test_repo"
+        self.test_repo_dir = Path(self.test_repo_name).absolute()
+        self.templates_dir = Path(__file__).parent.parent / "deep_code" / "templates"
+
+        # Mock GitHub credentials
+        self.github_username = "test_user"
+        self.github_token = "test_token"
+
+        # Ensure the repository directory does not exist before each test
+        if self.test_repo_dir.exists():
+            shutil.rmtree(self.test_repo_dir)
+
+    def tearDown(self):
+        """Clean up the temporary directory after each test."""
+        if self.test_repo_dir.exists():
+            shutil.rmtree(self.test_repo_dir)
+
+    @patch("deep_code.tools.new.read_git_access_file")
+    def test_missing_github_credentials(self, mock_read_git_access_file):
+        """Test that an error is raised if GitHub credentials are missing."""
+        # Mock the Git access file with missing credentials
+        mock_read_git_access_file.return_value = {}
+
+        # Verify that an error is raised
+        with self.assertRaises(ValueError) as context:
+            RepositoryInitializer(self.test_repo_name)
+        self.assertIn("GitHub credentials are missing", str(context.exception))
+
+    @patch("deep_code.tools.new.read_git_access_file")
+    def test_template_not_found(self, mock_read_git_access_file):
+        """Test that an error is raised if a template file is missing."""
+        # Mock the Git access file
+        mock_read_git_access_file.return_value = {
+            "github-username": self.github_username,
+            "github-token": self.github_token,
+        }
+
+        # Initialize the repository with a non-existent template
+        initializer = RepositoryInitializer(self.test_repo_name)
+        initializer.templates_dir = Path("/non/existent/path")
+
+        # Verify that an error is raised
+        with self.assertRaises(FileNotFoundError):
+            initializer.initialize()
+
+
+if __name__ == "__main__":
+    unittest.main()

deep_code/tools/new.py

@@ -10,4 +10,135 @@
 pyproject.toml), and a template setup for documentation (e.g., using mkdocs),
 setup of the build pipeline"""
 
+import subprocess
+from pathlib import Path
+import logging
 
+import requests
+
+from deep_code.utils.helper import read_git_access_file
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+
+class RepositoryInitializer:
+    """
+    A utility class to initialize a GitHub repository with configuration files,
+    a workflow notebook template, and a Python package template for DeepESDL experiment
+    """
+
+    def __init__(self, repo_name: str):
+        """
+        Initialize the RepositoryInitializer.
+        """
+        self.repo_name = repo_name
+        self.repo_dir = Path(repo_name).absolute()
+        self.templates_dir = Path(__file__).parent / "templates"
+        git_config = read_git_access_file()
+        self.github_username = git_config.get("github-username")
+        self.github_token = git_config.get("github-token")
+        if not self.github_username or not self.github_token:
+            raise ValueError("GitHub credentials are missing in the `.gitaccess` file.")
+
+    def create_local_repo(self) -> None:
+        """Create a local directory for the repository and initialize it as a Git repository."""
+        logger.info(f"Creating local repository: {self.repo_dir}")
+        self.repo_dir.mkdir(parents=True, exist_ok=True)
+        subprocess.run(["git", "init"], cwd=self.repo_dir, check=True)
+
+    def _load_template(self, template_name: str) -> str:
+        """Load a template file from the templates directory."""
+        template_path = self.templates_dir / template_name
+        if not template_path.exists():
+            raise FileNotFoundError(f"Template not found: {template_path}")
+        with open(template_path, "r") as file:
+            return file.read()
+
+    def create_config_files(self) -> None:
+        """Create configuration files in the repository."""
+        logger.info("Creating configuration files...")
+
+        # Create README.md
+        readme_content = (f"# {self.repo_name}\n\nThis repository contains workflows "
+                          f"and Python code for a DeepESDL Experiment.")
+        (self.repo_dir / "README.md").write_text(readme_content)
+
+        # Create .gitignore
+        gitignore_content = self._load_template(".gitignore")
+        (self.repo_dir / ".gitignore").write_text(gitignore_content)
+
+    def create_jupyter_notebook_template(self) -> None:
+        """Create a workflow notebook template (workflow.ipynb)."""
+        logger.info("Creating workflow notebook template...")
+        workflow_content = self._load_template("workflow.ipynb")
+        (self.repo_dir / "workflow.ipynb").write_text(workflow_content)
+
+    def create_python_package(self) -> None:
+        """Create a Python package template with a pyproject.toml file."""
+        logger.info("Creating Python package template...")
+
+        # Create package directory
+        package_dir = self.repo_dir / self.repo_name
+        package_dir.mkdir(exist_ok=True)
+
+        # Create __init__.py
+        (package_dir / "__init__.py").write_text("# Package initialization\n")
+
+        # Create pyproject.toml
+        pyproject_content = self._load_template("pyproject.toml")
+        pyproject_content = pyproject_content.replace("{repo_name}", self.repo_name)
+        (self.repo_dir / "pyproject.toml").write_text(pyproject_content)
+
+    def create_github_repo(self) -> None:
+        """Create a remote GitHub repository and push the local repository."""
+        if not self.github_username or not self.github_token:
+            logger.warning("GitHub credentials not provided. Skipping remote repository creation.")
+            return
+
+        logger.info("Creating remote GitHub repository...")
+
+        repo_url = "https://api.github.com/user/repos"
+        repo_data = {
+            "name": self.repo_name,
+            "description": "Repository for DeepESDL workflows and Python code.",
+            "private": True,
+        }
+        headers = {
+            "Authorization": f"token {self.github_token}",
+            "Accept": "application/vnd.github.v3+json",
+        }
+        response = requests.post(repo_url, json=repo_data, headers=headers)
+        response.raise_for_status()
+
+        remote_url = f"https://github.com/{self.github_username}/{self.repo_name}.git"
+        subprocess.run(["git", "remote", "add", "origin", remote_url], cwd=self.repo_dir, check=True)
+        subprocess.run(["git", "add", "."], cwd=self.repo_dir, check=True)
+        subprocess.run(["git", "commit", "-m", "Initial commit"], cwd=self.repo_dir, check=True)
+        subprocess.run(["git", "push", "-u", "origin", "main"], cwd=self.repo_dir, check=True)
+
+        logger.info(f"Remote repository created: {remote_url}")
+
+    def create_github_actions_workflow(self) -> None:
+        """Create a GitHub Actions workflow for running unit tests."""
+        logger.info("Creating GitHub Actions workflow...")
+
+        workflows_dir = self.repo_dir / ".github" / "workflows"
+        workflows_dir.mkdir(parents=True, exist_ok=True)
+
+        workflow_content = self._load_template("unit-tests.yml")
+        (workflows_dir / "unit-tests.yml").write_text(workflow_content)
+
+    def initialize(self) -> None:
+        """Initialize the repository with all templates and configurations."""
+        self.create_local_repo()
+        self.create_config_files()
+        self.create_jupyter_notebook_template()
+        self.create_python_package()
+        self.create_github_repo()
+        logger.info(f"Repository '{self.repo_name}' initialized successfully!")
+
+
+if __name__ == '__main__':
+    r = RepositoryInitializer("deepesdl-test-experiment")
+    r.initialize()

deep_code/tools/publish.py

@@ -22,7 +22,7 @@
 )
 from deep_code.utils.dataset_stac_generator import OscDatasetStacGenerator
 from deep_code.utils.github_automation import GitHubAutomation
-from deep_code.utils.helper import serialize
+from deep_code.utils.helper import serialize, read_git_access_file
 from deep_code.utils.ogc_api_record import (
     ExperimentAsOgcRecord,
     LinksBuilder,
@@ -42,8 +42,7 @@ class GitHubPublisher:
     """
 
     def __init__(self):
-        with fsspec.open(".gitaccess", "r") as file:
-            git_config = yaml.safe_load(file) or {}
+        git_config = read_git_access_file()
         self.github_username = git_config.get("github-username")
         self.github_token = git_config.get("github-token")
         if not self.github_username or not self.github_token:

deep_code/tools/templates/.gitignore

@@ -0,0 +1,14 @@
+# Ignore Python compiled files
+__pycache__/
+*.pyc
+*.pyo
+*.pyd
+
+# Ignore Jupyter notebook checkpoints
+.ipynb_checkpoints/
+
+# Ignore virtual environments
+venv/
+env/
+
+.gitaccess

deep_code/tools/templates/pyproject.toml

@@ -0,0 +1,15 @@
+[build-system]
+requires = ['setuptools>=61.2.0', 'wheel', 'build']
+build-backend = 'setuptools.build_meta'
+
+[project]
+name = '{repo_name}'
+version = '0.1.0'
+description = 'A Python package for DeepESDL workflows.'
+authors = [
+    {name = 'Your Name', email = '[email protected]'}
+]
+dependencies = [
+    'numpy',
+    'pandas',
+]

deep_code/tools/templates/workflow-template.yml

@@ -0,0 +1,31 @@
+name: Unit Tests
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.12'
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install .[dev]
+
+      - name: Run unit tests
+        run: |
+          pytest tests/

deep_code/tools/templates/workflow.ipynb

@@ -0,0 +1,47 @@
+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "# Workflow Notebook\n\n",
+        "This notebook provides a template for running DeepESDL workflows.\n",
+        "Modify this notebook to implement your specific workflow."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "# Import required libraries\n",
+        "import numpy as np\n",
+        "import pandas as pd\n",
+        "\n",
+        "# Add your workflow code here\n"
+      ]
+    }
+  ],
+  "metadata": {
+    "kernelspec": {
+      "display_name": "Python 3",
+      "language": "python",
+      "name": "python3"
+    },
+    "language_info": {
+      "codemirror_mode": {
+        "name": "ipython",
+        "version": 3
+      },
+      "file_extension": ".py",
+      "mimetype": "text/x-python",
+      "name": "python",
+      "nbconvert_exporter": "python",
+      "pygments_lexer": "ipython3",
+      "version": "3.10"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 4
+}

deep_code/utils/helper.py

@@ -1,3 +1,7 @@
+import fsspec
+import yaml
+
+
 def serialize(obj):
     """Convert non-serializable objects to JSON-compatible formats.
     Args:
@@ -12,3 +16,7 @@ def serialize(obj):
     if hasattr(obj, "__dict__"):
         return obj.__dict__
     raise TypeError(f"Object of type {type(obj).__name__} is not JSON serializable")
+
+def read_git_access_file():
+    with fsspec.open(".gitaccess", "r") as file:
+        return yaml.safe_load(file) or {}