add force_clean

.gitignore

@@ -127,4 +127,5 @@ venv.bak/
 dmypy.json
 
 # Pyre type checker
-.pyre/
+.pyre/
+.idea/

Dockerfile

@@ -1,5 +1,7 @@
 FROM ubuntu
 
+ENV PYTHONUNBUFFERED=1
+
 RUN apt update && apt install git python3 python3-pip -y && \
   echo "StrictHostKeyChecking no" >> /etc/ssh/ssh_config
 

action.yml

@@ -44,6 +44,9 @@ inputs:
   force_update:
     description: "Force to update the destination repo, use '-f' flag do 'git push'"
     default: false
+  force_clean:
+    description: "Force clean source repo finally"
+    default: false
   debug:
     description: "Enable the debug flag to show detail log"
     default: false
@@ -70,6 +73,7 @@ runs:
     - ${{ inputs.white_list }}
     - ${{ inputs.static_list }}
     - ${{ inputs.force_update}}
+    - ${{ inputs.force_clean}}
     - ${{ inputs.debug}}
     - ${{ inputs.timeout}}
     - ${{ inputs.mappings}}

entrypoint.sh

@@ -23,6 +23,7 @@ python3 /hub-mirror/hubmirror.py --src "${INPUT_SRC}" --dst "${INPUT_DST}" \
 --white-list "${INPUT_WHITE_LIST}" \
 --static-list "${INPUT_STATIC_LIST}" \
 --force-update "${INPUT_FORCE_UPDATE}" \
+--force-clean "${INPUT_FORCE_CLEAN}" \
 --debug "${INPUT_DEBUG}" \
 --timeout  "${INPUT_TIMEOUT}" \
 --mappings  "${INPUT_MAPPINGS}"

hub-mirror/hub.py

@@ -94,13 +94,12 @@ def create_dst_repo(self, repo_name):
             time.sleep(2)
         return result
 
-    def dynamic_list(self):
-        url = '/'.join(
-            [
-                self.src_base, self.src_account_type+'s', self.src_account,
-                'repos',
-            ]
-        )
+    def src_dynamic_list(self):
+        url = '/'.join([self.src_base, self.src_account_type+'s', self.src_account, 'repos'])
+        return self._get_all_repo_names(url)
+
+    def dst_dynamic_list(self):
+        url = '/'.join([self.dst_base, self.dst_account_type+'s', self.dst_account, 'repos'])
         return self._get_all_repo_names(url)
 
     @functools.lru_cache

hub-mirror/hubmirror.py

@@ -40,11 +40,11 @@ def _create_parser(self):
 
     def test_black_white_list(self, repo):
         if repo in self.black_list:
-            print("Skip, %s in black list: %s" % (repo, self.black_list))
+            print("Skip, %s in black list" % (repo,))
             return False
 
         if self.white_list and repo not in self.white_list:
-            print("Skip, %s not in white list: %s" % (repo, self.white_list))
+            print("Skip, %s not in white list" % (repo,))
             return False
 
         return True
@@ -63,7 +63,11 @@ def run(self):
 
         # Using static list when static_list is set
         repos = self.static_list
-        src_repos = repos if repos else hub.dynamic_list()
+        src_repos = repos if repos else hub.src_dynamic_list()
+
+        force_update = self.args.force_update
+        if force_update is False:
+            src_repos = list(set(src_repos) - set(hub.dst_dynamic_list()) )
 
         total, success, skip = len(src_repos), 0, 0
         failed_list = []
@@ -78,11 +82,13 @@ def run(self):
                         hub, src_repo, dst_repo,
                         cache=self.args.cache_path,
                         timeout=self.args.timeout,
-                        force_update=self.args.force_update,
+                        force_update=force_update,
+                        force_clean=self.args.force_clean,
                     )
                     mirror.download()
                     mirror.create()
                     mirror.push()
+                    mirror.clean()
                     success += 1
                 except Exception as e:
                     print(e)

hub-mirror/mirror.py

@@ -1,17 +1,19 @@
 import re
 import shutil
-import os
+from pathlib import Path
 
 import git
 from tenacity import retry, stop_after_attempt, wait_exponential
 
 from utils import cov2sec
 
+CUR_DIR = Path(__file__).resolve().parent
+
 
 class Mirror(object):
     def __init__(
-        self, hub, src_name, dst_name,
-        cache='.', timeout='0', force_update=False
+            self, hub, src_name, dst_name,
+            cache='.', timeout='0', force_update=False, force_clean=False
     ):
         self.hub = hub
         self.src_name = src_name
@@ -24,17 +26,18 @@ def __init__(
         else:
             self.timeout = 0
         self.force_update = force_update
+        self.force_clean = force_clean
 
     @retry(wait=wait_exponential(), reraise=True, stop=stop_after_attempt(3))
     def _clone(self):
         # TODO: process empty repo
         print("Starting git clone " + self.src_url)
-        mygit = git.cmd.Git(os.getcwd())
+        mygit = git.cmd.Git(CUR_DIR)
         mygit.clone(
             git.cmd.Git.polish_url(self.src_url), self.repo_path,
             kill_after_timeout=self.timeout
         )
-        print("Clone completed: %s" % os.getcwd() + self.repo_path)
+        print("Clone completed: %s" % self.repo_path)
 
     @retry(wait=wait_exponential(), reraise=True, stop=stop_after_attempt(3))
     def _update(self, local_repo):
@@ -94,3 +97,8 @@ def push(self, force=False):
             print("(3/3) Force pushing...")
             cmd = ['-f'] + cmd
             local_repo.git.push(*cmd, kill_after_timeout=self.timeout)
+
+    def clean(self):
+        if self.force_clean:
+            print("Force Clean...")
+            shutil.rmtree(self.repo_path, True)