[New features] complete auto-deploy (#4175)

* complete auto-deploy

* update manual deploy for paddlenlp

* fix typo

* Update paddlenlp/version.py

Co-authored-by: Sijun He <[email protected]>

* update auto-deploy

Co-authored-by: Sijun He <[email protected]>

Author: wj-Mcat

.flake8

@@ -1,5 +1,7 @@
 [flake8]
 ignore = E203, E501, E741, W503, W605
 max-line-length = 119
+
+# E402: module level import not at top of file
 per-file-ignores =
-    __init__.py:F401,F403
+    __init__.py:F401,F403,E402

.github/workflows/pypi.yml

@@ -1,63 +1,20 @@
 name: PyPI
 
-on: [push, pull_request]
+on: 
+  workflow_dispatch:
+    inputs:
+      package:
+        description: 'Package to be deployed'
+        required: true
+        default: 'paddlenlp'
+        type: choice
+        options:
+        - paddlenlp
+        - paddle-pipelines
+        - ppdiffusers
 
 jobs:
-  build:
-    name: Build
-    runs-on: ubuntu-20.04
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v1
-        with:
-          python-version: 3.8
-      - name: Install dependencies
-        run: |
-          python -m pip install --upgrade pip
-          make install
-
-  ppdiffusers:
-    name: Pack
-    needs: build
-    runs-on: ubuntu-20.04
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v1
-        with:
-          python-version: 3.8
-      - name: Install dependencies
-        run: |
-          python -m pip install --upgrade pip
-          pip install setuptools wheel twine
-          make install-ppdiffusers
-
-      - name: Should Deploy to Pypi Server
-        id: should-deploy
-        run: python scripts/should_deploy.py  --name ppdiffusers >> $GITHUB_OUTPUT
-      - name: Check Branch
-        id: check-branch
-        run: |
-          if [[ ${{ github.ref }} =~ ^refs/heads/(develop)$ ]]; then
-              echo "match=true" >> $GITHUB_OUTPUT
-          fi  # See: https://stackoverflow.com/a/58869470/1123955
-
-      - name: Is A Publish Branch
-        if: steps.check-branch.outputs.match == 'true' && steps.should-deploy.outputs.should_deploy == 'true'
-        env:
-          TWINE_USERNAME: paddle-dev
-          TWINE_PASSWORD: ${{ secrets.paddlenlp }}
-        run: |
-          make deploy-ppdiffusers
-
-      - name: Should Not Deploy To Pypi Server
-        if: steps.should-deploy.outputs.should_deploy != 'true'
-        run: echo 'should not deploy pypi server'
-
-      - name: Is Not A Publish Branch
-        if: steps.check-branch.outputs.match != 'true'
-        run: echo 'Not A Publish Branch'
-
-  paddle_pipelines:
+  Deployment:
     name: Pack
     needs: build
     runs-on: ubuntu-20.04
@@ -70,11 +27,12 @@ jobs:
         run: |
           python -m pip install --upgrade pip
           pip install setuptools wheel twine
-          make install-paddle-pipelines
+      - name: Deploy Package
+        id: deploy-package
+        env: 
+          PACKAGE: ${{ inputs.package }}
+        run: python scripts/should_deploy.py  --name $PACKAGE >> $GITHUB_OUTPUT
 
-      - name: Should Deploy to Pypi Server
-        id: should-deploy
-        run: python scripts/should_deploy.py  --name paddle-pipelines >> $GITHUB_OUTPUT
       - name: Check Branch
         id: check-branch
         run: |
@@ -87,8 +45,9 @@ jobs:
         env:
           TWINE_USERNAME: paddle-dev
           TWINE_PASSWORD: ${{ secrets.paddlenlp }}
+          PACKAGE: ${{ inputs.package }}
         run: |
-          make deploy-paddle-pipelines
+          make deploy-$PACKAGE
 
       - name: Should Not Deploy To Pypi Server
         if: steps.should-deploy.outputs.should_deploy != 'true'

Makefile

@@ -50,17 +50,19 @@ install:
 
 .PHONY: deploy-ppdiffusers
 deploy-ppdiffusers:
-	cd ppdiffusers && make
-
-.PHONY: install-ppdiffusers
-install-ppdiffusers:
-	cd ppdiffusers && make install
+	cd ppdiffusers && make install && make
 
 .PHONY: deploy-paddle-pipelines
 deploy-paddle-pipelines:
-	cd pipelines && make
-
-.PHONY: install-paddle-pipelines
-install-paddle-pipelines:
-	cd pipelines && make install
-
+	cd pipelines && make install && make
+
+.PHONY: deploy-paddlenlp
+deploy-paddlenlp:
+	# install related package
+	make install
+	# deploy version
+	echo "VERSION = '$$(cat VERSION)'" > paddlenlp/version.py
+	# build
+	python3 setup.py sdist bdist_wheel
+	# upload
+	twine upload --skip-existing dist/*

VERSION

@@ -1 +1 @@
-4.2.3
+2.4.5

paddlenlp/__init__.py

@@ -12,7 +12,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-__version__ = "2.4.0.dev"  # Maybe dev is better
+from .version import VERSION
+
+__version__ = VERSION  # Maybe dev is better
 import sys
 
 if "datasets" in sys.modules.keys():
@@ -23,47 +25,50 @@
         "This may cause PaddleNLP datasets to be unavalible in intranet. "
         "Please import paddlenlp before datasets module to avoid download issues"
     )
-from . import data
-from . import datasets
-from . import embeddings
-from . import ops
-from . import layers
-from . import metrics
-from . import seq2vec
-from . import transformers
-from . import utils
-from . import losses
-from . import experimental
-from .taskflow import Taskflow
-from .server import SimpleServer
-from . import trainer
-from . import prompt
-from . import dataaug
 import paddle
 
+from . import (
+    data,
+    dataaug,
+    datasets,
+    embeddings,
+    experimental,
+    layers,
+    losses,
+    metrics,
+    ops,
+    prompt,
+    seq2vec,
+    trainer,
+    transformers,
+    utils,
+)
+from .server import SimpleServer
+from .taskflow import Taskflow
+
 paddle.disable_signal_handler()
 
+import sys
+import warnings
+
+# BatchSampler use `_non_static_mode` which is not included in version <= 2.3,
+# thus use `in_dynamic_mode` instead.
+from paddle import in_dynamic_mode
+
 # Patches for DataLoader/BatchSamper to allow using other than paddle.io.Dataset
 # in Paddle version lower than 2.3
 from paddle.fluid.reader import (
-    _current_expected_place,
-    _get_paddle_place_list,
-    _get_paddle_place,
-    _convert_places,
+    BatchSampler,
     IterableDataset,
+    _convert_places,
+    _current_expected_place,
     _DatasetKind,
+    _get_paddle_place,
+    _get_paddle_place_list,
     _InfiniteIterableSampler,
-    BatchSampler,
     use_pinned_memory,
 )
 
-# BatchSampler use `_non_static_mode` which is not included in version <= 2.3,
-# thus use `in_dynamic_mode` instead.
-from paddle import in_dynamic_mode
-
-import warnings
-import sys
-
 
 def _patch_data_loader_init(
     self,
@@ -157,7 +162,11 @@ def _patch_data_loader_init(
     self._iterator = None
 
 
-from paddle.fluid.dataloader.batch_sampler import Sampler, RandomSampler, SequenceSampler
+from paddle.fluid.dataloader.batch_sampler import (
+    RandomSampler,
+    Sampler,
+    SequenceSampler,
+)
 
 
 def _patch_batch_sampler_init(self, dataset=None, sampler=None, shuffle=False, batch_size=1, drop_last=False):

paddlenlp/version.py

@@ -0,0 +1,17 @@
+# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This file is generated automatically by Github Actions
+# DO NOT MODIFY MANUALLY
+VERSION = "0.0.0"

scripts/should_deploy.py

@@ -18,6 +18,8 @@
 import subprocess
 import sys
 
+from packaging import version
+
 PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 
 
@@ -57,35 +59,23 @@ def read_version_of_local_package(version_file_path: str) -> str:
     return version
 
 
-def should_ppdiffusers_deploy():
-    """print the result to terminal"""
-    local_version_file = os.path.join(PROJECT_ROOT, "ppdiffusers/VERSION")
-    remote_version = read_version_of_remote_package("ppdiffusers")
-    local_version = read_version_of_local_package(local_version_file)
-
-    should_deploy = str(remote_version != local_version).lower()
-    print(f"should_deploy={should_deploy}")
-
-
-def should_paddle_pipelines_deploy():
-    """print the result to terminal"""
-    local_version_file = os.path.join(PROJECT_ROOT, "pipelines/VERSION")
-    remote_version = read_version_of_remote_package("pipelines")
-    local_version = read_version_of_local_package(local_version_file)
-
-    should_deploy = str(remote_version != local_version).lower()
-    print(f"should_deploy={should_deploy}")
-
-
 if __name__ == "__main__":
     parser = argparse.ArgumentParser()
     parser.add_argument("--name", required=True)
 
     args = parser.parse_args()
 
-    if args.name == "ppdiffusers":
-        should_ppdiffusers_deploy()
-    elif args.name == "paddle-pipelines":
-        should_paddle_pipelines_deploy()
-    else:
+    version_file_map = {
+        "paddlenlp": "VERSION",
+        "ppdiffusers": "ppdiffusers/VERSION",
+        "paddle-pipelines": "pipelines/VERSION",
+    }
+    if args.name not in version_file_map:
         raise ValueError(f"package<{args.name}> not supported")
+
+    local_version_file = os.path.join(PROJECT_ROOT, version_file_map[args.name])
+    remote_version = read_version_of_remote_package(args.name)
+    local_version = read_version_of_local_package(local_version_file)
+
+    should_deploy = str(version.parse(remote_version) < version.parse(local_version)).lower()
+    print(f"should_deploy={should_deploy}")