update 2.0 public api in vision (#33307)

* update 2.0 public api in vision

* fix some flake8 errors

Author: zhiboniu

python/paddle/hapi/callbacks.py

@@ -324,7 +324,7 @@ class ProgBarLogger(Callback):
             ])
             train_dataset = MNIST(mode='train', transform=transform)
 
-            lenet = paddle.vision.LeNet()
+            lenet = paddle.vision.models.LeNet()
             model = paddle.Model(lenet,
                 inputs, labels)
 
@@ -554,7 +554,7 @@ class ModelCheckpoint(Callback):
             ])
             train_dataset = MNIST(mode='train', transform=transform)
 
-            lenet = paddle.vision.LeNet()
+            lenet = paddle.vision.models.LeNet()
             model = paddle.Model(lenet,
                 inputs, labels)
 
@@ -614,7 +614,7 @@ class LRScheduler(Callback):
             ])
             train_dataset = paddle.vision.datasets.MNIST(mode='train', transform=transform)
 
-            lenet = paddle.vision.LeNet()
+            lenet = paddle.vision.models.LeNet()
             model = paddle.Model(lenet,
                 inputs, labels)
 
@@ -630,7 +630,7 @@ def make_optimizer(parameters=None):
                     boundaries=boundaries, values=values)
                 learning_rate = paddle.optimizer.lr.LinearWarmup(
                     learning_rate=learning_rate,
-                    warmup_steps=wamup_epochs,
+                    warmup_steps=wamup_steps,
                     start_lr=base_lr / 5.,
                     end_lr=base_lr,
                     verbose=True)
@@ -856,7 +856,7 @@ class VisualDL(Callback):
             train_dataset = paddle.vision.datasets.MNIST(mode='train', transform=transform)
             eval_dataset = paddle.vision.datasets.MNIST(mode='test', transform=transform)
 
-            net = paddle.vision.LeNet()
+            net = paddle.vision.models.LeNet()
             model = paddle.Model(net, inputs, labels)
 
             optim = paddle.optimizer.Adam(0.001, parameters=net.parameters())

python/paddle/hapi/model.py

@@ -30,20 +30,28 @@
 import paddle
 from paddle import fluid
 from paddle.fluid import core
-from paddle.fluid.framework import in_dygraph_mode, Variable, ParamBase, _current_expected_place
-from paddle.fluid.framework import in_dygraph_mode, Variable, _get_paddle_place
+from paddle.fluid.framework import in_dygraph_mode
+from paddle.fluid.framework import Variable
+from paddle.fluid.framework import ParamBase
+from paddle.fluid.framework import _current_expected_place
+from paddle.fluid.framework import _get_paddle_place
 from paddle.fluid.framework import _current_expected_place as _get_device
 from paddle.fluid.executor import global_scope
 from paddle.fluid.io import is_belong_to_optimizer
 from paddle.fluid.dygraph.base import to_variable
 from paddle.fluid.dygraph.parallel import ParallelEnv
-from paddle.fluid.dygraph.dygraph_to_static.program_translator import ProgramTranslator, FunctionSpec
-from paddle.fluid.dygraph.io import INFER_MODEL_SUFFIX, INFER_PARAMS_SUFFIX
+from paddle.fluid.dygraph.dygraph_to_static.program_translator import ProgramTranslator
+from paddle.fluid.dygraph.dygraph_to_static.program_translator import FunctionSpec
+from paddle.fluid.dygraph.io import INFER_MODEL_SUFFIX
+from paddle.fluid.dygraph.io import INFER_PARAMS_SUFFIX
 from paddle.fluid.layers.utils import flatten
 from paddle.fluid.layers import collective
 
-from paddle.io import DataLoader, Dataset, DistributedBatchSampler
-from paddle.fluid.executor import scope_guard, Executor
+from paddle.io import DataLoader
+from paddle.io import Dataset
+from paddle.io import DistributedBatchSampler
+from paddle.fluid.executor import scope_guard
+from paddle.fluid.executor import Executor
 from paddle.fluid.dygraph.layers import Layer
 from paddle.metric import Metric
 from paddle.static import InputSpec as Input
@@ -166,7 +174,6 @@ def init_communicator(program, rank, nranks, wait_port, current_endpoint,
             name=unique_name.generate('hccl_id'),
             persistable=True,
             type=core.VarDesc.VarType.RAW)
-        endpoint_to_index_map = {e: idx for idx, e in enumerate(endpoints)}
         block.append_op(
             type='c_gen_hccl_id',
             inputs={},
@@ -710,10 +717,10 @@ def train_batch(self, inputs, labels=None):
                 enable=self._amp_level != 'O0', **self._amp_custom_lists):
             if self._nranks > 1:
                 outputs = self.ddp_model.forward(
-                    * [to_variable(x) for x in inputs])
+                    *[to_variable(x) for x in inputs])
             else:
                 outputs = self.model.network.forward(
-                    * [to_variable(x) for x in inputs])
+                    *[to_variable(x) for x in inputs])
 
             losses = self.model._loss(*(to_list(outputs) + labels))
             losses = to_list(losses)
@@ -732,7 +739,7 @@ def train_batch(self, inputs, labels=None):
         metrics = []
         for metric in self.model._metrics:
             metric_outs = metric.compute(*(to_list(outputs) + labels))
-            m = metric.update(* [to_numpy(m) for m in to_list(metric_outs)])
+            m = metric.update(*[to_numpy(m) for m in to_list(metric_outs)])
             metrics.append(m)
 
         return ([to_numpy(l) for l in losses], metrics) \
@@ -746,7 +753,7 @@ def eval_batch(self, inputs, labels=None):
         labels = labels or []
         labels = [to_variable(l) for l in to_list(labels)]
 
-        outputs = self.model.network.forward(* [to_variable(x) for x in inputs])
+        outputs = self.model.network.forward(*[to_variable(x) for x in inputs])
         if self.model._loss:
             losses = self.model._loss(*(to_list(outputs) + labels))
             losses = to_list(losses)
@@ -777,7 +784,7 @@ def eval_batch(self, inputs, labels=None):
                     self._merge_count[self.mode + '_batch'] = samples
 
             metric_outs = metric.compute(*(to_list(outputs) + labels))
-            m = metric.update(* [to_numpy(m) for m in to_list(metric_outs)])
+            m = metric.update(*[to_numpy(m) for m in to_list(metric_outs)])
             metrics.append(m)
 
         if self.model._loss and len(metrics):
@@ -1363,8 +1370,9 @@ def _check_pure_fp16_configs():
             # pure float16 training has some restricts now
             if self._adapter._amp_level == "O2":
                 if in_dygraph_mode():
-                    warnings.warn("Pure float16 training is not supported in dygraph mode now, "\
-                        "and it will be supported in future version.")
+                    warnings.warn(
+                        "Pure float16 training is not supported in dygraph mode now, and it will be supported in future version."
+                    )
                 else:
                     # grad clip is not supported in pure fp16 training now
                     assert self._optimizer._grad_clip is None, \
@@ -1398,8 +1406,7 @@ def _check_pure_fp16_configs():
 
         if 'use_pure_fp16' in amp_configs:
             raise ValueError(
-                "''use_pure_fp16' is an invalid parameter, "
-                "the level of mixed precision training only depends on 'O1' or 'O2'."
+                "'use_pure_fp16' is an invalid parameter, the level of mixed precision training only depends on 'O1' or 'O2'."
             )
 
         _check_pure_fp16_configs()
@@ -1427,9 +1434,8 @@ def _check_amp_configs(amp_config_key_set):
             }
             if amp_config_key_set - accepted_param_set:
                 raise ValueError(
-                    "Except for 'level', the keys of 'amp_configs' must be accepted by mixed precision APIs, "
-                    "but {} could not be recognized.".format(
-                        tuple(amp_config_key_set - accepted_param_set)))
+                    "Except for 'level', the keys of 'amp_configs' must be accepted by mixed precision APIs, but {} could not be recognized.".
+                    format(tuple(amp_config_key_set - accepted_param_set)))
 
             if 'use_fp16_guard' in amp_config_key_set:
                 if in_dygraph_mode():
@@ -1501,8 +1507,9 @@ def prepare(self, optimizer=None, loss=None, metrics=None,
         self._optimizer = optimizer
         if loss is not None:
             if not isinstance(loss, paddle.nn.Layer) and not callable(loss):
-                raise TypeError("'loss' must be sub classes of " \
-                    "`paddle.nn.Layer` or any callable function.")
+                raise TypeError(
+                    "'loss' must be sub classes of `paddle.nn.Layer` or any callable function."
+                )
         self._loss = loss
 
         metrics = metrics or []
@@ -2080,7 +2087,7 @@ def summary(self, input_size=None, dtype=None):
               input = InputSpec([None, 1, 28, 28], 'float32', 'image')
               label = InputSpec([None, 1], 'int64', 'label')
            
-              model = paddle.Model(paddle.vision.LeNet(),
+              model = paddle.Model(paddle.vision.models.LeNet(),
                   input, label)
               optim = paddle.optimizer.Adam(
                   learning_rate=0.001, parameters=model.parameters())
@@ -2122,9 +2129,11 @@ def _verify_spec(self, specs, shapes=None, dtypes=None, is_input=False):
             else:
                 out_specs = to_list(specs)
         elif isinstance(specs, dict):
-            assert is_input == False
-            out_specs = [specs[n] \
-                for n in extract_args(self.network.forward) if n != 'self']
+            assert is_input is False
+            out_specs = [
+                specs[n] for n in extract_args(self.network.forward)
+                if n != 'self'
+            ]
         else:
             out_specs = to_list(specs)
         # Note: checks each element has specificed `name`.

python/paddle/metric/metrics.py

@@ -222,7 +222,7 @@ class Accuracy(Metric):
           transform = T.Compose([T.Transpose(), T.Normalize([127.5], [127.5])])
           train_dataset = MNIST(mode='train', transform=transform)
 
-          model = paddle.Model(paddle.vision.LeNet(), input, label)
+          model = paddle.Model(paddle.vision.models.LeNet(), input, label)
           optim = paddle.optimizer.Adam(
               learning_rate=0.001, parameters=model.parameters())
           model.prepare(

python/paddle/tests/test_callback_visualdl.py

@@ -55,7 +55,7 @@ def test_visualdl_callback(self):
         train_dataset = MnistDataset(mode='train', transform=transform)
         eval_dataset = MnistDataset(mode='test', transform=transform)
 
-        net = paddle.vision.LeNet()
+        net = paddle.vision.models.LeNet()
         model = paddle.Model(net, inputs, labels)
 
         optim = paddle.optimizer.Adam(0.001, parameters=net.parameters())

python/paddle/vision/__init__.py

@@ -11,22 +11,59 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+import paddle
+import paddle.nn as nn
+from . import models  # noqa: F401
+from . import transforms  # noqa: F401
+from . import datasets  # noqa: F401
+from . import ops  # noqa: F401
+from .image import set_image_backend  # noqa: F401
+from .image import get_image_backend  # noqa: F401
+from .image import image_load  # noqa: F401
+from .models import LeNet as models_LeNet
+import paddle.utils.deprecated as deprecated
 
-from . import models
-from .models import *
+__all__ = [  #noqa
+    'set_image_backend', 'get_image_backend', 'image_load'
+]
 
-from . import transforms
-from .transforms import *
 
-from . import datasets
-from .datasets import *
+class LeNet(models_LeNet):
+    """LeNet model from
+    `"LeCun Y, Bottou L, Bengio Y, et al. Gradient-based learning applied to document recognition[J]. Proceedings of the IEEE, 1998, 86(11): 2278-2324.`_
 
-from . import image
-from .image import *
+    Args:
+        num_classes (int): output dim of last fc layer. If num_classes <=0, last fc layer 
+                            will not be defined. Default: 10.
 
-from . import ops
+    Examples:
+        .. code-block:: python
 
-__all__ = models.__all__ \
-        + transforms.__all__ \
-        + datasets.__all__ \
-        + image.__all__
+            from paddle.vision.models import LeNet
+
+            model = LeNet()
+    """
+
+    @deprecated(
+        since="2.0.0",
+        update_to="paddle.vision.models.LeNet",
+        level=1,
+        reason="Please use new API in models, paddle.vision.LeNet will be removed in future"
+    )
+    def __init__(self, num_classes=10):
+        super(LeNet, self).__init__(num_classes=10)
+        self.num_classes = num_classes
+        self.features = nn.Sequential(
+            nn.Conv2D(
+                1, 6, 3, stride=1, padding=1),
+            nn.ReLU(),
+            nn.MaxPool2D(2, 2),
+            nn.Conv2D(
+                6, 16, 5, stride=1, padding=0),
+            nn.ReLU(),
+            nn.MaxPool2D(2, 2))
+
+        if num_classes > 0:
+            self.fc = nn.Sequential(
+                nn.Linear(400, 120),
+                nn.Linear(120, 84), nn.Linear(84, num_classes))

python/paddle/vision/datasets/__init__.py

@@ -12,20 +12,22 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from . import folder
-from . import mnist
-from . import flowers
-from . import cifar
-from . import voc2012
+from .folder import DatasetFolder  # noqa: F401
+from .folder import ImageFolder  # noqa: F401
+from .mnist import MNIST  # noqa: F401
+from .mnist import FashionMNIST  # noqa: F401
+from .flowers import Flowers  # noqa: F401
+from .cifar import Cifar10  # noqa: F401
+from .cifar import Cifar100  # noqa: F401
+from .voc2012 import VOC2012  # noqa: F401
 
-from .folder import *
-from .mnist import *
-from .flowers import *
-from .cifar import *
-from .voc2012 import *
-
-__all__ = folder.__all__ \
-          + mnist.__all__ \
-          + flowers.__all__ \
-          + cifar.__all__ \
-          + voc2012.__all__
+__all__ = [ #noqa
+    'DatasetFolder'
+    'ImageFolder',
+    'MNIST',
+    'FashionMNIST',
+    'Flowers',
+    'Cifar10',
+    'Cifar100',
+    'VOC2012'
+]

python/paddle/vision/datasets/cifar.py

@@ -24,7 +24,7 @@
 from paddle.io import Dataset
 from paddle.dataset.common import _check_exists_and_download
 
-__all__ = ['Cifar10', 'Cifar100']
+__all__ = []
 
 URL_PREFIX = 'https://dataset.bj.bcebos.com/cifar/'
 CIFAR10_URL = URL_PREFIX + 'cifar-10-python.tar.gz'

python/paddle/vision/datasets/flowers.py

@@ -25,7 +25,7 @@
 from paddle.utils import try_import
 from paddle.dataset.common import _check_exists_and_download
 
-__all__ = ["Flowers"]
+__all__ = []
 
 DATA_URL = 'http://paddlemodels.bj.bcebos.com/flowers/102flowers.tgz'
 LABEL_URL = 'http://paddlemodels.bj.bcebos.com/flowers/imagelabels.mat'

python/paddle/vision/datasets/folder.py

@@ -20,7 +20,7 @@
 from paddle.io import Dataset
 from paddle.utils import try_import
 
-__all__ = ["DatasetFolder", "ImageFolder"]
+__all__ = []
 
 
 def has_valid_extension(filename, extensions):

python/paddle/vision/datasets/mnist.py

@@ -24,7 +24,7 @@
 from paddle.io import Dataset
 from paddle.dataset.common import _check_exists_and_download
 
-__all__ = ["MNIST", "FashionMNIST"]
+__all__ = []
 
 
 class MNIST(Dataset):