Merge branch 'main' into p2p-rpc

Author: PaParaZz1

ding/framework/middleware/tests/test_trainer.py

@@ -1,7 +1,8 @@
 import pytest
 import random
-import torch
 import copy
+import torch
+import treetensor.torch as ttorch
 from unittest.mock import Mock, patch
 from ding.data.buffer import DequeBuffer
 from ding.framework import OnlineRLContext, task
@@ -10,6 +11,8 @@
 
 
 class MockPolicy(Mock):
+    _device = 'cpu'
+
     # MockPolicy class for train mode
     def forward(self, train_data, **kwargs):
         res = {
@@ -19,6 +22,8 @@ def forward(self, train_data, **kwargs):
 
 
 class MultiStepMockPolicy(Mock):
+    _device = 'cpu'
+
     # MockPolicy class for multi-step train mode
     def forward(self, train_data, **kwargs):
         res = [
@@ -34,7 +39,7 @@ def forward(self, train_data, **kwargs):
 
 def get_mock_train_input():
     data = {'obs': torch.rand(2, 2), 'next_obs': torch.rand(2, 2), 'reward': random.random(), 'info': {}}
-    return data
+    return ttorch.as_tensor(data)
 
 
 @pytest.mark.unittest

ding/torch_utils/loss/contrastive_loss.py

@@ -79,7 +79,7 @@ def forward(self, x: torch.Tensor, y: torch.Tensor):
         x_n = x.view(-1, self._encode_shape)
         y_n = y.view(-1, self._encode_shape)
 
-        # Use inner product to obtain postive samples.
+        # Use inner product to obtain positive samples.
         # [N, x_heads, encode_dim] * [N, encode_dim, y_heads] -> [N, x_heads, y_heads]
         u_pos = torch.matmul(x, y.permute(0, 2, 1)).unsqueeze(2)
         # Use outer product to obtain all sample permutations.
@@ -92,7 +92,7 @@ def forward(self, x: torch.Tensor, y: torch.Tensor):
         u_neg = (n_mask * u_all) - (10. * (1 - n_mask))
         u_neg = u_neg.view(N, N * x_heads, y_heads).unsqueeze(dim=1).expand(-1, x_heads, -1, -1)
 
-        # Concatenate postive and negative samples and apply log softmax.
+        # Concatenate positive and negative samples and apply log softmax.
         pred_lgt = torch.cat([u_pos, u_neg], dim=2)
         pred_log = F.log_softmax(pred_lgt * self._temperature, dim=2)
 

dizoo/beergame/__init__.py

@@ -3,7 +3,7 @@ FROM pytorch/pytorch:1.7.1-cuda11.0-cudnn8-runtime as base
 WORKDIR /ding
 
 RUN apt update \
-    && apt install libgl1-mesa-glx libglib2.0-0 libsm6 libxext6 libxrender-dev swig curl git vim gcc \g++ make wget locales dnsutils -y \
+    && apt install libgl1-mesa-glx libglib2.0-0 libsm6 libxext6 libxrender-dev swig curl git vim gcc \g++ make wget locales dnsutils zip unzip -y \
     && apt clean \
     && rm -rf /var/cache/apt/* \
     && sed -i '/en_US.UTF-8/s/^# //g' /etc/locale.gen \
@@ -33,7 +33,7 @@ RUN apt-get update && \
     python3.8 python3-pip python3.8-dev
 
 RUN apt update \
-    && apt install libgl1-mesa-glx libglib2.0-0 libsm6 libxext6 libxrender-dev swig curl git vim gcc \g++ make wget locales dnsutils -y \
+    && apt install libgl1-mesa-glx libglib2.0-0 libsm6 libxext6 libxrender-dev swig curl git vim gcc \g++ make wget locales dnsutils zip unzip -y \
     && apt clean \
     && rm -rf /var/cache/apt/* \
     && sed -i '/en_US.UTF-8/s/^# //g' /etc/locale.gen \

dizoo/procgen/__init__.py

@@ -91,9 +91,8 @@ WORKDIR /ding
 RUN mkdir tempfile \
     && cd tempfile \
     && wget https://github.com/rlworkgroup/metaworld/archive/refs/heads/master.zip -O metaworld_master.zip \
-    && apt-get install unzip \
     && unzip metaworld_master.zip \
-    && python3 -m pip install --no-cache-dir ./metaworld-master/ \
+    && python3 -m pip install --no-cache-dir ./Metaworld-master/ \
     && cd .. \
     && rm -rf tempfile