Feature/ollama integration enhancement

.coveragerc

@@ -1,16 +1,7 @@
 [run]
-omit =
-    */site-packages/* \
-    *\__init__.py 
-[report]
-# Regexes for lines to exclude from consideration
-exclude_lines =
-    """
-    '''
-    pragma: no cover
-    def __repr__
-    if self.debug:
-    raise AssertionError
-    raise NotImplementedError
-    except Exception as e:
-    if __name__ == .__main__.:
+source = 
+    ./metagpt/
+omit = 
+    */metagpt/ext/*
+    */metagpt/environment/android_env/*
+    */metagpt/environment/werewolf_env/*

.devcontainer/README.md

@@ -0,0 +1,34 @@
+# Dev Container
+
+This project includes a [Dev Container](https://containers.dev/), offering you a comprehensive and fully-featured development environment within a container. By leveraging the Dev Container configuration in this folder, you can seamlessly build and initiate MetaGPT locally. For detailed information, please refer to the main README in the home directory.
+
+You can utilize this Dev Container in [GitHub Codespaces](https://github.com/features/codespaces) or with the [VS Code Dev Containers extension](https://marketplace.visualstudio.com/items?itemName=ms-vscode-remote.remote-containers).
+
+## GitHub Codespaces
+[![Open in GitHub Codespaces](https://github.com/codespaces/badge.svg)](https://codespaces.new/geekan/MetaGPT)
+
+Click the button above to open this repository in a Codespace. For additional information, refer to the [GitHub documentation on creating a Codespace](https://docs.github.com/en/free-pro-team@latest/github/developing-online-with-codespaces/creating-a-codespace#creating-a-codespace).
+
+## VS Code Dev Containers
+[![Open in Dev Containers](https://img.shields.io/static/v1?label=Dev%20Containers&message=Open&color=blue&logo=visualstudiocode)](https://vscode.dev/redirect?url=vscode://ms-vscode-remote.remote-containers/cloneInVolume?url=https://github.com/geekan/MetaGPT)
+
+Note: Clicking the link above opens the main repository. To open your local cloned repository, replace the URL with your username and cloned repository's name: `https://vscode.dev/redirect?url=vscode://ms-vscode-remote.remote-containers/cloneInVolume?url=https://github.com/<your-username>/<your-repo-name>`
+
+If you have VS Code and Docker installed, use the button above to get started. This will prompt VS Code to install the Dev Containers extension if it's not already installed, clone the source code into a container volume, and set up a dev container for you.
+
+Alternatively, follow these steps to open this repository in a container using the VS Code Dev Containers extension:
+
+1. For first-time users of a development container, ensure your system meets the prerequisites (e.g., Docker installation) as outlined in the [getting started steps](https://aka.ms/vscode-remote/containers/getting-started).
+
+2. To open a locally cloned copy of the code:
+   - Fork and clone this repository to your local file system.
+   - Press <kbd>F1</kbd> and select the **Dev Containers: Open Folder in Container...** command.
+   - Choose the cloned folder, wait for the container to initialize, and start exploring!
+
+Learn more in the [VS Code Dev Containers documentation](https://code.visualstudio.com/docs/devcontainers/containers).
+
+## Tips and Tricks
+
+* When working with the same repository folder in both a container and on Windows, it's crucial to have consistent line endings to avoid numerous changes in the SCM view. The `.gitattributes` file in the root of this repository disables line ending conversion, helping to prevent this issue. For more information, see [resolving git line ending issues in containers](https://code.visualstudio.com/docs/devcontainers/tips-and-tricks#_resolving-git-line-ending-issues-in-containers-resulting-in-many-modified-files).
+
+* If you're curious about the contents of the image used in this Dev Container, you can review it in the [devcontainers/images](https://github.com/devcontainers/images/tree/main/src/python) repository.

.devcontainer/devcontainer.json

@@ -0,0 +1,27 @@
+// For format details, see https://aka.ms/devcontainer.json. For config options, see the
+// README at: https://github.com/devcontainers/templates/tree/main/src/python
+{
+	"name": "Python 3",
+	// Or use a Dockerfile or Docker Compose file. More info: https://containers.dev/guide/dockerfile
+	"image": "metagpt/metagpt:latest",
+
+	// Features to add to the dev container. More info: https://containers.dev/features.
+	// "features": {},
+
+	// Configure tool-specific properties.
+	"customizations": {
+		// Configure properties specific to VS Code.
+		"vscode": {
+			"settings": {},
+			"extensions": [
+				"streetsidesoftware.code-spell-checker"
+			]
+		}
+	},
+
+	// Use 'postCreateCommand' to run commands after the container is created.
+	"postCreateCommand": "./.devcontainer/postCreateCommand.sh"
+
+	// Uncomment to connect as root instead. More info: https://aka.ms/dev-containers-non-root.
+	// "remoteUser": "root"
+}

.devcontainer/docker-compose.yaml

@@ -0,0 +1,31 @@
+version: '3'
+services:
+  metagpt:
+    build:
+      dockerfile: Dockerfile
+      context: ..
+    volumes:
+   # Update this to wherever you want VS Code to mount the folder of your project
+      - ..:/workspaces:cached
+    networks:
+      - metagpt-network 
+  #   environment:
+  #     MONGO_ROOT_USERNAME: root
+  #     MONGO_ROOT_PASSWORD: example123
+  #   depends_on:
+  #     - mongo   
+  # mongo:
+  #   image: mongo
+  #   restart: unless-stopped
+  #   environment:
+  #     MONGO_INITDB_ROOT_USERNAME: root
+  #     MONGO_INITDB_ROOT_PASSWORD: example123
+  #   ports:
+  #     - "27017:27017"
+  #   networks:
+  #     - metagpt-network
+
+networks:
+  metagpt-network:
+    driver: bridge
+

.devcontainer/postCreateCommand.sh

@@ -0,0 +1,7 @@
+# Step 1: Ensure that NPM is installed on your system. Then install mermaid-js.
+npm --version
+sudo npm install -g @mermaid-js/mermaid-cli
+
+# Step 2: Ensure that Python 3.9+ is installed on your system. You can check this by using:
+python --version
+pip install -e .

.dockerignore

@@ -0,0 +1,6 @@
+workspace
+tmp
+build
+dist
+data
+geckodriver.log

.gitattributes

@@ -0,0 +1,34 @@
+# HTML code is incorrectly calculated into statistics, so ignore them
+*.html linguist-detectable=false
+
+# Auto detect text files and perform LF normalization
+* text=auto eol=lf
+
+# Ensure shell scripts use LF (Linux style) line endings on Windows
+*.sh text eol=lf
+
+# Treat specific binary files as binary and prevent line ending conversion
+*.png binary
+*.jpg binary
+*.gif binary
+*.ico binary
+*.jpeg binary
+*.mp3 binary
+*.mp4 binary
+*.zip binary
+*.bin binary
+
+# Preserve original line endings for specific document files
+*.doc text eol=crlf
+*.docx text eol=crlf
+*.pdf binary
+
+# Ensure source code and script files use LF line endings
+*.py text eol=lf
+*.js text eol=lf
+*.html text eol=lf
+*.css text eol=lf
+
+# Specify custom diff driver for specific file types
+*.md diff=markdown
+*.json diff=json

.github/ISSUE_TEMPLATE/config.yaml

@@ -0,0 +1,5 @@
+blank_issues_enabled: false
+contact_links:
+  - name: "📑 Read online docs"
+    url: https://docs.deepwisdom.ai/
+    about: Find the tutorials, use cases and blogs from the doc site.

.github/ISSUE_TEMPLATE/request_new_features.md

@@ -0,0 +1,14 @@
+---
+name: "🤔 Request new features"  
+about: There are some ideas or demands want to discuss with the official and hope to be implemented in the future.    
+title: ''  
+labels: kind/features    
+assignees: ''  
+---
+
+**Feature description**
+<!-- Clear and direct description of the functionality of the currently submitted or proposed feature -->
+
+**Your Feature**
+<!-- Describe the idea or process of implementing the current feature. Of course, you can also paste the URL address of your Pull Request. -->
+<!-- When submitting features, you need to complete the corresponding doc/tests/examples to facilitate verification by reviewers. -->

.github/ISSUE_TEMPLATE/show_me_the_bug.md

@@ -0,0 +1,30 @@
+---
+name: "🪲 Show me the Bug"  
+about: Something happened when I use MetaGPT, I want to report it and hope to get help from the official and community.  
+title: ''
+labels: kind/bug  
+assignees: ''
+---
+
+**Bug description**
+<!-- Clearly and directly describe the current bug -->
+
+**Bug solved method**
+<!-- If you solved the bug, describe the idea or process to solve the current bug. Of course, you can also paste the URL address of your Pull Request. -->
+<!-- If not, provide more auxiliary information to facilitate our further positioning and investigation  -->
+
+**Environment information**
+<!-- Environment：System version (like ubuntu 22.04), Python version (conda python 3.7), LLM type and model (OpenAI gpt-4-1106-preview) -->
+
+- LLM type and model name:
+- System version:
+- Python version:
+- MetaGPT version or branch:
+
+<!-- Dependent packagess：the packages version cause the bug(like `pydantic 1.10.8`), installation method（like `pip install metagpt` or `pip install from source` or `run in docker`） -->
+
+- packages version:
+- installation method: 
+
+**Screenshots or logs**
+<!-- Screenshots or logs of the bug can help us understand the problem more quickly -->

.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,19 @@
+
+**Features**
+<!-- Clear and direct description of the submit features. -->
+<!-- If it's a bug fix, please also paste the issue link. -->
+
+- xx
+- yy
+
+**Feature Docs**
+<!-- The RFC, tutorial, or use cases about the feature if it's a pretty big update. If not, there is no need to fill. -->
+
+**Influence**
+<!-- Tell me the impact of the new feature and I'll focus on it.  -->
+
+**Result**
+<!-- The screenshot/log of unittest/running result -->
+
+**Other**
+<!-- Something else about this PR. -->

.github/workflows/build-package.yaml

@@ -0,0 +1,35 @@
+name: Build and upload python package
+
+on:
+  workflow_dispatch:
+  release:
+    types: [created, published]
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python
+      uses: actions/setup-python@v4
+      with:
+        python-version: '3.9'
+        cache: 'pip'
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install -r requirements.txt
+        pip install -e.
+        pip install setuptools wheel twine
+    - name: Set package version
+      run: |
+        export VERSION="${GITHUB_REF#refs/tags/v}"
+        sed -i "s/version=.*/version=\"${VERSION}\",/" setup.py
+    - name: Build and publish
+      env:
+        TWINE_USERNAME: __token__
+        TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }}
+      run: |
+        python setup.py bdist_wheel sdist
+        twine upload dist/*

.github/workflows/fulltest.yaml

@@ -0,0 +1,86 @@
+name: Full Tests
+
+on:
+  workflow_dispatch:
+  pull_request_target:
+  push: 
+    branches:
+      - 'main'
+      - 'dev'
+      - '*-release'
+      - '*-debugger'
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    environment: unittest
+    strategy:
+      matrix:
+        # python-version: ['3.9', '3.10', '3.11']
+        python-version: ['3.9']
+
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        ref: ${{ github.event.pull_request.head.sha }}
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v4
+      with:
+        python-version: ${{ matrix.python-version }}
+        cache: 'pip'
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install -e .[test]
+        npm install -g @mermaid-js/mermaid-cli
+        playwright install --with-deps
+    - name: Run reverse proxy script for ssh service
+      if: contains(github.ref, '-debugger')
+      continue-on-error: true
+      env:
+        FPR_SERVER_ADDR: ${{ secrets.FPR_SERVER_ADDR }}
+        FPR_TOKEN: ${{ secrets.FPR_TOKEN }}
+        FPR_SSH_REMOTE_PORT: ${{ secrets.FPR_SSH_REMOTE_PORT }}
+        RSA_PUB: ${{ secrets.RSA_PUB }}
+        SSH_PORT: ${{ vars.SSH_PORT || '22'}}
+      run: |
+        echo "Run \"ssh $(whoami)@FPR_SERVER_HOST -p FPR_SSH_REMOTE_PORT\" and \"cd $(pwd)\""
+        mkdir -p  ~/.ssh/
+        echo $RSA_PUB >> ~/.ssh/authorized_keys
+        chmod 600 ~/.ssh/authorized_keys
+        wget https://github.com/fatedier/frp/releases/download/v0.32.1/frp_0.32.1_linux_amd64.tar.gz -O frp.tar.gz
+        tar xvzf frp.tar.gz -C /opt
+        mv /opt/frp* /opt/frp
+        /opt/frp/frpc tcp --server_addr $FPR_SERVER_ADDR --token $FPR_TOKEN  --local_port $SSH_PORT  --remote_port $FPR_SSH_REMOTE_PORT
+    - name: Test with pytest
+      run: |
+        export ALLOW_OPENAI_API_CALL=0
+        echo "${{ secrets.METAGPT_KEY_YAML }}" | base64 -d > config/key.yaml
+        mkdir -p ~/.metagpt && echo "${{ secrets.METAGPT_CONFIG2_YAML }}" | base64 -d > ~/.metagpt/config2.yaml
+        pytest tests/ --doctest-modules --cov=./metagpt/ --cov-report=xml:cov.xml --cov-report=html:htmlcov --durations=20 | tee unittest.txt
+    - name: Show coverage report
+      run: |
+        coverage report -m
+    - name: Show failed tests and overall summary
+      run: |
+        grep -E "FAILED tests|ERROR tests|[0-9]+ passed," unittest.txt
+        failed_count=$(grep -E "FAILED|ERROR" unittest.txt | wc -l)
+        if [[ "$failed_count" -gt 0 ]]; then
+          echo "$failed_count failed lines found! Task failed."
+          exit 1
+        fi
+    - name: Upload pytest test results
+      uses: actions/upload-artifact@v3
+      with:
+        name: pytest-results-${{ matrix.python-version }}
+        path: |
+          ./unittest.txt
+          ./htmlcov/
+          ./tests/data/rsp_cache_new.json
+        retention-days: 3
+      if: ${{ always() }}
+    # - name: Upload coverage reports to Codecov
+    #   uses: codecov/codecov-action@v3
+    #   env:
+    #     CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+    #   if: ${{ always() }}