From e68f822434e84e61b7bd19dfa7b5c2a16ae5d39b Mon Sep 17 00:00:00 2001 From: Konrad Molinski Date: Thu, 8 Aug 2024 18:54:31 +0000 Subject: [PATCH 001/227] cleanup/preparation --- .dependencies_installed | 0 {template => cancer_ai}/__init__.py | 0 {template/api => cancer_ai/base}/__init__.py | 0 {template => cancer_ai}/base/miner.py | 4 +- {template => cancer_ai}/base/neuron.py | 8 +- .../base => cancer_ai/base/utils}/__init__.py | 0 .../base/utils/weight_utils.py | 0 {template => cancer_ai}/base/validator.py | 8 +- {template => cancer_ai}/mock.py | 0 {template => cancer_ai}/protocol.py | 0 {template => cancer_ai}/utils/__init__.py | 0 {template => cancer_ai}/utils/config.py | 0 {template => cancer_ai}/utils/logging.py | 0 {template => cancer_ai}/utils/misc.py | 0 {template => cancer_ai}/utils/uids.py | 0 {template => cancer_ai}/validator/__init__.py | 0 {template => cancer_ai}/validator/forward.py | 6 +- {template => cancer_ai}/validator/reward.py | 0 contrib/CODE_REVIEW_DOCS.md | 72 --- contrib/CONTRIBUTING.md | 213 -------- contrib/DEVELOPMENT_WORKFLOW.md | 165 ------ contrib/STYLE.md | 348 ------------- docs/running_on_mainnet.md | 244 --------- docs/running_on_staging.md | 340 ------------ docs/running_on_testnet.md | 242 --------- docs/stream_tutorial/README.md | 490 ------------------ docs/stream_tutorial/client.py | 104 ---- docs/stream_tutorial/config.py | 116 ----- docs/stream_tutorial/miner.py | 398 -------------- docs/stream_tutorial/protocol.py | 154 ------ neurons/miner.py | 4 +- neurons/validator.py | 4 +- scripts/check_compatibility.sh | 76 --- scripts/check_requirements_changes.sh | 10 - scripts/install_staging.sh | 145 ------ template/api/dummy.py | 44 -- template/api/get_query_axons.py | 126 ----- template/base/utils/__init__.py | 0 template/subnet_links.py | 76 --- tests/test_template_validator.py | 8 +- verify/generate.py | 35 -- verify/verify.py | 41 -- 42 files changed, 21 insertions(+), 3460 deletions(-) delete mode 100644 .dependencies_installed rename {template => cancer_ai}/__init__.py (100%) rename {template/api => cancer_ai/base}/__init__.py (100%) rename {template => cancer_ai}/base/miner.py (98%) rename {template => cancer_ai}/base/neuron.py (96%) rename {template/base => cancer_ai/base/utils}/__init__.py (100%) rename {template => cancer_ai}/base/utils/weight_utils.py (100%) rename {template => cancer_ai}/base/validator.py (98%) rename {template => cancer_ai}/mock.py (100%) rename {template => cancer_ai}/protocol.py (100%) rename {template => cancer_ai}/utils/__init__.py (100%) rename {template => cancer_ai}/utils/config.py (100%) rename {template => cancer_ai}/utils/logging.py (100%) rename {template => cancer_ai}/utils/misc.py (100%) rename {template => cancer_ai}/utils/uids.py (100%) rename {template => cancer_ai}/validator/__init__.py (100%) rename {template => cancer_ai}/validator/forward.py (95%) rename {template => cancer_ai}/validator/reward.py (100%) delete mode 100644 contrib/CODE_REVIEW_DOCS.md delete mode 100644 contrib/CONTRIBUTING.md delete mode 100644 contrib/DEVELOPMENT_WORKFLOW.md delete mode 100644 contrib/STYLE.md delete mode 100644 docs/running_on_mainnet.md delete mode 100644 docs/running_on_staging.md delete mode 100644 docs/running_on_testnet.md delete mode 100644 docs/stream_tutorial/README.md delete mode 100644 docs/stream_tutorial/client.py delete mode 100644 docs/stream_tutorial/config.py delete mode 100644 docs/stream_tutorial/miner.py delete mode 100644 docs/stream_tutorial/protocol.py delete mode 100755 scripts/check_compatibility.sh delete mode 100755 scripts/check_requirements_changes.sh delete mode 100644 scripts/install_staging.sh delete mode 100644 template/api/dummy.py delete mode 100644 template/api/get_query_axons.py delete mode 100644 template/base/utils/__init__.py delete mode 100644 template/subnet_links.py delete mode 100644 verify/generate.py delete mode 100644 verify/verify.py diff --git a/.dependencies_installed b/.dependencies_installed deleted file mode 100644 index e69de29b..00000000 diff --git a/template/__init__.py b/cancer_ai/__init__.py similarity index 100% rename from template/__init__.py rename to cancer_ai/__init__.py diff --git a/template/api/__init__.py b/cancer_ai/base/__init__.py similarity index 100% rename from template/api/__init__.py rename to cancer_ai/base/__init__.py diff --git a/template/base/miner.py b/cancer_ai/base/miner.py similarity index 98% rename from template/base/miner.py rename to cancer_ai/base/miner.py index 1788e24b..528ae34f 100644 --- a/template/base/miner.py +++ b/cancer_ai/base/miner.py @@ -23,8 +23,8 @@ import bittensor as bt -from template.base.neuron import BaseNeuron -from template.utils.config import add_miner_args +from ..base.neuron import BaseNeuron +from ..utils.config import add_miner_args from typing import Union diff --git a/template/base/neuron.py b/cancer_ai/base/neuron.py similarity index 96% rename from template/base/neuron.py rename to cancer_ai/base/neuron.py index 9b2ce7b2..06f65a20 100644 --- a/template/base/neuron.py +++ b/cancer_ai/base/neuron.py @@ -23,10 +23,10 @@ from abc import ABC, abstractmethod # Sync calls set weights and also resyncs the metagraph. -from template.utils.config import check_config, add_args, config -from template.utils.misc import ttl_get_block -from template import __spec_version__ as spec_version -from template.mock import MockSubtensor, MockMetagraph +from ..utils.config import check_config, add_args, config +from ..utils.misc import ttl_get_block +from .. import __spec_version__ as spec_version +from ..mock import MockSubtensor, MockMetagraph class BaseNeuron(ABC): diff --git a/template/base/__init__.py b/cancer_ai/base/utils/__init__.py similarity index 100% rename from template/base/__init__.py rename to cancer_ai/base/utils/__init__.py diff --git a/template/base/utils/weight_utils.py b/cancer_ai/base/utils/weight_utils.py similarity index 100% rename from template/base/utils/weight_utils.py rename to cancer_ai/base/utils/weight_utils.py diff --git a/template/base/validator.py b/cancer_ai/base/validator.py similarity index 98% rename from template/base/validator.py rename to cancer_ai/base/validator.py index c1ca07ed..1c924fc1 100644 --- a/template/base/validator.py +++ b/cancer_ai/base/validator.py @@ -28,13 +28,13 @@ from typing import List, Union from traceback import print_exception -from template.base.neuron import BaseNeuron -from template.base.utils.weight_utils import ( +from ..base.neuron import BaseNeuron +from ..base.utils.weight_utils import ( process_weights_for_netuid, convert_weights_and_uids_for_emit, ) # TODO: Replace when bittensor switches to numpy -from template.mock import MockDendrite -from template.utils.config import add_validator_args +from ..mock import MockDendrite +from ..utils.config import add_validator_args class BaseValidatorNeuron(BaseNeuron): diff --git a/template/mock.py b/cancer_ai/mock.py similarity index 100% rename from template/mock.py rename to cancer_ai/mock.py diff --git a/template/protocol.py b/cancer_ai/protocol.py similarity index 100% rename from template/protocol.py rename to cancer_ai/protocol.py diff --git a/template/utils/__init__.py b/cancer_ai/utils/__init__.py similarity index 100% rename from template/utils/__init__.py rename to cancer_ai/utils/__init__.py diff --git a/template/utils/config.py b/cancer_ai/utils/config.py similarity index 100% rename from template/utils/config.py rename to cancer_ai/utils/config.py diff --git a/template/utils/logging.py b/cancer_ai/utils/logging.py similarity index 100% rename from template/utils/logging.py rename to cancer_ai/utils/logging.py diff --git a/template/utils/misc.py b/cancer_ai/utils/misc.py similarity index 100% rename from template/utils/misc.py rename to cancer_ai/utils/misc.py diff --git a/template/utils/uids.py b/cancer_ai/utils/uids.py similarity index 100% rename from template/utils/uids.py rename to cancer_ai/utils/uids.py diff --git a/template/validator/__init__.py b/cancer_ai/validator/__init__.py similarity index 100% rename from template/validator/__init__.py rename to cancer_ai/validator/__init__.py diff --git a/template/validator/forward.py b/cancer_ai/validator/forward.py similarity index 95% rename from template/validator/forward.py rename to cancer_ai/validator/forward.py index af5e7ee0..301664e9 100644 --- a/template/validator/forward.py +++ b/cancer_ai/validator/forward.py @@ -20,9 +20,9 @@ import time import bittensor as bt -from template.protocol import Dummy -from template.validator.reward import get_rewards -from template.utils.uids import get_random_uids +from ..protocol import Dummy +from ..validator.reward import get_rewards +from ..utils.uids import get_random_uids async def forward(self): diff --git a/template/validator/reward.py b/cancer_ai/validator/reward.py similarity index 100% rename from template/validator/reward.py rename to cancer_ai/validator/reward.py diff --git a/contrib/CODE_REVIEW_DOCS.md b/contrib/CODE_REVIEW_DOCS.md deleted file mode 100644 index 9909606a..00000000 --- a/contrib/CODE_REVIEW_DOCS.md +++ /dev/null @@ -1,72 +0,0 @@ -# Code Review -### Conceptual Review - -A review can be a conceptual review, where the reviewer leaves a comment - * `Concept (N)ACK`, meaning "I do (not) agree with the general goal of this pull - request", - * `Approach (N)ACK`, meaning `Concept ACK`, but "I do (not) agree with the - approach of this change". - -A `NACK` needs to include a rationale why the change is not worthwhile. -NACKs without accompanying reasoning may be disregarded. -After conceptual agreement on the change, code review can be provided. A review -begins with `ACK BRANCH_COMMIT`, where `BRANCH_COMMIT` is the top of the PR -branch, followed by a description of how the reviewer did the review. The -following language is used within pull request comments: - - - "I have tested the code", involving change-specific manual testing in - addition to running the unit, functional, or fuzz tests, and in case it is - not obvious how the manual testing was done, it should be described; - - "I have not tested the code, but I have reviewed it and it looks - OK, I agree it can be merged"; - - A "nit" refers to a trivial, often non-blocking issue. - -### Code Review -Project maintainers reserve the right to weigh the opinions of peer reviewers -using common sense judgement and may also weigh based on merit. Reviewers that -have demonstrated a deeper commitment and understanding of the project over time -or who have clear domain expertise may naturally have more weight, as one would -expect in all walks of life. - -Where a patch set affects consensus-critical code, the bar will be much -higher in terms of discussion and peer review requirements, keeping in mind that -mistakes could be very costly to the wider community. This includes refactoring -of consensus-critical code. - -Where a patch set proposes to change the Bittensor consensus, it must have been -discussed extensively on the discord server and other channels, be accompanied by a widely -discussed BIP and have a generally widely perceived technical consensus of being -a worthwhile change based on the judgement of the maintainers. - -### Finding Reviewers - -As most reviewers are themselves developers with their own projects, the review -process can be quite lengthy, and some amount of patience is required. If you find -that you've been waiting for a pull request to be given attention for several -months, there may be a number of reasons for this, some of which you can do something -about: - - - It may be because of a feature freeze due to an upcoming release. During this time, - only bug fixes are taken into consideration. If your pull request is a new feature, - it will not be prioritized until after the release. Wait for the release. - - It may be because the changes you are suggesting do not appeal to people. Rather than - nits and critique, which require effort and means they care enough to spend time on your - contribution, thundering silence is a good sign of widespread (mild) dislike of a given change - (because people don't assume *others* won't actually like the proposal). Don't take - that personally, though! Instead, take another critical look at what you are suggesting - and see if it: changes too much, is too broad, doesn't adhere to the - [developer notes](DEVELOPMENT_WORKFLOW.md), is dangerous or insecure, is messily written, etc. - Identify and address any of the issues you find. Then ask e.g. on IRC if someone could give - their opinion on the concept itself. - - It may be because your code is too complex for all but a few people, and those people - may not have realized your pull request even exists. A great way to find people who - are qualified and care about the code you are touching is the - [Git Blame feature](https://docs.github.com/en/github/managing-files-in-a-repository/managing-files-on-github/tracking-changes-in-a-file). Simply - look up who last modified the code you are changing and see if you can find - them and give them a nudge. Don't be incessant about the nudging, though. - - Finally, if all else fails, ask on IRC or elsewhere for someone to give your pull request - a look. If you think you've been waiting for an unreasonably long time (say, - more than a month) for no particular reason (a few lines changed, etc.), - this is totally fine. Try to return the favor when someone else is asking - for feedback on their code, and the universe balances out. - - Remember that the best thing you can do while waiting is give review to others! \ No newline at end of file diff --git a/contrib/CONTRIBUTING.md b/contrib/CONTRIBUTING.md deleted file mode 100644 index ba33ce3c..00000000 --- a/contrib/CONTRIBUTING.md +++ /dev/null @@ -1,213 +0,0 @@ -# Contributing to Bittensor Subnet Development - -The following is a set of guidelines for contributing to the Bittensor ecosystem. These are **HIGHLY RECOMMENDED** guidelines, but not hard-and-fast rules. Use your best judgment, and feel free to propose changes to this document in a pull request. - -## Table Of Contents -1. [How Can I Contribute?](#how-can-i-contribute) - 1. [Communication Channels](#communication-channels) - 1. [Code Contribution General Guideline](#code-contribution-general-guidelines) - 1. [Pull Request Philosophy](#pull-request-philosophy) - 1. [Pull Request Process](#pull-request-process) - 1. [Addressing Feedback](#addressing-feedback) - 1. [Squashing Commits](#squashing-commits) - 1. [Refactoring](#refactoring) - 1. [Peer Review](#peer-review) - 1. [Suggesting Features](#suggesting-enhancements-and-features) - - -## How Can I Contribute? -TODO(developer): Define your desired contribution procedure. - -## Communication Channels -TODO(developer): Place your communication channels here - -> Please follow the Bittensor Subnet [style guide](./STYLE.md) regardless of your contribution type. - -Here is a high-level summary: -- Code consistency is crucial; adhere to established programming language conventions. -- Use `black` to format your Python code; it ensures readability and consistency. -- Write concise Git commit messages; summarize changes in ~50 characters. -- Follow these six commit rules: - - Atomic Commits: Focus on one task or fix per commit. - - Subject and Body Separation: Use a blank line to separate the subject from the body. - - Subject Line Length: Keep it under 50 characters for readability. - - Imperative Mood: Write subject line as if giving a command or instruction. - - Body Text Width: Wrap text manually at 72 characters. - - Body Content: Explain what changed and why, not how. -- Make use of your commit messages to simplify project understanding and maintenance. - -> For clear examples of each of the commit rules, see the style guide's [rules](./STYLE.md#the-six-rules-of-a-great-commit) section. - -### Code Contribution General Guidelines - -> Review the Bittensor Subnet [style guide](./STYLE.md) and [development workflow](./DEVELOPMENT_WORKFLOW.md) before contributing. - - -#### Pull Request Philosophy - -Patchsets and enhancements should always be focused. A pull request could add a feature, fix a bug, or refactor code, but it should not contain a mixture of these. Please also avoid 'super' pull requests which attempt to do too much, are overly large, or overly complex as this makes review difficult. - -Specifically, pull requests must adhere to the following criteria: -- Contain fewer than 50 files. PRs with more than 50 files will be closed. -- If a PR introduces a new feature, it *must* include corresponding tests. -- Other PRs (bug fixes, refactoring, etc.) should ideally also have tests, as they provide proof of concept and prevent regression. -- Categorize your PR properly by using GitHub labels. This aids in the review process by informing reviewers about the type of change at a glance. -- Make sure your code includes adequate comments. These should explain why certain decisions were made and how your changes work. -- If your changes are extensive, consider breaking your PR into smaller, related PRs. This makes your contributions easier to understand and review. -- Be active in the discussion about your PR. Respond promptly to comments and questions to help reviewers understand your changes and speed up the acceptance process. - -Generally, all pull requests must: - - - Have a clear use case, fix a demonstrable bug or serve the greater good of the project (e.g. refactoring for modularisation). - - Be well peer-reviewed. - - Follow code style guidelines. - - Not break the existing test suite. - - Where bugs are fixed, where possible, there should be unit tests demonstrating the bug and also proving the fix. - - Change relevant comments and documentation when behaviour of code changes. - -#### Pull Request Process - -Please follow these steps to have your contribution considered by the maintainers: - -*Before* creating the PR: -1. Read the [development workflow](./DEVELOPMENT_WORKFLOW.md) defined for this repository to understand our workflow. -2. Ensure your PR meets the criteria stated in the 'Pull Request Philosophy' section. -3. Include relevant tests for any fixed bugs or new features as stated in the [testing guide](./TESTING.md). -4. Ensure your commit messages are clear and concise. Include the issue number if applicable. -5. If you have multiple commits, rebase them into a single commit using `git rebase -i`. -6. Explain what your changes do and why you think they should be merged in the PR description consistent with the [style guide](./STYLE.md). - -*After* creating the PR: -1. Verify that all [status checks](https://help.github.com/articles/about-status-checks/) are passing after you submit your pull request. -2. Label your PR using GitHub's labeling feature. The labels help categorize the PR and streamline the review process. -3. Document your code with comments that provide a clear understanding of your changes. Explain any non-obvious parts of your code or design decisions you've made. -4. If your PR has extensive changes, consider splitting it into smaller, related PRs. This reduces the cognitive load on the reviewers and speeds up the review process. - -Please be responsive and participate in the discussion on your PR! This aids in clarifying any confusion or concerns and leads to quicker resolution and merging of your PR. - -> Note: If your changes are not ready for merge but you want feedback, create a draft pull request. - -Following these criteria will aid in quicker review and potential merging of your PR. -While the prerequisites above must be satisfied prior to having your pull request reviewed, the reviewer(s) may ask you to complete additional design work, tests, or other changes before your pull request can be ultimately accepted. - -When you are ready to submit your changes, create a pull request: - -> **Always** follow the [style guide](./STYLE.md) and [development workflow](./DEVELOPMENT_WORKFLOW.md) before submitting pull requests. - -After you submit a pull request, it will be reviewed by the maintainers. They may ask you to make changes. Please respond to any comments and push your changes as a new commit. - -> Note: Be sure to merge the latest from "upstream" before making a pull request: - -```bash -git remote add upstream https://github.com/opentensor/bittensor.git # TODO(developer): replace with your repo URL -git fetch upstream -git merge upstream/ -git push origin -``` - -#### Addressing Feedback - -After submitting your pull request, expect comments and reviews from other contributors. You can add more commits to your pull request by committing them locally and pushing to your fork. - -You are expected to reply to any review comments before your pull request is merged. You may update the code or reject the feedback if you do not agree with it, but you should express so in a reply. If there is outstanding feedback and you are not actively working on it, your pull request may be closed. - -#### Squashing Commits - -If your pull request contains fixup commits (commits that change the same line of code repeatedly) or too fine-grained commits, you may be asked to [squash](https://git-scm.com/docs/git-rebase#_interactive_mode) your commits before it will be reviewed. The basic squashing workflow is shown below. - - git checkout your_branch_name - git rebase -i HEAD~n - # n is normally the number of commits in the pull request. - # Set commits (except the one in the first line) from 'pick' to 'squash', save and quit. - # On the next screen, edit/refine commit messages. - # Save and quit. - git push -f # (force push to GitHub) - -Please update the resulting commit message, if needed. It should read as a coherent message. In most cases, this means not just listing the interim commits. - -If your change contains a merge commit, the above workflow may not work and you will need to remove the merge commit first. See the next section for details on how to rebase. - -Please refrain from creating several pull requests for the same change. Use the pull request that is already open (or was created earlier) to amend changes. This preserves the discussion and review that happened earlier for the respective change set. - -The length of time required for peer review is unpredictable and will vary from pull request to pull request. - -#### Refactoring - -Refactoring is a necessary part of any software project's evolution. The following guidelines cover refactoring pull requests for the project. - -There are three categories of refactoring: code-only moves, code style fixes, and code refactoring. In general, refactoring pull requests should not mix these three kinds of activities in order to make refactoring pull requests easy to review and uncontroversial. In all cases, refactoring PRs must not change the behaviour of code within the pull request (bugs must be preserved as is). - -Project maintainers aim for a quick turnaround on refactoring pull requests, so where possible keep them short, uncomplex and easy to verify. - -Pull requests that refactor the code should not be made by new contributors. It requires a certain level of experience to know where the code belongs to and to understand the full ramification (including rebase effort of open pull requests). Trivial pull requests or pull requests that refactor the code with no clear benefits may be immediately closed by the maintainers to reduce unnecessary workload on reviewing. - -#### Peer Review - -Anyone may participate in peer review which is expressed by comments in the pull request. Typically reviewers will review the code for obvious errors, as well as test out the patch set and opine on the technical merits of the patch. Project maintainers take into account the peer review when determining if there is consensus to merge a pull request (remember that discussions may have taken place elsewhere, not just on GitHub). The following language is used within pull-request comments: - -- ACK means "I have tested the code and I agree it should be merged"; -- NACK means "I disagree this should be merged", and must be accompanied by sound technical justification. NACKs without accompanying reasoning may be disregarded; -- utACK means "I have not tested the code, but I have reviewed it and it looks OK, I agree it can be merged"; -- Concept ACK means "I agree in the general principle of this pull request"; -- Nit refers to trivial, often non-blocking issues. - -Reviewers should include the commit(s) they have reviewed in their comments. This can be done by copying the commit SHA1 hash. - -A pull request that changes consensus-critical code is considerably more involved than a pull request that adds a feature to the wallet, for example. Such patches must be reviewed and thoroughly tested by several reviewers who are knowledgeable about the changed subsystems. Where new features are proposed, it is helpful for reviewers to try out the patch set on a test network and indicate that they have done so in their review. Project maintainers will take this into consideration when merging changes. - -For a more detailed description of the review process, see the [Code Review Guidelines](CODE_REVIEW_DOCS.md). - -> **Note:** If you find a **Closed** issue that seems like it is the same thing that you're experiencing, open a new issue and include a link to the original issue in the body of your new one. - -#### How Do I Submit A (Good) Bug Report? - -Please track bugs as GitHub issues. - -Explain the problem and include additional details to help maintainers reproduce the problem: - -* **Use a clear and descriptive title** for the issue to identify the problem. -* **Describe the exact steps which reproduce the problem** in as many details as possible. For example, start by explaining how you started the application, e.g. which command exactly you used in the terminal, or how you started Bittensor otherwise. When listing steps, **don't just say what you did, but explain how you did it**. For example, if you ran with a set of custom configs, explain if you used a config file or command line arguments. -* **Provide specific examples to demonstrate the steps**. Include links to files or GitHub projects, or copy/pasteable snippets, which you use in those examples. If you're providing snippets in the issue, use [Markdown code blocks](https://help.github.com/articles/markdown-basics/#multiple-lines). -* **Describe the behavior you observed after following the steps** and point out what exactly is the problem with that behavior. -* **Explain which behavior you expected to see instead and why.** -* **Include screenshots and animated GIFs** which show you following the described steps and clearly demonstrate the problem. You can use [this tool](https://www.cockos.com/licecap/) to record GIFs on macOS and Windows, and [this tool](https://github.com/colinkeenan/silentcast) or [this tool](https://github.com/GNOME/byzanz) on Linux. -* **If you're reporting that Bittensor crashed**, include a crash report with a stack trace from the operating system. On macOS, the crash report will be available in `Console.app` under "Diagnostic and usage information" > "User diagnostic reports". Include the crash report in the issue in a [code block](https://help.github.com/articles/markdown-basics/#multiple-lines), a [file attachment](https://help.github.com/articles/file-attachments-on-issues-and-pull-requests/), or put it in a [gist](https://gist.github.com/) and provide link to that gist. -* **If the problem is related to performance or memory**, include a CPU profile capture with your report, if you're using a GPU then include a GPU profile capture as well. Look into the [PyTorch Profiler](https://pytorch.org/tutorials/recipes/recipes/profiler_recipe.html) to look at memory usage of your model. -* **If the problem wasn't triggered by a specific action**, describe what you were doing before the problem happened and share more information using the guidelines below. - -Provide more context by answering these questions: - -* **Did the problem start happening recently** (e.g. after updating to a new version) or was this always a problem? -* If the problem started happening recently, **can you reproduce the problem in an older version of Bittensor?** -* **Can you reliably reproduce the issue?** If not, provide details about how often the problem happens and under which conditions it normally happens. - -Include details about your configuration and environment: - -* **Which version of Bittensor Subnet are you using?** -* **What commit hash are you on?** You can get the exact commit hash by checking `git log` and pasting the full commit hash. -* **What's the name and version of the OS you're using**? -* **Are you running Bittensor Subnet in a virtual machine?** If so, which VM software are you using and which operating systems and versions are used for the host and the guest? -* **Are you running Bittensor Subnet in a dockerized container?** If so, have you made sure that your docker container contains your latest changes and is up to date with Master branch? - -### Suggesting Enhancements and Features - -This section guides you through submitting an enhancement suggestion, including completely new features and minor improvements to existing functionality. Following these guidelines helps maintainers and the community understand your suggestion :pencil: and find related suggestions :mag_right:. - -When you are creating an enhancement suggestion, please [include as many details as possible](#how-do-i-submit-a-good-enhancement-suggestion). Fill in [the template](https://bit.ly/atom-behavior-pr), including the steps that you imagine you would take if the feature you're requesting existed. - -#### Before Submitting An Enhancement Suggestion - -* **Check the [debugging guide](./DEBUGGING.md).** for tips — you might discover that the enhancement is already available. Most importantly, check if you're using the latest version of the project first. - -#### How Submit A (Good) Feature Suggestion - -* **Use a clear and descriptive title** for the issue to identify the problem. -* **Provide a step-by-step description of the suggested enhancement** in as many details as possible. -* **Provide specific examples to demonstrate the steps**. Include copy/pasteable snippets which you use in those examples, as [Markdown code blocks](https://help.github.com/articles/markdown-basics/#multiple-lines). -* **Describe the current behavior** and **explain which behavior you expected to see instead** and why. -* **Include screenshots and animated GIFs** which help you demonstrate the steps or point out the part of the project which the suggestion is related to. You can use [this tool](https://www.cockos.com/licecap/) to record GIFs on macOS and Windows, and [this tool](https://github.com/colinkeenan/silentcast) or [this tool](https://github.com/GNOME/byzanz) on Linux. -* **Explain why this enhancement would be useful** to most users. -* **List some other text editors or applications where this enhancement exists.** -* **Specify the name and version of the OS you're using.** - -Thank you for considering contributing to Bittensor! Any help is greatly appreciated along this journey to incentivize open and permissionless intelligence. diff --git a/contrib/DEVELOPMENT_WORKFLOW.md b/contrib/DEVELOPMENT_WORKFLOW.md deleted file mode 100644 index 13bb07b2..00000000 --- a/contrib/DEVELOPMENT_WORKFLOW.md +++ /dev/null @@ -1,165 +0,0 @@ -# Bittensor Subnet Development Workflow - -This is a highly advisable workflow to follow to keep your subtensor project organized and foster ease of contribution. - -## Table of contents - -- [Bittensor Subnet Development Workflow](#bittensor-subnet-development-workflow) - - [Main Branches](#main-branches) - - [Development Model](#development-model) - - [Feature Branches](#feature-branches) - - [Release Branches](#release-branches) - - [Hotfix Branches](#hotfix-branches) - - [Git Operations](#git-operations) - - [Creating a Feature Branch](#creating-a-feature-branch) - - [Merging Feature Branch into Staging](#merging-feature-branch-into-staging) - - [Creating a Release Branch](#creating-a-release-branch) - - [Finishing a Release Branch](#finishing-a-release-branch) - - [Creating a Hotfix Branch](#creating-a-hotfix-branch) - - [Finishing a Hotfix Branch](#finishing-a-hotfix-branch) - - [Continuous Integration (CI) and Continuous Deployment (CD)](#continuous-integration-ci-and-continuous-deployment-cd) - - [Versioning and Release Notes](#versioning-and-release-notes) - - [Pending Tasks](#pending-tasks) - -## Main Branches - -Bittensor's codebase consists of two main branches: **main** and **staging**. - -**main** -- This is Bittensor's live production branch, which should only be updated by the core development team. This branch is protected, so refrain from pushing or merging into it unless authorized. - -**staging** -- This branch is continuously updated and is where you propose and merge changes. It's essentially Bittensor's active development branch. - -## Development Model - -### Feature Branches - -- Branch off from: `staging` -- Merge back into: `staging` -- Naming convention: `feature//` - -Feature branches are used to develop new features for upcoming or future releases. They exist as long as the feature is in development, but will eventually be merged into `staging` or discarded. Always delete your feature branch after merging to avoid unnecessary clutter. - -### Release Branches - -- Branch off from: `staging` -- Merge back into: `staging` and then `main` -- Naming convention: `release///` - -Release branches support the preparation of a new production release, allowing for minor bug fixes and preparation of metadata (version number, configuration, etc). All new features should be merged into `staging` and wait for the next big release. - -### Hotfix Branches - -General workflow: - -- Branch off from: `main` or `staging` -- Merge back into: `staging` then `main` -- Naming convention: `hotfix///` - -Hotfix branches are meant for quick fixes in the production environment. When a critical bug in a production version must be resolved immediately, a hotfix branch is created. - -## Git Operations - -#### Create a feature branch - -1. Branch from the **staging** branch. - 1. Command: `git checkout -b feature/my-feature staging` - -> Rebase frequently with the updated staging branch so you do not face big conflicts before submitting your pull request. Remember, syncing your changes with other developers could also help you avoid big conflicts. - -#### Merge feature branch into staging - -In other words, integrate your changes into a branch that will be tested and prepared for release. - -1. Switch branch to staging: `git checkout staging` -2. Merging feature branch into staging: `git merge --no-ff feature/my-feature` -3. Pushing changes to staging: `git push origin staging` -4. Delete feature branch: `git branch -d feature/my-feature` (alternatively, this can be navigated on the GitHub web UI) - -This operation is done by Github when merging a PR. - -So, what you have to keep in mind is: -- Open the PR against the `staging` branch. -- After merging a PR you should delete your feature branch. This will be strictly enforced. - -#### Creating a release branch - -1. Create branch from staging: `git checkout -b release/3.4.0/descriptive-message/creator's_name staging` -2. Updating version with major or minor: `./scripts/update_version.sh major|minor` -3. Commit file changes with new version: `git commit -a -m "Updated version to 3.4.0"` - - -#### Finishing a Release Branch - -This involves releasing stable code and generating a new version for bittensor. - -1. Switch branch to main: `git checkout main` -2. Merge release branch into main: `git merge --no-ff release/3.4.0/optional-descriptive-message` -3. Tag changeset: `git tag -a v3.4.0 -m "Releasing v3.4.0: some comment about it"` -4. Push changes to main: `git push origin main` -5. Push tags to origin: `git push origin --tags` - -To keep the changes made in the __release__ branch, we need to merge those back into `staging`: - -- Switch branch to staging: `git checkout staging`. -- Merging release branch into staging: `git merge --no-ff release/3.4.0/optional-descriptive-message` - -This step may well lead to a merge conflict (probably even, since we have changed the version number). If so, fix it and commit. - - -#### Creating a hotfix branch -1. Create branch from main: `git checkout -b hotfix/3.3.4/descriptive-message/creator's-name main` -2. Update patch version: `./scripts/update_version.sh patch` -3. Commit file changes with new version: `git commit -a -m "Updated version to 3.3.4"` -4. Fix the bug and commit the fix: `git commit -m "Fixed critical production issue X"` - -#### Finishing a Hotfix Branch - -Finishing a hotfix branch involves merging the bugfix into both `main` and `staging`. - -1. Switch branch to main: `git checkout main` -2. Merge hotfix into main: `git merge --no-ff hotfix/3.3.4/optional-descriptive-message` -3. Tag new version: `git tag -a v3.3.4 -m "Releasing v3.3.4: descriptive comment about the hotfix"` -4. Push changes to main: `git push origin main` -5. Push tags to origin: `git push origin --tags` -6. Switch branch to staging: `git checkout staging` -7. Merge hotfix into staging: `git merge --no-ff hotfix/3.3.4/descriptive-message/creator's-name` -8. Push changes to origin/staging: `git push origin staging` -9. Delete hotfix branch: `git branch -d hotfix/3.3.4/optional-descriptive-message` - -The one exception to the rule here is that, **when a release branch currently exists, the hotfix changes need to be merged into that release branch, instead of** `staging`. Back-merging the bugfix into the __release__ branch will eventually result in the bugfix being merged into `develop` too, when the release branch is finished. (If work in develop immediately requires this bugfix and cannot wait for the release branch to be finished, you may safely merge the bugfix into develop now already as well.) - -Finally, we remove the temporary branch: - -- `git branch -d hotfix/3.3.4/optional-descriptive-message` -## Continuous Integration (CI) and Continuous Deployment (CD) - -Continuous Integration (CI) is a software development practice where members of a team integrate their work frequently. Each integration is verified by an automated build and test process to detect integration errors as quickly as possible. - -Continuous Deployment (CD) is a software engineering approach in which software functionalities are delivered frequently through automated deployments. - -- **CircleCI job**: Create jobs in CircleCI to automate the merging of staging into main and release version (needed to release code) and building and testing Bittensor (needed to merge PRs). - -> It is highly recommended to set up your own circleci pipeline with your subnet - -## Versioning and Release Notes - -Semantic versioning helps keep track of the different versions of the software. When code is merged into main, generate a new version. - -Release notes provide documentation for each version released to the users, highlighting the new features, improvements, and bug fixes. When merged into main, generate GitHub release and release notes. - -## Pending Tasks - -Follow these steps when you are contributing to the bittensor subnet: - -- Determine if main and staging are different -- Determine what is in staging that is not merged yet - - Document not released developments - - When merged into staging, generate information about what's merged into staging but not released. - - When merged into main, generate GitHub release and release notes. -- CircleCI jobs - - Merge staging into main and release version (needed to release code) - - Build and Test Bittensor (needed to merge PRs) - -This document can be improved as the Bittensor project continues to develop and change. diff --git a/contrib/STYLE.md b/contrib/STYLE.md deleted file mode 100644 index b7ac755f..00000000 --- a/contrib/STYLE.md +++ /dev/null @@ -1,348 +0,0 @@ -# Style Guide - -A project’s long-term success rests (among other things) on its maintainability, and a maintainer has few tools more powerful than his or her project’s log. It’s worth taking the time to learn how to care for one properly. What may be a hassle at first soon becomes habit, and eventually a source of pride and productivity for all involved. - -Most programming languages have well-established conventions as to what constitutes idiomatic style, i.e. naming, formatting and so on. There are variations on these conventions, of course, but most developers agree that picking one and sticking to it is far better than the chaos that ensues when everybody does their own thing. - -# Table of Contents -1. [Code Style](#code-style) -2. [Naming Conventions](#naming-conventions) -3. [Git Commit Style](#git-commit-style) -4. [The Six Rules of a Great Commit](#the-six-rules-of-a-great-commit) - - [1. Atomic Commits](#1-atomic-commits) - - [2. Separate Subject from Body with a Blank Line](#2-separate-subject-from-body-with-a-blank-line) - - [3. Limit the Subject Line to 50 Characters](#3-limit-the-subject-line-to-50-characters) - - [4. Use the Imperative Mood in the Subject Line](#4-use-the-imperative-mood-in-the-subject-line) - - [5. Wrap the Body at 72 Characters](#5-wrap-the-body-at-72-characters) - - [6. Use the Body to Explain What and Why vs. How](#6-use-the-body-to-explain-what-and-why-vs-how) -5. [Tools Worth Mentioning](#tools-worth-mentioning) - - [Using `--fixup`](#using---fixup) - - [Interactive Rebase](#interactive-rebase) -6. [Pull Request and Squashing Commits Caveats](#pull-request-and-squashing-commits-caveats) - - -### Code style - -#### General Style -Python's official style guide is PEP 8, which provides conventions for writing code for the main Python distribution. Here are some key points: - -- `Indentation:` Use 4 spaces per indentation level. - -- `Line Length:` Limit all lines to a maximum of 79 characters. - -- `Blank Lines:` Surround top-level function and class definitions with two blank lines. Method definitions inside a class are surrounded by a single blank line. - -- `Imports:` Imports should usually be on separate lines and should be grouped in the following order: - - - Standard library imports. - - Related third party imports. - - Local application/library specific imports. -- `Whitespace:` Avoid extraneous whitespace in the following situations: - - - Immediately inside parentheses, brackets or braces. - - Immediately before a comma, semicolon, or colon. - - Immediately before the open parenthesis that starts the argument list of a function call. -- `Comments:` Comments should be complete sentences and should be used to clarify code and are not a substitute for poorly written code. - -#### For Python - -- `List Comprehensions:` Use list comprehensions for concise and readable creation of lists. - -- `Generators:` Use generators when dealing with large amounts of data to save memory. - -- `Context Managers:` Use context managers (with statement) for resource management. - -- `String Formatting:` Use f-strings for formatting strings in Python 3.6 and above. - -- `Error Handling:` Use exceptions for error handling whenever possible. - -#### More details - -Use `black` to format your python code before commiting for consistency across such a large pool of contributors. Black's code [style](https://black.readthedocs.io/en/stable/the_black_code_style/current_style.html#code-style) ensures consistent and opinionated code formatting. It automatically formats your Python code according to the Black style guide, enhancing code readability and maintainability. - -Key Features of Black: - - Consistency: Black enforces a single, consistent coding style across your project, eliminating style debates and allowing developers to focus on code logic. - - Readability: By applying a standard formatting style, Black improves code readability, making it easier to understand and collaborate on projects. - - Automation: Black automates the code formatting process, saving time and effort. It eliminates the need for manual formatting and reduces the likelihood of inconsistencies. - -### Naming Conventions - -- `Classes:` Class names should normally use the CapWords Convention. -- `Functions and Variables:` Function names should be lowercase, with words separated by underscores as necessary to improve readability. Variable names follow the same convention as function names. - -- `Constants:` Constants are usually defined on a module level and written in all capital letters with underscores separating words. - -- `Non-public Methods and Instance Variables:` Use a single leading underscore (_). This is a weak "internal use" indicator. - -- `Strongly "private" methods and variables:` Use a double leading underscore (__). This triggers name mangling in Python. - - -### Git commit style - -Here’s a model Git commit message when contributing: -``` -Summarize changes in around 50 characters or less - -More detailed explanatory text, if necessary. Wrap it to about 72 -characters or so. In some contexts, the first line is treated as the -subject of the commit and the rest of the text as the body. The -blank line separating the summary from the body is critical (unless -you omit the body entirely); various tools like `log`, `shortlog` -and `rebase` can get confused if you run the two together. - -Explain the problem that this commit is solving. Focus on why you -are making this change as opposed to how (the code explains that). -Are there side effects or other unintuitive consequences of this -change? Here's the place to explain them. - -Further paragraphs come after blank lines. - - - Bullet points are okay, too - - - Typically a hyphen or asterisk is used for the bullet, preceded - by a single space, with blank lines in between, but conventions - vary here - -If you use an issue tracker, put references to them at the bottom, -like this: - -Resolves: #123 -See also: #456, #789 -``` - - -## The six rules of a great commit. - -#### 1. Atomic Commits -An “atomic” change revolves around one task or one fix. - -Atomic Approach - - Commit each fix or task as a separate change - - Only commit when a block of work is complete - - Commit each layout change separately - - Joint commit for layout file, code behind file, and additional resources - -Benefits - -- Easy to roll back without affecting other changes -- Easy to make other changes on the fly -- Easy to merge features to other branches - -#### Avoid trivial commit messages - -Commit messages like "fix", "fix2", or "fix3" don't provide any context or clear understanding of what changes the commit introduces. Here are some examples of good vs. bad commit messages: - -**Bad Commit Message:** - - $ git commit -m "fix" - -**Good Commit Message:** - - $ git commit -m "Fix typo in README file" - -> **Caveat**: When working with new features, an atomic commit will often consist of multiple files, since a layout file, code behind file, and additional resources may have been added/modified. You don’t want to commit all of these separately, because if you had to roll back the application to a state before the feature was added, it would involve multiple commit entries, and that can get confusing - -#### 2. Separate subject from body with a blank line - -Not every commit requires both a subject and a body. Sometimes a single line is fine, especially when the change is so simple that no further context is necessary. - -For example: - - Fix typo in introduction to user guide - -Nothing more need be said; if the reader wonders what the typo was, she can simply take a look at the change itself, i.e. use git show or git diff or git log -p. - -If you’re committing something like this at the command line, it’s easy to use the -m option to git commit: - - $ git commit -m"Fix typo in introduction to user guide" - -However, when a commit merits a bit of explanation and context, you need to write a body. For example: - - Derezz the master control program - - MCP turned out to be evil and had become intent on world domination. - This commit throws Tron's disc into MCP (causing its deresolution) - and turns it back into a chess game. - -Commit messages with bodies are not so easy to write with the -m option. You’re better off writing the message in a proper text editor. [See Pro Git](https://git-scm.com/book/en/v2/Customizing-Git-Git-Configuration). - -In any case, the separation of subject from body pays off when browsing the log. Here’s the full log entry: - - $ git log - commit 42e769bdf4894310333942ffc5a15151222a87be - Author: Kevin Flynn - Date: Fri Jan 01 00:00:00 1982 -0200 - - Derezz the master control program - - MCP turned out to be evil and had become intent on world domination. - This commit throws Tron's disc into MCP (causing its deresolution) - and turns it back into a chess game. - - -#### 3. Limit the subject line to 50 characters -50 characters is not a hard limit, just a rule of thumb. Keeping subject lines at this length ensures that they are readable, and forces the author to think for a moment about the most concise way to explain what’s going on. - -GitHub’s UI is fully aware of these conventions. It will warn you if you go past the 50 character limit. Git will truncate any subject line longer than 72 characters with an ellipsis, thus keeping it to 50 is best practice. - -#### 4. Use the imperative mood in the subject line -Imperative mood just means “spoken or written as if giving a command or instruction”. A few examples: - - Clean your room - Close the door - Take out the trash - -Each of the seven rules you’re reading about right now are written in the imperative (“Wrap the body at 72 characters”, etc.). - -The imperative can sound a little rude; that’s why we don’t often use it. But it’s perfect for Git commit subject lines. One reason for this is that Git itself uses the imperative whenever it creates a commit on your behalf. - -For example, the default message created when using git merge reads: - - Merge branch 'myfeature' - -And when using git revert: - - Revert "Add the thing with the stuff" - - This reverts commit cc87791524aedd593cff5a74532befe7ab69ce9d. - -Or when clicking the “Merge” button on a GitHub pull request: - - Merge pull request #123 from someuser/somebranch - -So when you write your commit messages in the imperative, you’re following Git’s own built-in conventions. For example: - - Refactor subsystem X for readability - Update getting started documentation - Remove deprecated methods - Release version 1.0.0 - -Writing this way can be a little awkward at first. We’re more used to speaking in the indicative mood, which is all about reporting facts. That’s why commit messages often end up reading like this: - - Fixed bug with Y - Changing behavior of X - -And sometimes commit messages get written as a description of their contents: - - More fixes for broken stuff - Sweet new API methods - -To remove any confusion, here’s a simple rule to get it right every time. - -**A properly formed Git commit subject line should always be able to complete the following sentence:** - - If applied, this commit will - -For example: - - If applied, this commit will refactor subsystem X for readability - If applied, this commit will update getting started documentation - If applied, this commit will remove deprecated methods - If applied, this commit will release version 1.0.0 - If applied, this commit will merge pull request #123 from user/branch - -#### 5. Wrap the body at 72 characters -Git never wraps text automatically. When you write the body of a commit message, you must mind its right margin, and wrap text manually. - -The recommendation is to do this at 72 characters, so that Git has plenty of room to indent text while still keeping everything under 80 characters overall. - -A good text editor can help here. It’s easy to configure Vim, for example, to wrap text at 72 characters when you’re writing a Git commit. - -#### 6. Use the body to explain what and why vs. how -This [commit](https://github.com/bitcoin/bitcoin/commit/eb0b56b19017ab5c16c745e6da39c53126924ed6) from Bitcoin Core is a great example of explaining what changed and why: - -``` -commit eb0b56b19017ab5c16c745e6da39c53126924ed6 -Author: Pieter Wuille -Date: Fri Aug 1 22:57:55 2014 +0200 - - Simplify serialize.h's exception handling - - Remove the 'state' and 'exceptmask' from serialize.h's stream - implementations, as well as related methods. - - As exceptmask always included 'failbit', and setstate was always - called with bits = failbit, all it did was immediately raise an - exception. Get rid of those variables, and replace the setstate - with direct exception throwing (which also removes some dead - code). - - As a result, good() is never reached after a failure (there are - only 2 calls, one of which is in tests), and can just be replaced - by !eof(). - - fail(), clear(n) and exceptions() are just never called. Delete - them. -``` - -Take a look at the [full diff](https://github.com/bitcoin/bitcoin/commit/eb0b56b19017ab5c16c745e6da39c53126924ed6) and just think how much time the author is saving fellow and future committers by taking the time to provide this context here and now. If he didn’t, it would probably be lost forever. - -In most cases, you can leave out details about how a change has been made. Code is generally self-explanatory in this regard (and if the code is so complex that it needs to be explained in prose, that’s what source comments are for). Just focus on making clear the reasons why you made the change in the first place—the way things worked before the change (and what was wrong with that), the way they work now, and why you decided to solve it the way you did. - -The future maintainer that thanks you may be yourself! - - - -#### Tools worth mentioning - -##### Using `--fixup` - -If you've made a commit and then realize you've missed something or made a minor mistake, you can use the `--fixup` option. - -For example, suppose you've made a commit with a hash `9fceb02`. Later, you realize you've left a debug statement in your code. Instead of making a new commit titled "remove debug statement" or "fix", you can do the following: - - $ git commit --fixup 9fceb02 - -This will create a new commit to fix the issue, with a message like "fixup! The original commit message". - -##### Interactive Rebase - -Interactive rebase, or `rebase -i`, can be used to squash these fixup commits into the original commits they're fixing, which cleans up your commit history. You can use the `autosquash` option to automatically squash any commits marked as "fixup" into their target commits. - -For example: - - $ git rebase -i --autosquash HEAD~5 - -This command starts an interactive rebase for the last 5 commits (`HEAD~5`). Any commits marked as "fixup" will be automatically moved to squash with their target commits. - -The benefit of using `--fixup` and interactive rebase is that it keeps your commit history clean and readable. It groups fixes with the commits they are related to, rather than having a separate "fix" commit that might not make sense to other developers (or even to you) in the future. - - ---- - -#### Pull Request and Squashing Commits Caveats - -While atomic commits are great for development and for understanding the changes within the branch, the commit history can get messy when merging to the main branch. To keep a cleaner and more understandable commit history in our main branch, we encourage squashing all the commits of a PR into one when merging. - -This single commit should provide an overview of the changes that the PR introduced. It should follow the guidelines for atomic commits (an atomic commit is complete, self-contained, and understandable) but on the scale of the entire feature, task, or fix that the PR addresses. This approach combines the benefits of atomic commits during development with a clean commit history in our main branch. - -Here is how you can squash commits: - -```bash -git rebase -i HEAD~n -``` - -where `n` is the number of commits to squash. After running the command, replace `pick` with `squash` for the commits you want to squash into the previous commit. This will combine the commits and allow you to write a new commit message. - -In this context, an atomic commit message could look like: - -``` -Add feature X - -This commit introduces feature X which does A, B, and C. It adds -new files for layout, updates the code behind the file, and introduces -new resources. This change is important because it allows users to -perform task Y more efficiently. - -It includes: -- Creation of new layout file -- Updates in the code-behind file -- Addition of new resources - -Resolves: #123 -``` - -In your PRs, remember to detail what the PR is introducing or fixing. This will be helpful for reviewers to understand the context and the reason behind the changes. diff --git a/docs/running_on_mainnet.md b/docs/running_on_mainnet.md deleted file mode 100644 index 38be00a6..00000000 --- a/docs/running_on_mainnet.md +++ /dev/null @@ -1,244 +0,0 @@ -# Running Subnet on Mainnet - -This tutorial shows how to use the bittensor `btcli` to create a subnetwork and connect your incentive mechanism to it. - -**IMPORTANT:** Before attempting to register on mainnet, we strongly recommend that you: -- First run [Running Subnet Locally](running_on_staging.md), and -- Then run [Running on the Testnet](running_on_testnet.md). - -Your incentive mechanisms running on the mainnet are open to anyone. They emit real TAO. Creating these mechanisms incur a `lock_cost` in TAO. - -**DANGER** -- Do not expose your private keys. -- Only use your testnet wallet. -- Do not reuse the password of your mainnet wallet. -- Make sure your incentive mechanism is resistant to abuse. - -## Prerequisites - -Before proceeding further, make sure that you have installed Bittensor. See the below instructions: - -- [Install `bittensor`](https://github.com/opentensor/bittensor#install). - -After installing `bittensor`, proceed as below: - -## Steps - -## 1. Install your subnet template - -**NOTE: Skip this step if** you already did this during local testing and development. - -In your project directory: - -```bash -git clone https://github.com/opentensor/bittensor-subnet-template.git -``` - -Next, `cd` into `bittensor-subnet-template` repo directory: - -```bash -cd bittensor-subnet-template -``` - -Install the Bittensor subnet template package: - -```bash -python -m pip install -e . # Install your subnet template package -``` - -## 2. Create wallets - -Create wallets for subnet owner, subnet validator and for subnet miner. - -This step creates local coldkey and hotkey pairs for your three identities: subnet owner, subnet validator and subnet miner. - -The owner will create and control the subnet. The owner must have at least 100 TAO before the owner can run next steps. - -The validator and miner will be registered to the subnet created by the owner. This ensures that the validator and miner can run the respective validator and miner scripts. - -**NOTE**: You can also use existing wallets to register. Creating new keys is shown here for reference. - -Create a coldkey for the owner wallet: - -```bash -btcli wallet new_coldkey --wallet.name owner -``` - -Create a coldkey and hotkey for the subnet miner wallet: -```bash -btcli wallet new_coldkey --wallet.name miner -``` - -and - -```bash -btcli wallet new_hotkey --wallet.name miner --wallet.hotkey default -``` - -Create a coldkey and hotkey for the subnet validator wallet: - -```bash -btcli wallet new_coldkey --wallet.name validator -``` - -and - -```bash -btcli wallet new_hotkey --wallet.name validator --wallet.hotkey default -``` - -## 3. Getting the price of subnet creation - -Creating subnets on mainnet is competitive. The cost is determined by the rate at which new subnets are being registered onto the Bittensor blockchain. - -By default you must have at least 100 TAO on your owner wallet to create a subnet. However, the exact amount will fluctuate based on demand. The below code shows how to get the current price of creating a subnet. - -```bash -btcli subnet lock_cost -``` - -The above command will show: - -```bash ->> Subnet lock cost: τ100.000000000 -``` - -## 4. Purchasing a slot - -Using your TAO balance, you can register your subnet to the mainchain. This will create a new subnet on the mainchain and give you the owner permissions to it. The below command shows how to purchase a slot. - -**NOTE**: Slots cost TAO to lock. You will get this TAO back when the subnet is deregistered. - -```bash -btcli subnet create -``` - -Enter the owner wallet name. This gives permissions to the coldkey. - -```bash ->> Enter wallet name (default): owner # Enter your owner wallet name ->> Enter password to unlock key: # Enter your wallet password. ->> Register subnet? [y/n]: # Select yes (y) ->> ⠇ 📡 Registering subnet... -✅ Registered subnetwork with netuid: 1 # Your subnet netuid will show here, save this for later. -``` - -## 5. (Optional) Register keys - -**NOTE**: While this is not enforced, we recommend subnet owners to run a subnet validator and a subnet miner on the subnet to demonstrate proper use to the community. - -This step registers your subnet validator and subnet miner keys to the subnet giving them the **first two slots** on the subnet. - -Register your miner key to the subnet: - -```bash -btcli subnet recycle_register --netuid 1 --subtensor.network finney --wallet.name miner --wallet.hotkey default -``` - -Follow the below prompts: - -```bash ->> Enter netuid [1] (1): # Enter netuid 1 to specify the subnet you just created. ->> Continue Registration? - hotkey: ... - coldkey: ... - network: finney [y/n]: # Select yes (y) ->> ✅ Registered -``` - -Next, register your validator key to the subnet: - -```bash -btcli subnet recycle_register --netuid 1 --subtensor.network finney --wallet.name validator --wallet.hotkey default -``` - -Follow the below prompts: - -```bash ->> Enter netuid [1] (1): # Enter netuid 1 to specify the subnet you just created. ->> Continue Registration? - hotkey: ... - coldkey: ... - network: finney [y/n]: # Select yes (y) ->> ✅ Registered -``` - -## 6. Check that your keys have been registered - -Check that your subnet validator key has been registered: - -```bash -btcli wallet overview --wallet.name validator -``` - -The output will be similar to the below: - -```bash -Subnet: 1 -COLDKEY HOTKEY UID ACTIVE STAKE(τ) RANK TRUST CONSENSUS INCENTIVE DIVIDENDS EMISSION(ρ) VTRUST VPERMIT UPDATED AXON HOTKEY_SS58 -miner default 0 True 0.00000 0.00000 0.00000 0.00000 0.00000 0.00000 0 0.00000 14 none 5GTFrsEQfvTsh3WjiEVFeKzFTc2xcf… -1 1 2 τ0.00000 0.00000 0.00000 0.00000 0.00000 0.00000 ρ0 0.00000 - Wallet balance: τ0.0 -``` - -Check that your subnet miner has been registered: - -```bash -btcli wallet overview --wallet.name miner -``` - -The output will be similar to the below: - -```bash -Subnet: 1 -COLDKEY HOTKEY UID ACTIVE STAKE(τ) RANK TRUST CONSENSUS INCENTIVE DIVIDENDS EMISSION(ρ) VTRUST VPERMIT UPDATED AXON HOTKEY_SS58 -miner default 1 True 0.00000 0.00000 0.00000 0.00000 0.00000 0.00000 0 0.00000 14 none 5GTFrsEQfvTsh3WjiEVFeKzFTc2xcf… -1 1 2 τ0.00000 0.00000 0.00000 0.00000 0.00000 0.00000 ρ0 0.00000 - Wallet balance: τ0.0 -``` - -## 7. Run subnet miner and subnet validator - -Run the subnet miner: - -```bash -python neurons/miner.py --netuid 1 --wallet.name miner --wallet.hotkey default --logging.debug -``` - -You will see the below terminal output: - -```bash ->> 2023-08-08 16:58:11.223 | INFO | Running miner for subnet: 1 on network: wss://entrypoint-finney.opentensor.ai:443 with config: ... -``` - -Run the subnet validator: - -```bash -python neurons/validator.py --netuid 1 --wallet.name validator --wallet.hotkey default --logging.debug -``` - -You will see the below terminal output: - -```bash ->> 2023-08-08 16:58:11.223 | INFO | Running validator for subnet: 1 on network: wss://entrypoint-finney.opentensor.ai:443 with config: ... -``` - -## 8. Get emissions flowing - -Register to the root subnet using the `btcli`: - -```bash -btcli root register -``` - -Then set your weights for the subnet: - -```bash -btcli root weights -``` - -## 9. Stopping your nodes - -To stop your nodes, press CTRL + C in the terminal where the nodes are running. - ---- \ No newline at end of file diff --git a/docs/running_on_staging.md b/docs/running_on_staging.md deleted file mode 100644 index 6eeb4d5e..00000000 --- a/docs/running_on_staging.md +++ /dev/null @@ -1,340 +0,0 @@ -# Running Subnet Locally - -This tutorial will guide you through: - -- Setting up a local blockchain that is not connected to either Bittensor testchain or mainchain -- Creating a subnet -- Run your incentive mechanism on the subnet. - -## Local blockchain vs local subtensor node - -Running a local blockchain is sometimes synonymously referred as running on staging. This is **different** from running a local subtensor node that connects to the Bittensor mainchain. - -A local subtensor node will connect to the mainchain and sync with the mainchain, giving you your own access point to the mainchain. - -Running a local blockchain spins up two authority nodes locally, not connected to any other nodes or testchain or mainchain. This tutorial is for running a local blockchain. - -## Prerequisites - -Before proceeding further, make sure that you have installed Bittensor. See the below instructions: - -- [Install `bittensor`](https://github.com/opentensor/bittensor#install). - -After installing `bittensor`, proceed as below: - -## 1. Install Substrate dependencies - -Begin by installing the required dependencies for running a Substrate node. - -Update your system packages: - -```bash -sudo apt update -``` - -Install additional required libraries and tools - -```bash -sudo apt install --assume-yes make build-essential git clang curl libssl-dev llvm libudev-dev protobuf-compiler -``` - -## 2. Install Rust and Cargo - -Rust is the programming language used in Substrate development. Cargo is Rust package manager. - -Install rust and cargo: - -```bash -curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -``` - -Update your shell's source to include Cargo's path: - -```bash -source "$HOME/.cargo/env" -``` - -## 3. Clone the subtensor repository - -This step fetches the subtensor codebase to your local machine. - -```bash -git clone https://github.com/opentensor/subtensor.git -``` - -## 4. Setup Rust - -This step ensures that you have the nightly toolchain and the WebAssembly (wasm) compilation target. Note that this step will run the subtensor chain on your terminal directly, hence we advise that you run this as a background process using PM2 or other software. - -Update to the nightly version of Rust: - -```bash -./subtensor/scripts/init.sh -``` - -## 5. Initialize - -These steps initialize your local subtensor chain in development mode. These commands will set up and run a local subtensor. - -Build the binary with the faucet feature enabled: - -```bash -cargo build --release --features pow-faucet -``` - -**NOTE**: The `--features pow-faucet` option in the above is required if we want to use the command `btcli wallet faucet` [See the below Mint tokens step](#8-mint-tokens-from-faucet). - -Next, run the localnet script and turn off the attempt to build the binary (as we have already done this above): - -```bash -BUILD_BINARY=0 ./scripts/localnet.sh -``` - -**NOTE**: Watch for any build or initialization outputs in this step. If you are building the project for the first time, this step will take a while to finish building, depending on your hardware. - -## 6. Install subnet template - -`cd` to your project directory and clone the bittensor subnet template repository: - -```bash -git clone https://github.com/opentensor/bittensor-subnet-template.git -``` - -Navigate to the cloned repository: - -```bash -cd bittensor-subnet-template -``` - -Install the bittensor-subnet-template Python package: - -```bash -python -m pip install -e . -``` - -## 7. Set up wallets - -You will need wallets for the different roles, i.e., subnet owner, subnet validator and subnet miner, in the subnet. - -- The owner wallet creates and controls the subnet. -- The validator and miner will be registered to the subnet created by the owner. This ensures that the validator and miner can run the respective validator and miner scripts. - -Create a coldkey for the owner role: - -```bash -btcli wallet new_coldkey --wallet.name owner -``` - -Set up the miner's wallets: - -```bash -btcli wallet new_coldkey --wallet.name miner -``` - -```bash -btcli wallet new_hotkey --wallet.name miner --wallet.hotkey default -``` - -Set up the validator's wallets: - -```bash -btcli wallet new_coldkey --wallet.name validator -``` -```bash -btcli wallet new_hotkey --wallet.name validator --wallet.hotkey default -``` - -## 8. Mint tokens from faucet - -You will need tokens to initialize the intentive mechanism on the chain as well as for registering the subnet. - -Run the following commands to mint faucet tokens for the owner and for the validator. - -Mint faucet tokens for the owner: - -```bash -btcli wallet faucet --wallet.name owner --subtensor.chain_endpoint ws://127.0.0.1:9946 -``` - -You will see: - -```bash ->> Balance: τ0.000000000 ➡ τ100.000000000 -``` - -Mint tokens for the validator: - -```bash -btcli wallet faucet --wallet.name validator --subtensor.chain_endpoint ws://127.0.0.1:9946 -``` - -You will see: - -```bash ->> Balance: τ0.000000000 ➡ τ100.000000000 -``` - -## 9. Create a subnet - -The below commands establish a new subnet on the local chain. The cost will be exactly τ1000.000000000 for the first subnet you create and you'll have to run the faucet several times to get enough tokens. - -```bash -btcli subnet create --wallet.name owner --subtensor.chain_endpoint ws://127.0.0.1:9946 -``` - -You will see: - -```bash ->> Your balance is: τ200.000000000 ->> Do you want to register a subnet for τ1000.000000000? [y/n]: ->> Enter password to unlock key: [YOUR_PASSWORD] ->> ✅ Registered subnetwork with netuid: 1 -``` - -**NOTE**: The local chain will now have a default `netuid` of 1. The second registration will create a `netuid` 2 and so on, until you reach the subnet limit of 8. If you register more than 8 subnets, then a subnet with the least staked TAO will be replaced by the 9th subnet you register. - -## 10. Register keys - -Register your subnet validator and subnet miner on the subnet. This gives your two keys unique slots on the subnet. The subnet has a current limit of 128 slots. - -Register the subnet miner: - -```bash -btcli subnet register --wallet.name miner --wallet.hotkey default --subtensor.chain_endpoint ws://127.0.0.1:9946 -``` - -Follow the below prompts: - -```bash ->> Enter netuid [1] (1): 1 ->> Continue Registration? [y/n]: y ->> ✅ Registered -``` - -Register the subnet validator: - -```bash - -btcli subnet register --wallet.name validator --wallet.hotkey default --subtensor.chain_endpoint ws://127.0.0.1:9946 -``` - -Follow the below prompts: - -``` ->> Enter netuid [1] (1): 1 ->> Continue Registration? [y/n]: y ->> ✅ Registered -``` - -## 11. Add stake - -This step bootstraps the incentives on your new subnet by adding stake into its incentive mechanism. - -```bash -btcli stake add --wallet.name validator --wallet.hotkey default --subtensor.chain_endpoint ws://127.0.0.1:9946 -``` - -Follow the below prompts: - -```bash ->> Stake all Tao from account: 'validator'? [y/n]: y ->> Stake: - τ0.000000000 ➡ τ100.000000000 -``` - -## 12. Validate key registrations - -Verify that both the miner and validator keys are successfully registered: - -```bash -btcli subnet list --subtensor.chain_endpoint ws://127.0.0.1:9946 -``` - -You will see the `2` entry under `NEURONS` column for the `NETUID` of 1, indicating that you have registered a validator and a miner in this subnet: - -```bash -NETUID NEURONS MAX_N DIFFICULTY TEMPO CON_REQ EMISSION BURN(τ) - 1 2 256.00 10.00 M 1000 None 0.00% τ1.00000 - 2 128 -``` - -See the subnet validator's registered details: - -```bash -btcli wallet overview --wallet.name validator --subtensor.chain_endpoint ws://127.0.0.1:9946 -``` - -You will see: - -``` -Subnet: 1 -COLDKEY HOTKEY UID ACTIVE STAKE(τ) RANK TRUST CONSENSUS INCENTIVE DIVIDENDS EMISSION(ρ) VTRUST VPERMIT UPDATED AXON HOTKEY_SS58 -miner default 0 True 100.00000 0.00000 0.00000 0.00000 0.00000 0.00000 0 0.00000 14 none 5GTFrsEQfvTsh3WjiEVFeKzFTc2xcf… -1 1 2 τ100.00000 0.00000 0.00000 0.00000 0.00000 0.00000 ρ0 0.00000 - Wallet balance: τ0.0 -``` - -See the subnet miner's registered details: - -```bash -btcli wallet overview --wallet.name miner --subtensor.chain_endpoint ws://127.0.0.1:9946 -``` - -You will see: - -```bash -Subnet: 1 -COLDKEY HOTKEY UID ACTIVE STAKE(τ) RANK TRUST CONSENSUS INCENTIVE DIVIDENDS EMISSION(ρ) VTRUST VPERMIT UPDATED AXON HOTKEY_SS58 -miner default 1 True 0.00000 0.00000 0.00000 0.00000 0.00000 0.00000 0 0.00000 14 none 5GTFrsEQfvTsh3WjiEVFeKzFTc2xcf… -1 1 2 τ0.00000 0.00000 0.00000 0.00000 0.00000 0.00000 ρ0 0.00000 - Wallet balance: τ0.0 - -``` - -## 13. Run subnet miner and subnet validator - -Run the subnet miner and subnet validator. Make sure to specify your subnet parameters. - -Run the subnet miner: - -```bash -python neurons/miner.py --netuid 1 --subtensor.chain_endpoint ws://127.0.0.1:9946 --wallet.name miner --wallet.hotkey default --logging.debug -``` - -Run the subnet validator: - -```bash -python neurons/validator.py --netuid 1 --subtensor.chain_endpoint ws://127.0.0.1:9946 --wallet.name validator --wallet.hotkey default --logging.debug -``` - -## 14. Set weights for your subnet - -Register a validator on the root subnet and boost to set weights for your subnet. This is a necessary step to ensure that the subnet is able to receive emmissions. - -### Register your validator on the root subnet - -```bash -btcli root register --wallet.name validator --wallet.hotkey default --subtensor.chain_endpoint ws://127.0.0.1:9946 -``` - -### Boost your subnet on the root subnet -```bash -btcli root boost --netuid 1 --increase 1 --wallet.name validator --wallet.hotkey default --subtensor.chain_endpoint ws://127.0.0.1:9946 -``` - -## 15. Verify your incentive mechanism - -After a few blocks the subnet validator will set weights. This indicates that the incentive mechanism is active. Then after a subnet tempo elapses (360 blocks or 72 minutes) you will see your incentive mechanism beginning to distribute TAO to the subnet miner. - -```bash -btcli wallet overview --wallet.name miner --subtensor.chain_endpoint ws://127.0.0.1:9946 -``` - -## Ending your session - -To halt your nodes: -```bash -# Press CTRL + C keys in the terminal. -``` - ---- diff --git a/docs/running_on_testnet.md b/docs/running_on_testnet.md deleted file mode 100644 index 9203d3a5..00000000 --- a/docs/running_on_testnet.md +++ /dev/null @@ -1,242 +0,0 @@ -# Running Subnet on Testnet - -This tutorial shows how to use the Bittensor testnet to create a subnet and run your incentive mechanism on it. - -**IMPORTANT:** We strongly recommend that you first run [Running Subnet Locally](running_on_staging.md) before running on the testnet. Incentive mechanisms running on the testnet are open to anyone, and although these mechanisms on testnet do not emit real TAO, they cost you test TAO which you must create. - -**DANGER** -- Do not expose your private keys. -- Only use your testnet wallet. -- Do not reuse the password of your mainnet wallet. -- Make sure your incentive mechanism is resistant to abuse. - -## Prerequisites - -Before proceeding further, make sure that you have installed Bittensor. See the below instructions: - -- [Install `bittensor`](https://github.com/opentensor/bittensor#install). - -After installing `bittensor`, proceed as below: - -## 1. Install Bittensor subnet template - -**NOTE: Skip this step if** you already did this during local testing and development. - -`cd` into your project directory and clone the bittensor-subnet-template repo: - -```bash -git clone https://github.com/opentensor/bittensor-subnet-template.git -``` - -Next, `cd` into bittensor-subnet-template repo directory: - -```bash -cd bittensor-subnet-template # Enter the -``` - -Install the bittensor-subnet-template package: - -```bash -python -m pip install -e . -``` - -## 2. Create wallets - -Create wallets for subnet owner, subnet validator and for subnet miner. - -This step creates local coldkey and hotkey pairs for your three identities: subnet owner, subnet validator and subnet miner. - -The owner will create and control the subnet. The owner must have at least 100 testnet TAO before the owner can run next steps. - -The validator and miner will be registered to the subnet created by the owner. This ensures that the validator and miner can run the respective validator and miner scripts. - -Create a coldkey for your owner wallet: - -```bash -btcli wallet new_coldkey --wallet.name owner -``` - -Create a coldkey and hotkey for your miner wallet: - -```bash -btcli wallet new_coldkey --wallet.name miner -``` - -and - -```bash -btcli wallet new_hotkey --wallet.name miner --wallet.hotkey default -``` - -Create a coldkey and hotkey for your validator wallet: - -```bash -btcli wallet new_coldkey --wallet.name validator -``` - -and - -```bash -btcli wallet new_hotkey --wallet.name validator --wallet.hotkey default -``` - -## 3. Get the price of subnet creation - -Creating subnets on the testnet is competitive. The cost is determined by the rate at which new subnets are being registered onto the chain. - -By default you must have at least 100 testnet TAO in your owner wallet to create a subnet. However, the exact amount will fluctuate based on demand. The below command shows how to get the current price of creating a subnet. - -```bash -btcli subnet lock_cost --subtensor.network test -``` - -The above command will show: - -```bash ->> Subnet lock cost: τ100.000000000 -``` - -## 4. (Optional) Get faucet tokens - -Faucet is disabled on the testnet. Hence, if you don't have sufficient faucet tokens, ask the [Bittensor Discord community](https://discord.com/channels/799672011265015819/830068283314929684) for faucet tokens. - -## 5. Purchase a slot - -Using the test TAO from the previous step you can register your subnet on the testnet. This will create a new subnet on the testnet and give you the owner permissions to it. - -The below command shows how to purchase a slot. - -**NOTE**: Slots cost TAO to lock. You will get this TAO back when the subnet is deregistered. - -```bash -btcli subnet create --subtensor.network test -``` - -Enter the owner wallet name which gives permissions to the coldkey: - -```bash ->> Enter wallet name (default): owner # Enter your owner wallet name ->> Enter password to unlock key: # Enter your wallet password. ->> Register subnet? [y/n]: # Select yes (y) ->> ⠇ 📡 Registering subnet... -✅ Registered subnetwork with netuid: 1 # Your subnet netuid will show here, save this for later. -``` - -## 6. Register keys - -This step registers your subnet validator and subnet miner keys to the subnet, giving them the **first two slots** on the subnet. - -Register your miner key to the subnet: - -```bash -btcli subnet register --netuid 13 --subtensor.network test --wallet.name miner --wallet.hotkey default -``` - -Follow the below prompts: - -```bash ->> Enter netuid [1] (1): # Enter netuid 1 to specify the subnet you just created. ->> Continue Registration? - hotkey: ... - coldkey: ... - network: finney [y/n]: # Select yes (y) ->> ✅ Registered -``` - -Next, register your validator key to the subnet: - -```bash -btcli subnet register --netuid 13 --subtensor.network test --wallet.name validator --wallet.hotkey default -``` - -Follow the prompts: - -```bash ->> Enter netuid [1] (1): # Enter netuid 1 to specify the subnet you just created. ->> Continue Registration? - hotkey: ... - coldkey: ... - network: finney [y/n]: # Select yes (y) ->> ✅ Registered -``` - -## 7. Check that your keys have been registered - -This step returns information about your registered keys. - -Check that your validator key has been registered: - -```bash -btcli wallet overview --wallet.name validator --subtensor.network test -``` - -The above command will display the below: - -```bash -Subnet: 1 -COLDKEY HOTKEY UID ACTIVE STAKE(τ) RANK TRUST CONSENSUS INCENTIVE DIVIDENDS EMISSION(ρ) VTRUST VPERMIT UPDATED AXON HOTKEY_SS58 -miner default 0 True 0.00000 0.00000 0.00000 0.00000 0.00000 0.00000 0 0.00000 14 none 5GTFrsEQfvTsh3WjiEVFeKzFTc2xcf… -1 1 2 τ0.00000 0.00000 0.00000 0.00000 0.00000 0.00000 ρ0 0.00000 - Wallet balance: τ0.0 -``` - -Check that your miner has been registered: - -```bash -btcli wallet overview --wallet.name miner --subtensor.network test -``` - -The above command will display the below: - -```bash -Subnet: 1 -COLDKEY HOTKEY UID ACTIVE STAKE(τ) RANK TRUST CONSENSUS INCENTIVE DIVIDENDS EMISSION(ρ) VTRUST VPERMIT UPDATED AXON HOTKEY_SS58 -miner default 1 True 0.00000 0.00000 0.00000 0.00000 0.00000 0.00000 0 0.00000 14 none 5GTFrsEQfvTsh3WjiEVFeKzFTc2xcf… -1 1 2 τ0.00000 0.00000 0.00000 0.00000 0.00000 0.00000 ρ0 0.00000 - Wallet balance: τ0.0 -``` - -## 8. Run subnet miner and subnet validator - -Run the subnet miner: - -```bash -python neurons/miner.py --netuid 1 --subtensor.network test --wallet.name miner --wallet.hotkey default --logging.debug -``` - -You will see the below terminal output: - -```bash ->> 2023-08-08 16:58:11.223 | INFO | Running miner for subnet: 1 on network: ws://127.0.0.1:9946 with config: ... -``` - -Next, run the subnet validator: - -```bash -python neurons/validator.py --netuid 1 --subtensor.network test --wallet.name validator --wallet.hotkey default --logging.debug -``` - -You will see the below terminal output: - -```bash ->> 2023-08-08 16:58:11.223 | INFO | Running validator for subnet: 1 on network: ws://127.0.0.1:9946 with config: ... -``` - - -## 9. Get emissions flowing - -Register to the root network using the `btcli`: - -```bash -btcli root register --subtensor.network test -``` - -Then set your weights for the subnet: - -```bash -btcli root weights --subtensor.network test -``` - -## 10. Stopping your nodes - -To stop your nodes, press CTRL + C in the terminal where the nodes are running. diff --git a/docs/stream_tutorial/README.md b/docs/stream_tutorial/README.md deleted file mode 100644 index f213fd3a..00000000 --- a/docs/stream_tutorial/README.md +++ /dev/null @@ -1,490 +0,0 @@ -# Bittensor Streaming Tutorial -This document is intented as a developer-friendly walkthrough of integrating streaming into your bittensor application. - -If you prefer to jump right into a complete stand-alone example, see: -- `miner.py` -- `protocol.py` -- `client.py` - -Start your miner: -```bash -python miner.py --netuid 8 --wallet.name default --wallet.hotkey miner --subtensor.network test --axon.port 10000 --logging.trace -``` - -Run the client: -```bash -python client.py --netuid 8 --my_uid 1 --network test -``` - -## Overview -This tutorial is designed to show you how to use the streaming API to integrate into your application. It will cover the following topics: -- writing your streaming protocol (inherits from bittensor.StreamingSynapse) -- writing your streaming server (uses your streaming protocol) -- writing your streaming client (uses your streaming protocol) - -### Defining your streaming protocol -When designing your protocol, it would be helpful to look at the bittensor.StreamingSynapse for reference. Below is a condensed snippet of the abstract methods that you will need to implement in your subclass. - -You will need to implement two methods: - -- `process_streaming_response` -- `extract_response_json` - -These two methods are the core of your streaming protocol. The first method process_streaming_response is called as the response is being streamed from the network. It is responsible for handling the streaming response, such as parsing and accumulating data. The second method extract_response_json is called after the response has been processed and is responsible for retrieving structured data to be post-processed in the dendrite in bittensor core code. - -```python -class StreamingSynapse(bittensor.Synapse, ABC): - ... - class BTStreamingResponse(_StreamingResponse): - ... - @abstractmethod - async def process_streaming_response(self, response: Response): - """ - Abstract method that must be implemented by the subclass. - This method should provide logic to handle the streaming response, such as parsing and accumulating data. - It is called as the response is being streamed from the network, and should be implemented to handle the specific - streaming data format and requirements of the subclass. - - Args: - response: The response object to be processed, typically containing chunks of data. - """ - ... - - @abstractmethod - def extract_response_json(self, response: Response) -> dict: - """ - Abstract method that must be implemented by the subclass. - This method should provide logic to extract JSON data from the response, including headers and content. - It is called after the response has been processed and is responsible for retrieving structured data - that can be used by the application. - - Args: - response: The response object from which to extract JSON data. - """ - ... - ... -``` - -See the full reference code at the bittensor [repo](https://github.com/opentensor/bittensor/blob/master/bittensor/stream.py). - - -#### Create your protocol -Let's walk through how to create a protocol using the bittensor.StreamingSynapse class. -```python -class MyStreamingSynapse(bt.StreamingSynapse): - # define your expected data fields here as pydantic field objects - # This allows you to control what information is passed along the network - messages: List[str] = pydantic.Field( - ..., # this ellipsis (...) indicates the object is required - title="Messages", # What is the name of this field? - description="A list of messages in the Prompting scenario. Immutable.", - allow_mutation=False, # disallow modification of this field after creation - ) - completion: str = pydantic.Field( - "", - title="Completion", - ) - # add fields as necessary - ... - - # This method controls how your synapse is deserialized from the network - # E.g. you can extract whatever information you want to receive at the final - # yield in the async generator returned by the server, without receiving - # the entire synapse object itself. - # In this example, we just want the completion string at the end. - def deserialize(self) -> str: - return self.completion - - # implement your `process_streaming_response` logic to actually yield objects to the streamer - # this effectively defines the async generator that you'll recieve on the client side - async def process_streaming_response(self, response: MyStreamingSynapse): - # this is an example of how you might process a streaming response - # iterate over the response content and yield each line - async for chunk in response.content.iter_any(): - tokens = chunk.decode("utf-8").split("\n") - yield tokens - - # implement `extract_response_json` to extract the JSON data from the response headers - # this will be dependent on the data you are streaming and how you want to structure it - # it MUST conform to the following format expected by the bittensor dendrite: - """ - { - # METADATA AND HEADERS - "name": ..., - "timeout": float(...), - "total_size": int(...), - "header_size": int(...), - "dendrite": ..., - "axon": ..., - # YOUR FIELDS - "messages": self.messages, - ... - } - """ - def extract_response_json(self, response: MyStreamingSynapse) -> dict: - # iterate over the response headers and extract the necessary data - headers = { - k.decode("utf-8"): v.decode("utf-8") - for k, v in response.__dict__["_raw_headers"] - } - # helper function to extract data from headers - def extract_info(prefix): - return { - key.split("_")[-1]: value - for key, value in headers.items() - if key.startswith(prefix) - } - # return the extracted data in the expected format - return { - "name": headers.get("name", ""), - "timeout": float(headers.get("timeout", 0)), - "total_size": int(headers.get("total_size", 0)), - "header_size": int(headers.get("header_size", 0)), - "dendrite": extract_info("bt_header_dendrite"), # dendrite info - "axon": extract_info("bt_header_axon"), # axon info - "messages": self.messages, # field object - } -``` - -[Here](https://github.com/opentensor/text-prompting/blob/main/prompting/protocol.py#L131) is a full example implementation of a streaming protocol based on the text-prompting network. - -Please read the docstrings provided, they can be very helpful! - -### Writing the server -Great! Now we have our protocol defined, let's see how to define our server. -This will generate the tokens to be streamed in this prompting example. - -For brevity we will not be building a full miner, but inspecting the central components. -```python -class MyStreamPromptingMiner(bt.Miner): - ... # any relevant methods you'd need for your miner - - # define your server forward here - # NOTE: It is crucial that your typehints are correct and reflect your streaming protocol object - # otherwise the axon will reject adding your route to the server. - def forward(self, synapse: MyStreamingSynapse) -> MyStreamingSynapse: - # Let's use a GPT2 tokenizer for this toy example - tokenizer = GPT2Tokenizer.from_pretrained("gpt2") - - # Simulated function to decode token IDs into strings. In a real-world scenario, - # this can be replaced with an actual model inference step. - def model(ids): - return (tokenizer.decode(id) for id in ids) - - # This function is called asynchronously to process the input text and send back tokens - # as a streaming response. It essentially produces the async generator that will be - # consumed by the client with an `async for` loop. - async def _forward(text: str, send: Send): - # `text` may be the input prompt to your model in a real-world scenario. - # let's tokenize them into IDs for the sake of this example. - input_ids = tokenizer(text, return_tensors="pt").input_ids.squeeze() - - # You may want to buffer your tokens before sending them back to the client. - # this can be useful so we aren't flooding the client with individual tokens - # and allows you more fine-grained control over how much data is sent back - # with each yield. - N = 3 # Number of tokens to send back to the client at a time - buffer = [] - # Iterate over the tokens and send the generationed tokens back to the client - # when we have sufficient (N) tokens in the buffer. - for token in model(input_ids): - buffer.append(token) # Add token to buffer - - # If buffer has N tokens, send them back to the client. - if len(buffer) == N: - joined_buffer = "".join(buffer) - # Send the tokens back to the client - # This is the core of the streaming response and the format - # is important. The `send` function is provided by the ASGI server - # and is responsible for sending the response back to the client. - # This buffer will be received by the client as a single chunk of - # data, which can then be split into individual tokens! - await send( - { - "type": "http.response.body", - "body": joined_buffer.encode("utf-8"), - "more_body": True, - } - ) - buffer = [] # Clear the buffer for next batch of tokens - - # Create a streaming response object using the `_forward` function - # It is useful to wrap your _forward function in a partial function - # to pass in the text argument lazily. - token_streamer = partial(_forward, synapse.messages[0]) - # Return the streaming response object, which is an instance of the - # `BTStreamingResponse` class. - return synapse.create_streaming_response(token_streamer) -``` - -#### Complete Example -Here is a full example for reference: -> This inherits from the prompting (text-prompting) miner base class. -> Take a look at the `prompting/baseminer/miner.py` file [here](https://github.com/opentensor/text-prompting/blob/main/prompting/baseminer/miner.py) for more details. - -```python -class StreamingTemplateMiner(prompting.Miner): - def config(self) -> "bt.Config": - """ - Returns the configuration object specific to this miner. - - Implement and extend this method to provide custom configurations for the miner. - Currently, it sets up a basic configuration parser. - - Returns: - bt.Config: A configuration object with the miner's operational parameters. - """ - parser = argparse.ArgumentParser(description="Streaming Miner Configs") - self.add_args(parser) - return bt.config(parser) - - def add_args(cls, parser: argparse.ArgumentParser): - """ - Adds custom arguments to the command line parser. - - Developers can introduce additional command-line arguments specific to the miner's - functionality in this method. These arguments can then be used to configure the miner's operation. - - Args: - parser (argparse.ArgumentParser): - The command line argument parser to which custom arguments should be added. - """ - pass - - def prompt(self, synapse: StreamPrompting) -> StreamPrompting: - """ - Generates a streaming response for the provided synapse. - - This function serves as the main entry point for handling streaming prompts. It takes - the incoming synapse which contains messages to be processed and returns a streaming - response. The function uses the GPT-2 tokenizer and a simulated model to tokenize and decode - the incoming message, and then sends the response back to the client token by token. - - Args: - synapse (StreamPrompting): The incoming StreamPrompting instance containing the messages to be processed. - - Returns: - StreamPrompting: The streaming response object which can be used by other functions to - stream back the response to the client. - - Usage: - This function can be extended and customized based on specific requirements of the - miner. Developers can swap out the tokenizer, model, or adjust how streaming responses - are generated to suit their specific applications. - """ - bt.logging.trace("In outer PROMPT()") - tokenizer = GPT2Tokenizer.from_pretrained("gpt2") - - # Simulated function to decode token IDs into strings. In a real-world scenario, - # this can be replaced with an actual model inference step. - def model(ids): - return (tokenizer.decode(id) for id in ids) - - async def _prompt(text: str, send: Send): - """ - Asynchronously processes the input text and sends back tokens as a streaming response. - - This function takes an input text, tokenizes it using the GPT-2 tokenizer, and then - uses the simulated model to decode token IDs into strings. It then sends each token - back to the client as a streaming response, with a delay between tokens to simulate - the effect of real-time streaming. - - Args: - text (str): The input text message to be processed. - send (Send): An asynchronous function that allows sending back the streaming response. - - Usage: - This function can be adjusted based on the streaming requirements, speed of - response, or the model being used. Developers can also introduce more sophisticated - processing steps or modify how tokens are sent back to the client. - """ - bt.logging.trace("In inner _PROMPT()") - input_ids = tokenizer(text, return_tensors="pt").input_ids.squeeze() - buffer = [] - bt.logging.debug(f"Input text: {text}") - bt.logging.debug(f"Input ids: {input_ids}") - - N = 3 # Number of tokens to send back to the client at a time - for token in model(input_ids): - bt.logging.trace(f"appending token: {token}") - buffer.append(token) - # If buffer has N tokens, send them back to the client. - if len(buffer) == N: - time.sleep(0.1) - joined_buffer = "".join(buffer) - bt.logging.debug(f"sedning tokens: {joined_buffer}") - await send( - { - "type": "http.response.body", - "body": joined_buffer.encode("utf-8"), - "more_body": True, - } - ) - bt.logging.debug(f"Streamed tokens: {joined_buffer}") - buffer = [] # Clear the buffer for next batch of tokens - - # Send any remaining tokens in the buffer - if buffer: - joined_buffer = "".join(buffer) - await send( - { - "type": "http.response.body", - "body": joined_buffer.encode("utf-8"), - "more_body": False, # No more tokens to send - } - ) - bt.logging.trace(f"Streamed tokens: {joined_buffer}") - - message = synapse.messages[0] - bt.logging.trace(f"message in _prompt: {message}") - token_streamer = partial(_prompt, message) - bt.logging.trace(f"token streamer: {token_streamer}") - return synapse.create_streaming_response(token_streamer) -``` - -### Writing the client -Excellent! Now we have defined our server, now we can define our client. - -This has assumed you have: -1. Registered your miner on the chain (`finney`/`test`) -2. Are serving your miner on an open port (e.g. `12345`) - -Steps: -- Instantiate your synapse subclass with the relevant information. E.g. `messages`, `roles`, etc. -- Instantiate your wallet and a dendrite client -- Query the dendrite client with your synapse object -- Iterate over the async generator to extract the yielded tokens on the server side - -```python - -# Import bittensor -import bittensor as bt - -# Create your streaming synapse subclass object to house the request body -syn = MyStreamingSynapse( - roles=["user"], - messages=["hello this is a test of a streaming response. Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum."] -) - -# Create a wallet instance that must be registered on the network -wallet = bt.wallet(name="default", hotkey="default") - -# Instantiate the metagraph -metagraph = bt.metagraph( - netuid=8, network="test", sync=True, lite=False -) - -# Grab the axon you're serving -my_uid = 1 -axon = metagraph.axons[my_uid] - -# Create a Dendrite instance to handle client-side communication. -dendrite = bt.dendrite(wallet=wallet) - - -This is an async function so we can use the `await` keyword when querying the server with the dendrite object. -async def main(): - # Send a request to the Axon using the Dendrite, passing in a StreamPrompting - # instance with roles and messages. The response is awaited, as the Dendrite - # communicates asynchronously with the Axon. Returns a list of async generator. - responses = await dendrite( - [axon], - syn, - deserialize=False, - streaming=True - ) - - # Now that we have our responses we want to iterate over the yielded tokens - # iterate over the async generator to extract the yielded tokens on server side - for resp in responses: - i=0 - async for chunk in resp: - i += 1 - if i % 5 == 0: - print() - if isinstance(chunk, list): - print(chunk[0], end="", flush=True) - else: - # last object yielded is the synapse itself with completion filled - synapse = chunk - break - - # The synapse object contains the completion attribute which contains the - # accumulated tokens from the streaming response. - -if __name__ == "__main__": - # Run the main function with asyncio - asyncio.run(main()) - -``` -There you have it! - -### Complete example -If you would like to see a complete standalone example that only depends on bittensor>=6.2.0, look below: - -- client.py -- streaming_miner.py -- - -# client.py -```python -# Import bittensor and the text-prompting packages -import bittensor as bt -import prompting - -# Create a StreamPrompting synapse object to house the request body -syn = prompting.protocol.StreamPrompting( - roles=["user"], - messages=["hello this is a test of a streaming response. Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum."]) -syn - -# create a wallet instance that must be registered on the network -wallet = bt.wallet(name="default", hotkey="default") -wallet - -# instantiate the metagraph -metagraph = bt.metagraph( - netuid=8, network="test", sync=True, lite=False -) -metagraph - -# Grab the axon you're serving -axon = metagraph.axons[62] -axon - -# Create a Dendrite instance to handle client-side communication. -d = bt.dendrite(wallet=wallet) -d - - -async def main(): - - # Send a request to the Axon using the Dendrite, passing in a StreamPrompting - # instance with roles and messages. The response is awaited, as the Dendrite - # communicates asynchronously with the Axon. Returns a list of async generator. - responses = await d( - [axon], - syn, - deserialize=False, - streaming=True - ) - responses - - # iterate over the async generator to extract the yielded tokens on server side - for resp in responses: - i=0 - async for chunk in resp: - i += 1 - if i % 5 == 0: - print() - if isinstance(chunk, list): - print(chunk[0], end="", flush=True) - else: - # last object yielded is the synapse itself with completion filled - synapse = chunk - break - -if __name__ == "__main__": - import asyncio - asyncio.run(main()) -``` diff --git a/docs/stream_tutorial/client.py b/docs/stream_tutorial/client.py deleted file mode 100644 index 67e6f05c..00000000 --- a/docs/stream_tutorial/client.py +++ /dev/null @@ -1,104 +0,0 @@ -import argparse -import asyncio -import bittensor as bt - -from protocol import StreamPrompting - -""" -This has assumed you have: -1. Registered your miner on the chain (finney/test) -2. Are serving your miner on an open port (e.g. 12345) - -Steps: -- Instantiate your synapse subclass with the relevant information. E.g. messages, roles, etc. -- Instantiate your wallet and a dendrite client -- Query the dendrite client with your synapse object -- Iterate over the async generator to extract the yielded tokens on the server side -""" - - -async def query_synapse(my_uid, wallet_name, hotkey, network, netuid): - syn = StreamPrompting( - roles=["user"], - messages=[ - "hello this is a test of a streaming response. Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua." - ], - ) - - # create a wallet instance with provided wallet name and hotkey - wallet = bt.wallet(name=wallet_name, hotkey=hotkey) - - # instantiate the metagraph with provided network and netuid - metagraph = bt.metagraph( - netuid=netuid, network=network, sync=True, lite=False - ) - - # Grab the axon you're serving - axon = metagraph.axons[my_uid] - - # Create a Dendrite instance to handle client-side communication. - dendrite = bt.dendrite(wallet=wallet) - - async def main(): - responses = await dendrite( - [axon], syn, deserialize=False, streaming=True - ) - - for resp in responses: - i = 0 - async for chunk in resp: - i += 1 - if i % 5 == 0: - print() - if isinstance(chunk, list): - print(chunk[0], end="", flush=True) - else: - # last object yielded is the synapse itself with completion filled - synapse = chunk - break - - # Run the main function with asyncio - await main() - - -if __name__ == "__main__": - parser = argparse.ArgumentParser( - description="Query a Bittensor synapse with given parameters." - ) - - # Adding arguments - parser.add_argument( - "--my_uid", - type=int, - required=True, - help="Your unique miner ID on the chain", - ) - parser.add_argument( - "--netuid", type=int, required=True, help="Network Unique ID" - ) - parser.add_argument( - "--wallet_name", type=str, default="default", help="Name of the wallet" - ) - parser.add_argument( - "--hotkey", type=str, default="default", help="Hotkey for the wallet" - ) - parser.add_argument( - "--network", - type=str, - default="test", - help='Network type, e.g., "test" or "mainnet"', - ) - - # Parse arguments - args = parser.parse_args() - - # Running the async function with provided arguments - asyncio.run( - query_synapse( - args.my_uid, - args.wallet_name, - args.hotkey, - args.network, - args.netuid, - ) - ) diff --git a/docs/stream_tutorial/config.py b/docs/stream_tutorial/config.py deleted file mode 100644 index 7cbe82ca..00000000 --- a/docs/stream_tutorial/config.py +++ /dev/null @@ -1,116 +0,0 @@ -import bittensor as bt -import argparse -import os - - -def check_config(cls, config: "bt.Config"): - bt.axon.check_config(config) - bt.logging.check_config(config) - full_path = os.path.expanduser( - "{}/{}/{}/{}".format( - config.logging.logging_dir, - config.wallet.get("name", bt.defaults.wallet.name), - config.wallet.get("hotkey", bt.defaults.wallet.hotkey), - config.miner.name, - ) - ) - config.miner.full_path = os.path.expanduser(full_path) - if not os.path.exists(config.miner.full_path): - os.makedirs(config.miner.full_path) - - -def get_config() -> "bt.Config": - parser = argparse.ArgumentParser() - parser.add_argument( - "--axon.port", type=int, default=8098, help="Port to run the axon on." - ) - # Subtensor network to connect to - parser.add_argument( - "--subtensor.network", - default="finney", - help="Bittensor network to connect to.", - ) - # Chain endpoint to connect to - parser.add_argument( - "--subtensor.chain_endpoint", - default="wss://entrypoint-finney.opentensor.ai:443", - help="Chain endpoint to connect to.", - ) - # Adds override arguments for network and netuid. - parser.add_argument( - "--netuid", type=int, default=1, help="The chain subnet uid." - ) - - parser.add_argument( - "--miner.root", - type=str, - help="Trials for this miner go in miner.root / (wallet_cold - wallet_hot) / miner.name ", - default="~/.bittensor/miners/", - ) - parser.add_argument( - "--miner.name", - type=str, - help="Trials for this miner go in miner.root / (wallet_cold - wallet_hot) / miner.name ", - default="Bittensor Miner", - ) - - # Run config. - parser.add_argument( - "--miner.blocks_per_epoch", - type=str, - help="Blocks until the miner repulls the metagraph from the chain", - default=100, - ) - - # Switches. - parser.add_argument( - "--miner.no_serve", - action="store_true", - help="If True, the miner doesnt serve the axon.", - default=False, - ) - parser.add_argument( - "--miner.no_start_axon", - action="store_true", - help="If True, the miner doesnt start the axon.", - default=False, - ) - - # Mocks. - parser.add_argument( - "--miner.mock_subtensor", - action="store_true", - help="If True, the miner will allow non-registered hotkeys to mine.", - default=False, - ) - - # Adds subtensor specific arguments i.e. --subtensor.chain_endpoint ... --subtensor.network ... - bt.subtensor.add_args(parser) - - # Adds logging specific arguments i.e. --logging.debug ..., --logging.trace .. or --logging.logging_dir ... - bt.logging.add_args(parser) - - # Adds wallet specific arguments i.e. --wallet.name ..., --wallet.hotkey ./. or --wallet.path ... - bt.wallet.add_args(parser) - - # Adds axon specific arguments i.e. --axon.port ... - bt.axon.add_args(parser) - - # Activating the parser to read any command-line inputs. - # To print help message, run python3 template/miner.py --help - config = bt.config(parser) - - # Logging captures events for diagnosis or understanding miner's behavior. - config.full_path = os.path.expanduser( - "{}/{}/{}/netuid{}/{}".format( - config.logging.logging_dir, - config.wallet.name, - config.wallet.hotkey, - config.netuid, - "miner", - ) - ) - # Ensure the directory for logging exists, else create one. - if not os.path.exists(config.full_path): - os.makedirs(config.full_path, exist_ok=True) - return config diff --git a/docs/stream_tutorial/miner.py b/docs/stream_tutorial/miner.py deleted file mode 100644 index a62814d2..00000000 --- a/docs/stream_tutorial/miner.py +++ /dev/null @@ -1,398 +0,0 @@ -import copy -import time -import asyncio -import argparse -import threading -import traceback -from abc import ABC, abstractmethod -from functools import partial -from starlette.types import Send - -import bittensor as bt -from transformers import GPT2Tokenizer -from typing import List, Dict, Tuple, Union, Callable, Awaitable - -from protocol import StreamPrompting -from config import get_config, check_config - - -class StreamMiner(ABC): - def __init__(self, config=None, axon=None, wallet=None, subtensor=None): - # Setup base config from Miner.config() and merge with subclassed config. - base_config = copy.deepcopy(config or get_config()) - self.config = self.config() - self.config.merge(base_config) - - check_config(StreamMiner, self.config) - bt.logging.info(self.config) # TODO: duplicate print? - - self.prompt_cache: Dict[str, Tuple[str, int]] = {} - - # Activating Bittensor's logging with the set configurations. - bt.logging.set_config(config=self.config.logging) - - # Wallet holds cryptographic information, ensuring secure transactions and communication. - self.wallet = wallet or bt.wallet(config=self.config) - bt.logging.info(f"Wallet {self.wallet}") - - # subtensor manages the blockchain connection, facilitating interaction with the Bittensor blockchain. - self.subtensor = subtensor or bt.subtensor(config=self.config) - bt.logging.info(f"Subtensor: {self.subtensor}") - bt.logging.info( - f"Running miner for subnet: {self.config.netuid} on network: {self.subtensor.chain_endpoint} with config:" - ) - - # metagraph provides the network's current state, holding state about other participants in a subnet. - self.metagraph = self.subtensor.metagraph(self.config.netuid) - bt.logging.info(f"Metagraph: {self.metagraph}") - - if self.wallet.hotkey.ss58_address not in self.metagraph.hotkeys: - bt.logging.error( - f"\nYour validator: {self.wallet} if not registered to chain connection: {self.subtensor} \nRun btcli register and try again. " - ) - exit() - else: - # Each miner gets a unique identity (UID) in the network for differentiation. - self.my_subnet_uid = self.metagraph.hotkeys.index( - self.wallet.hotkey.ss58_address - ) - bt.logging.info(f"Running miner on uid: {self.my_subnet_uid}") - - # The axon handles request processing, allowing validators to send this process requests. - self.axon = axon or bt.axon( - wallet=self.wallet, port=self.config.axon.port - ) - # Attach determiners which functions are called when servicing a request. - bt.logging.info(f"Attaching forward function to axon.") - print(f"Attaching forward function to axon. {self._prompt}") - self.axon.attach( - forward_fn=self._prompt, - ) - bt.logging.info(f"Axon created: {self.axon}") - - # Instantiate runners - self.should_exit: bool = False - self.is_running: bool = False - self.thread: threading.Thread = None - self.lock = asyncio.Lock() - self.request_timestamps: Dict = {} - - @abstractmethod - def config(self) -> "bt.Config": - ... - - @classmethod - @abstractmethod - def add_args(cls, parser: argparse.ArgumentParser): - ... - - def _prompt(self, synapse: StreamPrompting) -> StreamPrompting: - """ - A wrapper method around the `prompt` method that will be defined by the subclass. - - This method acts as an intermediary layer to perform pre-processing before calling the - actual `prompt` method implemented in the subclass. Specifically, it checks whether a - prompt is in cache to avoid reprocessing recent requests. If the prompt is not in the - cache, the subclass `prompt` method is called. - - Args: - synapse (StreamPrompting): The incoming request object encapsulating the details of the request. - - Returns: - StreamPrompting: The response object to be sent back in reply to the incoming request, essentially - the filled synapse request object. - - Raises: - ValueError: If the prompt is found in the cache indicating it was sent recently. - - Example: - This method is not meant to be called directly but is invoked internally when a request - is received, and it subsequently calls the `prompt` method of the subclass. - """ - return self.prompt(synapse) - - @abstractmethod - def prompt(self, synapse: StreamPrompting) -> StreamPrompting: - """ - Abstract method to handle and respond to incoming requests to the miner. - - Subclasses should implement this method to define their custom logic for processing and - responding to requests. This method is designed to be overridden, and its behavior will - be dependent on the specific implementation provided in the subclass. - - Args: - synapse (StreamPrompting): The incoming request object encapsulating the details - of the request. This must contain `messages` and `roles` as fields. - - Returns: - StreamPrompting: The response object that should be sent back in reply to the - incoming request. This is essentially the filled synapse request object. - - Example: - class CustomMiner(Miner): - def prompt(self, synapse: StreamPrompting) -> StreamPrompting: - # Custom logic to process and respond to the request. - synapse.completion = "The meaning of life is 42." - return synapse - """ - ... - - def run(self): - """ - Runs the miner logic. This method starts the miner's operations, including - listening for incoming requests and periodically updating the miner's knowledge - of the network graph. - """ - if not self.subtensor.is_hotkey_registered( - netuid=self.config.netuid, - hotkey_ss58=self.wallet.hotkey.ss58_address, - ): - bt.logging.error( - f"Wallet: {self.wallet} is not registered on netuid {self.config.netuid}" - f"Please register the hotkey using `btcli subnets register` before trying again" - ) - exit() - - # Serve passes the axon information to the network + netuid we are hosting on. - # This will auto-update if the axon port of external ip have changed. - bt.logging.info( - f"Serving axon {StreamPrompting} on network: {self.config.subtensor.chain_endpoint} with netuid: {self.config.netuid}" - ) - self.axon.serve(netuid=self.config.netuid, subtensor=self.subtensor) - - # Start starts the miner's axon, making it active on the network. - bt.logging.info( - f"Starting axon server on port: {self.config.axon.port}" - ) - self.axon.start() - - # --- Run until should_exit = True. - self.last_epoch_block = self.subtensor.get_current_block() - bt.logging.info(f"Miner starting at block: {self.last_epoch_block}") - - # This loop maintains the miner's operations until intentionally stopped. - bt.logging.info(f"Starting main loop") - step = 0 - try: - while not self.should_exit: - start_epoch = time.time() - - # --- Wait until next epoch. - current_block = self.subtensor.get_current_block() - while ( - current_block - self.last_epoch_block - < self.config.miner.blocks_per_epoch - ): - # --- Wait for next bloc. - time.sleep(1) - current_block = self.subtensor.get_current_block() - - # --- Check if we should exit. - if self.should_exit: - break - - # --- Update the metagraph with the latest network state. - self.last_epoch_block = self.subtensor.get_current_block() - - metagraph = self.subtensor.metagraph( - netuid=self.config.netuid, - lite=True, - block=self.last_epoch_block, - ) - log = ( - f"Step:{step} | " - f"Block:{metagraph.block.item()} | " - f"Stake:{metagraph.S[self.my_subnet_uid]} | " - f"Rank:{metagraph.R[self.my_subnet_uid]} | " - f"Trust:{metagraph.T[self.my_subnet_uid]} | " - f"Consensus:{metagraph.C[self.my_subnet_uid] } | " - f"Incentive:{metagraph.I[self.my_subnet_uid]} | " - f"Emission:{metagraph.E[self.my_subnet_uid]}" - ) - bt.logging.info(log) - - step += 1 - - # If someone intentionally stops the miner, it'll safely terminate operations. - except KeyboardInterrupt: - self.axon.stop() - bt.logging.success("Miner killed by keyboard interrupt.") - exit() - - # In case of unforeseen errors, the miner will log the error and continue operations. - except Exception as e: - bt.logging.error(traceback.format_exc()) - - def run_in_background_thread(self): - """ - Starts the miner's operations in a separate background thread. - This is useful for non-blocking operations. - """ - if not self.is_running: - bt.logging.debug("Starting miner in background thread.") - self.should_exit = False - self.thread = threading.Thread(target=self.run, daemon=True) - self.thread.start() - self.is_running = True - bt.logging.debug("Started") - - def stop_run_thread(self): - """ - Stops the miner's operations that are running in the background thread. - """ - if self.is_running: - bt.logging.debug("Stopping miner in background thread.") - self.should_exit = True - self.thread.join(5) - self.is_running = False - bt.logging.debug("Stopped") - - def __enter__(self): - """ - Starts the miner's operations in a background thread upon entering the context. - This method facilitates the use of the miner in a 'with' statement. - """ - self.run_in_background_thread() - - def __exit__(self, exc_type, exc_value, traceback): - """ - Stops the miner's background operations upon exiting the context. - This method facilitates the use of the miner in a 'with' statement. - - Args: - exc_type: The type of the exception that caused the context to be exited. - None if the context was exited without an exception. - exc_value: The instance of the exception that caused the context to be exited. - None if the context was exited without an exception. - traceback: A traceback object encoding the stack trace. - None if the context was exited without an exception. - """ - self.stop_run_thread() - - -class StreamingTemplateMiner(StreamMiner): - def config(self) -> "bt.Config": - """ - Returns the configuration object specific to this miner. - - Implement and extend this method to provide custom configurations for the miner. - Currently, it sets up a basic configuration parser. - - Returns: - bt.Config: A configuration object with the miner's operational parameters. - """ - parser = argparse.ArgumentParser(description="Streaming Miner Configs") - self.add_args(parser) - return bt.config(parser) - - def add_args(cls, parser: argparse.ArgumentParser): - """ - Adds custom arguments to the command line parser. - - Developers can introduce additional command-line arguments specific to the miner's - functionality in this method. These arguments can then be used to configure the miner's operation. - - Args: - parser (argparse.ArgumentParser): - The command line argument parser to which custom arguments should be added. - """ - pass - - def prompt(self, synapse: StreamPrompting) -> StreamPrompting: - """ - Generates a streaming response for the provided synapse. - - This function serves as the main entry point for handling streaming prompts. It takes - the incoming synapse which contains messages to be processed and returns a streaming - response. The function uses the GPT-2 tokenizer and a simulated model to tokenize and decode - the incoming message, and then sends the response back to the client token by token. - - Args: - synapse (StreamPrompting): The incoming StreamPrompting instance containing the messages to be processed. - - Returns: - StreamPrompting: The streaming response object which can be used by other functions to - stream back the response to the client. - - Usage: - This function can be extended and customized based on specific requirements of the - miner. Developers can swap out the tokenizer, model, or adjust how streaming responses - are generated to suit their specific applications. - """ - bt.logging.trace("HI. PROMPT()") - tokenizer = GPT2Tokenizer.from_pretrained("gpt2") - - # Simulated function to decode token IDs into strings. In a real-world scenario, - # this can be replaced with an actual model inference step. - def model(ids): - return (tokenizer.decode(id) for id in ids) - - async def _prompt(text: str, send: Send): - """ - Asynchronously processes the input text and sends back tokens as a streaming response. - - This function takes an input text, tokenizes it using the GPT-2 tokenizer, and then - uses the simulated model to decode token IDs into strings. It then sends each token - back to the client as a streaming response, with a delay between tokens to simulate - the effect of real-time streaming. - - Args: - text (str): The input text message to be processed. - send (Send): An asynchronous function that allows sending back the streaming response. - - Usage: - This function can be adjusted based on the streaming requirements, speed of - response, or the model being used. Developers can also introduce more sophisticated - processing steps or modify how tokens are sent back to the client. - """ - bt.logging.trace("HI. _PROMPT()") - input_ids = tokenizer( - text, return_tensors="pt" - ).input_ids.squeeze() - buffer = [] - bt.logging.debug(f"Input text: {text}") - bt.logging.debug(f"Input ids: {input_ids}") - - N = 3 # Number of tokens to send back to the client at a time - for token in model(input_ids): - bt.logging.trace(f"appending token: {token}") - buffer.append(token) - # If buffer has N tokens, send them back to the client. - if len(buffer) == N: - time.sleep(0.1) - joined_buffer = "".join(buffer) - bt.logging.debug(f"sedning tokens: {joined_buffer}") - await send( - { - "type": "http.response.body", - "body": joined_buffer.encode("utf-8"), - "more_body": True, - } - ) - bt.logging.debug(f"Streamed tokens: {joined_buffer}") - buffer = [] # Clear the buffer for next batch of tokens - - # Send any remaining tokens in the buffer - if buffer: - joined_buffer = "".join(buffer) - await send( - { - "type": "http.response.body", - "body": joined_buffer.encode("utf-8"), - "more_body": False, # No more tokens to send - } - ) - bt.logging.trace(f"Streamed tokens: {joined_buffer}") - - message = synapse.messages[0] - bt.logging.trace(f"message in _prompt: {message}") - token_streamer = partial(_prompt, message) - bt.logging.trace(f"token streamer: {token_streamer}") - return synapse.create_streaming_response(token_streamer) - - -# This is the main function, which runs the miner. -if __name__ == "__main__": - with StreamingTemplateMiner(): - while True: - time.sleep(1) diff --git a/docs/stream_tutorial/protocol.py b/docs/stream_tutorial/protocol.py deleted file mode 100644 index 26e91fdc..00000000 --- a/docs/stream_tutorial/protocol.py +++ /dev/null @@ -1,154 +0,0 @@ -import pydantic -import bittensor as bt - -from abc import ABC, abstractmethod -from typing import List, Union, Callable, Awaitable -from starlette.responses import StreamingResponse - - -class StreamPrompting(bt.StreamingSynapse): - """ - StreamPrompting is a specialized implementation of the `StreamingSynapse` tailored for prompting functionalities within - the Bittensor network. This class is intended to interact with a streaming response that contains a sequence of tokens, - which represent prompts or messages in a certain scenario. - - As a developer, when using or extending the `StreamPrompting` class, you should be primarily focused on the structure - and behavior of the prompts you are working with. The class has been designed to seamlessly handle the streaming, - decoding, and accumulation of tokens that represent these prompts. - - Attributes: - - `roles` (List[str]): A list of roles involved in the prompting scenario. This could represent different entities - or agents involved in the conversation or use-case. They are immutable to ensure consistent - interaction throughout the lifetime of the object. - - - `messages` (List[str]): These represent the actual prompts or messages in the prompting scenario. They are also - immutable to ensure consistent behavior during processing. - - - `completion` (str): Stores the processed result of the streaming tokens. As tokens are streamed, decoded, and - processed, they are accumulated in the completion attribute. This represents the "final" - product or result of the streaming process. - - `required_hash_fields` (List[str]): A list of fields that are required for the hash. - - Methods: - - `process_streaming_response`: This method asynchronously processes the incoming streaming response by decoding - the tokens and accumulating them in the `completion` attribute. - - - `deserialize`: Converts the `completion` attribute into its desired data format, in this case, a string. - - - `extract_response_json`: Extracts relevant JSON data from the response, useful for gaining insights on the response's - metadata or for debugging purposes. - - Note: While you can directly use the `StreamPrompting` class, it's designed to be extensible. Thus, you can create - subclasses to further customize behavior for specific prompting scenarios or requirements. - """ - - roles: List[str] = pydantic.Field( - ..., - title="Roles", - description="A list of roles in the StreamPrompting scenario. Immuatable.", - allow_mutation=False, - ) - - messages: List[str] = pydantic.Field( - ..., - title="Messages", - description="A list of messages in the StreamPrompting scenario. Immutable.", - allow_mutation=False, - ) - - required_hash_fields: List[str] = pydantic.Field( - ["messages"], - title="Required Hash Fields", - description="A list of required fields for the hash.", - allow_mutation=False, - ) - - completion: str = pydantic.Field( - "", - title="Completion", - description="Completion status of the current StreamPrompting object. This attribute is mutable and can be updated.", - ) - - async def process_streaming_response(self, response: StreamingResponse): - """ - `process_streaming_response` is an asynchronous method designed to process the incoming streaming response from the - Bittensor network. It's the heart of the StreamPrompting class, ensuring that streaming tokens, which represent - prompts or messages, are decoded and appropriately managed. - - As the streaming response is consumed, the tokens are decoded from their 'utf-8' encoded format, split based on - newline characters, and concatenated into the `completion` attribute. This accumulation of decoded tokens in the - `completion` attribute allows for a continuous and coherent accumulation of the streaming content. - - Args: - response: The streaming response object containing the content chunks to be processed. Each chunk in this - response is expected to be a set of tokens that can be decoded and split into individual messages or prompts. - """ - if self.completion is None: - self.completion = "" - bt.logging.debug( - "Processing streaming response (StreamingSynapse base class)." - ) - async for chunk in response.content.iter_any(): - bt.logging.debug(f"Processing chunk: {chunk}") - tokens = chunk.decode("utf-8").split("\n") - for token in tokens: - bt.logging.debug(f"--processing token: {token}") - if token: - self.completion += token - bt.logging.debug(f"yielding tokens {tokens}") - yield tokens - - def deserialize(self) -> str: - """ - Deserializes the response by returning the completion attribute. - - Returns: - str: The completion result. - """ - return self.completion - - def extract_response_json(self, response: StreamingResponse) -> dict: - """ - `extract_response_json` is a method that performs the crucial task of extracting pertinent JSON data from the given - response. The method is especially useful when you need a detailed insight into the streaming response's metadata - or when debugging response-related issues. - - Beyond just extracting the JSON data, the method also processes and structures the data for easier consumption - and understanding. For instance, it extracts specific headers related to dendrite and axon, offering insights - about the Bittensor network's internal processes. The method ultimately returns a dictionary with a structured - view of the extracted data. - - Args: - response: The response object from which to extract the JSON data. This object typically includes headers and - content which can be used to glean insights about the response. - - Returns: - dict: A structured dictionary containing: - - Basic response metadata such as name, timeout, total_size, and header_size. - - Dendrite and Axon related information extracted from headers. - - Roles and Messages pertaining to the current StreamPrompting instance. - - The accumulated completion. - """ - headers = { - k.decode("utf-8"): v.decode("utf-8") - for k, v in response.__dict__["_raw_headers"] - } - - def extract_info(prefix): - return { - key.split("_")[-1]: value - for key, value in headers.items() - if key.startswith(prefix) - } - - return { - "name": headers.get("name", ""), - "timeout": float(headers.get("timeout", 0)), - "total_size": int(headers.get("total_size", 0)), - "header_size": int(headers.get("header_size", 0)), - "dendrite": extract_info("bt_header_dendrite"), - "axon": extract_info("bt_header_axon"), - "roles": self.roles, - "messages": self.messages, - "completion": self.completion, - } diff --git a/neurons/miner.py b/neurons/miner.py index 5f7b9500..8fca52f2 100644 --- a/neurons/miner.py +++ b/neurons/miner.py @@ -22,10 +22,10 @@ import bittensor as bt # Bittensor Miner Template: -import template +import cancer_ai # import base miner class which takes care of most of the boilerplate -from template.base.miner import BaseMinerNeuron +from cancer_ai.base.miner import BaseMinerNeuron class Miner(BaseMinerNeuron): diff --git a/neurons/validator.py b/neurons/validator.py index e28b972c..07222c06 100644 --- a/neurons/validator.py +++ b/neurons/validator.py @@ -24,9 +24,9 @@ import bittensor as bt # import base validator class which takes care of most of the boilerplate -from template.base.validator import BaseValidatorNeuron +from cancer_ai.base.validator import BaseValidatorNeuron # Bittensor Validator Template: -from template.validator import forward +from cancer_ai.validator import forward class Validator(BaseValidatorNeuron): diff --git a/scripts/check_compatibility.sh b/scripts/check_compatibility.sh deleted file mode 100755 index b0bd6b43..00000000 --- a/scripts/check_compatibility.sh +++ /dev/null @@ -1,76 +0,0 @@ -#!/bin/bash - -if [ -z "$1" ]; then - echo "Please provide a Python version as an argument." - exit 1 -fi - -python_version="$1" -all_passed=true - -GREEN='\033[0;32m' -YELLOW='\033[0;33m' -RED='\033[0;31m' -NC='\033[0m' # No Color - -check_compatibility() { - all_supported=0 - - while read -r requirement; do - # Skip lines starting with git+ - if [[ "$requirement" == git+* ]]; then - continue - fi - - package_name=$(echo "$requirement" | awk -F'[!=<>]' '{print $1}' | awk -F'[' '{print $1}') # Strip off brackets - echo -n "Checking $package_name... " - - url="https://pypi.org/pypi/$package_name/json" - response=$(curl -s $url) - status_code=$(curl -s -o /dev/null -w "%{http_code}" $url) - - if [ "$status_code" != "200" ]; then - echo -e "${RED}Information not available for $package_name. Failure.${NC}" - all_supported=1 - continue - fi - - classifiers=$(echo "$response" | jq -r '.info.classifiers[]') - requires_python=$(echo "$response" | jq -r '.info.requires_python') - - base_version="Programming Language :: Python :: ${python_version%%.*}" - specific_version="Programming Language :: Python :: $python_version" - - if echo "$classifiers" | grep -q "$specific_version" || echo "$classifiers" | grep -q "$base_version"; then - echo -e "${GREEN}Supported${NC}" - elif [ "$requires_python" != "null" ]; then - if echo "$requires_python" | grep -Eq "==$python_version|>=$python_version|<=$python_version"; then - echo -e "${GREEN}Supported${NC}" - else - echo -e "${RED}Not compatible with Python $python_version due to constraint $requires_python.${NC}" - all_supported=1 - fi - else - echo -e "${YELLOW}Warning: Specific version not listed, assuming compatibility${NC}" - fi - done < requirements.txt - - return $all_supported -} - -echo "Checking compatibility for Python $python_version..." -check_compatibility -if [ $? -eq 0 ]; then - echo -e "${GREEN}All requirements are compatible with Python $python_version.${NC}" -else - echo -e "${RED}All requirements are NOT compatible with Python $python_version.${NC}" - all_passed=false -fi - -echo "" -if $all_passed; then - echo -e "${GREEN}All tests passed.${NC}" -else - echo -e "${RED}All tests did not pass.${NC}" - exit 1 -fi diff --git a/scripts/check_requirements_changes.sh b/scripts/check_requirements_changes.sh deleted file mode 100755 index a06d050f..00000000 --- a/scripts/check_requirements_changes.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash - -# Check if requirements files have changed in the last commit -if git diff --name-only HEAD~1 | grep -E 'requirements.txt|requirements.txt'; then - echo "Requirements files have changed. Running compatibility checks..." - echo 'export REQUIREMENTS_CHANGED="true"' >> $BASH_ENV -else - echo "Requirements files have not changed. Skipping compatibility checks..." - echo 'export REQUIREMENTS_CHANGED="false"' >> $BASH_ENV -fi diff --git a/scripts/install_staging.sh b/scripts/install_staging.sh deleted file mode 100644 index 24280ced..00000000 --- a/scripts/install_staging.sh +++ /dev/null @@ -1,145 +0,0 @@ -#!/bin/bash - -# Section 1: Build/Install -# This section is for first-time setup and installations. - -install_dependencies() { - # Function to install packages on macOS - install_mac() { - which brew > /dev/null - if [ $? -ne 0 ]; then - echo "Installing Homebrew..." - /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" - fi - echo "Updating Homebrew packages..." - brew update - echo "Installing required packages..." - brew install make llvm curl libssl protobuf tmux - } - - # Function to install packages on Ubuntu/Debian - install_ubuntu() { - echo "Updating system packages..." - sudo apt update - echo "Installing required packages..." - sudo apt install --assume-yes make build-essential git clang curl libssl-dev llvm libudev-dev protobuf-compiler tmux - } - - # Detect OS and call the appropriate function - if [[ "$OSTYPE" == "darwin"* ]]; then - install_mac - elif [[ "$OSTYPE" == "linux-gnu"* ]]; then - install_ubuntu - else - echo "Unsupported operating system." - exit 1 - fi - - # Install rust and cargo - curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh - - # Update your shell's source to include Cargo's path - source "$HOME/.cargo/env" -} - -# Call install_dependencies only if it's the first time running the script -if [ ! -f ".dependencies_installed" ]; then - install_dependencies - touch .dependencies_installed -fi - - -# Section 2: Test/Run -# This section is for running and testing the setup. - -# Create a coldkey for the owner role -wallet=${1:-owner} - -# Logic for setting up and running the environment -setup_environment() { - # Clone subtensor and enter the directory - if [ ! -d "subtensor" ]; then - git clone https://github.com/opentensor/subtensor.git - fi - cd subtensor - git pull - - # Update to the nightly version of rust - ./scripts/init.sh - - cd ../bittensor-subnet-template - - # Install the bittensor-subnet-template python package - python -m pip install -e . - - # Create and set up wallets - # This section can be skipped if wallets are already set up - if [ ! -f ".wallets_setup" ]; then - btcli wallet new_coldkey --wallet.name $wallet --no_password --no_prompt - btcli wallet new_coldkey --wallet.name miner --no_password --no_prompt - btcli wallet new_hotkey --wallet.name miner --wallet.hotkey default --no_prompt - btcli wallet new_coldkey --wallet.name validator --no_password --no_prompt - btcli wallet new_hotkey --wallet.name validator --wallet.hotkey default --no_prompt - touch .wallets_setup - fi - -} - -# Call setup_environment every time -setup_environment - -## Setup localnet -# assumes we are in the bittensor-subnet-template/ directory -# Initialize your local subtensor chain in development mode. This command will set up and run a local subtensor network. -cd ../subtensor - -# Start a new tmux session and create a new pane, but do not switch to it -echo "FEATURES='pow-faucet runtime-benchmarks' BT_DEFAULT_TOKEN_WALLET=$(cat ~/.bittensor/wallets/$wallet/coldkeypub.txt | grep -oP '"ss58Address": "\K[^"]+') bash scripts/localnet.sh" >> setup_and_run.sh -chmod +x setup_and_run.sh -tmux new-session -d -s localnet -n 'localnet' -tmux send-keys -t localnet 'bash ../subtensor/setup_and_run.sh' C-m - -# Notify the user -echo ">> localnet.sh is running in a detached tmux session named 'localnet'" -echo ">> You can attach to this session with: tmux attach-session -t localnet" - -# Register a subnet (this needs to be run each time we start a new local chain) -btcli subnet create --wallet.name $wallet --wallet.hotkey default --subtensor.chain_endpoint ws://127.0.0.1:9946 --no_prompt - -# Transfer tokens to miner and validator coldkeys -export BT_MINER_TOKEN_WALLET=$(cat ~/.bittensor/wallets/miner/coldkeypub.txt | grep -oP '"ss58Address": "\K[^"]+') -export BT_VALIDATOR_TOKEN_WALLET=$(cat ~/.bittensor/wallets/validator/coldkeypub.txt | grep -oP '"ss58Address": "\K[^"]+') - -btcli wallet transfer --subtensor.network ws://127.0.0.1:9946 --wallet.name $wallet --dest $BT_MINER_TOKEN_WALLET --amount 1000 --no_prompt -btcli wallet transfer --subtensor.network ws://127.0.0.1:9946 --wallet.name $wallet --dest $BT_VALIDATOR_TOKEN_WALLET --amount 10000 --no_prompt - -# Register wallet hotkeys to subnet -btcli subnet register --wallet.name miner --netuid 1 --wallet.hotkey default --subtensor.chain_endpoint ws://127.0.0.1:9946 --no_prompt -btcli subnet register --wallet.name validator --netuid 1 --wallet.hotkey default --subtensor.chain_endpoint ws://127.0.0.1:9946 --no_prompt - -# Add stake to the validator -btcli stake add --wallet.name validator --wallet.hotkey default --subtensor.chain_endpoint ws://127.0.0.1:9946 --amount 10000 --no_prompt - -# Ensure both the miner and validator keys are successfully registered. -btcli subnet list --subtensor.chain_endpoint ws://127.0.0.1:9946 -btcli wallet overview --wallet.name validator --subtensor.chain_endpoint ws://127.0.0.1:9946 --no_prompt -btcli wallet overview --wallet.name miner --subtensor.chain_endpoint ws://127.0.0.1:9946 --no_prompt - -cd ../bittensor-subnet-template - - -# Check if inside a tmux session -if [ -z "$TMUX" ]; then - # Start a new tmux session and run the miner in the first pane - tmux new-session -d -s bittensor -n 'miner' 'python neurons/miner.py --netuid 1 --subtensor.chain_endpoint ws://127.0.0.1:9946 --wallet.name miner --wallet.hotkey default --logging.debug' - - # Split the window and run the validator in the new pane - tmux split-window -h -t bittensor:miner 'python neurons/validator.py --netuid 1 --subtensor.chain_endpoint ws://127.0.0.1:9946 --wallet.name validator --wallet.hotkey default --logging.debug' - - # Attach to the new tmux session - tmux attach-session -t bittensor -else - # If already in a tmux session, create two panes in the current window - tmux split-window -h 'python neurons/miner.py --netuid 1 --subtensor.chain_endpoint ws://127.0.0.1:9946 --wallet.name miner --wallet.hotkey default --logging.debug' - tmux split-window -v -t 0 'python neurons/validator.py --netuid 1 --subtensor.chain_endpoint ws://127.0.0.1:9946 --wallet.name3 validator --wallet.hotkey default --logging.debug' -fi diff --git a/template/api/dummy.py b/template/api/dummy.py deleted file mode 100644 index f6a433f1..00000000 --- a/template/api/dummy.py +++ /dev/null @@ -1,44 +0,0 @@ -# The MIT License (MIT) -# Copyright © 2021 Yuma Rao -# Copyright © 2023 Opentensor Foundation -# Copyright © 2023 Opentensor Technologies Inc - -# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated -# documentation files (the “Software”), to deal in the Software without restriction, including without limitation -# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, -# and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all copies or substantial portions of -# the Software. - -# THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO -# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL -# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -# DEALINGS IN THE SOFTWARE. - -import bittensor as bt -from typing import List, Optional, Union, Any, Dict -from template.protocol import Dummy -from bittensor.subnets import SubnetsAPI - - -class DummyAPI(SubnetsAPI): - def __init__(self, wallet: "bt.wallet"): - super().__init__(wallet) - self.netuid = 33 - self.name = "dummy" - - def prepare_synapse(self, dummy_input: int) -> Dummy: - synapse.dummy_input = dummy_input - return synapse - - def process_responses( - self, responses: List[Union["bt.Synapse", Any]] - ) -> List[int]: - outputs = [] - for response in responses: - if response.dendrite.status_code != 200: - continue - return outputs.append(response.dummy_output) - return outputs diff --git a/template/api/get_query_axons.py b/template/api/get_query_axons.py deleted file mode 100644 index 5d51c8f3..00000000 --- a/template/api/get_query_axons.py +++ /dev/null @@ -1,126 +0,0 @@ -# The MIT License (MIT) -# Copyright © 2021 Yuma Rao -# Copyright © 2023 Opentensor Foundation -# Copyright © 2023 Opentensor Technologies Inc - -# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated -# documentation files (the “Software”), to deal in the Software without restriction, including without limitation -# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, -# and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all copies or substantial portions of -# the Software. - -# THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO -# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL -# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -# DEALINGS IN THE SOFTWARE. -import numpy as np -import random -import bittensor as bt - - -async def ping_uids(dendrite, metagraph, uids, timeout=3): - """ - Pings a list of UIDs to check their availability on the Bittensor network. - - Args: - dendrite (bittensor.dendrite): The dendrite instance to use for pinging nodes. - metagraph (bittensor.metagraph): The metagraph instance containing network information. - uids (list): A list of UIDs (unique identifiers) to ping. - timeout (int, optional): The timeout in seconds for each ping. Defaults to 3. - - Returns: - tuple: A tuple containing two lists: - - The first list contains UIDs that were successfully pinged. - - The second list contains UIDs that failed to respond. - """ - axons = [metagraph.axons[uid] for uid in uids] - try: - responses = await dendrite( - axons, - bt.Synapse(), # TODO: potentially get the synapses available back? - deserialize=False, - timeout=timeout, - ) - successful_uids = [ - uid - for uid, response in zip(uids, responses) - if response.dendrite.status_code == 200 - ] - failed_uids = [ - uid - for uid, response in zip(uids, responses) - if response.dendrite.status_code != 200 - ] - except Exception as e: - bt.logging.error(f"Dendrite ping failed: {e}") - successful_uids = [] - failed_uids = uids - bt.logging.debug(f"ping() successful uids: {successful_uids}") - bt.logging.debug(f"ping() failed uids : {failed_uids}") - return successful_uids, failed_uids - -async def get_query_api_nodes(dendrite, metagraph, n=0.1, timeout=3): - """ - Fetches the available API nodes to query for the particular subnet. - - Args: - wallet (bittensor.wallet): The wallet instance to use for querying nodes. - metagraph (bittensor.metagraph): The metagraph instance containing network information. - n (float, optional): The fraction of top nodes to consider based on stake. Defaults to 0.1. - timeout (int, optional): The timeout in seconds for pinging nodes. Defaults to 3. - - Returns: - list: A list of UIDs representing the available API nodes. - """ - bt.logging.debug( - f"Fetching available API nodes for subnet {metagraph.netuid}" - ) - vtrust_uids = [ - uid.item() - for uid in metagraph.uids - if metagraph.validator_trust[uid] > 0 - ] - top_uids = np.where(metagraph.S > np.quantile(metagraph.S, 1 - n))[0].tolist() - init_query_uids = set(top_uids).intersection(set(vtrust_uids)) - query_uids, _ = await ping_uids( - dendrite, metagraph, list(init_query_uids), timeout=timeout - ) - bt.logging.debug( - f"Available API node UIDs for subnet {metagraph.netuid}: {query_uids}" - ) - if len(query_uids) > 3: - query_uids = random.sample(query_uids, 3) - return query_uids - - -async def get_query_api_axons( - wallet, metagraph=None, n=0.1, timeout=3, uids=None -): - """ - Retrieves the axons of query API nodes based on their availability and stake. - - Args: - wallet (bittensor.wallet): The wallet instance to use for querying nodes. - metagraph (bittensor.metagraph, optional): The metagraph instance containing network information. - n (float, optional): The fraction of top nodes to consider based on stake. Defaults to 0.1. - timeout (int, optional): The timeout in seconds for pinging nodes. Defaults to 3. - uids (Union[List[int], int], optional): The specific UID(s) of the API node(s) to query. Defaults to None. - - Returns: - list: A list of axon objects for the available API nodes. - """ - dendrite = bt.dendrite(wallet=wallet) - - if metagraph is None: - metagraph = bt.metagraph(netuid=21) - - if uids is not None: - query_uids = [uids] if isinstance(uids, int) else uids - else: - query_uids = await get_query_api_nodes( - dendrite, metagraph, n=n, timeout=timeout - ) - return [metagraph.axons[uid] for uid in query_uids] diff --git a/template/base/utils/__init__.py b/template/base/utils/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/template/subnet_links.py b/template/subnet_links.py deleted file mode 100644 index c33f2e2d..00000000 --- a/template/subnet_links.py +++ /dev/null @@ -1,76 +0,0 @@ -SUBNET_LINKS = [ - {"name": "sn0", "url": ""}, - {"name": "sn1", "url": "https://github.com/opentensor/prompting/"}, - {"name": "sn2", "url": "https://github.com/inference-labs-inc/omron-subnet/"}, - { - "name": "sn3", - "url": "https://github.com/myshell-ai/MyShell-TTS-Subnet/", - }, - {"name": "sn4", "url": "https://github.com/manifold-inc/targon/"}, - {"name": "sn5", "url": "https://github.com/OpenKaito/openkaito/"}, - {"name": "sn6", "url": "https://github.com/amedeo-gigaver/infinite_games/"}, - {"name": "sn7", "url": "https://github.com/eclipsevortex/SubVortex/"}, - { - "name": "sn8", - "url": "https://github.com/taoshidev/proprietary-trading-network/", - }, - {"name": "sn9", "url": "https://github.com/unconst/pretrain-subnet/"}, - { - "name": "sn10", - "url": "https://github.com/Sturdy-Subnet/sturdy-subnet/", - }, - { - "name": "sn11", - "url": "https://github.com/impel-intelligence/dippy-bittensor-subnet/", - }, - {"name": "sn12", "url": "https://github.com/backend-developers-ltd/ComputeHorde/"}, - {"name": "sn13", "url": "https://github.com/macrocosm-os/data-universe/"}, - { - "name": "sn14", - "url": "https://github.com/synapsec-ai/llm-defender-subnet/", - }, - { - "name": "sn15", - "url": "https://github.com/blockchain-insights/blockchain-data-subnet/", - }, - {"name": "sn16", "url": "https://github.com/eseckft/BitAds.ai/"}, - {"name": "sn17", "url": "https://github.com/404-Repo/three-gen-subnet/"}, - {"name": "sn18", "url": "https://github.com/corcel-api/cortex.t/"}, - {"name": "sn19", "url": "https://github.com/namoray/vision/"}, - {"name": "sn20", "url": "https://github.com/RogueTensor/bitagent_subnet/"}, - { - "name": "sn21", - "url": "https://github.com/omegalabsinc/omegalabs-anytoany-bittensor", - }, - {"name": "sn22", "url": "https://github.com/Datura-ai/smart-scrape/"}, - {"name": "sn23", "url": "https://github.com/SocialTensor/SocialTensorSubnet/"}, - { - "name": "sn24", - "url": "https://github.com/omegalabsinc/omegalabs-bittensor-subnet/", - }, - {"name": "sn25", "url": "https://github.com/macrocosm-os/folding/"}, - { - "name": "sn26", - "url": "https://github.com/TensorAlchemy/TensorAlchemy/", - }, - { - "name": "sn27", - "url": "https://github.com/neuralinternet/compute-subnet/", - }, - {"name": "sn28", "url": "https://github.com/foundryservices/snpOracle/"}, - {"name": "sn29", "url": "https://github.com/fractal-net/fractal/"}, - {"name": "sn30", "url": "https://github.com/Bettensor/bettensor/"}, - { - "name": "sn31", - "url": "https://github.com/nimaaghli/NASChain/", - }, - {"name": "sn32", "url": "https://github.com/It-s-AI/llm-detection/"}, - { - "name": "sn33", - "url": "https://github.com/afterpartyai/bittensor-conversation-genome-project/", - }, - {"name": "sn34", "url": "https://github.com/Healthi-Labs/healthi-subnet/"}, - {"name": "sn35", "url": "https://github.com/LogicNet-Subnet/LogicNet-prod/"}, - {"name": "sn36", "url": "https://github.com/HIP-Labs/HIP-Subnet/"}, - {"name": "sn37", "url": "https://github.com/macrocosm-os/finetuning/"}, -] diff --git a/tests/test_template_validator.py b/tests/test_template_validator.py index 48e015a9..19b06b74 100644 --- a/tests/test_template_validator.py +++ b/tests/test_template_validator.py @@ -23,10 +23,10 @@ import torch from neurons.validator import Validator -from template.base.validator import BaseValidatorNeuron -from template.protocol import Dummy -from template.utils.uids import get_random_uids -from template.validator.reward import get_rewards +from cancer_ai.base.validator import BaseValidatorNeuron +from cancer_ai.protocol import Dummy +from cancer_ai.utils.uids import get_random_uids +from cancer_ai.validator.reward import get_rewards class TemplateValidatorNeuronTestCase(unittest.TestCase): diff --git a/verify/generate.py b/verify/generate.py deleted file mode 100644 index ad860359..00000000 --- a/verify/generate.py +++ /dev/null @@ -1,35 +0,0 @@ -from substrateinterface import Keypair -from os import getenv, environ -from datetime import datetime -import bittensor - -# Hardcode or set the environment variable WALLET_PASS to the password for the wallet -# environ["WALLET_PASS"] = "" - - -def main(args): - wallet = bittensor.wallet(name=args.name) - keypair = wallet.coldkey - - timestamp = datetime.now() - timezone = timestamp.astimezone().tzname() - - message = f"On {timestamp} {timezone} {args.message}" - signature = keypair.sign(data=message) - - file_contents = f"{message}\n\tSigned by: {keypair.ss58_address}\n\tSignature: {signature.hex()}" - print(file_contents) - open("message_and_signature.txt", "w").write(file_contents) - - print(f"Signature generated and saved to message_and_signature.txt") - - -if __name__ == "__main__": - import argparse - - parser = argparse.ArgumentParser(description="Generate a signature") - parser.add_argument("--message", help="The message to sign", type=str) - parser.add_argument("--name", help="The wallet name", type=str) - args = parser.parse_args() - - main(args) diff --git a/verify/verify.py b/verify/verify.py deleted file mode 100644 index 36ea50f7..00000000 --- a/verify/verify.py +++ /dev/null @@ -1,41 +0,0 @@ -from substrateinterface import Keypair -from binascii import unhexlify - - -def main(args): - file_data = open(args.file).read() - file_split = file_data.split("\n\t") - - address_line = file_split[1] - address_prefix = "Signed by: " - if address_line.startswith(address_prefix): - address = address_line[len(address_prefix) :] - else: - address = address_line - - keypair = Keypair(ss58_address=address, ss58_format=42) - - message = file_split[0] - - signature_line = file_split[2] - signature_prefix = "Signature: " - if signature_line.startswith(signature_prefix): - signature = signature_line[len(signature_prefix) :] - else: - signature = signature_line - - real_signature = unhexlify(signature.encode()) - - if not keypair.verify(data=message, signature=real_signature): - raise ValueError(f"Invalid signature for address={address}") - else: - print(f"Signature verified, signed by {address}") - - -if __name__ == "__main__": - import argparse - - parser = argparse.ArgumentParser(description="Verify a signature") - parser.add_argument("--file", help="The file containing the message and signature") - args = parser.parse_args() - main(args) From 7258eeeb693a93546accea76d3f83bfa3ebf28f2 Mon Sep 17 00:00:00 2001 From: Wojtek Jurkowlaniec Date: Thu, 8 Aug 2024 22:12:43 +0200 Subject: [PATCH 002/227] logic --- logic.py | 69 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100644 logic.py diff --git a/logic.py b/logic.py new file mode 100644 index 00000000..b62733f2 --- /dev/null +++ b/logic.py @@ -0,0 +1,69 @@ +from datetime import datetime +from time import sleep + + +import schedule + +database = {} # validator database +dataset_path = "/dataset" + +models_db = {} + +class Validator: + + def __init__(self, config=None, validator_hotkey): + self.config = config + self.dataset_refresh_date = datetime.now() + self.validator_hotkey = validator_hotkey + + async def forward(self): + pass + + async def reward(self): + pass + + def receive_model(self, miner, model_url): + """Get axon from miner""" + return self.save_model(miner, model_url) + + def save_model(self, miner, model_url): + """Save information about miner's models to database for later """ + database[miner.hotkey] = { + "model_url":model_url, + "timestamp": datetime.now() + } + + def download_new_dataset(dataset_url): + while True: + if new_dataset_available: + download_dataset() + self.dataset_refresh_date = datetime.now() + else: + # if it's not available yet, wait for 60 seconds + sleep(60) + + def download_save_model(self, hotkey, url): + models_db[hotkey] = download_model(url["model_url"]) + + def test_model(self, model) -> score: + return get_model_accuracy(model) + + def test_models(self): + for hotkey in models_db: + self.download_save_model(hotkey, models_db[hotkey]) + + for hotkey in models_db: + result = self.test_model() + save_result_to_wandb(self.validator_hotkey,hotkey, result) + + + + +validator = Validator() + +schedule.every.day.at("18:00").do(validator.download_new_dataset) + + + + + From 6628c4cf37ad9584b82aa3abde05dbec9f9fa139 Mon Sep 17 00:00:00 2001 From: Wojtek Jurkowlaniec Date: Thu, 8 Aug 2024 22:15:02 +0200 Subject: [PATCH 003/227] fix --- logic.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/logic.py b/logic.py index b62733f2..11c45ddf 100644 --- a/logic.py +++ b/logic.py @@ -15,6 +15,7 @@ def __init__(self, config=None, validator_hotkey): self.config = config self.dataset_refresh_date = datetime.now() self.validator_hotkey = validator_hotkey + async def forward(self): pass @@ -48,7 +49,9 @@ def download_save_model(self, hotkey, url): def test_model(self, model) -> score: return get_model_accuracy(model) - def test_models(self): + def evaluate_models(self): + self.download_new_dataset() + for hotkey in models_db: self.download_save_model(hotkey, models_db[hotkey]) @@ -61,7 +64,7 @@ def test_models(self): validator = Validator() -schedule.every.day.at("18:00").do(validator.download_new_dataset) +schedule.every.day.at("18:00").do(validator.evaluate_models) From 47f7215dc1a21c1c9412e2bb25d2f89e87e88d6a Mon Sep 17 00:00:00 2001 From: Wojtek Jurkowlaniec Date: Thu, 8 Aug 2024 22:58:30 +0200 Subject: [PATCH 004/227] dataset manager --- cancer_ai/validator/dataset_manager.py | 52 ++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 cancer_ai/validator/dataset_manager.py diff --git a/cancer_ai/validator/dataset_manager.py b/cancer_ai/validator/dataset_manager.py new file mode 100644 index 00000000..b4d20978 --- /dev/null +++ b/cancer_ai/validator/dataset_manager.py @@ -0,0 +1,52 @@ +from datetime import datetime +from time import sleep +from huggingface_hub import HfApi + +import schedule + +database = {} # validator database +dataset_path = "/dataset" +model_basepath = "./models" +models_db = {} + +class DatasetManager: + def __init__(self, config) -> None: + self.config = config + self.api = HfApi() + self.hotkey_store = {} + + def get_model_state(self): + return self.hotkey_store + + def load_model_state(self, hotkey_models: dict): + self.hotkey_store = hotkey_models + + def sync_hotkeys(self, hotkeys: list): + for hotkey in hotkeys: + if hotkey not in self.hotkey_store: + self.delete_model(hotkey) + + def download_miner_model(self, hotkey) -> str: + """Downloads newest model from Hugging Face and save to disk + Returns: + str: path to downloaded model + """ + return self.api.hf_hub_download(self.hotkey_store[hotkey]["repo_id"], self.hotkey_store[hotkey]["filename"], cache_dir=model_basepath,repo_type="space") + + def add_model(self,hotkey, repo_id, filename) -> None: + """Saves locally information about new model + """ + self.hotkey_store[hotkey] = { + "repo_id": repo_id, + "filename": filename + } + + def delete_model(self, hotkey): + print("Deleting model: ", hotkey) + del self.hotkey_store[hotkey] + + +if __name__ == "__main__": + dataset_manager = DatasetManager({}) + dataset_manager.add_model("wojtek", "vidhiparikh/House-Price-Estimator", "model_custom.pkcls") + print(dataset_manager.download_miner_model("wojtek")) From 96452251fae75afd78b64fe5b1f0a2ea4e9a4f12 Mon Sep 17 00:00:00 2001 From: Wojtek Jurkowlaniec Date: Fri, 9 Aug 2024 03:26:21 +0200 Subject: [PATCH 005/227] model manager with tests --- cancer_ai/validator/model_manager.py | 53 +++++++++++++++++++++++ cancer_ai/validator/model_manager_test.py | 47 ++++++++++++++++++++ 2 files changed, 100 insertions(+) create mode 100644 cancer_ai/validator/model_manager.py create mode 100644 cancer_ai/validator/model_manager_test.py diff --git a/cancer_ai/validator/model_manager.py b/cancer_ai/validator/model_manager.py new file mode 100644 index 00000000..58094d8c --- /dev/null +++ b/cancer_ai/validator/model_manager.py @@ -0,0 +1,53 @@ +from datetime import datetime +from time import sleep +from huggingface_hub import HfApi + +import schedule + +database = {} # validator database +dataset_path = "/dataset" +model_basepath = "./models" +models_db = {} + +class ModelManager: + def __init__(self, config) -> None: + self.config = config + self.api = HfApi() + self.hotkey_store = {} + + def get_model_state(self): + return self.hotkey_store + + def initialize_model_state(self, hotkey_models: dict): + self.hotkey_store = hotkey_models + + def sync_hotkeys(self, hotkeys: list): + hotkey_copy = list(self.hotkey_store.keys()) + for hotkey in hotkey_copy: + if hotkey not in hotkeys: + self.delete_model(hotkey) + + def download_miner_model(self, hotkey) -> str: + """Downloads newest model from Hugging Face and save to disk + Returns: + str: path to downloaded model + """ + return self.api.hf_hub_download(self.hotkey_store[hotkey]["repo_id"], self.hotkey_store[hotkey]["filename"], cache_dir=model_basepath,repo_type="space") + + def add_model(self,hotkey, repo_id, filename) -> None: + """Saves locally information about new model + """ + self.hotkey_store[hotkey] = { + "repo_id": repo_id, + "filename": filename + } + + def delete_model(self, hotkey): + print("Deleting model: ", hotkey) + del self.hotkey_store[hotkey] + + +if __name__ == "__main__": + model_manager = ModelManager({}) + model_manager.add_model("wojtek", "vidhiparikh/House-Price-Estimator", "model_custom.pkcls") + print(model_manager.download_miner_model("wojtek")) diff --git a/cancer_ai/validator/model_manager_test.py b/cancer_ai/validator/model_manager_test.py new file mode 100644 index 00000000..13d75a17 --- /dev/null +++ b/cancer_ai/validator/model_manager_test.py @@ -0,0 +1,47 @@ +import pytest +from unittest.mock import patch, MagicMock +from .model_manager import ModelManager # Replace with the actual module name + +hotkey = "test_hotkey" +repo_id = "test_repo_id" +filename = "test_filename" + + +@pytest.fixture +def model_manager(): + config = {} + return ModelManager(config) + + +def test_add_model(model_manager: ModelManager): + model_manager.add_model(hotkey, repo_id, filename) + + assert hotkey in model_manager.get_model_state() + assert model_manager.get_model_state()[hotkey]["repo_id"] == repo_id + assert model_manager.get_model_state()[hotkey]["filename"] == filename + + +def test_delete_model(model_manager: ModelManager): + model_manager.add_model(hotkey, repo_id, filename) + model_manager.delete_model(hotkey) + + assert hotkey not in model_manager.get_model_state() + + +def test_sync_hotkeys(model_manager: ModelManager): + model_manager.add_model(hotkey, repo_id, filename) + + model_manager.sync_hotkeys([]) + + assert hotkey not in model_manager.get_model_state() + + +def test_load_save_model_state(model_manager: ModelManager): + hotkey_models = { + "test_hotkey_1": {"repo_id": "repo_1", "filename": "file_1"}, + "test_hotkey_2": {"repo_id": "repo_2", "filename": "file_2"}, + } + + model_manager.initialize_model_state(hotkey_models) + + assert model_manager.get_model_state() == hotkey_models From f0947218adb42f922a7e5d2b8a50a6e4a81a27ec Mon Sep 17 00:00:00 2001 From: Wojtek Jurkowlaniec Date: Fri, 9 Aug 2024 04:30:23 +0200 Subject: [PATCH 006/227] fixed test, stub competition and dataset manager --- cancer_ai/__init__.py | 4 +- cancer_ai/validator/competition_manager.py | 7 ++ cancer_ai/validator/dataset_manager.py | 53 ++------------- cancer_ai/validator/manager.py | 11 ++++ cancer_ai/validator/model_manager.py | 77 ++++++++++++---------- cancer_ai/validator/model_manager_test.py | 48 ++++++++++---- 6 files changed, 104 insertions(+), 96 deletions(-) create mode 100644 cancer_ai/validator/competition_manager.py create mode 100644 cancer_ai/validator/manager.py diff --git a/cancer_ai/__init__.py b/cancer_ai/__init__.py index cb07b8c0..f914233f 100644 --- a/cancer_ai/__init__.py +++ b/cancer_ai/__init__.py @@ -31,5 +31,5 @@ from . import protocol from . import base from . import validator -from . import api -from .subnet_links import SUBNET_LINKS +# from . import api +# from .subnet_links import SUBNET_LINKS diff --git a/cancer_ai/validator/competition_manager.py b/cancer_ai/validator/competition_manager.py new file mode 100644 index 00000000..28bded89 --- /dev/null +++ b/cancer_ai/validator/competition_manager.py @@ -0,0 +1,7 @@ +from .manager import SerializableManager + + +class CompetitionManager(SerializableManager): + def __init__(self, config, competition_id: str) -> None: + self.config = config + self.competition_id = competition_id diff --git a/cancer_ai/validator/dataset_manager.py b/cancer_ai/validator/dataset_manager.py index b4d20978..e00be209 100644 --- a/cancer_ai/validator/dataset_manager.py +++ b/cancer_ai/validator/dataset_manager.py @@ -1,52 +1,7 @@ -from datetime import datetime -from time import sleep -from huggingface_hub import HfApi +from .manager import SerializableManager -import schedule -database = {} # validator database -dataset_path = "/dataset" -model_basepath = "./models" -models_db = {} - -class DatasetManager: - def __init__(self, config) -> None: +class DatasetManager(SerializableManager): + def __init__(self, config, competition_id: str) -> None: self.config = config - self.api = HfApi() - self.hotkey_store = {} - - def get_model_state(self): - return self.hotkey_store - - def load_model_state(self, hotkey_models: dict): - self.hotkey_store = hotkey_models - - def sync_hotkeys(self, hotkeys: list): - for hotkey in hotkeys: - if hotkey not in self.hotkey_store: - self.delete_model(hotkey) - - def download_miner_model(self, hotkey) -> str: - """Downloads newest model from Hugging Face and save to disk - Returns: - str: path to downloaded model - """ - return self.api.hf_hub_download(self.hotkey_store[hotkey]["repo_id"], self.hotkey_store[hotkey]["filename"], cache_dir=model_basepath,repo_type="space") - - def add_model(self,hotkey, repo_id, filename) -> None: - """Saves locally information about new model - """ - self.hotkey_store[hotkey] = { - "repo_id": repo_id, - "filename": filename - } - - def delete_model(self, hotkey): - print("Deleting model: ", hotkey) - del self.hotkey_store[hotkey] - - -if __name__ == "__main__": - dataset_manager = DatasetManager({}) - dataset_manager.add_model("wojtek", "vidhiparikh/House-Price-Estimator", "model_custom.pkcls") - print(dataset_manager.download_miner_model("wojtek")) + self.competition_id = competition_id diff --git a/cancer_ai/validator/manager.py b/cancer_ai/validator/manager.py new file mode 100644 index 00000000..118788dc --- /dev/null +++ b/cancer_ai/validator/manager.py @@ -0,0 +1,11 @@ + + +class SerializableManager: + def __init__(self, config) -> None: + self.config = config + + def get_state(self) -> dict: + raise NotImplementedError + + def set_state(self, state: dict): + raise NotImplementedError \ No newline at end of file diff --git a/cancer_ai/validator/model_manager.py b/cancer_ai/validator/model_manager.py index 58094d8c..50f970a3 100644 --- a/cancer_ai/validator/model_manager.py +++ b/cancer_ai/validator/model_manager.py @@ -1,25 +1,35 @@ +from dataclasses import dataclass, asdict, is_dataclass from datetime import datetime from time import sleep +import os from huggingface_hub import HfApi -import schedule +from .manager import SerializableManager -database = {} # validator database -dataset_path = "/dataset" -model_basepath = "./models" -models_db = {} -class ModelManager: +@dataclass +class ModelInfo: + repo_id: str + filename: str + file_path: str | None = None + + +class ModelManager(SerializableManager): def __init__(self, config) -> None: self.config = config + if "model_dir" not in self.config: + self.config["model_dir"] = "./models" + # create model_dir if it doesn't exist + if not os.path.exists(self.config["model_dir"]): + os.makedirs(self.config["model_dir"]) self.api = HfApi() - self.hotkey_store = {} - - def get_model_state(self): - return self.hotkey_store - - def initialize_model_state(self, hotkey_models: dict): - self.hotkey_store = hotkey_models + self.hotkey_store = {} # Now a dictionary mapping hotkeys to ModelInfo objects + + def get_state(self): + return {k: asdict(v) for k, v in self.hotkey_store.items() if is_dataclass(v)} + + def set_state(self, hotkey_models: dict): + self.hotkey_store = {k: ModelInfo(**v) for k, v in hotkey_models.items()} def sync_hotkeys(self, hotkeys: list): hotkey_copy = list(self.hotkey_store.keys()) @@ -27,27 +37,28 @@ def sync_hotkeys(self, hotkeys: list): if hotkey not in hotkeys: self.delete_model(hotkey) - def download_miner_model(self, hotkey) -> str: - """Downloads newest model from Hugging Face and save to disk + def download_miner_model(self, hotkey) -> None: + """Downloads the newest model from Hugging Face and saves it to disk. Returns: - str: path to downloaded model - """ - return self.api.hf_hub_download(self.hotkey_store[hotkey]["repo_id"], self.hotkey_store[hotkey]["filename"], cache_dir=model_basepath,repo_type="space") - - def add_model(self,hotkey, repo_id, filename) -> None: - """Saves locally information about new model + str: path to the downloaded model """ - self.hotkey_store[hotkey] = { - "repo_id": repo_id, - "filename": filename - } - - def delete_model(self, hotkey): - print("Deleting model: ", hotkey) - del self.hotkey_store[hotkey] + model_info = self.hotkey_store[hotkey] + model_path = self.api.hf_hub_download( + model_info.repo_id, + model_info.filename, + cache_dir=self.config["model_dir"], + repo_type="space", + ) + model_info.file_path = model_path + def add_model(self, hotkey, repo_id, filename) -> None: + """Saves locally information about a new model.""" + self.hotkey_store[hotkey] = ModelInfo(repo_id, filename) -if __name__ == "__main__": - model_manager = ModelManager({}) - model_manager.add_model("wojtek", "vidhiparikh/House-Price-Estimator", "model_custom.pkcls") - print(model_manager.download_miner_model("wojtek")) + def delete_model(self, hotkey): + """Deletes locally information about a model and the corresponding file on disk.""" + + print("Deleting model: ", hotkey) + if hotkey in self.hotkey_store and self.hotkey_store[hotkey].file_path: + os.remove(self.hotkey_store[hotkey].file_path) + self.hotkey_store[hotkey] = None diff --git a/cancer_ai/validator/model_manager_test.py b/cancer_ai/validator/model_manager_test.py index 13d75a17..3d63bfc6 100644 --- a/cancer_ai/validator/model_manager_test.py +++ b/cancer_ai/validator/model_manager_test.py @@ -1,6 +1,10 @@ import pytest from unittest.mock import patch, MagicMock -from .model_manager import ModelManager # Replace with the actual module name +from .model_manager import ( + ModelManager, + ModelInfo, +) # Replace with the actual module name +import os hotkey = "test_hotkey" repo_id = "test_repo_id" @@ -9,39 +13,59 @@ @pytest.fixture def model_manager(): - config = {} + config = { + "model_dir": "/tmp/models", + } return ModelManager(config) def test_add_model(model_manager: ModelManager): model_manager.add_model(hotkey, repo_id, filename) - assert hotkey in model_manager.get_model_state() - assert model_manager.get_model_state()[hotkey]["repo_id"] == repo_id - assert model_manager.get_model_state()[hotkey]["filename"] == filename + assert hotkey in model_manager.get_state() + assert model_manager.get_state()[hotkey]["repo_id"] == repo_id + assert model_manager.get_state()[hotkey]["filename"] == filename def test_delete_model(model_manager: ModelManager): model_manager.add_model(hotkey, repo_id, filename) model_manager.delete_model(hotkey) - assert hotkey not in model_manager.get_model_state() + assert hotkey not in model_manager.get_state() def test_sync_hotkeys(model_manager: ModelManager): model_manager.add_model(hotkey, repo_id, filename) - model_manager.sync_hotkeys([]) - assert hotkey not in model_manager.get_model_state() + assert hotkey not in model_manager.get_state() def test_load_save_model_state(model_manager: ModelManager): + # Create an instance of ModelManager + + # Create a dictionary of hotkey models hotkey_models = { - "test_hotkey_1": {"repo_id": "repo_1", "filename": "file_1"}, - "test_hotkey_2": {"repo_id": "repo_2", "filename": "file_2"}, + "test_hotkey_1": {"repo_id": "repo_1", "filename": "file_1", "file_path": None}, + "test_hotkey_2": {"repo_id": "repo_2", "filename": "file_2", "file_path": None}, } - model_manager.initialize_model_state(hotkey_models) + model_manager.set_state(hotkey_models) + + assert model_manager.get_state() == hotkey_models + + +def not_test_real_downloading(model_manager: ModelManager): + model_manager.add_model( + "example", "vidhiparikh/House-Price-Estimator", "model_custom.pkcls" + ) + model_manager.download_miner_model("example") + model_path = model_manager.hotkey_store["example"].file_path + + assert os.path.exists(model_path) + + # delete the file + model_manager.delete_model("example") - assert model_manager.get_model_state() == hotkey_models + # assert the file is deleted + assert not os.path.exists(model_path) From 72a65e9927f3d24b4bf54bd6f317afe006541b80 Mon Sep 17 00:00:00 2001 From: Wojtek Jurkowlaniec Date: Tue, 13 Aug 2024 12:13:30 +0200 Subject: [PATCH 007/227] fixes from PR --- cancer_ai/__init__.py | 2 - cancer_ai/utils/config.py | 8 +++ cancer_ai/validator/manager.py | 13 ++-- cancer_ai/validator/model_manager.py | 12 ++-- cancer_ai/validator/model_manager_test.py | 7 ++- logic.py | 72 ----------------------- 6 files changed, 25 insertions(+), 89 deletions(-) delete mode 100644 logic.py diff --git a/cancer_ai/__init__.py b/cancer_ai/__init__.py index f914233f..4854a3f1 100644 --- a/cancer_ai/__init__.py +++ b/cancer_ai/__init__.py @@ -31,5 +31,3 @@ from . import protocol from . import base from . import validator -# from . import api -# from .subnet_links import SUBNET_LINKS diff --git a/cancer_ai/utils/config.py b/cancer_ai/utils/config.py index 99c610e9..28f978ca 100644 --- a/cancer_ai/utils/config.py +++ b/cancer_ai/utils/config.py @@ -22,6 +22,7 @@ import bittensor as bt from .logging import setup_events_logger + def is_cuda_available(): try: output = subprocess.check_output(["nvidia-smi", "-L"], stderr=subprocess.STDOUT) @@ -37,6 +38,7 @@ def is_cuda_available(): pass return "cpu" + def check_config(cls, config: "bt.Config"): r"""Checks/validates the config namespace object.""" bt.logging.check_config(config) @@ -241,6 +243,12 @@ def add_validator_args(cls, parser): help="The name of the project where you are sending the new run.", default="opentensor-dev", ) + parser.add_argument( + "--models.model_dir", + type=str, + help="Path for storing hugging face models.", + default="./models", + ) def config(cls): diff --git a/cancer_ai/validator/manager.py b/cancer_ai/validator/manager.py index 118788dc..e8d44c22 100644 --- a/cancer_ai/validator/manager.py +++ b/cancer_ai/validator/manager.py @@ -1,11 +1,12 @@ +from abc import ABC, abstractmethod -class SerializableManager: - def __init__(self, config) -> None: - self.config = config +class SerializableManager(ABC): + @abstractmethod def get_state(self) -> dict: - raise NotImplementedError - + pass + + @abstractmethod def set_state(self, state: dict): - raise NotImplementedError \ No newline at end of file + pass diff --git a/cancer_ai/validator/model_manager.py b/cancer_ai/validator/model_manager.py index 50f970a3..c72ccff7 100644 --- a/cancer_ai/validator/model_manager.py +++ b/cancer_ai/validator/model_manager.py @@ -17,13 +17,11 @@ class ModelInfo: class ModelManager(SerializableManager): def __init__(self, config) -> None: self.config = config - if "model_dir" not in self.config: - self.config["model_dir"] = "./models" - # create model_dir if it doesn't exist - if not os.path.exists(self.config["model_dir"]): - os.makedirs(self.config["model_dir"]) + + if not os.path.exists(self.config["models"]["model_dir"]): + os.makedirs(self.config["models"]["model_dir"]) self.api = HfApi() - self.hotkey_store = {} # Now a dictionary mapping hotkeys to ModelInfo objects + self.hotkey_store = {} def get_state(self): return {k: asdict(v) for k, v in self.hotkey_store.items() if is_dataclass(v)} @@ -46,7 +44,7 @@ def download_miner_model(self, hotkey) -> None: model_path = self.api.hf_hub_download( model_info.repo_id, model_info.filename, - cache_dir=self.config["model_dir"], + cache_dir=self.config["models"]["model_dir"], repo_type="space", ) model_info.file_path = model_path diff --git a/cancer_ai/validator/model_manager_test.py b/cancer_ai/validator/model_manager_test.py index 3d63bfc6..5869a9e1 100644 --- a/cancer_ai/validator/model_manager_test.py +++ b/cancer_ai/validator/model_manager_test.py @@ -14,7 +14,7 @@ @pytest.fixture def model_manager(): config = { - "model_dir": "/tmp/models", + "models": {"model_dir": "/tmp/models"}, } return ModelManager(config) @@ -55,7 +55,10 @@ def test_load_save_model_state(model_manager: ModelManager): assert model_manager.get_state() == hotkey_models -def not_test_real_downloading(model_manager: ModelManager): +@pytest.mark.skip( + reason="we don't want to test every time with downloading data from huggingface" +) +def test_real_downloading(model_manager: ModelManager): model_manager.add_model( "example", "vidhiparikh/House-Price-Estimator", "model_custom.pkcls" ) diff --git a/logic.py b/logic.py deleted file mode 100644 index 11c45ddf..00000000 --- a/logic.py +++ /dev/null @@ -1,72 +0,0 @@ -from datetime import datetime -from time import sleep - - -import schedule - -database = {} # validator database -dataset_path = "/dataset" - -models_db = {} - -class Validator: - - def __init__(self, config=None, validator_hotkey): - self.config = config - self.dataset_refresh_date = datetime.now() - self.validator_hotkey = validator_hotkey - - - async def forward(self): - pass - - async def reward(self): - pass - - def receive_model(self, miner, model_url): - """Get axon from miner""" - return self.save_model(miner, model_url) - - def save_model(self, miner, model_url): - """Save information about miner's models to database for later """ - database[miner.hotkey] = { - "model_url":model_url, - "timestamp": datetime.now() - } - - def download_new_dataset(dataset_url): - while True: - if new_dataset_available: - download_dataset() - self.dataset_refresh_date = datetime.now() - else: - # if it's not available yet, wait for 60 seconds - sleep(60) - - def download_save_model(self, hotkey, url): - models_db[hotkey] = download_model(url["model_url"]) - - def test_model(self, model) -> score: - return get_model_accuracy(model) - - def evaluate_models(self): - self.download_new_dataset() - - for hotkey in models_db: - self.download_save_model(hotkey, models_db[hotkey]) - - for hotkey in models_db: - result = self.test_model() - save_result_to_wandb(self.validator_hotkey,hotkey, result) - - - - -validator = Validator() - -schedule.every.day.at("18:00").do(validator.evaluate_models) - - - - - From f39a5f375981f534e69132edac28bb81cd7c555b Mon Sep 17 00:00:00 2001 From: Wojtek Jurkowlaniec Date: Tue, 13 Aug 2024 13:34:12 +0200 Subject: [PATCH 008/227] fix using config --- cancer_ai/validator/model_manager.py | 8 ++++---- cancer_ai/validator/model_manager_test.py | 7 +++++-- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/cancer_ai/validator/model_manager.py b/cancer_ai/validator/model_manager.py index c72ccff7..ca249593 100644 --- a/cancer_ai/validator/model_manager.py +++ b/cancer_ai/validator/model_manager.py @@ -17,9 +17,9 @@ class ModelInfo: class ModelManager(SerializableManager): def __init__(self, config) -> None: self.config = config - - if not os.path.exists(self.config["models"]["model_dir"]): - os.makedirs(self.config["models"]["model_dir"]) + + if not os.path.exists(self.config.models.model_dir): + os.makedirs(self.config.models.model_dir) self.api = HfApi() self.hotkey_store = {} @@ -44,7 +44,7 @@ def download_miner_model(self, hotkey) -> None: model_path = self.api.hf_hub_download( model_info.repo_id, model_info.filename, - cache_dir=self.config["models"]["model_dir"], + cache_dir=self.config.models.model_dir, repo_type="space", ) model_info.file_path = model_path diff --git a/cancer_ai/validator/model_manager_test.py b/cancer_ai/validator/model_manager_test.py index 5869a9e1..04c150d4 100644 --- a/cancer_ai/validator/model_manager_test.py +++ b/cancer_ai/validator/model_manager_test.py @@ -1,4 +1,5 @@ import pytest +from types import SimpleNamespace from unittest.mock import patch, MagicMock from .model_manager import ( ModelManager, @@ -14,9 +15,11 @@ @pytest.fixture def model_manager(): config = { - "models": {"model_dir": "/tmp/models"}, + "models": SimpleNamespace(**{"model_dir": "/tmp/models"}), } - return ModelManager(config) + config_obj = SimpleNamespace(**config) + print(config_obj.models.model_dir) + return ModelManager(config=config_obj) def test_add_model(model_manager: ModelManager): From cd0114dc186a671b0df69abd1c1cb801ac4a7b76 Mon Sep 17 00:00:00 2001 From: Wojtek Jurkowlaniec Date: Tue, 13 Aug 2024 14:06:45 +0200 Subject: [PATCH 009/227] typing --- cancer_ai/validator/model_manager.py | 4 ++-- cancer_ai/validator/model_manager_test.py | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/cancer_ai/validator/model_manager.py b/cancer_ai/validator/model_manager.py index ca249593..e8d1c2c4 100644 --- a/cancer_ai/validator/model_manager.py +++ b/cancer_ai/validator/model_manager.py @@ -17,7 +17,7 @@ class ModelInfo: class ModelManager(SerializableManager): def __init__(self, config) -> None: self.config = config - + if not os.path.exists(self.config.models.model_dir): os.makedirs(self.config.models.model_dir) self.api = HfApi() @@ -53,7 +53,7 @@ def add_model(self, hotkey, repo_id, filename) -> None: """Saves locally information about a new model.""" self.hotkey_store[hotkey] = ModelInfo(repo_id, filename) - def delete_model(self, hotkey): + def delete_model(self, hotkey) -> None: """Deletes locally information about a model and the corresponding file on disk.""" print("Deleting model: ", hotkey) diff --git a/cancer_ai/validator/model_manager_test.py b/cancer_ai/validator/model_manager_test.py index 04c150d4..efdd8509 100644 --- a/cancer_ai/validator/model_manager_test.py +++ b/cancer_ai/validator/model_manager_test.py @@ -13,7 +13,7 @@ @pytest.fixture -def model_manager(): +def model_manager() -> ModelManager: config = { "models": SimpleNamespace(**{"model_dir": "/tmp/models"}), } @@ -22,7 +22,7 @@ def model_manager(): return ModelManager(config=config_obj) -def test_add_model(model_manager: ModelManager): +def test_add_model(model_manager: ModelManager) -> None: model_manager.add_model(hotkey, repo_id, filename) assert hotkey in model_manager.get_state() @@ -30,7 +30,7 @@ def test_add_model(model_manager: ModelManager): assert model_manager.get_state()[hotkey]["filename"] == filename -def test_delete_model(model_manager: ModelManager): +def test_delete_model(model_manager: ModelManager) -> None: model_manager.add_model(hotkey, repo_id, filename) model_manager.delete_model(hotkey) @@ -44,7 +44,7 @@ def test_sync_hotkeys(model_manager: ModelManager): assert hotkey not in model_manager.get_state() -def test_load_save_model_state(model_manager: ModelManager): +def test_load_save_model_state(model_manager: ModelManager) -> None: # Create an instance of ModelManager # Create a dictionary of hotkey models @@ -61,7 +61,7 @@ def test_load_save_model_state(model_manager: ModelManager): @pytest.mark.skip( reason="we don't want to test every time with downloading data from huggingface" ) -def test_real_downloading(model_manager: ModelManager): +def test_real_downloading(model_manager: ModelManager) -> None: model_manager.add_model( "example", "vidhiparikh/House-Price-Estimator", "model_custom.pkcls" ) From 42277aca68420503b7b3e0cf64cc2d46cc486a0f Mon Sep 17 00:00:00 2001 From: Wojtek Jurkowlaniec Date: Sun, 18 Aug 2024 03:20:52 +0200 Subject: [PATCH 010/227] not tested, competition and dataset manager --- cancer_ai/utils/config.py | 8 ++- cancer_ai/validator/competition_manager.py | 67 ++++++++++++++++++- .../validator/dataset_handlers/__init__.py | 0 .../dataset_handlers/base_handler.py | 62 +++++++++++++++++ .../validator/dataset_handlers/image_csv.py | 45 +++++++++++++ cancer_ai/validator/dataset_manager.py | 67 ++++++++++++++++++- cancer_ai/validator/model_manager.py | 1 + 7 files changed, 247 insertions(+), 3 deletions(-) create mode 100644 cancer_ai/validator/dataset_handlers/__init__.py create mode 100644 cancer_ai/validator/dataset_handlers/base_handler.py create mode 100644 cancer_ai/validator/dataset_handlers/image_csv.py diff --git a/cancer_ai/utils/config.py b/cancer_ai/utils/config.py index 28f978ca..5ebcf47e 100644 --- a/cancer_ai/utils/config.py +++ b/cancer_ai/utils/config.py @@ -246,9 +246,15 @@ def add_validator_args(cls, parser): parser.add_argument( "--models.model_dir", type=str, - help="Path for storing hugging face models.", + help="Path for storing competition participants models .", default="./models", ) + parser.add_argument( + "--models.dataset_dir", + type=str, + help="Path for storing datasets.", + default="./datasets", + ) def config(cls): diff --git a/cancer_ai/validator/competition_manager.py b/cancer_ai/validator/competition_manager.py index 28bded89..ddcb318f 100644 --- a/cancer_ai/validator/competition_manager.py +++ b/cancer_ai/validator/competition_manager.py @@ -1,7 +1,72 @@ from .manager import SerializableManager +from .model_manager import ModelManager, ModelInfo +from .dataset_manager import DatasetManager +from datetime import time class CompetitionManager(SerializableManager): - def __init__(self, config, competition_id: str) -> None: + """ + CompetitionManager is responsible for managing a competition. + + It handles the scoring, model management and synchronization with the chain. + """ + + def __init__( + self, + config, + competition_id: str, + category: str, + evaluation_time: list[time], + dataset_hf_id: str, + file_hf_id: str, + ) -> None: + """ + Initializes a CompetitionManager instance. + + Args: + config (dict): Config dictionary. + competition_id (str): Unique identifier for the competition. + category (str): Category of the competition. + evaluation_time (list[time]): List of times of a day at which the competition will be evaluated. + + Note: Times are in UTC time. + """ self.config = config self.competition_id = competition_id + self.category = category + self.model_manager = ModelManager(config) + self.evaluation_time = evaluation_time + self.dataset_manager: DatasetManager = DatasetManager( + config, competition_id, dataset_hf_id, file_hf_id + ) + + def get_state(self): + return { + "competition_id": self.competition_id, + "model_manager": self.model_manager.get_state(), + "category": self.category, + "evaluation_time": self.evaluation_time, + } + + def set_state(self, state: dict): + self.competition_id = state["competition_id"] + self.model_manager.set_state(state["model_manager"]) + self.category = state["category"] + self.evaluation_time = state["evaluation_time"] + + async def get_miner_model(self, hotkey): + # test data + return ModelInfo("vidhiparikh/House-Price-Estimator", "model_custom.pkcls") + + async def init_evaluation(self): + for hotkey in self.model_manager.hotkey_store: + self.model_manager.hotkey_store[hotkey] = await self.get_miner_model(hotkey) + await self.model_manager.download_miner_model(hotkey) + + # log event + + async def evaluate(self): + self.init_evaluation() + self.dataset + for hotkey in self.model_manager.hotkey_store: + print("Evaluating hotkey: ", hotkey) diff --git a/cancer_ai/validator/dataset_handlers/__init__.py b/cancer_ai/validator/dataset_handlers/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/cancer_ai/validator/dataset_handlers/base_handler.py b/cancer_ai/validator/dataset_handlers/base_handler.py new file mode 100644 index 00000000..cf38c379 --- /dev/null +++ b/cancer_ai/validator/dataset_handlers/base_handler.py @@ -0,0 +1,62 @@ +from typing import List, Tuple +from abc import abstractmethod + + +class BaseDatasetHandler: + """ + Base class for handling different dataset types. + + This class initializes the config and path attributes. + + Args: + config (dict): Configuration dictionary. + path (str): Path to the dataset. + + Attributes: + config (dict): Configuration dictionary. + path (str): Path to the dataset. + + """ + + def __init__(self, config, path) -> None: + """ + Initializes the BaseDatasetHandler object. + + Args: + config (dict): Configuration dictionary. + path (str): Path to the dataset. + + """ + # Initialize the config and path attributes + self.config = config # Configuration dictionary + self.path = path # Path to the dataset + self.entries = [] + + @abstractmethod + async def get_training_data(self) -> Tuple[List, List]: + """ + Abstract method to get the training data. + + This method is responsible for loading the training data and returning it as a tuple of two lists: the first list contains the input data and the second list contains the labels. + + Returns: + Tuple[List, List]: A tuple containing two lists: the first list contains the input data and the second list contains the labels. + """ + + @abstractmethod + async def sync_training_data(self): + """ + Abstract method to synchronize the training data. + + This method is responsible for reading the training data from the dataset and storing it in the self.entries attribute. + """ + + async def process_training_data(self): + """ + Process the training data. + + This method is responsible for preprocessing the training data and returning it as a tuple of two lists: the first list contains the input data and the second list contains the labels. + + Returns: + Tuple[List, List]: A tuple containing two lists: the first list contains the input data and the second list contains the labels. + """ diff --git a/cancer_ai/validator/dataset_handlers/image_csv.py b/cancer_ai/validator/dataset_handlers/image_csv.py new file mode 100644 index 00000000..8418c690 --- /dev/null +++ b/cancer_ai/validator/dataset_handlers/image_csv.py @@ -0,0 +1,45 @@ +from .base_handler import BaseDatasetHandler +from PIL import Image +from typing import List, Tuple +from dataclasses import dataclass +import csv +import aiofiles + + +@dataclass +class ImageEntry: + filepath: str + is_melanoma: bool + + +class DatasetImagesCSV(BaseDatasetHandler): + """ + DatasetImagesCSV is responsible for handling the CSV dataset where directory structure looks as follows: + . + ├── images + │ ├── image_1.jpg + │ ├── image_2.jpg + │ └── ... + ├── labels.csv + """ + + def __init__(self, config, path: str) -> None: + self.config = config + self.path = path + self.metadata_columns = ["filepath", "is_melanoma"] + + async def sync_training_data(self): + self.entries = [] + # go over csv file + async with aiofiles.open(self.path, "r") as f: + # "filepath" "is_melanoma" columns + reader = csv.reader(await f.readlines()) + for row in reader: + self.entries.append(ImageEntry(row[0], row[1])) + + async def get_training_data(self) -> Tuple[List, List]: + await self.sync_training_data() + pred_x = [Image(entry.filepath) for entry in self.entries] + pred_y = [entry.is_melanoma for entry in self.entries] + await self.process_training_data() + return pred_x, pred_y diff --git a/cancer_ai/validator/dataset_manager.py b/cancer_ai/validator/dataset_manager.py index e00be209..cb623af5 100644 --- a/cancer_ai/validator/dataset_manager.py +++ b/cancer_ai/validator/dataset_manager.py @@ -1,7 +1,72 @@ +import os +from zipfile import ZipFile +import shutil +from pathlib import Path from .manager import SerializableManager +from huggingface_hub import HfApi + +from typing import List, Tuple +import aiofiles +import aiozip +from io import BytesIO + +from .dataset_handlers.image_csv import DatasetImagesCSV class DatasetManager(SerializableManager): - def __init__(self, config, competition_id: str) -> None: + def __init__( + self, config, competition_id: str, dataset_hf_id: str, file_hf_id: str + ) -> None: self.config = config self.competition_id = competition_id + self.dataset_hf_id = dataset_hf_id + self.file_hf_id = file_hf_id + self.hf_api = HfApi() + self.path = "" + + def get_state(self) -> dict: + return {} + + def set_state(self, state: dict): + return {} + + def download_dataset(self): + if not os.path.exists( + Path(self.config.models.dataset_dir, self.competition_id) + ): + os.makedirs(Path(self.config.models.dataset_dir, self.competition_id)) + + self.path = self.hf_api.hf_hub_download( + self.dataset_hf_id, + self.file_hf_id, + cache_dir=Path(self.config.models.dataset_dir, self.competition_id), + ) + + def delete_dataset(self): + shutil.rmtree(self.path) + + async def unzip_dataset(self): + async with aiofiles.open(self.path, "rb") as f: + data = await f.read() + async with aiofiles.open(self.path, "wb") as f: + async with aiozip.AIOZipFile(BytesIO(data)) as z: + await z.extractall(path=Path(self.path).parent) + + def set_dataset_handler(self): + if not self.path: + raise Exception("Dataset not downloaded") + # is csv in directory + if os.path.exists(Path(self.path, "labels.csv")): + self.handler = DatasetImagesCSV(self.config, Path(self.path, "labels.csv")) + else: + print("Files in dataset: ", os.listdir(self.path)) + raise NotImplementedError("Dataset handler not implemented") + + async def prepare_dataset(self): + await self.download_dataset() + await self.unzip_dataset() + self.set_dataset_handler() + + async def get_training_data(self) -> Tuple[List, List]: + await self.prepare_dataset() + return await self.handler.get_training_data() diff --git a/cancer_ai/validator/model_manager.py b/cancer_ai/validator/model_manager.py index e8d1c2c4..75b09b89 100644 --- a/cancer_ai/validator/model_manager.py +++ b/cancer_ai/validator/model_manager.py @@ -34,6 +34,7 @@ def sync_hotkeys(self, hotkeys: list): for hotkey in hotkey_copy: if hotkey not in hotkeys: self.delete_model(hotkey) + def download_miner_model(self, hotkey) -> None: """Downloads the newest model from Hugging Face and saves it to disk. From 6bb82cff3b843f9a24ca62d9c05f603163e84d50 Mon Sep 17 00:00:00 2001 From: Wojtek Jurkowlaniec Date: Sun, 18 Aug 2024 04:02:31 +0200 Subject: [PATCH 011/227] still not tested, model runner --- cancer_ai/validator/competition_manager.py | 22 ++++++-- cancer_ai/validator/dataset_manager.py | 10 ++-- cancer_ai/validator/model_manager.py | 2 +- cancer_ai/validator/model_run_manager.py | 51 +++++++++++++++++++ cancer_ai/validator/model_runners/__init__.py | 0 cancer_ai/validator/utils.py | 43 ++++++++++++++++ 6 files changed, 120 insertions(+), 8 deletions(-) create mode 100644 cancer_ai/validator/model_run_manager.py create mode 100644 cancer_ai/validator/model_runners/__init__.py create mode 100644 cancer_ai/validator/utils.py diff --git a/cancer_ai/validator/competition_manager.py b/cancer_ai/validator/competition_manager.py index ddcb318f..ecc8d36f 100644 --- a/cancer_ai/validator/competition_manager.py +++ b/cancer_ai/validator/competition_manager.py @@ -1,8 +1,9 @@ from .manager import SerializableManager from .model_manager import ModelManager, ModelInfo from .dataset_manager import DatasetManager +from .model_run_manager import ModelRunManager from datetime import time - +import random class CompetitionManager(SerializableManager): """ @@ -39,6 +40,7 @@ def __init__( self.dataset_manager: DatasetManager = DatasetManager( config, competition_id, dataset_hf_id, file_hf_id ) + self.results = [] def get_state(self): return { @@ -61,12 +63,24 @@ async def get_miner_model(self, hotkey): async def init_evaluation(self): for hotkey in self.model_manager.hotkey_store: self.model_manager.hotkey_store[hotkey] = await self.get_miner_model(hotkey) - await self.model_manager.download_miner_model(hotkey) - + + self.dataset_manager.prepare_dataset() + # log event async def evaluate(self): self.init_evaluation() - self.dataset + pred_x, pred_y = self.dataset_manager.get_data() for hotkey in self.model_manager.hotkey_store: print("Evaluating hotkey: ", hotkey) + await self.model_manager.download_miner_model(hotkey) + pred_y = ModelRunManager(self.config, self.model_manager.hotkey_store[hotkey]).run(pred_x) + # print "make stats and send to wandb" + score = random.randint(0, 100) + self.results.append((hotkey, score)) + + # sort by score + self.results.sort(key=lambda x: x[1], reverse=True) + return self.results + + diff --git a/cancer_ai/validator/dataset_manager.py b/cancer_ai/validator/dataset_manager.py index cb623af5..e3aba2bf 100644 --- a/cancer_ai/validator/dataset_manager.py +++ b/cancer_ai/validator/dataset_manager.py @@ -23,6 +23,7 @@ def __init__( self.file_hf_id = file_hf_id self.hf_api = HfApi() self.path = "" + self.data = None def get_state(self) -> dict: return {} @@ -67,6 +68,9 @@ async def prepare_dataset(self): await self.unzip_dataset() self.set_dataset_handler() - async def get_training_data(self) -> Tuple[List, List]: - await self.prepare_dataset() - return await self.handler.get_training_data() + + async def get_data(self) -> Tuple[List, List]: + if not self.data: + await self.prepare_dataset() + self.data = await self.handler.get_training_data() + return self.data diff --git a/cancer_ai/validator/model_manager.py b/cancer_ai/validator/model_manager.py index 75b09b89..75421344 100644 --- a/cancer_ai/validator/model_manager.py +++ b/cancer_ai/validator/model_manager.py @@ -12,6 +12,7 @@ class ModelInfo: repo_id: str filename: str file_path: str | None = None + model_type: str | None = None class ModelManager(SerializableManager): @@ -34,7 +35,6 @@ def sync_hotkeys(self, hotkeys: list): for hotkey in hotkey_copy: if hotkey not in hotkeys: self.delete_model(hotkey) - def download_miner_model(self, hotkey) -> None: """Downloads the newest model from Hugging Face and saves it to disk. diff --git a/cancer_ai/validator/model_run_manager.py b/cancer_ai/validator/model_run_manager.py new file mode 100644 index 00000000..b7493afa --- /dev/null +++ b/cancer_ai/validator/model_run_manager.py @@ -0,0 +1,51 @@ +from .manager import SerializableManager +from .model_manager import ModelInfo +from typing import List, Tuple +from abc import abstractmethod +from .utils import detect_model_format, ModelType + + +class BaseRunnerHandler: + def __init__(self, config, model_path: str) -> None: + self.config = config + self.model_path = model_path + + @abstractmethod + def run(self): + """Exceutes the run process of the model in separate process.""" + + +class PytorchRunnerHandler(BaseRunnerHandler): + def run(self, pred_x: List, pred_y: List) -> List: + # example, might not work + from torch import load + + model = load(self.model_path) + model.eval() + output = model(pred_x) + return output + + +runner_handler_mapping = { + ModelType.PYTORCH: PytorchRunnerHandler, +} + + +class ModelRunManager(SerializableManager): + def __init__(self, config, model: ModelInfo) -> None: + self.config = config + self.model = model + + def get_state(self) -> dict: + return {} + + def set_state(self, state: dict): + pass + + def set_runner_handler(self) -> None: + self.handler = runner_handler_mapping[detect_model_format(self.model)]( + self.config, self.model.file_path + ) + + def run(self, pred_x: List) -> List: + return self.handler.run(pred_x) diff --git a/cancer_ai/validator/model_runners/__init__.py b/cancer_ai/validator/model_runners/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/cancer_ai/validator/utils.py b/cancer_ai/validator/utils.py new file mode 100644 index 00000000..2b7d7ae2 --- /dev/null +++ b/cancer_ai/validator/utils.py @@ -0,0 +1,43 @@ +from enum import Enum +import os + + +class ModelType(Enum): + ONNX = "ONNX" + TENSORFLOW_SAVEDMODEL = "TensorFlow SavedModel" + KERAS_H5 = "Keras H5" + PYTORCH = "PyTorch" + SCIKIT_LEARN = "Scikit-learn" + XGBOOST = "XGBoost" + UNKNOWN = "Unknown format" + + +def detect_model_format(file_path): + _, ext = os.path.splitext(file_path) + + if ext == ".onnx": + return ModelType.ONNX + elif ext == ".h5": + return ModelType.KERAS_H5 + elif ext in [".pt", ".pth"]: + return ModelType.PYTORCH + elif ext in [".pkl", ".joblib"]: + return ModelType.SCIKIT_LEARN + elif ext in [".model", ".json", ".txt"]: + return ModelType.XGBOOST + + try: + with open(file_path, "rb") as f: + # TODO check if it works + header = f.read(4) + if ( + header == b"PK\x03\x04" + ): # Magic number for ZIP files (common in TensorFlow SavedModel) + return ModelType.TENSORFLOW_SAVEDMODEL + elif header[:2] == b"\x89H": # Magic number for HDF5 files (used by Keras) + return ModelType.KERAS_H5 + + except Exception: + return ModelType.UNKNOWN + + return ModelType.UNKNOWN From 4acf64ae4326503d3263a83229668d1e0c71b628 Mon Sep 17 00:00:00 2001 From: Wojtek Jurkowlaniec Date: Mon, 19 Aug 2024 15:28:19 +0200 Subject: [PATCH 012/227] WIP model runner and some example code for testing --- cancer_ai/base/neuron.py | 4 +- cancer_ai/utils/config.py | 11 +++- cancer_ai/validator/__init__.py | 4 +- cancer_ai/validator/competition_manager.py | 59 +++++++++++++----- cancer_ai/validator/dataset_manager.py | 10 +-- cancer_ai/validator/model_manager.py | 3 +- cancer_ai/validator/model_run_manager.py | 28 +++++++-- cancer_ai/validator/utils.py | 2 +- cancer_ai/validator_tester.py | 71 ++++++++++++++++++++++ 9 files changed, 154 insertions(+), 38 deletions(-) create mode 100644 cancer_ai/validator_tester.py diff --git a/cancer_ai/base/neuron.py b/cancer_ai/base/neuron.py index 06f65a20..5b40199f 100644 --- a/cancer_ai/base/neuron.py +++ b/cancer_ai/base/neuron.py @@ -23,7 +23,7 @@ from abc import ABC, abstractmethod # Sync calls set weights and also resyncs the metagraph. -from ..utils.config import check_config, add_args, config +from ..utils.config import check_config, add_args, path_config from ..utils.misc import ttl_get_block from .. import __spec_version__ as spec_version from ..mock import MockSubtensor, MockMetagraph @@ -48,7 +48,7 @@ def add_args(cls, parser): @classmethod def config(cls): - return config(cls) + return path_config(cls) subtensor: "bt.subtensor" wallet: "bt.wallet" diff --git a/cancer_ai/utils/config.py b/cancer_ai/utils/config.py index 5ebcf47e..895282c6 100644 --- a/cancer_ai/utils/config.py +++ b/cancer_ai/utils/config.py @@ -167,6 +167,13 @@ def add_miner_args(cls, parser): help="Wandb entity to log to.", ) + parser.add_argument( + "--competition.entity", + type=str, + default="opentensor-dev", + help="Wandb entity to log to.", + ) + def add_validator_args(cls, parser): """Add validator specific arguments to the parser.""" @@ -257,10 +264,12 @@ def add_validator_args(cls, parser): ) -def config(cls): +def path_config(cls): """ Returns the configuration object specific to this miner or validator after adding relevant arguments. """ + + # config from huggingface parser = argparse.ArgumentParser() bt.wallet.add_args(parser) bt.subtensor.add_args(parser) diff --git a/cancer_ai/validator/__init__.py b/cancer_ai/validator/__init__.py index e43fa856..456f206d 100644 --- a/cancer_ai/validator/__init__.py +++ b/cancer_ai/validator/__init__.py @@ -1,2 +1,2 @@ -from .forward import forward -from .reward import reward +# from .forward import forward +# from .reward import reward diff --git a/cancer_ai/validator/competition_manager.py b/cancer_ai/validator/competition_manager.py index ecc8d36f..b7662591 100644 --- a/cancer_ai/validator/competition_manager.py +++ b/cancer_ai/validator/competition_manager.py @@ -1,9 +1,26 @@ +from datetime import time +import random +from typing import List + +import bittensor as bt + from .manager import SerializableManager from .model_manager import ModelManager, ModelInfo from .dataset_manager import DatasetManager from .model_run_manager import ModelRunManager -from datetime import time -import random + + +COMPETITION_MAPPING = { + "melaona-1": "melanoma", +} + + +class ImagePredictionCompetition: + def score_model( + self, model_info: ModelInfo, pred_y: List, model_pred_y: List + ) -> float: + pass + class CompetitionManager(SerializableManager): """ @@ -17,7 +34,7 @@ def __init__( config, competition_id: str, category: str, - evaluation_time: list[time], + evaluation_times: list[str], dataset_hf_id: str, file_hf_id: str, ) -> None: @@ -28,18 +45,24 @@ def __init__( config (dict): Config dictionary. competition_id (str): Unique identifier for the competition. category (str): Category of the competition. - evaluation_time (list[time]): List of times of a day at which the competition will be evaluated. + evaluation_time (list[str]): List of times of a day at which the competition will be evaluated in XX:XX format. Note: Times are in UTC time. """ + bt.logging.info(f"Initializing Competition: {competition_id}") self.config = config self.competition_id = competition_id self.category = category self.model_manager = ModelManager(config) - self.evaluation_time = evaluation_time - self.dataset_manager: DatasetManager = DatasetManager( + + self.evaluation_time = [ + time(hour_min.split(":")[0], hour_min.split(":")[1]) + for hour_min in evaluation_times + ] + self.dataset_manager = DatasetManager( config, competition_id, dataset_hf_id, file_hf_id ) + # self.model_evaluator = self.results = [] def get_state(self): @@ -57,30 +80,34 @@ def set_state(self, state: dict): self.evaluation_time = state["evaluation_time"] async def get_miner_model(self, hotkey): - # test data + # TODO get real data return ModelInfo("vidhiparikh/House-Price-Estimator", "model_custom.pkcls") async def init_evaluation(self): + # get models from chain for hotkey in self.model_manager.hotkey_store: self.model_manager.hotkey_store[hotkey] = await self.get_miner_model(hotkey) - - self.dataset_manager.prepare_dataset() - + + await self.dataset_manager.prepare_dataset() + # log event async def evaluate(self): self.init_evaluation() - pred_x, pred_y = self.dataset_manager.get_data() + pred_x, pred_y = await self.dataset_manager.get_data() for hotkey in self.model_manager.hotkey_store: - print("Evaluating hotkey: ", hotkey) + bt.logging.info("Evaluating hotkey: ", hotkey) await self.model_manager.download_miner_model(hotkey) - pred_y = ModelRunManager(self.config, self.model_manager.hotkey_store[hotkey]).run(pred_x) + + model_manager = ModelRunManager( + self.config, self.model_manager.hotkey_store[hotkey] + ) + model_pred_y = model_manager.run(pred_x) # print "make stats and send to wandb" score = random.randint(0, 100) + bt.logging.info(f"Hotkey {hotkey} model score: {score}") self.results.append((hotkey, score)) - # sort by score + # sort by score self.results.sort(key=lambda x: x[1], reverse=True) return self.results - - diff --git a/cancer_ai/validator/dataset_manager.py b/cancer_ai/validator/dataset_manager.py index e3aba2bf..d4f74142 100644 --- a/cancer_ai/validator/dataset_manager.py +++ b/cancer_ai/validator/dataset_manager.py @@ -6,8 +6,7 @@ from huggingface_hub import HfApi from typing import List, Tuple -import aiofiles -import aiozip +from async_unzip.unzipper import unzip from io import BytesIO from .dataset_handlers.image_csv import DatasetImagesCSV @@ -47,11 +46,7 @@ def delete_dataset(self): shutil.rmtree(self.path) async def unzip_dataset(self): - async with aiofiles.open(self.path, "rb") as f: - data = await f.read() - async with aiofiles.open(self.path, "wb") as f: - async with aiozip.AIOZipFile(BytesIO(data)) as z: - await z.extractall(path=Path(self.path).parent) + await unzip(self.path, Path(self.path).parent) def set_dataset_handler(self): if not self.path: @@ -68,7 +63,6 @@ async def prepare_dataset(self): await self.unzip_dataset() self.set_dataset_handler() - async def get_data(self) -> Tuple[List, List]: if not self.data: await self.prepare_dataset() diff --git a/cancer_ai/validator/model_manager.py b/cancer_ai/validator/model_manager.py index 75421344..b4dbe69f 100644 --- a/cancer_ai/validator/model_manager.py +++ b/cancer_ai/validator/model_manager.py @@ -42,13 +42,12 @@ def download_miner_model(self, hotkey) -> None: str: path to the downloaded model """ model_info = self.hotkey_store[hotkey] - model_path = self.api.hf_hub_download( + model_info.file_path = self.api.hf_hub_download( model_info.repo_id, model_info.filename, cache_dir=self.config.models.model_dir, repo_type="space", ) - model_info.file_path = model_path def add_model(self, hotkey, repo_id, filename) -> None: """Saves locally information about a new model.""" diff --git a/cancer_ai/validator/model_run_manager.py b/cancer_ai/validator/model_run_manager.py index b7493afa..1c34fc14 100644 --- a/cancer_ai/validator/model_run_manager.py +++ b/cancer_ai/validator/model_run_manager.py @@ -15,8 +15,13 @@ def run(self): """Exceutes the run process of the model in separate process.""" +class TensorflowRunnerHandler(BaseRunnerHandler): + def run(self, pred_x: List) -> List: + return [] + + class PytorchRunnerHandler(BaseRunnerHandler): - def run(self, pred_x: List, pred_y: List) -> List: + def run(self, pred_x: List) -> List: # example, might not work from torch import load @@ -26,8 +31,9 @@ def run(self, pred_x: List, pred_y: List) -> List: return output -runner_handler_mapping = { +MODEL_TYPE_HANDLERS = { ModelType.PYTORCH: PytorchRunnerHandler, + ModelType.TENSORFLOW_SAVEDMODEL: TensorflowRunnerHandler, } @@ -35,6 +41,7 @@ class ModelRunManager(SerializableManager): def __init__(self, config, model: ModelInfo) -> None: self.config = config self.model = model + self.set_runner_handler() def get_state(self) -> dict: return {} @@ -43,9 +50,18 @@ def set_state(self, state: dict): pass def set_runner_handler(self) -> None: - self.handler = runner_handler_mapping[detect_model_format(self.model)]( - self.config, self.model.file_path - ) + model_type = detect_model_format(self.model) + # initializing ml model handler object + model_handler = MODEL_TYPE_HANDLERS[model_type] + self.handler = model_handler(self.config, self.model.file_path) def run(self, pred_x: List) -> List: - return self.handler.run(pred_x) + """ + Run the model with the given input. + + Returns: + List: model predictions + """ + + model_predictions = self.handler.run(pred_x) + return model_predictions diff --git a/cancer_ai/validator/utils.py b/cancer_ai/validator/utils.py index 2b7d7ae2..cd7121a4 100644 --- a/cancer_ai/validator/utils.py +++ b/cancer_ai/validator/utils.py @@ -12,7 +12,7 @@ class ModelType(Enum): UNKNOWN = "Unknown format" -def detect_model_format(file_path): +def detect_model_format(file_path) -> ModelType: _, ext = os.path.splitext(file_path) if ext == ".onnx": diff --git a/cancer_ai/validator_tester.py b/cancer_ai/validator_tester.py new file mode 100644 index 00000000..f28d2e0f --- /dev/null +++ b/cancer_ai/validator_tester.py @@ -0,0 +1,71 @@ +from validator.competition_manager import CompetitionManager +from datetime import time, now +import asyncio +import json +import timeit +from types import SimpleNamespace + +from cancer_ai.utils.config import config + + +path_config = { + "models": SimpleNamespace(**{"model_dir": "/tmp/models", "dataset_dir": "/tmp/datasets"}), +} +path_config = SimpleNamespace(**path_config) + +# open competition config file +def get_competition_config(): + with open(config.validator.competition_config_path) as f: + return json.load(f) + +async def main_loop(): + competitions = get_competition_config() + for competition_config in competitions: + # get list of competition_config["evaluation_time"] and time it to run during specific time of day + eval_times = [ + competition_config["evaluation_time"]] + while True: + now = time.localtime() + for eval_time in eval_times: + if now.tm_hour == eval_time.hour and now.tm_min == eval_time.minute: + competition = CompetitionManager( + path_config, + competition_config["competition_id"], + competition_config["category"], + competition_config["evaluation_time"], + competition_config["dataset_hf_id"], + competition_config["file_hf_id"], + ) + await competition.evaluate() + print(competition.results) + break + await asyncio.sleep(60) + + +competition_config = [ + { + "competition_id": "melaona-1", + "category": "skin", + "evaluation_time": ["12:30", "15:30"], + "dataset_hf_id": "vidhiparikh/House-Price-Estimator", + "file_hf_id": "model_custom.pkcls", + } +] + + +def run(): + asyncio.run(main_loop()) + + +if False: + competition = CompetitionManager( + path_config, + "melaona-1", + "skin", + ["12:30", "15:30"], + "test-dataset-hf-id", + "test-file-hf-id", + ) + asyncio.run(competition.evaluate()) + # await competition.evaluate() + print(competition.results) From f29be144f414daabfedfd86ad90c12fbaa7c9e54 Mon Sep 17 00:00:00 2001 From: Wojtek Jurkowlaniec Date: Mon, 19 Aug 2024 18:07:56 +0200 Subject: [PATCH 013/227] lil refactor --- cancer_ai/validator/model_run_manager.py | 36 +++++-------------- cancer_ai/validator/model_runners/__init__.py | 10 ++++++ .../validator/model_runners/pytorch_runner.py | 13 +++++++ .../model_runners/tensorflow_runner.py | 6 ++++ 4 files changed, 37 insertions(+), 28 deletions(-) create mode 100644 cancer_ai/validator/model_runners/pytorch_runner.py create mode 100644 cancer_ai/validator/model_runners/tensorflow_runner.py diff --git a/cancer_ai/validator/model_run_manager.py b/cancer_ai/validator/model_run_manager.py index 1c34fc14..cd8f8987 100644 --- a/cancer_ai/validator/model_run_manager.py +++ b/cancer_ai/validator/model_run_manager.py @@ -1,34 +1,10 @@ +from typing import List + from .manager import SerializableManager from .model_manager import ModelInfo -from typing import List, Tuple -from abc import abstractmethod from .utils import detect_model_format, ModelType - - -class BaseRunnerHandler: - def __init__(self, config, model_path: str) -> None: - self.config = config - self.model_path = model_path - - @abstractmethod - def run(self): - """Exceutes the run process of the model in separate process.""" - - -class TensorflowRunnerHandler(BaseRunnerHandler): - def run(self, pred_x: List) -> List: - return [] - - -class PytorchRunnerHandler(BaseRunnerHandler): - def run(self, pred_x: List) -> List: - # example, might not work - from torch import load - - model = load(self.model_path) - model.eval() - output = model(pred_x) - return output +from .model_runners.pytorch_runner import PytorchRunnerHandler +from .model_runners.tensorflow_runner import TensorflowRunnerHandler MODEL_TYPE_HANDLERS = { @@ -50,6 +26,10 @@ def set_state(self, state: dict): pass def set_runner_handler(self) -> None: + """ + Sets the model runner handler based on the model type + """ + model_type = detect_model_format(self.model) # initializing ml model handler object model_handler = MODEL_TYPE_HANDLERS[model_type] diff --git a/cancer_ai/validator/model_runners/__init__.py b/cancer_ai/validator/model_runners/__init__.py index e69de29b..40fcb77b 100644 --- a/cancer_ai/validator/model_runners/__init__.py +++ b/cancer_ai/validator/model_runners/__init__.py @@ -0,0 +1,10 @@ +from abc import abstractmethod + +class BaseRunnerHandler: + def __init__(self, config, model_path: str) -> None: + self.config = config + self.model_path = model_path + + @abstractmethod + def run(self): + """Exceutes the run process of the model in separate process.""" diff --git a/cancer_ai/validator/model_runners/pytorch_runner.py b/cancer_ai/validator/model_runners/pytorch_runner.py new file mode 100644 index 00000000..4de57864 --- /dev/null +++ b/cancer_ai/validator/model_runners/pytorch_runner.py @@ -0,0 +1,13 @@ +from . import BaseRunnerHandler +from typing import List + +class PytorchRunnerHandler(BaseRunnerHandler): + def run(self, pred_x: List) -> List: + # example, might not work + from torch import load + + model = load(self.model_path) + model.eval() + output = model(pred_x) + return output + diff --git a/cancer_ai/validator/model_runners/tensorflow_runner.py b/cancer_ai/validator/model_runners/tensorflow_runner.py new file mode 100644 index 00000000..4a87190f --- /dev/null +++ b/cancer_ai/validator/model_runners/tensorflow_runner.py @@ -0,0 +1,6 @@ +from . import BaseRunnerHandler +from typing import List + +class TensorflowRunnerHandler(BaseRunnerHandler): + def run(self, pred_x: List) -> List: + return [] From 8552feb3dd6622b155cdf03dff07e19aa8893c40 Mon Sep 17 00:00:00 2001 From: Wojtek Jurkowlaniec Date: Tue, 20 Aug 2024 12:33:14 +0200 Subject: [PATCH 014/227] WIP --- cancer_ai/validator/competition_manager.py | 8 +-- .../validator/dataset_handlers/image_csv.py | 4 +- cancer_ai/validator/dataset_manager.py | 2 +- .../scripts/dataset_api_integration.py | 40 ++++++++++++ cancer_ai/validator_tester.py | 63 ++++++++++--------- 5 files changed, 79 insertions(+), 38 deletions(-) create mode 100644 cancer_ai/validator/scripts/dataset_api_integration.py diff --git a/cancer_ai/validator/competition_manager.py b/cancer_ai/validator/competition_manager.py index b7662591..7f655d06 100644 --- a/cancer_ai/validator/competition_manager.py +++ b/cancer_ai/validator/competition_manager.py @@ -55,10 +55,10 @@ def __init__( self.category = category self.model_manager = ModelManager(config) - self.evaluation_time = [ - time(hour_min.split(":")[0], hour_min.split(":")[1]) - for hour_min in evaluation_times - ] + # self.evaluation_time = [ + # time(hour_min.split(":")[0], hour_min.split(":")[1]) + # for hour_min in evaluation_times + # ] self.dataset_manager = DatasetManager( config, competition_id, dataset_hf_id, file_hf_id ) diff --git a/cancer_ai/validator/dataset_handlers/image_csv.py b/cancer_ai/validator/dataset_handlers/image_csv.py index 8418c690..2fc93392 100644 --- a/cancer_ai/validator/dataset_handlers/image_csv.py +++ b/cancer_ai/validator/dataset_handlers/image_csv.py @@ -15,7 +15,7 @@ class ImageEntry: class DatasetImagesCSV(BaseDatasetHandler): """ DatasetImagesCSV is responsible for handling the CSV dataset where directory structure looks as follows: - . + ├── images │ ├── image_1.jpg │ ├── image_2.jpg @@ -29,7 +29,7 @@ def __init__(self, config, path: str) -> None: self.metadata_columns = ["filepath", "is_melanoma"] async def sync_training_data(self): - self.entries = [] + self.entries: ImageEntry = [] # go over csv file async with aiofiles.open(self.path, "r") as f: # "filepath" "is_melanoma" columns diff --git a/cancer_ai/validator/dataset_manager.py b/cancer_ai/validator/dataset_manager.py index d4f74142..ab597322 100644 --- a/cancer_ai/validator/dataset_manager.py +++ b/cancer_ai/validator/dataset_manager.py @@ -22,7 +22,7 @@ def __init__( self.file_hf_id = file_hf_id self.hf_api = HfApi() self.path = "" - self.data = None + self.data: Tuple[List, List] = () def get_state(self) -> dict: return {} diff --git a/cancer_ai/validator/scripts/dataset_api_integration.py b/cancer_ai/validator/scripts/dataset_api_integration.py new file mode 100644 index 00000000..6b0979c5 --- /dev/null +++ b/cancer_ai/validator/scripts/dataset_api_integration.py @@ -0,0 +1,40 @@ +import os +import csv +import requests + + +# Base URL for downloading images (replace with actual base URL) +BASE_URL = "http://localhost:8001/" +API_GET_IMAGES = "dataset/skin/melanoma?amount=10" + +# Create the images directory +os.makedirs("images", exist_ok=True) + +# Open the CSV file for writing +with open("labels.csv", mode="w", newline="") as csv_file: + csv_writer = csv.writer(csv_file) + # Write the header row + csv_writer.writerow(["path", "is_melanoma"]) + data = requests.get(BASE_URL + API_GET_IMAGES).json() + # Process each entry in the JSON data + for entry in data["entries"]: + image_id = entry["id"] + image_url = BASE_URL + entry["image_url"] + is_melanoma = entry["label"]["melanoma"] + + # Define the local file path + image_filename = f"images/{image_id}.jpg" + + # Download the image + response = requests.get(image_url) + if response.status_code == 200: + with open(image_filename, "wb") as image_file: + image_file.write(response.content) + print(f"Downloaded {image_filename}") + else: + print(f"Failed to download {image_filename}") + + # Write the image path and label to the CSV file + csv_writer.writerow([image_filename, is_melanoma]) + +print("Process completed.") \ No newline at end of file diff --git a/cancer_ai/validator_tester.py b/cancer_ai/validator_tester.py index f28d2e0f..3f44485b 100644 --- a/cancer_ai/validator_tester.py +++ b/cancer_ai/validator_tester.py @@ -1,11 +1,12 @@ from validator.competition_manager import CompetitionManager -from datetime import time, now +from datetime import time, datetime + import asyncio import json import timeit from types import SimpleNamespace -from cancer_ai.utils.config import config +# from cancer_ai.utils.config import config path_config = { @@ -14,37 +15,37 @@ path_config = SimpleNamespace(**path_config) # open competition config file -def get_competition_config(): - with open(config.validator.competition_config_path) as f: - return json.load(f) +# def get_competition_config(): +# with open(config.validator.competition_config_path) as f: +# return json.load(f) -async def main_loop(): - competitions = get_competition_config() - for competition_config in competitions: - # get list of competition_config["evaluation_time"] and time it to run during specific time of day - eval_times = [ - competition_config["evaluation_time"]] - while True: - now = time.localtime() - for eval_time in eval_times: - if now.tm_hour == eval_time.hour and now.tm_min == eval_time.minute: - competition = CompetitionManager( - path_config, - competition_config["competition_id"], - competition_config["category"], - competition_config["evaluation_time"], - competition_config["dataset_hf_id"], - competition_config["file_hf_id"], - ) - await competition.evaluate() - print(competition.results) - break - await asyncio.sleep(60) +# async def main_loop(): +# competitions = get_competition_config() +# for competition_config in competitions: +# # get list of competition_config["evaluation_time"] and time it to run during specific time of day +# eval_times = [ +# competition_config["evaluation_time"]] +# while True: +# now = time.localtime() +# for eval_time in eval_times: +# if now.tm_hour == eval_time.hour and now.tm_min == eval_time.minute: +# competition = CompetitionManager( +# path_config, +# competition_config["competition_id"], +# competition_config["category"], +# competition_config["evaluation_time"], +# competition_config["dataset_hf_id"], +# competition_config["file_hf_id"], +# ) +# await competition.evaluate() +# print(competition.results) +# break +# await asyncio.sleep(60) competition_config = [ { - "competition_id": "melaona-1", + "competition_id": "melanoma-1", "category": "skin", "evaluation_time": ["12:30", "15:30"], "dataset_hf_id": "vidhiparikh/House-Price-Estimator", @@ -53,11 +54,11 @@ async def main_loop(): ] -def run(): - asyncio.run(main_loop()) +# def run(): + # asyncio.run(main_loop()) -if False: +if __name__ == "__main__": competition = CompetitionManager( path_config, "melaona-1", From a41d151eff6b63e06ffb82da817ea17131359ce9 Mon Sep 17 00:00:00 2001 From: Wojtek Jurkowlaniec Date: Tue, 20 Aug 2024 14:44:32 +0200 Subject: [PATCH 015/227] working dataset downloading --- cancer_ai/validator/competition_manager.py | 2 +- .../validator/dataset_handlers/image_csv.py | 14 +++--- cancer_ai/validator/dataset_manager.py | 44 +++++++++++-------- cancer_ai/validator/model_run_manager.py | 4 +- cancer_ai/validator/utils.py | 16 ++++++- cancer_ai/validator_tester.py | 4 +- 6 files changed, 54 insertions(+), 30 deletions(-) diff --git a/cancer_ai/validator/competition_manager.py b/cancer_ai/validator/competition_manager.py index 7f655d06..e228d2ed 100644 --- a/cancer_ai/validator/competition_manager.py +++ b/cancer_ai/validator/competition_manager.py @@ -93,7 +93,7 @@ async def init_evaluation(self): # log event async def evaluate(self): - self.init_evaluation() + await self.init_evaluation() pred_x, pred_y = await self.dataset_manager.get_data() for hotkey in self.model_manager.hotkey_store: bt.logging.info("Evaluating hotkey: ", hotkey) diff --git a/cancer_ai/validator/dataset_handlers/image_csv.py b/cancer_ai/validator/dataset_handlers/image_csv.py index 2fc93392..6887b1b8 100644 --- a/cancer_ai/validator/dataset_handlers/image_csv.py +++ b/cancer_ai/validator/dataset_handlers/image_csv.py @@ -15,7 +15,7 @@ class ImageEntry: class DatasetImagesCSV(BaseDatasetHandler): """ DatasetImagesCSV is responsible for handling the CSV dataset where directory structure looks as follows: - + ├── images │ ├── image_1.jpg │ ├── image_2.jpg @@ -25,21 +25,23 @@ class DatasetImagesCSV(BaseDatasetHandler): def __init__(self, config, path: str) -> None: self.config = config - self.path = path + self.label_path = path self.metadata_columns = ["filepath", "is_melanoma"] async def sync_training_data(self): - self.entries: ImageEntry = [] + self.entries: List[ImageEntry] = [] # go over csv file - async with aiofiles.open(self.path, "r") as f: - # "filepath" "is_melanoma" columns + async with aiofiles.open(self.label_path, "r") as f: + # "path" "is_melanoma" columns reader = csv.reader(await f.readlines()) + next(reader) # skip first line for row in reader: self.entries.append(ImageEntry(row[0], row[1])) async def get_training_data(self) -> Tuple[List, List]: await self.sync_training_data() - pred_x = [Image(entry.filepath) for entry in self.entries] + print(self.entries) + pred_x = [Image.open(entry.filepath) for entry in self.entries] pred_y = [entry.is_melanoma for entry in self.entries] await self.process_training_data() return pred_x, pred_y diff --git a/cancer_ai/validator/dataset_manager.py b/cancer_ai/validator/dataset_manager.py index ab597322..7b33f646 100644 --- a/cancer_ai/validator/dataset_manager.py +++ b/cancer_ai/validator/dataset_manager.py @@ -1,5 +1,4 @@ import os -from zipfile import ZipFile import shutil from pathlib import Path from .manager import SerializableManager @@ -8,7 +7,7 @@ from typing import List, Tuple from async_unzip.unzipper import unzip from io import BytesIO - +from .utils import run_command from .dataset_handlers.image_csv import DatasetImagesCSV @@ -21,7 +20,10 @@ def __init__( self.dataset_hf_id = dataset_hf_id self.file_hf_id = file_hf_id self.hf_api = HfApi() - self.path = "" + self.local_compressed_path = "" + self.local_extracted_dir = Path( + self.config.models.dataset_dir, self.competition_id + ) self.data: Tuple[List, List] = () def get_state(self) -> dict: @@ -31,40 +33,46 @@ def set_state(self, state: dict): return {} def download_dataset(self): - if not os.path.exists( - Path(self.config.models.dataset_dir, self.competition_id) - ): - os.makedirs(Path(self.config.models.dataset_dir, self.competition_id)) + if not os.path.exists(self.local_extracted_dir): + os.makedirs(self.local_extracted_dir) - self.path = self.hf_api.hf_hub_download( + self.local_compressed_path = self.hf_api.hf_hub_download( self.dataset_hf_id, self.file_hf_id, - cache_dir=Path(self.config.models.dataset_dir, self.competition_id), + cache_dir=Path(self.config.models.dataset_dir), + repo_type="dataset", ) def delete_dataset(self): - shutil.rmtree(self.path) + shutil.rmtree(self.local_compressed_path) async def unzip_dataset(self): - await unzip(self.path, Path(self.path).parent) + print("Unzipping dataset", self.local_compressed_path) + os.system(f"rm -R {self.local_extracted_dir}") + await run_command( + f"unzip {self.local_compressed_path} -d {self.local_extracted_dir}" + ) + print("Dataset unzipped") def set_dataset_handler(self): - if not self.path: + if not self.local_compressed_path: raise Exception("Dataset not downloaded") # is csv in directory - if os.path.exists(Path(self.path, "labels.csv")): - self.handler = DatasetImagesCSV(self.config, Path(self.path, "labels.csv")) + if os.path.exists(Path(self.local_extracted_dir, "labels.csv")): + self.handler = DatasetImagesCSV( + self.config, Path(self.local_extracted_dir, "labels.csv") + ) else: - print("Files in dataset: ", os.listdir(self.path)) + print("Files in dataset: ", os.listdir(self.local_extracted_dir)) raise NotImplementedError("Dataset handler not implemented") async def prepare_dataset(self): - await self.download_dataset() + self.download_dataset() await self.unzip_dataset() self.set_dataset_handler() + self.data = await self.handler.get_training_data() async def get_data(self) -> Tuple[List, List]: if not self.data: - await self.prepare_dataset() - self.data = await self.handler.get_training_data() + raise Exception("Dataset not initalized ") return self.data diff --git a/cancer_ai/validator/model_run_manager.py b/cancer_ai/validator/model_run_manager.py index cd8f8987..79b30d6a 100644 --- a/cancer_ai/validator/model_run_manager.py +++ b/cancer_ai/validator/model_run_manager.py @@ -27,9 +27,9 @@ def set_state(self, state: dict): def set_runner_handler(self) -> None: """ - Sets the model runner handler based on the model type + Sets the model runner handler based on the model type. """ - + model_type = detect_model_format(self.model) # initializing ml model handler object model_handler = MODEL_TYPE_HANDLERS[model_type] diff --git a/cancer_ai/validator/utils.py b/cancer_ai/validator/utils.py index cd7121a4..43eeff6e 100644 --- a/cancer_ai/validator/utils.py +++ b/cancer_ai/validator/utils.py @@ -1,5 +1,6 @@ from enum import Enum import os +import asyncio class ModelType(Enum): @@ -21,7 +22,7 @@ def detect_model_format(file_path) -> ModelType: return ModelType.KERAS_H5 elif ext in [".pt", ".pth"]: return ModelType.PYTORCH - elif ext in [".pkl", ".joblib"]: + elif ext in [".pkl", ".joblib", ""]: return ModelType.SCIKIT_LEARN elif ext in [".model", ".json", ".txt"]: return ModelType.XGBOOST @@ -41,3 +42,16 @@ def detect_model_format(file_path) -> ModelType: return ModelType.UNKNOWN return ModelType.UNKNOWN + + +async def run_command(cmd): + # Start the subprocess + process = await asyncio.create_subprocess_shell( + cmd, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE + ) + + # Wait for the subprocess to finish and capture the output + stdout, stderr = await process.communicate() + + # Return the output and error if any + return stdout.decode(), stderr.decode() diff --git a/cancer_ai/validator_tester.py b/cancer_ai/validator_tester.py index 3f44485b..91665e7b 100644 --- a/cancer_ai/validator_tester.py +++ b/cancer_ai/validator_tester.py @@ -64,8 +64,8 @@ "melaona-1", "skin", ["12:30", "15:30"], - "test-dataset-hf-id", - "test-file-hf-id", + "safescanai/test_dataset", + "skin_melanoma.zip", ) asyncio.run(competition.evaluate()) # await competition.evaluate() From 77db33eb2eca013c109e0a963faa1fa3ce804b6f Mon Sep 17 00:00:00 2001 From: Wojtek Jurkowlaniec Date: Tue, 20 Aug 2024 18:00:30 +0200 Subject: [PATCH 016/227] logging added --- cancer_ai/validator/dataset_manager.py | 46 ++++++++++++++++++++------ 1 file changed, 36 insertions(+), 10 deletions(-) diff --git a/cancer_ai/validator/dataset_manager.py b/cancer_ai/validator/dataset_manager.py index 7b33f646..3bbc28b7 100644 --- a/cancer_ai/validator/dataset_manager.py +++ b/cancer_ai/validator/dataset_manager.py @@ -1,20 +1,35 @@ import os import shutil from pathlib import Path -from .manager import SerializableManager +from typing import List, Tuple + from huggingface_hub import HfApi +import bittensor as bt -from typing import List, Tuple -from async_unzip.unzipper import unzip -from io import BytesIO +from .manager import SerializableManager from .utils import run_command from .dataset_handlers.image_csv import DatasetImagesCSV +class DatasetManagerException(Exception): + pass + class DatasetManager(SerializableManager): def __init__( self, config, competition_id: str, dataset_hf_id: str, file_hf_id: str ) -> None: + """ + Initializes a new instance of the DatasetManager class. + + Args: + config: The configuration object. + competition_id (str): The ID of the competition. + dataset_hf_id (str): The Hugging Face ID of the dataset. + file_hf_id (str): The Hugging Face ID of the file. + + Returns: + None + """ self.config = config self.competition_id = competition_id self.dataset_hf_id = dataset_hf_id @@ -25,6 +40,7 @@ def __init__( self.config.models.dataset_dir, self.competition_id ) self.data: Tuple[List, List] = () + self.handler = None def get_state(self) -> dict: return {} @@ -43,10 +59,12 @@ def download_dataset(self): repo_type="dataset", ) - def delete_dataset(self): + def delete_dataset(self) -> None: + """Delete dataset from disk""" shutil.rmtree(self.local_compressed_path) - async def unzip_dataset(self): + async def unzip_dataset(self) -> None: + """Unzip dataset""" print("Unzipping dataset", self.local_compressed_path) os.system(f"rm -R {self.local_extracted_dir}") await run_command( @@ -54,9 +72,10 @@ async def unzip_dataset(self): ) print("Dataset unzipped") - def set_dataset_handler(self): + def set_dataset_handler(self) -> None: + """Detect dataset type and set handler""" if not self.local_compressed_path: - raise Exception("Dataset not downloaded") + raise DatasetManagerException("Dataset not downloaded") # is csv in directory if os.path.exists(Path(self.local_extracted_dir, "labels.csv")): self.handler = DatasetImagesCSV( @@ -66,13 +85,20 @@ def set_dataset_handler(self): print("Files in dataset: ", os.listdir(self.local_extracted_dir)) raise NotImplementedError("Dataset handler not implemented") - async def prepare_dataset(self): + async def prepare_dataset(self) -> None: + """Download dataset, unzip and set dataset handler""" + + bt.logging.info("Downloading dataset") self.download_dataset() + bt.logging.info("Unzipping dataset") await self.unzip_dataset() + bt.logging.info("Setting dataset handler") self.set_dataset_handler() + bt.logging.info("Preprocessing dataset") self.data = await self.handler.get_training_data() async def get_data(self) -> Tuple[List, List]: + """Get data from dataset handler""" if not self.data: - raise Exception("Dataset not initalized ") + raise DatasetManagerException("Dataset not initalized ") return self.data From a3374aa9bf167f255145a5279ff808e1193213f3 Mon Sep 17 00:00:00 2001 From: notbulubula Date: Tue, 20 Aug 2024 18:53:12 +0200 Subject: [PATCH 017/227] tmp path correction --- cancer_ai/validator/dataset_handlers/image_csv.py | 3 ++- cancer_ai/validator/dataset_manager.py | 6 ++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/cancer_ai/validator/dataset_handlers/image_csv.py b/cancer_ai/validator/dataset_handlers/image_csv.py index 6887b1b8..d9ef4d72 100644 --- a/cancer_ai/validator/dataset_handlers/image_csv.py +++ b/cancer_ai/validator/dataset_handlers/image_csv.py @@ -4,6 +4,7 @@ from dataclasses import dataclass import csv import aiofiles +from pathlib import Path @dataclass @@ -41,7 +42,7 @@ async def sync_training_data(self): async def get_training_data(self) -> Tuple[List, List]: await self.sync_training_data() print(self.entries) - pred_x = [Image.open(entry.filepath) for entry in self.entries] + pred_x = [Image.open(f"{Path(self.label_path).parent}/{entry.filepath}") for entry in self.entries] pred_y = [entry.is_melanoma for entry in self.entries] await self.process_training_data() return pred_x, pred_y diff --git a/cancer_ai/validator/dataset_manager.py b/cancer_ai/validator/dataset_manager.py index 3bbc28b7..9ee35b96 100644 --- a/cancer_ai/validator/dataset_manager.py +++ b/cancer_ai/validator/dataset_manager.py @@ -67,9 +67,11 @@ async def unzip_dataset(self) -> None: """Unzip dataset""" print("Unzipping dataset", self.local_compressed_path) os.system(f"rm -R {self.local_extracted_dir}") - await run_command( + print(f"unzip {self.local_compressed_path} -d {self.local_extracted_dir}") + out, err = await run_command( f"unzip {self.local_compressed_path} -d {self.local_extracted_dir}" ) + print(err) print("Dataset unzipped") def set_dataset_handler(self) -> None: @@ -82,7 +84,7 @@ def set_dataset_handler(self) -> None: self.config, Path(self.local_extracted_dir, "labels.csv") ) else: - print("Files in dataset: ", os.listdir(self.local_extracted_dir)) + #print("Files in dataset: ", os.listdir(self.local_extracted_dir)) raise NotImplementedError("Dataset handler not implemented") async def prepare_dataset(self) -> None: From 53c95d809a91bdcc63d7d04b80032561be32ee10 Mon Sep 17 00:00:00 2001 From: Wojtek Jurkowlaniec Date: Tue, 20 Aug 2024 20:11:17 +0200 Subject: [PATCH 018/227] yeah working --- cancer_ai/validator/competition_manager.py | 29 +++++++++++++++---- .../validator/dataset_handlers/image_csv.py | 2 +- cancer_ai/validator/model_manager.py | 4 +-- cancer_ai/validator/model_run_manager.py | 4 +-- .../model_runners/tensorflow_runner.py | 14 ++++++++- 5 files changed, 42 insertions(+), 11 deletions(-) diff --git a/cancer_ai/validator/competition_manager.py b/cancer_ai/validator/competition_manager.py index e228d2ed..47e5a89e 100644 --- a/cancer_ai/validator/competition_manager.py +++ b/cancer_ai/validator/competition_manager.py @@ -81,13 +81,30 @@ def set_state(self, state: dict): async def get_miner_model(self, hotkey): # TODO get real data - return ModelInfo("vidhiparikh/House-Price-Estimator", "model_custom.pkcls") + return ModelInfo("safescanai/test_dataset", "melanoma.keras") async def init_evaluation(self): - # get models from chain - for hotkey in self.model_manager.hotkey_store: - self.model_manager.hotkey_store[hotkey] = await self.get_miner_model(hotkey) - + # TODO get models from chain + miner_models = [ + { + "hotkey": "wojtasy", + "hf_id": "safescanai/test_dataset", + "file_hf_id": "melanoma.keras", + }, + { + "hotkey": "wojtasyy", + "hf_id": "safescanai/test_dataset", + "file_hf_id": "melanoma.keras", + }, + ] + bt.logging.info( + f"Populating model manager with miner models. Got {len(miner_models)} models" + ) + for miner_info in miner_models: + self.model_manager.add_model( + miner_info["hotkey"], miner_info["hf_id"], miner_info["file_hf_id"] + ) + bt.logging.info("Initializing dataset") await self.dataset_manager.prepare_dataset() # log event @@ -103,6 +120,8 @@ async def evaluate(self): self.config, self.model_manager.hotkey_store[hotkey] ) model_pred_y = model_manager.run(pred_x) + print("Model prediction ", model_pred_y) + print("Ground truth: ", pred_y) # print "make stats and send to wandb" score = random.randint(0, 100) bt.logging.info(f"Hotkey {hotkey} model score: {score}") diff --git a/cancer_ai/validator/dataset_handlers/image_csv.py b/cancer_ai/validator/dataset_handlers/image_csv.py index d9ef4d72..834020d3 100644 --- a/cancer_ai/validator/dataset_handlers/image_csv.py +++ b/cancer_ai/validator/dataset_handlers/image_csv.py @@ -42,7 +42,7 @@ async def sync_training_data(self): async def get_training_data(self) -> Tuple[List, List]: await self.sync_training_data() print(self.entries) - pred_x = [Image.open(f"{Path(self.label_path).parent}/{entry.filepath}") for entry in self.entries] + pred_x = [f"{Path(self.label_path).parent}/{entry.filepath}" for entry in self.entries] pred_y = [entry.is_melanoma for entry in self.entries] await self.process_training_data() return pred_x, pred_y diff --git a/cancer_ai/validator/model_manager.py b/cancer_ai/validator/model_manager.py index b4dbe69f..eea6026e 100644 --- a/cancer_ai/validator/model_manager.py +++ b/cancer_ai/validator/model_manager.py @@ -36,7 +36,7 @@ def sync_hotkeys(self, hotkeys: list): if hotkey not in hotkeys: self.delete_model(hotkey) - def download_miner_model(self, hotkey) -> None: + async def download_miner_model(self, hotkey) -> None: """Downloads the newest model from Hugging Face and saves it to disk. Returns: str: path to the downloaded model @@ -46,7 +46,7 @@ def download_miner_model(self, hotkey) -> None: model_info.repo_id, model_info.filename, cache_dir=self.config.models.model_dir, - repo_type="space", + repo_type="dataset", ) def add_model(self, hotkey, repo_id, filename) -> None: diff --git a/cancer_ai/validator/model_run_manager.py b/cancer_ai/validator/model_run_manager.py index 79b30d6a..3873f744 100644 --- a/cancer_ai/validator/model_run_manager.py +++ b/cancer_ai/validator/model_run_manager.py @@ -30,7 +30,7 @@ def set_runner_handler(self) -> None: Sets the model runner handler based on the model type. """ - model_type = detect_model_format(self.model) + model_type = detect_model_format(self.model.file_path) # initializing ml model handler object model_handler = MODEL_TYPE_HANDLERS[model_type] self.handler = model_handler(self.config, self.model.file_path) @@ -42,6 +42,6 @@ def run(self, pred_x: List) -> List: Returns: List: model predictions """ - + print(" model handler is ", self.handler) model_predictions = self.handler.run(pred_x) return model_predictions diff --git a/cancer_ai/validator/model_runners/tensorflow_runner.py b/cancer_ai/validator/model_runners/tensorflow_runner.py index 4a87190f..8f588568 100644 --- a/cancer_ai/validator/model_runners/tensorflow_runner.py +++ b/cancer_ai/validator/model_runners/tensorflow_runner.py @@ -3,4 +3,16 @@ class TensorflowRunnerHandler(BaseRunnerHandler): def run(self, pred_x: List) -> List: - return [] + import tensorflow as tf + import numpy as np + from tensorflow.keras.preprocessing.image import load_img + print("imgs to test", len(pred_x)) + img_list = [load_img(img_path, target_size=(180, 180, 3)) for img_path in pred_x] + img_list = [np.expand_dims(test_img, axis=0) for test_img in img_list] + + model = tf.keras.models.load_model(self.model_path) + # batched_predictions = model.predict(np.array(img_list)) + # return [batched_predictions[i][0] for i in range(len(img_list))] + img_list = np.array(img_list) + img_list = np.squeeze(img_list, axis=1) + return model.predict(img_list, batch_size=10) From c4e211e2a2f7845f216fe7dc4cf71abcb4620e95 Mon Sep 17 00:00:00 2001 From: notbulubula Date: Wed, 21 Aug 2024 17:22:13 +0200 Subject: [PATCH 019/227] Adding onnx runner --- cancer_ai/validator/competition_manager.py | 19 ++++--- cancer_ai/validator/model_run_manager.py | 2 + .../validator/model_runners/onnx_runner.py | 51 +++++++++++++++++++ 3 files changed, 65 insertions(+), 7 deletions(-) create mode 100644 cancer_ai/validator/model_runners/onnx_runner.py diff --git a/cancer_ai/validator/competition_manager.py b/cancer_ai/validator/competition_manager.py index 47e5a89e..15c99fee 100644 --- a/cancer_ai/validator/competition_manager.py +++ b/cancer_ai/validator/competition_manager.py @@ -86,15 +86,20 @@ async def get_miner_model(self, hotkey): async def init_evaluation(self): # TODO get models from chain miner_models = [ + # { + # "hotkey": "wojtasy", + # "hf_id": "safescanai/test_dataset", + # "file_hf_id": "melanoma.keras", + # }, + # { + # "hotkey": "wojtasyy", + # "hf_id": "safescanai/test_dataset", + # "file_hf_id": "melanoma.keras", + # }, { - "hotkey": "wojtasy", + "hotkey": "obcy ludzie", "hf_id": "safescanai/test_dataset", - "file_hf_id": "melanoma.keras", - }, - { - "hotkey": "wojtasyy", - "hf_id": "safescanai/test_dataset", - "file_hf_id": "melanoma.keras", + "file_hf_id": "simple_cnn_model.onnx", }, ] bt.logging.info( diff --git a/cancer_ai/validator/model_run_manager.py b/cancer_ai/validator/model_run_manager.py index 3873f744..0d6258cb 100644 --- a/cancer_ai/validator/model_run_manager.py +++ b/cancer_ai/validator/model_run_manager.py @@ -5,11 +5,13 @@ from .utils import detect_model_format, ModelType from .model_runners.pytorch_runner import PytorchRunnerHandler from .model_runners.tensorflow_runner import TensorflowRunnerHandler +from .model_runners.onnx_runner import OnnxRunnerHandler MODEL_TYPE_HANDLERS = { ModelType.PYTORCH: PytorchRunnerHandler, ModelType.TENSORFLOW_SAVEDMODEL: TensorflowRunnerHandler, + ModelType.ONNX: OnnxRunnerHandler, } diff --git a/cancer_ai/validator/model_runners/onnx_runner.py b/cancer_ai/validator/model_runners/onnx_runner.py new file mode 100644 index 00000000..d5b36a09 --- /dev/null +++ b/cancer_ai/validator/model_runners/onnx_runner.py @@ -0,0 +1,51 @@ +from . import BaseRunnerHandler +from typing import List + +class OnnxRunnerHandler(BaseRunnerHandler): + def run(self, pred_x: List) -> List: + import onnxruntime + import numpy as np + import torch + from PIL import Image + from torchvision import transforms + + # Load the ONNX model + session = onnxruntime.InferenceSession(self.model_path) + input_name = session.get_inputs()[0].name + output_name = session.get_outputs()[0].name + + # Preprocess the input + img_list = [Image.open(img_path) for img_path in pred_x] + # Define your transformations (resize, normalize, etc.) + transform = transforms.Compose([ + transforms.Resize((224, 224)), + transforms.ToTensor(), + transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5]) + ]) + # preprocess = transforms.Compose([ + # transforms.Resize(256), + # transforms.CenterCrop(224), + # transforms.ToTensor(), + # transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]), + # ]) + results = [] + for img_path in pred_x: + # Preprocess the input + img = Image.open(img_path) + img = transform(img) + img = img.unsqueeze(0) # Add batch dimension + + # Convert to numpy array + input_data = img.numpy() + input_data = input_data.astype(np.float32) # Ensure type is float32 + + # Prepare input for ONNX model + input_data = {input_name: input_data} + + # Run the model + output = session.run([output_name], input_data) + + # Collect results + results.append(output[0].tolist()) + + return results From 627000ecfd91344e039b6a2ff432fb122f16ad55 Mon Sep 17 00:00:00 2001 From: notbulubula Date: Thu, 22 Aug 2024 12:09:35 +0200 Subject: [PATCH 020/227] Adding wandb loging --- cancer_ai/validator/competition_manager.py | 75 ++++++++++++++++--- cancer_ai/validator/model_run_manager.py | 4 +- .../validator/model_runners/onnx_runner.py | 38 ++-------- cancer_ai/validator_tester.py | 36 ++++++++- 4 files changed, 108 insertions(+), 45 deletions(-) diff --git a/cancer_ai/validator/competition_manager.py b/cancer_ai/validator/competition_manager.py index 15c99fee..6697b233 100644 --- a/cancer_ai/validator/competition_manager.py +++ b/cancer_ai/validator/competition_manager.py @@ -1,4 +1,4 @@ -from datetime import time +import time import random from typing import List @@ -9,6 +9,8 @@ from .dataset_manager import DatasetManager from .model_run_manager import ModelRunManager +from sklearn.metrics import accuracy_score, precision_score, recall_score, confusion_matrix, roc_curve, auc +from dataclasses import dataclass COMPETITION_MAPPING = { "melaona-1": "melanoma", @@ -21,6 +23,18 @@ def score_model( ) -> float: pass +@dataclass +class ModelEvaluationResult: + accuracy: float + precision: float + recall: float + confusion_matrix: any + fpr: any + tpr: any + roc_auc: float + run_time: float + tested_entries: int + class CompetitionManager(SerializableManager): """ @@ -115,23 +129,64 @@ async def init_evaluation(self): # log event async def evaluate(self): + from PIL import Image + import numpy as np await self.init_evaluation() - pred_x, pred_y = await self.dataset_manager.get_data() + path_X_test, y_test = await self.dataset_manager.get_data() + # Prepre X_test form paths to images + X_test = [] + target_size=(224, 224) #TODO: Change this to the correct size + + for img_path in path_X_test: + img = Image.open(img_path) + img = img.resize(target_size) + img_array = np.array(img, dtype=np.float32) / 255.0 + img_array = np.array(img) + if img_array.shape[-1] != 3: # Handle grayscale images + img_array = np.stack((img_array,) * 3, axis=-1) + + img_array = np.transpose(img_array, (2, 0, 1)) # Convert image to numpy array + img_array = np.expand_dims(img_array, axis=0) # Add batch dimension + X_test.append(img_array) + X_test = np.array(X_test, dtype=np.float32) + + # print("X_test shape: ", X_test.shape) + + # map y_test to 0, 1 + y_test = [1 if y == "True" else 0 for y in y_test] + for hotkey in self.model_manager.hotkey_store: bt.logging.info("Evaluating hotkey: ", hotkey) await self.model_manager.download_miner_model(hotkey) + start_time = time.time() model_manager = ModelRunManager( self.config, self.model_manager.hotkey_store[hotkey] ) - model_pred_y = model_manager.run(pred_x) - print("Model prediction ", model_pred_y) - print("Ground truth: ", pred_y) + y_pred = model_manager.run(X_test) + print("Model prediction ", y_pred) + print("Ground truth: ", y_test) # print "make stats and send to wandb" - score = random.randint(0, 100) - bt.logging.info(f"Hotkey {hotkey} model score: {score}") - self.results.append((hotkey, score)) + run_time = time.time() - start_time + tested_entries = len(y_test) + accuracy = accuracy_score(y_test, y_pred) + precision = precision_score(y_test, y_pred) + recall = recall_score(y_test, y_pred) + conf_matrix = confusion_matrix(y_test, y_pred) + fpr, tpr, _ = roc_curve(y_test, y_pred) + roc_auc = auc(fpr, tpr) + + model_result = ModelEvaluationResult( + tested_entries=tested_entries, + run_time=run_time, + accuracy=accuracy, + precision=precision, + recall=recall, + confusion_matrix=conf_matrix, + fpr=fpr, + tpr=tpr, + roc_auc=roc_auc, + ) + self.results.append((hotkey, model_result)) - # sort by score - self.results.sort(key=lambda x: x[1], reverse=True) return self.results diff --git a/cancer_ai/validator/model_run_manager.py b/cancer_ai/validator/model_run_manager.py index 0d6258cb..3b7f0cb5 100644 --- a/cancer_ai/validator/model_run_manager.py +++ b/cancer_ai/validator/model_run_manager.py @@ -37,7 +37,7 @@ def set_runner_handler(self) -> None: model_handler = MODEL_TYPE_HANDLERS[model_type] self.handler = model_handler(self.config, self.model.file_path) - def run(self, pred_x: List) -> List: + def run(self, X_test) -> List: """ Run the model with the given input. @@ -45,5 +45,5 @@ def run(self, pred_x: List) -> List: List: model predictions """ print(" model handler is ", self.handler) - model_predictions = self.handler.run(pred_x) + model_predictions = self.handler.run(X_test) return model_predictions diff --git a/cancer_ai/validator/model_runners/onnx_runner.py b/cancer_ai/validator/model_runners/onnx_runner.py index d5b36a09..a4c17c64 100644 --- a/cancer_ai/validator/model_runners/onnx_runner.py +++ b/cancer_ai/validator/model_runners/onnx_runner.py @@ -2,7 +2,7 @@ from typing import List class OnnxRunnerHandler(BaseRunnerHandler): - def run(self, pred_x: List) -> List: + def run(self, X_test: List) -> List: import onnxruntime import numpy as np import torch @@ -12,40 +12,14 @@ def run(self, pred_x: List) -> List: # Load the ONNX model session = onnxruntime.InferenceSession(self.model_path) input_name = session.get_inputs()[0].name - output_name = session.get_outputs()[0].name - - # Preprocess the input - img_list = [Image.open(img_path) for img_path in pred_x] - # Define your transformations (resize, normalize, etc.) - transform = transforms.Compose([ - transforms.Resize((224, 224)), - transforms.ToTensor(), - transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5]) - ]) - # preprocess = transforms.Compose([ - # transforms.Resize(256), - # transforms.CenterCrop(224), - # transforms.ToTensor(), - # transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]), - # ]) + results = [] - for img_path in pred_x: - # Preprocess the input - img = Image.open(img_path) - img = transform(img) - img = img.unsqueeze(0) # Add batch dimension - - # Convert to numpy array - input_data = img.numpy() - input_data = input_data.astype(np.float32) # Ensure type is float32 - + for img in X_test: # Prepare input for ONNX model - input_data = {input_name: input_data} - - # Run the model - output = session.run([output_name], input_data) + input_data = {input_name: img} + y_pred = session.run(None, input_data)[0][0] # Collect results - results.append(output[0].tolist()) + results.append(y_pred[0].tolist()) return results diff --git a/cancer_ai/validator_tester.py b/cancer_ai/validator_tester.py index 91665e7b..14524b71 100644 --- a/cancer_ai/validator_tester.py +++ b/cancer_ai/validator_tester.py @@ -7,6 +7,11 @@ from types import SimpleNamespace # from cancer_ai.utils.config import config +from validator.competition_manager import ModelEvaluationResult +import wandb + +from dotenv import load_dotenv +import os path_config = { @@ -50,6 +55,8 @@ "evaluation_time": ["12:30", "15:30"], "dataset_hf_id": "vidhiparikh/House-Price-Estimator", "file_hf_id": "model_custom.pkcls", + "wandb_project": "testing_integration", + "wandb_entity": "urbaniak-bruno-safescanai", # TODO: Update this line to official entity } ] @@ -57,6 +64,27 @@ # def run(): # asyncio.run(main_loop()) +def log_results_to_wandb(project, entity, hotkey, evaluation_result: ModelEvaluationResult): + wandb.init(project=project, entity=entity) # TODO: Update this line as needed + + wandb.log({ + "hotkey": hotkey, + "tested_entries": evaluation_result.tested_entries, + "model_test_run_time": evaluation_result.run_time, + "accuracy": evaluation_result.accuracy, + "precision": evaluation_result.precision, + "recall": evaluation_result.recall, + "confusion_matrix": evaluation_result.confusion_matrix.tolist(), + "roc_curve": { + "fpr": evaluation_result.fpr.tolist(), + "tpr": evaluation_result.tpr.tolist() + }, + "roc_auc": evaluation_result.roc_auc + }) + + wandb.finish() + return + if __name__ == "__main__": competition = CompetitionManager( @@ -69,4 +97,10 @@ ) asyncio.run(competition.evaluate()) # await competition.evaluate() - print(competition.results) + # print(competition.results) + load_dotenv() + wandb_api_key = os.getenv("WANDB_API_KEY") + wandb.login(key=wandb_api_key) + for competition_info in competition.results: + hotkey, result = competition_info + log_results_to_wandb(competition_config[0]["wandb_project"], competition_config[0]["wandb_entity"], hotkey, result) From 0cb794476775c3307985fe3146545e6c507b4f57 Mon Sep 17 00:00:00 2001 From: Konrad Date: Sun, 18 Aug 2024 01:26:22 +0200 Subject: [PATCH 021/227] snapshot commit --- cancer_ai/chain_models_store.py | 144 ++++++++++++++++++++++++ cancer_ai/utils/models_storage_utils.py | 48 ++++++++ neurons/miner.py | 70 ++++++++---- 3 files changed, 239 insertions(+), 23 deletions(-) create mode 100644 cancer_ai/chain_models_store.py create mode 100644 cancer_ai/utils/models_storage_utils.py diff --git a/cancer_ai/chain_models_store.py b/cancer_ai/chain_models_store.py new file mode 100644 index 00000000..b72be792 --- /dev/null +++ b/cancer_ai/chain_models_store.py @@ -0,0 +1,144 @@ +import functools +import bittensor as bt +import datetime +from typing import ClassVar, Optional, Type + +from utils import run_in_subprocess +from pydantic import BaseModel, Field, PositiveInt + +# The maximum bytes for metadata on the chain. +MAX_METADATA_BYTES = 128 +# The length, in bytes, of a git commit hash. +GIT_COMMIT_LENGTH = 40 +# The length, in bytes, of a base64 encoded sha256 hash. +SHA256_BASE_64_LENGTH = 44 +# The max length, in characters, of the competition id +MAX_COMPETITION_ID_LENGTH = 2 + +class ModelId(BaseModel): + """Uniquely identifies a trained model""" + + MAX_REPO_ID_LENGTH: ClassVar[int] = ( + MAX_METADATA_BYTES + - GIT_COMMIT_LENGTH + - SHA256_BASE_64_LENGTH + - MAX_COMPETITION_ID_LENGTH + - 4 # separators + ) + + namespace: str = Field( + description="Namespace where the model can be found. ex. Hugging Face username/org." + ) + name: str = Field(description="Name of the model.") + + epoch: str = Field(description="The epoch number to submit as your checkpoint to evaluate e.g. 10") + + date: datetime.datetime = Field(description="The datetime at which model was pushed to hugging face") + + # When handling a model locally the commit and hash are not necessary. + # Commit must be filled when trying to download from a remote store. + commit: Optional[str] = Field( + description="Commit of the model. May be empty if not yet committed." + ) + # Hash is filled automatically when uploading to or downloading from a remote store. + hash: Optional[str] = Field(description="Hash of the trained model.") + # Identifier for competition + competition_id: Optional[str] = Field(description="The competition id") + + def to_compressed_str(self) -> str: + """Returns a compressed string representation.""" + return f"{self.namespace}:{self.name}:{self.epoch}:{self.commit}:{self.hash}:{self.competition_id}" + + @classmethod + def from_compressed_str(cls, cs: str) -> Type["ModelId"]: + """Returns an instance of this class from a compressed string representation""" + tokens = cs.split(":") + return cls( + namespace=tokens[0], + name=tokens[1], + epoch=tokens[2] if tokens[2] != "None" else None, + commit=tokens[3] if tokens[3] != "None" else None, + hash=tokens[4] if tokens[4] != "None" else None, + competition_id=( + tokens[5] if len(tokens) >= 6 and tokens[5] != "None" else None + ), + ) + + +class Model(BaseModel): + """Represents a pre trained foundation model.""" + + class Config: + arbitrary_types_allowed = True + + id: ModelId = Field(description="Identifier for this model.") + local_repo_dir: str = Field(description="Local repository with the required files.") + + +class ModelMetadata(BaseModel): + id: ModelId = Field(description="Identifier for this trained model.") + block: PositiveInt = Field( + description="Block on which this model was claimed on the chain." + ) + + +class ChainModelMetadataStore(): + """Chain based implementation for storing and retrieving metadata about a model.""" + + def __init__( + self, + subtensor: bt.subtensor, + subnet_uid: int, + wallet: Optional[bt.wallet] = None, + ): + self.subtensor = subtensor + self.wallet = ( + wallet # Wallet is only needed to write to the chain, not to read. + ) + self.subnet_uid = subnet_uid + + async def store_model_metadata(self, model_id: ModelId): + """Stores model metadata on this subnet for a specific wallet.""" + if self.wallet is None: + raise ValueError("No wallet available to write to the chain.") + + # Wrap calls to the subtensor in a subprocess with a timeout to handle potential hangs. + partial = functools.partial( + self.subtensor.commit, + self.wallet, + self.subnet_uid, + model_id.to_compressed_str(), + ) + run_in_subprocess(partial, 60) + + async def retrieve_model_metadata(self, hotkey: str) -> Optional[ModelMetadata]: + """Retrieves model metadata on this subnet for specific hotkey""" + + # Wrap calls to the subtensor in a subprocess with a timeout to handle potential hangs. + partial = functools.partial( + bt.extrinsics.serving.get_metadata, self.subtensor, self.subnet_uid, hotkey + ) + + metadata = run_in_subprocess(partial, 60) + + if not metadata: + return None + + commitment = metadata["info"]["fields"][0] + hex_data = commitment[list(commitment.keys())[0]][2:] + + chain_str = bytes.fromhex(hex_data).decode() + + model_id = None + + try: + model_id = ModelId.from_compressed_str(chain_str) + except: + # If the metadata format is not correct on the chain then we return None. + bt.logging.trace( + f"Failed to parse the metadata on the chain for hotkey {hotkey}." + ) + return None + + model_metadata = ModelMetadata(id=model_id, block=metadata["block"]) + return model_metadata diff --git a/cancer_ai/utils/models_storage_utils.py b/cancer_ai/utils/models_storage_utils.py new file mode 100644 index 00000000..68e6ad2e --- /dev/null +++ b/cancer_ai/utils/models_storage_utils.py @@ -0,0 +1,48 @@ +import functools +import multiprocessing +from typing import Any + +def run_in_subprocess(func: functools.partial, ttl: int) -> Any: + """Runs the provided function on a subprocess with 'ttl' seconds to complete. + + Args: + func (functools.partial): Function to be run. + ttl (int): How long to try for in seconds. + + Returns: + Any: The value returned by 'func' + """ + + def wrapped_func(func: functools.partial, queue: multiprocessing.Queue): + try: + result = func() + queue.put(result) + except (Exception, BaseException) as e: + # Catch exceptions here to add them to the queue. + queue.put(e) + + # Use "fork" (the default on all POSIX except macOS), because pickling doesn't seem + # to work on "spawn". + ctx = multiprocessing.get_context("fork") + queue = ctx.Queue() + process = ctx.Process(target=wrapped_func, args=[func, queue]) + + process.start() + + process.join(timeout=ttl) + + if process.is_alive(): + process.terminate() + process.join() + raise TimeoutError(f"Failed to {func.func.__name__} after {ttl} seconds") + + # Raises an error if the queue is empty. This is fine. It means our subprocess timed out. + result = queue.get(block=False) + + # If we put an exception on the queue then raise instead of returning. + if isinstance(result, Exception): + raise result + if isinstance(result, BaseException): + raise Exception(f"BaseException raised in subprocess: {str(result)}") + + return result \ No newline at end of file diff --git a/neurons/miner.py b/neurons/miner.py index 8fca52f2..b1587253 100644 --- a/neurons/miner.py +++ b/neurons/miner.py @@ -1,26 +1,9 @@ -# The MIT License (MIT) -# Copyright © 2023 Yuma Rao -# TODO(developer): Set your name -# Copyright © 2023 - -# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated -# documentation files (the “Software”), to deal in the Software without restriction, including without limitation -# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, -# and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all copies or substantial portions of -# the Software. - -# THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO -# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL -# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -# DEALINGS IN THE SOFTWARE. import time import typing import bittensor as bt - +import datetime as dt +import os # Bittensor Miner Template: import cancer_ai @@ -43,8 +26,8 @@ def __init__(self, config=None): # TODO(developer): Anything specific to your use case you can do here async def forward( - self, synapse: template.protocol.Dummy - ) -> template.protocol.Dummy: + self, synapse: cancer_ai.protocol.Dummy + ) -> cancer_ai.protocol.Dummy: """ Processes the incoming 'Dummy' synapse by performing a predefined operation on the input data. This method should be replaced with actual logic relevant to the miner's purpose. @@ -63,7 +46,7 @@ async def forward( return synapse async def blacklist( - self, synapse: template.protocol.Dummy + self, synapse: cancer_ai.protocol.Dummy ) -> typing.Tuple[bool, str]: """ Determines whether an incoming request should be blacklisted and thus ignored. Your implementation should @@ -124,7 +107,7 @@ async def blacklist( ) return False, "Hotkey recognized!" - async def priority(self, synapse: template.protocol.Dummy) -> float: + async def priority(self, synapse: cancer_ai.protocol.Dummy) -> float: """ The priority function determines the order in which requests are handled. More valuable or higher-priority requests are processed before others. You should design your own priority mechanism with care. @@ -160,6 +143,46 @@ async def priority(self, synapse: template.protocol.Dummy) -> float: ) return priority +# miner moze brac udzial w kilku competitions, ale tylko w jednej na raz przy odpaleniu. config sprawdza flagą competition + + +async def compete(self, competition: str) -> None: + # Create a unique run id for this run. + run_id = dt.datetime.now().strftime("%Y-%m-%d_%H-%M-%S") + model_dir = model_path(self.config.save_model_dir, run_id) + os.makedirs(model_dir, exist_ok=True) + + model = await self.load_starting_model() + if model is None: + bt.logging.error("Failed to load the starting model. Exiting competition.") + return + + # TODO(Miner/Bruno): Implement the training logic here. + bt.logging.success(f"Saving model to path: {model_dir}.") + # TODO(Bruno): write the save_model_to_disk() function + # save_model_to_disk(model, model_dir) + + # TODO(Konrad): Push model to hugging face + # TODO(Konrad): Push model metadata to the chain + + +async def load_starting_model(self): + """Loads the model to train based on the provided config.""" + + # Check if we should load a model from a local directory. + if self.config.load_model_dir: + # TODO(Bruno): write the load_model_from_disk() function + # model = load_model_from_disk(config.load_model_dir) + # bt.logging.success(f"Training with model from disk. Model={str(model)}") + # return model + ... + # TODO(Bruno): possibly load model from scratch? if not handle the case where no model is loaded + +def model_path(base_dir: str, run_id: str) -> str: + """ + Constructs a file path for storing the model relating to a training run. + """ + return os.path.join(base_dir, "training", run_id) # This is the main function, which runs the miner. if __name__ == "__main__": @@ -167,3 +190,4 @@ async def priority(self, synapse: template.protocol.Dummy) -> float: while True: bt.logging.info(f"Miner running... {time.time()}") time.sleep(5) + From bd523f9f9a9c1fc91c50698bc6843338348d54da Mon Sep 17 00:00:00 2001 From: Konrad Date: Mon, 19 Aug 2024 14:52:20 +0200 Subject: [PATCH 022/227] commiting metadata on chain PoC --- cancer_ai/chain_models_store.py | 80 ++++++++++---------------------- cancer_ai/utils/config.py | 31 +++++++++++++ neurons/miner.py | 82 ++++++++++++++++++++------------- 3 files changed, 105 insertions(+), 88 deletions(-) diff --git a/cancer_ai/chain_models_store.py b/cancer_ai/chain_models_store.py index b72be792..fdaf3edf 100644 --- a/cancer_ai/chain_models_store.py +++ b/cancer_ai/chain_models_store.py @@ -3,65 +3,42 @@ import datetime from typing import ClassVar, Optional, Type -from utils import run_in_subprocess +from .utils.models_storage_utils import run_in_subprocess from pydantic import BaseModel, Field, PositiveInt -# The maximum bytes for metadata on the chain. -MAX_METADATA_BYTES = 128 -# The length, in bytes, of a git commit hash. -GIT_COMMIT_LENGTH = 40 -# The length, in bytes, of a base64 encoded sha256 hash. -SHA256_BASE_64_LENGTH = 44 -# The max length, in characters, of the competition id -MAX_COMPETITION_ID_LENGTH = 2 - -class ModelId(BaseModel): +class MinerModel(BaseModel): """Uniquely identifies a trained model""" - MAX_REPO_ID_LENGTH: ClassVar[int] = ( - MAX_METADATA_BYTES - - GIT_COMMIT_LENGTH - - SHA256_BASE_64_LENGTH - - MAX_COMPETITION_ID_LENGTH - - 4 # separators - ) - namespace: str = Field( description="Namespace where the model can be found. ex. Hugging Face username/org." ) name: str = Field(description="Name of the model.") - epoch: str = Field(description="The epoch number to submit as your checkpoint to evaluate e.g. 10") + epoch: int = Field(description="The epoch number to submit as your checkpoint to evaluate e.g. 10") date: datetime.datetime = Field(description="The datetime at which model was pushed to hugging face") - # When handling a model locally the commit and hash are not necessary. - # Commit must be filled when trying to download from a remote store. - commit: Optional[str] = Field( - description="Commit of the model. May be empty if not yet committed." - ) - # Hash is filled automatically when uploading to or downloading from a remote store. - hash: Optional[str] = Field(description="Hash of the trained model.") + block: Optional[str] = Field(description="Block on which this model was claimed on the chain.") + # Identifier for competition competition_id: Optional[str] = Field(description="The competition id") def to_compressed_str(self) -> str: """Returns a compressed string representation.""" - return f"{self.namespace}:{self.name}:{self.epoch}:{self.commit}:{self.hash}:{self.competition_id}" + return f"{self.namespace}:{self.name}:{self.epoch}:{self.competition_id}:{self.date}:{self.block}" @classmethod - def from_compressed_str(cls, cs: str) -> Type["ModelId"]: + def from_compressed_str(cls, cs: str) -> Type["MinerModel"]: """Returns an instance of this class from a compressed string representation""" tokens = cs.split(":") return cls( namespace=tokens[0], name=tokens[1], epoch=tokens[2] if tokens[2] != "None" else None, - commit=tokens[3] if tokens[3] != "None" else None, - hash=tokens[4] if tokens[4] != "None" else None, - competition_id=( - tokens[5] if len(tokens) >= 6 and tokens[5] != "None" else None - ), + date=tokens[3] if tokens[3] != "None" else None, + block=tokens[4] if tokens[4] != "None" else None, + competition_id=tokens[5] if tokens[5] != "None" else None, + block= ) @@ -71,17 +48,10 @@ class Model(BaseModel): class Config: arbitrary_types_allowed = True - id: ModelId = Field(description="Identifier for this model.") + id: MinerModel = Field(description="Identifier for this model.") local_repo_dir: str = Field(description="Local repository with the required files.") -class ModelMetadata(BaseModel): - id: ModelId = Field(description="Identifier for this trained model.") - block: PositiveInt = Field( - description="Block on which this model was claimed on the chain." - ) - - class ChainModelMetadataStore(): """Chain based implementation for storing and retrieving metadata about a model.""" @@ -97,7 +67,7 @@ def __init__( ) self.subnet_uid = subnet_uid - async def store_model_metadata(self, model_id: ModelId): + async def store_model_metadata(self, model_id: MinerModel): """Stores model metadata on this subnet for a specific wallet.""" if self.wallet is None: raise ValueError("No wallet available to write to the chain.") @@ -111,34 +81,32 @@ async def store_model_metadata(self, model_id: ModelId): ) run_in_subprocess(partial, 60) - async def retrieve_model_metadata(self, hotkey: str) -> Optional[ModelMetadata]: + async def retrieve_model_metadata(self, hotkey: str) -> Optional[MinerModel]: """Retrieves model metadata on this subnet for specific hotkey""" # Wrap calls to the subtensor in a subprocess with a timeout to handle potential hangs. - partial = functools.partial( - bt.extrinsics.serving.get_metadata, self.subtensor, self.subnet_uid, hotkey - ) - - metadata = run_in_subprocess(partial, 60) + # partial = functools.partial( + # bt.extrinsics.serving.get_metadata, self.subtensor, self.subnet_uid, hotkey + # ) + # metadata = run_in_subprocess(partial, 60) + metadata = bt.extrinsics.serving.get_metadata(self.subtensor, self.subnet_uid, hotkey) if not metadata: return None - + print("piwo", metadata["info"]["fields"]) commitment = metadata["info"]["fields"][0] hex_data = commitment[list(commitment.keys())[0]][2:] chain_str = bytes.fromhex(hex_data).decode() - model_id = None - + print("piwo", chain_str) try: - model_id = ModelId.from_compressed_str(chain_str) + model = MinerModel.from_compressed_str(chain_str) except: # If the metadata format is not correct on the chain then we return None. bt.logging.trace( f"Failed to parse the metadata on the chain for hotkey {hotkey}." ) return None - - model_metadata = ModelMetadata(id=model_id, block=metadata["block"]) - return model_metadata + model.block = metadata["block"] + return model diff --git a/cancer_ai/utils/config.py b/cancer_ai/utils/config.py index 895282c6..68cbbfd2 100644 --- a/cancer_ai/utils/config.py +++ b/cancer_ai/utils/config.py @@ -174,6 +174,37 @@ def add_miner_args(cls, parser): help="Wandb entity to log to.", ) + help="Path for storing trained model related to a training run.", + default="./models", + ) + + parser.add_argument( + "--models.load_model_dir", + type=str, + help="Path for for loading the starting model related to a training run.", + default="", + ) + + parser.add_argument( + "--models.namespace", + type=str, + help="Namespace where the model can be found.", + default="mock-namespace", + ) + + parser.add_argument( + "--models.model_name", + type=str, + help="Name of the model to push to hugging face.", + default="mock-name", + ) + + parser.add_argument( + "--models.epoch_checkpoint", + type=int, + help="The epoch number to submit as your checkpoint to evaluate e.g. 10", + default=10, + ) def add_validator_args(cls, parser): """Add validator specific arguments to the parser.""" diff --git a/neurons/miner.py b/neurons/miner.py index b1587253..a6045bba 100644 --- a/neurons/miner.py +++ b/neurons/miner.py @@ -4,11 +4,13 @@ import bittensor as bt import datetime as dt import os -# Bittensor Miner Template: +import datetime +import asyncio import cancer_ai # import base miner class which takes care of most of the boilerplate from cancer_ai.base.miner import BaseMinerNeuron +from cancer_ai.chain_models_store import MinerModel, ChainModelMetadataStore, ModelMetadata class Miner(BaseMinerNeuron): @@ -23,7 +25,9 @@ class Miner(BaseMinerNeuron): def __init__(self, config=None): super(Miner, self).__init__(config=config) - # TODO(developer): Anything specific to your use case you can do here + self.metadata_store = ChainModelMetadataStore(subtensor=self.subtensor, subnet_uid=126, wallet=self.wallet) + + asyncio.run(self.compete("mock_competition")) async def forward( self, synapse: cancer_ai.protocol.Dummy @@ -146,43 +150,57 @@ async def priority(self, synapse: cancer_ai.protocol.Dummy) -> float: # miner moze brac udzial w kilku competitions, ale tylko w jednej na raz przy odpaleniu. config sprawdza flagą competition -async def compete(self, competition: str) -> None: - # Create a unique run id for this run. - run_id = dt.datetime.now().strftime("%Y-%m-%d_%H-%M-%S") - model_dir = model_path(self.config.save_model_dir, run_id) - os.makedirs(model_dir, exist_ok=True) + async def compete(self, competition: str) -> None: + # Create a unique run id for this run. + run_id = dt.datetime.now().strftime("%Y-%m-%d_%H-%M-%S") + model_dir = os.path.join(self.config.models.save_model_dir, f"model_{run_id}") + os.makedirs(model_dir, exist_ok=True) + + # model = await self.load_starting_model() + # if model is None: + # bt.logging.error("Failed to load the starting model. Exiting competition.") + # return + + # TODO(Miner/Bruno): Implement the training logic here. + bt.logging.success(f"Saving model to path: {model_dir}.") + # TODO(Bruno): write the save_model_to_disk() function + # save_model_to_disk(model, model_dir) + + model_id = MinerModel(namespace=self.config.models.namespace, name=self.config.models.model_name, epoch=self.config.models.epoch_checkpoint, + date=datetime.datetime.now(), block=self.subtensor.block, competition_id=competition) + + # TODO(Konrad): Push model to hugging face + + await self.metadata_store.store_model_metadata(model_id) + bt.logging.success(f"Model successfully pushed model metadata on chain. Model ID: {model_id}") - model = await self.load_starting_model() - if model is None: - bt.logging.error("Failed to load the starting model. Exiting competition.") - return - - # TODO(Miner/Bruno): Implement the training logic here. - bt.logging.success(f"Saving model to path: {model_dir}.") - # TODO(Bruno): write the save_model_to_disk() function - # save_model_to_disk(model, model_dir) + time.sleep(10) - # TODO(Konrad): Push model to hugging face - # TODO(Konrad): Push model metadata to the chain + model_metadata: ModelMetadata = await self.metadata_store.retrieve_model_metadata(self.wallet.hotkey.ss58_address) + time.sleep(10) + print("MODEL METADATA......", model_metadata.id.name) + + -async def load_starting_model(self): - """Loads the model to train based on the provided config.""" + async def load_starting_model(self): + """Loads the model to train based on the provided config.""" - # Check if we should load a model from a local directory. - if self.config.load_model_dir: - # TODO(Bruno): write the load_model_from_disk() function - # model = load_model_from_disk(config.load_model_dir) - # bt.logging.success(f"Training with model from disk. Model={str(model)}") - # return model + # Check if we should load a model from a local directory. + if self.config.load_model_dir: + # TODO(Bruno): write the load_model_from_disk() function + # model = load_model_from_disk(config.load_model_dir) + # bt.logging.success(f"Training with model from disk. Model={str(model)}") + # return model + ... ... - # TODO(Bruno): possibly load model from scratch? if not handle the case where no model is loaded + # TODO(Bruno): possibly load model from scratch? if not handle the case where no model is loaded -def model_path(base_dir: str, run_id: str) -> str: - """ - Constructs a file path for storing the model relating to a training run. - """ - return os.path.join(base_dir, "training", run_id) + def model_path(base_dir: str, run_id: str) -> str: + """ + Constructs a file path for storing the model relating to a training run. + """ + return os.path.join(base_dir, "training", run_id) # This is the main function, which runs the miner. if __name__ == "__main__": From 81db375b29dda55ed05d7b5a7fab1c2920d63592 Mon Sep 17 00:00:00 2001 From: Konrad Date: Tue, 20 Aug 2024 08:15:12 +0200 Subject: [PATCH 023/227] store and retrieve metadata on chain PoC --- cancer_ai/chain_models_store.py | 32 ++++++------------- neurons/miner.py | 56 ++++++--------------------------- 2 files changed, 19 insertions(+), 69 deletions(-) diff --git a/cancer_ai/chain_models_store.py b/cancer_ai/chain_models_store.py index fdaf3edf..cf66aa30 100644 --- a/cancer_ai/chain_models_store.py +++ b/cancer_ai/chain_models_store.py @@ -18,14 +18,14 @@ class MinerModel(BaseModel): date: datetime.datetime = Field(description="The datetime at which model was pushed to hugging face") - block: Optional[str] = Field(description="Block on which this model was claimed on the chain.") - # Identifier for competition competition_id: Optional[str] = Field(description="The competition id") + block: Optional[str] = Field(description="Block on which this model was claimed on the chain.") + def to_compressed_str(self) -> str: """Returns a compressed string representation.""" - return f"{self.namespace}:{self.name}:{self.epoch}:{self.competition_id}:{self.date}:{self.block}" + return f"{self.namespace}:{self.name}:{self.epoch}:{self.competition_id}:{self.date}" @classmethod def from_compressed_str(cls, cs: str) -> Type["MinerModel"]: @@ -36,22 +36,9 @@ def from_compressed_str(cls, cs: str) -> Type["MinerModel"]: name=tokens[1], epoch=tokens[2] if tokens[2] != "None" else None, date=tokens[3] if tokens[3] != "None" else None, - block=tokens[4] if tokens[4] != "None" else None, - competition_id=tokens[5] if tokens[5] != "None" else None, - block= + competition_id=tokens[4] if tokens[4] != "None" else None, ) - -class Model(BaseModel): - """Represents a pre trained foundation model.""" - - class Config: - arbitrary_types_allowed = True - - id: MinerModel = Field(description="Identifier for this model.") - local_repo_dir: str = Field(description="Local repository with the required files.") - - class ChainModelMetadataStore(): """Chain based implementation for storing and retrieving metadata about a model.""" @@ -85,12 +72,11 @@ async def retrieve_model_metadata(self, hotkey: str) -> Optional[MinerModel]: """Retrieves model metadata on this subnet for specific hotkey""" # Wrap calls to the subtensor in a subprocess with a timeout to handle potential hangs. - # partial = functools.partial( - # bt.extrinsics.serving.get_metadata, self.subtensor, self.subnet_uid, hotkey - # ) + partial = functools.partial( + bt.extrinsics.serving.get_metadata, self.subtensor, self.subnet_uid, hotkey + ) - # metadata = run_in_subprocess(partial, 60) - metadata = bt.extrinsics.serving.get_metadata(self.subtensor, self.subnet_uid, hotkey) + metadata = run_in_subprocess(partial, 60) if not metadata: return None print("piwo", metadata["info"]["fields"]) @@ -99,7 +85,6 @@ async def retrieve_model_metadata(self, hotkey: str) -> Optional[MinerModel]: chain_str = bytes.fromhex(hex_data).decode() - print("piwo", chain_str) try: model = MinerModel.from_compressed_str(chain_str) except: @@ -108,5 +93,6 @@ async def retrieve_model_metadata(self, hotkey: str) -> Optional[MinerModel]: f"Failed to parse the metadata on the chain for hotkey {hotkey}." ) return None + # The block id at which the metadata is stored model.block = metadata["block"] return model diff --git a/neurons/miner.py b/neurons/miner.py index a6045bba..e190fe40 100644 --- a/neurons/miner.py +++ b/neurons/miner.py @@ -10,7 +10,7 @@ # import base miner class which takes care of most of the boilerplate from cancer_ai.base.miner import BaseMinerNeuron -from cancer_ai.chain_models_store import MinerModel, ChainModelMetadataStore, ModelMetadata +from cancer_ai.chain_models_store import MinerModel, ChainModelMetadataStore class Miner(BaseMinerNeuron): @@ -25,9 +25,9 @@ class Miner(BaseMinerNeuron): def __init__(self, config=None): super(Miner, self).__init__(config=config) - self.metadata_store = ChainModelMetadataStore(subtensor=self.subtensor, subnet_uid=126, wallet=self.wallet) + self.metadata_store = ChainModelMetadataStore(subtensor=self.subtensor, subnet_uid=163, wallet=self.wallet) - asyncio.run(self.compete("mock_competition")) + asyncio.run(self.store_and_retrieve_metadata_on_chain("mock_competition")) async def forward( self, synapse: cancer_ai.protocol.Dummy @@ -147,60 +147,24 @@ async def priority(self, synapse: cancer_ai.protocol.Dummy) -> float: ) return priority -# miner moze brac udzial w kilku competitions, ale tylko w jednej na raz przy odpaleniu. config sprawdza flagą competition - - - async def compete(self, competition: str) -> None: - # Create a unique run id for this run. - run_id = dt.datetime.now().strftime("%Y-%m-%d_%H-%M-%S") - model_dir = os.path.join(self.config.models.save_model_dir, f"model_{run_id}") - os.makedirs(model_dir, exist_ok=True) - - # model = await self.load_starting_model() - # if model is None: - # bt.logging.error("Failed to load the starting model. Exiting competition.") - # return - - # TODO(Miner/Bruno): Implement the training logic here. - bt.logging.success(f"Saving model to path: {model_dir}.") - # TODO(Bruno): write the save_model_to_disk() function - # save_model_to_disk(model, model_dir) + async def store_and_retrieve_metadata_on_chain(self, competition: str) -> None: + """ + PoC function to integrate with the structured business logic + """ model_id = MinerModel(namespace=self.config.models.namespace, name=self.config.models.model_name, epoch=self.config.models.epoch_checkpoint, - date=datetime.datetime.now(), block=self.subtensor.block, competition_id=competition) - - # TODO(Konrad): Push model to hugging face + date=datetime.datetime.now(), competition_id=competition, block=None) await self.metadata_store.store_model_metadata(model_id) bt.logging.success(f"Model successfully pushed model metadata on chain. Model ID: {model_id}") time.sleep(10) - model_metadata: ModelMetadata = await self.metadata_store.retrieve_model_metadata(self.wallet.hotkey.ss58_address) + model_metadata = await self.metadata_store.retrieve_model_metadata(self.wallet.hotkey.ss58_address) time.sleep(10) - print("MODEL METADATA......", model_metadata.id.name) - - - - async def load_starting_model(self): - """Loads the model to train based on the provided config.""" + print("Model Metadata name: ", model_metadata.id.name) - # Check if we should load a model from a local directory. - if self.config.load_model_dir: - # TODO(Bruno): write the load_model_from_disk() function - # model = load_model_from_disk(config.load_model_dir) - # bt.logging.success(f"Training with model from disk. Model={str(model)}") - # return model - ... - ... - # TODO(Bruno): possibly load model from scratch? if not handle the case where no model is loaded - - def model_path(base_dir: str, run_id: str) -> str: - """ - Constructs a file path for storing the model relating to a training run. - """ - return os.path.join(base_dir, "training", run_id) # This is the main function, which runs the miner. if __name__ == "__main__": From 166ed29b52fc45d7e8a233e444291cca8be097ff Mon Sep 17 00:00:00 2001 From: Wojtek Jurkowlaniec Date: Thu, 22 Aug 2024 00:07:25 +0200 Subject: [PATCH 024/227] miner work in progress --- neurons/miner2.py | 143 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 143 insertions(+) create mode 100644 neurons/miner2.py diff --git a/neurons/miner2.py b/neurons/miner2.py new file mode 100644 index 00000000..32b0ff54 --- /dev/null +++ b/neurons/miner2.py @@ -0,0 +1,143 @@ +import argparse + +import bittensor as bt +from dotenv import load_dotenv + +from cancer_ai.validator.utils import ModelType +from huggingface_hub import HfApi +import onnx + +help = """ +How to run it: + +python3 neurons/miner2.py \ + evaluate \ + --logging.debug \ + --model_type pytorch \ + --model_path /path/to/model \ + --competition_id "your_competition_id" + +python3 upload neurons/miner2.py \ + --model_path /path/to/model + --hf_repo_id "hf_org_id/your_hf_repo_id" + +python3 neurons/miner2.py \ + submit \ + --netuid 163 \ + --subtensor.network test \ + --wallet.name miner \ + --wallet.hotkey hot_validator \ + --model_type pytorch \ + --model_path /path/to/model +""" + + +def get_config() -> bt.config: + parser = argparse.ArgumentParser() + + # always required + parser.add_argument( + "--model_path", + type=str, + required=True, + help="Path to ONNX model, used for evaluation", + ) + parser.add_argument( + "--model_type", + type=str, + help="Type of model to use", + required=True, + choices=list(ModelType), + ) + parser.add_argument( + "--competition_id", + type=str, + help="Competition ID", + required=True, + ) + + # common arguments for subparsers + def add_hf_arguments(parser): + parser.add_argument( + "--hf_repo_id", + type=str, + default=None, + help="Hugging Face model repository ID", + ) + parser.add_argument( + "--hf_file_path", + type=str, + default=None, + help="Hugging Face model file path", + ) + + subparsers = parser.add_subparsers(required=True) + + subparser_evaluate = subparsers.add_parser("evaluate") + + subparser_upload = subparsers.add_parser("upload") + add_hf_arguments(subparser_upload) + + subparser_submit = subparsers.add_parser("submit") + add_hf_arguments(subparser_submit) + + bt.wallet.add_args(parser) + bt.subtensor.add_args(parser) + bt.logging.add_args(parser) + + config = bt.config(parser) + return config + + +async def test_model(config: bt.config, model_path: str, model_type: str): + pass + + +def upload_model_to_hf(self) -> None: + """Uploads model to Hugging Face.""" + hf_api = HfApi() + hf_api.upload_file( + path_or_fileobj=self.config.models.model_path, + ) + + +def is_it_onnx_model(model_path: str) -> None: + """Checks if model is an ONNX model.""" + model = onnx.load(model_path) + assert model is not None, "Failed to load model" + assert model.ir_version > 0, "Model is not an ONNX model" + + +async def main(config: bt.config) -> None: + bt.logging(config=config) + if not is_it_onnx_model(config.models.model_path): + bt.logging.error("Provided model with --model_type is not in ONNX format") + return + match config.action: + case "submit": + # The wallet holds the cryptographic key pairs for the miner. + bt.logging.info("Initializing connection with Bittensor subnet {config.netuid} - Safe-Scan Project") + bt.logging.info(f"Subtensor network: {config.subtensor.network}") + bt.logging.info(f"Wallet hotkey: {config.wallet.hotke.ss58_address}") + wallet = bt.wallet(config=config) + subtensor = bt.subtensor(config=config) + metagraph = subtensor.metagraph(config.netuid) + + # Start miner + bt.logging.info("Starting miner.") + case "evaluate": + pass + case "upload": + bt.logging.info("Uploading model to Hugging Face.") + bt.huggingface(config=config) + case _: + bt.logging.error(f"Unrecognized action: {action}") + + else: + bt.logging.error(f"Unrecognized action: {action}") + + +if __name__ == "__main__": + config = get_config() + load_dotenv() + main(config) From 4283840daf837843bcf22e93a53a00ec4d177111 Mon Sep 17 00:00:00 2001 From: Wojtek Jurkowlaniec Date: Thu, 22 Aug 2024 00:23:40 +0200 Subject: [PATCH 025/227] moved config --- neurons/miner2.py | 98 +++++------------------------------------ neurons/miner_config.py | 72 ++++++++++++++++++++++++++++++ 2 files changed, 82 insertions(+), 88 deletions(-) create mode 100644 neurons/miner_config.py diff --git a/neurons/miner2.py b/neurons/miner2.py index 32b0ff54..e5286e50 100644 --- a/neurons/miner2.py +++ b/neurons/miner2.py @@ -6,90 +6,12 @@ from cancer_ai.validator.utils import ModelType from huggingface_hub import HfApi import onnx +import asyncio -help = """ -How to run it: +from .miner_config import get_config -python3 neurons/miner2.py \ - evaluate \ - --logging.debug \ - --model_type pytorch \ - --model_path /path/to/model \ - --competition_id "your_competition_id" -python3 upload neurons/miner2.py \ - --model_path /path/to/model - --hf_repo_id "hf_org_id/your_hf_repo_id" - -python3 neurons/miner2.py \ - submit \ - --netuid 163 \ - --subtensor.network test \ - --wallet.name miner \ - --wallet.hotkey hot_validator \ - --model_type pytorch \ - --model_path /path/to/model -""" - - -def get_config() -> bt.config: - parser = argparse.ArgumentParser() - - # always required - parser.add_argument( - "--model_path", - type=str, - required=True, - help="Path to ONNX model, used for evaluation", - ) - parser.add_argument( - "--model_type", - type=str, - help="Type of model to use", - required=True, - choices=list(ModelType), - ) - parser.add_argument( - "--competition_id", - type=str, - help="Competition ID", - required=True, - ) - - # common arguments for subparsers - def add_hf_arguments(parser): - parser.add_argument( - "--hf_repo_id", - type=str, - default=None, - help="Hugging Face model repository ID", - ) - parser.add_argument( - "--hf_file_path", - type=str, - default=None, - help="Hugging Face model file path", - ) - - subparsers = parser.add_subparsers(required=True) - - subparser_evaluate = subparsers.add_parser("evaluate") - - subparser_upload = subparsers.add_parser("upload") - add_hf_arguments(subparser_upload) - - subparser_submit = subparsers.add_parser("submit") - add_hf_arguments(subparser_submit) - - bt.wallet.add_args(parser) - bt.subtensor.add_args(parser) - bt.logging.add_args(parser) - - config = bt.config(parser) - return config - - -async def test_model(config: bt.config, model_path: str, model_type: str): +async def test_model(config: bt.config, model_path: str): pass @@ -101,7 +23,7 @@ def upload_model_to_hf(self) -> None: ) -def is_it_onnx_model(model_path: str) -> None: +def is_onnx_model(model_path: str) -> None: """Checks if model is an ONNX model.""" model = onnx.load(model_path) assert model is not None, "Failed to load model" @@ -110,7 +32,8 @@ def is_it_onnx_model(model_path: str) -> None: async def main(config: bt.config) -> None: bt.logging(config=config) - if not is_it_onnx_model(config.models.model_path): + print(config) + if not is_onnx_model(config.model_path): bt.logging.error("Provided model with --model_type is not in ONNX format") return match config.action: @@ -131,13 +54,12 @@ async def main(config: bt.config) -> None: bt.logging.info("Uploading model to Hugging Face.") bt.huggingface(config=config) case _: - bt.logging.error(f"Unrecognized action: {action}") - - else: - bt.logging.error(f"Unrecognized action: {action}") + bt.logging.error(f"Unrecognized action: {config.action}") if __name__ == "__main__": config = get_config() + print(config) load_dotenv() - main(config) + asyncio.run(main(config)) + diff --git a/neurons/miner_config.py b/neurons/miner_config.py new file mode 100644 index 00000000..01192639 --- /dev/null +++ b/neurons/miner_config.py @@ -0,0 +1,72 @@ +import argparse + +import bittensor as bt + +help = """ +How to run it: + +python3 neurons/miner2.py \ + evaluate \ + --logging.debug \ + --model_path /path/to/model \ + --competition_id "your_competition_id" + +python3 upload neurons/miner2.py \ + --model_path /path/to/model + --hf_repo_id "hf_org_id/your_hf_repo_id" + +python3 neurons/miner2.py \ + submit \ + --netuid 163 \ + --subtensor.network test \ + --wallet.name miner \ + --wallet.hotkey hot_validator \ + --model_path /path/to/model +""" + + +def get_config() -> bt.config: + main_parser = argparse.ArgumentParser() + # always required + main_parser.add_argument( + "--model_path", + type=str, + required=True, + help="Path to ONNX model, used for evaluation", + ) + main_parser.add_argument( + "--competition_id", + type=str, + help="Competition ID", + required=True, + ) + # common arguments for subparsers + def add_hf_arguments(parser: argparse.ArgumentParser) -> None: + """Adds Hugging Face arguments to the parser.""" + parser.add_argument( + "--hf_repo_id", + type=str, + default=None, + help="Hugging Face model repository ID", + ) + parser.add_argument( + "--hf_file_path", + type=str, + default=None, + help="Hugging Face model file path", + ) + subparsers = main_parser.add_subparsers(title="action") + subparser_evaluate = subparsers.add_parser("evaluate") + + subparser_upload = subparsers.add_parser("upload") + add_hf_arguments(subparser_upload) + + subparser_submit = subparsers.add_parser("submit") + add_hf_arguments(subparser_submit) + + bt.wallet.add_args(main_parser) + bt.subtensor.add_args(main_parser) + bt.logging.add_args(main_parser) + + config = bt.config(main_parser) + return config From 08934600e87ffff9d0d0af37c01ea49cfd012646 Mon Sep 17 00:00:00 2001 From: Wojtek Jurkowlaniec Date: Thu, 22 Aug 2024 04:02:07 +0200 Subject: [PATCH 026/227] evaluating model locally from dataset --- .../validator/dataset_handlers/image_csv.py | 22 +++-- cancer_ai/validator/dataset_manager.py | 36 ++++++-- cancer_ai/validator/model_manager.py | 5 +- cancer_ai/validator/model_run_manager.py | 14 +-- .../validator/model_runners/onnx_runner.py | 13 +++ .../validator/model_runners/pytorch_runner.py | 4 +- .../model_runners/tensorflow_runner.py | 3 +- cancer_ai/validator/utils.py | 15 ++++ neurons/miner2.py | 59 ++++++++++--- neurons/miner_config.py | 86 +++++++++++++------ 10 files changed, 191 insertions(+), 66 deletions(-) create mode 100644 cancer_ai/validator/model_runners/onnx_runner.py diff --git a/cancer_ai/validator/dataset_handlers/image_csv.py b/cancer_ai/validator/dataset_handlers/image_csv.py index 6887b1b8..0e22ce76 100644 --- a/cancer_ai/validator/dataset_handlers/image_csv.py +++ b/cancer_ai/validator/dataset_handlers/image_csv.py @@ -4,11 +4,14 @@ from dataclasses import dataclass import csv import aiofiles +from pathlib import Path + +from ..utils import log_time @dataclass class ImageEntry: - filepath: str + relative_path: str is_melanoma: bool @@ -23,11 +26,13 @@ class DatasetImagesCSV(BaseDatasetHandler): ├── labels.csv """ - def __init__(self, config, path: str) -> None: + def __init__(self, config, dataset_path, label_path: str) -> None: self.config = config - self.label_path = path + self.dataset_path = dataset_path + self.label_path = label_path self.metadata_columns = ["filepath", "is_melanoma"] + @log_time async def sync_training_data(self): self.entries: List[ImageEntry] = [] # go over csv file @@ -38,10 +43,17 @@ async def sync_training_data(self): for row in reader: self.entries.append(ImageEntry(row[0], row[1])) + @log_time async def get_training_data(self) -> Tuple[List, List]: await self.sync_training_data() - print(self.entries) - pred_x = [Image.open(entry.filepath) for entry in self.entries] + pred_x = [ + Image.open( + str( + Path(self.dataset_path, entry.relative_path).resolve(), + ), + ) + for entry in self.entries + ] pred_y = [entry.is_melanoma for entry in self.entries] await self.process_training_data() return pred_x, pred_y diff --git a/cancer_ai/validator/dataset_manager.py b/cancer_ai/validator/dataset_manager.py index 3bbc28b7..609c5665 100644 --- a/cancer_ai/validator/dataset_manager.py +++ b/cancer_ai/validator/dataset_manager.py @@ -7,26 +7,27 @@ import bittensor as bt from .manager import SerializableManager -from .utils import run_command +from .utils import run_command, log_time from .dataset_handlers.image_csv import DatasetImagesCSV class DatasetManagerException(Exception): pass + class DatasetManager(SerializableManager): def __init__( self, config, competition_id: str, dataset_hf_id: str, file_hf_id: str ) -> None: """ Initializes a new instance of the DatasetManager class. - + Args: config: The configuration object. competition_id (str): The ID of the competition. dataset_hf_id (str): The Hugging Face ID of the dataset. file_hf_id (str): The Hugging Face ID of the file. - + Returns: None """ @@ -36,8 +37,9 @@ def __init__( self.file_hf_id = file_hf_id self.hf_api = HfApi() self.local_compressed_path = "" + print(self.config) self.local_extracted_dir = Path( - self.config.models.dataset_dir, self.competition_id + self.config.models_dataset_dir, self.competition_id ) self.data: Tuple[List, List] = () self.handler = None @@ -48,23 +50,37 @@ def get_state(self) -> dict: def set_state(self, state: dict): return {} - def download_dataset(self): + @log_time + async def download_dataset(self): if not os.path.exists(self.local_extracted_dir): os.makedirs(self.local_extracted_dir) self.local_compressed_path = self.hf_api.hf_hub_download( self.dataset_hf_id, self.file_hf_id, - cache_dir=Path(self.config.models.dataset_dir), + cache_dir=Path(self.config.models_dataset_dir), repo_type="dataset", ) def delete_dataset(self) -> None: """Delete dataset from disk""" - shutil.rmtree(self.local_compressed_path) + bt.logging.info("Deleting dataset: ") + + try: + shutil.rmtree(self.local_compressed_path) + bt.logging.info("Dataset deleted") + except OSError as e: + bt.logging.error(f"Failed to delete dataset from disk: {e}") + + @log_time async def unzip_dataset(self) -> None: """Unzip dataset""" + + self.local_extracted_dir = Path( + self.config.models_dataset_dir, self.competition_id + ) + print("Unzipping dataset", self.local_compressed_path) os.system(f"rm -R {self.local_extracted_dir}") await run_command( @@ -79,7 +95,9 @@ def set_dataset_handler(self) -> None: # is csv in directory if os.path.exists(Path(self.local_extracted_dir, "labels.csv")): self.handler = DatasetImagesCSV( - self.config, Path(self.local_extracted_dir, "labels.csv") + self.config, + self.local_extracted_dir, + Path(self.local_extracted_dir, "labels.csv"), ) else: print("Files in dataset: ", os.listdir(self.local_extracted_dir)) @@ -89,7 +107,7 @@ async def prepare_dataset(self) -> None: """Download dataset, unzip and set dataset handler""" bt.logging.info("Downloading dataset") - self.download_dataset() + await self.download_dataset() bt.logging.info("Unzipping dataset") await self.unzip_dataset() bt.logging.info("Setting dataset handler") diff --git a/cancer_ai/validator/model_manager.py b/cancer_ai/validator/model_manager.py index b4dbe69f..effe8912 100644 --- a/cancer_ai/validator/model_manager.py +++ b/cancer_ai/validator/model_manager.py @@ -9,8 +9,8 @@ @dataclass class ModelInfo: - repo_id: str - filename: str + repo_id: str | None = None + filename: str | None = None file_path: str | None = None model_type: str | None = None @@ -48,6 +48,7 @@ def download_miner_model(self, hotkey) -> None: cache_dir=self.config.models.model_dir, repo_type="space", ) + def add_model(self, hotkey, repo_id, filename) -> None: """Saves locally information about a new model.""" diff --git a/cancer_ai/validator/model_run_manager.py b/cancer_ai/validator/model_run_manager.py index 79b30d6a..69e6d160 100644 --- a/cancer_ai/validator/model_run_manager.py +++ b/cancer_ai/validator/model_run_manager.py @@ -5,11 +5,13 @@ from .utils import detect_model_format, ModelType from .model_runners.pytorch_runner import PytorchRunnerHandler from .model_runners.tensorflow_runner import TensorflowRunnerHandler +from .model_runners.onnx_runner import ONNXRunnerHandler MODEL_TYPE_HANDLERS = { ModelType.PYTORCH: PytorchRunnerHandler, ModelType.TENSORFLOW_SAVEDMODEL: TensorflowRunnerHandler, + ModelType.ONNX: ONNXRunnerHandler, } @@ -26,16 +28,14 @@ def set_state(self, state: dict): pass def set_runner_handler(self) -> None: - """ - Sets the model runner handler based on the model type. - """ + """Sets the model runner handler based on the model type.""" - model_type = detect_model_format(self.model) - # initializing ml model handler object + model_type = detect_model_format(self.model.file_path) + # Initializing ml model handler object model_handler = MODEL_TYPE_HANDLERS[model_type] self.handler = model_handler(self.config, self.model.file_path) - def run(self, pred_x: List) -> List: + async def run(self, pred_x: List) -> List: """ Run the model with the given input. @@ -43,5 +43,5 @@ def run(self, pred_x: List) -> List: List: model predictions """ - model_predictions = self.handler.run(pred_x) + model_predictions = await self.handler.run(pred_x) return model_predictions diff --git a/cancer_ai/validator/model_runners/onnx_runner.py b/cancer_ai/validator/model_runners/onnx_runner.py new file mode 100644 index 00000000..74143d14 --- /dev/null +++ b/cancer_ai/validator/model_runners/onnx_runner.py @@ -0,0 +1,13 @@ +from . import BaseRunnerHandler +from typing import List +from ..utils import log_time + + +class ONNXRunnerHandler(BaseRunnerHandler): + @log_time + async def run(self, pred_x: List) -> List: + # example, might not work + import random + + output = [random.randrange(0, 1) for _ in range(len(pred_x))] + return output diff --git a/cancer_ai/validator/model_runners/pytorch_runner.py b/cancer_ai/validator/model_runners/pytorch_runner.py index 4de57864..9862ca61 100644 --- a/cancer_ai/validator/model_runners/pytorch_runner.py +++ b/cancer_ai/validator/model_runners/pytorch_runner.py @@ -1,8 +1,9 @@ from . import BaseRunnerHandler from typing import List + class PytorchRunnerHandler(BaseRunnerHandler): - def run(self, pred_x: List) -> List: + async def run(self, pred_x: List) -> List: # example, might not work from torch import load @@ -10,4 +11,3 @@ def run(self, pred_x: List) -> List: model.eval() output = model(pred_x) return output - diff --git a/cancer_ai/validator/model_runners/tensorflow_runner.py b/cancer_ai/validator/model_runners/tensorflow_runner.py index 4a87190f..a76e0cd5 100644 --- a/cancer_ai/validator/model_runners/tensorflow_runner.py +++ b/cancer_ai/validator/model_runners/tensorflow_runner.py @@ -1,6 +1,7 @@ from . import BaseRunnerHandler from typing import List + class TensorflowRunnerHandler(BaseRunnerHandler): - def run(self, pred_x: List) -> List: + async def run(self, pred_x: List) -> List: return [] diff --git a/cancer_ai/validator/utils.py b/cancer_ai/validator/utils.py index 43eeff6e..b7aa3b69 100644 --- a/cancer_ai/validator/utils.py +++ b/cancer_ai/validator/utils.py @@ -1,6 +1,7 @@ from enum import Enum import os import asyncio +import bittensor as bt class ModelType(Enum): @@ -13,6 +14,20 @@ class ModelType(Enum): UNKNOWN = "Unknown format" +import time +from functools import wraps + +def log_time(func): + @wraps(func) + async def wrapper(*args, **kwargs): + start_time = time.time() + result = await func(*args, **kwargs) + end_time = time.time() + module_name = func.__module__ + bt.logging.debug(f"'{module_name}.{func.__name__}' took {end_time - start_time:.4f}s") + return result + return wrapper + def detect_model_format(file_path) -> ModelType: _, ext = os.path.splitext(file_path) diff --git a/neurons/miner2.py b/neurons/miner2.py index e5286e50..05b7408c 100644 --- a/neurons/miner2.py +++ b/neurons/miner2.py @@ -1,14 +1,18 @@ import argparse +import sys import bittensor as bt from dotenv import load_dotenv - -from cancer_ai.validator.utils import ModelType from huggingface_hub import HfApi import onnx import asyncio -from .miner_config import get_config +from neurons.miner_config import get_config + +from cancer_ai.validator.utils import ModelType +from cancer_ai.validator.model_run_manager import ModelRunManager, ModelInfo +from cancer_ai.validator.dataset_manager import DatasetManager +from cancer_ai.validator.model_manager import ModelManager async def test_model(config: bt.config, model_path: str): @@ -25,21 +29,51 @@ def upload_model_to_hf(self) -> None: def is_onnx_model(model_path: str) -> None: """Checks if model is an ONNX model.""" - model = onnx.load(model_path) - assert model is not None, "Failed to load model" - assert model.ir_version > 0, "Model is not an ONNX model" + try: + onnx.checker.check_model(model_path) + except onnx.checker.ValidationError as e: + print(e) + return False + return True + + +async def evaluate_model(config: bt.config) -> None: + bt.logging.info("Evaluate model mode") + run_manager = ModelRunManager( + config=config, model=ModelInfo(file_path=config.model_path) + ) + dataset_manager = DatasetManager( + config, + config.competition_id, + "safescanai/test_dataset", + "skin_melanoma.zip", + ) + await dataset_manager.prepare_dataset() + + pred_x, pred_y = await dataset_manager.get_data() + + model_predictions = await run_manager.run(pred_x) + + print(pred_y) + print(model_predictions) + + if config.clean_after_run: + dataset_manager.delete_dataset() async def main(config: bt.config) -> None: bt.logging(config=config) - print(config) + if not is_onnx_model(config.model_path): bt.logging.error("Provided model with --model_type is not in ONNX format") return + match config.action: - case "submit": + case "submit": # The wallet holds the cryptographic key pairs for the miner. - bt.logging.info("Initializing connection with Bittensor subnet {config.netuid} - Safe-Scan Project") + bt.logging.info( + "Initializing connection with Bittensor subnet {config.netuid} - Safe-Scan Project" + ) bt.logging.info(f"Subtensor network: {config.subtensor.network}") bt.logging.info(f"Wallet hotkey: {config.wallet.hotke.ss58_address}") wallet = bt.wallet(config=config) @@ -48,8 +82,9 @@ async def main(config: bt.config) -> None: # Start miner bt.logging.info("Starting miner.") - case "evaluate": - pass + case "evaluate": + await evaluate_model(config) + case "upload": bt.logging.info("Uploading model to Hugging Face.") bt.huggingface(config=config) @@ -59,7 +94,5 @@ async def main(config: bt.config) -> None: if __name__ == "__main__": config = get_config() - print(config) load_dotenv() asyncio.run(main(config)) - diff --git a/neurons/miner_config.py b/neurons/miner_config.py index 01192639..22c3312a 100644 --- a/neurons/miner_config.py +++ b/neurons/miner_config.py @@ -23,50 +23,82 @@ --wallet.hotkey hot_validator \ --model_path /path/to/model """ +import argparse +import bittensor as bt + +import argparse def get_config() -> bt.config: main_parser = argparse.ArgumentParser() - # always required + + main_parser.add_argument( + "--action", choices=["submit", "evaluate", "upload"], + # required=True, + default="evaluate", + ) main_parser.add_argument( "--model_path", type=str, - required=True, + # required=True, help="Path to ONNX model, used for evaluation", + default="neurons/simple_cnn_model.onnx", ) main_parser.add_argument( "--competition_id", type=str, + # required=True, help="Competition ID", - required=True, + default="your_competition_id", + ) + + main_parser.add_argument( + "--models_dataset_dir", + type=str, + help="Path for storing datasets.", + default="./datasets", + ) + # Subparser for upload command + + main_parser.add_argument( + "--hf_repo_id", + type=str, + required=False, + help="Hugging Face model repository ID", + ) + main_parser.add_argument( + "--hf_file_path", + type=str, + help="Hugging Face model file path", ) - # common arguments for subparsers - def add_hf_arguments(parser: argparse.ArgumentParser) -> None: - """Adds Hugging Face arguments to the parser.""" - parser.add_argument( - "--hf_repo_id", - type=str, - default=None, - help="Hugging Face model repository ID", - ) - parser.add_argument( - "--hf_file_path", - type=str, - default=None, - help="Hugging Face model file path", - ) - subparsers = main_parser.add_subparsers(title="action") - subparser_evaluate = subparsers.add_parser("evaluate") - - subparser_upload = subparsers.add_parser("upload") - add_hf_arguments(subparser_upload) - - subparser_submit = subparsers.add_parser("submit") - add_hf_arguments(subparser_submit) + main_parser.add_argument( + "--clean-after-run", + action="store_true", + help="Whether to clean up (dataset, temporary files) after running", + default=False, + ) + + # Add additional args from bt modules bt.wallet.add_args(main_parser) bt.subtensor.add_args(main_parser) bt.logging.add_args(main_parser) - config = bt.config(main_parser) + # Parse the arguments and return the config + # config = bt.config(main_parser) + # parsed = main_parser.parse_args() + # config = bt.config(main_parser) + # print(config) + + config = main_parser.parse_args() + # print(config) + config.logging_dir = "./" + config.record_log = True + config.trace = False + config.debug = True return config + + +if __name__ == "__main__": + config = get_config() + print(config) From 101b7b41ff188e36817ce83a18a29252e5562a1d Mon Sep 17 00:00:00 2001 From: Wojtek Jurkowlaniec Date: Thu, 22 Aug 2024 04:38:49 +0200 Subject: [PATCH 027/227] miner manager + override logging formats --- cancer_ai/validator/dataset_manager.py | 17 +++-- neurons/miner2.py | 98 ------------------------ neurons/miner3.py | 102 +++++++++++++++++++++++++ neurons/miner_config.py | 34 ++++++++- 4 files changed, 142 insertions(+), 109 deletions(-) delete mode 100644 neurons/miner2.py create mode 100644 neurons/miner3.py diff --git a/cancer_ai/validator/dataset_manager.py b/cancer_ai/validator/dataset_manager.py index 609c5665..cf6c8e45 100644 --- a/cancer_ai/validator/dataset_manager.py +++ b/cancer_ai/validator/dataset_manager.py @@ -81,7 +81,8 @@ async def unzip_dataset(self) -> None: self.config.models_dataset_dir, self.competition_id ) - print("Unzipping dataset", self.local_compressed_path) + bt.logging.info(f"Unzipping dataset '{self.competition_id}'") + bt.logging.debug(f"Dataset extracted to: { self.local_compressed_path}") os.system(f"rm -R {self.local_extracted_dir}") await run_command( f"unzip {self.local_compressed_path} -d {self.local_extracted_dir}" @@ -91,7 +92,7 @@ async def unzip_dataset(self) -> None: def set_dataset_handler(self) -> None: """Detect dataset type and set handler""" if not self.local_compressed_path: - raise DatasetManagerException("Dataset not downloaded") + raise DatasetManagerException(f"Dataset '{self.competition_id}' not downloaded") # is csv in directory if os.path.exists(Path(self.local_extracted_dir, "labels.csv")): self.handler = DatasetImagesCSV( @@ -105,18 +106,18 @@ def set_dataset_handler(self) -> None: async def prepare_dataset(self) -> None: """Download dataset, unzip and set dataset handler""" - - bt.logging.info("Downloading dataset") + bt.logging.info(f"Preparing dataset '{self.competition_id}'") + bt.logging.info(f"Downloading dataset '{self.competition_id}'") await self.download_dataset() - bt.logging.info("Unzipping dataset") + bt.logging.info(f"Unzipping dataset '{self.competition_id}'") await self.unzip_dataset() - bt.logging.info("Setting dataset handler") + bt.logging.info(f"Setting dataset handler '{self.competition_id}'") self.set_dataset_handler() - bt.logging.info("Preprocessing dataset") + bt.logging.info(f"Preprocessing dataset '{self.competition_id}'") self.data = await self.handler.get_training_data() async def get_data(self) -> Tuple[List, List]: """Get data from dataset handler""" if not self.data: - raise DatasetManagerException("Dataset not initalized ") + raise DatasetManagerException(f"Dataset '{self.competition_id}' not initalized ") return self.data diff --git a/neurons/miner2.py b/neurons/miner2.py deleted file mode 100644 index 05b7408c..00000000 --- a/neurons/miner2.py +++ /dev/null @@ -1,98 +0,0 @@ -import argparse -import sys - -import bittensor as bt -from dotenv import load_dotenv -from huggingface_hub import HfApi -import onnx -import asyncio - -from neurons.miner_config import get_config - -from cancer_ai.validator.utils import ModelType -from cancer_ai.validator.model_run_manager import ModelRunManager, ModelInfo -from cancer_ai.validator.dataset_manager import DatasetManager -from cancer_ai.validator.model_manager import ModelManager - - -async def test_model(config: bt.config, model_path: str): - pass - - -def upload_model_to_hf(self) -> None: - """Uploads model to Hugging Face.""" - hf_api = HfApi() - hf_api.upload_file( - path_or_fileobj=self.config.models.model_path, - ) - - -def is_onnx_model(model_path: str) -> None: - """Checks if model is an ONNX model.""" - try: - onnx.checker.check_model(model_path) - except onnx.checker.ValidationError as e: - print(e) - return False - return True - - -async def evaluate_model(config: bt.config) -> None: - bt.logging.info("Evaluate model mode") - run_manager = ModelRunManager( - config=config, model=ModelInfo(file_path=config.model_path) - ) - dataset_manager = DatasetManager( - config, - config.competition_id, - "safescanai/test_dataset", - "skin_melanoma.zip", - ) - await dataset_manager.prepare_dataset() - - pred_x, pred_y = await dataset_manager.get_data() - - model_predictions = await run_manager.run(pred_x) - - print(pred_y) - print(model_predictions) - - if config.clean_after_run: - dataset_manager.delete_dataset() - - -async def main(config: bt.config) -> None: - bt.logging(config=config) - - if not is_onnx_model(config.model_path): - bt.logging.error("Provided model with --model_type is not in ONNX format") - return - - match config.action: - case "submit": - # The wallet holds the cryptographic key pairs for the miner. - bt.logging.info( - "Initializing connection with Bittensor subnet {config.netuid} - Safe-Scan Project" - ) - bt.logging.info(f"Subtensor network: {config.subtensor.network}") - bt.logging.info(f"Wallet hotkey: {config.wallet.hotke.ss58_address}") - wallet = bt.wallet(config=config) - subtensor = bt.subtensor(config=config) - metagraph = subtensor.metagraph(config.netuid) - - # Start miner - bt.logging.info("Starting miner.") - case "evaluate": - await evaluate_model(config) - - case "upload": - bt.logging.info("Uploading model to Hugging Face.") - bt.huggingface(config=config) - case _: - bt.logging.error(f"Unrecognized action: {config.action}") - - -if __name__ == "__main__": - config = get_config() - load_dotenv() - asyncio.run(main(config)) diff --git a/neurons/miner3.py b/neurons/miner3.py new file mode 100644 index 00000000..6dbb740b --- /dev/null +++ b/neurons/miner3.py @@ -0,0 +1,102 @@ +import argparse +import sys +import asyncio +from typing import Optional + +import bittensor as bt +from dotenv import load_dotenv +from huggingface_hub import HfApi +import onnx + +from neurons.miner_config import get_config, set_log_formatting +from cancer_ai.validator.utils import ModelType +from cancer_ai.validator.model_run_manager import ModelRunManager, ModelInfo +from cancer_ai.validator.dataset_manager import DatasetManager +from cancer_ai.validator.model_manager import ModelManager + + +class ModelManagerCLI: + def __init__(self, config: bt.config): + self.config = config + self.hf_api = HfApi() + + async def test_model(self, model_path: str) -> None: + # Placeholder for actual implementation + pass + + def upload_model_to_hf(self) -> None: + """Uploads model to Hugging Face.""" + bt.logging.info("Uploading model to Hugging Face.") + path = self.hf_api.upload_file( + path_or_fileobj=self.config.models.model_path, + ) + bt.logging.info(f"Uploaded model to Hugging Face: {path}") + + @staticmethod + def is_onnx_model(model_path: str) -> bool: + """Checks if model is an ONNX model.""" + try: + onnx.checker.check_model(model_path) + except onnx.checker.ValidationError as e: + bt.logging.warning(e) + return False + return True + + async def evaluate_model(self) -> None: + bt.logging.info("Evaluate model mode") + run_manager = ModelRunManager( + config=self.config, model=ModelInfo(file_path=self.config.model_path) + ) + dataset_manager = DatasetManager( + self.config, + self.config.competition_id, + "safescanai/test_dataset", + "skin_melanoma.zip", + ) + await dataset_manager.prepare_dataset() + + pred_x, pred_y = await dataset_manager.get_data() + + model_predictions = await run_manager.run(pred_x) + + print(pred_y) + print(model_predictions) + + if self.config.clean_after_run: + dataset_manager.delete_dataset() + + async def submit_model(self) -> None: + # The wallet holds the cryptographic key pairs for the miner. + bt.logging.info( + f"Initializing connection with Bittensor subnet {self.config.netuid} - Safe-Scan Project" + ) + bt.logging.info(f"Subtensor network: {self.config.subtensor.network}") + bt.logging.info(f"Wallet hotkey: {self.config.wallet.hotkey.ss58_address}") + wallet = bt.wallet(config=self.config) + subtensor = bt.subtensor(config=self.config) + metagraph = subtensor.metagraph(self.config.netuid) + + async def main(self) -> None: + bt.logging(config=self.config) + + if not self.is_onnx_model(self.config.model_path): + bt.logging.error("Provided model with --model_type is not in ONNX format") + return + + match self.config.action: + case "submit": + await self.submit_model() + case "evaluate": + await self.evaluate_model() + case "upload": + self.upload_model_to_hf() + case _: + bt.logging.error(f"Unrecognized action: {self.config.action}") + + +if __name__ == "__main__": + config = get_config() + set_log_formatting() + load_dotenv() + cli_manager = ModelManagerCLI(config) + asyncio.run(cli_manager.main()) \ No newline at end of file diff --git a/neurons/miner_config.py b/neurons/miner_config.py index 22c3312a..6e2924a8 100644 --- a/neurons/miner_config.py +++ b/neurons/miner_config.py @@ -1,6 +1,9 @@ import argparse +from colorama import init, Fore, Back, Style import bittensor as bt +from bittensor.btlogging import format + help = """ How to run it: @@ -29,11 +32,36 @@ import argparse +def set_log_formatting() -> None: + """Override bittensor logging formats.""" + + + format.LOG_TRACE_FORMATS = { + level: f"{Fore.BLUE}%(asctime)s{Fore.RESET}" + f" | {Style.BRIGHT}{color}%(levelname)s{Fore.RESET}{Back.RESET}{Style.RESET_ALL}" + f" |%(message)s" + for level, color in format.log_level_color_prefix.items() + } + + format.DEFAULT_LOG_FORMAT = ( + f"{Fore.BLUE}%(asctime)s{Fore.RESET} | " + f"{Style.BRIGHT}{Fore.WHITE}%(levelname)s{Style.RESET_ALL} | " + "%(message)s" + ) + + format.DEFAULT_TRACE_FORMAT = ( + f"{Fore.BLUE}%(asctime)s{Fore.RESET} | " + f"{Style.BRIGHT}{Fore.WHITE}%(levelname)s{Style.RESET_ALL} | " + f" %(message)s" + ) + + def get_config() -> bt.config: main_parser = argparse.ArgumentParser() main_parser.add_argument( - "--action", choices=["submit", "evaluate", "upload"], + "--action", + choices=["submit", "evaluate", "upload"], # required=True, default="evaluate", ) @@ -94,8 +122,8 @@ def get_config() -> bt.config: # print(config) config.logging_dir = "./" config.record_log = True - config.trace = False - config.debug = True + config.trace = True + config.debug = False return config From f21a14b30b5561237c5491a4b0d4aea5aa8cf525 Mon Sep 17 00:00:00 2001 From: Wojtek Jurkowlaniec Date: Fri, 23 Aug 2024 03:48:03 +0200 Subject: [PATCH 028/227] support for multiple competitions --- cancer_ai/validator/model_manager.py | 6 +- cancer_ai/validator/scripts/__init__.py | 0 neurons/competition_config.py | 10 ++++ neurons/miner3.py | 4 +- {cancer_ai => neurons}/validator_tester.py | 65 ++++++++++++---------- 5 files changed, 51 insertions(+), 34 deletions(-) create mode 100644 cancer_ai/validator/scripts/__init__.py create mode 100644 neurons/competition_config.py rename {cancer_ai => neurons}/validator_tester.py (51%) diff --git a/cancer_ai/validator/model_manager.py b/cancer_ai/validator/model_manager.py index effe8912..68d6c6ab 100644 --- a/cancer_ai/validator/model_manager.py +++ b/cancer_ai/validator/model_manager.py @@ -19,8 +19,8 @@ class ModelManager(SerializableManager): def __init__(self, config) -> None: self.config = config - if not os.path.exists(self.config.models.model_dir): - os.makedirs(self.config.models.model_dir) + if not os.path.exists(self.config.model_dir): + os.makedirs(self.config.model_dir) self.api = HfApi() self.hotkey_store = {} @@ -45,7 +45,7 @@ def download_miner_model(self, hotkey) -> None: model_info.file_path = self.api.hf_hub_download( model_info.repo_id, model_info.filename, - cache_dir=self.config.models.model_dir, + cache_dir=self.config.model_dir, repo_type="space", ) diff --git a/cancer_ai/validator/scripts/__init__.py b/cancer_ai/validator/scripts/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/neurons/competition_config.py b/neurons/competition_config.py new file mode 100644 index 00000000..dbe0511b --- /dev/null +++ b/neurons/competition_config.py @@ -0,0 +1,10 @@ + +competitions = [ + { + "competition_id": "melanoma-1", + "category": "skin", + "evaluation_time": ["12:30", "15:30"], + "dataset_hf_id": "safescanai/test_dataset", + "file_hf_id": "skin_melanoma.zip", + } +] \ No newline at end of file diff --git a/neurons/miner3.py b/neurons/miner3.py index 6dbb740b..87f1d3f0 100644 --- a/neurons/miner3.py +++ b/neurons/miner3.py @@ -15,7 +15,7 @@ from cancer_ai.validator.model_manager import ModelManager -class ModelManagerCLI: +class MinerManagerCLI: def __init__(self, config: bt.config): self.config = config self.hf_api = HfApi() @@ -98,5 +98,5 @@ async def main(self) -> None: config = get_config() set_log_formatting() load_dotenv() - cli_manager = ModelManagerCLI(config) + cli_manager = MinerManagerCLI(config) asyncio.run(cli_manager.main()) \ No newline at end of file diff --git a/cancer_ai/validator_tester.py b/neurons/validator_tester.py similarity index 51% rename from cancer_ai/validator_tester.py rename to neurons/validator_tester.py index 91665e7b..a16d38e1 100644 --- a/cancer_ai/validator_tester.py +++ b/neurons/validator_tester.py @@ -1,4 +1,4 @@ -from validator.competition_manager import CompetitionManager +from cancer_ai.validator.competition_manager import CompetitionManager from datetime import time, datetime import asyncio @@ -6,15 +6,19 @@ import timeit from types import SimpleNamespace -# from cancer_ai.utils.config import config +from competition_config import competitions +# from cancer_ai.utils.config import config -path_config = { - "models": SimpleNamespace(**{"model_dir": "/tmp/models", "dataset_dir": "/tmp/datasets"}), -} +path_config = {"model_dir": "/tmp/models", "models_dataset_dir": "/tmp/datasets"} +# path_config = { +# "models": SimpleNamespace( +# **{"model_dir": "/tmp/models", "models_dataset_dir": "/tmp/datasets"} +# ), +# } path_config = SimpleNamespace(**path_config) -# open competition config file +# open competition config file # def get_competition_config(): # with open(config.validator.competition_config_path) as f: # return json.load(f) @@ -43,30 +47,33 @@ # await asyncio.sleep(60) -competition_config = [ - { - "competition_id": "melanoma-1", - "category": "skin", - "evaluation_time": ["12:30", "15:30"], - "dataset_hf_id": "vidhiparikh/House-Price-Estimator", - "file_hf_id": "model_custom.pkcls", - } -] - - # def run(): - # asyncio.run(main_loop()) - +# asyncio.run(main_loop()) if __name__ == "__main__": - competition = CompetitionManager( - path_config, - "melaona-1", - "skin", - ["12:30", "15:30"], - "safescanai/test_dataset", - "skin_melanoma.zip", + competition_managers_async = [] + for competition_config in competitions: + competition_manager = CompetitionManager( + path_config, + competition_config["competition_id"], + competition_config["category"], + competition_config["evaluation_time"], + competition_config["dataset_hf_id"], + competition_config["file_hf_id"], + ) + competition_managers_async.append(competition_manager) + competition_managers_async_gathered = asyncio.gather( + [ + competition_manager.evaluate() + for competition_manager in competition_managers_async + ] ) - asyncio.run(competition.evaluate()) - # await competition.evaluate() - print(competition.results) + loop = asyncio.get_event_loop() + try: + loop.run_until_complete(competition_managers_async_gathered) + finally: + loop.close() + for competition_manager in competition_managers_async: + print( + f"Results of competition {competition_manager.competition_id}: {competition_manager.results}" + ) From 5afc1014876471fc209fda96ae5d7cd2602a1469 Mon Sep 17 00:00:00 2001 From: Wojtek Jurkowlaniec Date: Fri, 23 Aug 2024 04:15:53 +0200 Subject: [PATCH 029/227] multi competition scheduler and debug runner --- .gitignore | 3 + cancer_ai/validator/competition_manager.py | 24 ++--- cancer_ai/validator/model_manager.py | 4 +- neurons/competition_config.py | 2 +- neurons/competition_runner.py | 114 +++++++++++++++++++++ neurons/miner_config.py | 2 +- neurons/validator_tester.py | 79 -------------- 7 files changed, 129 insertions(+), 99 deletions(-) create mode 100644 neurons/competition_runner.py delete mode 100644 neurons/validator_tester.py diff --git a/.gitignore b/.gitignore index 72a70c3a..861e8d0f 100644 --- a/.gitignore +++ b/.gitignore @@ -163,3 +163,6 @@ testing/ # Editors .vscode/settings.json + + +datasets \ No newline at end of file diff --git a/cancer_ai/validator/competition_manager.py b/cancer_ai/validator/competition_manager.py index e228d2ed..6227f244 100644 --- a/cancer_ai/validator/competition_manager.py +++ b/cancer_ai/validator/competition_manager.py @@ -34,36 +34,26 @@ def __init__( config, competition_id: str, category: str, - evaluation_times: list[str], dataset_hf_id: str, file_hf_id: str, ) -> None: """ - Initializes a CompetitionManager instance. + Responsible for managing a competition. Args: config (dict): Config dictionary. competition_id (str): Unique identifier for the competition. category (str): Category of the competition. - evaluation_time (list[str]): List of times of a day at which the competition will be evaluated in XX:XX format. - - Note: Times are in UTC time. """ bt.logging.info(f"Initializing Competition: {competition_id}") self.config = config self.competition_id = competition_id self.category = category + self.results = [] self.model_manager = ModelManager(config) - - # self.evaluation_time = [ - # time(hour_min.split(":")[0], hour_min.split(":")[1]) - # for hour_min in evaluation_times - # ] self.dataset_manager = DatasetManager( config, competition_id, dataset_hf_id, file_hf_id ) - # self.model_evaluator = - self.results = [] def get_state(self): return { @@ -77,15 +67,17 @@ def set_state(self, state: dict): self.competition_id = state["competition_id"] self.model_manager.set_state(state["model_manager"]) self.category = state["category"] - self.evaluation_time = state["evaluation_time"] async def get_miner_model(self, hotkey): # TODO get real data - return ModelInfo("vidhiparikh/House-Price-Estimator", "model_custom.pkcls") + return ModelInfo("safescanai/test_dataset", "simple_cnn_model.onnx") async def init_evaluation(self): # get models from chain - for hotkey in self.model_manager.hotkey_store: + hotkeys = [ + "example_hotkey", + ] + for hotkey in hotkeys: self.model_manager.hotkey_store[hotkey] = await self.get_miner_model(hotkey) await self.dataset_manager.prepare_dataset() @@ -102,7 +94,7 @@ async def evaluate(self): model_manager = ModelRunManager( self.config, self.model_manager.hotkey_store[hotkey] ) - model_pred_y = model_manager.run(pred_x) + model_pred_y = await model_manager.run(pred_x) # print "make stats and send to wandb" score = random.randint(0, 100) bt.logging.info(f"Hotkey {hotkey} model score: {score}") diff --git a/cancer_ai/validator/model_manager.py b/cancer_ai/validator/model_manager.py index 68d6c6ab..3bf60a09 100644 --- a/cancer_ai/validator/model_manager.py +++ b/cancer_ai/validator/model_manager.py @@ -36,7 +36,7 @@ def sync_hotkeys(self, hotkeys: list): if hotkey not in hotkeys: self.delete_model(hotkey) - def download_miner_model(self, hotkey) -> None: + async def download_miner_model(self, hotkey) -> None: """Downloads the newest model from Hugging Face and saves it to disk. Returns: str: path to the downloaded model @@ -46,7 +46,7 @@ def download_miner_model(self, hotkey) -> None: model_info.repo_id, model_info.filename, cache_dir=self.config.model_dir, - repo_type="space", + repo_type="dataset", ) diff --git a/neurons/competition_config.py b/neurons/competition_config.py index dbe0511b..1912b365 100644 --- a/neurons/competition_config.py +++ b/neurons/competition_config.py @@ -3,7 +3,7 @@ { "competition_id": "melanoma-1", "category": "skin", - "evaluation_time": ["12:30", "15:30"], + "evaluation_time": ["02:05", "15:30"], "dataset_hf_id": "safescanai/test_dataset", "file_hf_id": "skin_melanoma.zip", } diff --git a/neurons/competition_runner.py b/neurons/competition_runner.py new file mode 100644 index 00000000..4406e715 --- /dev/null +++ b/neurons/competition_runner.py @@ -0,0 +1,114 @@ +from cancer_ai.validator.competition_manager import CompetitionManager +from datetime import time, datetime + +import asyncio +import json +import timeit +from types import SimpleNamespace +from datetime import datetime, timezone, timedelta +import bittensor as bt +from typing import List + +from competition_config import competitions + +# from cancer_ai.utils.config import config + +# TODO integrate with bt config +path_config = SimpleNamespace( + **{"model_dir": "/tmp/models", "models_dataset_dir": "/tmp/datasets"} +) + + +def calculate_next_evaluation_times(evaluation_times) -> List[datetime]: + """Calculate the next evaluation times for a given list of times in UTC.""" + now_utc = datetime.now(timezone.utc) + next_times = [] + + for time_str in evaluation_times: + # Parse the evaluation time to a datetime object in UTC + evaluation_time_utc = datetime.strptime(time_str, "%H:%M").replace( + tzinfo=timezone.utc, year=now_utc.year, month=now_utc.month, day=now_utc.day + ) + + # If the evaluation time has already passed today, schedule it for tomorrow + if evaluation_time_utc < now_utc: + evaluation_time_utc += timedelta(days=1) + + next_times.append(evaluation_time_utc) + + return next_times + + +async def schedule_competitions( + competitions: CompetitionManager, path_config: str +) -> None: + # Cache the next evaluation times for each competition + print("Initializing competitions") + next_evaluation_times = {} + + # Calculate initial evaluation times + for competition_config in competitions: + competition_id = competition_config["competition_id"] + evaluation_times = competition_config["evaluation_time"] + next_evaluation_times[competition_id] = calculate_next_evaluation_times( + evaluation_times + ) + print( + f"Next evaluation times for competition {competition_id}: {next_evaluation_times[competition_id]}" + ) + + while True: + now_utc = datetime.now(timezone.utc) + + for competition_config in competitions: + competition_id = competition_config["competition_id"] + # Get the cached next evaluation times + next_times = next_evaluation_times[competition_id] + + for next_time in next_times: + if now_utc >= next_time: + print( + f"Next evaluation time for competition {competition_id} is {next_time}" + ) + # If it's time to run the competition + competition_manager = CompetitionManager( + path_config, + competition_config["competition_id"], + competition_config["category"], + competition_config["dataset_hf_id"], + competition_config["file_hf_id"], + ) + print(f"Evaluating competition {competition_id} at {now_utc}") + await competition_manager.evaluate() + print( + f"Results for competition {competition_id}: {competition_manager.results}" + ) + + # Calculate the next evaluation time for this specific time + next_times.remove(next_time) + next_times.append(next_time + timedelta(days=1)) + + # Update the cache with the next evaluation times + next_evaluation_times[competition_id] = next_times + if now_utc.minute % 5 == 0: + print("Waiting for next scheduled competition") + await asyncio.sleep(60) + +def run_all_competitions(path_config: str, competitions: List[dict]) -> None: + for competition_config in competitions: + print("Starting competition: ", competition_config) + competition_manager = CompetitionManager( + path_config, + competition_config["competition_id"], + competition_config["category"], + competition_config["dataset_hf_id"], + competition_config["file_hf_id"], + ) + asyncio.run(competition_manager.evaluate()) + +if __name__ == "__main__": + if True: # run them right away + run_all_competitions(path_config, competitions) + + else: # Run the scheduling coroutine + asyncio.run(schedule_competitions(competitions, path_config)) diff --git a/neurons/miner_config.py b/neurons/miner_config.py index 6e2924a8..67d384d3 100644 --- a/neurons/miner_config.py +++ b/neurons/miner_config.py @@ -77,7 +77,7 @@ def get_config() -> bt.config: type=str, # required=True, help="Competition ID", - default="your_competition_id", + default="melanoma-1", ) main_parser.add_argument( diff --git a/neurons/validator_tester.py b/neurons/validator_tester.py deleted file mode 100644 index a16d38e1..00000000 --- a/neurons/validator_tester.py +++ /dev/null @@ -1,79 +0,0 @@ -from cancer_ai.validator.competition_manager import CompetitionManager -from datetime import time, datetime - -import asyncio -import json -import timeit -from types import SimpleNamespace - -from competition_config import competitions - -# from cancer_ai.utils.config import config - -path_config = {"model_dir": "/tmp/models", "models_dataset_dir": "/tmp/datasets"} -# path_config = { -# "models": SimpleNamespace( -# **{"model_dir": "/tmp/models", "models_dataset_dir": "/tmp/datasets"} -# ), -# } -path_config = SimpleNamespace(**path_config) - -# open competition config file -# def get_competition_config(): -# with open(config.validator.competition_config_path) as f: -# return json.load(f) - -# async def main_loop(): -# competitions = get_competition_config() -# for competition_config in competitions: -# # get list of competition_config["evaluation_time"] and time it to run during specific time of day -# eval_times = [ -# competition_config["evaluation_time"]] -# while True: -# now = time.localtime() -# for eval_time in eval_times: -# if now.tm_hour == eval_time.hour and now.tm_min == eval_time.minute: -# competition = CompetitionManager( -# path_config, -# competition_config["competition_id"], -# competition_config["category"], -# competition_config["evaluation_time"], -# competition_config["dataset_hf_id"], -# competition_config["file_hf_id"], -# ) -# await competition.evaluate() -# print(competition.results) -# break -# await asyncio.sleep(60) - - -# def run(): -# asyncio.run(main_loop()) - -if __name__ == "__main__": - competition_managers_async = [] - for competition_config in competitions: - competition_manager = CompetitionManager( - path_config, - competition_config["competition_id"], - competition_config["category"], - competition_config["evaluation_time"], - competition_config["dataset_hf_id"], - competition_config["file_hf_id"], - ) - competition_managers_async.append(competition_manager) - competition_managers_async_gathered = asyncio.gather( - [ - competition_manager.evaluate() - for competition_manager in competition_managers_async - ] - ) - loop = asyncio.get_event_loop() - try: - loop.run_until_complete(competition_managers_async_gathered) - finally: - loop.close() - for competition_manager in competition_managers_async: - print( - f"Results of competition {competition_manager.competition_id}: {competition_manager.results}" - ) From ee788773f96885907df3d54c74cd1f6f5035890b Mon Sep 17 00:00:00 2001 From: Wojtek Jurkowlaniec Date: Fri, 23 Aug 2024 05:10:02 +0200 Subject: [PATCH 030/227] integration (not tested) with chain storage --- cancer_ai/chain_models_store.py | 37 +++++++++++------ cancer_ai/validator/competition_manager.py | 46 ++++++++++++++-------- cancer_ai/validator/model_manager.py | 11 +++--- neurons/miner.py | 4 +- 4 files changed, 64 insertions(+), 34 deletions(-) diff --git a/cancer_ai/chain_models_store.py b/cancer_ai/chain_models_store.py index cf66aa30..b20158e6 100644 --- a/cancer_ai/chain_models_store.py +++ b/cancer_ai/chain_models_store.py @@ -6,7 +6,8 @@ from .utils.models_storage_utils import run_in_subprocess from pydantic import BaseModel, Field, PositiveInt -class MinerModel(BaseModel): + +class ChainMinerModel(BaseModel): """Uniquely identifies a trained model""" namespace: str = Field( @@ -14,21 +15,34 @@ class MinerModel(BaseModel): ) name: str = Field(description="Name of the model.") - epoch: int = Field(description="The epoch number to submit as your checkpoint to evaluate e.g. 10") + epoch: int = Field( + description="The epoch number to submit as your checkpoint to evaluate e.g. 10" + ) - date: datetime.datetime = Field(description="The datetime at which model was pushed to hugging face") + date: datetime.datetime = Field( + description="The datetime at which model was pushed to hugging face" + ) # Identifier for competition competition_id: Optional[str] = Field(description="The competition id") - block: Optional[str] = Field(description="Block on which this model was claimed on the chain.") + block: Optional[str] = Field( + description="Block on which this model was claimed on the chain." + ) + + hf_repo_id: Optional[str] = Field(description="Hugging Face repo id.") + hf_filename: Optional[str] = Field(description="Hugging Face filename.") + hf_repo_type: Optional[str] = Field(description="Hugging Face repo type.") + + class Config: + arbitrary_types_allowed = True def to_compressed_str(self) -> str: """Returns a compressed string representation.""" return f"{self.namespace}:{self.name}:{self.epoch}:{self.competition_id}:{self.date}" @classmethod - def from_compressed_str(cls, cs: str) -> Type["MinerModel"]: + def from_compressed_str(cls, cs: str) -> Type["ChainMinerModel"]: """Returns an instance of this class from a compressed string representation""" tokens = cs.split(":") return cls( @@ -39,7 +53,8 @@ def from_compressed_str(cls, cs: str) -> Type["MinerModel"]: competition_id=tokens[4] if tokens[4] != "None" else None, ) -class ChainModelMetadataStore(): + +class ChainModelMetadataStore: """Chain based implementation for storing and retrieving metadata about a model.""" def __init__( @@ -54,7 +69,7 @@ def __init__( ) self.subnet_uid = subnet_uid - async def store_model_metadata(self, model_id: MinerModel): + async def store_model_metadata(self, model_id: ChainMinerModel): """Stores model metadata on this subnet for a specific wallet.""" if self.wallet is None: raise ValueError("No wallet available to write to the chain.") @@ -68,7 +83,7 @@ async def store_model_metadata(self, model_id: MinerModel): ) run_in_subprocess(partial, 60) - async def retrieve_model_metadata(self, hotkey: str) -> Optional[MinerModel]: + async def retrieve_model_metadata(self, hotkey: str) -> Optional[ChainMinerModel]: """Retrieves model metadata on this subnet for specific hotkey""" # Wrap calls to the subtensor in a subprocess with a timeout to handle potential hangs. @@ -86,11 +101,11 @@ async def retrieve_model_metadata(self, hotkey: str) -> Optional[MinerModel]: chain_str = bytes.fromhex(hex_data).decode() try: - model = MinerModel.from_compressed_str(chain_str) + model = ChainMinerModel.from_compressed_str(chain_str) except: # If the metadata format is not correct on the chain then we return None. - bt.logging.trace( - f"Failed to parse the metadata on the chain for hotkey {hotkey}." + bt.logging.error( + f"Failed to parse the metadata on the chain for hotkey {hotkey}. Raw value: {chain_str}" ) return None # The block id at which the metadata is stored diff --git a/cancer_ai/validator/competition_manager.py b/cancer_ai/validator/competition_manager.py index 6227f244..9e1be7c8 100644 --- a/cancer_ai/validator/competition_manager.py +++ b/cancer_ai/validator/competition_manager.py @@ -8,7 +8,7 @@ from .model_manager import ModelManager, ModelInfo from .dataset_manager import DatasetManager from .model_run_manager import ModelRunManager - +from cancer_ai.chain_models_store import ChainMinerModel, ChainModelMetadataStore COMPETITION_MAPPING = { "melaona-1": "melanoma", @@ -32,6 +32,8 @@ class CompetitionManager(SerializableManager): def __init__( self, config, + subtensor: bt.Subtensor, + subnet_uid: str, competition_id: str, category: str, dataset_hf_id: str, @@ -54,6 +56,10 @@ def __init__( self.dataset_manager = DatasetManager( config, competition_id, dataset_hf_id, file_hf_id ) + self.chain_model_metadata_store = ChainModelMetadataStore(subtensor, subnet_uid ) + + self.hotkeys = [] + self.chain_miner_models = {} def get_state(self): return { @@ -68,24 +74,32 @@ def set_state(self, state: dict): self.model_manager.set_state(state["model_manager"]) self.category = state["category"] - async def get_miner_model(self, hotkey): - # TODO get real data - return ModelInfo("safescanai/test_dataset", "simple_cnn_model.onnx") - - async def init_evaluation(self): - # get models from chain - hotkeys = [ - "example_hotkey", - ] - for hotkey in hotkeys: - self.model_manager.hotkey_store[hotkey] = await self.get_miner_model(hotkey) - - await self.dataset_manager.prepare_dataset() + async def get_miner_model(self, chain_miner_model: ChainMinerModel): + model_info = ModelInfo( + hf_repo_id=chain_miner_model.hf_repo_id, + hf_filename=chain_miner_model.hf_filename, + hf_repo_type=chain_miner_model.hf_repo_type, + ) + return model_info - # log event + # return ModelInfo(hf_repo_id="safescanai/test_dataset", hf_filename="simple_cnn_model.onnx", hf_repo_type="dataset") + async def sync_chain_miners(self, hotkeys: list[str]): + """ + Updates hotkeys and downloads information of models from the chain + """ + bt.logging.info("Synchronizing miners from the chain") + self.hotkeys = hotkeys + bt.logging.info(f"Amount of hotkeys: {len(hotkeys)}") + for hotkey in hotkeys: + hotkey_metadata = await self.chain_model_metadata_store.retrieve_model_metadata(hotkey) + if hotkey_metadata: + self.chain_miner_models[hotkey] = hotkey_metadata + self.model_manager.hotkey_store[hotkey] = await self.get_miner_model(hotkey) + bt.logging.info(f"Amount of chain miners with models: {len(self.chain_miner_models)}") + async def evaluate(self): - await self.init_evaluation() + await self.dataset_manager.prepare_dataset() pred_x, pred_y = await self.dataset_manager.get_data() for hotkey in self.model_manager.hotkey_store: bt.logging.info("Evaluating hotkey: ", hotkey) diff --git a/cancer_ai/validator/model_manager.py b/cancer_ai/validator/model_manager.py index 3bf60a09..24fac58a 100644 --- a/cancer_ai/validator/model_manager.py +++ b/cancer_ai/validator/model_manager.py @@ -9,8 +9,9 @@ @dataclass class ModelInfo: - repo_id: str | None = None - filename: str | None = None + hf_repo_id: str | None = None + hf_filename: str | None = None + hf_repo_type: str | None = None file_path: str | None = None model_type: str | None = None @@ -43,10 +44,10 @@ async def download_miner_model(self, hotkey) -> None: """ model_info = self.hotkey_store[hotkey] model_info.file_path = self.api.hf_hub_download( - model_info.repo_id, - model_info.filename, + model_info.hf_repo_id, + model_info.hf_filename, cache_dir=self.config.model_dir, - repo_type="dataset", + repo_type=model_info.hf_repo_type, ) diff --git a/neurons/miner.py b/neurons/miner.py index e190fe40..9360a4af 100644 --- a/neurons/miner.py +++ b/neurons/miner.py @@ -10,7 +10,7 @@ # import base miner class which takes care of most of the boilerplate from cancer_ai.base.miner import BaseMinerNeuron -from cancer_ai.chain_models_store import MinerModel, ChainModelMetadataStore +from cancer_ai.chain_models_store import ChainMinerModel, ChainModelMetadataStore class Miner(BaseMinerNeuron): @@ -152,7 +152,7 @@ async def store_and_retrieve_metadata_on_chain(self, competition: str) -> None: PoC function to integrate with the structured business logic """ - model_id = MinerModel(namespace=self.config.models.namespace, name=self.config.models.model_name, epoch=self.config.models.epoch_checkpoint, + model_id = ChainMinerModel(namespace=self.config.models.namespace, name=self.config.models.model_name, epoch=self.config.models.epoch_checkpoint, date=datetime.datetime.now(), competition_id=competition, block=None) await self.metadata_store.store_model_metadata(model_id) From 22289f00c9d376141bb2d8bc027b77498353d845 Mon Sep 17 00:00:00 2001 From: Wojtek Jurkowlaniec Date: Fri, 23 Aug 2024 05:23:00 +0200 Subject: [PATCH 031/227] little refactoring, added repo type to dataset as well --- cancer_ai/utils/config.py | 6 +----- cancer_ai/validator/competition_manager.py | 5 +++-- cancer_ai/validator/dataset_manager.py | 18 +++++++++--------- neurons/competition_config.py | 5 +++-- neurons/competition_runner.py | 14 ++++++++++---- 5 files changed, 26 insertions(+), 22 deletions(-) diff --git a/cancer_ai/utils/config.py b/cancer_ai/utils/config.py index 68cbbfd2..5967d6f3 100644 --- a/cancer_ai/utils/config.py +++ b/cancer_ai/utils/config.py @@ -174,15 +174,11 @@ def add_miner_args(cls, parser): help="Wandb entity to log to.", ) - help="Path for storing trained model related to a training run.", - default="./models", - ) - parser.add_argument( "--models.load_model_dir", type=str, help="Path for for loading the starting model related to a training run.", - default="", + default="./models", ) parser.add_argument( diff --git a/cancer_ai/validator/competition_manager.py b/cancer_ai/validator/competition_manager.py index 9e1be7c8..df8a57e7 100644 --- a/cancer_ai/validator/competition_manager.py +++ b/cancer_ai/validator/competition_manager.py @@ -36,8 +36,9 @@ def __init__( subnet_uid: str, competition_id: str, category: str, + dataset_hf_repo: str, dataset_hf_id: str, - file_hf_id: str, + dataset_hf_repo_type: str, ) -> None: """ Responsible for managing a competition. @@ -54,7 +55,7 @@ def __init__( self.results = [] self.model_manager = ModelManager(config) self.dataset_manager = DatasetManager( - config, competition_id, dataset_hf_id, file_hf_id + config, competition_id, dataset_hf_repo, dataset_hf_id, dataset_hf_repo_type ) self.chain_model_metadata_store = ChainModelMetadataStore(subtensor, subnet_uid ) diff --git a/cancer_ai/validator/dataset_manager.py b/cancer_ai/validator/dataset_manager.py index cf6c8e45..3952fe60 100644 --- a/cancer_ai/validator/dataset_manager.py +++ b/cancer_ai/validator/dataset_manager.py @@ -17,7 +17,7 @@ class DatasetManagerException(Exception): class DatasetManager(SerializableManager): def __init__( - self, config, competition_id: str, dataset_hf_id: str, file_hf_id: str + self, config, competition_id: str, hf_repo_id: str, hf_filename: str, hf_repo_type: str ) -> None: """ Initializes a new instance of the DatasetManager class. @@ -33,9 +33,9 @@ def __init__( """ self.config = config self.competition_id = competition_id - self.dataset_hf_id = dataset_hf_id - self.file_hf_id = file_hf_id - self.hf_api = HfApi() + self.hf_repo_id = hf_repo_id + self.hf_filename = hf_filename + self.hf_repo_type = hf_repo_type self.local_compressed_path = "" print(self.config) self.local_extracted_dir = Path( @@ -54,12 +54,12 @@ def set_state(self, state: dict): async def download_dataset(self): if not os.path.exists(self.local_extracted_dir): os.makedirs(self.local_extracted_dir) - - self.local_compressed_path = self.hf_api.hf_hub_download( - self.dataset_hf_id, - self.file_hf_id, + + self.local_compressed_path = HfApi().hf_hub_download( + self.hf_repo_id, + self.hf_filename, cache_dir=Path(self.config.models_dataset_dir), - repo_type="dataset", + repo_type=self.hf_repo_type, ) def delete_dataset(self) -> None: diff --git a/neurons/competition_config.py b/neurons/competition_config.py index 1912b365..aa232922 100644 --- a/neurons/competition_config.py +++ b/neurons/competition_config.py @@ -4,7 +4,8 @@ "competition_id": "melanoma-1", "category": "skin", "evaluation_time": ["02:05", "15:30"], - "dataset_hf_id": "safescanai/test_dataset", - "file_hf_id": "skin_melanoma.zip", + "dataset_hf_repo": "safescanai/test_dataset", + "dataset_hf_filename": "skin_melanoma.zip", + "dataset_hf_repo_type": "dataset" } ] \ No newline at end of file diff --git a/neurons/competition_runner.py b/neurons/competition_runner.py index 4406e715..74793a22 100644 --- a/neurons/competition_runner.py +++ b/neurons/competition_runner.py @@ -73,10 +73,13 @@ async def schedule_competitions( # If it's time to run the competition competition_manager = CompetitionManager( path_config, + None, + 7, competition_config["competition_id"], competition_config["category"], - competition_config["dataset_hf_id"], - competition_config["file_hf_id"], + competition_config["dataset_hf_repo"], + competition_config["dataset_hf_filename"], + competition_config["dataset_hf_repo_type"], ) print(f"Evaluating competition {competition_id} at {now_utc}") await competition_manager.evaluate() @@ -99,10 +102,13 @@ def run_all_competitions(path_config: str, competitions: List[dict]) -> None: print("Starting competition: ", competition_config) competition_manager = CompetitionManager( path_config, + None, + 7, competition_config["competition_id"], competition_config["category"], - competition_config["dataset_hf_id"], - competition_config["file_hf_id"], + competition_config["dataset_hf_repo"], + competition_config["dataset_hf_filename"], + competition_config["dataset_hf_repo_type"], ) asyncio.run(competition_manager.evaluate()) From f04412e5f1ecb1a011cc141363af2794600c1523 Mon Sep 17 00:00:00 2001 From: notbulubula Date: Fri, 23 Aug 2024 10:52:35 +0200 Subject: [PATCH 032/227] Restructuring competition handlers --- .../competition_handlers/base_handler.py | 49 +++++++++++++ .../competition_handlers/melanoma_handler.py | 56 ++++++++++++++ cancer_ai/validator/competition_manager.py | 73 ++++--------------- cancer_ai/validator_tester.py | 2 +- 4 files changed, 120 insertions(+), 60 deletions(-) create mode 100644 cancer_ai/validator/competition_handlers/base_handler.py create mode 100644 cancer_ai/validator/competition_handlers/melanoma_handler.py diff --git a/cancer_ai/validator/competition_handlers/base_handler.py b/cancer_ai/validator/competition_handlers/base_handler.py new file mode 100644 index 00000000..c0b4aca4 --- /dev/null +++ b/cancer_ai/validator/competition_handlers/base_handler.py @@ -0,0 +1,49 @@ +from abc import abstractmethod + +from dataclasses import dataclass + +@dataclass +class ModelEvaluationResult: + accuracy: float + precision: float + recall: float + confusion_matrix: any + fpr: any + tpr: any + roc_auc: float + run_time: float + tested_entries: int + +class BaseCompetitionHandler: + """ + Base class for handling different competition types. + + This class initializes the config and competition_id attributes. + """ + + def __init__(self, path_X_test, y_test) -> None: + """ + Initializes the BaseCompetitionHandler object. + + Args: + path_X_test (str): Path to the test data. + y_test (list): List of test labels. + """ + self.path_X_test = path_X_test + self.y_test = y_test + + @abstractmethod + def preprocess_data(self): + """ + Abstract method to prepare the data. + + This method is responsible for preprocessing the data for the competition. + """ + + @abstractmethod + def evaluate(self, y_pred) -> ModelEvaluationResult: + """ + Abstract method to evaluate the competition. + + This method is responsible for evaluating the competition. + """ \ No newline at end of file diff --git a/cancer_ai/validator/competition_handlers/melanoma_handler.py b/cancer_ai/validator/competition_handlers/melanoma_handler.py new file mode 100644 index 00000000..d4161f4b --- /dev/null +++ b/cancer_ai/validator/competition_handlers/melanoma_handler.py @@ -0,0 +1,56 @@ +from .base_handler import BaseCompetitionHandler +from .base_handler import ModelEvaluationResult + +from PIL import Image +import numpy as np +from sklearn.metrics import accuracy_score, precision_score, recall_score, confusion_matrix, roc_curve, auc + +class MelanomaCompetitionHandler(BaseCompetitionHandler): + """ + """ + def __init__(self, path_X_test, y_test) -> None: + super().__init__(path_X_test, y_test) + + def preprocess_data(self): + X_test = [] + target_size=(224, 224) #TODO: Change this to the correct size + + for img_path in self.path_X_test: + img = Image.open(img_path) + img = img.resize(target_size) + img_array = np.array(img, dtype=np.float32) / 255.0 + img_array = np.array(img) + if img_array.shape[-1] != 3: # Handle grayscale images + img_array = np.stack((img_array,) * 3, axis=-1) + + img_array = np.transpose(img_array, (2, 0, 1)) # Transpose image to (C, H, W) + img_array = np.expand_dims(img_array, axis=0) # Add batch dimension + X_test.append(img_array) + + X_test = np.array(X_test, dtype=np.float32) + + # Map y_test to 0, 1 + y_test = [1 if y == "True" else 0 for y in self.y_test] + + return X_test, y_test + + def evaluate(self, y_test, y_pred, run_time) -> ModelEvaluationResult: + y_pred_binary = [1 if y > 0.5 else 0 for y in y_pred] + tested_entries = len(y_test) + accuracy = accuracy_score(y_test, y_pred_binary) + precision = precision_score(y_test, y_pred_binary) + recall = recall_score(y_test, y_pred_binary) + conf_matrix = confusion_matrix(y_test, y_pred_binary) + fpr, tpr, _ = roc_curve(y_test, y_pred) + roc_auc = auc(fpr, tpr) + return ModelEvaluationResult( + tested_entries=tested_entries, + run_time=run_time, + accuracy=accuracy, + precision=precision, + recall=recall, + confusion_matrix=conf_matrix, + fpr=fpr, + tpr=tpr, + roc_auc=roc_auc, + ) \ No newline at end of file diff --git a/cancer_ai/validator/competition_manager.py b/cancer_ai/validator/competition_manager.py index 6697b233..6d91173a 100644 --- a/cancer_ai/validator/competition_manager.py +++ b/cancer_ai/validator/competition_manager.py @@ -9,12 +9,15 @@ from .dataset_manager import DatasetManager from .model_run_manager import ModelRunManager -from sklearn.metrics import accuracy_score, precision_score, recall_score, confusion_matrix, roc_curve, auc -from dataclasses import dataclass +from .competition_handlers.melanoma_handler import MelanomaCompetitionHandler + COMPETITION_MAPPING = { "melaona-1": "melanoma", } +COMPETITION_HANDLER_MAPPING = { + "melaona-1": MelanomaCompetitionHandler, +} class ImagePredictionCompetition: @@ -23,18 +26,6 @@ def score_model( ) -> float: pass -@dataclass -class ModelEvaluationResult: - accuracy: float - precision: float - recall: float - confusion_matrix: any - fpr: any - tpr: any - roc_auc: float - run_time: float - tested_entries: int - class CompetitionManager(SerializableManager): """ @@ -129,64 +120,28 @@ async def init_evaluation(self): # log event async def evaluate(self): - from PIL import Image - import numpy as np await self.init_evaluation() path_X_test, y_test = await self.dataset_manager.get_data() - # Prepre X_test form paths to images - X_test = [] - target_size=(224, 224) #TODO: Change this to the correct size - - for img_path in path_X_test: - img = Image.open(img_path) - img = img.resize(target_size) - img_array = np.array(img, dtype=np.float32) / 255.0 - img_array = np.array(img) - if img_array.shape[-1] != 3: # Handle grayscale images - img_array = np.stack((img_array,) * 3, axis=-1) - - img_array = np.transpose(img_array, (2, 0, 1)) # Convert image to numpy array - img_array = np.expand_dims(img_array, axis=0) # Add batch dimension - X_test.append(img_array) - X_test = np.array(X_test, dtype=np.float32) - - # print("X_test shape: ", X_test.shape) - - # map y_test to 0, 1 - y_test = [1 if y == "True" else 0 for y in y_test] + + competition_handler = COMPETITION_HANDLER_MAPPING[self.competition_id]( + path_X_test=path_X_test, y_test=y_test + ) + + X_test, y_test = competition_handler.preprocess_data() for hotkey in self.model_manager.hotkey_store: bt.logging.info("Evaluating hotkey: ", hotkey) await self.model_manager.download_miner_model(hotkey) - start_time = time.time() model_manager = ModelRunManager( self.config, self.model_manager.hotkey_store[hotkey] ) + start_time = time.time() y_pred = model_manager.run(X_test) + run_time = time.time() - start_time print("Model prediction ", y_pred) print("Ground truth: ", y_test) - # print "make stats and send to wandb" - run_time = time.time() - start_time - tested_entries = len(y_test) - accuracy = accuracy_score(y_test, y_pred) - precision = precision_score(y_test, y_pred) - recall = recall_score(y_test, y_pred) - conf_matrix = confusion_matrix(y_test, y_pred) - fpr, tpr, _ = roc_curve(y_test, y_pred) - roc_auc = auc(fpr, tpr) - - model_result = ModelEvaluationResult( - tested_entries=tested_entries, - run_time=run_time, - accuracy=accuracy, - precision=precision, - recall=recall, - confusion_matrix=conf_matrix, - fpr=fpr, - tpr=tpr, - roc_auc=roc_auc, - ) + model_result = competition_handler.evaluate(y_test, y_pred, run_time) self.results.append((hotkey, model_result)) return self.results diff --git a/cancer_ai/validator_tester.py b/cancer_ai/validator_tester.py index 14524b71..6f39e7f8 100644 --- a/cancer_ai/validator_tester.py +++ b/cancer_ai/validator_tester.py @@ -7,7 +7,7 @@ from types import SimpleNamespace # from cancer_ai.utils.config import config -from validator.competition_manager import ModelEvaluationResult +from validator.competition_handlers.base_handler import ModelEvaluationResult import wandb from dotenv import load_dotenv From 4fd067ef2135b16ea21498022c640f8597842bfd Mon Sep 17 00:00:00 2001 From: notbulubula Date: Fri, 23 Aug 2024 12:00:46 +0200 Subject: [PATCH 033/227] Dynamic batch size for onnx --- .../competition_handlers/melanoma_handler.py | 2 +- cancer_ai/validator/competition_manager.py | 11 +++-------- .../validator/model_runners/onnx_runner.py | 19 ++++++++----------- 3 files changed, 12 insertions(+), 20 deletions(-) diff --git a/cancer_ai/validator/competition_handlers/melanoma_handler.py b/cancer_ai/validator/competition_handlers/melanoma_handler.py index d4161f4b..153e9101 100644 --- a/cancer_ai/validator/competition_handlers/melanoma_handler.py +++ b/cancer_ai/validator/competition_handlers/melanoma_handler.py @@ -24,7 +24,7 @@ def preprocess_data(self): img_array = np.stack((img_array,) * 3, axis=-1) img_array = np.transpose(img_array, (2, 0, 1)) # Transpose image to (C, H, W) - img_array = np.expand_dims(img_array, axis=0) # Add batch dimension + X_test.append(img_array) X_test = np.array(X_test, dtype=np.float32) diff --git a/cancer_ai/validator/competition_manager.py b/cancer_ai/validator/competition_manager.py index 6d91173a..f0dee77d 100644 --- a/cancer_ai/validator/competition_manager.py +++ b/cancer_ai/validator/competition_manager.py @@ -92,19 +92,14 @@ async def init_evaluation(self): # TODO get models from chain miner_models = [ # { - # "hotkey": "wojtasy", + # "hotkey": "obcy ludzie_2", # "hf_id": "safescanai/test_dataset", - # "file_hf_id": "melanoma.keras", - # }, - # { - # "hotkey": "wojtasyy", - # "hf_id": "safescanai/test_dataset", - # "file_hf_id": "melanoma.keras", + # "file_hf_id": "model_dynamic.onnx", # }, { "hotkey": "obcy ludzie", "hf_id": "safescanai/test_dataset", - "file_hf_id": "simple_cnn_model.onnx", + "file_hf_id": "model_dynamic.onnx", }, ] bt.logging.info( diff --git a/cancer_ai/validator/model_runners/onnx_runner.py b/cancer_ai/validator/model_runners/onnx_runner.py index a4c17c64..6159b82e 100644 --- a/cancer_ai/validator/model_runners/onnx_runner.py +++ b/cancer_ai/validator/model_runners/onnx_runner.py @@ -5,21 +5,18 @@ class OnnxRunnerHandler(BaseRunnerHandler): def run(self, X_test: List) -> List: import onnxruntime import numpy as np - import torch - from PIL import Image - from torchvision import transforms # Load the ONNX model session = onnxruntime.InferenceSession(self.model_path) + + # Stack input images into a single batch + input_batch = np.stack(X_test) + + # Prepare input for ONNX model input_name = session.get_inputs()[0].name - - results = [] - for img in X_test: - # Prepare input for ONNX model - input_data = {input_name: img} - y_pred = session.run(None, input_data)[0][0] + input_data = {input_name: input_batch} - # Collect results - results.append(y_pred[0].tolist()) + # Perform inference on the batch + results = session.run(None, input_data)[0] return results From fb2e9b24fab9feb2cd1dc61fef63434dd9f8b588 Mon Sep 17 00:00:00 2001 From: notbulubula Date: Fri, 23 Aug 2024 14:05:36 +0200 Subject: [PATCH 034/227] Renaming and adding types --- cancer_ai/validator/competition_handlers/base_handler.py | 2 +- .../validator/competition_handlers/melanoma_handler.py | 5 +++-- cancer_ai/validator/competition_manager.py | 5 +++-- cancer_ai/validator/dataset_handlers/image_csv.py | 8 ++++++++ 4 files changed, 15 insertions(+), 5 deletions(-) diff --git a/cancer_ai/validator/competition_handlers/base_handler.py b/cancer_ai/validator/competition_handlers/base_handler.py index c0b4aca4..bf473dcc 100644 --- a/cancer_ai/validator/competition_handlers/base_handler.py +++ b/cancer_ai/validator/competition_handlers/base_handler.py @@ -41,7 +41,7 @@ def preprocess_data(self): """ @abstractmethod - def evaluate(self, y_pred) -> ModelEvaluationResult: + def get_model_result(self) -> ModelEvaluationResult: """ Abstract method to evaluate the competition. diff --git a/cancer_ai/validator/competition_handlers/melanoma_handler.py b/cancer_ai/validator/competition_handlers/melanoma_handler.py index 153e9101..ab205363 100644 --- a/cancer_ai/validator/competition_handlers/melanoma_handler.py +++ b/cancer_ai/validator/competition_handlers/melanoma_handler.py @@ -1,6 +1,7 @@ from .base_handler import BaseCompetitionHandler from .base_handler import ModelEvaluationResult +from typing import List from PIL import Image import numpy as np from sklearn.metrics import accuracy_score, precision_score, recall_score, confusion_matrix, roc_curve, auc @@ -34,7 +35,7 @@ def preprocess_data(self): return X_test, y_test - def evaluate(self, y_test, y_pred, run_time) -> ModelEvaluationResult: + def get_model_result(self, y_test: List[float], y_pred: np.ndarray, run_time_s: float) -> ModelEvaluationResult: y_pred_binary = [1 if y > 0.5 else 0 for y in y_pred] tested_entries = len(y_test) accuracy = accuracy_score(y_test, y_pred_binary) @@ -45,7 +46,7 @@ def evaluate(self, y_test, y_pred, run_time) -> ModelEvaluationResult: roc_auc = auc(fpr, tpr) return ModelEvaluationResult( tested_entries=tested_entries, - run_time=run_time, + run_time=run_time_s, accuracy=accuracy, precision=precision, recall=recall, diff --git a/cancer_ai/validator/competition_manager.py b/cancer_ai/validator/competition_manager.py index f0dee77d..ec871cc3 100644 --- a/cancer_ai/validator/competition_manager.py +++ b/cancer_ai/validator/competition_manager.py @@ -133,10 +133,11 @@ async def evaluate(self): ) start_time = time.time() y_pred = model_manager.run(X_test) - run_time = time.time() - start_time + run_time_s = time.time() - start_time print("Model prediction ", y_pred) print("Ground truth: ", y_test) - model_result = competition_handler.evaluate(y_test, y_pred, run_time) + + model_result = competition_handler.get_model_result(y_test, y_pred, run_time_s) self.results.append((hotkey, model_result)) return self.results diff --git a/cancer_ai/validator/dataset_handlers/image_csv.py b/cancer_ai/validator/dataset_handlers/image_csv.py index 834020d3..7d71cbb2 100644 --- a/cancer_ai/validator/dataset_handlers/image_csv.py +++ b/cancer_ai/validator/dataset_handlers/image_csv.py @@ -40,6 +40,14 @@ async def sync_training_data(self): self.entries.append(ImageEntry(row[0], row[1])) async def get_training_data(self) -> Tuple[List, List]: + """ + Get the training data. + + This method is responsible for loading the training data and returning it as a tuple of two lists: the first list contains the input data and the second list contains the labels. + + Returns: + Tuple[List, List]: A tuple containing two lists: the first list contains paths to the images and the second list contains the labels. + """ await self.sync_training_data() print(self.entries) pred_x = [f"{Path(self.label_path).parent}/{entry.filepath}" for entry in self.entries] From 71b7e7ad99f04fabb1dc5f8ca378ba8152ece48a Mon Sep 17 00:00:00 2001 From: notbulubula Date: Fri, 23 Aug 2024 14:20:34 +0200 Subject: [PATCH 035/227] Fix --- cancer_ai/validator/dataset_handlers/image_csv.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/cancer_ai/validator/dataset_handlers/image_csv.py b/cancer_ai/validator/dataset_handlers/image_csv.py index 7d71cbb2..a9d68c2c 100644 --- a/cancer_ai/validator/dataset_handlers/image_csv.py +++ b/cancer_ai/validator/dataset_handlers/image_csv.py @@ -43,10 +43,7 @@ async def get_training_data(self) -> Tuple[List, List]: """ Get the training data. - This method is responsible for loading the training data and returning it as a tuple of two lists: the first list contains the input data and the second list contains the labels. - - Returns: - Tuple[List, List]: A tuple containing two lists: the first list contains paths to the images and the second list contains the labels. + This method is responsible for loading the training data and returning a tuple containing two lists: the first list contains paths to the images and the second list contains the labels. """ await self.sync_training_data() print(self.entries) From f18a283217b0c6e6b21a9e6f9062824210477f99 Mon Sep 17 00:00:00 2001 From: notbulubula Date: Fri, 23 Aug 2024 14:37:57 +0200 Subject: [PATCH 036/227] Removing redundant mapping --- cancer_ai/validator/competition_manager.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/cancer_ai/validator/competition_manager.py b/cancer_ai/validator/competition_manager.py index ec871cc3..e0f60576 100644 --- a/cancer_ai/validator/competition_manager.py +++ b/cancer_ai/validator/competition_manager.py @@ -12,9 +12,7 @@ from .competition_handlers.melanoma_handler import MelanomaCompetitionHandler -COMPETITION_MAPPING = { - "melaona-1": "melanoma", -} + COMPETITION_HANDLER_MAPPING = { "melaona-1": MelanomaCompetitionHandler, } From 4721df83153cfff94558c425e70258b9fa560944 Mon Sep 17 00:00:00 2001 From: LEMSTUDI0 <105062843+LEMSTUDI0@users.noreply.github.com> Date: Fri, 23 Aug 2024 17:19:02 +0200 Subject: [PATCH 037/227] Update README.md --- README.md | 357 ++++++++++++++++++++++++++++-------------------------- 1 file changed, 186 insertions(+), 171 deletions(-) diff --git a/README.md b/README.md index ba69bdae..4426af08 100644 --- a/README.md +++ b/README.md @@ -1,213 +1,228 @@ -
+# SAFE SCAN -# **Bittensor Subnet Template** -[![Discord Chat](https://img.shields.io/discord/308323056592486420.svg)](https://discord.gg/bittensor) -[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) +*Bittensor Subnet for improving cancer detection algorithms* ---- +![image.png](OUR%20GITHUB%20481afe11c91843b796c4afc6b838d8fb/image.png) -## The Incentivized Internet +- [Introduction](https://github.com/impel-intelligence/dippy-bittensor-subnet#introduction) +- Features +- Vision +- [Roadmap](https://github.com/impel-intelligence/dippy-bittensor-subnet#roadmap) +- [Overview of Miner and Validator Functionality](https://github.com/impel-intelligence/dippy-bittensor-subnet#overview-of-miner-and-validator-functionality) + - [Miner](https://github.com/impel-intelligence/dippy-bittensor-subnet#miner) + - [Validator](https://github.com/impel-intelligence/dippy-bittensor-subnet#validator) +- [Running Miners and Validators](https://github.com/impel-intelligence/dippy-bittensor-subnet#running-miners-and-validators) + - [Running a Miner](https://github.com/impel-intelligence/dippy-bittensor-subnet#running-a-miner) + - [Running a Validator](https://github.com/impel-intelligence/dippy-bittensor-subnet#running-a-validator) +- [Contributing](https://github.com/impel-intelligence/dippy-bittensor-subnet#contributing) +- [License](https://github.com/impel-intelligence/dippy-bittensor-subnet#license) -[Discord](https://discord.gg/bittensor) • [Network](https://taostats.io/) • [Research](https://bittensor.com/whitepaper) -
+# **👋 INTRODUCTION** ---- -- [Quickstarter template](#quickstarter-template) -- [Introduction](#introduction) - - [Example](#example) -- [Installation](#installation) - - [Before you proceed](#before-you-proceed) - - [Install](#install) -- [Writing your own incentive mechanism](#writing-your-own-incentive-mechanism) -- [Writing your own subnet API](#writing-your-own-subnet-api) -- [Subnet Links](#subnet-links) -- [License](#license) +Welcome to Safe Scan Cancer AI Detection, a groundbreaking initiative leveraging the power of AI and blockchain technology to revolutionize cancer detection. Our mission is to make advanced cancer detection algorithms accessible and free for everyone. Through our project, we aim to provide cutting-edge, open-source tools that support early cancer diagnosis for patients and healthcare professionals worldwide. ---- -## Quickstarter template +This repository contains subnet code to run on Bittensor network. -This template contains all the required installation instructions, scripts, and files and functions for: -- Building Bittensor subnets. -- Creating custom incentive mechanisms and running these mechanisms on the subnets. +# **⚙️ FEATURES** -In order to simplify the building of subnets, this template abstracts away the complexity of the underlying blockchain and other boilerplate code. While the default behavior of the template is sufficient for a simple subnet, you should customize the template in order to meet your specific requirements. ---- +🤗 Validator-friendly code -## Introduction +🏆 Rewards for best-performing algorithms -**IMPORTANT**: If you are new to Bittensor subnets, read this section before proceeding to [Installation](#installation) section. +👑 Royalties for algorithms used in our real-world software solutions -The Bittensor blockchain hosts multiple self-contained incentive mechanisms called **subnets**. Subnets are playing fields in which: -- Subnet miners who produce value, and -- Subnet validators who produce consensus +🤳 Free app for skin cancer detection -determine together the proper distribution of TAO for the purpose of incentivizing the creation of value, i.e., generating digital commodities, such as intelligence or data. +⚔️ Various cancer detection algorithm competitions -Each subnet consists of: -- Subnet miners and subnet validators. -- A protocol using which the subnet miners and subnet validators interact with one another. This protocol is part of the incentive mechanism. -- The Bittensor API using which the subnet miners and subnet validators interact with Bittensor's onchain consensus engine [Yuma Consensus](https://bittensor.com/documentation/validating/yuma-consensus). The Yuma Consensus is designed to drive these actors: subnet validators and subnet miners, into agreement on who is creating value and what that value is worth. +📊 WandB Visualization Dashboard -This starter template is split into three primary files. To write your own incentive mechanism, you should edit these files. These files are: -1. `template/protocol.py`: Contains the definition of the protocol used by subnet miners and subnet validators. -2. `neurons/miner.py`: Script that defines the subnet miner's behavior, i.e., how the subnet miner responds to requests from subnet validators. -3. `neurons/validator.py`: This script defines the subnet validator's behavior, i.e., how the subnet validator requests information from the subnet miners and determines the scores. +💻 Specialized software for detecting other types of cancer -### Example +💸 Self-sustaining economy -The Bittensor Subnet 1 for Text Prompting is built using this template. See [prompting](https://github.com/macrocosm-os/prompting) for how to configure the files and how to add monitoring and telemetry and support multiple miner types. Also see this Subnet 1 in action on [Taostats](https://taostats.io/subnets/netuid-1/) explorer. +# 👁️ VISION ---- +Cancer is one of the most significant challenges of our time, and we believe that AI holds the key to addressing it. However, this solution should be accessible and free for everyone. Machine vision technology has long proven effective in early diagnosis, which is crucial for curing cancer. Yet, until now, it has largely remained in the realm of whitepapers. SAFESCAN is a project dedicated to aggregating and enhancing the best algorithms for detecting various types of cancer and providing free computational power for practical cancer detection. We aim to create open-source products that support cancer diagnosis for both patients and doctors. -## Installation +To date, many crypto and AI projects, particularly those focused on medicine, have struggled to achieve real-world implementation due to various barriers. Our solution focuses on: -### Before you proceed -Before you proceed with the installation of the subnet, note the following: +**🛠️ Development of Applications and Software:** Invest in the ongoing development and enhancement of our cancer detection applications and software to ensure they are at the cutting edge of technology. -- Use these instructions to run your subnet locally for your development and testing, or on Bittensor testnet or on Bittensor mainnet. -- **IMPORTANT**: We **strongly recommend** that you first run your subnet locally and complete your development and testing before running the subnet on Bittensor testnet. Furthermore, make sure that you next run your subnet on Bittensor testnet before running it on the Bittensor mainnet. -- You can run your subnet either as a subnet owner, or as a subnet validator or as a subnet miner. -- **IMPORTANT:** Make sure you are aware of the minimum compute requirements for your subnet. See the [Minimum compute YAML configuration](./min_compute.yml). -- Note that installation instructions differ based on your situation: For example, installing for local development and testing will require a few additional steps compared to installing for testnet. Similarly, installation instructions differ for a subnet owner vs a validator or a miner. +**📝 Medical Device Registration:** Allocate funds to cover the costs associated with registering our solutions as medical devices, ensuring they meet all regulatory requirements for safety and efficacy. -### Install +**📢 Marketing and Awareness:** Implement comprehensive marketing strategies to raise awareness about our solutions and Bittensor project, making them known to both potential users and healthcare professionals. -- **Running locally**: Follow the step-by-step instructions described in this section: [Running Subnet Locally](./docs/running_on_staging.md). -- **Running on Bittensor testnet**: Follow the step-by-step instructions described in this section: [Running on the Test Network](./docs/running_on_testnet.md). -- **Running on Bittensor mainnet**: Follow the step-by-step instructions described in this section: [Running on the Main Network](./docs/running_on_mainnet.md). +**🤝 Collaboration and Networking:** Build strong networks with cancer organizations, researchers, and healthcare providers to facilitate the practical implementation and continuous improvement of our technology. ---- +**📈Continuous Improvement of Algorithms:** Reward top researchers, maintain algorithms in the open domain, and constantly expand our anonymized cancer detection dataset through partnerships and user contributions. -## Writing your own incentive mechanism +**⚖️ Legislative Efforts:** Engage in legislative activities to support the recognition and adoption of AI-driven cancer detection technologies within the medical community. -As described in [Quickstarter template](#quickstarter-template) section above, when you are ready to write your own incentive mechanism, update this template repository by editing the following files. The code in these files contains detailed documentation on how to update the template. Read the documentation in each of the files to understand how to update the template. There are multiple **TODO**s in each of the files identifying sections you should update. These files are: -- `template/protocol.py`: Contains the definition of the wire-protocol used by miners and validators. -- `neurons/miner.py`: Script that defines the miner's behavior, i.e., how the miner responds to requests from validators. -- `neurons/validator.py`: This script defines the validator's behavior, i.e., how the validator requests information from the miners and determines the scores. -- `template/forward.py`: Contains the definition of the validator's forward pass. -- `template/reward.py`: Contains the definition of how validators reward miner responses. +By focusing on these areas, we aim to overcome the barriers to the practical use of AI in cancer detection and provide a solution that is accessible to everyone. -In addition to the above files, you should also update the following files: -- `README.md`: This file contains the documentation for your project. Update this file to reflect your project's documentation. -- `CONTRIBUTING.md`: This file contains the instructions for contributing to your project. Update this file to reflect your project's contribution guidelines. -- `template/__init__.py`: This file contains the version of your project. -- `setup.py`: This file contains the metadata about your project. Update this file to reflect your project's metadata. -- `docs/`: This directory contains the documentation for your project. Update this directory to reflect your project's documentation. +To expedite the process and navigate the complexities of medical certification, we are beginning our initiatives with authorized clinical trials. After completing clinical trials of our first project, **SELFSCAN** – an application for detecting skin cancer through self-made pictures – we will focus on its deployment as a Class II medical device in the USA and Europe, obtaining the necessary FDA and CE approvals. -__Note__ -The `template` directory should also be renamed to your project name. ---- +Concurrently, with the help of the Bittensor community and our unique tokenomics supporting researchers, we will continuously improve the best cancer detection algorithms. This ensures that, by the time our products are brought to market, our solutions surpass all existing algorithms. -# Writing your own subnet API -To leverage the abstract `SubnetsAPI` in Bittensor, you can implement a standardized interface. This interface is used to interact with the Bittensor network and can be used by a client to interact with the subnet through its exposed axons. +Subsequently, we will focus on detecting other types of cancer, starting with breast and lung cancer. -What does Bittensor communication entail? Typically two processes, (1) preparing data for transit (creating and filling `synapse`s) and (2), processing the responses received from the `axon`(s). +For more information about our project visit our website: -This protocol uses a handler registry system to associate bespoke interfaces for subnets by implementing two simple abstract functions: -- `prepare_synapse` -- `process_responses` +[safe-scan.ai](https://www.safe-scan.ai/) -These can be implemented as extensions of the generic `SubnetsAPI` interface. E.g.: +[skin-scan.ai](https://skin-scan.ai/) +# **🌍 REAL-WORLD APPLICATIONS** -This is abstract, generic, and takes(`*args`, `**kwargs`) for flexibility. See the extremely simple base class: -```python -class SubnetsAPI(ABC): - def __init__(self, wallet: "bt.wallet"): - self.wallet = wallet - self.dendrite = bt.dendrite(wallet=wallet) +Our SKIN SCAN app, accessible at [www.skin-scan.ai](http://www.skin-scan.ai/), is designed to bridge the gap between AI's proven efficiency in cancer detection and its limited real-world application. Despite numerous studies validating AI's potential in cancer detection, its use in everyday healthcare is still not widespread. Our app aims to change this by providing a user-friendly, accessible tool for early skin cancer detection. - async def __call__(self, *args, **kwargs): - return await self.query_api(*args, **kwargs) +Building on this foundation, we are developing dedicated software for breast cancer detection, utilizing advanced AI to offer accurate assessments. Following this, we will expand our focus to include lung and brain cancer detection solutions, aiming to make these life-saving technologies widely available and effective in clinical settings. - @abstractmethod - def prepare_synapse(self, *args, **kwargs) -> Any: - """ - Prepare the synapse-specific payload. - """ - ... +SKIN SCAN app live demo: - @abstractmethod - def process_responses(self, responses: List[Union["bt.Synapse", Any]]) -> Any: - """ - Process the responses from the network. - """ - ... +[https://x.com/SAFESCAN_AI/status/1819351129362149876](https://x.com/SAFESCAN_AI/status/1819351129362149876) -``` +# ⚠️ WHY IS SAFESCAN SUBNET IMPORTANT? +tutaj będzie jak protein folding opisane czemu nas wspierać i jak to dobrze zrobi dla bittensor -Here is a toy example: - -```python -from bittensor.subnets import SubnetsAPI -from MySubnet import MySynapse - -class MySynapseAPI(SubnetsAPI): - def __init__(self, wallet: "bt.wallet"): - super().__init__(wallet) - self.netuid = 99 - - def prepare_synapse(self, prompt: str) -> MySynapse: - # Do any preparatory work to fill the synapse - data = do_prompt_injection(prompt) - - # Fill the synapse for transit - synapse = StoreUser( - messages=[data], - ) - # Send it along - return synapse - - def process_responses(self, responses: List[Union["bt.Synapse", Any]]) -> str: - # Look through the responses for information required by your application - for response in responses: - if response.dendrite.status_code != 200: - continue - # potentially apply post processing - result_data = postprocess_data_from_response(response) - # return data to the client - return result_data -``` +SAFE SCAN harnesses the power of the Bittensor network to address one of the world's most pressing issues: cancer detection. Researchers can contribute to refining detection algorithms and earn TAO, with additional royalties for those whose algorithms are integrated into our software. By focusing on obtaining large datasets, including paid and hard-to-access medical data, we ensure the development of superior models. Our decentralized, transparent system guarantees fair competition and protects against model overfitting. With strong community and validator support, we can expand to create and register standalone software for detecting other types of cancer. -You can use a subnet API to the registry by doing the following: -1. Download and install the specific repo you want -1. Import the appropriate API handler from bespoke subnets -1. Make the query given the subnet specific API - - - -# Subnet Links -In order to see real-world examples of subnets in-action, see the `subnet_links.py` document or access them from inside the `template` package by: -```python -import template -template.SUBNET_LINKS -[{'name': 'sn0', 'url': ''}, - {'name': 'sn1', 'url': 'https://github.com/opentensor/prompting/'}, - {'name': 'sn2', 'url': 'https://github.com/bittranslateio/bittranslate/'}, - {'name': 'sn3', 'url': 'https://github.com/gitphantomman/scraping_subnet/'}, - {'name': 'sn4', 'url': 'https://github.com/manifold-inc/targon/'}, -... -] -``` +Additionally with Safe Scan, we can significantly broaden awareness of Bittensor's capabilities and resonate with a more general audience. This will be crucial for the network's growth and increasing market cap, attracting both large and microinvestors. + +# 📢 MARKETING + +Our first goal is to develop the best skin cancer detection algorithm and establish ourselves as a recognized leader in cancer detection. We aim not only to create the most popular and widely accessible skin cancer detection app but also to demonstrate Bittensor's power. We plan to spread awareness through partnerships with skin cancer foundations, growth hacking strategies like affiliate links for unlocking premium features, and promotional support from Apple and Google stores, aiming to reach over 1 million users within 18 months. And every app launch will display “proudly powered by BITTENSOR.” + +However, brand recognition is just the beginning. Our marketing strategy will focus on creating hype by engaging bloggers, reaching to celebrities affected by skin cancer, and sending articles to major tech, health, and news outlets. We will leverage the current interest in AI and blockchain to showcase the life-saving potential of these technologies. + +# 💰 TOKENOMY & ECONOMY + +**🪙 UNIQUE TOKENOMY** + +Our tokenomics are uniquely designed to drive research and development of new algorithms while also supporting real-life applications. + +**Incentives**: Miners with the best-performing algorithms in our ongoing competitions are rewarded through our leaderboard system. The top-ranked miner receives significant incentives, promoting continuous improvement and innovation. + +**Royalties**: Miners whose algorithms are integrated into our app and software for real-life cancer detection applications earn additional 1% of emission royalties. This ensures ongoing motivation for developers to create cutting-edge solutions that contribute to our mission of saving lives + +**📈 SELF-SUSTAINING ECONOMY** + +Although our primary focus is on using our subnet to save lives with state-of-the-art algorithms and custom-made software while promoting the power of Bittensor computing worldwide, our long-term goal is to establish a self-sustaining economy. + +We aim to keep our cancer detection app and software free for those who need it most: regular people and public hospitals, especially in developing countries with limited medical personnel, while offering paid solutions for the private healthcare sector and developed countries. + +- Premium features for paid users (e.g., more lesions for detection, exporting data to doctors, etc.) +- Support from sponsors, donors, cancer foundations, and companies that align with our mission +- Rewards for miners and validators for generating economic value (e.g., analyzing mammography data for private healthcare) +- Proceeds generated from sponsors and end-users will be distributed among the network's participants. + +# **🖧 PARTNERSHIP SUBNETS** + +To create SOTA cancer detection algorithms, we aim to fully leverage the power of the Bittensor community and closely collaborate with other subnets. Therefore, we are establishing partnerships with: + +POTENTIAL COLLABS + +- Subnet 2 - fingerprinting modeli ML - czy miner odpala na tym modelu co trzeba +- subnet 31 - NASChain - optymalizacja layerów neuronów modeli + +# **👨‍👨‍👦‍👦 TEAM COMPOSITION** + +The SafeScan team is not only composed of professionals with diverse expertise in crypto, software development, machine learning, marketing, ux design and business, but we are also close friends united by a shared vision. + +Our team is deeply committed to supporting and improving the Bittensor network with passion and dedication. While we are still in development, we are actively engaging with the Bittensor community and contributing to the overall experience, continuously striving to make a meaningful difference. + +Team members: + +- **@Q. -** Business development +- **@czlowiek** - Project manager & HEAD DEV +- **@Konrad -** Subnet developer +- **@bulubula -** Machine learning engineer +- **@Izuael -** Mobile software Engineer + +# **🛣️ ROADMAP** + +zrobić plan + +Given the complexity of creating a state of the art roleplay LLM, we plan to divide the process into 3 distinct phases. + +**Phase 1:** + +- [x] Subnet launch with robust pipeline for roleplay LLM evaluation on public datasets and response length +- [x] Public model leaderboard based on evaluation criteria +- [x] Introduce Coherence and Creativity as a criteria for live model evaluation + +**Phase 2:** + +- [ ] Publicly release front-end powered by top miner submitted model of the week +- [ ] Segment model submission into different "expert" categories (funny, romantic, therapeutic etc) +- [ ] Models with the highest score in each personality type are chosen as "expert" models and made publicly available on the front-end + +**Phase 3:** + +- [ ] New Mixture of Experts model made as a baseline based on the "expert" models chosen from Phase 2 +- [ ] Robust pipeline to evaluate new MOE model submissions against live evaluation criteria +- [ ] Expand the state of the art in roleplay LLMs through continuous iteration and data collection + +# 👷🏻‍♂️ ENGINEERING ROADMAP + +tu potrzebuje wsparcia + +edgemaxxing for mobile phones + +# **📊 SETUP WandB (HIGHLY RECOMMENDED - VALIDATORS PLEASE READ)** + +Before running your miner and validator, you may also choose to set up Weights & Biases (WANDB). It is a popular tool for tracking and visualizing machine learning experiments, and we use it for logging and tracking key metrics across miners and validators, all of which is available publicly [here](https://wandb.ai/shr1ftyy/sturdy-subnet/table?nw=nwusershr1ftyy). We ***highly recommend*** validators use wandb, as it allows subnet developers and miners to diagnose issues more quickly and effectively, say, in the event a validator were to be set abnormal weights. Wandb logs are collected by default, and done so in an anonymous fashion, but we recommend setting up an account to make it easier to differentiate between validators when searching for runs on our dashboard. If you would *not* like to run WandB, you can do so by adding the flag `--wandb.off` when running your miner/validator. + +Before getting started, as mentioned previously, you'll first need to [register](https://wandb.ai/login?signup=true) for a WANDB account, and then set your API key on your system. Here's a step-by-step guide on how to do this on Ubuntu: + +**Step 1: Installation of WANDB** + +Before logging in, make sure you have the WANDB Python package installed. If you haven't installed it yet, you can do so using pip: -## License -This repository is licensed under the MIT License. -```text -# The MIT License (MIT) -# Copyright © 2024 Opentensor Foundation - -# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated -# documentation files (the “Software”), to deal in the Software without restriction, including without limitation -# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, -# and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all copies or substantial portions of -# the Software. - -# THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO -# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL -# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -# DEALINGS IN THE SOFTWARE. ``` +# Should already be installed with the sturdy repo +pip install wandb +``` + +**Step 2: Obtain Your API Key** + +1. Log in to your Weights & Biases account through your web browser. +2. Go to your account settings, usually accessible from the top right corner under your profile. +3. Find the section labeled "API keys". +4. Copy your API key. It's a long string of characters unique to your account. + +**Step 3: Setting Up the API Key in Ubuntu** + +To configure your WANDB API key on your Ubuntu machine, follow these steps: + +1. **Log into WANDB**: Run the following command in the terminal: + + ``` + wandb login + ``` + +2. **Enter Your API Key**: When prompted, paste the API key you copied from your WANDB account settings. + - After pasting your API key, press `Enter`. + - WANDB should display a message confirming that you are logged in. +3. **Verifying the Login**: To verify that the API key was set correctly, you can start a small test script in Python that uses WANDB. If everything is set up correctly, the script should run without any authentication errors. +4. **Setting API Key Environment Variable (Optional)**: If you prefer not to log in every time, you can set your API key as an environment variable in your `~/.bashrc` or `~/.bash_profile` file: + + ``` + echo 'export WANDB_API_KEY=your_api_key' >> ~/.bashrc + source ~/.bashrc + ``` + + Replace `your_api_key` with the actual API key. This method automatically authenticates you with wandb every time you open a new terminal session. + + +# **👍** RUNNING VALIDATOR + +# ⛏️ RUNNING MINER + +# **🚀 GET INVOLVED** + +1. Visit our  to explore the code behind TensorAlchemy + + [https://camo.githubusercontent.com/e8608a6316b9d88ea49559b15837c90b1c14fb172ca6743b50150cd54f208e26/68747470733a2f2f696d672e736869656c64732e696f2f62616467652f4769744875622d3130303030303f7374796c653d666f722d7468652d6261646765266c6f676f3d676974687562266c6f676f436f6c6f723d7768697465](https://camo.githubusercontent.com/e8608a6316b9d88ea49559b15837c90b1c14fb172ca6743b50150cd54f208e26/68747470733a2f2f696d672e736869656c64732e696f2f62616467652f4769744875622d3130303030303f7374796c653 From 43c8ebe5a0a1e03534c46132703d63d28e061f07 Mon Sep 17 00:00:00 2001 From: LEMSTUDI0 <105062843+LEMSTUDI0@users.noreply.github.com> Date: Fri, 23 Aug 2024 17:25:09 +0200 Subject: [PATCH 038/227] Update README.md --- README.md | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 4426af08..4caee7d1 100644 --- a/README.md +++ b/README.md @@ -3,19 +3,21 @@ *Bittensor Subnet for improving cancer detection algorithms* ![image.png](OUR%20GITHUB%20481afe11c91843b796c4afc6b838d8fb/image.png) - -- [Introduction](https://github.com/impel-intelligence/dippy-bittensor-subnet#introduction) -- Features -- Vision -- [Roadmap](https://github.com/impel-intelligence/dippy-bittensor-subnet#roadmap) -- [Overview of Miner and Validator Functionality](https://github.com/impel-intelligence/dippy-bittensor-subnet#overview-of-miner-and-validator-functionality) - - [Miner](https://github.com/impel-intelligence/dippy-bittensor-subnet#miner) - - [Validator](https://github.com/impel-intelligence/dippy-bittensor-subnet#validator) -- [Running Miners and Validators](https://github.com/impel-intelligence/dippy-bittensor-subnet#running-miners-and-validators) - - [Running a Miner](https://github.com/impel-intelligence/dippy-bittensor-subnet#running-a-miner) - - [Running a Validator](https://github.com/impel-intelligence/dippy-bittensor-subnet#running-a-validator) -- [Contributing](https://github.com/impel-intelligence/dippy-bittensor-subnet#contributing) -- [License](https://github.com/impel-intelligence/dippy-bittensor-subnet#license) +[![Discord Chat](https://img.shields.io/discord/308323056592486420.svg)]([https://discord.gg/bittensor](https://discord.com/channels/1259812760280236122/1262383307832823809)) +[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) + +- [Introduction](#introduction) +- [Features](#Features) +- [Vision](#Vision) +- [Roadmap](#roadmap) +- [Overview of Miner and Validator Functionality](#overview-of-miner-and-validator-functionality) + - [Miner](#miner) + - [Validator](#validator) +- [Running Miners and Validators](#running-miners-and-validators) + - [Running a Miner](#running-a-miner) + - [Running a Validator](#running-a-validator) +- [Contributing](#contributing) +- [License](#license) # **👋 INTRODUCTION** From e3dfbfacb5dea8dcf6d0103336117f40c9ffb241 Mon Sep 17 00:00:00 2001 From: LEMSTUDI0 <105062843+LEMSTUDI0@users.noreply.github.com> Date: Fri, 23 Aug 2024 17:28:02 +0200 Subject: [PATCH 039/227] Update README.md --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 4caee7d1..cb6a511d 100644 --- a/README.md +++ b/README.md @@ -6,9 +6,10 @@ [![Discord Chat](https://img.shields.io/discord/308323056592486420.svg)]([https://discord.gg/bittensor](https://discord.com/channels/1259812760280236122/1262383307832823809)) [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) -- [Introduction](#introduction) +👋 [Introduction](#introduction) - [Features](#Features) - [Vision](#Vision) +- REAL-WORLD APPLICATIONS - [Roadmap](#roadmap) - [Overview of Miner and Validator Functionality](#overview-of-miner-and-validator-functionality) - [Miner](#miner) From 97b8c0eba34abfe7f1c919228b3e08cc243c9061 Mon Sep 17 00:00:00 2001 From: Wojtek Jurkowlaniec Date: Fri, 23 Aug 2024 17:32:28 +0200 Subject: [PATCH 040/227] merge fixes in progress --- cancer_ai/validator/competition_manager.py | 30 +++++++++------ cancer_ai/validator/model_run_manager.py | 1 - neurons/competition_config.py | 4 +- neurons/competition_runner.py | 44 +++++++++++++++++----- 4 files changed, 56 insertions(+), 23 deletions(-) diff --git a/cancer_ai/validator/competition_manager.py b/cancer_ai/validator/competition_manager.py index a89fbb2f..50905764 100644 --- a/cancer_ai/validator/competition_manager.py +++ b/cancer_ai/validator/competition_manager.py @@ -9,12 +9,13 @@ from .dataset_manager import DatasetManager from .model_run_manager import ModelRunManager -from .competition_handlers.melanoma_handler import MelanomaCompetitionHandler +from .competition_handlers.melanoma_handler import MelanomaCompetitionHandler +from cancer_ai.chain_models_store import ChainModelMetadataStore, ChainMinerModel COMPETITION_HANDLER_MAPPING = { - "melaona-1": MelanomaCompetitionHandler, + "melanoma-1": MelanomaCompetitionHandler, } @@ -60,7 +61,7 @@ def __init__( self.dataset_manager = DatasetManager( config, competition_id, dataset_hf_repo, dataset_hf_id, dataset_hf_repo_type ) - self.chain_model_metadata_store = ChainModelMetadataStore(subtensor, subnet_uid ) + self.chain_model_metadata_store = ChainModelMetadataStore(subtensor, subnet_uid) self.hotkeys = [] self.chain_miner_models = {} @@ -70,7 +71,6 @@ def get_state(self): "competition_id": self.competition_id, "model_manager": self.model_manager.get_state(), "category": self.category, - "evaluation_time": self.evaluation_time, } def set_state(self, state: dict): @@ -96,16 +96,22 @@ async def sync_chain_miners(self, hotkeys: list[str]): self.hotkeys = hotkeys bt.logging.info(f"Amount of hotkeys: {len(hotkeys)}") for hotkey in hotkeys: - hotkey_metadata = await self.chain_model_metadata_store.retrieve_model_metadata(hotkey) + hotkey_metadata = ( + await self.chain_model_metadata_store.retrieve_model_metadata(hotkey) + ) if hotkey_metadata: self.chain_miner_models[hotkey] = hotkey_metadata - self.model_manager.hotkey_store[hotkey] = await self.get_miner_model(hotkey) - bt.logging.info(f"Amount of chain miners with models: {len(self.chain_miner_models)}") - + self.model_manager.hotkey_store[hotkey] = await self.get_miner_model( + hotkey + ) + bt.logging.info( + f"Amount of chain miners with models: {len(self.chain_miner_models)}" + ) + async def evaluate(self): await self.dataset_manager.prepare_dataset() path_X_test, y_test = await self.dataset_manager.get_data() - + competition_handler = COMPETITION_HANDLER_MAPPING[self.competition_id]( path_X_test=path_X_test, y_test=y_test ) @@ -124,8 +130,10 @@ async def evaluate(self): run_time_s = time.time() - start_time print("Model prediction ", y_pred) print("Ground truth: ", y_test) - - model_result = competition_handler.get_model_result(y_test, y_pred, run_time_s) + + model_result = competition_handler.get_model_result( + y_test, y_pred, run_time_s + ) self.results.append((hotkey, model_result)) return self.results diff --git a/cancer_ai/validator/model_run_manager.py b/cancer_ai/validator/model_run_manager.py index 374f7786..dce6c6a9 100644 --- a/cancer_ai/validator/model_run_manager.py +++ b/cancer_ai/validator/model_run_manager.py @@ -5,7 +5,6 @@ from .utils import detect_model_format, ModelType from .model_runners.pytorch_runner import PytorchRunnerHandler from .model_runners.tensorflow_runner import TensorflowRunnerHandler -from .model_runners.onnx_runner import ONNXRunnerHandler from .model_runners.onnx_runner import OnnxRunnerHandler diff --git a/neurons/competition_config.py b/neurons/competition_config.py index aa232922..bae72d9d 100644 --- a/neurons/competition_config.py +++ b/neurons/competition_config.py @@ -6,6 +6,8 @@ "evaluation_time": ["02:05", "15:30"], "dataset_hf_repo": "safescanai/test_dataset", "dataset_hf_filename": "skin_melanoma.zip", - "dataset_hf_repo_type": "dataset" + "dataset_hf_repo_type": "dataset", + "wandb_project": "testing_integration", + "wandb_entity": "urbaniak-bruno-safescanai", # TODO: Update this line to official entity } ] \ No newline at end of file diff --git a/neurons/competition_runner.py b/neurons/competition_runner.py index 74793a22..94919e50 100644 --- a/neurons/competition_runner.py +++ b/neurons/competition_runner.py @@ -1,4 +1,5 @@ from cancer_ai.validator.competition_manager import CompetitionManager +from cancer_ai.validator.competition_handlers.base_handler import ModelEvaluationResult from datetime import time, datetime import asyncio @@ -9,7 +10,9 @@ import bittensor as bt from typing import List -from competition_config import competitions +from competition_config import competitions as competitions_cfg + +import wandb # from cancer_ai.utils.config import config @@ -38,6 +41,27 @@ def calculate_next_evaluation_times(evaluation_times) -> List[datetime]: return next_times +def log_results_to_wandb(project, entity, hotkey, evaluation_result: ModelEvaluationResult): + wandb.init(project=project, entity=entity) # TODO: Update this line as needed + + wandb.log({ + "hotkey": hotkey, + "tested_entries": evaluation_result.tested_entries, + "model_test_run_time": evaluation_result.run_time, + "accuracy": evaluation_result.accuracy, + "precision": evaluation_result.precision, + "recall": evaluation_result.recall, + "confusion_matrix": evaluation_result.confusion_matrix.tolist(), + "roc_curve": { + "fpr": evaluation_result.fpr.tolist(), + "tpr": evaluation_result.tpr.tolist() + }, + "roc_auc": evaluation_result.roc_auc + }) + + wandb.finish() + return + async def schedule_competitions( competitions: CompetitionManager, path_config: str @@ -97,24 +121,24 @@ async def schedule_competitions( print("Waiting for next scheduled competition") await asyncio.sleep(60) -def run_all_competitions(path_config: str, competitions: List[dict]) -> None: - for competition_config in competitions: - print("Starting competition: ", competition_config) +def run_all_competitions(path_config: str, competitions_cfg: List[dict]) -> None: + for competition_cfg in competitions_cfg: + print("Starting competition: ", competition_cfg) competition_manager = CompetitionManager( path_config, None, 7, - competition_config["competition_id"], - competition_config["category"], - competition_config["dataset_hf_repo"], - competition_config["dataset_hf_filename"], - competition_config["dataset_hf_repo_type"], + competition_cfg["competition_id"], + competition_cfg["category"], + competition_cfg["dataset_hf_repo"], + competition_cfg["dataset_hf_filename"], + competition_cfg["dataset_hf_repo_type"], ) asyncio.run(competition_manager.evaluate()) if __name__ == "__main__": if True: # run them right away - run_all_competitions(path_config, competitions) + run_all_competitions(path_config, competitions_cfg) else: # Run the scheduling coroutine asyncio.run(schedule_competitions(competitions, path_config)) From 70f3db2c7aebaf299bb0636ea093b83bc639ea15 Mon Sep 17 00:00:00 2001 From: LEMSTUDI0 <105062843+LEMSTUDI0@users.noreply.github.com> Date: Fri, 23 Aug 2024 17:47:47 +0200 Subject: [PATCH 041/227] Update README.md --- README.md | 26 ++++++++++---------------- 1 file changed, 10 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index cb6a511d..32a74976 100644 --- a/README.md +++ b/README.md @@ -7,10 +7,16 @@ [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) 👋 [Introduction](#introduction) -- [Features](#Features) -- [Vision](#Vision) -- REAL-WORLD APPLICATIONS -- [Roadmap](#roadmap) +⚙️ [Features](#Features) +👁️ [Vision](#👁️ VISION) +🌍 [Real-world appliactions](#REAL-WORLD APPLICATIONS) +⚠️ Why is safescan subnet important? +📢 Marketing +💰 [Tokenomy & economy](#Tokenomy & economy) +👨‍👨‍👦‍👦 [Team composition](#Team composition) +🛣️ [Roadmap](#roadmap) +📊 [SETUP WandB (HIGHLY RECOMMENDED - VALIDATORS PLEASE READ)](#SETUP WandB (HIGHLY RECOMMENDED - VALIDATORS PLEASE READ) + - [Overview of Miner and Validator Functionality](#overview-of-miner-and-validator-functionality) - [Miner](#miner) - [Validator](#validator) @@ -121,14 +127,6 @@ We aim to keep our cancer detection app and software free for those who need it - Rewards for miners and validators for generating economic value (e.g., analyzing mammography data for private healthcare) - Proceeds generated from sponsors and end-users will be distributed among the network's participants. -# **🖧 PARTNERSHIP SUBNETS** - -To create SOTA cancer detection algorithms, we aim to fully leverage the power of the Bittensor community and closely collaborate with other subnets. Therefore, we are establishing partnerships with: - -POTENTIAL COLLABS - -- Subnet 2 - fingerprinting modeli ML - czy miner odpala na tym modelu co trzeba -- subnet 31 - NASChain - optymalizacja layerów neuronów modeli # **👨‍👨‍👦‍👦 TEAM COMPOSITION** @@ -146,8 +144,6 @@ Team members: # **🛣️ ROADMAP** -zrobić plan - Given the complexity of creating a state of the art roleplay LLM, we plan to divide the process into 3 distinct phases. **Phase 1:** @@ -170,8 +166,6 @@ Given the complexity of creating a state of the art roleplay LLM, we plan to div # 👷🏻‍♂️ ENGINEERING ROADMAP -tu potrzebuje wsparcia - edgemaxxing for mobile phones # **📊 SETUP WandB (HIGHLY RECOMMENDED - VALIDATORS PLEASE READ)** From 032a79ef906f28a5523058801677e93daaf3f61b Mon Sep 17 00:00:00 2001 From: LEMSTUDI0 <105062843+LEMSTUDI0@users.noreply.github.com> Date: Fri, 23 Aug 2024 17:52:04 +0200 Subject: [PATCH 042/227] Update README.md --- README.md | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 32a74976..505bf4a5 100644 --- a/README.md +++ b/README.md @@ -6,16 +6,16 @@ [![Discord Chat](https://img.shields.io/discord/308323056592486420.svg)]([https://discord.gg/bittensor](https://discord.com/channels/1259812760280236122/1262383307832823809)) [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) -👋 [Introduction](#introduction) -⚙️ [Features](#Features) -👁️ [Vision](#👁️ VISION) -🌍 [Real-world appliactions](#REAL-WORLD APPLICATIONS) -⚠️ Why is safescan subnet important? -📢 Marketing -💰 [Tokenomy & economy](#Tokenomy & economy) -👨‍👨‍👦‍👦 [Team composition](#Team composition) -🛣️ [Roadmap](#roadmap) -📊 [SETUP WandB (HIGHLY RECOMMENDED - VALIDATORS PLEASE READ)](#SETUP WandB (HIGHLY RECOMMENDED - VALIDATORS PLEASE READ) +- [👋 Introduction](#-introduction) +- [⚙️ Features](#️-features) +- [👁️ Vision](#️-vision) +- [🌍 Real-world appliactions](#REAL-WORLD APPLICATIONS) +- [⚠️ Why is safescan subnet important?] +- [📢 Marketing] +- [💰Tokenomy & economy](#Tokenomy & economy) +- [👨‍👨‍👦‍👦 Team composition](#Team composition) +- [🛣️ Roadmap](#roadmap) +- [📊 SETUP WandB (HIGHLY RECOMMENDED - VALIDATORS PLEASE READ)](#SETUP WandB (HIGHLY RECOMMENDED - VALIDATORS PLEASE READ) - [Overview of Miner and Validator Functionality](#overview-of-miner-and-validator-functionality) - [Miner](#miner) From 052cae725488a47abb0011f3fd10281f3a9bf4d5 Mon Sep 17 00:00:00 2001 From: LEMSTUDI0 <105062843+LEMSTUDI0@users.noreply.github.com> Date: Fri, 23 Aug 2024 17:53:17 +0200 Subject: [PATCH 043/227] Update README.md --- README.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 505bf4a5..5ea2f9ed 100644 --- a/README.md +++ b/README.md @@ -9,13 +9,13 @@ - [👋 Introduction](#-introduction) - [⚙️ Features](#️-features) - [👁️ Vision](#️-vision) -- [🌍 Real-world appliactions](#REAL-WORLD APPLICATIONS) -- [⚠️ Why is safescan subnet important?] -- [📢 Marketing] -- [💰Tokenomy & economy](#Tokenomy & economy) -- [👨‍👨‍👦‍👦 Team composition](#Team composition) +- [🌍 Real-world Applications](#real-world-applications) +- [⚠️ Why is SafeScan Subnet Important?](#why-is-safescan-subnet-important) +- [📢 Marketing](#marketing) +- [💰 Tokenomy & Economy](#tokenomy--economy) +- [👨‍👨‍👦‍👦 Team Composition](#team-composition) - [🛣️ Roadmap](#roadmap) -- [📊 SETUP WandB (HIGHLY RECOMMENDED - VALIDATORS PLEASE READ)](#SETUP WandB (HIGHLY RECOMMENDED - VALIDATORS PLEASE READ) +- [📊 SETUP WandB (HIGHLY RECOMMENDED - VALIDATORS PLEASE READ)](#setup-wandb-highly-recommended---validators-please-read) - [Overview of Miner and Validator Functionality](#overview-of-miner-and-validator-functionality) - [Miner](#miner) From de2d16111eeecc61f67f061045cc6051ea0b743d Mon Sep 17 00:00:00 2001 From: LEMSTUDI0 <105062843+LEMSTUDI0@users.noreply.github.com> Date: Fri, 23 Aug 2024 17:55:08 +0200 Subject: [PATCH 044/227] Update README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 5ea2f9ed..2df347e0 100644 --- a/README.md +++ b/README.md @@ -3,6 +3,7 @@ *Bittensor Subnet for improving cancer detection algorithms* ![image.png](OUR%20GITHUB%20481afe11c91843b796c4afc6b838d8fb/image.png) + [![Discord Chat](https://img.shields.io/discord/308323056592486420.svg)]([https://discord.gg/bittensor](https://discord.com/channels/1259812760280236122/1262383307832823809)) [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) From 51c3b5e013dde563300e22f1935e6808a7e7b0bc Mon Sep 17 00:00:00 2001 From: LEMSTUDI0 <105062843+LEMSTUDI0@users.noreply.github.com> Date: Fri, 23 Aug 2024 17:58:53 +0200 Subject: [PATCH 045/227] Update README.md --- README.md | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 2df347e0..6dd04d58 100644 --- a/README.md +++ b/README.md @@ -17,15 +17,10 @@ - [👨‍👨‍👦‍👦 Team Composition](#team-composition) - [🛣️ Roadmap](#roadmap) - [📊 SETUP WandB (HIGHLY RECOMMENDED - VALIDATORS PLEASE READ)](#setup-wandb-highly-recommended---validators-please-read) - -- [Overview of Miner and Validator Functionality](#overview-of-miner-and-validator-functionality) - - [Miner](#miner) - - [Validator](#validator) -- [Running Miners and Validators](#running-miners-and-validators) - - [Running a Miner](#running-a-miner) - - [Running a Validator](#running-a-validator) -- [Contributing](#contributing) -- [License](#license) +- [👍 RUNNING VALIDATOR](#running-validator) +- [⛏️ RUNNING MINER](#running-miner) +- [🚀 GET INVOLVED](#get-involved) +- [📝 LICENSE](#license) # **👋 INTRODUCTION** @@ -221,6 +216,6 @@ To configure your WANDB API key on your Ubuntu machine, follow these steps: # **🚀 GET INVOLVED** -1. Visit our  to explore the code behind TensorAlchemy - - [https://camo.githubusercontent.com/e8608a6316b9d88ea49559b15837c90b1c14fb172ca6743b50150cd54f208e26/68747470733a2f2f696d672e736869656c64732e696f2f62616467652f4769744875622d3130303030303f7374796c653d666f722d7468652d6261646765266c6f676f3d676974687562266c6f676f436f6c6f723d7768697465](https://camo.githubusercontent.com/e8608a6316b9d88ea49559b15837c90b1c14fb172ca6743b50150cd54f208e26/68747470733a2f2f696d672e736869656c64732e696f2f62616467652f4769744875622d3130303030303f7374796c653 +# **📝 LICENSE** + + From c13d185af1c5788cec7146d4a7b54b8e7474fcd1 Mon Sep 17 00:00:00 2001 From: Wojtek Jurkowlaniec Date: Fri, 23 Aug 2024 18:28:01 +0200 Subject: [PATCH 046/227] Update README.md --- README.md | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 6dd04d58..b00a322f 100644 --- a/README.md +++ b/README.md @@ -144,21 +144,22 @@ Given the complexity of creating a state of the art roleplay LLM, we plan to div **Phase 1:** -- [x] Subnet launch with robust pipeline for roleplay LLM evaluation on public datasets and response length -- [x] Public model leaderboard based on evaluation criteria -- [x] Introduce Coherence and Creativity as a criteria for live model evaluation +- [ ] Launch competition for melanoma skin cancer +- [ ] Public model leaderboard based on evaluation criteria +- [ ] Start marketing of Skin Scan app and Bittensor **Phase 2:** -- [ ] Publicly release front-end powered by top miner submitted model of the week -- [ ] Segment model submission into different "expert" categories (funny, romantic, therapeutic etc) -- [ ] Models with the highest score in each personality type are chosen as "expert" models and made publicly available on the front-end +- [ ] Run mutliple competitions at once for other skin cancer types +- [ ] Integrate skin cancer detection models within our Skin Scan app +- [ ] Publicly release website for testing models **Phase 3:** -- [ ] New Mixture of Experts model made as a baseline based on the "expert" models chosen from Phase 2 -- [ ] Robust pipeline to evaluate new MOE model submissions against live evaluation criteria -- [ ] Expand the state of the art in roleplay LLMs through continuous iteration and data collection +- [ ] Optimize skin cancer detection models to create one mixture-of-experts model which will run on mobile device +- [ ] Start a process for certifying models - FDA approval +- [ ] Make competitions for breast cancer + # 👷🏻‍♂️ ENGINEERING ROADMAP From 36bb63c4149adf3cd15ff19fdab1c5941f75a37a Mon Sep 17 00:00:00 2001 From: notbulubula Date: Fri, 23 Aug 2024 18:52:23 +0200 Subject: [PATCH 047/227] fixing melanoma_handler --- .../competition_handlers/base_handler.py | 6 +++--- .../competition_handlers/melanoma_handler.py | 17 ++++++++--------- cancer_ai/validator/competition_manager.py | 4 ++-- 3 files changed, 13 insertions(+), 14 deletions(-) diff --git a/cancer_ai/validator/competition_handlers/base_handler.py b/cancer_ai/validator/competition_handlers/base_handler.py index bf473dcc..e19f76b9 100644 --- a/cancer_ai/validator/competition_handlers/base_handler.py +++ b/cancer_ai/validator/competition_handlers/base_handler.py @@ -21,15 +21,15 @@ class BaseCompetitionHandler: This class initializes the config and competition_id attributes. """ - def __init__(self, path_X_test, y_test) -> None: + def __init__(self, X_test, y_test) -> None: """ Initializes the BaseCompetitionHandler object. Args: - path_X_test (str): Path to the test data. + X_test (list): List of test images. y_test (list): List of test labels. """ - self.path_X_test = path_X_test + self.X_test = X_test self.y_test = y_test @abstractmethod diff --git a/cancer_ai/validator/competition_handlers/melanoma_handler.py b/cancer_ai/validator/competition_handlers/melanoma_handler.py index ab205363..f6ef85b5 100644 --- a/cancer_ai/validator/competition_handlers/melanoma_handler.py +++ b/cancer_ai/validator/competition_handlers/melanoma_handler.py @@ -9,15 +9,14 @@ class MelanomaCompetitionHandler(BaseCompetitionHandler): """ """ - def __init__(self, path_X_test, y_test) -> None: - super().__init__(path_X_test, y_test) + def __init__(self, X_test, y_test) -> None: + super().__init__(X_test, y_test) def preprocess_data(self): - X_test = [] + new_X_test = [] target_size=(224, 224) #TODO: Change this to the correct size - for img_path in self.path_X_test: - img = Image.open(img_path) + for img in self.X_test: img = img.resize(target_size) img_array = np.array(img, dtype=np.float32) / 255.0 img_array = np.array(img) @@ -26,14 +25,14 @@ def preprocess_data(self): img_array = np.transpose(img_array, (2, 0, 1)) # Transpose image to (C, H, W) - X_test.append(img_array) + new_X_test.append(img_array) - X_test = np.array(X_test, dtype=np.float32) + new_X_test = np.array(new_X_test, dtype=np.float32) # Map y_test to 0, 1 - y_test = [1 if y == "True" else 0 for y in self.y_test] + new_y_test = [1 if y == "True" else 0 for y in self.y_test] - return X_test, y_test + return new_X_test, new_y_test def get_model_result(self, y_test: List[float], y_pred: np.ndarray, run_time_s: float) -> ModelEvaluationResult: y_pred_binary = [1 if y > 0.5 else 0 for y in y_pred] diff --git a/cancer_ai/validator/competition_manager.py b/cancer_ai/validator/competition_manager.py index 50905764..3ee9a786 100644 --- a/cancer_ai/validator/competition_manager.py +++ b/cancer_ai/validator/competition_manager.py @@ -110,10 +110,10 @@ async def sync_chain_miners(self, hotkeys: list[str]): async def evaluate(self): await self.dataset_manager.prepare_dataset() - path_X_test, y_test = await self.dataset_manager.get_data() + X_test, y_test = await self.dataset_manager.get_data() competition_handler = COMPETITION_HANDLER_MAPPING[self.competition_id]( - path_X_test=path_X_test, y_test=y_test + X_test=X_test, y_test=y_test ) X_test, y_test = competition_handler.preprocess_data() From 4fa3699a88088c4cbe179fadfa06ff1b0b7780ef Mon Sep 17 00:00:00 2001 From: Wojtek Jurkowlaniec Date: Sat, 24 Aug 2024 04:45:45 +0200 Subject: [PATCH 048/227] upload code with model, working --- cancer_ai/validator/dataset_manager.py | 7 ++-- neurons/miner3.py | 57 +++++++++++++++++--------- neurons/miner_config.py | 17 ++++---- 3 files changed, 49 insertions(+), 32 deletions(-) diff --git a/cancer_ai/validator/dataset_manager.py b/cancer_ai/validator/dataset_manager.py index 62ec9a2d..f26d42b6 100644 --- a/cancer_ai/validator/dataset_manager.py +++ b/cancer_ai/validator/dataset_manager.py @@ -39,7 +39,7 @@ def __init__( self.local_compressed_path = "" print(self.config) self.local_extracted_dir = Path( - self.config.models_dataset_dir, self.competition_id + self.config.dataset_dir, self.competition_id ) self.data: Tuple[List, List] = () self.handler = None @@ -58,7 +58,7 @@ async def download_dataset(self): self.local_compressed_path = HfApi().hf_hub_download( self.hf_repo_id, self.hf_filename, - cache_dir=Path(self.config.models_dataset_dir), + cache_dir=Path(self.config.dataset_dir), repo_type=self.hf_repo_type, ) @@ -78,10 +78,9 @@ async def unzip_dataset(self) -> None: """Unzip dataset""" self.local_extracted_dir = Path( - self.config.models_dataset_dir, self.competition_id + self.config.dataset_dir, self.competition_id ) - bt.logging.info(f"Unzipping dataset '{self.competition_id}'") bt.logging.debug(f"Dataset extracted to: { self.local_compressed_path}") os.system(f"rm -R {self.local_extracted_dir}") print(f"unzip {self.local_compressed_path} -d {self.local_extracted_dir}") diff --git a/neurons/miner3.py b/neurons/miner3.py index 87f1d3f0..cac027fe 100644 --- a/neurons/miner3.py +++ b/neurons/miner3.py @@ -9,10 +9,11 @@ import onnx from neurons.miner_config import get_config, set_log_formatting -from cancer_ai.validator.utils import ModelType +from cancer_ai.validator.utils import ModelType, run_command from cancer_ai.validator.model_run_manager import ModelRunManager, ModelInfo from cancer_ai.validator.dataset_manager import DatasetManager from cancer_ai.validator.model_manager import ModelManager +from datetime import datetime class MinerManagerCLI: @@ -20,16 +21,23 @@ def __init__(self, config: bt.config): self.config = config self.hf_api = HfApi() - async def test_model(self, model_path: str) -> None: - # Placeholder for actual implementation - pass - - def upload_model_to_hf(self) -> None: - """Uploads model to Hugging Face.""" + async def upload_to_hf(self) -> None: + """Uploads model and code to Hugging Face.""" bt.logging.info("Uploading model to Hugging Face.") + now_str = datetime.now().strftime("%Y-%m-%d_%H-%M-%S") + path = self.hf_api.upload_file( + path_or_fileobj=self.config.model_path, + path_in_repo=f"{now_str}-{self.config.competition_id}.onnx", + repo_id=self.config.hf_repo_id, + repo_type="model", + ) path = self.hf_api.upload_file( - path_or_fileobj=self.config.models.model_path, + path_or_fileobj=f"{self.config.code_directory}/code.zip", + path_in_repo=f"{now_str}-{self.config.competition_id}.zip", + repo_id=self.config.hf_repo_id, + repo_type="model", ) + bt.logging.info(f"Uploaded model to Hugging Face: {path}") @staticmethod @@ -52,6 +60,7 @@ async def evaluate_model(self) -> None: self.config.competition_id, "safescanai/test_dataset", "skin_melanoma.zip", + "dataset", ) await dataset_manager.prepare_dataset() @@ -65,16 +74,23 @@ async def evaluate_model(self) -> None: if self.config.clean_after_run: dataset_manager.delete_dataset() + async def compress_code(self) -> str: + bt.logging.info("Compressing code") + out, err = await run_command( + f"zip {self.config.code_directory}/code.zip {self.config.code_directory}/*" + ) + return f"{self.config.code_directory}/code.zip" + async def submit_model(self) -> None: - # The wallet holds the cryptographic key pairs for the miner. - bt.logging.info( - f"Initializing connection with Bittensor subnet {self.config.netuid} - Safe-Scan Project" - ) - bt.logging.info(f"Subtensor network: {self.config.subtensor.network}") - bt.logging.info(f"Wallet hotkey: {self.config.wallet.hotkey.ss58_address}") - wallet = bt.wallet(config=self.config) - subtensor = bt.subtensor(config=self.config) - metagraph = subtensor.metagraph(self.config.netuid) + # The wallet holds the cryptographic key pairs for the miner. + bt.logging.info( + f"Initializing connection with Bittensor subnet {self.config.netuid} - Safe-Scan Project" + ) + bt.logging.info(f"Subtensor network: {self.config.subtensor.network}") + bt.logging.info(f"Wallet hotkey: {self.config.wallet.hotkey.ss58_address}") + wallet = bt.wallet(config=self.config) + subtensor = bt.subtensor(config=self.config) + metagraph = subtensor.metagraph(self.config.netuid) async def main(self) -> None: bt.logging(config=self.config) @@ -85,11 +101,12 @@ async def main(self) -> None: match self.config.action: case "submit": - await self.submit_model() + await self.submit_model() case "evaluate": await self.evaluate_model() case "upload": - self.upload_model_to_hf() + await self.compress_code() + await self.upload_to_hf() case _: bt.logging.error(f"Unrecognized action: {self.config.action}") @@ -99,4 +116,4 @@ async def main(self) -> None: set_log_formatting() load_dotenv() cli_manager = MinerManagerCLI(config) - asyncio.run(cli_manager.main()) \ No newline at end of file + asyncio.run(cli_manager.main()) diff --git a/neurons/miner_config.py b/neurons/miner_config.py index 67d384d3..f82c3859 100644 --- a/neurons/miner_config.py +++ b/neurons/miner_config.py @@ -81,7 +81,7 @@ def get_config() -> bt.config: ) main_parser.add_argument( - "--models_dataset_dir", + "--dataset_dir", type=str, help="Path for storing datasets.", default="./datasets", @@ -93,11 +93,7 @@ def get_config() -> bt.config: type=str, required=False, help="Hugging Face model repository ID", - ) - main_parser.add_argument( - "--hf_file_path", - type=str, - help="Hugging Face model file path", + default="eatcats/melanoma-test", ) main_parser.add_argument( @@ -106,6 +102,12 @@ def get_config() -> bt.config: help="Whether to clean up (dataset, temporary files) after running", default=False, ) + main_parser.add_argument( + "--code-directory", + type=str, + help="Path to code directory", + default=".", + ) # Add additional args from bt modules bt.wallet.add_args(main_parser) @@ -116,10 +118,9 @@ def get_config() -> bt.config: # config = bt.config(main_parser) # parsed = main_parser.parse_args() # config = bt.config(main_parser) - # print(config) + config = main_parser.parse_args() - # print(config) config.logging_dir = "./" config.record_log = True config.trace = True From db3fb9da038610bc7dbf8f4adfb881fdf7d42527 Mon Sep 17 00:00:00 2001 From: LEMSTUDI0 <105062843+LEMSTUDI0@users.noreply.github.com> Date: Mon, 26 Aug 2024 12:30:25 +0200 Subject: [PATCH 049/227] Update README.md --- README.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index b00a322f..9f90c496 100644 --- a/README.md +++ b/README.md @@ -7,9 +7,9 @@ [![Discord Chat](https://img.shields.io/discord/308323056592486420.svg)]([https://discord.gg/bittensor](https://discord.com/channels/1259812760280236122/1262383307832823809)) [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) -- [👋 Introduction](#-introduction) -- [⚙️ Features](#️-features) -- [👁️ Vision](#️-vision) +- [👋 Introduction](#introduction) +- [⚙️ Features](#features) +- [👁️ Vision](#vision) - [🌍 Real-world Applications](#real-world-applications) - [⚠️ Why is SafeScan Subnet Important?](#why-is-safescan-subnet-important) - [📢 Marketing](#marketing) @@ -22,6 +22,7 @@ - [🚀 GET INVOLVED](#get-involved) - [📝 LICENSE](#license) + # **👋 INTRODUCTION** Welcome to Safe Scan Cancer AI Detection, a groundbreaking initiative leveraging the power of AI and blockchain technology to revolutionize cancer detection. Our mission is to make advanced cancer detection algorithms accessible and free for everyone. Through our project, we aim to provide cutting-edge, open-source tools that support early cancer diagnosis for patients and healthcare professionals worldwide. From 650ebaa2e0302c495cad06f8e3e518bba4dd5791 Mon Sep 17 00:00:00 2001 From: Wojtek Jurkowlaniec Date: Mon, 26 Aug 2024 12:30:48 +0200 Subject: [PATCH 050/227] here you go --- DOCS/validator.md | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 DOCS/validator.md diff --git a/DOCS/validator.md b/DOCS/validator.md new file mode 100644 index 00000000..e69de29b From efb8edc3989683ce6f696c2c70df353edd7e8ed3 Mon Sep 17 00:00:00 2001 From: LEMSTUDI0 <105062843+LEMSTUDI0@users.noreply.github.com> Date: Mon, 26 Aug 2024 12:33:47 +0200 Subject: [PATCH 051/227] Update README.md --- README.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 9f90c496..6789b121 100644 --- a/README.md +++ b/README.md @@ -89,7 +89,7 @@ SKIN SCAN app live demo: [https://x.com/SAFESCAN_AI/status/1819351129362149876](https://x.com/SAFESCAN_AI/status/1819351129362149876) -# ⚠️ WHY IS SAFESCAN SUBNET IMPORTANT? +# **⚠️ WHY IS SAFESCAN SUBNET IMPORTANT?** tutaj będzie jak protein folding opisane czemu nas wspierać i jak to dobrze zrobi dla bittensor @@ -97,13 +97,14 @@ SAFE SCAN harnesses the power of the Bittensor network to address one of the wor Additionally with Safe Scan, we can significantly broaden awareness of Bittensor's capabilities and resonate with a more general audience. This will be crucial for the network's growth and increasing market cap, attracting both large and microinvestors. -# 📢 MARKETING +# **📢 MARKETING** + Our first goal is to develop the best skin cancer detection algorithm and establish ourselves as a recognized leader in cancer detection. We aim not only to create the most popular and widely accessible skin cancer detection app but also to demonstrate Bittensor's power. We plan to spread awareness through partnerships with skin cancer foundations, growth hacking strategies like affiliate links for unlocking premium features, and promotional support from Apple and Google stores, aiming to reach over 1 million users within 18 months. And every app launch will display “proudly powered by BITTENSOR.” However, brand recognition is just the beginning. Our marketing strategy will focus on creating hype by engaging bloggers, reaching to celebrities affected by skin cancer, and sending articles to major tech, health, and news outlets. We will leverage the current interest in AI and blockchain to showcase the life-saving potential of these technologies. -# 💰 TOKENOMY & ECONOMY +# **💰 TOKENOMY & ECONOMY** **🪙 UNIQUE TOKENOMY** From baf36773461bc5f388c1b629064148fb1f8c6114 Mon Sep 17 00:00:00 2001 From: LEMSTUDI0 <105062843+LEMSTUDI0@users.noreply.github.com> Date: Mon, 26 Aug 2024 12:34:24 +0200 Subject: [PATCH 052/227] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 6789b121..969b43cb 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ - [👁️ Vision](#vision) - [🌍 Real-world Applications](#real-world-applications) - [⚠️ Why is SafeScan Subnet Important?](#why-is-safescan-subnet-important) -- [📢 Marketing](#marketing) +- [📢 Marketing](#MARKETING) - [💰 Tokenomy & Economy](#tokenomy--economy) - [👨‍👨‍👦‍👦 Team Composition](#team-composition) - [🛣️ Roadmap](#roadmap) From 0e7f6f1a6d8d067d82f10ecb5e73cf146c69f1e2 Mon Sep 17 00:00:00 2001 From: LEMSTUDI0 <105062843+LEMSTUDI0@users.noreply.github.com> Date: Mon, 26 Aug 2024 12:38:28 +0200 Subject: [PATCH 053/227] Update README.md --- README.md | 68 +++++++++++++++++++++++++++++++------------------------ 1 file changed, 39 insertions(+), 29 deletions(-) diff --git a/README.md b/README.md index 969b43cb..507486fb 100644 --- a/README.md +++ b/README.md @@ -12,15 +12,15 @@ - [👁️ Vision](#vision) - [🌍 Real-world Applications](#real-world-applications) - [⚠️ Why is SafeScan Subnet Important?](#why-is-safescan-subnet-important) -- [📢 Marketing](#MARKETING) +- [📢 Marketing](#marketing) - [💰 Tokenomy & Economy](#tokenomy--economy) - [👨‍👨‍👦‍👦 Team Composition](#team-composition) - [🛣️ Roadmap](#roadmap) - [📊 SETUP WandB (HIGHLY RECOMMENDED - VALIDATORS PLEASE READ)](#setup-wandb-highly-recommended---validators-please-read) -- [👍 RUNNING VALIDATOR](#running-validator) -- [⛏️ RUNNING MINER](#running-miner) -- [🚀 GET INVOLVED](#get-involved) -- [📝 LICENSE](#license) +- [👍 RUNNING VALIDATOR](#-running-validator) +- [⛏️ RUNNING MINER](#-running-miner) +- [🚀 GET INVOLVED](#-get-involved) +- [📝 LICENSE](#-license) # **👋 INTRODUCTION** @@ -47,7 +47,7 @@ This repository contains subnet code to run on Bittensor network. 💸 Self-sustaining economy -# 👁️ VISION +# **👁️ VISION** Cancer is one of the most significant challenges of our time, and we believe that AI holds the key to addressing it. However, this solution should be accessible and free for everyone. Machine vision technology has long proven effective in early diagnosis, which is crucial for curing cancer. Yet, until now, it has largely remained in the realm of whitepapers. SAFESCAN is a project dedicated to aggregating and enhancing the best algorithms for detecting various types of cancer and providing free computational power for practical cancer detection. We aim to create open-source products that support cancer diagnosis for both patients and doctors. @@ -99,7 +99,6 @@ Additionally with Safe Scan, we can significantly broaden awareness of Bittensor # **📢 MARKETING** - Our first goal is to develop the best skin cancer detection algorithm and establish ourselves as a recognized leader in cancer detection. We aim not only to create the most popular and widely accessible skin cancer detection app but also to demonstrate Bittensor's power. We plan to spread awareness through partnerships with skin cancer foundations, growth hacking strategies like affiliate links for unlocking premium features, and promotional support from Apple and Google stores, aiming to reach over 1 million users within 18 months. And every app launch will display “proudly powered by BITTENSOR.” However, brand recognition is just the beginning. Our marketing strategy will focus on creating hype by engaging bloggers, reaching to celebrities affected by skin cancer, and sending articles to major tech, health, and news outlets. We will leverage the current interest in AI and blockchain to showcase the life-saving potential of these technologies. @@ -112,7 +111,7 @@ Our tokenomics are uniquely designed to drive research and development of new al **Incentives**: Miners with the best-performing algorithms in our ongoing competitions are rewarded through our leaderboard system. The top-ranked miner receives significant incentives, promoting continuous improvement and innovation. -**Royalties**: Miners whose algorithms are integrated into our app and software for real-life cancer detection applications earn additional 1% of emission royalties. This ensures ongoing motivation for developers to create cutting-edge solutions that contribute to our mission of saving lives +**Royalties**: Miners whose algorithms are integrated into our app and software for real-life cancer detection applications earn additional 1% of emission royalties. This ensures ongoing motivation for developers to create cutting-edge solutions that contribute to our mission of saving lives. **📈 SELF-SUSTAINING ECONOMY** @@ -125,10 +124,14 @@ We aim to keep our cancer detection app and software free for those who need it - Rewards for miners and validators for generating economic value (e.g., analyzing mammography data for private healthcare) - Proceeds generated from sponsors and end-users will be distributed among the network's participants. +# **👨‍👨‍👦‍👦 TEAM COMPOSITION** + +The SafeScan team is not only composed of professionals with diverse expertise in crypto, software development, machine learning, marketing, UX designIt seems like the message was cut off. Let me continue the full content with properly functioning links: +```markdown # **👨‍👨‍👦‍👦 TEAM COMPOSITION** -The SafeScan team is not only composed of professionals with diverse expertise in crypto, software development, machine learning, marketing, ux design and business, but we are also close friends united by a shared vision. +The SafeScan team is not only composed of professionals with diverse expertise in crypto, software development, machine learning, marketing, UX design, and business, but we are also close friends united by a shared vision. Our team is deeply committed to supporting and improving the Bittensor network with passion and dedication. While we are still in development, we are actively engaging with the Bittensor community and contributing to the overall experience, continuously striving to make a meaningful difference. @@ -142,46 +145,47 @@ Team members: # **🛣️ ROADMAP** -Given the complexity of creating a state of the art roleplay LLM, we plan to divide the process into 3 distinct phases. +Given the complexity of creating a state-of-the-art roleplay LLM, we plan to divide the process into 3 distinct phases. **Phase 1:** -- [ ] Launch competition for melanoma skin cancer -- [ ] Public model leaderboard based on evaluation criteria -- [ ] Start marketing of Skin Scan app and Bittensor +- [ ] Launch competition for melanoma skin cancer +- [ ] Public model leaderboard based on evaluation criteria +- [ ] Start marketing of Skin Scan app and Bittensor **Phase 2:** -- [ ] Run mutliple competitions at once for other skin cancer types -- [ ] Integrate skin cancer detection models within our Skin Scan app -- [ ] Publicly release website for testing models +- [ ] Run multiple competitions at once for other skin cancer types +- [ ] Integrate skin cancer detection models within our Skin Scan app +- [ ] Publicly release website for testing models **Phase 3:** -- [ ] Optimize skin cancer detection models to create one mixture-of-experts model which will run on mobile device -- [ ] Start a process for certifying models - FDA approval -- [ ] Make competitions for breast cancer - +- [ ] Optimize skin cancer detection models to create one mixture-of-experts model which will run on mobile devices +- [ ] Start the process for certifying models - FDA approval +- [ ] Make competitions for breast cancer # 👷🏻‍♂️ ENGINEERING ROADMAP -edgemaxxing for mobile phones +Edgemaxxing for mobile phones. # **📊 SETUP WandB (HIGHLY RECOMMENDED - VALIDATORS PLEASE READ)** -Before running your miner and validator, you may also choose to set up Weights & Biases (WANDB). It is a popular tool for tracking and visualizing machine learning experiments, and we use it for logging and tracking key metrics across miners and validators, all of which is available publicly [here](https://wandb.ai/shr1ftyy/sturdy-subnet/table?nw=nwusershr1ftyy). We ***highly recommend*** validators use wandb, as it allows subnet developers and miners to diagnose issues more quickly and effectively, say, in the event a validator were to be set abnormal weights. Wandb logs are collected by default, and done so in an anonymous fashion, but we recommend setting up an account to make it easier to differentiate between validators when searching for runs on our dashboard. If you would *not* like to run WandB, you can do so by adding the flag `--wandb.off` when running your miner/validator. +Before running your miner and validator, you may also choose to set up Weights & Biases (WANDB). It is a popular tool for tracking and visualizing machine learning experiments, and we use it for logging and tracking key metrics across miners and validators, all of which is available publicly [here](https://wandb.ai/shr1ftyy/sturdy-subnet/table?nw=nwusershr1ftyy). We ***highly recommend*** validators use WandB, as it allows subnet developers and miners to diagnose issues more quickly and effectively, say, in the event a validator were to be set abnormal weights. WandB logs are collected by default and done so in an anonymous fashion, but we recommend setting up an account to make it easier to differentiate between validators when searching for runs on our dashboard. If you would *not* like to run WandB, you can do so by adding the flag `--wandb.off` when running your miner/validator. -Before getting started, as mentioned previously, you'll first need to [register](https://wandb.ai/login?signup=true) for a WANDB account, and then set your API key on your system. Here's a step-by-step guide on how to do this on Ubuntu: +Before getting started, as mentioned previously, you'll first need to [register](https://wandb.ai/login?signup=true) for a WANDB account, and then set your API key on your system. Here's a step-by-step guide on how to do this on Ubuntu: **Step 1: Installation of WANDB** Before logging in, make sure you have the WANDB Python package installed. If you haven't installed it yet, you can do so using pip: + ``` # Should already be installed with the sturdy repo pip install wandb ``` + **Step 2: Obtain Your API Key** 1. Log in to your Weights & Biases account through your web browser. @@ -200,25 +204,31 @@ To configure your WANDB API key on your Ubuntu machine, follow these steps: ``` 2. **Enter Your API Key**: When prompted, paste the API key you copied from your WANDB account settings. - - After pasting your API key, press `Enter`. + - After pasting your API key, press `Enter`. - WANDB should display a message confirming that you are logged in. 3. **Verifying the Login**: To verify that the API key was set correctly, you can start a small test script in Python that uses WANDB. If everything is set up correctly, the script should run without any authentication errors. -4. **Setting API Key Environment Variable (Optional)**: If you prefer not to log in every time, you can set your API key as an environment variable in your `~/.bashrc` or `~/.bash_profile` file: +4. **Setting API Key Environment Variable (Optional)**: If you prefer not to log in every time, you can set your API key as an environment variable in your `~/.bashrc` or `~/.bash_profile` file: ``` echo 'export WANDB_API_KEY=your_api_key' >> ~/.bashrc source ~/.bashrc ``` - Replace `your_api_key` with the actual API key. This method automatically authenticates you with wandb every time you open a new terminal session. - + Replace `your_api_key` with the actual API key. This method automatically authenticates you with WandB every time you open a new terminal session. + +# **👍 RUNNING VALIDATOR** -# **👍** RUNNING VALIDATOR +... -# ⛏️ RUNNING MINER +# **⛏️ RUNNING MINER** + +... # **🚀 GET INVOLVED** +... + # **📝 LICENSE** +... From 47a758a69a3a0d6d1e131cd6e83d5f9d90219470 Mon Sep 17 00:00:00 2001 From: LEMSTUDI0 <105062843+LEMSTUDI0@users.noreply.github.com> Date: Mon, 26 Aug 2024 12:53:34 +0200 Subject: [PATCH 054/227] Add files via upload --- DOCS/header.png | Bin 0 -> 50570 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 DOCS/header.png diff --git a/DOCS/header.png b/DOCS/header.png new file mode 100644 index 0000000000000000000000000000000000000000..837fdbd0dacf6c58eaaa0f9b1a789a174b818f6b GIT binary patch literal 50570 zcmYhi1yqz<`#wx}ccXN7BOoB%Al*{Z3@I%j-O`P8NypHg(j7w$(lAIjeB=53&w0PK zhBb?4_OtWecV1Vdx~d!|8W|c43=F1%{0B`K82AGi7}x|9IOvml&UFFk7p$A6oD}TJ zC|e-(A5<54eK!~wbo{@6urQg~Brq_8oC+T#KYqzR$w5xXUm*HgTEynTtaPTB#Gt)B zGa#iwfD4;g;GVGIYwUAIiCCnR$Pygp4W>Z$)P?=W%Bh) zK>La%&+g=S;90gW)>?-TM#vkK;J+UQm8j(Lmh~8gPNE?6XmH&BJw|}*9KHxuDgagn z|NDqeOuM3C^M-5id4%D-0)d4I@_!x3<0Xj+9`bHvGf<|@v*JzX18M(X&kA~qCH{e3 zUxYEf1K^`X{}~SDdvb?qmLq6ppq}LI4ibSEf+P;~LpLf<-K(r;7nkHo;3fFKE<$>H z=Iwiz%4{MYF$dQ%{~L2wy^(LtA~~`a%SI01^}o*f+^Nl(bS4w#?Gqi|1LXd8jUf5` z>!;!s)&3Y+{F1Np=>JUw0V8v4pVgrUh4lHq0V5|2=~r?sl+%@mXw^Zu|Gb{?A#^8w zV+R8Va zd8q5Kz9+=aFZtg#1laU^fJ}TUDKd%vjrs4)z@{5)AKK!1O64c$p*=YN*Td_D0(`TH z5-&rN!kd3jC?b%QYvqqMuC!*2Og-|y3qGeUpG&Zn2<2vrYhon$Z)WiWyGh)P*9Hi% z2>)vtQhMQyZm~{(E6)pK@@!gvbzt*pUTbe0cYDyy)hz9K(h=*w#>CD9Z`S3qv`TvY zcpz`T)nYu}1-TqHesPI+fzVyPW7+s`Z*T%NT9Gz8M6p`h_q_>(A6ow0I^Vc4e%$=; zsP3X~;4!Q!|FBsH6-k2OET}e#b*7r~#NJYy>i%~b?xI5tRoI4qw6NCckJdf+<(dv? z$$iM7mi$-do}}t#m2YRP+1lvBxsbgKTdEnBIB?^|!~d0%yHKQ`cbV7RJ?pS z9wz%Q_-%vror!|}*W~S7tT);P^cZ2`yk9Su5EYm{SJZessylABzfj(a(A$P?z5 z9m5BoBJ4fg?3(NH=o6g!?SG9}0t{Wk(hVdD4-1yMbsmKsxFVUz0H5Zlc!Zcv1av+nQ!p9v9b)KIkmsNki6YO5>vj8loKjsunQ$n9g?Bu``;ab9^LG(Mp-z zQ@smRm{9Q)b+!!L1xr*PAIQ94iiErxPe-Mb|Ed_ey2E$rf$as{^BrP)^cfP1o(|p4 zt|LtUjI=e;0t`q}&i;fp9sA4j3uCmIT9k(r98zHZbiFi2nXR@42c$Bcqv&}6rJw6jSlbI+cY&aTHm|d0x~JENj}r?BU2iadAi#$ARP_$ z%@8}coaGNw3PFyB*A9eTWtw;JS?r}#PH(Z#lES@`UfN9IvmD6 zI#kTho?s2<6;LI)F%Hbh4e|~TAbWiuo{>z*r}K3ZaLxVb!|Xord=3xL3_LzPetAyC z!Mi23pJ3K{EV+)LnXAyLQX>j|BX0i3U27xw3imb?-{H}7E290wg29uZacq~tYHG?& zlLQ_1F(w%xu#!%t({g*uNL-SPr{^j4R7aI9n=X6)l%PSnIJrHMxeP83uF~R ztMQWtTv4FA+L5!EKf&-i?!ei+ZQqhD84Flkbk_B=GmSrGRQq2mDDR3eXe|3Cx?LD? zF0fu$SNxXvG@W>SC!FqvUV*yTdgS-sbD#rms+Rvg&!;HgN%-#L0=re^V}NHMGkp=(^8!r`kUs-N->v?T zXC#gKIUCe*apV$KY_`EP-rF6czkEen>?JG45KsMGD||4zw25kckG-eUI5#A|XFW}j zON}Z4*=saZ3w)2?Ms=5|#dVLS4hgg|)|}ZNog!o!v!0v`wK3ByHqVow*H9aY{q=h} z;C%2Bfqyu;Jfooa-pILjn{>JHP%k7XsGaSp-8U&)oW-ux_E$pWM7BzolIaH}*1a3w zhF$CaxxvS(iJ^mx%v5gPEau85rET(U-vx0va_usNv~H9%4yjeyqzu4GkD=~p_v^?{ zDR{f`AW^|08R5W*X9i>wI9Af!+?8&vcO7RQJx<$r$%Pcx$)HvaQ&=IU*3JBm`zHmB zCueL)C05NS){ogrX}Dd{v9WLCsT0`t@@>A51=;uqaKop5_!L}Uvy z&~JElA>VA1hl2PFIOQz?;acKyCLGXvN>k~6QGyz-t5^OkVMVn;yTlo#%Mn53Pi&TI zwWY^o+6@R5no|Fwwh?SJGfR;zWIM;*?M#r}KK~QgU$IwgOmF7sY&_=+Z20}f!^&-U zS817{*uGH@|#qP%1T>;yT2UV8i=bFJvLT=k^A!~9DY~Fs< zs@{jI0fvB89|XQIztAH3{s4K($s*NXeNBjykjnjHy4(G@<8?Gr7voT*RAKTo2FfIc z>{xGwT(Ly5%R}0N7rc&)`Z`iVY+1`wI&Hg~Q?FJdXGPtQm7)6Y2-^)u$uSW=$;y-hmkDv)vdYV+}>lMqLI0#;o z%~15a=KNC27rTmHyVu@~((RmYV+WsjDyY!4SGz!YG#CZZu>teAJ)qnZ-+p%RzL=g^ zS@16^*IW70VhH@@>4Zam8GK=V@Kbm%giM<;ZtS!%#a1F?`-R09u>E1J z>lE&!-!AW>>4@AYem+Y0hVs~SJzyIXy(N@6tu?Te%em?)dfRiu_nlSjS|AmAyC4nv z$F{b35`ifwV;k86Iu9ImAOtpk!&W31tmE;YUhJFqFTmyRw|8y#ZfVf9@l4@lPJK6>bA zml0mRjIRA0WfE&YKY6ENd{}?tQ_mDMSJ8*;cYo_$>v@XUr~h;&F?;BD;w&hN$l(uP zqCQzaiODvjfzWln6NYpr`^T4MS}b+4o;|oU46}1UL+vbn+ge-*o)jls z*1hJ$(3aTahY(D+1XWJNUH26}I!&tEK~~XA2WHlbeXXkf96|(Sf!xB*6fV7_^GI&q z3=%|6*7_EIqJ5PcVZbkU8esY5Et6)N+2*2ceum>SvzpgR3zML2zNKpNaOlyxP2XbgE=#*Pm_KeMzQr zaemw?J+oe^9yf7yy_Rp_ub^I~5yiFGI%I|c1NFjd)<=2{N0Q+g`>xCdTKX}uwvIHq zQ+{ZAU^gR*AhGUsv>ldd6cfD;wuChvUU9hJ>X;uYXUWQaQ~?cN)h;^QmYo+rJ%oUB zmG1-4r$~Y&KJG1At>xcDW>%sNu6Kh2cF+`3G6ABdW2h19jc*rSweN1=-`PW4Mnwa^ zbA+~T{tlZg07-1ME2iGQ8C{M3%v0WdIws71{Toxc)0tVBf%_QYa8A#F+ZXrMgh!bx1&aw98t_uCu>hv zt9-(=1}W#!+K}hlOQxW4u&LDH>*0d^7DR=oDrB(08?N}*h$ofZ$I7D$q|aA2Q9F|e z9YAT~&d;Qr_{4oq6dR)jdk{NtmsRm|hzQbA1FqU2>f zfyDXbdp8X{^PaRh_FSH6MM?&>luXBs{;io+5g{+5Y=QZTW}7lFmRIU9wtR+S z@|hGstpuSH*e~3#*y3?KIT4vz_Rn@h5Q$Zz5U&WYJP|p(U2F?YK$pMw!=`0HojrK{ z7*5XKF5v+?2wo>Vpx?%}(#Y6niRPR7#)YG__4PYMze4-1twfgMwJ&9W>dLRo<5hTj z+(kLNLxcbvI6~HFFMqNvghZTcRutdLd28_;W&R{&&Awnk4rwz&R%VDsx&bM5pjsiC1tKAQ5he?Gp{+`_d;6P|tCUAw;BNa$HbZce@JnJ^#UI^1)BdIQ3oTtUQ%qFJdL*ulCtzFh7i0vQMH%7X1>Y zPy6K~c67<|&6AY={Q&-LOY6pt-y&W9aG5|L;134A$5`PK0uUJD9%uSV_4pKpqfk7E z2QXkf!SCfv*l2^iXzz6QoAtgGd;emjIybZlrkM4D5?L!wV;sGtpSWuZ2Fd0*;y#%p zJYb&Y){hC&Q;Vh3w30WCeI`6qH9wR%_8~u5}8kY zRj$J|_vPhQVkZm8x0E*X8-xnydod$+am8))ySU3VL$SjFt`T1eW3GX@*pV&{2rq4u z;o8k?DtJ)l{ zYBt2dWB*~qNQi#dK3KwLyl;ujOU?pdILePV5Elh!4coPv8iO^;4|13u#WAy zWa$aLHHJ5Qc9yi}UvQOC4E>nPvk{HV=N6BG2d`?2;PP zq|lOx0s?mUXC~6x9_FZ(VxIXucqYa^@eU=Qz{>|0V$NS4g&wL-#J}j5puO8AXMpevxa(-c5&Wb{9VL3Qwke3RwzOj0Zvu6a4X*h# zr`{wyVwn!G_{JpZrZ0T3-1IcxacyVYLla@A4wziBE%1=Lor`EMWy`T$YfY>!GF!F? z;IbJXr@B{K?vxSozSExifOK^rg2AIxR5S$r&^`gt^}KjZpz3r`AdBYt;Nl=MXQlFZ zf9@9uBfCB*9ghqFM+rB#3-0{XD@QXb^c;$~BX!;;csaFsEZMDb;&XYXXkV^UegiM( z)+@9YU|@s6@ggRxWe4ixFPds0(tlvKgRUglI1?3Si9;^=9Y}QkVO0JN{uP)b$KYFI zRmO6sB7xV`A5~EN3BL98P!yjme>za3I?HTE18F{6~Y?vrZ<%D@`1qwY!=mNr9v7Fa& zYx*OdhL%l8mNu(70^$Q?iB#&SS4!Iy9|kFl9A{k>8>gN{i_TiEls5BiOZ=jW9_!uY zByz$s-*cqYOp%$3UX^xh3i|ya4SXnjghRwMAhla8C1K>hQ;`NqS~A|Wv=MQ*!1?0c zFkWlTh3Q$@mTMMUmsasy@*yLghlqiG5*BviO@n80ogDmULQVQU7!Mg#u+Lr*a^3_U-y-NX)$k7C+(e#IkH}$a=SgU((H&Gbk6?%`rnoXKZU7w^cq@CIVOE3t5^O+Y(^sRGSindnO1v z#Hy1jVenQJ$_Ui5^cRkm+l7uXUNr@;kC_z3p?~G{v^AS4(=UUtzLucK+|2m`WC5$mqr@M`uCC9gRA(%o6RDmKnru%Qu#*`em%bl>68 z=il@^u6W||Ah%_yM%iui)1jCdoQOfRlhxQ|uEAy5(fA%f*P{>2uxJWF`!3tuh%8OLF9z|IaX)Z zso}Rptm4DGyi1z(K)p{HRI$u6Y&^-4Mif`h_p35npu9E_Wl|#Sb`;O#OWmdlF~t&$ zdMb%O25@D+Bm$)<<}R-%lLt9OYxOhEj)v^gCSkX5SZUuU&*3Z#7^@lkL#NCvM}LVCmXcW<9850 zLD)Tp<|a}nVp?>2SAWgb!yI~x!0r?C*I7p;yWE5_%bioEY&E83a1_qFT5^iV(@aOZ z`9u3CEG+uNa;Dfu13S{E!bX0QqNM?PR?>`o9I0MZviW!>XeT}Sanscb^r?YOHBlqO zyIcTVZF7>Ou~|I|Z>sG-mYGD^l;G+L)yV1GW@UDf{9w@(iC0IF^iUWx+bd#N_iLeW<{7GgHk6BHH>>^UoiB379jlfbW065GBXBOry2~X>06mGO0@0werl;{u#b7^ ztNR*Ykdo=Q-O}m6m2)(@qdf0fKA9Yy8|7OqxNP3Qe3n*Dc(9Zfm0uEFI;gS!ZW8YB z!^BbY&V)FEQG+XHmm<6Lq~8Vxt`|#@ zCwx(#=*BE4-?F&oT~=Q7360nf*DvQph99sb9|}Qh`YybK=b~Rivl(r zv>;`bTv+?@9if+Oaj%?2Qr{Pfq>AmkxYtBHIYFD`(;la@@FDH}1<)IKgL`^kcq6dE z71vRX^UncVfY$`LVyJP!+G^;hmfS$(vq_$K?duQXoP>rLGpWYZp67 z#6EFNqT<^R_t=~V`jxPH#N05bby+1!t-@p@jpkXhwnX%2CxYz+NPm%Y67%uLUZgIA z%hwHbe&+TiCTt>)C*^&|Wkztd4(S-77#K_0SS7==yxU!vpDPme3m}|m);2=nr(I3J zRXp%GhT&EUwCNx3ycSKc8DPNAy@Q#S6 zA|LBBnm5-6^$&5qU>;n@%-oNv{GaJ+QCJD}Kan=!dWmrd$@3JbT2!m@epV8%%%=RD z0Ccq1fFE3U{#FI7j0M`y<9O`$PudBPtXd#cDDCmrLGN6SEDUr?uk?4;Z_XFtj zu8#CBZ2Ag&CpA1kMx5Q&ak?mAyuRgRRXSXCB2Wt(5vraU?}Rw?Q*q@T2Pg-{MwoOX zZ3Rcz@>cL25^_xqsU#HVYd!$5jR8x-S_x!H{lEd*ieli*{bxhRZsy^Rptrm#W21hw zZF;;`7HcfC1+C(F#c`GFIkut!?^C=3AS`N=;T#$%l; zr&ad(*8b8VI+2-Jvy5P@HFU$De$OlT^?SaLks$@H6F-zNC@ps7uENLe)Ig*ucnCdT z7m3oG*#5dwNWM4!rIFdKbIwrROW} ztOeFH2jbq-GnLDNd}H}LwZFRpkO19*!@mj>xOvf^)pn1&#)<~=f#Db{m$kHe8Q`Jf zkd<|t*hZ?=q%t>Nbx?>Yi2nkJefGK2N3yclYJ|>sAcZk!oddPf&6TkU4!zK#USn*2 ze4v;Bmq5l=I+=8?%&F0$Gl8G+xQY?H<(*6rb4&Y4-K0}&%ujkJl6%o9lER2N7Y z3tIiA+Tsnm##aVoiNoqh8V5JSmQ9qq;EaGbyb8+m3&e@tP37D**mrQYpgMBncRV{> z_9OK|N=@tJ2W_X(>{!Y|Ja%~9lkFTfuj*q*_(Puh>3PmP!W38;J8uW{c+wx2vjyq`&EcseaQ!lJP8FkOswy2A}r` zc{&R1HjG?)yYQKsq=N9n2Szyr$_2bBAU4vXWd`VlCe!=l+yFo7F%=p|k#f!a6B!qG zE5G1b`LcMdT(P(HW-xLzWV^abE%zSgW6CyvP7`x}R(t#^C&BpctIxM)z!fRV^UHY)#nguO3 zik0N@$dkGc)8zmw>0fbWY}NN3=ofyDKvU`h!xWFf4n|I!P5KGDRN8Cztb5CzRbHG- zF1SgEKBMoUITFoMbDI4OaCKDg38aNnA1NTM?38Fw>~7GQ{o_y&PZkir$)_xaX0%PV z>>yNYb|TEc@x#u>2}ZuI)X$^Rfu?{Y_$D(SQZmW5O?Pgc=X7Xw<1BGRb#ecQzs+sW zV(o@ktWB=RiGS}DF$1W3_tkmb&_29BAf$RH=lIhQn?tpqQ8Ng_2?=6@%!?Ir-=KC* z|Mm7Vs)BHKAn#Q$Q|Me&i(=+&i&1ySGVN94PVjX&Yd5=9oH3yJvneuiOPk}B9ZdmE zn@Qg$I0CY%>3O4f<)1*XLLx3PWt9kGI7dbsM)%8SNE!9Nzg(9;dW}2`-UJ%KCFI+{ zJ>wrm370Zxac5*8{fr?L|1IY}z-z>h zPEPbe*nN-}`j0M6YMg#n9t@zS`052gL7umdLU3Ub`FdH-bfYCs`;rwwKxBDuf zOTmQgnjPV^nJ_fL6|nYZqynuE&b&qNW%57>aZ8hWWt7FYXb)jf%snF8Vk9vQ8$xoM z{@L7&{5VN2vLzPkqllMhAARud#?S*SQ9%=-M-c=h`y=-ZR#<3ixQNytbdR-VB3kje zax;$mVlv5#Jna!|;XUT=wpw?<%O(b?RRXXxu%HDeQ`$)>FtetjsOb%rP?d{IIW@L8 zKrJzflrJ{-?SLp;k!wW~>`&8pwhA%yV8JJ$b)W>4bMW|F6%c`d93J2w>( z(vWv`7;oX%^qfr9IygT7Zoga7L7e13V&FSiz}TBMxxKOBn>;D%gvI#WBX&EP8A=q- z61NaNJG0KAgJgIyUX>Ko9Wfb4PwPLCqWFTUE8b?&@hKvTS$wdxKHnO)0PUn?C z>B_IoKW-+8r%)XTjDl(EYsb!2Nuf)fZe*y*{~hWsW0PUYsh$;hNXc0^`bSmA=Br9l z+|aFrwpd#4C+ZE_>94(?n&dE&bU?8TNUHk?;;Wje*IyBDa?D)Uboha;suXi`c3RK? z!uLSEnP(}$tTtK{B*HjwvJylK33y{UxE@XmX{kvXnIwvS*dt({@E~!RjZj{APmqz~ z=Fsiv^4e(p^LG-w)r^*I*{&{S4`zwmv)<`_p|VlH@nUD}na<_L`VfoK2aEjpyAgm>RaCsLDTpW?h_ATUYk3~3+6dbm*L5ddk_^V|0+qx z4Jzn@ZNS#H*x8>7WKEfZa*v+s1nfL!-{X7z=^JI8m={O-vtQGPo%AFs$f_cQUYfOP z_{)0hzL%0S=Ix2K++$mR&J}_cpw%!xWZ%p9Di3$fW6 z{=RHp%bn7`?z+QszsuNePe$svMaOQK@!C;-bfu+Rkel$c@|$tisi&ILvmL#G<>6Y& z2g}5IZS~h8$0x=2_4e~S5@i^J zdo?kPJ(6+>E}k2(trQzh?~Bfv60 zQgpf_tIk!WO%VIDRpP|_E&jA?*UMx=%U2sb)UNhZM$sqTQ%tD)zk0)8EWc|lyp_)z zZa8OU*&dsmCmQqyf>po!Ny7fqmr|CR;bf@aAfdX{MkD-HCZ&8pI_+y_muw^xSGQwr zjj4mKuTLbHO7p{{!0#{pHZxAJXt<(&=CR%x@|$5`2z~Ec2sH1jRg!u{tMesKS3EL| ze4+al1&W5*SM#~VK;gI@`~?^syBHjE89R)0nYz2BKEKj9KiGuh*Dn&V(?8n6>huXB z$0Ze7xRN!$ul6NX5%E8qI1yP6npR@|nC)9hhb-u^OODW{na zg5fF~)GoMc7U%AMV%A|RoeY*En1|8u7lM~%8c_T0OI4Wo?_tb)mL`mOW|8T-r%PS) zW0u1wc9T?eOMq7bJ+fg5jcE~|Za!0K_ONpc_r7Wb@~wPE@_f`6^`I{InOV&mQMG`$ zRc>HlI257M=P1)IMHJ;L|07XXL@$Bt^klmZ`Ide*BovApOOP=H<#fwhS~cuj^IBVL z6FWqBs~nAIk^0;_sDE! zjaoVyK3e}E?HDE`M>CTqvU5~2GF4K@R!NRWfM{ZWMOz+$0$k-a^awP!!5^Mv(pb zC6nv1or}prB7$Z!-sO7!x7{jvEC*8yLt%-;>TgVt-gpC@4DHL4?&@nL8WaQooPRSw zX}n!vZON~uvM%T_F5S6gGQ1GWnKrV&G#1k%Yt&ckjhz@-xMvN4B#uh`zPa4^rDw3) z#rBrd?CU{mA&p6inK%0<-J1Fo64gw2a{J~b z6Ram`Q$yqy9GX$b!eOqM`eK`1j+iFWI0zbp6tXn?nY8r}^?0URk|Y`w%ZsT(D-=1%*pp5^pD`$mPVGS%|r@waT*5 z5s&HYb(1qqzye(q-+37!_^V~*vKwR;hLsHsafy-5p)){1<3dop@11qx7Pea+p{;PE zAa@rA)X|0FE5Ao!g~VF1JZ$;+|K>Cw;(BGE_@I|c%DtJ*p0;tXZRarng=ia<&&{is zD%3Kjnve9ZOMyT9$_FA#FhS(M=cj7-Ye9?{rA74`@h{$>?yuYVt#$c89|_UNLE^%p z;Lh7*AKeqacXz+IxIZ{(@N!p2{l8tL>(KsgKhs}^`&oNR4W3_^{vOT z_mneHiE?_jN#-ZXev0eDO5zNpgm#D3edFpiiL}++TuzdEeqsOjJm#I~b{!V=3l)dj zWNs!{YaV&dZO9^oXXE0OK%{eFQl7eNL(&a@H4oLk9b|+D zjX;N9peKUlyFw4_YAFv3s7-J6&?Z^>b@P-FD2NZ;Ueu``O{KQncXTEqA+{n31GF9n z0)ZL=rjn4@KBHB4hYCK;R4+HQy$6}|7fRQCkyf`5P#Xi`tKtzqrtxfv4b($LOWShY z+u^Y`M-*-K&jEGIOy=O+I>}kd;S7+EY45FKWB34UP~IjXEvr)A)ad{C7v29tLxWDr zHA+E+3t9oe)}dRzGK+W|!Z=+!#X z)O09V4iOV$%H*HKJO<)n^ZvsWQ+rpYFWY}Yv2?RZPjPcxTe{~q_#Ysd(@lUx#az6s zm~so$#e(5*Jgm9-->ZR#e9P(;pAy1bMM>sG_7*(?i>LpG%l<{Ivb?I|k?gJX^3*oj zWmnx@QdaQqv|frGWG>d<^Y$b9Ys&B45I**J{@bnm#egMEzcn<_0S9;D;lHIJogeaf z3^PrQ`w!-vbMt)XdPbV$!~TRB^m+<1GZtCnRCucaN=%?t5i-1U!^`vzrxqMj;dj#r9&&dTSM@K_16CIofS%XJrW{6*paqzn?#w&-tw9ll_ ztJ}x~wcjPtZ?!X*(qbE4quhp<3#bZ-kVNu>Ds^0nVDSiq>GLc!2c`uc%m1Zf z1kn3ZL6N_SDdS=hyE#HHytT2rHy)dul!KJ0Z_$YMA7TXSqJ9lRO?a7;PH(q%6s~d4 zOppbFoZ^kh+wtU}Mf`BTr!RCC)&g2OW6$uJ-{==xOhezG;2n{~DDVFm`eYR%ktGUh z5@4%y*4ukLXMoQ0+X463$3_f)@yD3$X4*%W_kxGadNIeMi6%tkmLl~@3oWL%0O|fT zb|&vox?lt@zVaRAQ39Mye$A;$*F(C`E?%C&>hzhjwX&x+6c zJ~%QU?Dp)9`9ILRhn-d^QkXno;O%YEey7@*Z7#dnUoL9DTqK}KLvjdwRtP%o4if|uc)HIAf(GYstB+ow;B{pIJ!3I&qIFk+A1c=T0I zcvi+g_bMq{@5M?H;QPCzy=$Y+Ud7D~nn+D@rKmB+voHbPJA^ETWtwWTO+ca0_lp-N z3NDbk--r~O=St&9nQhJ{=B6zX7a!?N-%eJMbVYiP8nqvECH;f_|32ck5LWXNnrN<| zuBMUl<>te|LrT1bm6SZFw1&S_u@)!_!+7Tr3&r{t#aE5|Qwq5lIozj*t@C-wKN^Ns z;F$~ro71jA0Ko#J=j@lH-yx81T$^f*r)xp|r$JKpxA#!Dzg~DCe&cY;8dwGe&=*1*y|~UnjaO08xZRG zG?n=)ha|fq=3kBn9cdy%c4W4VVpbegE16M(Soboq%K7ey8A@b3wj3yDpu^G{tA)WF z5o+BSMWA9W{?8G8C;p)!rCssk;WsSl!otn6S4hX-NS7MmWfo~Ads}O=gsXCltq#h8 zrt#SRx#y$N1;y!*!ilQ-YDFCI&Nm3O{oGQz@v)T%=~A?-*M6+>LyW@%ih#2uMgn4dCf7r z560@;e0Sg4N8cL3R{II#qz>v)Hk1WZa6%bljgr&=Wpi~x0UaaG^|ymfKE3@GMb~>YSC^C7q9g%WAE&829&AXIOTu~EU0QF6;xl+Yp|ovt^|3hnnjY&g$j@w2j1{`Qa&8F_3$r67P7szVyh?9(qA z0-qV!NY1(hjXKzL8@gbX4NjN20r95!qz=Z8fuyI_w*OcD$*JDh)T7#SUgl`J)4uRKW%QqF`&r4(?M=H;+V%xr>FW!vD&>i6+W*1q z1pUrDw;v;GuzJ(jF{uUfN8G0vpuj#8{*E~5m`P?<7qoN&QK&I!^^^>FaWFBL@Sxj> zUIk-`-H51Q3SX8BtTwzVvfpqmf@Gl7lmw~43~_uWuiyD}PEi%k;5SfJ0xby{%>Q}s z_r4uj`Nxt4avI`pl=fy0uKqFuWi0*1%%j=Cj6VXLUdq0p<)6*6-e)t4TB2ODlxH

zF^PuO=!z_K^UF3YkSDg%@=gqOmifm#Y$JSIS^PSkF|*^Ox?8Ff52dSa@o`GDJ1CYM zzjX+U`6{*vxNBA9Kf};gm7d5fZ%kfWG>Nk33G(_a>ILu(U)Se8p9Muy0rHwxvkiE^ z#*RHfmHSTGil)6mzo>zPqXUz+KXeg1jSC3==FR_0SkxtC=lQ6CzU0QEWBl z1O-u{1^t1XnPmCtiNKfq_|7137Rp25@_y{cg*MN?90u*Az&&Y5)nSc%BQXI zSu`{h-!ZtPGrZKs@Rpm1dH1~U)U!O9`7U8+iH^W4)yi(3*KY=7_UG=g7^Xr$QJ`rW zmmeU}q06MD5Z>@AH*diM%(-q2pL(C&gQRL=tK(TYbk&$E@niM$tN2DtlleQxJZLWA z(IR>gn%LL(nC1o#GQWuI3 ze6uKRleN{o@soUH?COV3ljLsW;mTWH^eHIEDUBysk$PD6A@&SQ5CCU-*HtXs>37k$K zvJ_a~aqXmUYx>xK%whH>qMev6&++lpCME+)ot*i#wkw}D0>qubNw`fy4po5n=aS+7 zHek#i-(E@!d4F%`m(@jJ9~mGkvXmx)#taRRxL1=ClI6^Xpb9{ZGJt-h3(qSlgOqC_ zTS+WH7yHo*^>Rp^tUx&CP*B69a9%4{j2_;=!1O~>5#H^QDM^=ImU?BoS2V`iXAM=M zn;$@UEo8$V)Vu38mqc4gQwc3h%cnWrjb+!ikE(oxY$ZM}zI@GPgbenpqk2D^JCdf` zGX|TSlavS7@%r7LSpC8v1QY+S_-@mlCdkT`B1HPnP zIhq5MSFTckOaA~rqw2p_Qf{zP+$+7SHO!<#Gme`jEGA?%%qUeIqu3nw*-JMf?S7fO zGpAh8;yywZ{u6$8fvAvy_M|g$NcCT8UOH9d;&W%o1wx18V^$D)NCLd5f0$jwonO-B>O?dC~Q}QK-W3-8mdRrTDP2 z$N7PcZO(&*iGWz z#O`xV>x2h{C;lZ~`;=>j+?N9VXgMTbiG&A9-H(gRgv{<*l1PVT(5?e_i{>DxOALfjvv6ig&PD0;2*~G&xa*t z0EOY{?}A~F)n_0RW}Y21s@Y9oHyD6xHm zqMB`T`P?vj$<(=v)qDHsi}zk6mdO+Cgfb;_X&2?hurv^8xeL&FaEWiXD9HC&=Jn%J z4V%vx?q81j`v;7oR|2=P->K&5)TKj^)y#Z)C=mIP>tmP(@g_7ka!k}&-$qjm%(Y;; z*w^8nD@kHg%&0G1(@ibAQ{8F;YDv@1Gc2`1!##_YF;`}@t%0*|UHRX3HbnC&t|Ei&D^xRdR062=-#H1BQHdGyN`|E+~B+x`iC&I|^kT z7ZjnSwb~jZ({h98SJ}h}Bp=CX;_sa#C}4!Sv!li(K0u-qm5gvplsY3=z}<=V|xW(KQ(D^bA+Ua?lB`)C&#rC$<{Ye?qz zRbQ<4pq1I;QqWD(I=EV`_fqp+A?b%M0&z|EL}WFkT*PUAo{N3 z{AYT4yLPWr>>GAgQ)&XJx%JS?UN${8glgCg>#-wxE@nE94@R}T#$-dsGE7s2P@GU{ ze8h$D_yJ@Qz0`yn*NTP=3g|8urzc><8K6|<6I$mJmycbtfxa00CL-K2!t7Cv+j@im zZ-(dFTXJGULAG1*0=&U>B|v<(&17PDJ>OE5p9N8dSj9FtRXPx@LH6!Fh@Zy2v&5oP z*QeClxZY3HBnbta%grnLA~$BTpy~*L(sw-v{Kkg8oCs1U{hk3(DyLP`b7`m(jY&zh zK&i&>k5G0Q#SFb9FKszMz1n}~J0WmVdUuEBPgoga(IwaYv8W)bl1;~|i5*R*L<_2f za>>cmdmOLRQL6V{QlNa>RX5i)$fLnk|2x1L9FYJ zz;{$qU`T%z50nKv%6d368nSBOWK6%hdB9(d%Z$~5@744n0oB~h{?{pRG-9Hily#}- zU9`OXB>vz1FZZ4Z`dN8m-{o~?X}2O3|IDsR{_yYzDpU52DP;4^XsEOP)aWc2hCZQf$ls z=n&py+8NdW3O7FYZx}u40%#n*eEG7Y{(V3q>q0lCY}JM-6<=6qc-1v<)@oxAJeb^aiu|Ku(uj$vToX zfoBP)toRXUOd%gJv?VwJ8qAzB%3@@SFQ8eaQp8#K8PS(RxO7|bN)A$#&boRVQrEgz zJ3`j>6{_Oo_EPo}$;6Q3cHdt0JN8~Zn>B!Ki%)NrAKu^7I~i5=WIpX%NO5K#Aa}JW zFD`(hgEdV$7nrttWuE*jei*~seWBVWuHJIawB@6J6E34Yb+VN}CJW7|z`6|nWqk`L z6){0z(uiNXK?4x7%1g||Ax&iOdDk~lbIAH)0p{9}VUr1{2JscARgPfP^1VYt^@wd& zyQ`XFgA3o%MWa;osyip;-7N%VlgCt~4v&9|_cr=GV9qetsN$hrtsr@oEaN(4^7a7; zU}~rRJ)i*kgH`8YFK9JoG;O2#>FDu@_3yoW3mpp_6D*UP$7@-x?N!C#&@!?$mUZH( zrlnQ5hjks)qe2Mp($Hz7@noI3-x3oBipSFxtDN%+8+7*pFKgSdOO2=jBt=%;dYEhiZK(pZFa^g{u zI|I5WD{6boU>KV#+lyz}!9}A(;d?d#)2g>`=skxsq&^A=>P*Wv`y3IiG$05EE@QEj zdM%lMCH66Mh8cIFC)lktOp%|eJd9x<+Urp>&|Mg`qoP9M!P9K82`l3EDY?=|k8J^> z$w}FR`od9dQ*#l53gCH7IfVWF7^}ToDL zrgmjBl!$I&*XS@m>i3^V`;S=4m%6c$3Q8;%3FJnwJn8IW8OLi`#NPDzia5taZo;Iq za*2~{XrDYraP;z`w4_Gp zTgso*kWV*lfQSPtB}O2v^!#nRZKa<5=Rd1NKU)y#n-H4X%}V}RX8r_^knRsEJ~5~T z<6Zot&DGQ_$uJ|AMiG@H9ine%S+z6Ht{Zq$S3 z`aY5r_2HPY9tO63rhYCIH=RM07VFu+1R$x&E(57p05}n?LylxDp3ojr-z=HlIwi;! z+yjcqr?wAKmM^{O%0*^69>)dR_n{HsGARp}k>6Kip^f0`;Lt^w90 zlxp(cDs0aPE~4D zY1oCLh>2D%d9(u1BcQP$gnI+pX9Wt={Fl&@(b4H~0IJ`yjf;d!!oNz$c>62EZl!MP zd|MbB^iLBKKvAenlAG_CEm7TW;o-&tv~R&rYmlT7xf3A102LRzBC&AD1$o$Vo&vJ~ zno3R7lNRZWf_s9iI2wzBdJcwl8Z^Nwo;2cvPI4seOUw1yZvQ7QUhb7as7=BJJ{Uc`ayZ8rTAT zO9t)jA{Ni*E_|pW38#xrlEepy7#%NjrLESMSdu637y#fniUO0jyuv`}KuimJz-shH zrB%fQTgoOY=pV-SWK}LPK>B4ISmwJH?{u&@J4g^OC_3BLuAKa8kp(bh02r|h;b}|z z2|bI+eq^hi-!X?Pv__sq7f4Bbg8Q7hn)G88=rdN2(`A65*cb4u&}dbQy)1!iHd!W2 z&E><2|A*j`GyV$QZwEsT$d+4Ei)gKV{WwkVz{`lag)Z-VQ609?BGbvj50|AazxR%`MTA0zE+C4w>e9k+l@4xI6%=s|r$KCesj;4h3&L61%z*pwamB zp&(Ej&5b+R2~mL2#}-Fpq}IaB%M9AJqjSkqjMywV-!SrU>eJZQiUUFd8Z|R{xm%Ry zT8aBwzSNc1+ZlZN7_u(=ElhB~B-yHaq~sxIpvvUr%p7fAZ zDo9quakqf2<3ADzd%7#ow76%7C2aFEvT*|Dwe=eq91zbnD?n(~Q%mQgQ z<-{%KDLE(*rQaQ~&bNQM0wgg~#3GSKvI*O$i5sDD&^RY!GDwrG0tH@ODb~Bd?xJsP zi4PP{Vu>75=?I}lr}VqqsH0mcu}AMf6OE!l%`%ID^Oq*Rnqwh=LUI3E67zBIisl1^ ztV5aU7+tf5xm=Ev{5~8|QDlfPl0&E)sRV4n^|l5`quH*{7D@#(V3;YqAg>|z^mLvOR+?3} zdd9^%(fqGc>?f94U}Ll4d?)QgiT^r-!AHJZlRI?iBXwVj>p`czu&!oH`K8|bzY%`0 z^y=ibwvyYel@THr#gulb71m%=i25(h{DNxUiGzA8bzNoHU&e#TbQooCL9)0v={YF_ z2Jqvbe6Wv1V{vvYL5$&xY%{LK^`JA+e<^M_*^sKdy3@@xJSyQBvbN=-Kd+|B5uFCd1JpFdP+adI0Q@s8GFEZd^9|XKvq@l zmHXX$(y1~0Q|P%BFiq^Vyr?gLv>KoJLg4ryYA1InomF0BGhhqXR|_;#7R0BODyN4lUWve$jw^K^j-4mzFqgQf2Ee z0)>E_W}s2nbV!z{pwUskrib=n_X*G_|Vv$p5E#_BvRG9bPW`9-)Tq~ zy*lb)@*e}a-M^jpWV^BI|HNJ~#1yu{Z^2nUOgu-m7P%f$D(5#zGkqr52*vz?7fonK zv^Vat^hZ$+G2LNqQ;)!@I8fXX&<`EAARR_U80t&KCbA{%n7OG+!^7W0A3hs(Z;$R^ z;ADzWEw$HOncTyt0_eAONF~=kDA)07@WYRLXB+X%|^Z&j|!bI*c$njF2Ng!cgUxGViktAS<{2 zb~DK$Q17)oz9wn<8@-rd!G4ZdjikvxJ>X~4DIT^2BL49-Y77%!VPc&Op&sRzFT}oQ zz7kD=o`?q``%@aLi0%#M{$ZnsWx!qb9_&478E2G8tK<|mykkEl%Rc0c_UQ`5h70Q7 z^a|?$=4WNexq34|)8C<%UBMmY+(&GO@++*BD!NuAlX@;alD@X^3%BH_eeTcal5*sd z5-5wO&M=lvDauv}sM^>m-vV54it1Y}2pK>)d+Q;1vg01mHBNwX2D!6C5hjR#q|$t* z$OIsWbi@f%-wj@QA19{~IU zmxY8e8YZVZGLk0q2|48UjC)JXf*#cD&`M9l<3=4TLY*q|eN^Yiuz|lkPMOOs`2SdS zzxt7X;CsoUqX(c5@|s4|+D+IK3j21#E^~g}&sK`?YGq?Qhqma^YpzI%3JXu9r4&SG zCj2eT*42dZ#HF3vc)WB$;JH!L{m0kw{~w;cL;mvSt>eiL%u|)e70`No>RNmCZ{Nr@ZtI9qGNj$4Ip=il`Y z2GSw|YGYaf62Dn z4JHQUaH~$`Sfa=`mr{nTdN1b&_v6Hhr77xo0p*SkD}q0I@q+p9EUX%W3-NcbK(^SQ|7yHt9tKiqJk~);V_%-}j=?J^ zKd?O-L7!gOv^r6X^}VwB=`4xPLnGAc%=cw53Kp$1I~afiy?^wWqTqe!HBSZD_qp&1 z+SGo&7O3es!#ZgNpZwq(jE&KHJ_){c{$pwOE`@*QpnLRi^R{qp$t!WUz7a0vuB5FfP2zI0%Mx{Ii5m>vRNY z_WQ?OYu5(1mLeuN?C;WU6$&(2q~yT-etxv{c1z zy@;68hj9iwQKCf6}LQB@qZ1er_eAUpyDYjmezvK2c5CobI2>7oo zvdI)8dayt{zvVwmCZn`FmVs{tZvFF3Hzf*yHk(7bU6#IQ=Thh!EA+oJ<0KNsJ$mrH zOI!{}9pY`)AK7R{0b9>Z(Vg|kENPY|67H8>Mc=-R5}63!)RTbN^b@S+hh<8~3P($= zjECMcxg?R+;=$ zQD;N|{0zJO^eej2OGREY)u_L-TYV;+O@jAp2CvqV?DT* z)Osf=@T=%g1z?7P7OK}~(Fnf(suxB4oO7nd(GNS56&)7YDq)HPXk-7po#74Nvlg6yl`W4*9tUzHlOCi0I3E2(E?F>X7!0d) z)dV=?{)-w$v(6hONG^FSxjb_j92YTO^Sa%7~*4h2<&{b)Lt-@|Zkjf1|!Il>D zXP_gK%hd1`4_d{*nct#sLjcMa*#X(#eRH#JudoYRWf$Gy`F0`3E5(n7d*#q%Gi!S7 z2QLh@4KXCOu;f!9)|p0*(&zhKH=ge%4={!kb}>cof6w*<+Iq(egJYQd@i7<>N3KQ4 zwLMmY$iJxI*pMo&2=aMEFH1^_eNe(uhLe1cs4V7j*m~(`f41yd`35<(^~)pkT;&&} zJXtKIP~>`+>xcQzRgE+{WgWGbmseNJp3U=v#+;|AgMwV@0m!Fd5O~)hV`KG~rZ^<~ zTdfgtf4Pc$LuGZA`T0gNr$Q^)Zjs|H7B$<-1N;(yrS9GDx8g>4q%NzhOQ*jjv~aVnovl*n zWpN2`z5f+^?WGSpIB4UoztQ?i6O!8a+k`+?eRF2yL|enaFpj<(BeQ6(9X^O~esEzv zm>IS`W?pV{(b*d@L6*1O$)ge$Ic9LZ_Q~a#W_!N&?`QNs4(ddxqTt*dgzlO5Q}EIE zTkX(-=x-aJAdmW&;H$tDhX@~KsJ812o$bP-KLoZ&H7{a!B^&I?yo#-;yGz>e2#sx< ztz*(uO*uwoy>#sQ-So9d)64*vr5x zDW`*h6+L;P%_pxU8OYGWQ!1A!-5Ws6uTood8SKIe#2>xXws$sTd(z(^cgeNEd-Nui ze<{<#<0J_2l=7_PcdIEPe%ARnQ!BDsHywugoumm z57-tL<~#O1=8vNp_6=Iqfu0xs28SmL zss5FxWG}OEZu615{&GvT<@$Di%%&M3-0e%>bUV@pg(A=fwPpcU!b?HJ=ho9*0jD0X z!#BNOU#gi68joKZd`J#gfGS|VFwCBOf4m&z*o!-@O9LY!WJJ(*O(3`3{z6x7ciehq ztTb4%L=~R2xk^^S1iB&$q6c29Wbtw@xbl_k?%H zB?4Cd{mYNex-fR#j;3Uh^^HYXCG8w%PWxRxU%#)9XG3Iw{n9Je^&VjZIPgqSE1OLj z&g+R`y^7udSp6pWY=-pGkw@Y`t6-1ivRr+v%Sz}pE(y)3q=v1%ILj{pmkX0v=Ph0) zl5p4atkXSAeJNCqTJa3`YPK={Ewq1N&i`dz?XX@j*ME=}ca^NZaWJ-nWgg#*Hwx{v zH8YUH)(n@i7{tSD_u#2O0uksP3>>a!6FDkRCi}QAfEIHIo>=oDlm6MrL=7v>Bw6Yu zbBQfd%Wh~;FBkZ5+7iQ&^>AYE(WCv*{EJ*Ow|B{E zLeeQy{rB6xGJh$b;!XvZb&8v6dYb5t0yIO)0poJup-6H$Y8eGM3nW9qSbOWz@A_pN zP$Qk-Ry=##-6fT_NE6de-5W(lV<&~kz4XuwYev%=I%HkV9wIZO<#j%Wvb2G;1TQ~N zg1SRI{ofuDt^FJc1aK$hc(Ewt`}Xxd_wA1e zcy?mn<&dVYzm!+DTwXicc_Yyy4Egy9Yv}b&(vknY;_FgkbdE6zft>I`Ro+p8GUw(Fkuui6!opfs!IOEZ_P3N;0xcMf}c zUmrt=AUf#+6GqE-+-VI#VQw7A^+Oqr1;$&l!Ga$0JA#t;0@Ma@ZJiQOy$H$vouk6L zmQU%TV;&6UKO0v3UuSMrgcq$8U}QcXmpC0_rvK<~GM%{DO(Z(fI_X8dLG3RM>uBM~ zFlY-~){!h<{mrUrxvi$V7l}Z|D!);KmQf7xYOakd_O|`dkGrIYN&^FcxyaYajOy)r zucxB;HCCK}(rLAr6A|SYHPcCdp^F)A&6{JC&=BE!N2%xVTG1n{d`c2$k`=ZV_-WG5 zq)Ly^f#P@P{dkegSm%sIWjh7n$Hj!jSnX8|(yU>p=_5nyy|~Xm)4dK}4BWGn`Dy?UpX1d@prbJ6_ zuadj$c4Rq~f~m(uJkKwecK6M>7O608jf&2`By$rgK!WOAs`K%>9k_e%$8LMbsmV^@ zXb&3D5`Y6Qjo?fU4*)%`QFew0=ID~X&t{yro;lOtfE!w;s#ITwd4Ba7UjyIF2as@r zbYV*h4Kk@`VV#*hssL{JRVpZXfbK6h&%pDC~Y)!5JmmBV);JcIdYQ+Qm14}wZ9y18eYJ$mVs=e zyK`?WeMU#QM(g*}0MdQ1`-8mh*Gb!L%<0GIr)BideLxVOH6*g;{2cle#kry6wxJ|u zIz>$@FB)CJCH%cRk;+8envKhYcv2t-X}zyPq`k5y)tS!7aE=fJZkr7^hs+8s2T18@ zf!!3y;Nreb=X}oF=eLID!}pn(SFaR0DN8jlJf4@>75!7)r$?u)4PU*jTURSm~r zqEf9cW2gVUwwl>~Xdp;;lehM97`NXhWRd}fb>T0IL0_^{3DCCBFQ0!mUIEJr3_I^N zx!j<@29V+e6N38DGzc&B$Pd9v`|k$Cs!h>P_wV_ERwP#Ie*)3QBkz_h>ydALRAbZ2 z?jUBQb@KP?nl3j#YNS__8a<#QibdK>9xK{kle?l#9YDG`I4!M9^){t!5WnG~%s{)E;3lHB9o;5wI*+;s(sa1^*DT3(Y+Y z8U@u7Ni11sha+dDIvk6#q#Xq4uu)NSY#W2_8wxK&3~-XU0>b9GAtyzEH z*eQXIf3oj*cv~(7Aj9a7mv%HR4VF+5uRzE3(z&jI&mBoD>*&@X12My+QQ!-{wvWtyG!F7v7Y|BBq~o>@lz_yEc>q5Q0aM{BXmCGE zW4libU5_j35fs5l6X~Jsmw~+c{)Xz!N8LY+$0=1+38yvbJ?W^``Xpr@*}-G(<6-4Y^}WH}*zE_toL-MfX4qPjumO0(fV;bs zfwE|4fggXkgdg~w4ff0i-Ok4Yib#`5UBR`SL9hsHo%jB*70QJ|f?yj#6;uTW>KBdL zpEX4T88q#99rZ5iUZe17_U>@B-Bul*V%)*rmuh5TM%@-FJ8}4A%cz4=di6AZB=LFP ze~ZJgiM7rg!SuENG$p72cozYgL$F-D zR}P_;j`(saEms6#?+>XWJ^c7P#UZ!n-2Tdj9^ATFvuB6=rOxhfLsyf$yWfu)**1E6TrXpAP>yO3vB~%FT}v-ZXki0zQ?{Fo1yrK?p z(sf6(HkcvM7;B_%y+F^&s*fyB$HNC>w~W*_4Mt@izeKt*6ySKk;Du(8qr;RF>D<>~ zSRx2)L>ydtCA~$A^=>(S$qv38?Hs4LZYn7lWBfKEp;J_&7cNkv?T*9UUs&XxGJWbe zAV|$APj5}62*i8<2j(O1F#Y$ldLKvE;uR(G-h6v)&g@ZS_o8W8N&{$zjn_J3xh+!? z(ilF6vwAx|ov$i%!B5^$mAfq|@(!&TbYz!9#RwB10 zl7Empu<3*->aJCldtt+6CRStA>T^3#uzxI5XIt+jgRH2m6v)X~6$SYzknB6WQ;xeH z9LBvpk+z_=fqGjBS3c4NSsD6#!=c72LBUDy+7+_3J;E8ZRTY+hO5?HOy{>EXxMkbK zzoM&NKAnVo(6(Eq^K+IqaC_d@K61ITk4!}-W`k`i`oRHDbsBs`+u|b8s`)&?!7=u`BpE+cjqWgxmE_1lFqQ#U^^-lvX^fr{{q5 zy`y&L4duP^>5FBHktDf-1}9%(5ugM*Sl_{)xYXfa(%Dmhqo})MFau_bj3>dFB20`_LMe3b7P$XhrrnD8J|H z!V}uL8i=$tB2vW$0peo9&ks4+iHQvF(UmVDG%!P+FDU)MKA-gY`%)Mo zaW?Zx)N}Tb@05*{cLuw7i3(yLrqmX?O5aHrW!3IgVuwy_$RYzP4in7P`q(zOu9AKu z{@sQf+xQ$-erIQQnM$zqsU27dR5+bdQ05MwG|tbf-#TrV2EWVga<{&s+4Z3u#>uaui*pEmWswQ zf6rotf<<%Ff|w$i^u*Z2CCk@h*>)v&?moBLk-G0qYIQ8BHNhLy#eD0|xYsc#;La6T z1S*mbt~84JwhP*tom8X|PwCT>e&9pG&qbw&GFpB!dC=1%eclVV?0N z<^P5iq*_SN`$#0YG9)PYj1fE!%Zq%EN@z}Fa9=GC_1_U}t2WRl)<}g6Ga0XZ``^Gc zC$ih;DjG)|w0ZOl1ONRpyq@i~g!Z_I*#MehiFqafp9vHy*gK)EuM$Ki9@L+H3^C+xW*d|GYTSBO{kP=st< zYM$)BYrtx`(tAl-Iv~r#^}id;ayTY9ceBhh{{d2;|63H7DVwYdQk=}z8RzG%`1}Z{ zPcsrw3}Vw?E`5B;nw|y(ky%gZ!T}pAP`HefumA7cZD~X>c-@nATWWsj!JGbX{Id5* zJ*?du``HKo4z%p80i5@gZlkk1Wi<{q&cA<$yP}MNzMP0cJ9We)9~sJ?~Djmvo`Lv!gjrpQj||Ca{_%SOL#J z9w>$8>{ip!H~ztk1)R=HO&45Y4&7YIdEqiJkm=20na-30U_mqbjDPO>_J zbloW1el~fjws`vBnajOPA}84B;wRoITz&7p%Uc*zOtR*8pw5p(*nGnAG=A$*;5ZyZ zR*QxLP8Uw{_7jcdwLkK%(6&6J9d7H*{|v@%O!J6nfKlsxcH;V(#td`ExyoPEtModF zmv7U|IIi7eTDc6qdM<1UGA8yE+R<7IA0 z>qE+Ue4;M#Od@tinLe3QQm)QsmZ8CCf82v#Q?E2k+zv+4zpa+HD>}~o^V4J*L%*5I z#BFrXA!g*mh@r<~-b-={=5t?ttX}x!9F?xMF#&AKuK44|_|n zy-Y_MzI=yMUv><>iF|DLQB8mK0Uyx9;d6TRr1uE(0-2Q0uM&Ti%PxVK{lIw4roC`7X8{sNE;D&2 ziBzs0FjOV~tGc0@+2vmx;{*N|*q3s?PhU~moBpPX`q;f8dopi`Hx$lkTpZZOy-sh3QKT`7my?)Ed_#r+(jfpxNLj*{tPvZkpT}qL z>xIvk@?9Q0B(c>YHSDpBnXydyFr7rk{#!57y#C{Ir%Z=6>EFodVM(p{CwQx<1nUln zwdMxcZ_DHm*QQ>bLjzbE*U6Z0 zGt^7$E4P0IZ^+cLFf+^FfX=c^$NdMOt1R3b_v<4eBM%v5NuII!xE`RW_zrsXn>yW{;p(vD0dHG>*PqIk=qlZN+R%*sqa2KDON4}=+ZCv;oS^DDi&p?dC9 zoce9%S6onBTFWa)rULJ;O0rn2!2vfBjb+`{5K}oIp1pREUYac{*R9e$QaN~#)f4QuDu*1}GK!1HB8vPhhCHlH z9gyUXVC?ZJ*&h8ccRvLfsZuL1jM{ke1FA}M&r^d#OJ-LRR-dzb4o9}9{$n6yCmU?>qX+(L*B4A&526^dvxt8m^+2R4_TN1FJO zJ74EzHEMlGK;bVekg^f9_{oJNl#lP>uY*&&SAp%yD)-_mYrA8fR>Rb};Yk~R-PXwE znVh$L4Vp_09k<(Xdz%uub9GT_xskzA1n*;9+tz)qi8A}Am|Y3q@Oo)^Sw>%Gs%kH` zRdKMa1+h(YR|}iIVl|L7pO(m^HqRJxTodvi_j@Ecd|yHCru)m@=E&&i+oe7b1L{^F zUq>OU-?PGZH>B!PafF^(LBl?!@iv3+Ce{$~d|7V8-S^}2^(gdB*YE@mWAx*8qB|6Y zO6umn#kw@1P(SJSqLv5NhqDwxC-amz%jdb6O_OR&gizR}Qxqt6uJ=34>!|o~vjDv1 z*e)PY6tVueHJn=p)dUVQ#Pu12mB+KM<#hQNOswFlb279K zr-=VSCUIXW{6@KI1h?;&b}VO#Ym%C^+J~W%<$Y{jDonI5r#y{*RhUnm##zi(BUXrX zM9fPK8%pH8POx^u1dwP~a6WN;$UaZM%i|rwW0+!q?HadTPo=okR;f+f_ACfcX$;&aR^mB&75|!6K4a}8=kN=$QW8hPKrD>gJk$}uvVZ$6NZVa2kWb1c z;HA{zpAu~R)=paFLFKl^VbrICc$Wv)U58=Z`;bjr(Ywu!H%}4vlA#GPmud)O@fgV| z1Za6(XFmwIIw)$de@}lri_k7jI9i|G{q0x$#_`d!JIbkOR%>72Lm*G&Wp-Uu4S}gB9JAvj!$2xqb%lgnH);My|=p^ChLBK$;-t)waU&_<2s<_{Bn+c zxlHliwcZq9c4IY|%pbEH_?S~r-fhV1d}4n!H?B#6$OTu5PRw(6K4LL;I0?YCQDyL? zh_=2427Jra{ZlD9$%mrKVqxbGB>cif z47aL&i_C-M8c6J_iaIN4?{GYVUaSf{JmiML2TtX%lsnZYy_+zq@nMd6k+^lv@?aD+ zg>7<88@1j`CyUE8!*6!H%WN12e|5eS`jY5N;V!MpCVXyUL8rcC`3bduZfd@Yg`O?sG{I zfZqx>UT+I-3Pr{N2`^e9m)jg?XXPj@&5ASX`AWA@UWO~?Tzamvz>5q9o+{G-K`P6% z;%aAoDc`j8?U{hRK;cV=pS_<>1`?=m)w?f#)G0f~$KtM_G-s3dDX#0k?|*&$y8Qmg<&o=?1W^F`L0mk_ zu8%Qo-t~q9fE8zHELJ{v;s0Azo`JGG@N8i7_ zdF*uSJyVX`9B^D2(&|^wF3@exc{W(I5))o_uQ2R&LJmzg;>b!o|ZBbJ%d#2C>DA zk!ug3xAG&}(N9%RQp0+0ol6_pIFgn6)G8*!09xC3wkqEFeU&KWX3&m_y{(YtkU+S>}9Hk{HXU&OPzkdXOM zxBQ|TMC?25V{?oz8c1asVuh7@%1oRIixxZeyM-G%&b=R3KDp*v*kSENO`-Emtra_@ zo~QmBp}MXTD!B3GN{(pranz=bU?#|xmF88WKLM6wNCW_8&4pjDw=laR)=O-6o-E1t zrXOC-j$>?M-;gIbxudkD&WWnA13C?TYyCCwVf<>UJxKRWYtp&QZ{dpWrx~lG za9W;lYF5VFoO3&mW{9&}mceW4#1Im9NQp^0NM`6j`$r7SHeawlKrF2ipdJwi`1|(> z^xcXE+v08Vy9XTbBf!xhr}gU%Np7fTfK>oM9L62}l~9dSVcp!U%Orq+A>xSjbhaw% z@;uN;0;YnX48d$aVVByl$p*uNC8-+q_E(MvH&mR)z3-yN)hJSZ0OFz&>_bH0aImk$ z^lb=E6OHW$0&x1hk7)W*{V zQUQZ6jUVDymgwN0kp6Z+ApE*Wva^ebQtjmoI)*-w&+4NDPAAjz&X>k$5KKlR0?@?~5mlR;N})!-pc}eAw~#Ren*_yk7hi{n*&m)&o;J^^pUaUzOYC`Cn`a%1S&L|1<*(+SpjQJNXje1Gj$&H<33AJ)Hd@1gi%)EzlmB`zot-Ew5^ycIr5IvZSkh(PW7N#{swPCQ%SUW{;*+(3f0_HsgYU^Dvu z=Dmn#KP#+6KSEWP^kn|mn=iYJdN*tRf(Q<$6Pv6O#_L#Ke6JnYRKh9vrNurjvadAZ zl6~s#x4X{N#pOwS?Z9gWAnq|PLyVeJ#)N;RO^Krk*LukM`^%Hs7d?M@-M0QVOXqZ` zPyFMV60-B90~GsGQg)*h6ta_$ikCJj8d4#}0&C8jE<=5^>|SVMaTwg6(aOLjbq+e{ zyjB{_D(3JB=XahOERs81_L+XP-ffPKO<=?;Mv=&{Cn0-zk=yKglc*@~P_xDrPh$0| zIuOaTedm;UksQf^bef-j#dU(@!y%c+(qpCZC*98_FqO%fKMCr~AOn<STv)8;ezplFLvL?GvxGH}QG-ME=;NogL(v?zN6C)ZEM+iLzsq#v@6#%!Ji!o8WLlV=(Dq^*hvY#YE|OgpUG=YYuBcVw3XdZnyQ@rn z?4smB6zG`ljEI=)edK%a!tfdvjWMzcN#}M_Ccg&6Xp2eRPL`vXAh&^lEuZ%X!I*`g zwvJP(9vgN4Ylp)}^QyPZeDqa zXYu9S%AqO;gb+wG7}LX+K^lI`%jH<-C?OB%krn;l?j|)f$2NLL(WIiqr(>ZE{)qeJ zl~w=E%72nUDVP90_Oq65+O<0%n2{!u_{c>O(Maq?tU7^C!FGR>!ZyfWW#mMCo~_5aJ~{e z>3%G%^b#s4UhWmQUgbr}kAG;66l-Q6JFEg-G5f;1A+jpR^6BZ72yBhp<0Lra5n*8tMpo%0^?ckg}P zKj+Lj`|Q1T@AdhvgRY>UyC?OKDt2Tq^vAs?91+y{+B7V=PzG~{2kV7Ox1)=TCHP-?`s9GRe{#yUex^Are>S=Y-Hmj zqlHwEY=G|7Cn!DI+mIGAS{A;MMthp2u*TGo0%7H6Gq4}lA=62ycxh>$ScI*aL!)7j z4s8}0Z-cJ(5gOXQBuKk7blH&1_;0FdmSV$_U*y$J}jE^%nY4#j0_zc$y$UA(H)|L@AvuX%tQjX$; zR?61(kTR5GX`{rba$vXSr!@3j{YksEkH=2Z#q4dCK_uYAr4j6;J-|(5#4pEXji!RG zf486n^+=g_?6}yYu#V8}1$lha?k;uO!ra^q+V%Ji_9{ASdOxII3@p`~-N_BNURp&P zS4>S|fiK2gK!ne27e0a*o0};Ch@=dmASkaCFElaUqVxV!EY->o#c9?c={Q|D<~CW^ z{X>Gvyl9MwQXm^9PBvuNZBG4+BoHx0uCDQf{s`Exp~hA;8n9Np_>PptZRQoM44ti_ zFcdeXHA%rXF!29{=E^!-MU8?Q&n3;S6>6Itxh`AOxc~=#W%AvWq28Kq5h>qL0E=A> zVSjh4zbm-(%ChyPMeV3-5B0%vyTibrYR45&>lQXL#|HEHa)=k84aBnrjC((cR)aYG6=HtX{{Jh}nfCiM6hm-LZjdXESv`L99@zJWD z_3G##q^u)Zu`>*fwjdjW)pg)xaflR`t~0YQw>hYrh`Hn7Yi}0!faw0P`V4_xf1Z2m zLb4#p+`R6R!Fl7T!qPYy@YwP^nJZ%KTHY7AgFLZehV8{d@{A|0u zf+0eok#)mwjU>X%K;_Q)uM8Y4(8kb0^D17(T1%pH7-55_toK~o>TGO^F6$UIv}AR% zewqc;yyrWd{36=XUdqn&K2|k)*lKAcvsUq7Rn#bChl@U+r0rV(WZqA7mZF(WF?Bmv=&qAVb#JtK zUL5Szkx_aP{a;tCibDjux#BDeS0BbM6SH|Bx;-z@ z3Q0w+BO<*sn_U6zf}J4CdI*lDw`*z_-av0(kON%&Jn|#qB8)`vD7#z=5P3^1FhShP z>2|d4@=ydm-23rqZ{#`_TpFNy5_UmJX(X;otOxRqY{kw3I0qnh7My}uhI5x&>Z!{T zM0~f8$M2?hL*WJVR6`-8!(Uj@u@;l><3uk^7TLVNKJ0}~ffeNw8xek@Vw>A98#+$ndb_Fp{novI3BIon@ec}h@mT@F-O@AZaQ;wCF z@T%lMoV9^?Y8W4H+D(4N6-N!MoRk6v5k$)BV`m8{Oc;n^W2B8yaU)`G0R7$liKGOX-? zn{E0G_SCceZL)rodLBv?3ZRaBhS5*;(?qDdxTAG`eZ?>yc&;<0?0809X<9S9C+!jj zJgmB_1A_W@HF%~bPW&im?m$8Ic6!ja6^Bb?>NRJRJ%zl_r+jz5YVE zsfS(ZWqhgtv>L|0Y1KKk$R%d~GxH`w<7nJ99Rak-4$e5PjAu&pdzr{JAqGZlD3rqy zN$8`sfLA)w%+dPPB?1wqGuOp%*kPtq{I&i9q?}hiZf6K>g@md?+n)yRv_pcxiwqY; z+niI`iof-C#nq3lZ{2T=W_#=1ZQ2+WN(jQ6$Z{$-1RDqvY6AB9-&AeVT0DvNkp87L z8AvTyYh^E>j02{YQ!d9|^1`!KWOH2j)429lA{ubQsXGZL$Llh$XLlyz%L&Vxl|O%z z#8Cd-9wY>=cQGD4EkG|;!+im@AQ^Q}?$V7pV*q7*8*C$&Jmd?16W%hbWxZe}hB zDYsGs@L1ZGT#G-O4ew4GD*%JX!HJ_fJ66HXVE#=ly@f?Flu@nz&F4=utBwO(==+}V z8+E6;u**zDK7$4cNV9K{8LOpz2vy}w7rCC0bB%3X0(uf8?zWVKlryWl)t(*l1oxz3 z78hsdvI|$FHQ7iyoVdF;w_NOmCbs_r!I7a(n{l9cbRt*C?LFK1cK@q&%9gVZ?$E@I z@=|)JTWgYMbSkg#**7>F25qHV1o>Cx_1O~eIC+f*p%mWtgA5PZfoGve*N$DgHnlCx zE@&$lQJV_^efHqmC%7%4{a$jqqfJj#!;*h)_%3%faZ!X0YjL_;tQT(q*^3O!K%W1M ze2L`WXEC(D67gNgCOoDUE3*Fw&pLv9f&ttao1wvDN^wYTK}H0I&*<5IAEYDnXP9^m zdeAg}!|0{M)2MwC8!gw3AYU0O6pH(rMAW+q)eiY*}`>}j1Q?YoK z7cfY7SH?w^sw*SQS;k>|4fu`dVEKeNGi>a*i!Yx>M3e3=hIN#N2nW9EULO@9T?{DU z`%fImAtzlKnAV9JlFe1bJ?Ns>8sWC4Q7zN&fW7HX3HSZ=2LD# zla!W5V!#;A+_rQ&MrT-Pb}n7JrEs=y&&)>#rA;Hqq{DS<8d?I2LDe}Qp!NNyf~mIYC=~ zIFUH7qrB^q%R(5b!;IB!s^nxRWDFk^l1+R-wf*V;3w^uL5p4gA{(`tR1^s|}u5X zZN@5-hHCJiH+GCW=7cE(9b=><^Hw*RMFP>zFszY9!D0FRSj6G%gfsc%;?;&3)1v&? z?^#oDWj|lP|Iub=gZiE)NEtfXD*{#EX2A4p9LHSeMpXEwE90{|I+VrjxAum(PHg)$ zACG6t6&CE!M&0%!OJwoK0OFH|GHLl(RpPCpnnR9k;fmxcIISm?yon8(+`fIqUT)q( zkBq{&pDil>YHCf4j<3kfsPgr1by)xK)qtP93sCjC<%`*>I;PO4R^}f23kE+fU$wJ2 z!(R~tOJcGHM(Jh1BSA<6?%c-&XNs}^z)(C^5w^YCwR0KDo#{$PwMLn0$oKzI>C^6& zsk?1;SKw+Ta?BhgL_g z^@)3Ofg$gHnONjjju4l;v41W=lu)2?=1&-biP4e&23r3!76nhv$#+J~u$vKqXKMF+ zzQTnmLbbAJ#0GLs(A|77b;?vrDiI%{JqpR&80-zfo0jVeuPN#8p8@Yn<<1oa`s?F8 zlS5fn`R+B_G-f4C%&3(h)8Wrfe4DRa0Bj=Qq0N^>$s)2QX+qgqM5&uRfZ9SUoIdiT z)T25js8hs)`mOu^v$hyh4E58#)lu@7%vee<@k848VY=rzmNzU-SYrVQt1Iwq4tJ){ zNAT8J3OGX27&sEs2FPd2Z)gzS%(Z@m6g)_-3@puEtFa?eU@$5#e^1+68BW7sCo}BE z0#8-M7gKk3Wg}~_;i|I%{xwv*DcIrZ&Sp|AnmHa@8zFkhw?GB>cV9RLVv*yL0mSeQ z-c6HP z817%`vhhw+>a(haId;L9SCJe~geIT)}%5`}{hT5CUjepgI?Z`x6zf1KNVrAb? zCf}nj#}hTpz3wLK=wJH#IN9pmw)#}`Gi^w&?&$uTOsw45{f!5KLaC$}BsYM5L{hF$ zB%5q!Q|W}iBh3uEOGet7E})KFSx&Nsa%}l6U&&UOMov0?d3ZY;@};MPe6Cn9!6fZf z3f#-*y<@G4migS@@eO`g(7ii_rA<+@dUlT3EsNj8zjqV~7-v-qGMD>AAyV*l-Xj!s z;Dq_(+t;Yh7MVK=b$l{qlgQMl+?ST{r91%2m_tagY8&x{j@w@^dp*^TP{t{Y8~R8{ z3S=N^qbQy!fA={Ed9$Lhh9wIv#{$$3!)a43p%7caM_3WTM2l2pxNGXJ7f%)%O!oBu z;7E-W!Sk;zxZF0tax%S_3bF#BFU?sCC7VAA`a{PdQFjCncc7=Us~6In{yO;=v3riq z;(V}jt1yuEKZG1LK5c1&oJOyIxY|EJAQCnHL`&!)|C?pke+bDhz3pp##qV{q@z_5t z0D%tw4z-WA7R@g%jx`=<&k8DD(I%?u>vm0|bUpVX7@{`}wdaY*h+ zUR@e-+y0w>07E2d6gS)-IlKEI( zQyY-v4QGZBp2ie@N9g{<+H%I)T1r1KN!X zX#ZY)1>{&xn~ji}-DD+A?b*Mn*yjx`h-V%<{G9#I-o;fWsWD0~VhLlp9#!OzwTMU} zO6x>aVB*3o{;0e9V@iK~kp{1D6^P~sV%yDs%iFGyy&>NC@NNUKHuc}*i^KKT zLgav^^_TQSqTW4`2SsPIJivMqpkd#|c#6Cp;YD&5A+TDNOB^ksnvEG;(1egHv zzZ7H(Pj2iz9=>SVLegpe!C#U5*d;m}Q-6`=n|>`-Vf!};e+6+-n`6GOg#N4+`~Ot~ zr{MYKf#t!9(|`145bN8Pr_djh0Qs+FQLOe?6#1vl+C-oUcC%7_DQ~#{tsUyRScXVG za4f(VPLcnQ9llEmwMdz30I(^_Kg~(9jNa!ByJdKc(&yn6M-}vc3KFFAzIaF{)}@Tq z_P>lpr!uKYrq(e<5LPwRIR2Z2ZQgsutm9RO5C7ZpdUNqO`V{rbIyVOqriLE%SC7DI z!lTiUW{v*GJPr)GzfJ+^;Wet+cFp|Yzviir=408;m*`U_y%;&|QrWV-|Nrv# zh`#lR&aamLF{eO=$~wf-O6LaWVD>*QJMdH%G0UxIuij7yu3n{t!W_=Nn#lGB<9EXbRMShB6bz%a`jym>*t>H2uM>gZ3mI z9iCrS9;Wz~PIpxq{wNBo`5OHDQGY*psgRCO?kg^E{TY&1cCdi~OyDE4zWTtyV*w>? zY0_6R_v2|BT?pV^`QM?q*`i1Eb;x<)zLd>FMT`2&OZ{(9*d zb;#KpKU>(#3G@JJFyNeU%jlOD93ohIulNQ24a}V+B5H%fLC1o8`7B4+LzfBQ!iytm z9mhYg_;#5%AOWp2y3ro*(*LPv^KIwcIsBb^90WL21foYA+)TjwFN?fZwrwth_?Or@ zjXl?imkL*J%BxD2?%AN+e>8YdNS6kDy znCVtZlV|CezWMMfGg0V=;=+AAu)};<4J4PZwX`afhZlJdEUuly`LAQjnq)i3(a-&W z6^VooOt1*xRPJ|t9>l!WX=dyJ=2D?rn=Q*ool3O)VQ3q7ZZUK6u=S^pcR9na;J9My zi3ZgIm(UCc6L=Nxwa+)F?B@Vl186!fhF2Se$CN|*vs*=*6g@VjL@-hD*LN2&{NXQdcr(%x~a+kfm)KQ`{kg_PH;u;AZc>mEh zTTMXJ=gq?e^8R70GJ*ir7pc(G%^_f%P2L7hRCnHs!`?cyJjFPl<5((G2>_T(lX6=%O=;4nOX)_`$%b9@A`&xe$a9I`-LP&{c^{5uUlcpTg39~y0VKK z>xK9$VJJ;{j`ScMmHdg~U14eRz6hk}x_cls(r#L#Xr6Je-sip&RzKOoldqc7BnzS- z4Amt$Lr;*ufNIJCLMnw`N_qFXzcWKS_aSk;Jr|X^yqxJ83aGUvLo+XC`->$MfMm^K z%Ojl`4=gN2@ojPHWmWjs>O4*QUU*Qt{ITwZ4}&bKIzpNrnV@&)g31)N?oj9+wtIl)B8N^GrHBR-0o(c(E_>#V z2!#L4ITT{2wVwV%?G5G6E4V-NzT~SNzBP}CA%W5TFdeFcX6b(LODBtb_4r#r%{ni5 zmjV~2+0X;W^;Mztsit_(jlcTkzyN9cpY`~N4c2t=)RcG z2UQ(U;8+11s)u{Wj*&VuywA0eUgl~$rlQeTmE|}OOKrF`KZlCxh(EcH=-wt}h_D?4 z0watwd=axDxTI`RyB7t}!RYh{{ho2pFGlj9iS3ICe$%$C(9PB9h~0-2SJ_}=K1mQ9(OPtMB&mNus-O9$l=RCAzX zGTY>yhHWd+DsXM-IpBeEU2y?{$*cmU8A_D_ZNTd9)&=6Gb26=acjz(A9O`b9*#0}N z4>l3JcB5}TjdERO2VNAWCR~4O-N1}1>;+R&Ww-iH7iCIp_}qhz%D6&rD?3P7VfPyM z<#rcsP2uJGswtG5T1>j4Ns^wwXV=^M6Zt8{yrbK1&;SX<2l??;!vV4Il$@pK#DS(T z9l&FAa5@s4RE?URX|8eAayxL?BCN#Dq7yD?bo-zJ?BO#Xj-2oFwd7m1pgv2b3CzRU z{l{}=uuJ``;mpw{y@=WtYNRjrISO8thtHXMBzuVf%_?l0)Vb|yWGn84LpoyLdFqyK zW#Kd7jh=BWM4ZbKi3mI}68o>Ql|H>5CS2h3(S?Gw-*-4eIPX2_N<9|QtGXVD&;yny zcuHEN?3q}laNq6jrJ-G=vt3u4%yFfqp9sT)Q;=SN?9V|jdUjDx8C=UG?w_l##=n_p zS(NqTCtcihB<`FPQV$8NZ=w$YEC1iNj;qi0W^ z5k5@1wBX*b>}83odnpGw!YnLwV1g-za$1S(!y5ID#DkaAOq??iPDTfJ=cE zH2DcR_ZO-Ug%a#aYSNhs4N#SYOdsd|Le7Vc0kYuip*eZd-Y6VXHxt*k_B((|dRS6Z z{Gl2s)#m>#jIwD*u0`i3D9woOd=;-bK+NGaCTg(#Ct!U59LS;&EMMN?=^Xe4-^u}_ zUFdsg>y=XE)bFQ>dvl~;fob*DTbWH2lLx*U;s>Dhc6MM<8YkP^h}dwVB)Z^?!)MR> z3h}{B4|^i8hI+$BE_vt^L8&d0Suv%$=ov>f&R2`V7k2rNkqO+tD9h4cIU{$UfSRYdg4h!dQRXxqT~6aE@*Ukl=a|6f;SDYG8#Ri%IbJ6u&E`K*xyCYI#W_N z4v_jjW`&F2B6Pp~tjy3}yWdTk^70S+fNPf_mW^(StJVdGpR}D7D@!^lTZ{Vh@{*7d zH^g?C)#5V6rcO`mn9{npn9=MG`GnF|RuJS->fs5I#uhk-h0MJ1K!IUHZgXFqZ2!d7 z2&_4L67!+apmLu7X3>>h#RF5|b9zbYsuV}=Jg0z9mVwr#TRos1YIM<#L>v6Hp!(^q zz{L9$VE5H|gl)%7>H)12pm<8|5b^LqS(iAhJlVh85qQ-SczYGC^r!WyHA+=DeIH5Q z7JJJ@o$5Q?A=0{d`@rH|EessUeVWx07vzVhP6Cl#h~n)ZQQjyLvUg)9v96MZmlrT! z-Vo^eO~f$-s_wp$ZOOOXB3Z>LPuj2<|M;#H(t4GHx$^P2VI*>8qP7IhQh%31CHBJ@ zupT(=ekIAwZ&ONGn~hi#+`Isfa(XFRz_5LKd6&SerxL0qk(Ey))Ih^(6r_5S~!e4cE@D9gpGa>@T2<7&4Fm7p1@ zhv~OuyVt@MZ5;W^kT=rhNgVz=ao^uxVt}f#ZJc_}gRa$4K1Ns4g_WU0(=q z)GhT(Y{7`LKBt0@jsXGq&QZm#ZWZ@WH*9|$X{!(Hcj=FopT4Z-sh5EE^nzbv`7GuR zQJSj<@|O3!b?B8%g7{oywA#{OqtiN_0w+ zxz%1Fm5a5D=0PnO=#2Kq4(iWe?f^>ObipiqmOg$r9S8lf{2Kh`fIwdF)3-C&mx%MS zK;wO(sKnSmt6st^uT%6|e)PD4HOkyK_%>3%I10@PF3MU{ye37wW8>z!>`|o~wYVY; zsO%E0TO)6mSz!7;pX?)2caZFzOxnY;C|2u!d)6(`Xa^`K+JM)QB^6T1$u$hSvw#RT z%!Gb@J*FyJOX2oB2|_uOO~BEYZGO@>@14^mFO}uy(TCsTb&=35>!5bS~nR>c&(;uZymmjx(}cE2vt+(hZ5 zbduNM+}JI9cVS(Yi=LFHEJtM|lj=aw>pw%PGc;eVoE~+_!CURH6*nEWnBZR`AtP*^ zSK^a`F2lDs%*Y*dLND&m*J;8dm&+NY7!!aY(jEjFeyFE({?gIGuOrE4haX5X$M-Ay z`?i%XUH>+v^g5^M=gm9fLDmCTM{9oYM9=#_tLsN0V)o1{q+pK=T4>NMI`7x$YG*$! zk@4Gg*cn1bun-{@(oDPn%Ee;_z8^n4 zQs}up!`rJ>0bCPStS*%7`V9q$dnA>kvPVH$H!5BmaQXxc6^DfQe8Y=OfBol&a&gPi z;c_^BhuNf`%R77nY8*-3c&)Noe5fc6Y5;jllD|ex!^7Jd;fP?GYW`$kp5hhR2tR&Z z6|Ra!cVGF;8^bG-fzU0y`t>{MfwAchK64Hh9LPbnjK2apdCYOBU-m;~B*0up-*1i9 zFC-pvrma>Gqo6r&a1JkK`XQRrRN2{{Hb*%issi7|GO_K&?Yn~E<6PQf&m%Bu@Juz% zpoB3FKvwL>fna2E8DL_T?#>7<2rx}M;2eY+5gf3HNdE#an&X4nIkzIjvRwUdsomYT z27FF~Z;mlC-QtTT%eyMyK8?_|LM_P-C1l)Rb>HeZH3P|jQlN=HfL$gviX&f$lw7Xu z_Oi9)0X{g$bqNHxB7~ol*w9uNgHYp{C!aS_jPcE;a~tC%?)t4{?j70bB#E;E60iKS zDTcB^V*C^r0zi|boO$}A_%)SxKQkZ!%2len{TO0%yG6&bv!>?qg<#XA3lv)0y$l0_ zHBt|tn!HBI3V6lZG%K8gD3);>R{YxV;VU{2B5|iL*5rQA0O#T;m81??A_h-=z1s;? zO(6>aS$hn+O3eTySdEoJA1PPr@ot#?Qglo!hPDNk-N&ku3+$>at@?yX?jzqw@bc;d7U; zIQvtqT&Z9{h~p;XBl)(IQU%q4^$*fqBq@^V%U{8_{Fzr8@_@jIz!ATHHu;S{k_a{X zdJ(sA1AP9BAdUjyV8Xo4Ff(etm2&)@kS=jM0nB>cr*-E-D>K=1y#hbC_;gx|D<5_l zkBl1|6!Y-F{abLrb>`>1sfs3!j!SxN$Y}+dIISE#^)B5|eMmb|%Ib+xr(m zYc-k4o+tdmXH30wyM`ujfFskpMligiUif|^YP8P`S*efr#gs`;3mHOaAmd#1jbylG zXJ~4ZFzWF=(U&>z*;TF%3Wt20$UN56*ly#C$sz{=Ku8Zx;tChUOI!`Mvo)-Dz|Su3 z!7XB5a}xBDib~W2^dO)gIK>#-zYOV~;n^iDeGHKKJDJ^GfzmcGMPh`lRpp%CaKO_P zX1DXB?M);A_4TTll&p7@C>jl1@O0eZch`MOlG7h4Kbi2QxP3^p7L?qhIoN6l+c9}B zKe^MMj~`*NrTbEx`qIG|%@cEbRthSnUKeP15t zafR(n`{tRg0HG_v%QBBRq3B6k9fS3>w_YOVu&F(v{BGAjhvoHA!fy3#s<~EnIF2S! zXf%l-LSd&+VlUD-Al}qKWtVlQ*JBtK)m~y$f}T(+8|db+=DdT;oo?n$B15h=)UJ>P zO&+cB9hxxCB5uD!)LSc^-2PKiLRT?2p{uuiY`pC$RAxyFBs6hm2Y*6!$@2YpR=eO) zvIlbXN)fW24N^dAptiF)74Z=`%I?RyKFFiqove6$5el;efaz*Rl8tdU9Ej=sxiwdn zM%>vpV$@uRPr=20rHRxqq9e&n&~5orv7^k#!GPrP%8t3#TH^3mqVgmh2*L@T$6s2K zexErvKxo`9wj=i@j3`0;n8l=c%^MC%9_DqGeG1i6W2+0Q^2(-%6_$n3J}l?}$@odA z#XKhyQWo&JgASpN8k(MYDxOF%|u~Juhf+XHb45#&incF2UEU%o*d4n z)?TF99viOlv+eYip^e59h(D|+i`}1aCNwTTWoR_0M)ktylbR&*^Kpp2G@c(1I5Q>y zr_sM-x}pvpMV0v*+a`$7VRa)I zzxZuR0!?BpG7t^$i~S0QM}Wj?*GMgfpZ%l20c&tKt>Ox+5wgjV6ZkqQvZLYoGD5}d zQ`X8bNCy(3X#Sy6M_z#oIsLTH>${a_u#{P@F8m5?a8LY+=8LOkuTMrkB9}$e*Z~UT z4BHnL8&L)XG`6A&i_g8cUGC##ob36n+4-#4h>tlPGsadaZ^TldW#yv5Q%xFln%53# zEE%c1(CW?ntx9U_POlq5uR1s52hj7gmZ@sPa_%bfWXeVe&1|)G(XTI#ujaprhI;up z79V~^7w!Ksf!0$MAyl@v7fsO<{d)}m3Y#bX>DOnCuscK<%rYt0^Dc7!7iBcXLD2z) z&!Bw@=L8Dqx*QjkV&0RKtrf_m+hibfFs_H=qGsf(nL1ooqXevPbdp^%QdfiFt{WPR_cA&!7Qe zp?1%eQ0UB6$!GA?&z@OL*D3H0y&hQcHfarztG;D%?s!^Ghk(|cZrfo>8+L!XT6e}} zQ0s6Vj|=W$={4xTtDctSL@6?T@_uIj8&lSEyGJ3ILEqbaNVdkTv&Bd+a3b>EHM6>_ zf7wMMa7xp~%)0LPqN2{PF2>RHj{<`?&S#sAn2F)*i_sdW46d5Pk3m_erRmudnDscM zU^;w(og(mn&;%{z`$Ut3t4E{VZVKLFK!Wp>v=_y)e@526e-i5)9p<28^Uosejyd@0 z96weC{rsMenGUru5kh0U%az=-I7sZd`qA7)v{72ZFJJ}P(wS7ol%VIUuwDElVI|zq zf!1~BlSl@|#`95r%C!cP_O1FGrDK2_SaF=W;k1r+brc@3x!g*we z<^U1I-b(8?3Vwl1H>i?vYU$fJ8relI)Up1Ky=q!G2^Z)r)K)ip=QEt;#gtP!{^EKO zQL4u#V1^Ywt~uhF30^+1VU)qti!OB;Sb2CTe)zb9vc;QzrRo902jYqpQACQ?BZ>q9 zdK&9z-0Z&_9i=@m^+gj|7!Ee@#o6Lmd$JK3b<1L50=f5x=wdgqjFwSm&{`n2>n}Al z`l}g3n<3fAV|l=Np;v(wBxoncV|Na_i6+vx`Vzy z*`PBb;;he^yzzc{6+Fn|xj|)5i{=GKM5WTd82sV@ZUNbeIf7y=8V{g#ixE)XpdEi9cS2`tX%s@IG%hA^ z9uZ(vw_J71a<#A)3e63D0*aK;V8BxhR-i9SH%%;wTGp}1wK{q)yk2Zkv8U1KQ*gS(ID*{X7Q)~`TZhy zHw*IvAn1DVo9e{9+jGQN9X_y!hsWCK?E<(}%)D@7?)5U0Zhf*mO4~hA>2om1#`IfB z`!!E$rNgItTdyTW((y*C`8Uq69i_U1?q$DEyBn^;MML!?ad;1bHOEybo8@44`??sG z{CC`;WYHcGhY#X}_@olU)h^bG{-^1EO&5sZOL3L!@Arwh4NyR56Ml@^UotE@N@?Tx zL4n*ek7^b1dOJj48g??r#gea(zVG>9Kk%!tBc1mC8bicKdu_zm_(10=RDxrzzL0K` zaEv#fZ*3?hkD&bhWvPneuVEYh8rhTE-wXbd?AI<^NU9jhH~EyMh?gzT!nRNS4(^T* zIL^fuK?H{Jh8@Wr^29_YtT(HCN{MS{2OFK6LO3Tk2^m(>)C!h0)=Lk+**8Bfjib3u zo!H;*IxUT-)o82tlM|Vi)a+?(GQ&qN5{#b@!e4GuM)tX$gdF>taV*E*-Kg29DTGPR z8Q1baLXlNd2*?JBP)+TZvY3g3%oQW{t{`0oBGx(1t7o5uLYG+Gctl%h8}`#BZ|HIu zD4^?UiSz3|pYJ}*_w{G@*T-q=vMgwArt&Y?Hp&(8?i%>%-7D?wi;+7PARoYuUl72)S3-L?v%gVOqOJStzfPvXzpwz4o%-V0;;G_53>Y>fR@VFslhG~fnv!I5v%2# ztI!7!y?ilshuWG8!cjK6myR!6gy!nU)T#=FrWt)*#}8L`GsAi3i-v8K(seOoNl;D8 z)z@NZcrF?~({3+%MYPahQmRl|hr=!kU3Z>^O!+mw>t5+EM+sxCR@eqRIjCK%=^rxa zdDAC6j1&$NtJGP;IFFB-ws`r48b=>~w_Mji(4=fjD;I1(mx;9y`83pa)zVID%x3p! zaYIx19%Q?hVEpN@+Akuu&MF-OU`xl7}23kw^76>$^LO9lg$@_TAxb=*9Q5u9;>ithA7X{O`-Hsu>BCKA56JzV~rLyps z{QAND)huuImKh{MMf7*}lH%EkVyHsut@1@(f0HuxD)@YRpvxnBE3n5<b8G! z64%)XyXWU&$Xx@gHoKGJFq%e1Dd&4P(K_~9xqo|eIGg)X3=#S+pHG8F*2jF?3#>S^jl?TZPvcq(p2}T0Qd>3)=XF>2jv|Ht8Ob?|h;0FrvE?L;&>WW^VU1 z=f$q6pLVHF=IgPH4kpY^SaN$z;7q|SaWR}_G1m_#?U`Ed{!_X!BTI2fhXE0gU`+M z_a2hT(~>dlC;IWV`B=~=aBw36XXM{DL`cR0mV0It>=EJMUYm2(cTkoUgL945ZwH(bRzedW>hRnL$T?we_ZS4Tn&%O z=wpu8%HFDX)2)i@9e&+R+xnZsh#lRMjfntpLw&_M2Fp^Wc%UG!%ZidLBGL_Rn|>_R z>{b)~%~U~!ESolY%!n-(w8q=$?cdkxsEM|*W?C?`o_W|WU4yKb(+42WfV#fKyHp3W zqFk_YH!osGTIQmgmbk6;b`utY{+Q(;S<_TckbUc1Z_WSy{5V-Gm_k(Y(O?n8FD2DE zkOCBq8Wt>u~=xlHbj?@Slc4 z3?juPh`kRPaBU{~&G}nerOcs^_3`rWR#T>bhk5KK|f#${jW9_-Jrj;j@ zBPV5jo)%gWFi-G$Z{xH&CwVdanB61IYiH@}NK1N}!i9oc>V|KEAL{axBPss=sWPnR zeh{MM{bbO@U+>i3WVK<#=P|3F6T8)trq$*G4*TcRuPyA8ekkcqy-rue{aTL8XrA4)Vx#Pl6GQA%%+(0)qrcy}!_{J?IN2_P| z2r3H)adl!aO_Tf6{e`s0-Wb7S5u{wrDr?zT0ONMK&C$HAWBAh=zxh44=C%9kU4Zi$8vT3wh literal 0 HcmV?d00001 From 6f9c8fe5d171bc3c4a74ab392e5c7d4ce024dbb8 Mon Sep 17 00:00:00 2001 From: LEMSTUDI0 <105062843+LEMSTUDI0@users.noreply.github.com> Date: Mon, 26 Aug 2024 12:54:19 +0200 Subject: [PATCH 055/227] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 507486fb..6a92b45e 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ *Bittensor Subnet for improving cancer detection algorithms* -![image.png](OUR%20GITHUB%20481afe11c91843b796c4afc6b838d8fb/image.png) +![image.png](OUR%20GITHUB%20481afe11c91843b796c4afc6b838d8fb/docs/header.png) [![Discord Chat](https://img.shields.io/discord/308323056592486420.svg)]([https://discord.gg/bittensor](https://discord.com/channels/1259812760280236122/1262383307832823809)) [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) From e31cb7cd81eed8e140ec9b2a0fc6e898c8df11c3 Mon Sep 17 00:00:00 2001 From: LEMSTUDI0 <105062843+LEMSTUDI0@users.noreply.github.com> Date: Mon, 26 Aug 2024 12:55:16 +0200 Subject: [PATCH 056/227] Update README.md --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 6a92b45e..28390759 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,8 @@ *Bittensor Subnet for improving cancer detection algorithms* -![image.png](OUR%20GITHUB%20481afe11c91843b796c4afc6b838d8fb/docs/header.png) +![header.png](https://github.com/safe-scan-ai/cancer-ai-3/blob/LEMSTUDI0-patch-1/DOCS/header.png) + [![Discord Chat](https://img.shields.io/discord/308323056592486420.svg)]([https://discord.gg/bittensor](https://discord.com/channels/1259812760280236122/1262383307832823809)) [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) From 435d3301e6f24686dbbf80e7c239ce16b32a7b66 Mon Sep 17 00:00:00 2001 From: LEMSTUDI0 <105062843+LEMSTUDI0@users.noreply.github.com> Date: Mon, 26 Aug 2024 12:57:54 +0200 Subject: [PATCH 057/227] Delete DOCS/header.png --- DOCS/header.png | Bin 50570 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 DOCS/header.png diff --git a/DOCS/header.png b/DOCS/header.png deleted file mode 100644 index 837fdbd0dacf6c58eaaa0f9b1a789a174b818f6b..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 50570 zcmYhi1yqz<`#wx}ccXN7BOoB%Al*{Z3@I%j-O`P8NypHg(j7w$(lAIjeB=53&w0PK zhBb?4_OtWecV1Vdx~d!|8W|c43=F1%{0B`K82AGi7}x|9IOvml&UFFk7p$A6oD}TJ zC|e-(A5<54eK!~wbo{@6urQg~Brq_8oC+T#KYqzR$w5xXUm*HgTEynTtaPTB#Gt)B zGa#iwfD4;g;GVGIYwUAIiCCnR$Pygp4W>Z$)P?=W%Bh) zK>La%&+g=S;90gW)>?-TM#vkK;J+UQm8j(Lmh~8gPNE?6XmH&BJw|}*9KHxuDgagn z|NDqeOuM3C^M-5id4%D-0)d4I@_!x3<0Xj+9`bHvGf<|@v*JzX18M(X&kA~qCH{e3 zUxYEf1K^`X{}~SDdvb?qmLq6ppq}LI4ibSEf+P;~LpLf<-K(r;7nkHo;3fFKE<$>H z=Iwiz%4{MYF$dQ%{~L2wy^(LtA~~`a%SI01^}o*f+^Nl(bS4w#?Gqi|1LXd8jUf5` z>!;!s)&3Y+{F1Np=>JUw0V8v4pVgrUh4lHq0V5|2=~r?sl+%@mXw^Zu|Gb{?A#^8w zV+R8Va zd8q5Kz9+=aFZtg#1laU^fJ}TUDKd%vjrs4)z@{5)AKK!1O64c$p*=YN*Td_D0(`TH z5-&rN!kd3jC?b%QYvqqMuC!*2Og-|y3qGeUpG&Zn2<2vrYhon$Z)WiWyGh)P*9Hi% z2>)vtQhMQyZm~{(E6)pK@@!gvbzt*pUTbe0cYDyy)hz9K(h=*w#>CD9Z`S3qv`TvY zcpz`T)nYu}1-TqHesPI+fzVyPW7+s`Z*T%NT9Gz8M6p`h_q_>(A6ow0I^Vc4e%$=; zsP3X~;4!Q!|FBsH6-k2OET}e#b*7r~#NJYy>i%~b?xI5tRoI4qw6NCckJdf+<(dv? z$$iM7mi$-do}}t#m2YRP+1lvBxsbgKTdEnBIB?^|!~d0%yHKQ`cbV7RJ?pS z9wz%Q_-%vror!|}*W~S7tT);P^cZ2`yk9Su5EYm{SJZessylABzfj(a(A$P?z5 z9m5BoBJ4fg?3(NH=o6g!?SG9}0t{Wk(hVdD4-1yMbsmKsxFVUz0H5Zlc!Zcv1av+nQ!p9v9b)KIkmsNki6YO5>vj8loKjsunQ$n9g?Bu``;ab9^LG(Mp-z zQ@smRm{9Q)b+!!L1xr*PAIQ94iiErxPe-Mb|Ed_ey2E$rf$as{^BrP)^cfP1o(|p4 zt|LtUjI=e;0t`q}&i;fp9sA4j3uCmIT9k(r98zHZbiFi2nXR@42c$Bcqv&}6rJw6jSlbI+cY&aTHm|d0x~JENj}r?BU2iadAi#$ARP_$ z%@8}coaGNw3PFyB*A9eTWtw;JS?r}#PH(Z#lES@`UfN9IvmD6 zI#kTho?s2<6;LI)F%Hbh4e|~TAbWiuo{>z*r}K3ZaLxVb!|Xord=3xL3_LzPetAyC z!Mi23pJ3K{EV+)LnXAyLQX>j|BX0i3U27xw3imb?-{H}7E290wg29uZacq~tYHG?& zlLQ_1F(w%xu#!%t({g*uNL-SPr{^j4R7aI9n=X6)l%PSnIJrHMxeP83uF~R ztMQWtTv4FA+L5!EKf&-i?!ei+ZQqhD84Flkbk_B=GmSrGRQq2mDDR3eXe|3Cx?LD? zF0fu$SNxXvG@W>SC!FqvUV*yTdgS-sbD#rms+Rvg&!;HgN%-#L0=re^V}NHMGkp=(^8!r`kUs-N->v?T zXC#gKIUCe*apV$KY_`EP-rF6czkEen>?JG45KsMGD||4zw25kckG-eUI5#A|XFW}j zON}Z4*=saZ3w)2?Ms=5|#dVLS4hgg|)|}ZNog!o!v!0v`wK3ByHqVow*H9aY{q=h} z;C%2Bfqyu;Jfooa-pILjn{>JHP%k7XsGaSp-8U&)oW-ux_E$pWM7BzolIaH}*1a3w zhF$CaxxvS(iJ^mx%v5gPEau85rET(U-vx0va_usNv~H9%4yjeyqzu4GkD=~p_v^?{ zDR{f`AW^|08R5W*X9i>wI9Af!+?8&vcO7RQJx<$r$%Pcx$)HvaQ&=IU*3JBm`zHmB zCueL)C05NS){ogrX}Dd{v9WLCsT0`t@@>A51=;uqaKop5_!L}Uvy z&~JElA>VA1hl2PFIOQz?;acKyCLGXvN>k~6QGyz-t5^OkVMVn;yTlo#%Mn53Pi&TI zwWY^o+6@R5no|Fwwh?SJGfR;zWIM;*?M#r}KK~QgU$IwgOmF7sY&_=+Z20}f!^&-U zS817{*uGH@|#qP%1T>;yT2UV8i=bFJvLT=k^A!~9DY~Fs< zs@{jI0fvB89|XQIztAH3{s4K($s*NXeNBjykjnjHy4(G@<8?Gr7voT*RAKTo2FfIc z>{xGwT(Ly5%R}0N7rc&)`Z`iVY+1`wI&Hg~Q?FJdXGPtQm7)6Y2-^)u$uSW=$;y-hmkDv)vdYV+}>lMqLI0#;o z%~15a=KNC27rTmHyVu@~((RmYV+WsjDyY!4SGz!YG#CZZu>teAJ)qnZ-+p%RzL=g^ zS@16^*IW70VhH@@>4Zam8GK=V@Kbm%giM<;ZtS!%#a1F?`-R09u>E1J z>lE&!-!AW>>4@AYem+Y0hVs~SJzyIXy(N@6tu?Te%em?)dfRiu_nlSjS|AmAyC4nv z$F{b35`ifwV;k86Iu9ImAOtpk!&W31tmE;YUhJFqFTmyRw|8y#ZfVf9@l4@lPJK6>bA zml0mRjIRA0WfE&YKY6ENd{}?tQ_mDMSJ8*;cYo_$>v@XUr~h;&F?;BD;w&hN$l(uP zqCQzaiODvjfzWln6NYpr`^T4MS}b+4o;|oU46}1UL+vbn+ge-*o)jls z*1hJ$(3aTahY(D+1XWJNUH26}I!&tEK~~XA2WHlbeXXkf96|(Sf!xB*6fV7_^GI&q z3=%|6*7_EIqJ5PcVZbkU8esY5Et6)N+2*2ceum>SvzpgR3zML2zNKpNaOlyxP2XbgE=#*Pm_KeMzQr zaemw?J+oe^9yf7yy_Rp_ub^I~5yiFGI%I|c1NFjd)<=2{N0Q+g`>xCdTKX}uwvIHq zQ+{ZAU^gR*AhGUsv>ldd6cfD;wuChvUU9hJ>X;uYXUWQaQ~?cN)h;^QmYo+rJ%oUB zmG1-4r$~Y&KJG1At>xcDW>%sNu6Kh2cF+`3G6ABdW2h19jc*rSweN1=-`PW4Mnwa^ zbA+~T{tlZg07-1ME2iGQ8C{M3%v0WdIws71{Toxc)0tVBf%_QYa8A#F+ZXrMgh!bx1&aw98t_uCu>hv zt9-(=1}W#!+K}hlOQxW4u&LDH>*0d^7DR=oDrB(08?N}*h$ofZ$I7D$q|aA2Q9F|e z9YAT~&d;Qr_{4oq6dR)jdk{NtmsRm|hzQbA1FqU2>f zfyDXbdp8X{^PaRh_FSH6MM?&>luXBs{;io+5g{+5Y=QZTW}7lFmRIU9wtR+S z@|hGstpuSH*e~3#*y3?KIT4vz_Rn@h5Q$Zz5U&WYJP|p(U2F?YK$pMw!=`0HojrK{ z7*5XKF5v+?2wo>Vpx?%}(#Y6niRPR7#)YG__4PYMze4-1twfgMwJ&9W>dLRo<5hTj z+(kLNLxcbvI6~HFFMqNvghZTcRutdLd28_;W&R{&&Awnk4rwz&R%VDsx&bM5pjsiC1tKAQ5he?Gp{+`_d;6P|tCUAw;BNa$HbZce@JnJ^#UI^1)BdIQ3oTtUQ%qFJdL*ulCtzFh7i0vQMH%7X1>Y zPy6K~c67<|&6AY={Q&-LOY6pt-y&W9aG5|L;134A$5`PK0uUJD9%uSV_4pKpqfk7E z2QXkf!SCfv*l2^iXzz6QoAtgGd;emjIybZlrkM4D5?L!wV;sGtpSWuZ2Fd0*;y#%p zJYb&Y){hC&Q;Vh3w30WCeI`6qH9wR%_8~u5}8kY zRj$J|_vPhQVkZm8x0E*X8-xnydod$+am8))ySU3VL$SjFt`T1eW3GX@*pV&{2rq4u z;o8k?DtJ)l{ zYBt2dWB*~qNQi#dK3KwLyl;ujOU?pdILePV5Elh!4coPv8iO^;4|13u#WAy zWa$aLHHJ5Qc9yi}UvQOC4E>nPvk{HV=N6BG2d`?2;PP zq|lOx0s?mUXC~6x9_FZ(VxIXucqYa^@eU=Qz{>|0V$NS4g&wL-#J}j5puO8AXMpevxa(-c5&Wb{9VL3Qwke3RwzOj0Zvu6a4X*h# zr`{wyVwn!G_{JpZrZ0T3-1IcxacyVYLla@A4wziBE%1=Lor`EMWy`T$YfY>!GF!F? z;IbJXr@B{K?vxSozSExifOK^rg2AIxR5S$r&^`gt^}KjZpz3r`AdBYt;Nl=MXQlFZ zf9@9uBfCB*9ghqFM+rB#3-0{XD@QXb^c;$~BX!;;csaFsEZMDb;&XYXXkV^UegiM( z)+@9YU|@s6@ggRxWe4ixFPds0(tlvKgRUglI1?3Si9;^=9Y}QkVO0JN{uP)b$KYFI zRmO6sB7xV`A5~EN3BL98P!yjme>za3I?HTE18F{6~Y?vrZ<%D@`1qwY!=mNr9v7Fa& zYx*OdhL%l8mNu(70^$Q?iB#&SS4!Iy9|kFl9A{k>8>gN{i_TiEls5BiOZ=jW9_!uY zByz$s-*cqYOp%$3UX^xh3i|ya4SXnjghRwMAhla8C1K>hQ;`NqS~A|Wv=MQ*!1?0c zFkWlTh3Q$@mTMMUmsasy@*yLghlqiG5*BviO@n80ogDmULQVQU7!Mg#u+Lr*a^3_U-y-NX)$k7C+(e#IkH}$a=SgU((H&Gbk6?%`rnoXKZU7w^cq@CIVOE3t5^O+Y(^sRGSindnO1v z#Hy1jVenQJ$_Ui5^cRkm+l7uXUNr@;kC_z3p?~G{v^AS4(=UUtzLucK+|2m`WC5$mqr@M`uCC9gRA(%o6RDmKnru%Qu#*`em%bl>68 z=il@^u6W||Ah%_yM%iui)1jCdoQOfRlhxQ|uEAy5(fA%f*P{>2uxJWF`!3tuh%8OLF9z|IaX)Z zso}Rptm4DGyi1z(K)p{HRI$u6Y&^-4Mif`h_p35npu9E_Wl|#Sb`;O#OWmdlF~t&$ zdMb%O25@D+Bm$)<<}R-%lLt9OYxOhEj)v^gCSkX5SZUuU&*3Z#7^@lkL#NCvM}LVCmXcW<9850 zLD)Tp<|a}nVp?>2SAWgb!yI~x!0r?C*I7p;yWE5_%bioEY&E83a1_qFT5^iV(@aOZ z`9u3CEG+uNa;Dfu13S{E!bX0QqNM?PR?>`o9I0MZviW!>XeT}Sanscb^r?YOHBlqO zyIcTVZF7>Ou~|I|Z>sG-mYGD^l;G+L)yV1GW@UDf{9w@(iC0IF^iUWx+bd#N_iLeW<{7GgHk6BHH>>^UoiB379jlfbW065GBXBOry2~X>06mGO0@0werl;{u#b7^ ztNR*Ykdo=Q-O}m6m2)(@qdf0fKA9Yy8|7OqxNP3Qe3n*Dc(9Zfm0uEFI;gS!ZW8YB z!^BbY&V)FEQG+XHmm<6Lq~8Vxt`|#@ zCwx(#=*BE4-?F&oT~=Q7360nf*DvQph99sb9|}Qh`YybK=b~Rivl(r zv>;`bTv+?@9if+Oaj%?2Qr{Pfq>AmkxYtBHIYFD`(;la@@FDH}1<)IKgL`^kcq6dE z71vRX^UncVfY$`LVyJP!+G^;hmfS$(vq_$K?duQXoP>rLGpWYZp67 z#6EFNqT<^R_t=~V`jxPH#N05bby+1!t-@p@jpkXhwnX%2CxYz+NPm%Y67%uLUZgIA z%hwHbe&+TiCTt>)C*^&|Wkztd4(S-77#K_0SS7==yxU!vpDPme3m}|m);2=nr(I3J zRXp%GhT&EUwCNx3ycSKc8DPNAy@Q#S6 zA|LBBnm5-6^$&5qU>;n@%-oNv{GaJ+QCJD}Kan=!dWmrd$@3JbT2!m@epV8%%%=RD z0Ccq1fFE3U{#FI7j0M`y<9O`$PudBPtXd#cDDCmrLGN6SEDUr?uk?4;Z_XFtj zu8#CBZ2Ag&CpA1kMx5Q&ak?mAyuRgRRXSXCB2Wt(5vraU?}Rw?Q*q@T2Pg-{MwoOX zZ3Rcz@>cL25^_xqsU#HVYd!$5jR8x-S_x!H{lEd*ieli*{bxhRZsy^Rptrm#W21hw zZF;;`7HcfC1+C(F#c`GFIkut!?^C=3AS`N=;T#$%l; zr&ad(*8b8VI+2-Jvy5P@HFU$De$OlT^?SaLks$@H6F-zNC@ps7uENLe)Ig*ucnCdT z7m3oG*#5dwNWM4!rIFdKbIwrROW} ztOeFH2jbq-GnLDNd}H}LwZFRpkO19*!@mj>xOvf^)pn1&#)<~=f#Db{m$kHe8Q`Jf zkd<|t*hZ?=q%t>Nbx?>Yi2nkJefGK2N3yclYJ|>sAcZk!oddPf&6TkU4!zK#USn*2 ze4v;Bmq5l=I+=8?%&F0$Gl8G+xQY?H<(*6rb4&Y4-K0}&%ujkJl6%o9lER2N7Y z3tIiA+Tsnm##aVoiNoqh8V5JSmQ9qq;EaGbyb8+m3&e@tP37D**mrQYpgMBncRV{> z_9OK|N=@tJ2W_X(>{!Y|Ja%~9lkFTfuj*q*_(Puh>3PmP!W38;J8uW{c+wx2vjyq`&EcseaQ!lJP8FkOswy2A}r` zc{&R1HjG?)yYQKsq=N9n2Szyr$_2bBAU4vXWd`VlCe!=l+yFo7F%=p|k#f!a6B!qG zE5G1b`LcMdT(P(HW-xLzWV^abE%zSgW6CyvP7`x}R(t#^C&BpctIxM)z!fRV^UHY)#nguO3 zik0N@$dkGc)8zmw>0fbWY}NN3=ofyDKvU`h!xWFf4n|I!P5KGDRN8Cztb5CzRbHG- zF1SgEKBMoUITFoMbDI4OaCKDg38aNnA1NTM?38Fw>~7GQ{o_y&PZkir$)_xaX0%PV z>>yNYb|TEc@x#u>2}ZuI)X$^Rfu?{Y_$D(SQZmW5O?Pgc=X7Xw<1BGRb#ecQzs+sW zV(o@ktWB=RiGS}DF$1W3_tkmb&_29BAf$RH=lIhQn?tpqQ8Ng_2?=6@%!?Ir-=KC* z|Mm7Vs)BHKAn#Q$Q|Me&i(=+&i&1ySGVN94PVjX&Yd5=9oH3yJvneuiOPk}B9ZdmE zn@Qg$I0CY%>3O4f<)1*XLLx3PWt9kGI7dbsM)%8SNE!9Nzg(9;dW}2`-UJ%KCFI+{ zJ>wrm370Zxac5*8{fr?L|1IY}z-z>h zPEPbe*nN-}`j0M6YMg#n9t@zS`052gL7umdLU3Ub`FdH-bfYCs`;rwwKxBDuf zOTmQgnjPV^nJ_fL6|nYZqynuE&b&qNW%57>aZ8hWWt7FYXb)jf%snF8Vk9vQ8$xoM z{@L7&{5VN2vLzPkqllMhAARud#?S*SQ9%=-M-c=h`y=-ZR#<3ixQNytbdR-VB3kje zax;$mVlv5#Jna!|;XUT=wpw?<%O(b?RRXXxu%HDeQ`$)>FtetjsOb%rP?d{IIW@L8 zKrJzflrJ{-?SLp;k!wW~>`&8pwhA%yV8JJ$b)W>4bMW|F6%c`d93J2w>( z(vWv`7;oX%^qfr9IygT7Zoga7L7e13V&FSiz}TBMxxKOBn>;D%gvI#WBX&EP8A=q- z61NaNJG0KAgJgIyUX>Ko9Wfb4PwPLCqWFTUE8b?&@hKvTS$wdxKHnO)0PUn?C z>B_IoKW-+8r%)XTjDl(EYsb!2Nuf)fZe*y*{~hWsW0PUYsh$;hNXc0^`bSmA=Br9l z+|aFrwpd#4C+ZE_>94(?n&dE&bU?8TNUHk?;;Wje*IyBDa?D)Uboha;suXi`c3RK? z!uLSEnP(}$tTtK{B*HjwvJylK33y{UxE@XmX{kvXnIwvS*dt({@E~!RjZj{APmqz~ z=Fsiv^4e(p^LG-w)r^*I*{&{S4`zwmv)<`_p|VlH@nUD}na<_L`VfoK2aEjpyAgm>RaCsLDTpW?h_ATUYk3~3+6dbm*L5ddk_^V|0+qx z4Jzn@ZNS#H*x8>7WKEfZa*v+s1nfL!-{X7z=^JI8m={O-vtQGPo%AFs$f_cQUYfOP z_{)0hzL%0S=Ix2K++$mR&J}_cpw%!xWZ%p9Di3$fW6 z{=RHp%bn7`?z+QszsuNePe$svMaOQK@!C;-bfu+Rkel$c@|$tisi&ILvmL#G<>6Y& z2g}5IZS~h8$0x=2_4e~S5@i^J zdo?kPJ(6+>E}k2(trQzh?~Bfv60 zQgpf_tIk!WO%VIDRpP|_E&jA?*UMx=%U2sb)UNhZM$sqTQ%tD)zk0)8EWc|lyp_)z zZa8OU*&dsmCmQqyf>po!Ny7fqmr|CR;bf@aAfdX{MkD-HCZ&8pI_+y_muw^xSGQwr zjj4mKuTLbHO7p{{!0#{pHZxAJXt<(&=CR%x@|$5`2z~Ec2sH1jRg!u{tMesKS3EL| ze4+al1&W5*SM#~VK;gI@`~?^syBHjE89R)0nYz2BKEKj9KiGuh*Dn&V(?8n6>huXB z$0Ze7xRN!$ul6NX5%E8qI1yP6npR@|nC)9hhb-u^OODW{na zg5fF~)GoMc7U%AMV%A|RoeY*En1|8u7lM~%8c_T0OI4Wo?_tb)mL`mOW|8T-r%PS) zW0u1wc9T?eOMq7bJ+fg5jcE~|Za!0K_ONpc_r7Wb@~wPE@_f`6^`I{InOV&mQMG`$ zRc>HlI257M=P1)IMHJ;L|07XXL@$Bt^klmZ`Ide*BovApOOP=H<#fwhS~cuj^IBVL z6FWqBs~nAIk^0;_sDE! zjaoVyK3e}E?HDE`M>CTqvU5~2GF4K@R!NRWfM{ZWMOz+$0$k-a^awP!!5^Mv(pb zC6nv1or}prB7$Z!-sO7!x7{jvEC*8yLt%-;>TgVt-gpC@4DHL4?&@nL8WaQooPRSw zX}n!vZON~uvM%T_F5S6gGQ1GWnKrV&G#1k%Yt&ckjhz@-xMvN4B#uh`zPa4^rDw3) z#rBrd?CU{mA&p6inK%0<-J1Fo64gw2a{J~b z6Ram`Q$yqy9GX$b!eOqM`eK`1j+iFWI0zbp6tXn?nY8r}^?0URk|Y`w%ZsT(D-=1%*pp5^pD`$mPVGS%|r@waT*5 z5s&HYb(1qqzye(q-+37!_^V~*vKwR;hLsHsafy-5p)){1<3dop@11qx7Pea+p{;PE zAa@rA)X|0FE5Ao!g~VF1JZ$;+|K>Cw;(BGE_@I|c%DtJ*p0;tXZRarng=ia<&&{is zD%3Kjnve9ZOMyT9$_FA#FhS(M=cj7-Ye9?{rA74`@h{$>?yuYVt#$c89|_UNLE^%p z;Lh7*AKeqacXz+IxIZ{(@N!p2{l8tL>(KsgKhs}^`&oNR4W3_^{vOT z_mneHiE?_jN#-ZXev0eDO5zNpgm#D3edFpiiL}++TuzdEeqsOjJm#I~b{!V=3l)dj zWNs!{YaV&dZO9^oXXE0OK%{eFQl7eNL(&a@H4oLk9b|+D zjX;N9peKUlyFw4_YAFv3s7-J6&?Z^>b@P-FD2NZ;Ueu``O{KQncXTEqA+{n31GF9n z0)ZL=rjn4@KBHB4hYCK;R4+HQy$6}|7fRQCkyf`5P#Xi`tKtzqrtxfv4b($LOWShY z+u^Y`M-*-K&jEGIOy=O+I>}kd;S7+EY45FKWB34UP~IjXEvr)A)ad{C7v29tLxWDr zHA+E+3t9oe)}dRzGK+W|!Z=+!#X z)O09V4iOV$%H*HKJO<)n^ZvsWQ+rpYFWY}Yv2?RZPjPcxTe{~q_#Ysd(@lUx#az6s zm~so$#e(5*Jgm9-->ZR#e9P(;pAy1bMM>sG_7*(?i>LpG%l<{Ivb?I|k?gJX^3*oj zWmnx@QdaQqv|frGWG>d<^Y$b9Ys&B45I**J{@bnm#egMEzcn<_0S9;D;lHIJogeaf z3^PrQ`w!-vbMt)XdPbV$!~TRB^m+<1GZtCnRCucaN=%?t5i-1U!^`vzrxqMj;dj#r9&&dTSM@K_16CIofS%XJrW{6*paqzn?#w&-tw9ll_ ztJ}x~wcjPtZ?!X*(qbE4quhp<3#bZ-kVNu>Ds^0nVDSiq>GLc!2c`uc%m1Zf z1kn3ZL6N_SDdS=hyE#HHytT2rHy)dul!KJ0Z_$YMA7TXSqJ9lRO?a7;PH(q%6s~d4 zOppbFoZ^kh+wtU}Mf`BTr!RCC)&g2OW6$uJ-{==xOhezG;2n{~DDVFm`eYR%ktGUh z5@4%y*4ukLXMoQ0+X463$3_f)@yD3$X4*%W_kxGadNIeMi6%tkmLl~@3oWL%0O|fT zb|&vox?lt@zVaRAQ39Mye$A;$*F(C`E?%C&>hzhjwX&x+6c zJ~%QU?Dp)9`9ILRhn-d^QkXno;O%YEey7@*Z7#dnUoL9DTqK}KLvjdwRtP%o4if|uc)HIAf(GYstB+ow;B{pIJ!3I&qIFk+A1c=T0I zcvi+g_bMq{@5M?H;QPCzy=$Y+Ud7D~nn+D@rKmB+voHbPJA^ETWtwWTO+ca0_lp-N z3NDbk--r~O=St&9nQhJ{=B6zX7a!?N-%eJMbVYiP8nqvECH;f_|32ck5LWXNnrN<| zuBMUl<>te|LrT1bm6SZFw1&S_u@)!_!+7Tr3&r{t#aE5|Qwq5lIozj*t@C-wKN^Ns z;F$~ro71jA0Ko#J=j@lH-yx81T$^f*r)xp|r$JKpxA#!Dzg~DCe&cY;8dwGe&=*1*y|~UnjaO08xZRG zG?n=)ha|fq=3kBn9cdy%c4W4VVpbegE16M(Soboq%K7ey8A@b3wj3yDpu^G{tA)WF z5o+BSMWA9W{?8G8C;p)!rCssk;WsSl!otn6S4hX-NS7MmWfo~Ads}O=gsXCltq#h8 zrt#SRx#y$N1;y!*!ilQ-YDFCI&Nm3O{oGQz@v)T%=~A?-*M6+>LyW@%ih#2uMgn4dCf7r z560@;e0Sg4N8cL3R{II#qz>v)Hk1WZa6%bljgr&=Wpi~x0UaaG^|ymfKE3@GMb~>YSC^C7q9g%WAE&829&AXIOTu~EU0QF6;xl+Yp|ovt^|3hnnjY&g$j@w2j1{`Qa&8F_3$r67P7szVyh?9(qA z0-qV!NY1(hjXKzL8@gbX4NjN20r95!qz=Z8fuyI_w*OcD$*JDh)T7#SUgl`J)4uRKW%QqF`&r4(?M=H;+V%xr>FW!vD&>i6+W*1q z1pUrDw;v;GuzJ(jF{uUfN8G0vpuj#8{*E~5m`P?<7qoN&QK&I!^^^>FaWFBL@Sxj> zUIk-`-H51Q3SX8BtTwzVvfpqmf@Gl7lmw~43~_uWuiyD}PEi%k;5SfJ0xby{%>Q}s z_r4uj`Nxt4avI`pl=fy0uKqFuWi0*1%%j=Cj6VXLUdq0p<)6*6-e)t4TB2ODlxH

zF^PuO=!z_K^UF3YkSDg%@=gqOmifm#Y$JSIS^PSkF|*^Ox?8Ff52dSa@o`GDJ1CYM zzjX+U`6{*vxNBA9Kf};gm7d5fZ%kfWG>Nk33G(_a>ILu(U)Se8p9Muy0rHwxvkiE^ z#*RHfmHSTGil)6mzo>zPqXUz+KXeg1jSC3==FR_0SkxtC=lQ6CzU0QEWBl z1O-u{1^t1XnPmCtiNKfq_|7137Rp25@_y{cg*MN?90u*Az&&Y5)nSc%BQXI zSu`{h-!ZtPGrZKs@Rpm1dH1~U)U!O9`7U8+iH^W4)yi(3*KY=7_UG=g7^Xr$QJ`rW zmmeU}q06MD5Z>@AH*diM%(-q2pL(C&gQRL=tK(TYbk&$E@niM$tN2DtlleQxJZLWA z(IR>gn%LL(nC1o#GQWuI3 ze6uKRleN{o@soUH?COV3ljLsW;mTWH^eHIEDUBysk$PD6A@&SQ5CCU-*HtXs>37k$K zvJ_a~aqXmUYx>xK%whH>qMev6&++lpCME+)ot*i#wkw}D0>qubNw`fy4po5n=aS+7 zHek#i-(E@!d4F%`m(@jJ9~mGkvXmx)#taRRxL1=ClI6^Xpb9{ZGJt-h3(qSlgOqC_ zTS+WH7yHo*^>Rp^tUx&CP*B69a9%4{j2_;=!1O~>5#H^QDM^=ImU?BoS2V`iXAM=M zn;$@UEo8$V)Vu38mqc4gQwc3h%cnWrjb+!ikE(oxY$ZM}zI@GPgbenpqk2D^JCdf` zGX|TSlavS7@%r7LSpC8v1QY+S_-@mlCdkT`B1HPnP zIhq5MSFTckOaA~rqw2p_Qf{zP+$+7SHO!<#Gme`jEGA?%%qUeIqu3nw*-JMf?S7fO zGpAh8;yywZ{u6$8fvAvy_M|g$NcCT8UOH9d;&W%o1wx18V^$D)NCLd5f0$jwonO-B>O?dC~Q}QK-W3-8mdRrTDP2 z$N7PcZO(&*iGWz z#O`xV>x2h{C;lZ~`;=>j+?N9VXgMTbiG&A9-H(gRgv{<*l1PVT(5?e_i{>DxOALfjvv6ig&PD0;2*~G&xa*t z0EOY{?}A~F)n_0RW}Y21s@Y9oHyD6xHm zqMB`T`P?vj$<(=v)qDHsi}zk6mdO+Cgfb;_X&2?hurv^8xeL&FaEWiXD9HC&=Jn%J z4V%vx?q81j`v;7oR|2=P->K&5)TKj^)y#Z)C=mIP>tmP(@g_7ka!k}&-$qjm%(Y;; z*w^8nD@kHg%&0G1(@ibAQ{8F;YDv@1Gc2`1!##_YF;`}@t%0*|UHRX3HbnC&t|Ei&D^xRdR062=-#H1BQHdGyN`|E+~B+x`iC&I|^kT z7ZjnSwb~jZ({h98SJ}h}Bp=CX;_sa#C}4!Sv!li(K0u-qm5gvplsY3=z}<=V|xW(KQ(D^bA+Ua?lB`)C&#rC$<{Ye?qz zRbQ<4pq1I;QqWD(I=EV`_fqp+A?b%M0&z|EL}WFkT*PUAo{N3 z{AYT4yLPWr>>GAgQ)&XJx%JS?UN${8glgCg>#-wxE@nE94@R}T#$-dsGE7s2P@GU{ ze8h$D_yJ@Qz0`yn*NTP=3g|8urzc><8K6|<6I$mJmycbtfxa00CL-K2!t7Cv+j@im zZ-(dFTXJGULAG1*0=&U>B|v<(&17PDJ>OE5p9N8dSj9FtRXPx@LH6!Fh@Zy2v&5oP z*QeClxZY3HBnbta%grnLA~$BTpy~*L(sw-v{Kkg8oCs1U{hk3(DyLP`b7`m(jY&zh zK&i&>k5G0Q#SFb9FKszMz1n}~J0WmVdUuEBPgoga(IwaYv8W)bl1;~|i5*R*L<_2f za>>cmdmOLRQL6V{QlNa>RX5i)$fLnk|2x1L9FYJ zz;{$qU`T%z50nKv%6d368nSBOWK6%hdB9(d%Z$~5@744n0oB~h{?{pRG-9Hily#}- zU9`OXB>vz1FZZ4Z`dN8m-{o~?X}2O3|IDsR{_yYzDpU52DP;4^XsEOP)aWc2hCZQf$ls z=n&py+8NdW3O7FYZx}u40%#n*eEG7Y{(V3q>q0lCY}JM-6<=6qc-1v<)@oxAJeb^aiu|Ku(uj$vToX zfoBP)toRXUOd%gJv?VwJ8qAzB%3@@SFQ8eaQp8#K8PS(RxO7|bN)A$#&boRVQrEgz zJ3`j>6{_Oo_EPo}$;6Q3cHdt0JN8~Zn>B!Ki%)NrAKu^7I~i5=WIpX%NO5K#Aa}JW zFD`(hgEdV$7nrttWuE*jei*~seWBVWuHJIawB@6J6E34Yb+VN}CJW7|z`6|nWqk`L z6){0z(uiNXK?4x7%1g||Ax&iOdDk~lbIAH)0p{9}VUr1{2JscARgPfP^1VYt^@wd& zyQ`XFgA3o%MWa;osyip;-7N%VlgCt~4v&9|_cr=GV9qetsN$hrtsr@oEaN(4^7a7; zU}~rRJ)i*kgH`8YFK9JoG;O2#>FDu@_3yoW3mpp_6D*UP$7@-x?N!C#&@!?$mUZH( zrlnQ5hjks)qe2Mp($Hz7@noI3-x3oBipSFxtDN%+8+7*pFKgSdOO2=jBt=%;dYEhiZK(pZFa^g{u zI|I5WD{6boU>KV#+lyz}!9}A(;d?d#)2g>`=skxsq&^A=>P*Wv`y3IiG$05EE@QEj zdM%lMCH66Mh8cIFC)lktOp%|eJd9x<+Urp>&|Mg`qoP9M!P9K82`l3EDY?=|k8J^> z$w}FR`od9dQ*#l53gCH7IfVWF7^}ToDL zrgmjBl!$I&*XS@m>i3^V`;S=4m%6c$3Q8;%3FJnwJn8IW8OLi`#NPDzia5taZo;Iq za*2~{XrDYraP;z`w4_Gp zTgso*kWV*lfQSPtB}O2v^!#nRZKa<5=Rd1NKU)y#n-H4X%}V}RX8r_^knRsEJ~5~T z<6Zot&DGQ_$uJ|AMiG@H9ine%S+z6Ht{Zq$S3 z`aY5r_2HPY9tO63rhYCIH=RM07VFu+1R$x&E(57p05}n?LylxDp3ojr-z=HlIwi;! z+yjcqr?wAKmM^{O%0*^69>)dR_n{HsGARp}k>6Kip^f0`;Lt^w90 zlxp(cDs0aPE~4D zY1oCLh>2D%d9(u1BcQP$gnI+pX9Wt={Fl&@(b4H~0IJ`yjf;d!!oNz$c>62EZl!MP zd|MbB^iLBKKvAenlAG_CEm7TW;o-&tv~R&rYmlT7xf3A102LRzBC&AD1$o$Vo&vJ~ zno3R7lNRZWf_s9iI2wzBdJcwl8Z^Nwo;2cvPI4seOUw1yZvQ7QUhb7as7=BJJ{Uc`ayZ8rTAT zO9t)jA{Ni*E_|pW38#xrlEepy7#%NjrLESMSdu637y#fniUO0jyuv`}KuimJz-shH zrB%fQTgoOY=pV-SWK}LPK>B4ISmwJH?{u&@J4g^OC_3BLuAKa8kp(bh02r|h;b}|z z2|bI+eq^hi-!X?Pv__sq7f4Bbg8Q7hn)G88=rdN2(`A65*cb4u&}dbQy)1!iHd!W2 z&E><2|A*j`GyV$QZwEsT$d+4Ei)gKV{WwkVz{`lag)Z-VQ609?BGbvj50|AazxR%`MTA0zE+C4w>e9k+l@4xI6%=s|r$KCesj;4h3&L61%z*pwamB zp&(Ej&5b+R2~mL2#}-Fpq}IaB%M9AJqjSkqjMywV-!SrU>eJZQiUUFd8Z|R{xm%Ry zT8aBwzSNc1+ZlZN7_u(=ElhB~B-yHaq~sxIpvvUr%p7fAZ zDo9quakqf2<3ADzd%7#ow76%7C2aFEvT*|Dwe=eq91zbnD?n(~Q%mQgQ z<-{%KDLE(*rQaQ~&bNQM0wgg~#3GSKvI*O$i5sDD&^RY!GDwrG0tH@ODb~Bd?xJsP zi4PP{Vu>75=?I}lr}VqqsH0mcu}AMf6OE!l%`%ID^Oq*Rnqwh=LUI3E67zBIisl1^ ztV5aU7+tf5xm=Ev{5~8|QDlfPl0&E)sRV4n^|l5`quH*{7D@#(V3;YqAg>|z^mLvOR+?3} zdd9^%(fqGc>?f94U}Ll4d?)QgiT^r-!AHJZlRI?iBXwVj>p`czu&!oH`K8|bzY%`0 z^y=ibwvyYel@THr#gulb71m%=i25(h{DNxUiGzA8bzNoHU&e#TbQooCL9)0v={YF_ z2Jqvbe6Wv1V{vvYL5$&xY%{LK^`JA+e<^M_*^sKdy3@@xJSyQBvbN=-Kd+|B5uFCd1JpFdP+adI0Q@s8GFEZd^9|XKvq@l zmHXX$(y1~0Q|P%BFiq^Vyr?gLv>KoJLg4ryYA1InomF0BGhhqXR|_;#7R0BODyN4lUWve$jw^K^j-4mzFqgQf2Ee z0)>E_W}s2nbV!z{pwUskrib=n_X*G_|Vv$p5E#_BvRG9bPW`9-)Tq~ zy*lb)@*e}a-M^jpWV^BI|HNJ~#1yu{Z^2nUOgu-m7P%f$D(5#zGkqr52*vz?7fonK zv^Vat^hZ$+G2LNqQ;)!@I8fXX&<`EAARR_U80t&KCbA{%n7OG+!^7W0A3hs(Z;$R^ z;ADzWEw$HOncTyt0_eAONF~=kDA)07@WYRLXB+X%|^Z&j|!bI*c$njF2Ng!cgUxGViktAS<{2 zb~DK$Q17)oz9wn<8@-rd!G4ZdjikvxJ>X~4DIT^2BL49-Y77%!VPc&Op&sRzFT}oQ zz7kD=o`?q``%@aLi0%#M{$ZnsWx!qb9_&478E2G8tK<|mykkEl%Rc0c_UQ`5h70Q7 z^a|?$=4WNexq34|)8C<%UBMmY+(&GO@++*BD!NuAlX@;alD@X^3%BH_eeTcal5*sd z5-5wO&M=lvDauv}sM^>m-vV54it1Y}2pK>)d+Q;1vg01mHBNwX2D!6C5hjR#q|$t* z$OIsWbi@f%-wj@QA19{~IU zmxY8e8YZVZGLk0q2|48UjC)JXf*#cD&`M9l<3=4TLY*q|eN^Yiuz|lkPMOOs`2SdS zzxt7X;CsoUqX(c5@|s4|+D+IK3j21#E^~g}&sK`?YGq?Qhqma^YpzI%3JXu9r4&SG zCj2eT*42dZ#HF3vc)WB$;JH!L{m0kw{~w;cL;mvSt>eiL%u|)e70`No>RNmCZ{Nr@ZtI9qGNj$4Ip=il`Y z2GSw|YGYaf62Dn z4JHQUaH~$`Sfa=`mr{nTdN1b&_v6Hhr77xo0p*SkD}q0I@q+p9EUX%W3-NcbK(^SQ|7yHt9tKiqJk~);V_%-}j=?J^ zKd?O-L7!gOv^r6X^}VwB=`4xPLnGAc%=cw53Kp$1I~afiy?^wWqTqe!HBSZD_qp&1 z+SGo&7O3es!#ZgNpZwq(jE&KHJ_){c{$pwOE`@*QpnLRi^R{qp$t!WUz7a0vuB5FfP2zI0%Mx{Ii5m>vRNY z_WQ?OYu5(1mLeuN?C;WU6$&(2q~yT-etxv{c1z zy@;68hj9iwQKCf6}LQB@qZ1er_eAUpyDYjmezvK2c5CobI2>7oo zvdI)8dayt{zvVwmCZn`FmVs{tZvFF3Hzf*yHk(7bU6#IQ=Thh!EA+oJ<0KNsJ$mrH zOI!{}9pY`)AK7R{0b9>Z(Vg|kENPY|67H8>Mc=-R5}63!)RTbN^b@S+hh<8~3P($= zjECMcxg?R+;=$ zQD;N|{0zJO^eej2OGREY)u_L-TYV;+O@jAp2CvqV?DT* z)Osf=@T=%g1z?7P7OK}~(Fnf(suxB4oO7nd(GNS56&)7YDq)HPXk-7po#74Nvlg6yl`W4*9tUzHlOCi0I3E2(E?F>X7!0d) z)dV=?{)-w$v(6hONG^FSxjb_j92YTO^Sa%7~*4h2<&{b)Lt-@|Zkjf1|!Il>D zXP_gK%hd1`4_d{*nct#sLjcMa*#X(#eRH#JudoYRWf$Gy`F0`3E5(n7d*#q%Gi!S7 z2QLh@4KXCOu;f!9)|p0*(&zhKH=ge%4={!kb}>cof6w*<+Iq(egJYQd@i7<>N3KQ4 zwLMmY$iJxI*pMo&2=aMEFH1^_eNe(uhLe1cs4V7j*m~(`f41yd`35<(^~)pkT;&&} zJXtKIP~>`+>xcQzRgE+{WgWGbmseNJp3U=v#+;|AgMwV@0m!Fd5O~)hV`KG~rZ^<~ zTdfgtf4Pc$LuGZA`T0gNr$Q^)Zjs|H7B$<-1N;(yrS9GDx8g>4q%NzhOQ*jjv~aVnovl*n zWpN2`z5f+^?WGSpIB4UoztQ?i6O!8a+k`+?eRF2yL|enaFpj<(BeQ6(9X^O~esEzv zm>IS`W?pV{(b*d@L6*1O$)ge$Ic9LZ_Q~a#W_!N&?`QNs4(ddxqTt*dgzlO5Q}EIE zTkX(-=x-aJAdmW&;H$tDhX@~KsJ812o$bP-KLoZ&H7{a!B^&I?yo#-;yGz>e2#sx< ztz*(uO*uwoy>#sQ-So9d)64*vr5x zDW`*h6+L;P%_pxU8OYGWQ!1A!-5Ws6uTood8SKIe#2>xXws$sTd(z(^cgeNEd-Nui ze<{<#<0J_2l=7_PcdIEPe%ARnQ!BDsHywugoumm z57-tL<~#O1=8vNp_6=Iqfu0xs28SmL zss5FxWG}OEZu615{&GvT<@$Di%%&M3-0e%>bUV@pg(A=fwPpcU!b?HJ=ho9*0jD0X z!#BNOU#gi68joKZd`J#gfGS|VFwCBOf4m&z*o!-@O9LY!WJJ(*O(3`3{z6x7ciehq ztTb4%L=~R2xk^^S1iB&$q6c29Wbtw@xbl_k?%H zB?4Cd{mYNex-fR#j;3Uh^^HYXCG8w%PWxRxU%#)9XG3Iw{n9Je^&VjZIPgqSE1OLj z&g+R`y^7udSp6pWY=-pGkw@Y`t6-1ivRr+v%Sz}pE(y)3q=v1%ILj{pmkX0v=Ph0) zl5p4atkXSAeJNCqTJa3`YPK={Ewq1N&i`dz?XX@j*ME=}ca^NZaWJ-nWgg#*Hwx{v zH8YUH)(n@i7{tSD_u#2O0uksP3>>a!6FDkRCi}QAfEIHIo>=oDlm6MrL=7v>Bw6Yu zbBQfd%Wh~;FBkZ5+7iQ&^>AYE(WCv*{EJ*Ow|B{E zLeeQy{rB6xGJh$b;!XvZb&8v6dYb5t0yIO)0poJup-6H$Y8eGM3nW9qSbOWz@A_pN zP$Qk-Ry=##-6fT_NE6de-5W(lV<&~kz4XuwYev%=I%HkV9wIZO<#j%Wvb2G;1TQ~N zg1SRI{ofuDt^FJc1aK$hc(Ewt`}Xxd_wA1e zcy?mn<&dVYzm!+DTwXicc_Yyy4Egy9Yv}b&(vknY;_FgkbdE6zft>I`Ro+p8GUw(Fkuui6!opfs!IOEZ_P3N;0xcMf}c zUmrt=AUf#+6GqE-+-VI#VQw7A^+Oqr1;$&l!Ga$0JA#t;0@Ma@ZJiQOy$H$vouk6L zmQU%TV;&6UKO0v3UuSMrgcq$8U}QcXmpC0_rvK<~GM%{DO(Z(fI_X8dLG3RM>uBM~ zFlY-~){!h<{mrUrxvi$V7l}Z|D!);KmQf7xYOakd_O|`dkGrIYN&^FcxyaYajOy)r zucxB;HCCK}(rLAr6A|SYHPcCdp^F)A&6{JC&=BE!N2%xVTG1n{d`c2$k`=ZV_-WG5 zq)Ly^f#P@P{dkegSm%sIWjh7n$Hj!jSnX8|(yU>p=_5nyy|~Xm)4dK}4BWGn`Dy?UpX1d@prbJ6_ zuadj$c4Rq~f~m(uJkKwecK6M>7O608jf&2`By$rgK!WOAs`K%>9k_e%$8LMbsmV^@ zXb&3D5`Y6Qjo?fU4*)%`QFew0=ID~X&t{yro;lOtfE!w;s#ITwd4Ba7UjyIF2as@r zbYV*h4Kk@`VV#*hssL{JRVpZXfbK6h&%pDC~Y)!5JmmBV);JcIdYQ+Qm14}wZ9y18eYJ$mVs=e zyK`?WeMU#QM(g*}0MdQ1`-8mh*Gb!L%<0GIr)BideLxVOH6*g;{2cle#kry6wxJ|u zIz>$@FB)CJCH%cRk;+8envKhYcv2t-X}zyPq`k5y)tS!7aE=fJZkr7^hs+8s2T18@ zf!!3y;Nreb=X}oF=eLID!}pn(SFaR0DN8jlJf4@>75!7)r$?u)4PU*jTURSm~r zqEf9cW2gVUwwl>~Xdp;;lehM97`NXhWRd}fb>T0IL0_^{3DCCBFQ0!mUIEJr3_I^N zx!j<@29V+e6N38DGzc&B$Pd9v`|k$Cs!h>P_wV_ERwP#Ie*)3QBkz_h>ydALRAbZ2 z?jUBQb@KP?nl3j#YNS__8a<#QibdK>9xK{kle?l#9YDG`I4!M9^){t!5WnG~%s{)E;3lHB9o;5wI*+;s(sa1^*DT3(Y+Y z8U@u7Ni11sha+dDIvk6#q#Xq4uu)NSY#W2_8wxK&3~-XU0>b9GAtyzEH z*eQXIf3oj*cv~(7Aj9a7mv%HR4VF+5uRzE3(z&jI&mBoD>*&@X12My+QQ!-{wvWtyG!F7v7Y|BBq~o>@lz_yEc>q5Q0aM{BXmCGE zW4libU5_j35fs5l6X~Jsmw~+c{)Xz!N8LY+$0=1+38yvbJ?W^``Xpr@*}-G(<6-4Y^}WH}*zE_toL-MfX4qPjumO0(fV;bs zfwE|4fggXkgdg~w4ff0i-Ok4Yib#`5UBR`SL9hsHo%jB*70QJ|f?yj#6;uTW>KBdL zpEX4T88q#99rZ5iUZe17_U>@B-Bul*V%)*rmuh5TM%@-FJ8}4A%cz4=di6AZB=LFP ze~ZJgiM7rg!SuENG$p72cozYgL$F-D zR}P_;j`(saEms6#?+>XWJ^c7P#UZ!n-2Tdj9^ATFvuB6=rOxhfLsyf$yWfu)**1E6TrXpAP>yO3vB~%FT}v-ZXki0zQ?{Fo1yrK?p z(sf6(HkcvM7;B_%y+F^&s*fyB$HNC>w~W*_4Mt@izeKt*6ySKk;Du(8qr;RF>D<>~ zSRx2)L>ydtCA~$A^=>(S$qv38?Hs4LZYn7lWBfKEp;J_&7cNkv?T*9UUs&XxGJWbe zAV|$APj5}62*i8<2j(O1F#Y$ldLKvE;uR(G-h6v)&g@ZS_o8W8N&{$zjn_J3xh+!? z(ilF6vwAx|ov$i%!B5^$mAfq|@(!&TbYz!9#RwB10 zl7Empu<3*->aJCldtt+6CRStA>T^3#uzxI5XIt+jgRH2m6v)X~6$SYzknB6WQ;xeH z9LBvpk+z_=fqGjBS3c4NSsD6#!=c72LBUDy+7+_3J;E8ZRTY+hO5?HOy{>EXxMkbK zzoM&NKAnVo(6(Eq^K+IqaC_d@K61ITk4!}-W`k`i`oRHDbsBs`+u|b8s`)&?!7=u`BpE+cjqWgxmE_1lFqQ#U^^-lvX^fr{{q5 zy`y&L4duP^>5FBHktDf-1}9%(5ugM*Sl_{)xYXfa(%Dmhqo})MFau_bj3>dFB20`_LMe3b7P$XhrrnD8J|H z!V}uL8i=$tB2vW$0peo9&ks4+iHQvF(UmVDG%!P+FDU)MKA-gY`%)Mo zaW?Zx)N}Tb@05*{cLuw7i3(yLrqmX?O5aHrW!3IgVuwy_$RYzP4in7P`q(zOu9AKu z{@sQf+xQ$-erIQQnM$zqsU27dR5+bdQ05MwG|tbf-#TrV2EWVga<{&s+4Z3u#>uaui*pEmWswQ zf6rotf<<%Ff|w$i^u*Z2CCk@h*>)v&?moBLk-G0qYIQ8BHNhLy#eD0|xYsc#;La6T z1S*mbt~84JwhP*tom8X|PwCT>e&9pG&qbw&GFpB!dC=1%eclVV?0N z<^P5iq*_SN`$#0YG9)PYj1fE!%Zq%EN@z}Fa9=GC_1_U}t2WRl)<}g6Ga0XZ``^Gc zC$ih;DjG)|w0ZOl1ONRpyq@i~g!Z_I*#MehiFqafp9vHy*gK)EuM$Ki9@L+H3^C+xW*d|GYTSBO{kP=st< zYM$)BYrtx`(tAl-Iv~r#^}id;ayTY9ceBhh{{d2;|63H7DVwYdQk=}z8RzG%`1}Z{ zPcsrw3}Vw?E`5B;nw|y(ky%gZ!T}pAP`HefumA7cZD~X>c-@nATWWsj!JGbX{Id5* zJ*?du``HKo4z%p80i5@gZlkk1Wi<{q&cA<$yP}MNzMP0cJ9We)9~sJ?~Djmvo`Lv!gjrpQj||Ca{_%SOL#J z9w>$8>{ip!H~ztk1)R=HO&45Y4&7YIdEqiJkm=20na-30U_mqbjDPO>_J zbloW1el~fjws`vBnajOPA}84B;wRoITz&7p%Uc*zOtR*8pw5p(*nGnAG=A$*;5ZyZ zR*QxLP8Uw{_7jcdwLkK%(6&6J9d7H*{|v@%O!J6nfKlsxcH;V(#td`ExyoPEtModF zmv7U|IIi7eTDc6qdM<1UGA8yE+R<7IA0 z>qE+Ue4;M#Od@tinLe3QQm)QsmZ8CCf82v#Q?E2k+zv+4zpa+HD>}~o^V4J*L%*5I z#BFrXA!g*mh@r<~-b-={=5t?ttX}x!9F?xMF#&AKuK44|_|n zy-Y_MzI=yMUv><>iF|DLQB8mK0Uyx9;d6TRr1uE(0-2Q0uM&Ti%PxVK{lIw4roC`7X8{sNE;D&2 ziBzs0FjOV~tGc0@+2vmx;{*N|*q3s?PhU~moBpPX`q;f8dopi`Hx$lkTpZZOy-sh3QKT`7my?)Ed_#r+(jfpxNLj*{tPvZkpT}qL z>xIvk@?9Q0B(c>YHSDpBnXydyFr7rk{#!57y#C{Ir%Z=6>EFodVM(p{CwQx<1nUln zwdMxcZ_DHm*QQ>bLjzbE*U6Z0 zGt^7$E4P0IZ^+cLFf+^FfX=c^$NdMOt1R3b_v<4eBM%v5NuII!xE`RW_zrsXn>yW{;p(vD0dHG>*PqIk=qlZN+R%*sqa2KDON4}=+ZCv;oS^DDi&p?dC9 zoce9%S6onBTFWa)rULJ;O0rn2!2vfBjb+`{5K}oIp1pREUYac{*R9e$QaN~#)f4QuDu*1}GK!1HB8vPhhCHlH z9gyUXVC?ZJ*&h8ccRvLfsZuL1jM{ke1FA}M&r^d#OJ-LRR-dzb4o9}9{$n6yCmU?>qX+(L*B4A&526^dvxt8m^+2R4_TN1FJO zJ74EzHEMlGK;bVekg^f9_{oJNl#lP>uY*&&SAp%yD)-_mYrA8fR>Rb};Yk~R-PXwE znVh$L4Vp_09k<(Xdz%uub9GT_xskzA1n*;9+tz)qi8A}Am|Y3q@Oo)^Sw>%Gs%kH` zRdKMa1+h(YR|}iIVl|L7pO(m^HqRJxTodvi_j@Ecd|yHCru)m@=E&&i+oe7b1L{^F zUq>OU-?PGZH>B!PafF^(LBl?!@iv3+Ce{$~d|7V8-S^}2^(gdB*YE@mWAx*8qB|6Y zO6umn#kw@1P(SJSqLv5NhqDwxC-amz%jdb6O_OR&gizR}Qxqt6uJ=34>!|o~vjDv1 z*e)PY6tVueHJn=p)dUVQ#Pu12mB+KM<#hQNOswFlb279K zr-=VSCUIXW{6@KI1h?;&b}VO#Ym%C^+J~W%<$Y{jDonI5r#y{*RhUnm##zi(BUXrX zM9fPK8%pH8POx^u1dwP~a6WN;$UaZM%i|rwW0+!q?HadTPo=okR;f+f_ACfcX$;&aR^mB&75|!6K4a}8=kN=$QW8hPKrD>gJk$}uvVZ$6NZVa2kWb1c z;HA{zpAu~R)=paFLFKl^VbrICc$Wv)U58=Z`;bjr(Ywu!H%}4vlA#GPmud)O@fgV| z1Za6(XFmwIIw)$de@}lri_k7jI9i|G{q0x$#_`d!JIbkOR%>72Lm*G&Wp-Uu4S}gB9JAvj!$2xqb%lgnH);My|=p^ChLBK$;-t)waU&_<2s<_{Bn+c zxlHliwcZq9c4IY|%pbEH_?S~r-fhV1d}4n!H?B#6$OTu5PRw(6K4LL;I0?YCQDyL? zh_=2427Jra{ZlD9$%mrKVqxbGB>cif z47aL&i_C-M8c6J_iaIN4?{GYVUaSf{JmiML2TtX%lsnZYy_+zq@nMd6k+^lv@?aD+ zg>7<88@1j`CyUE8!*6!H%WN12e|5eS`jY5N;V!MpCVXyUL8rcC`3bduZfd@Yg`O?sG{I zfZqx>UT+I-3Pr{N2`^e9m)jg?XXPj@&5ASX`AWA@UWO~?Tzamvz>5q9o+{G-K`P6% z;%aAoDc`j8?U{hRK;cV=pS_<>1`?=m)w?f#)G0f~$KtM_G-s3dDX#0k?|*&$y8Qmg<&o=?1W^F`L0mk_ zu8%Qo-t~q9fE8zHELJ{v;s0Azo`JGG@N8i7_ zdF*uSJyVX`9B^D2(&|^wF3@exc{W(I5))o_uQ2R&LJmzg;>b!o|ZBbJ%d#2C>DA zk!ug3xAG&}(N9%RQp0+0ol6_pIFgn6)G8*!09xC3wkqEFeU&KWX3&m_y{(YtkU+S>}9Hk{HXU&OPzkdXOM zxBQ|TMC?25V{?oz8c1asVuh7@%1oRIixxZeyM-G%&b=R3KDp*v*kSENO`-Emtra_@ zo~QmBp}MXTD!B3GN{(pranz=bU?#|xmF88WKLM6wNCW_8&4pjDw=laR)=O-6o-E1t zrXOC-j$>?M-;gIbxudkD&WWnA13C?TYyCCwVf<>UJxKRWYtp&QZ{dpWrx~lG za9W;lYF5VFoO3&mW{9&}mceW4#1Im9NQp^0NM`6j`$r7SHeawlKrF2ipdJwi`1|(> z^xcXE+v08Vy9XTbBf!xhr}gU%Np7fTfK>oM9L62}l~9dSVcp!U%Orq+A>xSjbhaw% z@;uN;0;YnX48d$aVVByl$p*uNC8-+q_E(MvH&mR)z3-yN)hJSZ0OFz&>_bH0aImk$ z^lb=E6OHW$0&x1hk7)W*{V zQUQZ6jUVDymgwN0kp6Z+ApE*Wva^ebQtjmoI)*-w&+4NDPAAjz&X>k$5KKlR0?@?~5mlR;N})!-pc}eAw~#Ren*_yk7hi{n*&m)&o;J^^pUaUzOYC`Cn`a%1S&L|1<*(+SpjQJNXje1Gj$&H<33AJ)Hd@1gi%)EzlmB`zot-Ew5^ycIr5IvZSkh(PW7N#{swPCQ%SUW{;*+(3f0_HsgYU^Dvu z=Dmn#KP#+6KSEWP^kn|mn=iYJdN*tRf(Q<$6Pv6O#_L#Ke6JnYRKh9vrNurjvadAZ zl6~s#x4X{N#pOwS?Z9gWAnq|PLyVeJ#)N;RO^Krk*LukM`^%Hs7d?M@-M0QVOXqZ` zPyFMV60-B90~GsGQg)*h6ta_$ikCJj8d4#}0&C8jE<=5^>|SVMaTwg6(aOLjbq+e{ zyjB{_D(3JB=XahOERs81_L+XP-ffPKO<=?;Mv=&{Cn0-zk=yKglc*@~P_xDrPh$0| zIuOaTedm;UksQf^bef-j#dU(@!y%c+(qpCZC*98_FqO%fKMCr~AOn<STv)8;ezplFLvL?GvxGH}QG-ME=;NogL(v?zN6C)ZEM+iLzsq#v@6#%!Ji!o8WLlV=(Dq^*hvY#YE|OgpUG=YYuBcVw3XdZnyQ@rn z?4smB6zG`ljEI=)edK%a!tfdvjWMzcN#}M_Ccg&6Xp2eRPL`vXAh&^lEuZ%X!I*`g zwvJP(9vgN4Ylp)}^QyPZeDqa zXYu9S%AqO;gb+wG7}LX+K^lI`%jH<-C?OB%krn;l?j|)f$2NLL(WIiqr(>ZE{)qeJ zl~w=E%72nUDVP90_Oq65+O<0%n2{!u_{c>O(Maq?tU7^C!FGR>!ZyfWW#mMCo~_5aJ~{e z>3%G%^b#s4UhWmQUgbr}kAG;66l-Q6JFEg-G5f;1A+jpR^6BZ72yBhp<0Lra5n*8tMpo%0^?ckg}P zKj+Lj`|Q1T@AdhvgRY>UyC?OKDt2Tq^vAs?91+y{+B7V=PzG~{2kV7Ox1)=TCHP-?`s9GRe{#yUex^Are>S=Y-Hmj zqlHwEY=G|7Cn!DI+mIGAS{A;MMthp2u*TGo0%7H6Gq4}lA=62ycxh>$ScI*aL!)7j z4s8}0Z-cJ(5gOXQBuKk7blH&1_;0FdmSV$_U*y$J}jE^%nY4#j0_zc$y$UA(H)|L@AvuX%tQjX$; zR?61(kTR5GX`{rba$vXSr!@3j{YksEkH=2Z#q4dCK_uYAr4j6;J-|(5#4pEXji!RG zf486n^+=g_?6}yYu#V8}1$lha?k;uO!ra^q+V%Ji_9{ASdOxII3@p`~-N_BNURp&P zS4>S|fiK2gK!ne27e0a*o0};Ch@=dmASkaCFElaUqVxV!EY->o#c9?c={Q|D<~CW^ z{X>Gvyl9MwQXm^9PBvuNZBG4+BoHx0uCDQf{s`Exp~hA;8n9Np_>PptZRQoM44ti_ zFcdeXHA%rXF!29{=E^!-MU8?Q&n3;S6>6Itxh`AOxc~=#W%AvWq28Kq5h>qL0E=A> zVSjh4zbm-(%ChyPMeV3-5B0%vyTibrYR45&>lQXL#|HEHa)=k84aBnrjC((cR)aYG6=HtX{{Jh}nfCiM6hm-LZjdXESv`L99@zJWD z_3G##q^u)Zu`>*fwjdjW)pg)xaflR`t~0YQw>hYrh`Hn7Yi}0!faw0P`V4_xf1Z2m zLb4#p+`R6R!Fl7T!qPYy@YwP^nJZ%KTHY7AgFLZehV8{d@{A|0u zf+0eok#)mwjU>X%K;_Q)uM8Y4(8kb0^D17(T1%pH7-55_toK~o>TGO^F6$UIv}AR% zewqc;yyrWd{36=XUdqn&K2|k)*lKAcvsUq7Rn#bChl@U+r0rV(WZqA7mZF(WF?Bmv=&qAVb#JtK zUL5Szkx_aP{a;tCibDjux#BDeS0BbM6SH|Bx;-z@ z3Q0w+BO<*sn_U6zf}J4CdI*lDw`*z_-av0(kON%&Jn|#qB8)`vD7#z=5P3^1FhShP z>2|d4@=ydm-23rqZ{#`_TpFNy5_UmJX(X;otOxRqY{kw3I0qnh7My}uhI5x&>Z!{T zM0~f8$M2?hL*WJVR6`-8!(Uj@u@;l><3uk^7TLVNKJ0}~ffeNw8xek@Vw>A98#+$ndb_Fp{novI3BIon@ec}h@mT@F-O@AZaQ;wCF z@T%lMoV9^?Y8W4H+D(4N6-N!MoRk6v5k$)BV`m8{Oc;n^W2B8yaU)`G0R7$liKGOX-? zn{E0G_SCceZL)rodLBv?3ZRaBhS5*;(?qDdxTAG`eZ?>yc&;<0?0809X<9S9C+!jj zJgmB_1A_W@HF%~bPW&im?m$8Ic6!ja6^Bb?>NRJRJ%zl_r+jz5YVE zsfS(ZWqhgtv>L|0Y1KKk$R%d~GxH`w<7nJ99Rak-4$e5PjAu&pdzr{JAqGZlD3rqy zN$8`sfLA)w%+dPPB?1wqGuOp%*kPtq{I&i9q?}hiZf6K>g@md?+n)yRv_pcxiwqY; z+niI`iof-C#nq3lZ{2T=W_#=1ZQ2+WN(jQ6$Z{$-1RDqvY6AB9-&AeVT0DvNkp87L z8AvTyYh^E>j02{YQ!d9|^1`!KWOH2j)429lA{ubQsXGZL$Llh$XLlyz%L&Vxl|O%z z#8Cd-9wY>=cQGD4EkG|;!+im@AQ^Q}?$V7pV*q7*8*C$&Jmd?16W%hbWxZe}hB zDYsGs@L1ZGT#G-O4ew4GD*%JX!HJ_fJ66HXVE#=ly@f?Flu@nz&F4=utBwO(==+}V z8+E6;u**zDK7$4cNV9K{8LOpz2vy}w7rCC0bB%3X0(uf8?zWVKlryWl)t(*l1oxz3 z78hsdvI|$FHQ7iyoVdF;w_NOmCbs_r!I7a(n{l9cbRt*C?LFK1cK@q&%9gVZ?$E@I z@=|)JTWgYMbSkg#**7>F25qHV1o>Cx_1O~eIC+f*p%mWtgA5PZfoGve*N$DgHnlCx zE@&$lQJV_^efHqmC%7%4{a$jqqfJj#!;*h)_%3%faZ!X0YjL_;tQT(q*^3O!K%W1M ze2L`WXEC(D67gNgCOoDUE3*Fw&pLv9f&ttao1wvDN^wYTK}H0I&*<5IAEYDnXP9^m zdeAg}!|0{M)2MwC8!gw3AYU0O6pH(rMAW+q)eiY*}`>}j1Q?YoK z7cfY7SH?w^sw*SQS;k>|4fu`dVEKeNGi>a*i!Yx>M3e3=hIN#N2nW9EULO@9T?{DU z`%fImAtzlKnAV9JlFe1bJ?Ns>8sWC4Q7zN&fW7HX3HSZ=2LD# zla!W5V!#;A+_rQ&MrT-Pb}n7JrEs=y&&)>#rA;Hqq{DS<8d?I2LDe}Qp!NNyf~mIYC=~ zIFUH7qrB^q%R(5b!;IB!s^nxRWDFk^l1+R-wf*V;3w^uL5p4gA{(`tR1^s|}u5X zZN@5-hHCJiH+GCW=7cE(9b=><^Hw*RMFP>zFszY9!D0FRSj6G%gfsc%;?;&3)1v&? z?^#oDWj|lP|Iub=gZiE)NEtfXD*{#EX2A4p9LHSeMpXEwE90{|I+VrjxAum(PHg)$ zACG6t6&CE!M&0%!OJwoK0OFH|GHLl(RpPCpnnR9k;fmxcIISm?yon8(+`fIqUT)q( zkBq{&pDil>YHCf4j<3kfsPgr1by)xK)qtP93sCjC<%`*>I;PO4R^}f23kE+fU$wJ2 z!(R~tOJcGHM(Jh1BSA<6?%c-&XNs}^z)(C^5w^YCwR0KDo#{$PwMLn0$oKzI>C^6& zsk?1;SKw+Ta?BhgL_g z^@)3Ofg$gHnONjjju4l;v41W=lu)2?=1&-biP4e&23r3!76nhv$#+J~u$vKqXKMF+ zzQTnmLbbAJ#0GLs(A|77b;?vrDiI%{JqpR&80-zfo0jVeuPN#8p8@Yn<<1oa`s?F8 zlS5fn`R+B_G-f4C%&3(h)8Wrfe4DRa0Bj=Qq0N^>$s)2QX+qgqM5&uRfZ9SUoIdiT z)T25js8hs)`mOu^v$hyh4E58#)lu@7%vee<@k848VY=rzmNzU-SYrVQt1Iwq4tJ){ zNAT8J3OGX27&sEs2FPd2Z)gzS%(Z@m6g)_-3@puEtFa?eU@$5#e^1+68BW7sCo}BE z0#8-M7gKk3Wg}~_;i|I%{xwv*DcIrZ&Sp|AnmHa@8zFkhw?GB>cV9RLVv*yL0mSeQ z-c6HP z817%`vhhw+>a(haId;L9SCJe~geIT)}%5`}{hT5CUjepgI?Z`x6zf1KNVrAb? zCf}nj#}hTpz3wLK=wJH#IN9pmw)#}`Gi^w&?&$uTOsw45{f!5KLaC$}BsYM5L{hF$ zB%5q!Q|W}iBh3uEOGet7E})KFSx&Nsa%}l6U&&UOMov0?d3ZY;@};MPe6Cn9!6fZf z3f#-*y<@G4migS@@eO`g(7ii_rA<+@dUlT3EsNj8zjqV~7-v-qGMD>AAyV*l-Xj!s z;Dq_(+t;Yh7MVK=b$l{qlgQMl+?ST{r91%2m_tagY8&x{j@w@^dp*^TP{t{Y8~R8{ z3S=N^qbQy!fA={Ed9$Lhh9wIv#{$$3!)a43p%7caM_3WTM2l2pxNGXJ7f%)%O!oBu z;7E-W!Sk;zxZF0tax%S_3bF#BFU?sCC7VAA`a{PdQFjCncc7=Us~6In{yO;=v3riq z;(V}jt1yuEKZG1LK5c1&oJOyIxY|EJAQCnHL`&!)|C?pke+bDhz3pp##qV{q@z_5t z0D%tw4z-WA7R@g%jx`=<&k8DD(I%?u>vm0|bUpVX7@{`}wdaY*h+ zUR@e-+y0w>07E2d6gS)-IlKEI( zQyY-v4QGZBp2ie@N9g{<+H%I)T1r1KN!X zX#ZY)1>{&xn~ji}-DD+A?b*Mn*yjx`h-V%<{G9#I-o;fWsWD0~VhLlp9#!OzwTMU} zO6x>aVB*3o{;0e9V@iK~kp{1D6^P~sV%yDs%iFGyy&>NC@NNUKHuc}*i^KKT zLgav^^_TQSqTW4`2SsPIJivMqpkd#|c#6Cp;YD&5A+TDNOB^ksnvEG;(1egHv zzZ7H(Pj2iz9=>SVLegpe!C#U5*d;m}Q-6`=n|>`-Vf!};e+6+-n`6GOg#N4+`~Ot~ zr{MYKf#t!9(|`145bN8Pr_djh0Qs+FQLOe?6#1vl+C-oUcC%7_DQ~#{tsUyRScXVG za4f(VPLcnQ9llEmwMdz30I(^_Kg~(9jNa!ByJdKc(&yn6M-}vc3KFFAzIaF{)}@Tq z_P>lpr!uKYrq(e<5LPwRIR2Z2ZQgsutm9RO5C7ZpdUNqO`V{rbIyVOqriLE%SC7DI z!lTiUW{v*GJPr)GzfJ+^;Wet+cFp|Yzviir=408;m*`U_y%;&|QrWV-|Nrv# zh`#lR&aamLF{eO=$~wf-O6LaWVD>*QJMdH%G0UxIuij7yu3n{t!W_=Nn#lGB<9EXbRMShB6bz%a`jym>*t>H2uM>gZ3mI z9iCrS9;Wz~PIpxq{wNBo`5OHDQGY*psgRCO?kg^E{TY&1cCdi~OyDE4zWTtyV*w>? zY0_6R_v2|BT?pV^`QM?q*`i1Eb;x<)zLd>FMT`2&OZ{(9*d zb;#KpKU>(#3G@JJFyNeU%jlOD93ohIulNQ24a}V+B5H%fLC1o8`7B4+LzfBQ!iytm z9mhYg_;#5%AOWp2y3ro*(*LPv^KIwcIsBb^90WL21foYA+)TjwFN?fZwrwth_?Or@ zjXl?imkL*J%BxD2?%AN+e>8YdNS6kDy znCVtZlV|CezWMMfGg0V=;=+AAu)};<4J4PZwX`afhZlJdEUuly`LAQjnq)i3(a-&W z6^VooOt1*xRPJ|t9>l!WX=dyJ=2D?rn=Q*ool3O)VQ3q7ZZUK6u=S^pcR9na;J9My zi3ZgIm(UCc6L=Nxwa+)F?B@Vl186!fhF2Se$CN|*vs*=*6g@VjL@-hD*LN2&{NXQdcr(%x~a+kfm)KQ`{kg_PH;u;AZc>mEh zTTMXJ=gq?e^8R70GJ*ir7pc(G%^_f%P2L7hRCnHs!`?cyJjFPl<5((G2>_T(lX6=%O=;4nOX)_`$%b9@A`&xe$a9I`-LP&{c^{5uUlcpTg39~y0VKK z>xK9$VJJ;{j`ScMmHdg~U14eRz6hk}x_cls(r#L#Xr6Je-sip&RzKOoldqc7BnzS- z4Amt$Lr;*ufNIJCLMnw`N_qFXzcWKS_aSk;Jr|X^yqxJ83aGUvLo+XC`->$MfMm^K z%Ojl`4=gN2@ojPHWmWjs>O4*QUU*Qt{ITwZ4}&bKIzpNrnV@&)g31)N?oj9+wtIl)B8N^GrHBR-0o(c(E_>#V z2!#L4ITT{2wVwV%?G5G6E4V-NzT~SNzBP}CA%W5TFdeFcX6b(LODBtb_4r#r%{ni5 zmjV~2+0X;W^;Mztsit_(jlcTkzyN9cpY`~N4c2t=)RcG z2UQ(U;8+11s)u{Wj*&VuywA0eUgl~$rlQeTmE|}OOKrF`KZlCxh(EcH=-wt}h_D?4 z0watwd=axDxTI`RyB7t}!RYh{{ho2pFGlj9iS3ICe$%$C(9PB9h~0-2SJ_}=K1mQ9(OPtMB&mNus-O9$l=RCAzX zGTY>yhHWd+DsXM-IpBeEU2y?{$*cmU8A_D_ZNTd9)&=6Gb26=acjz(A9O`b9*#0}N z4>l3JcB5}TjdERO2VNAWCR~4O-N1}1>;+R&Ww-iH7iCIp_}qhz%D6&rD?3P7VfPyM z<#rcsP2uJGswtG5T1>j4Ns^wwXV=^M6Zt8{yrbK1&;SX<2l??;!vV4Il$@pK#DS(T z9l&FAa5@s4RE?URX|8eAayxL?BCN#Dq7yD?bo-zJ?BO#Xj-2oFwd7m1pgv2b3CzRU z{l{}=uuJ``;mpw{y@=WtYNRjrISO8thtHXMBzuVf%_?l0)Vb|yWGn84LpoyLdFqyK zW#Kd7jh=BWM4ZbKi3mI}68o>Ql|H>5CS2h3(S?Gw-*-4eIPX2_N<9|QtGXVD&;yny zcuHEN?3q}laNq6jrJ-G=vt3u4%yFfqp9sT)Q;=SN?9V|jdUjDx8C=UG?w_l##=n_p zS(NqTCtcihB<`FPQV$8NZ=w$YEC1iNj;qi0W^ z5k5@1wBX*b>}83odnpGw!YnLwV1g-za$1S(!y5ID#DkaAOq??iPDTfJ=cE zH2DcR_ZO-Ug%a#aYSNhs4N#SYOdsd|Le7Vc0kYuip*eZd-Y6VXHxt*k_B((|dRS6Z z{Gl2s)#m>#jIwD*u0`i3D9woOd=;-bK+NGaCTg(#Ct!U59LS;&EMMN?=^Xe4-^u}_ zUFdsg>y=XE)bFQ>dvl~;fob*DTbWH2lLx*U;s>Dhc6MM<8YkP^h}dwVB)Z^?!)MR> z3h}{B4|^i8hI+$BE_vt^L8&d0Suv%$=ov>f&R2`V7k2rNkqO+tD9h4cIU{$UfSRYdg4h!dQRXxqT~6aE@*Ukl=a|6f;SDYG8#Ri%IbJ6u&E`K*xyCYI#W_N z4v_jjW`&F2B6Pp~tjy3}yWdTk^70S+fNPf_mW^(StJVdGpR}D7D@!^lTZ{Vh@{*7d zH^g?C)#5V6rcO`mn9{npn9=MG`GnF|RuJS->fs5I#uhk-h0MJ1K!IUHZgXFqZ2!d7 z2&_4L67!+apmLu7X3>>h#RF5|b9zbYsuV}=Jg0z9mVwr#TRos1YIM<#L>v6Hp!(^q zz{L9$VE5H|gl)%7>H)12pm<8|5b^LqS(iAhJlVh85qQ-SczYGC^r!WyHA+=DeIH5Q z7JJJ@o$5Q?A=0{d`@rH|EessUeVWx07vzVhP6Cl#h~n)ZQQjyLvUg)9v96MZmlrT! z-Vo^eO~f$-s_wp$ZOOOXB3Z>LPuj2<|M;#H(t4GHx$^P2VI*>8qP7IhQh%31CHBJ@ zupT(=ekIAwZ&ONGn~hi#+`Isfa(XFRz_5LKd6&SerxL0qk(Ey))Ih^(6r_5S~!e4cE@D9gpGa>@T2<7&4Fm7p1@ zhv~OuyVt@MZ5;W^kT=rhNgVz=ao^uxVt}f#ZJc_}gRa$4K1Ns4g_WU0(=q z)GhT(Y{7`LKBt0@jsXGq&QZm#ZWZ@WH*9|$X{!(Hcj=FopT4Z-sh5EE^nzbv`7GuR zQJSj<@|O3!b?B8%g7{oywA#{OqtiN_0w+ zxz%1Fm5a5D=0PnO=#2Kq4(iWe?f^>ObipiqmOg$r9S8lf{2Kh`fIwdF)3-C&mx%MS zK;wO(sKnSmt6st^uT%6|e)PD4HOkyK_%>3%I10@PF3MU{ye37wW8>z!>`|o~wYVY; zsO%E0TO)6mSz!7;pX?)2caZFzOxnY;C|2u!d)6(`Xa^`K+JM)QB^6T1$u$hSvw#RT z%!Gb@J*FyJOX2oB2|_uOO~BEYZGO@>@14^mFO}uy(TCsTb&=35>!5bS~nR>c&(;uZymmjx(}cE2vt+(hZ5 zbduNM+}JI9cVS(Yi=LFHEJtM|lj=aw>pw%PGc;eVoE~+_!CURH6*nEWnBZR`AtP*^ zSK^a`F2lDs%*Y*dLND&m*J;8dm&+NY7!!aY(jEjFeyFE({?gIGuOrE4haX5X$M-Ay z`?i%XUH>+v^g5^M=gm9fLDmCTM{9oYM9=#_tLsN0V)o1{q+pK=T4>NMI`7x$YG*$! zk@4Gg*cn1bun-{@(oDPn%Ee;_z8^n4 zQs}up!`rJ>0bCPStS*%7`V9q$dnA>kvPVH$H!5BmaQXxc6^DfQe8Y=OfBol&a&gPi z;c_^BhuNf`%R77nY8*-3c&)Noe5fc6Y5;jllD|ex!^7Jd;fP?GYW`$kp5hhR2tR&Z z6|Ra!cVGF;8^bG-fzU0y`t>{MfwAchK64Hh9LPbnjK2apdCYOBU-m;~B*0up-*1i9 zFC-pvrma>Gqo6r&a1JkK`XQRrRN2{{Hb*%issi7|GO_K&?Yn~E<6PQf&m%Bu@Juz% zpoB3FKvwL>fna2E8DL_T?#>7<2rx}M;2eY+5gf3HNdE#an&X4nIkzIjvRwUdsomYT z27FF~Z;mlC-QtTT%eyMyK8?_|LM_P-C1l)Rb>HeZH3P|jQlN=HfL$gviX&f$lw7Xu z_Oi9)0X{g$bqNHxB7~ol*w9uNgHYp{C!aS_jPcE;a~tC%?)t4{?j70bB#E;E60iKS zDTcB^V*C^r0zi|boO$}A_%)SxKQkZ!%2len{TO0%yG6&bv!>?qg<#XA3lv)0y$l0_ zHBt|tn!HBI3V6lZG%K8gD3);>R{YxV;VU{2B5|iL*5rQA0O#T;m81??A_h-=z1s;? zO(6>aS$hn+O3eTySdEoJA1PPr@ot#?Qglo!hPDNk-N&ku3+$>at@?yX?jzqw@bc;d7U; zIQvtqT&Z9{h~p;XBl)(IQU%q4^$*fqBq@^V%U{8_{Fzr8@_@jIz!ATHHu;S{k_a{X zdJ(sA1AP9BAdUjyV8Xo4Ff(etm2&)@kS=jM0nB>cr*-E-D>K=1y#hbC_;gx|D<5_l zkBl1|6!Y-F{abLrb>`>1sfs3!j!SxN$Y}+dIISE#^)B5|eMmb|%Ib+xr(m zYc-k4o+tdmXH30wyM`ujfFskpMligiUif|^YP8P`S*efr#gs`;3mHOaAmd#1jbylG zXJ~4ZFzWF=(U&>z*;TF%3Wt20$UN56*ly#C$sz{=Ku8Zx;tChUOI!`Mvo)-Dz|Su3 z!7XB5a}xBDib~W2^dO)gIK>#-zYOV~;n^iDeGHKKJDJ^GfzmcGMPh`lRpp%CaKO_P zX1DXB?M);A_4TTll&p7@C>jl1@O0eZch`MOlG7h4Kbi2QxP3^p7L?qhIoN6l+c9}B zKe^MMj~`*NrTbEx`qIG|%@cEbRthSnUKeP15t zafR(n`{tRg0HG_v%QBBRq3B6k9fS3>w_YOVu&F(v{BGAjhvoHA!fy3#s<~EnIF2S! zXf%l-LSd&+VlUD-Al}qKWtVlQ*JBtK)m~y$f}T(+8|db+=DdT;oo?n$B15h=)UJ>P zO&+cB9hxxCB5uD!)LSc^-2PKiLRT?2p{uuiY`pC$RAxyFBs6hm2Y*6!$@2YpR=eO) zvIlbXN)fW24N^dAptiF)74Z=`%I?RyKFFiqove6$5el;efaz*Rl8tdU9Ej=sxiwdn zM%>vpV$@uRPr=20rHRxqq9e&n&~5orv7^k#!GPrP%8t3#TH^3mqVgmh2*L@T$6s2K zexErvKxo`9wj=i@j3`0;n8l=c%^MC%9_DqGeG1i6W2+0Q^2(-%6_$n3J}l?}$@odA z#XKhyQWo&JgASpN8k(MYDxOF%|u~Juhf+XHb45#&incF2UEU%o*d4n z)?TF99viOlv+eYip^e59h(D|+i`}1aCNwTTWoR_0M)ktylbR&*^Kpp2G@c(1I5Q>y zr_sM-x}pvpMV0v*+a`$7VRa)I zzxZuR0!?BpG7t^$i~S0QM}Wj?*GMgfpZ%l20c&tKt>Ox+5wgjV6ZkqQvZLYoGD5}d zQ`X8bNCy(3X#Sy6M_z#oIsLTH>${a_u#{P@F8m5?a8LY+=8LOkuTMrkB9}$e*Z~UT z4BHnL8&L)XG`6A&i_g8cUGC##ob36n+4-#4h>tlPGsadaZ^TldW#yv5Q%xFln%53# zEE%c1(CW?ntx9U_POlq5uR1s52hj7gmZ@sPa_%bfWXeVe&1|)G(XTI#ujaprhI;up z79V~^7w!Ksf!0$MAyl@v7fsO<{d)}m3Y#bX>DOnCuscK<%rYt0^Dc7!7iBcXLD2z) z&!Bw@=L8Dqx*QjkV&0RKtrf_m+hibfFs_H=qGsf(nL1ooqXevPbdp^%QdfiFt{WPR_cA&!7Qe zp?1%eQ0UB6$!GA?&z@OL*D3H0y&hQcHfarztG;D%?s!^Ghk(|cZrfo>8+L!XT6e}} zQ0s6Vj|=W$={4xTtDctSL@6?T@_uIj8&lSEyGJ3ILEqbaNVdkTv&Bd+a3b>EHM6>_ zf7wMMa7xp~%)0LPqN2{PF2>RHj{<`?&S#sAn2F)*i_sdW46d5Pk3m_erRmudnDscM zU^;w(og(mn&;%{z`$Ut3t4E{VZVKLFK!Wp>v=_y)e@526e-i5)9p<28^Uosejyd@0 z96weC{rsMenGUru5kh0U%az=-I7sZd`qA7)v{72ZFJJ}P(wS7ol%VIUuwDElVI|zq zf!1~BlSl@|#`95r%C!cP_O1FGrDK2_SaF=W;k1r+brc@3x!g*we z<^U1I-b(8?3Vwl1H>i?vYU$fJ8relI)Up1Ky=q!G2^Z)r)K)ip=QEt;#gtP!{^EKO zQL4u#V1^Ywt~uhF30^+1VU)qti!OB;Sb2CTe)zb9vc;QzrRo902jYqpQACQ?BZ>q9 zdK&9z-0Z&_9i=@m^+gj|7!Ee@#o6Lmd$JK3b<1L50=f5x=wdgqjFwSm&{`n2>n}Al z`l}g3n<3fAV|l=Np;v(wBxoncV|Na_i6+vx`Vzy z*`PBb;;he^yzzc{6+Fn|xj|)5i{=GKM5WTd82sV@ZUNbeIf7y=8V{g#ixE)XpdEi9cS2`tX%s@IG%hA^ z9uZ(vw_J71a<#A)3e63D0*aK;V8BxhR-i9SH%%;wTGp}1wK{q)yk2Zkv8U1KQ*gS(ID*{X7Q)~`TZhy zHw*IvAn1DVo9e{9+jGQN9X_y!hsWCK?E<(}%)D@7?)5U0Zhf*mO4~hA>2om1#`IfB z`!!E$rNgItTdyTW((y*C`8Uq69i_U1?q$DEyBn^;MML!?ad;1bHOEybo8@44`??sG z{CC`;WYHcGhY#X}_@olU)h^bG{-^1EO&5sZOL3L!@Arwh4NyR56Ml@^UotE@N@?Tx zL4n*ek7^b1dOJj48g??r#gea(zVG>9Kk%!tBc1mC8bicKdu_zm_(10=RDxrzzL0K` zaEv#fZ*3?hkD&bhWvPneuVEYh8rhTE-wXbd?AI<^NU9jhH~EyMh?gzT!nRNS4(^T* zIL^fuK?H{Jh8@Wr^29_YtT(HCN{MS{2OFK6LO3Tk2^m(>)C!h0)=Lk+**8Bfjib3u zo!H;*IxUT-)o82tlM|Vi)a+?(GQ&qN5{#b@!e4GuM)tX$gdF>taV*E*-Kg29DTGPR z8Q1baLXlNd2*?JBP)+TZvY3g3%oQW{t{`0oBGx(1t7o5uLYG+Gctl%h8}`#BZ|HIu zD4^?UiSz3|pYJ}*_w{G@*T-q=vMgwArt&Y?Hp&(8?i%>%-7D?wi;+7PARoYuUl72)S3-L?v%gVOqOJStzfPvXzpwz4o%-V0;;G_53>Y>fR@VFslhG~fnv!I5v%2# ztI!7!y?ilshuWG8!cjK6myR!6gy!nU)T#=FrWt)*#}8L`GsAi3i-v8K(seOoNl;D8 z)z@NZcrF?~({3+%MYPahQmRl|hr=!kU3Z>^O!+mw>t5+EM+sxCR@eqRIjCK%=^rxa zdDAC6j1&$NtJGP;IFFB-ws`r48b=>~w_Mji(4=fjD;I1(mx;9y`83pa)zVID%x3p! zaYIx19%Q?hVEpN@+Akuu&MF-OU`xl7}23kw^76>$^LO9lg$@_TAxb=*9Q5u9;>ithA7X{O`-Hsu>BCKA56JzV~rLyps z{QAND)huuImKh{MMf7*}lH%EkVyHsut@1@(f0HuxD)@YRpvxnBE3n5<b8G! z64%)XyXWU&$Xx@gHoKGJFq%e1Dd&4P(K_~9xqo|eIGg)X3=#S+pHG8F*2jF?3#>S^jl?TZPvcq(p2}T0Qd>3)=XF>2jv|Ht8Ob?|h;0FrvE?L;&>WW^VU1 z=f$q6pLVHF=IgPH4kpY^SaN$z;7q|SaWR}_G1m_#?U`Ed{!_X!BTI2fhXE0gU`+M z_a2hT(~>dlC;IWV`B=~=aBw36XXM{DL`cR0mV0It>=EJMUYm2(cTkoUgL945ZwH(bRzedW>hRnL$T?we_ZS4Tn&%O z=wpu8%HFDX)2)i@9e&+R+xnZsh#lRMjfntpLw&_M2Fp^Wc%UG!%ZidLBGL_Rn|>_R z>{b)~%~U~!ESolY%!n-(w8q=$?cdkxsEM|*W?C?`o_W|WU4yKb(+42WfV#fKyHp3W zqFk_YH!osGTIQmgmbk6;b`utY{+Q(;S<_TckbUc1Z_WSy{5V-Gm_k(Y(O?n8FD2DE zkOCBq8Wt>u~=xlHbj?@Slc4 z3?juPh`kRPaBU{~&G}nerOcs^_3`rWR#T>bhk5KK|f#${jW9_-Jrj;j@ zBPV5jo)%gWFi-G$Z{xH&CwVdanB61IYiH@}NK1N}!i9oc>V|KEAL{axBPss=sWPnR zeh{MM{bbO@U+>i3WVK<#=P|3F6T8)trq$*G4*TcRuPyA8ekkcqy-rue{aTL8XrA4)Vx#Pl6GQA%%+(0)qrcy}!_{J?IN2_P| z2r3H)adl!aO_Tf6{e`s0-Wb7S5u{wrDr?zT0ONMK&C$HAWBAh=zxh44=C%9kU4Zi$8vT3wh From 1e2708d78fa6de4fcfd1000e05e8e14b2d45b1af Mon Sep 17 00:00:00 2001 From: LEMSTUDI0 <105062843+LEMSTUDI0@users.noreply.github.com> Date: Mon, 26 Aug 2024 12:58:15 +0200 Subject: [PATCH 058/227] Add files via upload --- DOCS/header.png | Bin 0 -> 52071 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 DOCS/header.png diff --git a/DOCS/header.png b/DOCS/header.png new file mode 100644 index 0000000000000000000000000000000000000000..3db2325af5709b89c2911ebee9c47d057ce6144b GIT binary patch literal 52071 zcmY(r1yogAxWBE^-5?;{Qqsz%yE~*+Qo6f))7{-5otqZv5Tpe*-L;W!_?GA1bMF5e z0|x^)Yt1>|S#LZ)z$!}87^ozuFJ8RBkd={8d+`Ep@5PIkamcTL|EXtR<^{gJbW)QR zd$};o;s^YIVlSiP^x_2?-rs*OUu0yxe(_=yT~nQlg=P+CRE0^jAj0mc~b;9II1EU7b~EAc(H``rDc!R9rr;~H`kgi zw$`n#=gnJrUN0{_^CzzRpL|F6cKw{lfs}v05)@3(X;@eBD&}y;zW&}?NzDtDbdbh| z7X`i&&|=IFc-Ie_u&Es#hdbGOFe54i@QXo?WJ~+#rPEtfC(7dCP7u=ze7ApE!KNi1%FeZ!)nvR!9Xv;m zXlMApr$jn;^dD2i=8j%3!I&6f|MM;xwL}obQU$3|GwKA|@A?1tgXuC1?Us&Y5ixlr4_ zOr!$E`Des&MU>ix&C5t)!)M+P8s*{h{|#7HG$_TolTd8O`c0&nsh-;D6mjZ3wv_G4;IF4Cq}@ z!^8OR{fu(jpH_~)j_bV+E53XAUvE?z_r_)|X-&qWHelqhET0~pUob~*?W^y=Fg_gM zw0h`tIl=vB9CV!@=}M14AKiO`U&3Hi!L{h5q%_hiRH+lgeC&y!mb5*_FVX5x_bep; z_e}pt?eSgO8PU$iBYQWb-?u-R&u$Z^wi~uoX~CWnIRaXQxVQcM|CvpsY$?j_cg92p zMpHH>7%K`KO+R=gzpI>(IiEHJI!ly}Vw{?n{&(JAiJeS}h0MEWxq%~Sq97$&$D;IV(VjdGO9Urw7T@U(i(>_EEk2#ZCzBP*G(@XHuP9QUdf9rG|+W3q` zC$KZ8NzXPu$eRD}M)sv=VV*W`Y?UF7LA#V8{?fvsJt5!y$dU~rrywE`#CDQPGzq1%FJe(4_&o!=5D-*8Nd9fAW@wwG~P5Hxk&D{8ZjXf2A zByK)`p$>bM|DJQ!Hu3p4?Nf(b4EXAKK=pq&Ft(psJI;JrHj`Cz99mM{((sePR5Q=U zvYv0GmD~T#EJOAkn&#o3uhSt}kX%9fst1h@mt^=Ki^PD5WED-2jpe(>B)3ee>b78t zjtAiiGL9#B!1;>%#s1B-aZ$M#4_lGi2zIDIn{<}}Be^i`J>5AC!QW@;l4=HA$i9rk z?MpIwz=9uc(eDdolKvj2wEIJG;C-$EjYbv*WGY%%9f!#h$$hg+JuTuNJz$U0^88Vg zwY?B%aMj&EOdU#kn)4^5mic4M$(3xW;s}CvNmUynk1grLi;AU%XIZbJlsvH4vEYeq z_*%q5&bT-ZsPKNdEd%yUAbI`1|KaT0MbCX?=_1x6L2at5j2xvpp=-}I;qjT#T%=Xk zcP+9&O5Xu07=j^ceiA1ZETLw1N$FWy#vdduCIgL3*>M*x#-kfqgF7k_3k+Xa{YdTj z3&hJ?jqi7dxUE3N!ko!mv)blIeG)$eVxmE3N(Otip(`wmc&E$h!vd|_F0Aj+NA@rW za|_558V0x3rMIS~x9?lj2WCa%W~3*ltW!lxpts^uP?Uu>DJVp@z8~S?8_+UQRZG$6 zS}wm{N;(V!&d&k6&zPQtn}|h3 zb^6JV(m&#E0{5$u|Ei{AQ~DI9$n6d~M*u6%|9q zcQXRET41!+a&XHM^5XW~d$XDQxw0@uHdIE~TeUdI?#M*fQm+P#z)0ZJlJ4p5iA3h? z2`GdTIXeRbQbHz92L^@TrzBrRe>35bl~e!U_hBTa6Z6dLGkz`waX+R`w4B@`v@hib zZB$bihkI?h{s5h*E*9sUuaWDV-s89rO{`Q!f@SW(Ef%(hRj_B z>hrRn3`Gn+;Om@h*$oLrPEe;;WEMZmF_f`5iSnrEWxs*spGdEdgw4B*>ceMG9_vs0 z-1j|#zhflh{MAcQu#S(%)~Tpq^W#el*Qp8CbNK($#keHoi1r9dqHBfpg;TVj>kE%s z*W#0cc2UPOI7e>Mao?wkDX!(n)E6(xji6wKE(42uV^Y}+m@8ucGT+~|N*4%>Car%t zA%&Q0HK8W%UJP_aeEu$>uRo3jr*cXA+|N3`>$VS8>LY<2JpQ}&NWNGRJ@XE!bg}ZR z&#<|@B@sp{ZUC)vP{=haTSLah-2>k%uZELqBbImHg(NY;f=bgzwKys+v5rzOAL(*$ zm3>K)^Gd%t(^ZcgdjAYfI_hrcm0x9(u#%4R$76??g(^M@!mv)jcwS4ng7R%Nk`C9I zklnFSv+;@*-wSoe&9iekIz`B3%Pff@TT1eED(x{MMkaTwvCI9yBl_aE~c;TqEwYE~>F6wA{kQTV} ze8F#)8HBwu%I1alIo}ln*A;5RXPpIwbRE%8c_|@&Znkmi(k7eSHaZP;3r^+A`wbSR zgHT_)>IdtOJMg(pOT@~dB_ZbK4ci)iOx`r!H7n9<3a!)^! zk|(m`wx4sUAk%W1YZ0m2oMF9gbq;k~>pqT$BmwQ1uEV`F1!roqn9W(5AKIi!S-(DnEU^|;aO+~}clfezx{PblA@&#~!4?OTfow`U#; z2Rbj&XDRg2j?WHL!RII;E~tc?a@t#KJ;O=q3$3b-IchB%nC-f8T5{i4Ryq#$x5wjm zJ%)GW6wsZ1-Vf`HYHuPca2`Hho_+A>e|5SjS@`pZ?4AInC}GCWZ~Yjf%lfA8RqZ<+ zi9R(*F+P*Ndx`k2J6#W*KT2c8vUg_%( zj42GfjpQIBdfH?pdB2_}1^lZtBXV;Pge`gCs;Yl1X!I%~dW%M5XLHEDgOtCO`byC-LGW zcV#*CPoq{vPxmDzR*Ko5tsglji`2&t={e;X60R2lmpQ$v*EYlq?MvCM%&a=pGe!c9 zL6@pUnFsbm&Q9qgC>45Trbvz8NFOO3VqL44(>!{E!B1@2@z>U=YjR*%MUJv>t;l1A zF2uUp=cEQVt9McHCbSA+p1H{r4A~sqon^1Awe})8=DmJo^Wl(x>{rQ-O%q?(6#BWj zb|JO(iL{BpufnCi>Kr6$=2<8ND6Zy~Qz;b370c4ymx{L01ZE+wor;&bjQ#b8y#)y5-z;5jD_q=bT{kg)s6VXwZaWCw#bzeH zl3e|bkW^|dLTcrVA7JNwZ}74U=OG7`J2tu0I&!P(vp92|j&%MPmt|J2kx`$WJ)y6M zy2HJ^1l&cdp3?4{HY<50{>>Okydd8sa;VdJu$t3+jxf{2>w=l&Thuxm&y2h;j#_V2 z)}l_ptpe(=^C@BWg?M0=T)Ss0=>*J7Q3ostM4XK4Lfk1Q5N2n@c-fU>*TQT{q+ZD~ zU!FUhsN6sBPF|>y>{bMF+H$cxT$>ey2_3Tt4v-u=N?$l9#u#5b-11=8k z9fJkA#k0iaNl2o#G`f}I?|^u2j^$c?Xvbn1$=K_Gz%F|fm&0F1Qyb-$sxr9&HuKW{ zBP+d*C=~}aI}5Qeu0z61X$TKzFfZ(_YUhS zX;T!;hTM!UTrux%0Y|z0%36TsicYNxTL=yxTjGQew`Ui6pWp$`v{w{UshQNrnm#g% zNlA;T95oVeq3JHa)O&i8w7J^!gG|&6SCw)9sn$mbbIttjDuozDg#AXIfL-O!Z&fa& zmi$iop>yn)HNW_67G^a{!KGOBcoyuenZ|tYx#1lA8qyQ&gU%`Mh{%IO5|3@$N#BU< zPqwwIY_hV)Adnz_OBJYbz9?IW^V=1qQa4lPYxDYcvHYc7R9r(DYL3~|1f!p^D`xXk zfvQsjQ{&GVuOn_vG(r3IiWmR-d&K?aD6J3Tz_~^VrE9K2qh|T=HSysxHb3Lpcy^xr z=ciPNY_8e?)=l-9X6NhoXDk_Gq{iLeb25S+0bg&)i?=R=0jG%Ftb+LmVRi^T*^J~v zx1U_DBHw|dmtXDvB_jS`FD2^mv&Kt*m!{+5Cw?1Hz6CL+0>1(tu_-#$m|NsgJMSSs zwkA}xD$hftisz9H7WSO-wri){5DvI;hJC8EdXX5N-h)0R$l7UX3yKtGT2_PbE;cX~ zFQ$^c-$WslI>`Bb%V|FZ->01PxBY}aXveYZCxg{_d}GPlxdXL}Mep-X7Ik_&Eij34 zjUAh7?qy3^tjF$~1wDJViwcern$4lge-he5-N{zZ>ihd5+T2P<02b9(_b3D)XSCZuTHZwmuMR07Xs|J^mD(l9Ah34*C z_6ZmS^5II;d@>h*iEbHi-oj;CaeGK8xn7|*+5ff~^Ph9@>|#ty3lT1=+Y9zkII`*V z)oIztZTww%@H+-f54ZTyz*r2pb3-RMy{7b}OqV$w4t3jT*z~2rn(w}!Y{20BPLfg% zGKt@e2Z!uf^K)PcVQ+1$)JFem%RD)UCuSt!26sa@4EVaNlv~4s)V-oPJ9RVF!lRG# zmkR9T=mH=2K#|Qcp}nUj5)sVwTn#sa+^>#Vs_(|0mXv*C=#IJOG>S`3wy#e8wr@B0 zlACiN_x?-O5!x$jUa_Zw2Dl&ZA>;>y5IPL5&-rwz?Z2r6*V-mte~B)a#?A(*k#I>?9X& zBis8r_>Dj$z4)R|(`1Av2kb3y9op&B|1m&_tVf7&G?Qa}ep7LuQa*-}UVYDMpHpL$ zVrNJ{sf=R%#ze_(jRuc`{w4MHLzAs~m$;FOr0uUs9*YXqn&(0p&PKRSOp94f_j>~u z7fzczjPa;*?J{t_(3~&t9m+8?-lB$?=Oc5JpDq6T(~?EvGNxHDf{e zvi55+g!n_z0Z|7oNORUh-R5cBWaG2x&nAP`P9M(eYvvql$iet-od*0)3d0TSx2`eA zsqDFhHAbaOtCQQV(O`P>SiJX%K!7)hl}yZ^cdT8GSVQvSxQ}EM{ZOKnFF^_#Ae)%xB>bFU$8u)(96wL^)jWG5xRXgTB5VWmI<)XtsPuQryKx-Jasyl0 z(J@_Hj_^3ZwH}^__rsj%A$&VPe394Q6&<|s?NH^e*9%Co9Qj|&*CZ6)Bcdk0wfm!K zb^di)T`P3p1viQYnkrO0AP(D2hj^>g-chYo0F}z&dK6wkB?;?9f?YJK|M3TtNAG-r z6=yrJaoRDsh)rldZ&;lwe(N{Ll-ONz`t`U{sOVY89fOZo5n}5F?KY4>>8REXJK_9Q z#5XV=A3t!*JOiEXYP1L&)p2@An|Je3L%_X@AwSy4u~pdlUIw(cTh%F8>Hznzn9FERSuX&KO>-M$v~t zV-h}rBCz`|yiyyEaU+{ay!g&;IZ|zKMH;h3W5+_4-P~lE6qZVCdy99=@27jIwuHdX z6Az2HIoW~_Esb80&Kq$G*+`_i&OU>g<<*-cnJ@|Vf#u|OzTp)zT$$2M?O$ZO;W!ih zow^2HgUBOHoXF}PRcxyU=kW}&LvdF*C4~+MY`;hw@sGXh4GTwUa$gak$T zOo5B#9I?lg3+bJk?z02BD-a*<(Z>)={q~iT@L*!Qb)2VT28Zd2T>^}x+ywJKK<79MP?3zEHri0K!TgYKhWS$D zN#vzLsz_wxfL;<$4{pj&7xgh>^iu_#4H%XE@b@-OOGAExc1q=gZ(Vg5Vy1vAvgNC# zIaq;OW2R&?r)n_=c6<^hF@SpEd zK4|Upd7R%Me0=BG$$z%)!&zBq+nUKM*x(LVOjxRQ>nRAER37r5gTW!?7Zj8E{g9*6k(1EhPcn#pECM!D%~#yo_@vN6#P)@gi=sxc5F;{79iCz2_S;IOpe|ul z3mNE-*sHQ7VK@Dp4@dpO8vSgb+NCrs-9BRFY`E`qb5i~mVN?g;c82q*dr9be%Im3d z%XjOMhQe*lA-qccs2=Z@`-qRCc~)MT=vwV_3T_pr2F7#LVg9UNrO`RIndd8hcX>{ZJ92T!`Gb zp`W@3_Hs_bHQZiGs+Or$ZFsrbSh(iFc7z{ck?O@e&{O`8W$!4L`ls72qYYEJW?S^r zJAV{JsGB%T86I3hyr!;)_bBEc945?tM$%Is?TPzQFrQgj&(;VM45tw=DU<@wwJl>I zZ=0xHJ}w>5x!U|0don=Tm?{&zVyUuTm1K*=Qh*{p4!)n<5^Aid`XrX0o>n_4V; zt$d6cgX(M)+dDP+YVU(=oMSn=z$K52%e--IjH3lM!HZAiWr$Ub4Y93VZ6+7uro2_S ze)<86IdWwS{#J{VKt{!&o=cGAUU(^)1IU@rz!b_N#>uV=h>(gt4~Jdgsw&$@Yl_#{ z&yS++n#@S=d@-HX^wnoM7j?Q6ESlf6eS^O)!M<{O=y4_3%}Bwn8%nB(=ELhdyC%U- z7A-eYUX9}TB4X`96V!Gwj=@=yso4&-HNI=|=EzYa6~|hHXNF*w@hE9+!Y!KMYdFMq z$V)l9K7Z+B-+%$J^=4Y}6631l1%+u>GSRpFk2`+%+p!Gpv@p+d$yZ65h)QQc?S|G@ z^B4u9C+zxY7NxV+-&5Te21!f&Q~whLM|a%$dzZn;M8-Zs5@v@AxkY_pep zvadZcQ1R3%J+`}|S<3)oUn^exGs3feF4(G8=%)(8gXClTSL@s)c7GySlN7EPu!fxm zA$AA%DIwFc{qr}zc#ppzE(jTi)$zWbsc9sUAw?<`b6-|HR`KJfbCb<#Uz#?uyqz*t zUi_4fjeET{vboYCnblL|=7CzTE?lY+XItg=oz}_YCpOy+^$q{5n4klnOb86sh2Cj^ zZmnL;rHlQz_$7}{bPDt~Szu&w8z?hX-EsF!=Uv*pIFOfi#68H9lKvr4f#CZ=kN9wS z&!lSdDq^25@FIPW(bSS*o^GbOIP6^OjxM&nrxulArrKGvn!k38y`^Zf1n-Y`m#3vY z4&H?aG(i7$fxNdfhe<0$3ksaljh(d`9RzMtyP>hZ1;MB95jA@+v(H)SVr+-QuHf2z zPDtj-w4Jh}j>+S){8jo~$TG^RMQi$&c_P1&=TT{ZGFhm`%O2zAMw*>GH9c%1^N=PopQ>uqBvPJbimqQ+rt~rZZycVrbX9Nz%wec1w z;}V9@ZS4?*V}^RhpQL;evg6;*2%Ij$WB&5Yr#HPOKJG8L%Juk&0UU{!W}%M#cn(NH zhVsO&`?oXOGv8_H_kiF$?g{UwXTz0e%&s7eTq^ON5>_*};T2ty#cZC#c;Jte+aSUZ zl+v=BvV_!3#q5}Qw2i;GHxREr1@(DWXm4~m*7=mSzh_qNBlB<0;(eXXZ@KD(`vHZS zK=TW6C%K#QmsjygViHQttMKYiKdj$QATx2uoBhUw$xiV}GQxmzz@;Y?V+Y%1`{$1L zy-<=Pbip{g^abb=ZG@_9=gq7*J5vs?9kk|r+5E9sg~ZC0VTPDDp9tL=B=-a6c(uJ> z*b2|SQ2?$cFE^P{A=Gfl;dY3#CzT6+Iqx|bYj1l|&cvub<)}5+++LY>(L7xi*CDqR znqa^M52bV{Wx!`1jlLVQ!|UQ#F@>KxbdYvXRy6rJ;QP*yQ6oxucBMMJug$di_)rB3 z2N%(fiY1MzX6*Pn)m@?!pO3t2aSmj+C}j6(<4c3Kg4LdI!6Cy|Vd0HNJK-7YuF-*2evuqJ7FoYM6b8 z{mkOFGI>0Z0v>!$Zt}uZ{9^IiN_fMgOku4rdon%n#p|Wsw~NUhB^sKXEWv17dW16C z==214U+8^aQNjTU=u6k~T^aRR{-kri?W~Slul);E*~|gAF>j}MwF7F#R!?Zy1ZOIX zGE;bBwegpQ-VG2E3_p8Gum6O&ma0N6$kO-vC zpo=JX#PnFzA{fK?^Ue5=D&PI6N({!SQDZOr*hUjXp95<8p}tO4z5MNae7KVF_2O6L z-0xXu+Q1p3vxx#Q^1uyp>Weo_K>3@mRQBh_IQGss##b>rFSQa5Ne8U24&4sv3j3v* zd_?;8Gtx<|VgZ{-+qZ1I<*8FR2Mw}5E>$6uxxaQVR6WLJ54BU{LKaR zeT`9ZZZ`o1RNj8-yT*!Tk9y+pdlWN=($H&bU8bj-Xjls9l5f`bO~=Og9y+f-VUq>J z;s!T|4yC5^o>j$C5_vjeF?5&9`uW&xwO_OB^}O4UbJbGmr+6YIIs3?iZQe(dgyS;b z5lWk1WQpDz)u|8CDalIdRzxnb)Y?L>XSniEu6l=Lh*d+;h+u`=nKpaf$<3q>0|Ne+ z3K4byZ=e#pUwN>)ym2O0ySzLm@S#Inw_T&dEz+0E1pFz}6=k5&Ppx4nC@f*Uq1{LoRks@Htq*$xBBb}JU`o6sK(D87{jyAQG+ zSIC*J{99w|M?)Xjz2}p+4jhmolGmjsOC~>U+%K<3e8=GVdk=-9m|fOEICGoTv>_f40mD|K0t zK`QMX$|;n6Hgb3cxt{PmL$Ozp6VlZISL3w6T=m2NyYkPm80e;WQ(GlVjQ}ELgJ2`l zk(h_;rfjGw=pljFqR#L|PQdfSGG35V{X2IPAw%&~V|w2T8i>jGX|{cDYRyRZwB#=- zUyzKHgZV?!AJoNMdt_CtTnfiI>!Tz(vmlc5uSrubl25sl6sY{Ohw&i3q)9T4$)t$_ zNEk9PXPoFXYlwW2uY>SSn9nc^O8@e6a3!Su*x@?D7-QajczJ}D81(!uPlei|tbqwPn8P1D}{uiVJ5hJ)$V z@0PM&@{--BA8=3lr1X5DIl+Yl^Z?A)B3%t6a|~Tz9PHgxoIMA}`~HBYVaaROf7E^w=9*#qdyz0?D4@Q zR2F18CtikPpMIlL=M}}BE=aN|^75cir`KAWz04>Vm_p1Gtc`CSR7J9>6?miXYqNE=a%dsN1OY| zXS{;Pz^fsTgkFF0NGI{hN6@V>8n0J3XTacfKw{5&?hIKR zcyaYW?ISpWr9+7^^m^dOW(l9x3LfT3a@VbdGkSwVu?B+eP?*_x!^T_w2%t+1o zPtZQ5unXM*4nP+zHtFDZmfu~LSg01;&Ev+44{M1&Cf1!=y&SFv*p(#(nauIROA08> zDmza8V_}Qf0ihcMW_DG*r?jTex;3bzbUS}jAUbaaSqzDbTM#?m_6S>~0Tdt~zzkB%#mUTsej1cL` z5Wm*G`WklEvGZdF&s=mh)xw%{XRGwXbWSD3ok#WRUlmvEwXMdRkM+ zLjr(sG^juLD{5Bsh$s4IbV)_BZ~$S~6GUvr%L}|g&X3R9?@Vqnp13$z`l|cc#ON}l zi0?VbPZTiM3>B=~7JJ%MWk3(%qfM@sAm--cF|XR;A41poMx zRXRIeCb2|K_BvVY2+-^Gi{d2KEKqeHhgUs<-^=2W(V9P|74{1*p%MK4u4g@hQxtkd z?R?9^v%f-tQhb-0cmTfy5=n}eZ+vsLYR+nXaw`xsAbmKiO|CdEe(NYH@3iRfP>}t> zOaJLwnJzIwyJ0{?ugzJqc-9~fS=#0`v|Uk&A~|1|BZmsDU(k6dL(u~ci=euNJy7k< z;4=Y{GJ#JVF$FS}1C~xy|MIPsgc7ftFRuV7lb6ivvEKU;cXBjh4{o2UwiS)>d8)|N zPCv!YfTq0OA5k#yEgn9Dx*1U7@wn0;UUl8>$l}Iw`t{0&)Y1_}kfafJ@gi^LkbL|Z zmsUg)0i!kijrH8o$exsrnk{1OaAc`zkp7b>+Uw&$o}{^%jq)pTKe`FY;q=s>Ihl&7 z0gAaVVH|Vu4g|VuSTIhNz~$~^7Zf->|5v|Gv8uCqQX~Ywz*eYp!u}wT+DpiRlTt_1 zV|}iIq21!t>#KqZL%yOteOpZ5dxj@cB0~nylU$Swhki1bH)u%X1`SI9_1ff`kxbU| zB`6BbC~$n&7%D_q5aVCZ#2RX)^B9G9S1SL|@r;jyU$k!Mt~**}slFd6%*0S|TTCT9 zaDQ)k&6~i@y-(S=S81ylP=;}gy;!9NFN*$kUcT&!@MO?EEKj05t?Oy1&5a=u{*e<1 zDk<>YXo(%h$F|eL4r6~Nhzbp!X|^goD$X+~cM?G4-|2AR#H<9{@2| zegu%!VMzedl$*r^mD1SX*nOSc`cD5w;$uqsRWY5mkTPiCVh2YY8{WYbpmO#jSVoKQ zl-$ftiy$=jaOA8(fOid$>In#)l_V>U_ax~v^Ojk0NU{Pob30zqXWZN4Kx`F74ux;Sl2%JzdSO^u9WJ?xdUY>cur@KBUxF1sZRXZ{ksXsFEmp5y_AxOy$SFOn!+y zScFKJVru4VGrXho=B<9rX{XKPoiPExvu6QW1c@2ZyD0#kvvkSd2U3cbE6tEjUHN?!&m-ewoK z^;z9EV_fnThF>vv#%KdgG{_qWpLJwXpGu}Q-4j2|hP5WLu11n}C%z*2$7Cp@BWk72 zf(|NO?j+OBl(N#H3fA#c*7;xb*Kzer#;ot6+RN3mdO+XJbtRJG1af@ zCLC;WX3_t0-S83fhYit7>Gj)kmy?`*w#F%AwPZ`mMgA_%{~^o))JCFbE7(r?8@@xB zZvg(!cC||h7mg6<|G%<^fz!)kiW*NFvIEFfO|AUzq%5p9yvL9g3#qX$p6F*L3y}ZaT+r*V|Wk;r~xd z-=}U2s%^Q{Ke|_(I57O_Ud}vTWlx5~`F{qJV(Id`giPVXS;-x_o4){rM(KcAF!JiT zPr?t20t&~Qmp>0k->I)FKA(?8R5av6soPw8x2|6O)6L-8C@tf!nr@rg|X2_U|q za+-kQ-@&voUrbw}w+qhu3rN?)3&v-gNVKij$bcf_l&q(Z`-HY*#Xn5Q(}>TdtiWwy3f_c$TR-H;3qTq8qYrm9LCh*dp591R`j=ZR1>R2yJ|?kSKsy{6{|%z2`HghdsNCrSwNFULqU) z(b-7H%J%(`r3-A5#)6L-JTb(#IhVtQGadtKg!VoFk%D7!(B;tt`TU{y53@#A!^ask zW8<`ML{Pr^e->jx!Ub{qI9xF5tb0$=T|e23&$ZTRSg~3ZNcc1h$dZQ~vhL%t!Z5$1 z`06m&>Zd=%695DT1>5v{wBCmcMU^8%N`U$68Wo9{`$=L8@*0aS69t)%3AmJksK&IN zj=kJIxp~#9G-0k92g?$?L?ICbA{QkzhuqU-I%)FDLI6Y+=nwLD{~vS1k6cNeIJ#tH zaF#wXX(WHpI9a8Sq+hTzIA6k^#m05(zJRnlZg+IpCp@Z}IL^TsVM+ksB7&V}mK_{X zbGrWFcYDiFdGC~+ux$C|vBXF*Kt6l(Yx~Ywn@y0XVm&b(FVXEMd;X)pUHO5egwRwsE}gMD-W+%`&zpQ-%3F6 zWSSA+XpWzbP0Dsuq1kzmFqZ!rL*$SC^yW@QNiiB-#NQ1D51-RtKlWi|0)WW;p~J}> zWent79<%1IUI)Y0c+Izy6(tN~+1{k;AtIK6P*YPbQ?m~_9(QTLN}Fd(z(Xqj{aU$Y z)6KzLr$hv&TEc^k8tLdYocYm$`q>tT({+jWS)BL1=NC z0Ma0(Hwm#6NrE2nNv|Ek3Yn|Z6Y#=4B?N@IfA1!2w~F<@hqw}RF^9ciZ7GdSzjp(a z5vCJ1SVG98lWg7YJCcQ2n`s-{?{D}mq<+jVRBnByb=nQ}hCzsoOvdEtJ|ylquR!Dq z3ZZu05+5#~ywb+YWSEp>EJn4yB{mnSgZV~wjV)J~B$4`}^92C_p{_P8`(VXN8Z(`QQS2{l}7tAzs_TdH2ZDu2g&bXWoT&@jIV}E%0nB zbz0hDbcn9qdFutK@b_7?v+>;GZZ|o{A9%NxtX*1#yT^`y=Cc}|Ky$qnQb&=xQ8; z%6K>AH)flzVom&6hTij?OD5M^`vBs@9k%?_o6TO{o>LS{JTNHGC=_BN7ucN{i1h%1ZtOe33Bb^)jf_H$1L}1o; zJ4;sx{i-Le>!eSWD?2EZ*Rr%MA!MTwpEiD>Ug}|=RN3poicjkZ2uYOnW?O0!C(M6~ z_GQa|S{2z5cz`4Va-%*W6y3K%kzQRtMw*y)7pJX|T6&r(`dGh723;s4ZWtERubv48 zoUajV{QPFJF0s@p5&Si|>IX(nNmq7`0DaJk3zw#7ZXNw^lT>&;xxexEnM5Wo7lgr0Z)(Icr1$IrA`X7VR<8B-H;ML{b08PeJh-))O$acG@D zfPS(FIU@D5t{Wc#TTj>kvgS^=&(u8H!#fFr{$-GZIXWG#v(q!hMHin+R<*c7N~56< zfpH7RUl#h7A>i>=8`@xI-1GfY=8CP&>d91ntLx_Q_aF7Of>IfBntwbu3(6EsQH8c{ zc!>M6O@XR;h}#Mgd`-6zO5UAwg#o*>gfbNCkxsK>UGlfljQBPJRGob_;C%obkoS;W zs?7K><_d$@Hoz^qN6wY5f8E{6R6g>v<~Rt*X@VxOY6sSM;md})x;qpnXh2;&q4}&y zf~iDH*ISx!fQD=X{vdw*F$BQVZsXFxU~o(VkwUG}2j#84&O!_-2W6A8C_HSkcd_AF zNs0k@{=^okz;-#|PZEl3*ZZ_8lx>ET?!&Sun@S*(6z{kFDq9u!^VUcGo>M$3vovc<6)s4A=n}O%NsmX-gZmXYFFMRHWk2GQb(MI6&t(^fHiAX7A4# zvqHmoNuZ=SUCUVU^v1^5jlv-b8``A4W#xQMe>&Fl1?3C(Kc|9Kd^KGv*gbt?w{v30 zQ@-}G5BH@PyS)437^aWt>A&Ym|7cn|eo|dbra6gIHK>Z2XMV!HPB}q~UQ|m$&_4zM z`}OW^xb|exf?r!@9}O1j0FQJ=G~G#R5zyT*8{cCo*zr#L;1`m|!}$$yOvtodtkuZP z;dh4q1TOP$LJbU&5YkAVtl~`Vq3F@G%`n<$dHn$D%AP#K`S?_F+tuM@r}|z5pm2io z!MPh4hQ}S{DrmWBo%ws`CL|4-O99f@_ zi(Psi6bd;hJo$q6_~nvi_4#FL`h5D{H1ICx2OSHC>&?kbWKGZcHD61ILsW$MAmP`!YvF#I3|C{5n;2k6rYL!@qM`TAAxBI zDeK zw*1ypJ5ElcFndtJhq;J!R;s(#010vr@co)P@x~J5CZ_EdnY=CKBe8#kq~kDa6*sEA zl9!d0X2I!Gdwj)nx9^G}(&m$0`8cH%{U}4?@E3FuOnuT7uK}flDY)Lj@VFYq8~ykJ z6x6~ou;v}M8-$8Tsh(_E`oo(bsmb~CBQM`mjsbo^EZ|n`+2ERamLiqO?{kHE_Zmxk z;HuV=-FfwteiovqV|Le~hSiCT`m*X z8ps%_f_Kkrs`SeVO`j(&eZE)XmY(8gH*Nv?qtL?`p=m8)~3;kI%l&jZ2mZVe1 zXfqPiV^gJylKtHrD1u7uVN|7$rz zWnX*lXPbJVs}87#Y&?-MbXBWQ{zpl{FNqN8fJdmt*2bg6p`ne!3k=&8ae)_v_3lH% zVtf6jw8JIt>Lz!dtd;^7D4ADPZ5tK@*aAF^qHlKh$Q{XEv z=kvVDw^8y;%4K1(Tw>YN<`fjQzazBH<)WKq3Pc5dil0=wz+Nk2ay3=1w^8}c=XcA; zm~tt*p%ZFeq~4YT&v@V@_jFQ@OlnsK%#rhfHXNowZvKJW+C(WR00d4dC81pphmUKu za0OlEnw0E3YcXZ#K$dI(;!Udj*X4`{a03!GK$U5V2M;lBu93beA5Hq4G2r3p^Bfpd z(VejcIBmCWMTdG5DN>VIAc*S8H2pp1Z3@UcOWH-1J_6ch2?X|ccD&oG);)vWl}e8= zrPq$T8I7MhoS$QAWV2MEK6uwTg$kNHUoa{^ep>B3_Gl7x4baJ?cmzOA1!Y^2cs2d~ zh=YlztA@sdX?U}pOvqtzDfB7G!ezchg?ehS+y_CU-EDf8oh?rgu8QOs2@h{*T@E)K zltg1>n_npvtu|y%U}2(Tz4b z38J@X5iNQz(YxrqcLp({GrG~ehv)t0zAm4}nK{3H)?RzK8YeUmP|Ah`>x)lo%lJi>5j1qh2VW64$+Pe;nEv1rs_&mrHvb zS5j}jmRt{oO_8$N@zqIPI{zT=Tzn%sV*v=pN+BN8IsLV(oHB3Vqww@&mbGHyJZ{K0 zkT7UZuD{23nclnhysPK5(;KKG4l=W&G!Qbd?#u5haNG(Pu>F`RDXF^pQ*R-2tq6=v zTL8K;3e1->{cU=Bw9F&3%;MqGG%3#IStDmSl9w{_5gW4F_WFGW40^6azc1*cM`5QE z7-9y{cLFrF{JGxb8CGOz1frr)A~SrAh1#&cK?v^)8`O|S^2+`IGouY(r6|SM%L5+A z&)IJ!)4>ta;6K2G(0q}{SoP4d@4}MwdMH4OgW!bpxx?qcdo&e?&3w-N-}S-;`sHTxvu@A(f^~lu3N7uEQ(Sd2 zB*1+90m*LV+668vXS|L%mFa@B3@lE5VRDC1#Ves)o@BL7Z+V^~Yjd%5naAK6P!Whn zmWiNoA3TEH?NgC(bco!TLqp!M57v~JR7Q)gQ#&s9cgOOWx#SAXR>1Pr=o#l#lTM6K{d z{?nPm0ujV`4-Hr=4q$>iY3;rLi!T~~8en&^?R32}tcwCGNrIp8pK(rEW}ey4+Rk3+ zM4S&JPg*ST!SL-)4}UoGM;=y83fHe2sy;;GFIB%!ye(;b{%-p?fh)rZj^Xa6nZ~2o zloR`A!SB9>r*&B0ZCDNCOIHi)w5&SIMWnK@&S~Ku?u;9ezfuL_v}%2-xtFc+B#H&m z7S95*`xqWQb%v#>xYz!2-3bC?bB2z1n_~ zx%7=a{N_78{SmWH2vr5~lHJOj_9wl9Tv-i~Ax6|eQgaXlY7gkRoskTOs)1lFroKaS z@MidBLcC*M#GYNV_e~)B)4aMJ+v_Y9Zd()k3E4xbM6csX`~iZ>U9zj^g^BvzjoT%4 zVhb7|qrEk!BlQPT22WHAKlkQYF5|n!JQ77RY*9t>n3g-y2ABBWC3++J5Kou~l*+Nf zyZZ-*sA1Ja^(GBny-S1_@2pjH{sIygr*Rt2>f5y9x!I_eEUzwj$#`f~1fJJJS?;#U z=lx5!_T!(XCclT%CAZ*nK$Y2-D%>0eivr_LQm4ZT=&f+J%Yb-cCQ{kdT4U7D;41>o zy(`!HEewPn^i-9q^4u8Gz}kwD13Ockycy-WKV{ ziRAg_he9e3jAkkwokpmsIltFO$23!!Cj$gLK6KsP`7_Lj=RBVnX@)oUQ^!>y6D=OB z@KLKD$Ekt*cRPF5rt5D>suwePtdcS=^hQHLRVo%`Kj{=4*Pv+b>&mAy5dpcHd^~2Q z%uF~^zeb!j4?aMp2Dua6l8$n4ue8bx9dU(%@7eg5)Of7JHM@*)%;bz{i=SI-!i2q< z1W#x;UWbB?k0Eh7AF#hO&&?L&*t8jwPA%!xdx)?TTKS_135jqrj{sGOkA0qYm;IpU z5&$h^--NlgHnVuk`KTm`y|eI@HSJPVp0`MdfrG>GbD**wifH^Byz@oU6^OGY zB1O-j+o$Ogu`UChi@LbZL)&ID7O;4N;rLZDl;RI9#AW<0K&6Purxo@w(;0KX(B+#6 zzDAucK>WH?&QAbAuQNfxDUL2C;Wjg^u*2$@x(2>M5V z62g{VFiA&F*;H_=M_N8{&I1Q0B6=AvUxWyp*#^@(d@6D+#^53(}g#Js0ZQ1sEF{`&_twfxaA5#il67G#Y#1P-kNG%I>f?B+l z4@eSCKb4CAHPueIr#m*w2o@OBQ%FF|gIlfAk7OEruP(rcU&c{Y0=v=$bLFRG_GZ`) zIe5Gh_zvAu^94_yx^ri+JM0u5&n~3X$n3ly{G>Ko?MnV@95g|(-yod89c1H`jwAysRzSa24)?D898Lyrp z9BeT9)ckPeF$AOzxavzE?4MpzJCLfaDtwUHOk-20T(tdAu?4zU~}Dha;?h@F|$ zG{Vx9*tV}^i=`fZcbLg5LO#DAVUI@jEJTuL2bIO$mm1p9%ic*P50zWlw$QO4hi{PH zNEz_XNn&=+kR%KIZbjP08ro5_K>NcCZ6^S{77qBe1u8N z)umSzz7Mw(m~|e{Tfb(jU*_H2{^R;V!{ID>=p3MCQ_33gPow3x>?e|6R;%dJ3wl9T(05dEew;v3#=59Gb?<=@T@sV}L^BIU#nBF?s6#nWJY)PhIRh2=24JuN*epn)+>FniwL)6lK0T28U!Il6 zn3G*J3{BSesbSf^!2PHI6whE&hK>*mO$YQ}gt2ja%2>?L~z(v=V;w2};C|V?3OJd<=0qfZG41c?Y(g>QXQ@!50tMB&4g*-E3bQ%Jo z1aY6$I~u$zAI93wRDFK#FciHBX@$rcaL@Q0R!ISqC<_Rl1yq1R$2SoO z@2gP+kYfQvCToQblfAJEGrhi7IjJNxBO{zPmFCE|r%y#&+T$<{gqZd(Brctj|EA2p zVTZvVmFm=M){V!z+rkbu>HeOmTukCzEDO`r%3LP>B!K)Jcr!urgE`+p?1iQMyIxZ) zJm!7{E%A^96@}UCAba5Z@P)6oPwFPBaF@HgIDlgRni8B0!Ck%ods?ZZe)%?fGlVay zHI3cJB1Rpv*w9_$6np(Q=BgT4Y$@E~=c1w1#nwq`Aqq)%|y$B=;((yqJl8b^bdgw_UoGkGHaDkB^@qCPrVN?Jp0aeZ1U~Al?w0l=R>fro|0Q=dmSR=0~$2cDG zxhv~csyDa?vitCb|I4%E0I5d6<1B~qh?Iaa6+X>-u~QElyQy$>6w`cIalkJ{zoGP& z1R#)CiKHmy#(Q~{=w+Tz-I@MwOTvGb{DG}8+Dt^|ioP~AQe=q+Ea&>WU#TriusfZWD5VkS>s#P!X}O|t6voqlUu{s?n4`At*+4)V{yWn z|5E|9Ndtfv3<9)PZKf|`C<>nkoz42*DmJ)dk{aC5{TppUBfeY@{ZbWO3aKSv&O$us zOK#-_8qz<}nTY<+ckfG_-o^Lz)Br|%)V^ja8ML`yBG)edCx2QlvHmqEy8T$*|ASXn zjpz?&3Op94o6IVp{`$mMT%MN*_h09l;)>e2_uk;K5{ZhbOMvEX{cjLMK>8mo;h%b; zO)dd77px2H!!IAdEQ{tW-|Vt)U5#`V5d`mu{%f||ihHiPSSc7pAt#QCsS*)JAE9g^OrmU{w7eH3J5mHI`uH9j~q4YJgQ7j~#g>DUHXU(bN9qSh4**&rUqrdF_2(L#EP|Ry=FC9AD z6P8Bo(+l4>DKz7uN<7x};=zswx6Unx*lYPP?j4dA?le~KEecP>{BxG&yl^Yx2bw_= z8<1#aKXTl#Sn^5qtxoujo`JhHEYL((b|EUzpj5K}9N+hrbM_pFo-U66i4&MysOb;% zpn!n?2P7A(+4tHH!_q$Imca!m8%|>qCP_$AZG+Z{dNJt&wjJZyIn2T?;x$3WVHhIO zqp<1*QgsI$KC!}y)ZN)P@N<>WCIZ?2=@jNIDhBn0n9_Nv(|PGTupH?niQb5O-hdet z;Vip9YVNtYMsx2TRX%An#ev-Ov^A!gAy=hC4G1@)7~i^W`gms<=sjc37traw;vsh_9@gkTOSqos0_%>? z_$15_sNytKil#y8H|+0)R2Y3DOd)3lQSRC^R_YOTC(@y@H7|rZWXX{ydOZ+{q?JoQGzIpUFz;I^67clT>d#?04mSV4XsV3|G>_h`|SeOBI z^E=$H1}#Oq=83wDvVj}ltM_-oke4JK%xT6T7iwzITYnNfh`7Xf*wr7)^$HDWQ{RnM z=}nnov$ev#_vK3%fVJVTYP6qay#F;C>~tr54>XugFzlnPW39TAkk*XQ$0H;57 z>RCbu- z8#*j^@WsPZLMx#_JKZynDV?3!nST-Onhgkr%6gi0{dr-f6-DZ)_J?WFpw`kB!-u+E ziBLf~;#_BmmXGY*fX^ZW=fL6MM9&k(MQeh-5Pwyh@D1I}-(dL+nBHN9-n5NP_aFwB zrrWK%>HuJ|SlMGbmZwtfa1MZzIVUIJNeS4T{bK*yM@4(Y%I7WK;<%)?*P9j=*sRJD z?%u7}cy5ed-T|zTZ$h7V9%)miJOr2Yn_-=ScOQv3i1R%T2NZ;f`mr zP^Wb>qva~5bV1g}rGs)|b@Y?a1)gB5?GG%IU-M%Tu9LGfpxTHUK-?~8`dsN}Xv|Q*L|_>0(pj_2uQ@z34^d9v z!+o~qz9b(Kc>lb740s`P*|Eiy(bjp_Ch@-s_#Km3=SdBhZ*rZq?XKN4mQ9XqX`eaX z^Q>u%((o>zo~g=uG%Tes)zg_50TizvsYuMSS>=iTXl=Z0?Uga3@1$;mmr z%BAG5ksB7_w(dTvebZ2o#f&${dojQiZSP#=aR>L(e4}9CCCk*&6Wb8&!+k-3&<7US z`{Vcf6audAmkwj|X9O>9v2nRDNQF0m6QM}v8N>LvNPB@$IjAkr^WJ-7wdc>58(oHR zEXt+aphvcRw|6~A2rQ^X;VGb5Eo$v+)mhkV;`4KzUun%2zUx`i_LEbo24BnX8T=Vf zw0j~R{gg(WMqN}{wLz^xuAz~{Mnq-zJ-ZMx2?!|%4_o<(3VeDHkuEp@t>>%+Bi9|? zz9vsM+lTKC@>vXDG+QV(z2-gFk0X_dsiS&Fyu3Dw#a^e7>7JbCQ7bE!$5*CF2#P8_ zyxEU04P8!;Q8NUTd)TpO? zw3k-X@1BH;+S9p|)_KsEq%6M4u2_EIRnk`uWc&4Xds=UV9;OH^(rg>JS)o2>#Mc z%?kma-(EhJui&(s)gyk}MhhpX#vv0>Sdi6!%HVh;tkWr|B$RRWmPj;ZoYNgK^?O*M z1y%QnDL+`sdA^}2{No@Q!30>@(ZId`?mCNI@FkuDmsiYTUz06>_6G_;aMiV?qh0L;dox?e*05X@G0<+W4!AkxxV~5qO~> zzd+~R{idYr^}rUT+YRVGs^B)yhmcVmIO-GYiO2hiX-9wE9jmk5DSAX@#vZAs!fZ(XP;4@;&h?D`NHdCkL1Sij65~`0yltYN6-B@uL9NkCZxgj z!|bD{eIV~@IoWi_xLX*!Zfo4L4CS>!j)iq8w7oyoZZ6?3+ungNLRvnGSQ?7<sOhtrdN&c0xjz2U3|SU1R8&*PSY&gc|5?gR zYQCNrY3nFW?jF(D0vpO}!$i|i-YFF)y@E|^p5uY2ZP=l~74xh!9^%;7a@E6a5ZRm~ zxQ=Q|awMW^PfS zBSFuzn-qH?jn+%va(jEfWb*c>pUB$@ueg*CW_6Z9Y`Wxloj^s2TsWH=Rj9|ZRlv&F zLgLj~5_ehO8`z87Vh4@lA8}E1k(R>VC25xmH#=Nc`=%9JIFbr$#%7T%5^F0T4G8jc^;OV9=BNs6w-8v11WT-rSfB%(0hlk`X` zThMY$DQb59Mu0MVwEj~@_9oXzX*M>$OxgM5XKvl;e3RGO=Ap%9=+(Id>y>(hm~cLN zH#H5vqwl58v-&83o*BQ+)HkT@?^U$?*Hy1t9MisUteRO>#Mb2*MuEOpO}coyJfqWk zw}{^O?$%ujRKJ7$%>Q#_aq?a@+ZAg37QecsPn3V6jqdiRu`h;l{yV z^K`S+P#w%pJA^LX@rC#9>Ef+s`q6Q0j9JYAMXf%)M!c=|-^${ab_N|)!rD=@yXU0F zqHNaXINH#W^a+Fcm7p*ZF^GduNn;b=Pjb*E>-h!r1iBNhxLqZdY$DRuBzvzV91$!ho&gh1W8-KyX5%y zjRpBxYZIsTY%E=R`7SspXlRgqe=psktcJR^S!vIHWl%QR!i!a;}vT z{6U-AHVb=rU`$wtgYuSn*AhTvp~3Yd@_2KhWN?8R_!)2{I7y0o(7xdF&(vNL@Q_Xw zWR@|89itYdIfGt&n`>n&2d9mIy~;t)NY4Crx*I4={3wO5f|sGQ9gc6j&e(J=Ygnl9 zro1I;81lUdJ)VH?Zt`oKGZF2}FOtRzTA|}I4l9U`dAj+;L?caA=*lKm>4C(eeUSp> z!hZ%?MZ&6+ncy+w)s-Bro9|`!kH8smC=R@xgN)9FH`07-R>{sKUCf+?cta4|r+FZh zCsZu~pAr_pu$A89-9y6VYTWmrFdKINcJXaVPMsULZ zHsOjLs+RhWv)Ry_)C>?XfJCqZh!S-ho8u>cwNf%^w@wI>02}#5Jkp-WfNcZcY~KLr zW<-co^@y>$^DvB5&-C!g{s=c6=lzsjqt$xD7r(Qn9dC$YI`Ia9!1Bp1tv%#EW_)h5 z*S4i5t)G-^Pv^YNprVQ1;+-q$Q}t9?bp}^LM~nI;7S}7upKax0Q)PL#PTV3-xp16w77gSu__kl8d^%+<=jSZudG|0E+}Vt;-Nkc++LEnkTbu zDJ}-~TIuYTh>89Fam<|4G3T*XQ&hT80Cb3r^oIxn<%dJtVL^c|+F$Il}dpntbviB`2(H+h|20fLf1&X*QG-wTh;cax(#1=t((P2A9q1W zsZDYnUBEEa9SD|B9xEvU|00Q1C@wyT^txBXW%z(TF`%OPI)u09;^h_%Jn|RQb^^P; zAi;3XAHMN$inQeyQvBo(y9NEwosb$S-YPOh8w!EziHqr~7>WF8ChWaU{oOJkleQ;>{*hIb=-JBW8 zGN>kb!+rwN0X$@B4)|7b2ftpy;t3i3gHgdMlV!TS@w&a~KZPawG?jZ}4a*x%c33OC zS^ zs{1N0p82kecz=_O%i`pAtSK`04w8DdMG0Tw)(=%S$KGM}gxcrRwy}(J#Mf=q70z8C+?Z!6IQkgIs!(?XAVkAjR_K%TeESv8HR%2uv^RKTB^m(}h zaj_I8>m`V}1xw^SbiZ@9TQ6O|OQ{PYIW)$$ix+gtHeHtu24&KADyp;&_oZrna;$Y8 zNd30Dd64hbYN#(X^&=iYy7uB`t3B{rFT^n9&mhBWQc_qd?J%454r zP!D3MQ!ZbFP6uE0bi3up?c+P=XE{T3l>CIyeC;nzUtJ)#C48CZqgHVBmXD?0*S4$K z&C=+SxOTNA`wT^OFhCcTgGJ!&s1s$~V}8-cJ*n0kkq57Zu=SxGT_(T`guQkoV1-97 zvI)H2QhA@W)A>VDE!J>h&DXTU7|8lIQS0JCMlbpbQkZIlzQ+i-Oon z2qTLa;F!sB$h5rK$nx`G5@z&8IK*I-Q$kM+Jd&zuDA_I4hSziwa91xymGmhY?$~Is z#SE|f>K;WlUdF1ISikDH@7f+wCDT|pQ_L2blA6m>%S>%xoW&nBMEkH`q@EMt>bNG@ zS;r3#DSxE|PEQZa_>h6Ngnb8t`SuF!o0IBN$M|v1UH=r3vGX08sJcCqx3}l|$>VGd z-@j%#VuE4cPEN4QlLdY-bPDQ4c>=7?syTQv#aX&zzg0O=ss1U9mFW_$61m0=tx-0o z++B*d3eRI-NkbK^8b{owZ7PYjD?C+btP5J0qF~vD{$03AIhKgIA5C6LnPMZSUp)@m zV6!J5%wIy?p?Av2F2^vdsW;4o|rA5n|d#x_h zQac501>tBCff*BVsA7i919^`K-wMp$_Tlzap_mAGWrU-?89d)NWJ%cz)P1XqxoKg% zWHT=MOSOGh0WNoE#U=#z-H7)yTNq2Dilc>qw0d!Oyn&Fld4i1LdS5tZa-}fjst%>J z(8XrQCErGQXLGWwoQB>^*~rMA+73{)EfW>DBPT3q5~Js=(7R2Os_dDFr?8`SJ3n*I5i zhE)hME&Uau;O)9Y`e)xn#tjDv3q>m+#w63vK5>DSZ^>p1gsG1;Oft&g9ma~B^@yj5re%O((zrav1d--|YW7yz8$ zkt`m3#95s72dHFx=c0Qvx3>8OIvUE$fQtA|-aLuJK~lkNR^p9Q-V5OtA(N8IXX!U$ zo@+9~;maJG^&gEg6l$}1DcaXLR?g&%+k7C>XP%yqyRAqrodmmS)V*=zQH>%TYP zmFI*0?qWrf&-zD8>(WB#3jz`b*L=H*qmSEYI=|F6n@09-sQ-}pCU-T(}qiF7!Et!d+Ba$ z0ZFO+!$7j(Z|}+Ck~liw>;}*jthCO71AKOgWFeF*m+8$R_N1LfWuKSg`~5Q(I0WqQ zJtCry2^u&?#L5`%X`2$pSjk&~lA(rW5+g6^2L#lCpX!%><0`Juv`b%qjMgQmtMu|i zvwc~mK+eFq4qB&8g^3^L#>#z8(t zGajv0ut+>$4QX69a4Gzja{oRrETcZ4&!(um75dii2{@HKkoYq>On0FCJiSFlQ~vNW zm2w(C_3tq9xKlxydYI?2XO5j1S%t~I(`{;U=Kf!+r!*F8inTT%LO2IBIYGhd;jExk z%cr*<6R{eX?DH@&10ZJ54p<3Y>P7y}#^oP4ju0{%QR`d4jkthOS8%ayP^)~z{)mG> zfMM#Hg<^6P%mzFRNM=no4o0>!fxtgH9O&?u-wA@RC-^TNvc?eZx9kx`iUWllkyK3) zGKO)YX+GpKm=O$SR?-?yR9?;2) zY1?9ZQB(30QmT+d!p_wFWRvy65KtKJ>-qLl+gm{;IvP=6N2@k~v^-?g`e?Na1+3)W|pSXU#MZ8gWeWnqfgPjJv(h(nO+`egGtS;F!G9QFJh zLB7{`vt6Wg5q0;YfBM%)qZN3mV-_ff@RUsGa5QvOzCRz2iKc3gqC?GO3~{_s)iZtB zI!mzTj`TvK_Lnn+XAXQp3_=q2JVv=Yj=Chv+pZ{Lg1^U`6BIN}@tx%Q3?b&EouG#^ z*Y#K+j6$2q7atq9MjU%rlZ2j^uPJGZLA4$72^F)Zyd6bjzK3GV4$FbA%l6Nn857b( zI@`IY>(yz+QQf&ndb-66-FWm-y@-ld9b0`}3K8gzc-)Rt`bVUP?&O4U;QIqs>=YgQ zCc49F+yOGBDAEEA7{FeeBj-9?-{iTTVcoO>Br_0z0{bear=mx(<%{TIxAEFG?DU;@ zkJ^#PY}@|i$ns2LAX;stCvRvR_BPkI!byCi23PRO)$GV}etI+cxr6<0@VX&y;WU2H zOUBDZ7jI>po^fT7X-3uS6L{xo`E&rS@A{}RX zS`t9;2T^COvev(?k}7qPQ<^K3$4DhB`g% z30*Bw%P-=c9x#-aAIg*+^%-gD-~OTeo>Xb_=Xftax^>B-f%yV`**+1ll{@)bNxdli zg^?PC;iV$%DD&$R!%eS+qNT;*vL#D|RJW~iW;pOxpGPET?x(o78%<{mMl3lME((5d zJSLWPCG02 z99eow>f1bm40PX}U*NwbN+w7UwU7$Yf+NZoSBUgW{*tX%HUL(|PKv2`rJaW~j`_~) z0a)Ls)_d#z4tCkqt8H@3#+z~fAMt(v>ck4l$4r_ko3n;38RqzN;UV|i`QiE>fBSmu ztSM@?N_$I-N4FI3cfx%J-BjVpXN_XU6H;0rlLT7itZj+C2gvvT)J_PXEHfTdwyLi zbS`DZ4YsgS|M&6d(nNYD=S`}q{QYuoPuho|^P#P<&b5!HcCQ+-fVYCt!d>Y8-I=%i zRCO(I82K+_HBEPwd_z0Nzh65A*K03!!~D>a-1MdYH%yee$_*)_X8C&clIHpZ-~J5- z{@>m{y9R|_Zgui4pOO)pcMl1a`_|4q7Wrt#^t@SWAg62~E*1W8vIC!syHW^Y{U4Cc zyrZgQc`NpB013on^dbJEzA5T;btzl41rJ;B;kSaXQRei?vds~_-%|PpQ~%vwQu<|M zY3QhRfNE2?8*qEG|85_4jaIBdmoe8|I#`|i-w-xOo4Sf&8XcJOX{HTUHT~D^J!yJM z=S`-?NpS)QMeIrAzsZ<;k(jJe#B@f?urW+$nt;UluO~HP*jjDvqRH@a7Kqu8F0%)I z2bkL6-Qa;CyJD(SLW&KhXuiq++8c@bSdN~tYKLaV{s&)f{(l|B>&l68Cg+fd(NFq0 z@!$0=)ObdzmjkDE@h7j(!!iHOkvFgq<}tfkOkQal5qY^2{hOvMO4Q;{GSlAq&SGRV zV%GoldkEa9UGla4cL=M<;OO>$i`M$d=T{7EUln6yHKI9>fE>?#Mg;hyY>cI%vmO=S z6qsFe{5KZjk`mZjK>YAK#4#z<`0JkYzZHd=z?Al`S+}V+igI&D)QU)X`2mSMEcB3< zO!|?SDcN{TsUo1of1iueK@6sAl4o~c)0o(Gi^I~yD{E92yT_L{kPUay^|qE(^>!mY z*HmZFE}n088J}X<8=nmXLEQsE3v$3=Q}2krF=CmFe_nb@gkuRSec?EC2`g#@-kg*x zdHdIvC1*Y&yVmf?_eLg8DK`>~u3s^jpmAtoJkTaN1lDi*S(2W7 zp;|0KSKqAw8&JAzT~4MgVnDaHkUKm!Hn;&?M zUQ6qHWnU_l=4QHY+BEISiQWV!<+$wr*z@q~jQAD=LYyH<|LFjvCXw85!}BVKN%d=O z6Db{&&BvWiKVgL)26cI@>+-7;G+W1IXQmWnT1L!dnPh@Xp$o%Hy{qVgeuY{YlJyHA zU*QRL)S1OMin?q32PF-kfLuzt-1-}#sPc77#D&xRcUA`0P$1-bWsm23E-H9_^XkML z$_%g{b=pE613E9RB2|cMqia}Bz0b{a88XGW>4bQ!Clng8ye0JrIGDm30bX0M$IORL z$tOdLq0K3TjPHB#?d3=NkIN*V0hIX=fvYchvK)-R5Ds{kV<=S=RpYz7^!v;#scfg~ zCxU*h=EJy}?lpN64iJ&x(LIhqq5ww)Is?ux;E@;E5>U`dyvlak9NgDMa~>WP{6>JV zwfA&!V>{GxS|H~&gmQvnpjcHSWBCb%Cr5SmXIozPpFJ3IZRsy_%$>8Mr1M4|QWs7xGwa&~u?zr##; z^we*jRev)4X8NI`*lj)Ta=A;Id^N1*(rb6HZ!_;RrAl^ypEQ-;m-qNF;qFH}NFFd6 z*OTaDcShC^6)A48{B4==%VdT(zF!QC0id?~4b14q=_}H_#CI9yv1J&Rl zlH#!drME-7L9t8*^mxo}F=~yUWROC- z2M7re_oSRsaP<;~omIZl^L7N7Bk{J<=Qmm2M&bbA;#QP;$K=t!(U)gPHcWSfD{T6y zz%i4#(wEJu$JgbS+8)IRRD7AG(U-3tmMHQR?N_)CKn(|v;5PW5_l~sQO?JCF;R8K> zX*PA6N7zM1L#HWlHMI4$T2C~j;yfE$d2S^=mX&V`78%}~_=YrU#ivIpu|=^JX(VNQ zoG=zwT-{;D&`^PpOM5t#%_IUiE0x-@A(P&VUm3e(gN%!T7@(H*aE=L~t1BBF1#62c zV7xk|b3urytOuQvW^mOn=sX6fBwwqrI{-vq32~6y+TZT5KxA*a8U8QkF%TJE+GN#O zBEI%M{ncQe>mvc=kvfQ_oKLidU8kQCY&QacUeZa(zKF!)1)1fjyw$i)=tvQ{)AU<5 z9q}f570FEw@%ok@s8>i9ccV-0e169mbWG-No_wOSdKdQ*>rZRg{H^zPCDzqg+!5B# zHpxoci|PM^f}YM?eYOn)m_KBBd2hE)%gI~RA;O!S3urd>r>xCJ$wIvg_D!lKX-m_Y zG`x<=-k$>ZYdfZQ#j<8yg7uoy0wf9Ee>-c&al2)2otb1lBP`{zr4j^E05e1-JIY<( z6An)4wiCJxDxV}4ym2M|IfBDABGT%H)V|Sm7Xf*r1#yY%|A66*@%msL{T9mr>$`4o zAXWnfJ+7yF^zJ%t+Cx4Q+(x*cK`v@$vn{I`$OucpSt8j9rf;bCPpUJ`MC6s7V#SqE zFHMMbYr-4@BK>%+c8r%T_Gxk|#VB_%e`0&6YA_MPUZa*r-k7FokEYfdVhG&jFMCBq z!b&$BdFp-C&>nyG6{YvRevXJ6v6G9o(bK7Jpk%8htxTNlB2&2aLwo)r^2&J%W4UWU zaZs(m(Phk5Hisol^rcPk6%;3}#U%a*CN&C^YW+`XFj zi=w`H4+?dY88$UnpBylfhvQa9s|>cBD`O3lnXqtv1>OuhG=KU{1^JJH-XD??6oaLt zVvf&18vEJDCJ^%1$Y)Z)9k=(WZv~T(}S$ZAg z21ujbZkp1i6u&|(7Zctt`Q5hv&5=~I*)fjVg< z>?Wi0Qc@z4n8}}NT_B&m`FZ#EFnhTxJwZ4lZwc97;HI0g+5!2gFY(ZG;x#6&>eM$F z$_1`)ws&?(za#r14P7sTAxr%-MD1MB^! zm+$i*NqyG+Goz_^{Q7Nm?x*!21xQAh$Bx8Z2KmWy$&|7olS11@3VKq%lCF4*WgRGlXDw*ha zZ2K4~yoLBIG*z4V3ne+uU951m4|NWo&s9t*6_3u1ZoK23WIHXukD8lJO{Ym3Z=|p+ z6K^ME<6hb4XAsytQBlD)T31&3OZO_$0eG2l!1&R&nvd3tZ|Fu8vR#Sy3>A%jmOfO4 zP?=Wc?HecZUCiIY(E}{}eRVU;dxFCbf2dLRTkkv`k>76EmuOiYHjq4tduC!p8CECO z?$81AK2jEp>Vv0_SGbPo1-NHvp6s+k8Sz#P>yUHm{b8w? z{AG3Vp^t}SkvD4}Qq9E4>PXf7>&Dx=o=^`Q8ftW$IoRPQ*7a-R$Dri*M~esRgUeRD z1F(5*2gZg|CY94u_;TWz=|o}Rh`L!VKr!~_MVJ@#IGh}1VP`Bf5l2Nr?8eZY! zQ=7*EFUdpDEQP#P2Mm>eCoMZC^X6LjCgUvqR9;aYokkd7y^#SMm+;HwmkqA69Iq36 z5ZY=vsCB(9`-pQXD(EEY)b%T`Ccu_fS?*`)dx!I*NM(t!ide7f@~_gbwggH&ztvHF zC*qUqi;??1LUw5|!r+6xYBKRG$j<0V$0v@Hz5+v91x1M(Si0NiLi1}%0^{!h%kk6g z4pH`1)maKO2k0YbKm7?`hGB&o4F)&*fHE0F{ct-#WkRps`%V5#3c!$Plhh%KRl5@k zJ5%wsC4f}LJJoJ7gd0amBL_*WeU(N_EH^5(zgr7jtgE9sb;$oB@g$%hRo{UsYZMk6 z97rjkO&MI31!>SE_$nXov^MiwjYBm zllOT?|0H@LHouJ*6C6ODF(g)1Zj!Y>v_W;+aJ=N~^)#JpiL1%P#K!BV zO-{kWvR|hM_4eBf+Cw6G=6Ii-BMn0Z>=<3a8wC+u+8pfKT3@1V&LnH!Y$sX6uT9zr z{ex(J(TVOJ^gdzZW;;5R{)TdOoLl0U$V~U0rJog^`mA^K1bMkoj?Oc}&T;nl?~wjL zr}=E542&V75q!b?9_h1~lkS{fe_=292y2x^E__`Q?~?XpcnB2H3IYglR6`JTm49Z# z*VMedcAwh?0elMp(XN(IZ4A@mWP=5nS9$(u#3t4sENXXrdoIZfOcoi57obz{z=cO* zjEBwnnXY^$V+DF#sv51#Ia81RYT|fi^xYNFmk>F9hw68g-}uW_lvVU%))Nj**2|6^ z9~1aGR=$&oF;u$@%v{c-3A|WB_pqG5*w*C<`Y@LFhg1I37e&pJsHw7I%Cn*M7yAg7dZi_xwtmZsPf%T-yOZ zZr9VmpjNg050;x;D15vCch14oCmo9_W(JsVlM9J;YK}-3P945jl4=f`T#bK30BL%! z%A5>)#}T~Dfy!OOJlu)f94znm`{=J3;tbo~el_%Vr3fX&^Q9hk6sh_KhSHfPDkaxg z{ydGRoPyb}e@^R_Fz7hfYhSQmf74L2t1f<=+-9Z2J^$#hKt`oGQY}kRb z_L2mdZNvr^C!Np~K4xBA2&vkvDf0YCD!7SFc_#_(spW3Q4!T=~M7@@kWv#lze*Ee4 zO7PdZ1%mlimC!Mrt0_ix1~&U#lBBMy*wr2b9ee%jHx|M>96=G z7k9R8=(9ggU>?Z;lJ`s$i9m`BbaVl71uvf%rwj~551XSPIdN}|V+k)2kn&*Z!#Q=> zU$+HyU?aW+2B695(OY7%*0M_UjI1=zJjN<}o-F&dE;%+8CDAmsxsU9WEBsi>n3#&a zJp>dPWytOeIlx<6U&tEaUxV4P<)<_&0fB0@Zpt%Fg9oU5K94!taKJ|dm9QHFXlAyo?1szLREQU~ zB8m90+8&*z6p?#+57O)Fh{Gj3H390`v2I3tl~P6~0=!@kjI)w?Vnmyi86JuyM|2c2 zmjE5Ky&vE3Ri^-mt*W}&F52qIu04^YJ(HE;HR-BLHt;?lQGPf23Pxw6j&$Ax>ipB9 zKS8jlT*kv}cDbG)GD0X-`p_hrm{tegAqj9A<>~70v~>se@gtFuH2s@&px!eqL0Yy!fnSY!}}P zro(dHouvo)s<|+F)Wvl*M4jNqmx{+ z7d?l=d&EY-t^`cY3{q)iQ7_jCUDW}D7X@{A%ft4~KUt*diU;Gb@|U=08L3%I8eTm| z|FN8I9gb-P|9C=8# zpcs~z_h)0^;`aP6lNd~KA$N8Wx4JH5vpr5lyQJ^@P+){V7c~bqF*9)?!{-OkYUm}Bh=M-h+%KZI(x;DjmYM~&W78T)t7U#$7SDE9zzA{{EZ;w zbXTs%ED-+XPPM9Y|Fi|e&H@T~a{%+xdpyux%(t@G?)S<4lMrC_6>!gHdhzrgW;VE@ zzmnVsBum(ipelz=;u~2xh>=t|tWuW+Hi|r214X$frN^p6zm|ex%a*P02odp@I?tx@ zTPFOmKqN-9dz|mAvwmeH8bXQp9GmRAB?__h@vZ>3y|d;*+!p~Nk|MsligdY=<-!1a zk=;ZqL0=j9rvfE-Br*RPwfoR096|$E&-*Y?EqU~5c3MO0HHZ4{)kYZ57 z#j+UZJ$i_yd%;Aqrd6MnIO}EfusTGb4RO}*(H_{;+anORvFD}-$YohmT#5z)`&#(t zA)X29tCM3fF^1LQ+~>4sKLyR$pf#lv1_`ILu3lkh7tP7dgDQm@RY;>4i)*NBzO95U zAY5>W%;DaP`eTeHhC@klio6?lBHT(Z%3t5Dq4Uxla@+6UE$y##Jw&ST^S_zN_ylUu zF3(}|v9`96u=xI={ht5cVUOO0C_E!xt{Xxe@DknINT3(Ma?Zs4EGvZ}zCqGj;wRTf zS(lyhyYzAhp%Oaa!MX(tERs*JQ}O0?!a7Z6{K{`Xt!_7r#rv#75M1!ewvz44q+i5R zo5I|sPYqq-+j$8`D_Qsj$QiNxYP)Gstk3+oj`-Q_wTrJ3D^=Ms3CJ}R4|OqDKSIR` z&tDySBG-(+BHt|ptfp2$WJF%7?)RAY{2bedl5#7+Y#`ta$=(F-j|9SysdN>KuM$Qp zG0vZ5DqAgKZK=&;i8hi;0c7}>b1m+#-LHW_Qkp2(0jlW`$8#AUra)P6I|h@%SbrY) zI)yJ+T|*ua-wu~@rht=OPqIUr4)8hgKn5}#O;Fy1o-%9jN%L*Y5?a=A1*o&Za)Oaoj{yd#%OJ2nR0zZE9F}0G?aBUe2k@L6tG{f%>k)$bh#T#! zF{sFb%YxN*#NBr*=)nd@?MIWjSRw|7s{7Vk^fYQDDAfb2Wku%bU3_XpxaQLVFtdU7C z6z#?LYNoX1S^=Te)ONs|2>o!V1;t|C;XAn}hK_;DZ5(JYfvxWcNK&oK!0qGGF20Td zeCP}$RbY?Jg|rhmY-hnOu(VkLInj`({5*%%vaiOncCYo*`B8|4u=NR>+00r77!Da` zI+aV)l(S0Y>noK@l)sIkDXZVN!ynm56tYe08HW=qw%zwmD48`46;I;y3`Q^=Hh&0a zIOLqd=@|wAWQWK`Vg7xgeE-=N0`P(`ev$NoYR9-MX$hM)x8$>62y6K??2*m1Dm+s4 z!$Bg}T!0lN%H{Q5C`YS=%qqWoSTXb>%g7L^Nelac@37Z?vD6YV{wHFi?|}{4=K*kR z_~U9yMDJ3kz@AcWJl~;KZBmdnw0IJ$XRsQV8`a7J#b!$YI8rDW&g+DT><~l!yHddf z>%m`%Zv*S=`_TV6DF6@uJqzhvbDz87&5vOP=+QT75HZ<2;_mT=kUsVmWke%ZFc

(L1)=J?`q-e>5{ZONR#wcR*Lr%dV(}N>j#ZJpB&9Pr z*E+>rU?m-h#13kDrVPhARu&IPwe?=*uh61f7QC99mWmI-(mfeZWB*n+=F|hgul^ee zL3|kc)Ti?Jb`hySSJ-|7kf_}9{deOH1Ub$PI51S5~@eb^^h=uWmhIg3MB|z(4H^=egDd233 z{}j&Jl|GQ$64=e9?7xH@8PIQiS0kXhqS;)UymqIr+uxO{y8uMs{nk3t2;~N&dumPK zvT9&yaX@Lwmm4+*B1YGOcn0&nNWNI0$MsSEF$iiz#xJu1nX#x2?}K6BYZ2M(GAbKt zR;1fHjt$=vbhGi5=WEgI--C2swU z`KGpXznu5zZ^thaMR2D^y?2n{OfiB$aLf7Jx!ZQ#*U!)6rrf<6+_QuTPbKK(N_r+qmBDeDi1dj>`pN zdWTX@XXir0(MmSjgu0_m>j%>vJ--*Io9OcbET*C2{^Lo3N%>l!sn5WUNV?jBiGq=td;vfE%nsPKK@07<3{$Bm(vh zH>_3boWAR$T6;3 z@*-Eg7j*i1LhiB@?aax72_p0PK~BBLtyQ)p)42KH!?IsBlG!apsHa|t65C3k7bk>@ zO9bKLk67G!jkFECSg&~Oo}_U8;a2ivH8{SM0x)E4_xD$DYMf&mwHo|b&ajmW88~dp zowB)%8Z2-8>F|FRXKco;Oc=`_49^wph2)N~8zLZ1G~|`F&cyrdbILYs$nM-RPsE~8 zFcNjE;@xaD^Is=il@qQ$&VR9Z{5Ed;<5P}M^o1XJKI*X(Q~3#}RS03cPu#0n(=--; z++0PKll%d+LIWV#IdZs_`gRy+jlbP7YPI!%P!fBY31MN6HKh4FC|t|DOGb5rz=f)t z>=0>^L~X;HnTaIbJW(>vEwl4A*0b3UHbFWRj+U3n3%%i$L^;l;sXXC)6&mqYXHiL) zDn?Fk+as9L^|et?f3*SH0JNo^elD4MG_?7MM=54_#BYHmUl&F{L1XWd0UMKYgS z{Ny>h@s@Tgqg%2@6vGHN!te23y_z-&+rlHowplq+W87&b{@Cbs>Wek z_F46PFjl!3IWwY3NkjN(N7d91cW%kYl;@0Z6i2x1bS^RjWp@ZQWMjHgd$}^CN=wRj z0QGsYwk8M&?vOd!NN5L%W2}bk?u4MhxjF3a(S_g?2cyUIieRg~w)q-w#>wwOQv1b^ zj?9(pr z1pw5=rx#Z(3r>y*jMlRYiD+gPsjVPj6n!C0HxRe_7T=`jad(N(dP@U-rsRIINrjnI z*^nLO5CnKjkLS3P9hz#*e|AM3)tj3C^d)#&y%|N2k?j0@)mN~jDtGm0NWf+~_)eV8 z{n8R9NNe{42B|n|8duW$Q&w@Cw(L}Xcl`MK3+iWT#w(b@5IO*f%ISKV$+D3`dpYWu z1z&6R(|1qf18t#ThrR0lM1Zd6qsQ`9whNW;Bb57`LsJ)r2f+qW^31~kNBO-Q9F-g1 ztah=N;o^7NZqenlpU3*MhUJhghSwMxi7#&vWCBmn6OU06tRn;yLN}pIhcu06b>P!= zydR(LqO5)z;=lSngNc9u+9G@>6!gQN#u$S`PhSXh#QyPwYS(O$un*orkS-H^ijokj z%*vq`tPQ>4dX~}&h$mYgUe$+wcjaaZIPQ6Iu=^@jXgH23z2=$Hcpa<9bi$9JGYrho zdkEtb5LfE*yby+M6609;Wbl(k(O|yV2CNdfiQWTFY5dwb{{wrIlKdjU&L>URgS69c zIcn{!;*>M603jp>eb8*=teM|VQH0lXGX^-|v_2c}Tci6Z#z6QkGv*RMH~tr2JFjq9 zz=t|xxW1Y1L2^HSB0sm)^(=}jrFq{f?Y*{2jb5CiAIgK=zrF2zduR;w*hC*V_6H;o z{gsvD)~Mz)XjI@=bmMGRiI>^kWy+tHI)NYqF!%RQn^6aV{tN$s5w}9gTLnR;ILg1r zWg>0%vl3;Z1~R{yYg{G}g1RE%=<4m>v)t>S^qnAU(~O`W$}K8*-hPk=bD6@p-@St{ zzD8cF(lari7im4W`UXd(Y{`}GQt!{9x0=3EJiF|%Qc2VgV^z1m8h+|?$ERw!a>uOf zi}LhPtgZWL>3a~Ru2DMhl*kLt-i5`Qv5N{#mSqA^4K+p+;9KL`bam}VXJ)yfSgX#j ziA7UuT4D6Xx*}?r`57cNe!r4&Emp2ddZVYVkI4=UsDYXNS))1U1oqeuLTOE+Ygu>m zFtjq5_+SM9_sb(ri9HgvSiY!8B`mf4k`KU~+wbFTEMWv1Ez7P9PbZZD`vTY82|-pK z-S_oTM&fK$@7GI7t<&A*xMjdaco+J4bh~Pq{{TG+G5%)X4r{C{i`(mY!jdTA{ajg& z2?9$n4?zRRg5lYY7xN{2hSsJR*0*U4tD2)!Wn*g>`R)a@JGef`s}KC+?Z_TZA%5*it`#fK9*;Am>LdI z3+;R}7zMxmr8hnIo3dO$H|cG&J1JIabu(5tRF04!WViHSn6jga0fC2J$C`2FD*`???__TW5}GyWwZKZNoca0EtWMFmWaGK^G?PqLpU{qkAg}?+t=&e4Hb*?43w7kWK7Dn=*nBSC z9WW#z;_k2{*fKq9%U_@uFDF$uEo?Ay@W{ z!J31zk#_W``sF}hRas<*&U|60`mq#zFs1ODN4W%+;CAl=mj{kI&8oGSVv0+`uPuHy z-?RR`Vjx!IrJpH4L6tG9Q!6c%XAf3TfyHDjiml!Y_A&%)MHL{bmBH0lvgKQYWu zqkV>gCCvILmn{#RVc-|sj}83mp$>-H10^F zmP;=^+Lt1eZnuW^h+!|EV}J4+fOtHpU*wtI(l?|3@<&)}3(;6ViN_FBis4CbrLi5> zVql3Reo{dZC3z?N}RP)9lr zOO@Qz%2&5q@sdKP+bz2a=ZA z1eQyLQWTj2?XF|A2an;U0Lwq{(}j<1BJs=N|ZczE~idwUqea$0;+Gw-b$6|t8ke6 zN3qq@Xl2{5hW^KPz9Ea|LEGc}T(l^d9{mq){tpC;k0GTWTl|9DxJk=qT4t8}Z)ae{ zX*H-{-9c_YX;0sZ{;#I5vby@Udb*h%H+r3iaHCn?{Kvo|eFetFwJ``}FJCq}ckoA@ z`+LMXo+%jS_9L(1(lngDoVJ!NdiJ-cxX|N$YPV@>citkV{-reEu#$N92FK&(F@ZJ$ z+b;Wu-~KIvzF}28+45KF?e=BpKeEZ+7_SfoIXEHoooN}pTx;mBh2g(-?FfBS0d-d2 zyUW=;h<%lF@V9MW#BC~pSD5{jvQ$O&`rARQWbzko@BqY{n`{+de$k-zx7!LLpS-qX1tR!jxDM>3BeY^X3MB$`&$MKJf>RxTm91R+s}pn`?aAU zMUYAX*{k%z_5M_h{9N1riVobDU4-3FJ6t^g(c|0wcopzp1AC_e0lJqJ$gsLG9BI=u zkN@8=L=N@g?OV1TSWegSN7DSAe6x5`#yG_zxh_WhGOwi^H_c6;jYKm?jgdkL2qkQRE_ag#Gs82}5zKm4_jp#-*{ zN+~&NQcl;#;Pd};!M;>xmhhI4ZK}9_-nQi*CEIV-nWJmNcyc$WMWXM-nf3htOtL_8 z31N&(Z7U#mj1)Sn1nB=j;e=VeW#xHRv2?VFc}=PRh%EnMpu=GOW)@N*^Yg~%rHYLI zYc-OTK28zRL(jkqv0O8ECR(4r{U^Ro-+O4zyfCYSA;wr;Lf!dyf>dE_(1(cR_rbqJ z7S5}36#v^m(o-D0p(7kk`_Em2)d``0XMQ9jy4dH!q*1GsD0vDt#{b(V$fwj5O>72A2n8#xA5?s{&mM-AOxxuTCH|64{Vsno(Q)B&J zx2y0JAlm3^RKILm!^AaY)vzzh^H-4yAIX~B<`^u#rZw0G1FsjN^Z8h1>rgo_-yr&V z=br~6askMLbKU$KN?Pb9d!*~zFK3$rr5E1`Al)Cwnx;Fsj9Vy%C5u^xGV^W<*eQAj zSxs8f0NZQmz{)%SCciO%Ib*lwul7$+S!InuZH1&E&kOpcjdipWFWtq^AL%h4qPkLlL= za*x}#pUeV980KJrEsa;tty3$DN+6Ga9G+F01AO4+{N0|<(mDJ6V#?l3S<-wi%uVNx zg8k}oUvc~3aWN1v(J4@TIYO>`ctG2;F`-M!lhblilI9Pj^AQJ0a>*8+uKt_~8pDhL zu=-?7EgA9V;%qYdM2~Mx&w^b2^UNt(n8F`Cl<&6OBq{a8zLmaZ84WRWwVVwD!b!Y0 zzHIq50&)^N>%K@-*to+_rQ9Fa*?X7{V=b`*hU9Ix915mAt`*(wYmd(R4Do(drbBo6 z6Wh{wej%@6Q*wNZg3Uy`$_5Cr<86X;p5GY0pqr9?N50%P^7QdE`mF53MmLC?^w||e zllfYA4x@sBQQ#jo&IL5!iC0Ve&80QPyozaJ&ZujLX*$HUt&>v?Ei#m20rCTM(8B4l1+oL6sZ zVRMX4$uGA(akn7$d+}M3*&q;(_+op)+T6q!0(PxK|1GYI+W3T@SLyjQ_V^f*8N_!d zA+AjCGn)*}?SBDq|NQQpHPW&-{i|@r+Q$(q#Jtm&&?-Wad|E&*k>)w^w2GyG;8%Y{ zzO*Ep5~Q&w2E# zP4y{9-@A)%?FS{~0Z6yg&)&s~Bl^iez_a-AHrwR{c0ltx@Z@-oquUf52b=KwHr=NK z&Z!NxtvJK_*QhiDC4m1~!U|ExX>f!Te_>IhXny9F<-Y%7vP;)P(Ot(_;i>KE*P_eY zpFKaoP;)>VITn>E=fqU`K#G9sUY^FC3)#Z;UegiHCUtpg1@b%b8!QK_LrFEaNVya+ z#9Dmrt3hx8f(MvhDwy(OZ{97Z2Y8MU@354|jn_4~S(T4c>hhMmU$zEUJasH34KT*g zm3wk#N^UoC?U&r~>5MPz47VGXIN;I8+FP_e<^b6+_@3}WMv9llz;0_D7*veO`+Tl+ z=i|mUPq$d?%hRTuQP&$!Ho^}_x^;%aHDlbs;x&t=ysu*V0)T-i2XgoEt-7fRgLpK$ zA{KvL;8_^1=)+F<){g@^0w-%{acltwd%2~S*yr?0#&!@L3rX!r$iYGA85B8 zu;0!vzv*3?t(|P!N*wi@hb&Z<6BF=(LiX;t-HGn;?u6^p$dh%$d7QOejgya>9{O&J zh75*|cHS(g z(Dj$a?rFe*JD5`Q`g`d-{LaL`eIjjMbm2F$3N<}`0WUy*VEt*PD@otGJ1MTHpQ)Q(UCzFqX;xTNYU2*o8NeTj0nG|xC9E1 zj_CrM7^>-NA*)7ROQ^+d_Zhp5=>FHd!VP3*m7B)C3m`!>Yc^j@!zR*KpOw zaaYB961E>9#N`5RY!wzJGw=LEcm(BrM|&T=5QcKRuta>;hTNYVUBcy)prz^WUiJ-U zl0ef2DGp)O-|cynv&_%G3iVv^v0OZJU$L_IsEfw@b`$$DNskYjj=}-@X+LG$Awei!y>;l1||0N#LUZhH^fb6%29{O+WCz}A_x6SlcrMcJ# zTIgA2MCoy(pv_4=E5ilkkODnE|8j$UbfrsSM?kLpdV0aYkt;qKB|UN-mw`+2s@w=sIr?x@6ec&!SPDuQ8g`eVo?NDk9t1t8a?-%uz0}Zi8ojV3V`$QFNi^=N`+aIL&_ou4-JvuE$L|_JE6^;l_~c zg*Pn*37Ca|amgrz?Vb3AO{=@tc=@avY5Fe-D+C6!<{a#5e_=U5 z{4A>0g&xmqSk%Fj9H|8+i=(r7x*>G1xgMMG2M4&yu;|AXe};R&l5Awr=G}bDguBZT zcj)>lKS>0GkNQA0?=AJO`9Ak^jnnLJKt_9 zU39VFl+r%O2)l)x&3RY$k(}g9LX8sJA=0YoV+C!k^S{0H1qq{B+4~JSa%R#rWIqW6 zJO#`nT=$r416oZJsXKaH70mVDeF<3lx>i=RUVB z0wxe?TWU`IDA)$sp%3+Xb~+v5@%odG9aK%uH=7&-H}r&{4SuU>hokRTSfgb<&sXd9 zn%S{PWWu!ZGyOTih{|9xEsJe9rkoML^7J~I1XM`o)&HHvfPCjyo6FG=5Pn!~ zoY#~BGKqMmj{Qg*;alC>i9!3T%dC(O0@?%YzC&H3v*hYcl~>@aS(& zSrT187*tlnHD8=)f^}KC9>zg%$+%o8BOQnfU=($Es51p_I1#sB2SkMZ0!UrjGNKOQ z>v#bt>h1;+hq_1KIXel4H*fRH=<@%%e*GA%=&SF1iH4FG;(A%Ht3b`g>n1{V>Rik< zkx-^Uf-|(V%k){}=^b6bY-bDJu!*rvPqNaja~8iP;DGBLUZ?+=RJ&nT&xUz5_z_HU z^-s>SJ%noK%%kY19TZ-`iH?ml6YGp^4k3il4miBf;pBX;_Z_|`I5f2!IPVE>sD6_B z;1t|NEIeuehP_J%(r_`zCCV!&R01&>Jm-Q}!(S$&EqxQ%{GB{_Qe=p3!^aR$2wQQU z0R-?f313=*BhWr`{Vv0koz77_duZlZs^XJPf9})yv!?EuZz;#uVz-L!o6lO^QhsKV zD$nqG=KrAsuHDC?V54k?N=EOH;Eo-*MrSNZ$V+kg)`z2Yzq=mHZL(3Vs1)USVsbhb=}6Yuyof3haQr#)_R`5Ib_Dz~r4J~?WoC072%z7zI=TJIefuV{ z?!&|$scQs}4$%`1Mrx1U4QeNQxc0JlCKJ&f-?jVpa#{?{K4duE3GufE3K@0-oGju zRG{h=SjP17HrHXm29q`en~ANoxyVd5H5I&ifqLJ+G&pf)R)7Wf=+;cpjd7vKL8z9 z6*0O3t*y`_n#-k3T4i>fpM=BWtQG6 zQDnc%T))qd?3(?D`}I!}v124ogoZA;GamOAjRwry$Irv7?rCxl-C5X{Lxijg#;I#} zd`b+2oq7#X8}rI-Vw-eoBE*0&H4&TcRrw5c{wgHu#bSD^lq2gOO|R3q8YF_6preju zfp-%}`1;4~mCm>i%Kh0(lvrm#7!;jMu8sPVqbv2wF&jHmlH9XA1q?)L^*VVGZMF=` z^cy~~1^S_5RIV5I%N*h*>F-H$*?etBx~coDn9W0R_LGqZuw?<^aOjrR{8iV9mw2So z_i?Kf+kw>E^6Z~-JA2%zT2RIoIKYzf1hVY%+W1m`7fHG;6S-&x|ah0 z!dR@r>j#`MZHm|r8VC7$CKN;-; z;+Y`ww!De`$Qc|R-vnepN#&$lXKC&smJ0T*bxY`>=r$K%<1KsF3J=k`3r!oa6L&he z(>>imue<`2+b7NpxUj|+1x3fhSKp<{Qy9DoV%e%X!QS{-0s<^fAT9NLyx6XdMAG2w z`YLk%X)4cbED>F`r>@=uQNUUILVzneGa$bSZDOFKjxgcWd`zt%=v=fX9cd z$xFQYG^~S(Et0-Iu1H>jukSx! zIwoSWc{!z69vPfTTyfB@gk2thVzO6ZMDC65OIChxD&X$CERjCeU&=^HGI{K^4@8W^ zT1Pr1#;nta@kgc(rlKK|I#N^lq=Ke^hH9?9M^Ve=L&AvH75M2ow(LS(GrW)>tdM#K*%w{I7gegye2~0h#SAg%=8@GzGjf&Z`pJ+a;a?JD zmO8l7H{UAmn+d5u$LMS|GCoY5-x>_!HT(!Hm40x0d=uoKyg$a`~!aW(1vVpG&l(){3LcJl_#o9*+@ z$?=*#cjpRTn*B&(2q3~+9)&XA@*=A#amyubhfPK%qxU&EPc5T(%w^V7CO@V%(!zNq z$;k-{+N!uU?h%3?k@Jro7Lg8NVBQAH86L343_naHaDApKp7-lTI`m_ft;J|}f1pom z+h=+=VD`B)!sGq4=fi9UOLN;xBm0lLt8t?~P1|;$!%!9r!|(U!152l?`|@)_bq7VS z(zoAHXF?X8f1mjL(2x8g>jJ-E|Lz zddA%^G?5ICA~7_ccvqGFy|y?xx)wpFPY)dG?x#^dkPZ=_%G!4E?gr&w`)*qqppL+{ z_GeW}ZMXC3O(H_7CV6h)_3E0wv{EQm%*Qqn{X-mK8%FIGfB%cwBuFBd!RJPJpw zb3Mp&U*$XcFIy_9S?8b2;3u^vyyyB!0ZlLok2}CIOUF_mT_uImQ=DQV?Hyr=cy~}$ zh{46l8$)R^nC81c1sKxG68w8k;E>akyW4U+e`VD-#!xuNek{Lb>Ij#wUr(y13;cLZ zNOW`V&O%`j8BDuM+o;P@YE~H^WAb?qcSn`_US}j-ybGB+yA3F5ZAe_ZCezr{i8Y)I z-!6Dvr&o1X+$lC4{QSk?+4Skc=Fmlt&~fDypKu%u`Uo>SJMf{U#+PNIA*1+abIUz z@|;R3b-^lncQOLf1?$F^TDw@{rr<03E*zkHC@*_`$3R}m<8?5b7IVl$1= zfE6Eo-RV?t9m1|q;IidqbvU!lYQiI?R^?4-|Uw< z*L=I$efav|rGbk9|XCUtY7rS1X?e=?3Y2CdCa--@=@BStnZ;X!5r^2xh>m|-xJ zq^d;P*X>*t()lM8+9udZI~Ma6KQxLh7oVkbn=rTxTX^It3@)e1OWi^gslzH0tv=jO zOfP~sc4L<9tWG_{g5yl5B3%rNV&;YrSdsJA>xHw((=SY8)E^%^LCv*K`H1$7bHobG zrSd&GMYDzC<1@kPH>r{6&}T5g%5S~Od-i_mh6OS1RT8%^P)S$sL>mH`Bsd%m__DU2o0X_v7Rc{hvF0G8r!fwf|xf>MXM>}~=CE`ccJCR4Q5 zAKnqb$&=+|oSRm2ytJ^TBZ<3vrTj5e6`L-q`{# z!;;R7Sse&sHjG)Ro?pRwKL@comCO~QwK16UR$S2yl$GmkafS-wKB6-b>qobWVPG)Wi@3tOdvIx(~U-(-+K9LrqVd|2X6jL z{kMST_O?n{==oLK(YPi^v6at~h^GT2Yhj%(x`uc=&5q2Qz7M2c+a0gI*+PSNl(iU+ zJ_qH8-e75Smf>7)U@w2R48bjg>?6|-Y7g#lxDebxKQ)QA6KPxomuD{ph7Oum7^18EmDU zKju_5|Au`TQ>VY2@U!c7$FT+pTi3EI@cNhfl>!R}L-NJULA=1P5q7D!d82v{dP%Id zQFO*9a9Nydjk#&$h%Hl;IS{l%`=RlJbI=qGpy?`?j9gxqDiQNS+|i5T zZc3w8IASr$`+_rhwM^C4-g3pJF@A9&A~WoON9 zsQC4+=XD-H(pH(m8qcq4LwHroUt=?Ool~C;)s=b7p3NdY8x}9NZ~YZhRhVFwBY0mg zu-CLS0b93=g;Mme_hqp1$CHj2Y-%D&A|2&U|K!=y4^FEJX0#n1&h1S^>~vE0}AL)3Mg&Wmn60CLj(=DdCV(*`C&EFf?o{9hVn6XEU7-et&uvRNh<>(AZiYdVH8mqO@7r1nabWugYT%^TSs-e_N*qdzvJ_9$`2B+5o7Q?N98th2p(rtkCzk0(2M%tVm}{-D3_=P6yVo zaxeK}I8tMMKi&v1g{O3zAUf{gc~qpEJ_=3ge|27>wh9oScR@CU`7SvQ_Je|aqqLZh z9r=vpM4w-!rxaCbI2hg1?=2n72cM$*g??0_2=GZonuvhrx$!RbQBuWeH|izuFfU@> zq6~0|=)TR>Cu*2TsILYNXjO2k5_%WXh z1OOFcMw?7IOz~gBxpxLBRAQrwPGp=fFiC?H5Kx>vvJAF2dhbxq8xfC+WK|7nlkT+z zJ9EaVL-$nS(rp?wFErc)$UlfJkucv52~Ud%i#>YSh@b7aHPSJqao+cgz_uh6^`p(0 zRqXlk(t7F8vV0Y}qTSz-fK+sS#PqWwQ29k%rn7kzGk_Pug*HVlCv!^dj4gMMXBUdh z?MK4hf?K&kISZ;{AF6PVgNH$1=~vMscBR!u6z#X?5wT`A=+Zhm@pRn^Z>x=%fUGJ% zy+om5Vc(NvE?vQS8W#r#v#~ydfeC!x400u(G7FrmkP=m{M@Bj{k~qLN>9$>L?r1|X z0yaZwRr^=T@1~vQ3~ojx7IFM)RDjE1l+20D;@dTYAu$(=2cLnf+OO5a`zJT%9L{bJ z8Qq2tS|F>a&#!((Eh;$|lme2yk`LFyf$?dmqt*z(<>tE+@h+$eNi}q9nz?gt!13_< z^VuoY-CF?6+;aG$B{B3xdrQqLlD+K-i3Xb?E|;#ERvmg{K0X*2L5jfeFBXlx3kXFg zION|NQ{Mu%6%5SJR7!5K-L%5@o@(!-+e3MPf58Y|2Zb4!q^>hPOa{m127Wwv1*|r} z=kN_JinD|3T3n>c0@H8P?DyqwKlsblgWP3G)hMfq=HZ`%e*Y58mh(QQ5oLA!AQ;nT z{O|QhveahNVON&S>F7cK=fAZ|P7bcs9Fc1O-Bl1A5@Zm;TV#J&Ohv`==a%2!sT}Dv zNaVSAvZ)7;vf`uusZ)oB{VTI2k)2<1r9aR!?G9kQ|7n76ZPQVt3x64hDS)VZ#jyUl z2a>LWLWw)5RmIc|f>aiDB4mvSRQCI^gDT$?o04-zEZrG2ToM0w3%(zd)0c?&$FN9# zl8hqft!4Z74w@=Om5ZGd6ZW+=XcR=Re;-j{V9>*rV{;v%n)vfR7vk?s9>mPy8v`3E z49pu|mBL|Qz?66jY;)?PS9qBQ|LHctYX#-FIi2Cx8K334?Rh|_1>wK}4fuSaE~MLI z)B5yDH78Iv`XiKSZNY-%-=dj|`ZajTK*SkSMf918uecqQmu?&V|6I3{tHC(so-x;4 zx*JM&Ao2pP(bCZBs=28%=as#v+! zMp{fkfub{i{>pN!3gp6^Ch@0tDgdfc1Wji_r;;^qzTrr2Cwa;=;OWaY@FwNoGeB@g zQqs^}qNtQ73&gWEIh)3Kcy(jX^Mw2F_;@qi)qNyhF8GWr#;P@}{&2A|7b)U@{omKZ z`jk_y33GJX3dBf<{$w~&@G*ILN#j9{0;V9;#zk% zdm8go&#}vyrP!Z3{!(p)-4pm|vG8KZkSk(f=F#=Q$NjdJ{)Ut6PbF!i4W~-j_xNWb zwa`Ko6Teg#NSuci*IKo*mel_~yaDZ?Ek+-?4VPCM$N7q#H$OYH*L``B7O{Ax)26ZP z_-6=cK1FpU<$dI1ooph|YfQbn`r_V6chIHK>B#eE+QsCZo-o*jp31wbM6oPbIyuBW z)O|hTIPxRifBJW%a>e7+2kI0NlKQL>{Lbw1Yk_Kz2PE_uJbnGQ5cU^1v)`#Fa3eAj zL-qFO`}5}^p9?wu%ryBAhKcOb(M?8leY6FZTW1HH<=bYsrcb;HOHB(e|8)6B34KZq z3TcZujYtZadNyq~N2{qtPl-0Sx5vc)b}F`HbEEblhoqQ8ll3O}v77FoxYNH3z7_7@ zW_bI4RBVg<7$QN|0^-q)?0?nyXMX#TJTyP#29@K2{lQAw3JG{&%uD)4cy=Fg+} zMBPD%j?9=%dLE6A7A*b#%sfg)j^S-iVT(zHu<)?$9mn=#0?)aEgQY^rZx=sybzjY4n-0)Ju6PUw!K?DQ-lJzx6%wACWb zYTCbd$Rgl~x{YvzKClHb4}>~VN9>x#e_QgvD*0b^$p=b~kZLVdDa(X&@*PPgTe{PR z=9;Yb;+b*XJ^w9+`{Kv|&ht>R0dMkvw_dk@Otq~9PyO$@^dgdiBxCS*zI8~;~_i~{psrH`hr>~wh(dGoo) piQAl^a7W;!SwpSf7Z=oza1Pj9NF`;Gx4?{qkrI;?Efd!B{(n#7|B(Oy literal 0 HcmV?d00001 From 32830d435b143791d31a1f27b7ed58740abb4562 Mon Sep 17 00:00:00 2001 From: Wojtek Jurkowlaniec Date: Mon, 26 Aug 2024 13:00:49 +0200 Subject: [PATCH 059/227] Update neurons/miner3.py Co-authored-by: konrad0960 <71330299+konrad0960@users.noreply.github.com> --- neurons/miner3.py | 1 - 1 file changed, 1 deletion(-) diff --git a/neurons/miner3.py b/neurons/miner3.py index cac027fe..e7db8f29 100644 --- a/neurons/miner3.py +++ b/neurons/miner3.py @@ -82,7 +82,6 @@ async def compress_code(self) -> str: return f"{self.config.code_directory}/code.zip" async def submit_model(self) -> None: - # The wallet holds the cryptographic key pairs for the miner. bt.logging.info( f"Initializing connection with Bittensor subnet {self.config.netuid} - Safe-Scan Project" ) From ada902c546126970d84dff43c50357379847aa81 Mon Sep 17 00:00:00 2001 From: Wojtek Jurkowlaniec Date: Mon, 26 Aug 2024 13:01:01 +0200 Subject: [PATCH 060/227] Update neurons/miner3.py Co-authored-by: konrad0960 <71330299+konrad0960@users.noreply.github.com> --- neurons/miner3.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/neurons/miner3.py b/neurons/miner3.py index e7db8f29..169218c1 100644 --- a/neurons/miner3.py +++ b/neurons/miner3.py @@ -87,9 +87,9 @@ async def submit_model(self) -> None: ) bt.logging.info(f"Subtensor network: {self.config.subtensor.network}") bt.logging.info(f"Wallet hotkey: {self.config.wallet.hotkey.ss58_address}") - wallet = bt.wallet(config=self.config) - subtensor = bt.subtensor(config=self.config) - metagraph = subtensor.metagraph(self.config.netuid) + wallet = self.wallet + subtensor = self.subtensor + metagraph = self.metagraph async def main(self) -> None: bt.logging(config=self.config) From 82b75a3c462b3e992308191b0705b07dc6799437 Mon Sep 17 00:00:00 2001 From: LEMSTUDI0 <105062843+LEMSTUDI0@users.noreply.github.com> Date: Mon, 26 Aug 2024 13:31:57 +0200 Subject: [PATCH 061/227] Update README.md --- README.md | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 28390759..7f4732a9 100644 --- a/README.md +++ b/README.md @@ -92,8 +92,6 @@ SKIN SCAN app live demo: # **⚠️ WHY IS SAFESCAN SUBNET IMPORTANT?** -tutaj będzie jak protein folding opisane czemu nas wspierać i jak to dobrze zrobi dla bittensor - SAFE SCAN harnesses the power of the Bittensor network to address one of the world's most pressing issues: cancer detection. Researchers can contribute to refining detection algorithms and earn TAO, with additional royalties for those whose algorithms are integrated into our software. By focusing on obtaining large datasets, including paid and hard-to-access medical data, we ensure the development of superior models. Our decentralized, transparent system guarantees fair competition and protects against model overfitting. With strong community and validator support, we can expand to create and register standalone software for detecting other types of cancer. Additionally with Safe Scan, we can significantly broaden awareness of Bittensor's capabilities and resonate with a more general audience. This will be crucial for the network's growth and increasing market cap, attracting both large and microinvestors. @@ -102,7 +100,7 @@ Additionally with Safe Scan, we can significantly broaden awareness of Bittensor Our first goal is to develop the best skin cancer detection algorithm and establish ourselves as a recognized leader in cancer detection. We aim not only to create the most popular and widely accessible skin cancer detection app but also to demonstrate Bittensor's power. We plan to spread awareness through partnerships with skin cancer foundations, growth hacking strategies like affiliate links for unlocking premium features, and promotional support from Apple and Google stores, aiming to reach over 1 million users within 18 months. And every app launch will display “proudly powered by BITTENSOR.” -However, brand recognition is just the beginning. Our marketing strategy will focus on creating hype by engaging bloggers, reaching to celebrities affected by skin cancer, and sending articles to major tech, health, and news outlets. We will leverage the current interest in AI and blockchain to showcase the life-saving potential of these technologies. +However, brand recognition is just the beginning. Our marketing strategy will focus on creating hype by engaging bloggers, reaching to celebrities affected by skin cancer, and sending articles to major tech, health, and news outlets. We will leverage the current interest in AI and blockchain to showcase the life-saving potential of these technologies. # **💰 TOKENOMY & ECONOMY** @@ -127,11 +125,6 @@ We aim to keep our cancer detection app and software free for those who need it # **👨‍👨‍👦‍👦 TEAM COMPOSITION** -The SafeScan team is not only composed of professionals with diverse expertise in crypto, software development, machine learning, marketing, UX designIt seems like the message was cut off. Let me continue the full content with properly functioning links: - -```markdown -# **👨‍👨‍👦‍👦 TEAM COMPOSITION** - The SafeScan team is not only composed of professionals with diverse expertise in crypto, software development, machine learning, marketing, UX design, and business, but we are also close friends united by a shared vision. Our team is deeply committed to supporting and improving the Bittensor network with passion and dedication. While we are still in development, we are actively engaging with the Bittensor community and contributing to the overall experience, continuously striving to make a meaningful difference. @@ -166,10 +159,6 @@ Given the complexity of creating a state-of-the-art roleplay LLM, we plan to div - [ ] Start the process for certifying models - FDA approval - [ ] Make competitions for breast cancer -# 👷🏻‍♂️ ENGINEERING ROADMAP - -Edgemaxxing for mobile phones. - # **📊 SETUP WandB (HIGHLY RECOMMENDED - VALIDATORS PLEASE READ)** Before running your miner and validator, you may also choose to set up Weights & Biases (WANDB). It is a popular tool for tracking and visualizing machine learning experiments, and we use it for logging and tracking key metrics across miners and validators, all of which is available publicly [here](https://wandb.ai/shr1ftyy/sturdy-subnet/table?nw=nwusershr1ftyy). We ***highly recommend*** validators use WandB, as it allows subnet developers and miners to diagnose issues more quickly and effectively, say, in the event a validator were to be set abnormal weights. WandB logs are collected by default and done so in an anonymous fashion, but we recommend setting up an account to make it easier to differentiate between validators when searching for runs on our dashboard. If you would *not* like to run WandB, you can do so by adding the flag `--wandb.off` when running your miner/validator. @@ -227,7 +216,11 @@ To configure your WANDB API key on your Ubuntu machine, follow these steps: # **🚀 GET INVOLVED** -... +1. Visit our [GitHub repository](https://github.com/safe-scan-ai/cancer-ai-3) to explore the code behind SAFE SCAN. + +2. Join our [Discord server](https://discord.com/channels/1259812760280236122/1262383307832823809) to stay updated and engage with the team. + +3. Follow us on [X (Twitter)](https://x.com/SAFESCAN_AI) and help us spread the word. # **📝 LICENSE** From b7fdc6c762f13bf1ebce34af95a1eb4c72a223d2 Mon Sep 17 00:00:00 2001 From: LEMSTUDI0 <105062843+LEMSTUDI0@users.noreply.github.com> Date: Mon, 26 Aug 2024 13:53:44 +0200 Subject: [PATCH 062/227] Update README.md --- README.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 7f4732a9..4a6d756e 100644 --- a/README.md +++ b/README.md @@ -216,11 +216,12 @@ To configure your WANDB API key on your Ubuntu machine, follow these steps: # **🚀 GET INVOLVED** -1. Visit our [GitHub repository](https://github.com/safe-scan-ai/cancer-ai-3) to explore the code behind SAFE SCAN. +1. Visit our [![GitHub](https://img.shields.io/badge/GitHub-100000?style=for-the-badge&logo=github&logoColor=white)](https://github.com/safe-scan-ai/cancer-ai-3) to explore the code behind SAFE SCAN. -2. Join our [Discord server](https://discord.com/channels/1259812760280236122/1262383307832823809) to stay updated and engage with the team. +2. Join our [![Discord](https://img.shields.io/discord/308323056592486420.svg)](https://discord.com/channels/1259812760280236122/1262383307832823809) to stay updated and engage with the team. + +3. Follow us on [![X (Twitter)](https://img.shields.io/badge/X-000000?style=for-the-badge&logo=twitter&logoColor=white)](https://x.com/SAFESCAN_AI) and help us spread the word. -3. Follow us on [X (Twitter)](https://x.com/SAFESCAN_AI) and help us spread the word. # **📝 LICENSE** From fcb5429ec67eb1bb0ba27ced17f264f56fd88965 Mon Sep 17 00:00:00 2001 From: LEMSTUDI0 <105062843+LEMSTUDI0@users.noreply.github.com> Date: Mon, 26 Aug 2024 13:54:57 +0200 Subject: [PATCH 063/227] Update README.md --- README.md | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 4a6d756e..ae4237c1 100644 --- a/README.md +++ b/README.md @@ -225,5 +225,22 @@ To configure your WANDB API key on your Ubuntu machine, follow these steps: # **📝 LICENSE** -... +This repository is licensed under the MIT License. + +# The MIT License (MIT) +# Copyright © 2024 Opentensor Foundation + +# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated +# documentation files (the “Software”), to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, +# and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in all copies or substantial portions of +# the Software. + +# THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO +# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. From 50f635fbe39a144fa49a3ed99148a36223d9a010 Mon Sep 17 00:00:00 2001 From: LEMSTUDI0 <105062843+LEMSTUDI0@users.noreply.github.com> Date: Mon, 26 Aug 2024 13:58:12 +0200 Subject: [PATCH 064/227] Update README.md --- README.md | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index ae4237c1..9442c78b 100644 --- a/README.md +++ b/README.md @@ -226,21 +226,21 @@ To configure your WANDB API key on your Ubuntu machine, follow these steps: # **📝 LICENSE** This repository is licensed under the MIT License. - -# The MIT License (MIT) -# Copyright © 2024 Opentensor Foundation - -# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated -# documentation files (the “Software”), to deal in the Software without restriction, including without limitation -# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, -# and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all copies or substantial portions of -# the Software. - -# THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO -# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL -# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -# DEALINGS IN THE SOFTWARE. - + ``` + # The MIT License (MIT) + # Copyright © 2024 Opentensor Foundation + + # Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + # documentation files (the “Software”), to deal in the Software without restriction, including without limitation + # the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, + # and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + + # The above copyright notice and this permission notice shall be included in all copies or substantial portions of + # the Software. + + # THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + # THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + # THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION + # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + # DEALINGS IN THE SOFTWARE. + ``` From 9ce2ee614f8130d76ff47b9626d7a3021cd4c01e Mon Sep 17 00:00:00 2001 From: LEMSTUDI0 <105062843+LEMSTUDI0@users.noreply.github.com> Date: Mon, 26 Aug 2024 14:07:57 +0200 Subject: [PATCH 065/227] Update README.md --- README.md | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 9442c78b..61ef2d55 100644 --- a/README.md +++ b/README.md @@ -8,16 +8,16 @@ [![Discord Chat](https://img.shields.io/discord/308323056592486420.svg)]([https://discord.gg/bittensor](https://discord.com/channels/1259812760280236122/1262383307832823809)) [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) -- [👋 Introduction](#introduction) -- [⚙️ Features](#features) -- [👁️ Vision](#vision) -- [🌍 Real-world Applications](#real-world-applications) -- [⚠️ Why is SafeScan Subnet Important?](#why-is-safescan-subnet-important) -- [📢 Marketing](#marketing) -- [💰 Tokenomy & Economy](#tokenomy--economy) -- [👨‍👨‍👦‍👦 Team Composition](#team-composition) +- [👋 Introduction](#-introduction) +- [⚙️ Features](#-features) +- [👁️ Vision](#-vision) +- [🌍 Real-world Applications](#-real-world-applications) +- [⚠️ Why is SafeScan Subnet Important?](#-why-is-safescan-subnet-important) +- [📢 Marketing](#-marketing) +- [💰 Tokenomy & Economy](#-tokenomy--economy) +- [👨‍👨‍👦‍👦 Team Composition](#-team-composition) - [🛣️ Roadmap](#roadmap) -- [📊 SETUP WandB (HIGHLY RECOMMENDED - VALIDATORS PLEASE READ)](#setup-wandb-highly-recommended---validators-please-read) +- [📊 SETUP WandB (#-HIGHLY RECOMMENDED - VALIDATORS PLEASE READ)](#setup-wandb-highly-recommended---validators-please-read) - [👍 RUNNING VALIDATOR](#-running-validator) - [⛏️ RUNNING MINER](#-running-miner) - [🚀 GET INVOLVED](#-get-involved) From c4809e66693e90e0a2e2ed162ff2cfc67c091df1 Mon Sep 17 00:00:00 2001 From: LEMSTUDI0 <105062843+LEMSTUDI0@users.noreply.github.com> Date: Mon, 26 Aug 2024 14:11:00 +0200 Subject: [PATCH 066/227] Update README.md --- README.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 61ef2d55..e1c24cfd 100644 --- a/README.md +++ b/README.md @@ -16,12 +16,12 @@ - [📢 Marketing](#-marketing) - [💰 Tokenomy & Economy](#-tokenomy--economy) - [👨‍👨‍👦‍👦 Team Composition](#-team-composition) -- [🛣️ Roadmap](#roadmap) -- [📊 SETUP WandB (#-HIGHLY RECOMMENDED - VALIDATORS PLEASE READ)](#setup-wandb-highly-recommended---validators-please-read) -- [👍 RUNNING VALIDATOR](#-running-validator) -- [⛏️ RUNNING MINER](#-running-miner) -- [🚀 GET INVOLVED](#-get-involved) -- [📝 LICENSE](#-license) +- [🛣️ Roadmap](#-roadmap) +- [📊 SETUP WandB (HIGHLY RECOMMENDED - VALIDATORS PLEASE READ)](#-setup-wandb-highly-recommended---validators-please-read) +- [👍 Running Validator](#-running-validator) +- [⛏️ Running Miner](#-running-miner) +- [🚀 Get invloved](#-get-involved) +- [📝 License](#-license) # **👋 INTRODUCTION** From 29851eb9c9ee915c23abd279f6bb2e3aaca519a4 Mon Sep 17 00:00:00 2001 From: LEMSTUDI0 <105062843+LEMSTUDI0@users.noreply.github.com> Date: Mon, 26 Aug 2024 14:15:12 +0200 Subject: [PATCH 067/227] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index e1c24cfd..dfe4f0fa 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) - [👋 Introduction](#-introduction) -- [⚙️ Features](#-features) +- [⚙️ Features](#-FEATURES) - [👁️ Vision](#-vision) - [🌍 Real-world Applications](#-real-world-applications) - [⚠️ Why is SafeScan Subnet Important?](#-why-is-safescan-subnet-important) From f3e90f71198743f19df1cd902326971f4b51b26a Mon Sep 17 00:00:00 2001 From: LEMSTUDI0 <105062843+LEMSTUDI0@users.noreply.github.com> Date: Mon, 26 Aug 2024 14:16:08 +0200 Subject: [PATCH 068/227] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index dfe4f0fa..e1c24cfd 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) - [👋 Introduction](#-introduction) -- [⚙️ Features](#-FEATURES) +- [⚙️ Features](#-features) - [👁️ Vision](#-vision) - [🌍 Real-world Applications](#-real-world-applications) - [⚠️ Why is SafeScan Subnet Important?](#-why-is-safescan-subnet-important) From a91c700f59e65099073ba39685537292885a08b4 Mon Sep 17 00:00:00 2001 From: LEMSTUDI0 <105062843+LEMSTUDI0@users.noreply.github.com> Date: Mon, 26 Aug 2024 14:17:21 +0200 Subject: [PATCH 069/227] Update README.md From 116efce4d15dc4229206db85067f504e0d886bdd Mon Sep 17 00:00:00 2001 From: LEMSTUDI0 <105062843+LEMSTUDI0@users.noreply.github.com> Date: Mon, 26 Aug 2024 14:17:37 +0200 Subject: [PATCH 070/227] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index e1c24cfd..f3c1746d 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ - [👋 Introduction](#-introduction) - [⚙️ Features](#-features) -- [👁️ Vision](#-vision) +- [👁️ Vision](#-Vision) - [🌍 Real-world Applications](#-real-world-applications) - [⚠️ Why is SafeScan Subnet Important?](#-why-is-safescan-subnet-important) - [📢 Marketing](#-marketing) From dcfc23627211325c9bb429ad17aa0c8c58339e4c Mon Sep 17 00:00:00 2001 From: LEMSTUDI0 <105062843+LEMSTUDI0@users.noreply.github.com> Date: Mon, 26 Aug 2024 14:18:06 +0200 Subject: [PATCH 071/227] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f3c1746d..3a237cc1 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ - [👋 Introduction](#-introduction) - [⚙️ Features](#-features) -- [👁️ Vision](#-Vision) +- [👁️ Vision](#vision) - [🌍 Real-world Applications](#-real-world-applications) - [⚠️ Why is SafeScan Subnet Important?](#-why-is-safescan-subnet-important) - [📢 Marketing](#-marketing) From 0faa0ab5bb2b8dc6f7237137d9121d23aad5bb71 Mon Sep 17 00:00:00 2001 From: LEMSTUDI0 <105062843+LEMSTUDI0@users.noreply.github.com> Date: Mon, 26 Aug 2024 14:18:20 +0200 Subject: [PATCH 072/227] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 3a237cc1..e1c24cfd 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ - [👋 Introduction](#-introduction) - [⚙️ Features](#-features) -- [👁️ Vision](#vision) +- [👁️ Vision](#-vision) - [🌍 Real-world Applications](#-real-world-applications) - [⚠️ Why is SafeScan Subnet Important?](#-why-is-safescan-subnet-important) - [📢 Marketing](#-marketing) From e841f6cd56b8da8730a9d78ad20e7621ad3e938c Mon Sep 17 00:00:00 2001 From: LEMSTUDI0 <105062843+LEMSTUDI0@users.noreply.github.com> Date: Mon, 26 Aug 2024 14:20:47 +0200 Subject: [PATCH 073/227] Update README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index e1c24cfd..49a2f4d3 100644 --- a/README.md +++ b/README.md @@ -48,6 +48,7 @@ This repository contains subnet code to run on Bittensor network. 💸 Self-sustaining economy + # **👁️ VISION** Cancer is one of the most significant challenges of our time, and we believe that AI holds the key to addressing it. However, this solution should be accessible and free for everyone. Machine vision technology has long proven effective in early diagnosis, which is crucial for curing cancer. Yet, until now, it has largely remained in the realm of whitepapers. SAFESCAN is a project dedicated to aggregating and enhancing the best algorithms for detecting various types of cancer and providing free computational power for practical cancer detection. We aim to create open-source products that support cancer diagnosis for both patients and doctors. From cde2ce9cbc378652d2d4cd4fecccec719d89d7d6 Mon Sep 17 00:00:00 2001 From: LEMSTUDI0 <105062843+LEMSTUDI0@users.noreply.github.com> Date: Mon, 26 Aug 2024 14:21:08 +0200 Subject: [PATCH 074/227] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 49a2f4d3..83297baf 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ - [👋 Introduction](#-introduction) - [⚙️ Features](#-features) -- [👁️ Vision](#-vision) +- [👁️ Vision](#vision) - [🌍 Real-world Applications](#-real-world-applications) - [⚠️ Why is SafeScan Subnet Important?](#-why-is-safescan-subnet-important) - [📢 Marketing](#-marketing) From 30ed8885dde372519d0012c45aa6c32da1828420 Mon Sep 17 00:00:00 2001 From: LEMSTUDI0 <105062843+LEMSTUDI0@users.noreply.github.com> Date: Mon, 26 Aug 2024 14:21:37 +0200 Subject: [PATCH 075/227] Update README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 83297baf..4752cedd 100644 --- a/README.md +++ b/README.md @@ -48,8 +48,8 @@ This repository contains subnet code to run on Bittensor network. 💸 Self-sustaining economy - -# **👁️ VISION** + +# **👁️ VISION** Cancer is one of the most significant challenges of our time, and we believe that AI holds the key to addressing it. However, this solution should be accessible and free for everyone. Machine vision technology has long proven effective in early diagnosis, which is crucial for curing cancer. Yet, until now, it has largely remained in the realm of whitepapers. SAFESCAN is a project dedicated to aggregating and enhancing the best algorithms for detecting various types of cancer and providing free computational power for practical cancer detection. We aim to create open-source products that support cancer diagnosis for both patients and doctors. From 3c4807d2a3a724706f06e785d10a9f8aeecae2bd Mon Sep 17 00:00:00 2001 From: LEMSTUDI0 <105062843+LEMSTUDI0@users.noreply.github.com> Date: Mon, 26 Aug 2024 14:22:41 +0200 Subject: [PATCH 076/227] Update README.md --- README.md | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 4752cedd..5a78e9b5 100644 --- a/README.md +++ b/README.md @@ -9,14 +9,14 @@ [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) - [👋 Introduction](#-introduction) -- [⚙️ Features](#-features) +- [⚙️ Features](#features) - [👁️ Vision](#vision) - [🌍 Real-world Applications](#-real-world-applications) - [⚠️ Why is SafeScan Subnet Important?](#-why-is-safescan-subnet-important) - [📢 Marketing](#-marketing) - [💰 Tokenomy & Economy](#-tokenomy--economy) - [👨‍👨‍👦‍👦 Team Composition](#-team-composition) -- [🛣️ Roadmap](#-roadmap) +- [🛣️ Roadmap](#roadmap) - [📊 SETUP WandB (HIGHLY RECOMMENDED - VALIDATORS PLEASE READ)](#-setup-wandb-highly-recommended---validators-please-read) - [👍 Running Validator](#-running-validator) - [⛏️ Running Miner](#-running-miner) @@ -30,6 +30,7 @@ Welcome to Safe Scan Cancer AI Detection, a groundbreaking initiative leveraging This repository contains subnet code to run on Bittensor network. + # **⚙️ FEATURES** 🤗 Validator-friendly code @@ -48,8 +49,8 @@ This repository contains subnet code to run on Bittensor network. 💸 Self-sustaining economy - -# **👁️ VISION** + +# **👁️ VISION** Cancer is one of the most significant challenges of our time, and we believe that AI holds the key to addressing it. However, this solution should be accessible and free for everyone. Machine vision technology has long proven effective in early diagnosis, which is crucial for curing cancer. Yet, until now, it has largely remained in the realm of whitepapers. SAFESCAN is a project dedicated to aggregating and enhancing the best algorithms for detecting various types of cancer and providing free computational power for practical cancer detection. We aim to create open-source products that support cancer diagnosis for both patients and doctors. @@ -138,6 +139,7 @@ Team members: - **@bulubula -** Machine learning engineer - **@Izuael -** Mobile software Engineer + # **🛣️ ROADMAP** Given the complexity of creating a state-of-the-art roleplay LLM, we plan to divide the process into 3 distinct phases. From 6da68eab6c9c72ab24285f940c81e4a501b2971a Mon Sep 17 00:00:00 2001 From: LEMSTUDI0 <105062843+LEMSTUDI0@users.noreply.github.com> Date: Mon, 26 Aug 2024 14:24:24 +0200 Subject: [PATCH 077/227] Update README.md --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 5a78e9b5..61df2ad7 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ - [⚙️ Features](#features) - [👁️ Vision](#vision) - [🌍 Real-world Applications](#-real-world-applications) -- [⚠️ Why is SafeScan Subnet Important?](#-why-is-safescan-subnet-important) +- [⚠️ Why is SAFESCAN Subnet Important?](#why-is-safescan-subnet-important) - [📢 Marketing](#-marketing) - [💰 Tokenomy & Economy](#-tokenomy--economy) - [👨‍👨‍👦‍👦 Team Composition](#-team-composition) @@ -92,6 +92,7 @@ SKIN SCAN app live demo: [https://x.com/SAFESCAN_AI/status/1819351129362149876](https://x.com/SAFESCAN_AI/status/1819351129362149876) + # **⚠️ WHY IS SAFESCAN SUBNET IMPORTANT?** SAFE SCAN harnesses the power of the Bittensor network to address one of the world's most pressing issues: cancer detection. Researchers can contribute to refining detection algorithms and earn TAO, with additional royalties for those whose algorithms are integrated into our software. By focusing on obtaining large datasets, including paid and hard-to-access medical data, we ensure the development of superior models. Our decentralized, transparent system guarantees fair competition and protects against model overfitting. With strong community and validator support, we can expand to create and register standalone software for detecting other types of cancer. From 25179d28e980ae30e56b157bb2ec6a483f97dbbd Mon Sep 17 00:00:00 2001 From: LEMSTUDI0 <105062843+LEMSTUDI0@users.noreply.github.com> Date: Mon, 26 Aug 2024 14:24:55 +0200 Subject: [PATCH 078/227] Update README.md --- README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.md b/README.md index 61df2ad7..c9716141 100644 --- a/README.md +++ b/README.md @@ -30,8 +30,7 @@ Welcome to Safe Scan Cancer AI Detection, a groundbreaking initiative leveraging This repository contains subnet code to run on Bittensor network. - -# **⚙️ FEATURES** +# **⚙️ FEATURES** 🤗 Validator-friendly code From d2aef5e4e2fd0bb73143f6dde64fb2d0841d68b6 Mon Sep 17 00:00:00 2001 From: LEMSTUDI0 <105062843+LEMSTUDI0@users.noreply.github.com> Date: Mon, 26 Aug 2024 14:25:18 +0200 Subject: [PATCH 079/227] Update README.md --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index c9716141..61df2ad7 100644 --- a/README.md +++ b/README.md @@ -30,7 +30,8 @@ Welcome to Safe Scan Cancer AI Detection, a groundbreaking initiative leveraging This repository contains subnet code to run on Bittensor network. -# **⚙️ FEATURES** + +# **⚙️ FEATURES** 🤗 Validator-friendly code From f5c3e059407c3c2b0026ed490292656661e18928 Mon Sep 17 00:00:00 2001 From: Konrad Date: Mon, 26 Aug 2024 16:03:19 +0200 Subject: [PATCH 080/227] plugged in config and integrated pushing model metadata on chain from miner --- cancer_ai/base/miner.py | 54 -------- cancer_ai/chain_models_store.py | 23 ++-- cancer_ai/utils/config.py | 64 +++++++-- cancer_ai/validator/dataset_manager.py | 4 +- neurons/miner.py | 175 ------------------------- neurons/miner3.py | 71 ++++++---- neurons/miner_config.py | 133 ------------------- 7 files changed, 107 insertions(+), 417 deletions(-) delete mode 100644 neurons/miner.py delete mode 100644 neurons/miner_config.py diff --git a/cancer_ai/base/miner.py b/cancer_ai/base/miner.py index 528ae34f..2990e77a 100644 --- a/cancer_ai/base/miner.py +++ b/cancer_ai/base/miner.py @@ -43,27 +43,6 @@ def add_args(cls, parser: argparse.ArgumentParser): def __init__(self, config=None): super().__init__(config=config) - # Warn if allowing incoming requests from anyone. - if not self.config.blacklist.force_validator_permit: - bt.logging.warning( - "You are allowing non-validators to send requests to your miner. This is a security risk." - ) - if self.config.blacklist.allow_non_registered: - bt.logging.warning( - "You are allowing non-registered entities to send requests to your miner. This is a security risk." - ) - # The axon handles request processing, allowing validators to send this miner requests. - self.axon = bt.axon(wallet=self.wallet, config=self.config() if callable(self.config) else self.config) - - # Attach determiners which functions are called when servicing a request. - bt.logging.info(f"Attaching forward function to miner axon.") - self.axon.attach( - forward_fn=self.forward, - blacklist_fn=self.blacklist, - priority_fn=self.priority, - ) - bt.logging.info(f"Axon created: {self.axon}") - # Instantiate runners self.should_exit: bool = False self.is_running: bool = False @@ -71,41 +50,8 @@ def __init__(self, config=None): self.lock = asyncio.Lock() def run(self): - """ - Initiates and manages the main loop for the miner on the Bittensor network. The main loop handles graceful shutdown on keyboard interrupts and logs unforeseen errors. - - This function performs the following primary tasks: - 1. Check for registration on the Bittensor network. - 2. Starts the miner's axon, making it active on the network. - 3. Periodically resynchronizes with the chain; updating the metagraph with the latest network state and setting weights. - - The miner continues its operations until `should_exit` is set to True or an external interruption occurs. - During each epoch of its operation, the miner waits for new blocks on the Bittensor network, updates its - knowledge of the network (metagraph), and sets its weights. This process ensures the miner remains active - and up-to-date with the network's latest state. - - Note: - - The function leverages the global configurations set during the initialization of the miner. - - The miner's axon serves as its interface to the Bittensor network, handling incoming and outgoing requests. - - Raises: - KeyboardInterrupt: If the miner is stopped by a manual interruption. - Exception: For unforeseen errors during the miner's operation, which are logged for diagnosis. - """ - # Check that miner is registered on the network. self.sync() - - # Serve passes the axon information to the network + netuid we are hosting on. - # This will auto-update if the axon port of external ip have changed. - bt.logging.info( - f"Serving miner axon {self.axon} on network: {self.config.subtensor.chain_endpoint} with netuid: {self.config.netuid}" - ) - self.axon.serve(netuid=self.config.netuid, subtensor=self.subtensor) - - # Start starts the miner's axon, making it active on the network. - self.axon.start() - bt.logging.info(f"Miner starting at block: {self.block}") # This loop maintains the miner's operations until intentionally stopped. diff --git a/cancer_ai/chain_models_store.py b/cancer_ai/chain_models_store.py index b20158e6..25b10062 100644 --- a/cancer_ai/chain_models_store.py +++ b/cancer_ai/chain_models_store.py @@ -10,15 +10,11 @@ class ChainMinerModel(BaseModel): """Uniquely identifies a trained model""" - namespace: str = Field( + hf_repo_id: str = Field( description="Namespace where the model can be found. ex. Hugging Face username/org." ) name: str = Field(description="Name of the model.") - epoch: int = Field( - description="The epoch number to submit as your checkpoint to evaluate e.g. 10" - ) - date: datetime.datetime = Field( description="The datetime at which model was pushed to hugging face" ) @@ -30,27 +26,25 @@ class ChainMinerModel(BaseModel): description="Block on which this model was claimed on the chain." ) - hf_repo_id: Optional[str] = Field(description="Hugging Face repo id.") - hf_filename: Optional[str] = Field(description="Hugging Face filename.") - hf_repo_type: Optional[str] = Field(description="Hugging Face repo type.") + # hf_filename: Optional[str] = Field(description="Hugging Face filename.") + # hf_repo_type: Optional[str] = Field(description="Hugging Face repo type.") class Config: arbitrary_types_allowed = True def to_compressed_str(self) -> str: """Returns a compressed string representation.""" - return f"{self.namespace}:{self.name}:{self.epoch}:{self.competition_id}:{self.date}" + return f"{self.hf_repo_id}:{self.name}:{self.date}:{self.competition_id}" @classmethod def from_compressed_str(cls, cs: str) -> Type["ChainMinerModel"]: """Returns an instance of this class from a compressed string representation""" tokens = cs.split(":") return cls( - namespace=tokens[0], + hf_repo_id=tokens[0], name=tokens[1], - epoch=tokens[2] if tokens[2] != "None" else None, - date=tokens[3] if tokens[3] != "None" else None, - competition_id=tokens[4] if tokens[4] != "None" else None, + date=tokens[2] if tokens[2] != "None" else None, + competition_id=tokens[3] if tokens[3] != "None" else None, ) @@ -94,7 +88,6 @@ async def retrieve_model_metadata(self, hotkey: str) -> Optional[ChainMinerModel metadata = run_in_subprocess(partial, 60) if not metadata: return None - print("piwo", metadata["info"]["fields"]) commitment = metadata["info"]["fields"][0] hex_data = commitment[list(commitment.keys())[0]][2:] @@ -110,4 +103,4 @@ async def retrieve_model_metadata(self, hotkey: str) -> Optional[ChainMinerModel return None # The block id at which the metadata is stored model.block = metadata["block"] - return model + return model \ No newline at end of file diff --git a/cancer_ai/utils/config.py b/cancer_ai/utils/config.py index 5967d6f3..8bc73d0f 100644 --- a/cancer_ai/utils/config.py +++ b/cancer_ai/utils/config.py @@ -175,31 +175,73 @@ def add_miner_args(cls, parser): ) parser.add_argument( - "--models.load_model_dir", + "--load_model_dir", type=str, help="Path for for loading the starting model related to a training run.", default="./models", ) parser.add_argument( - "--models.namespace", + "--hf_model_name", type=str, - help="Namespace where the model can be found.", - default="mock-namespace", + help="Name of the model to push to hugging face.", + default="", + ) + + parser.add_argument( + "--action", + choices=["submit", "evaluate", "upload"], + default="submit", ) parser.add_argument( - "--models.model_name", + "--model_path", type=str, - help="Name of the model to push to hugging face.", - default="mock-name", + help="Path to ONNX model, used for evaluation", + default="", ) parser.add_argument( - "--models.epoch_checkpoint", - type=int, - help="The epoch number to submit as your checkpoint to evaluate e.g. 10", - default=10, + "--competition_id", + type=str, + help="Competition ID", + default="melanoma-1", + ) + + parser.add_argument( + "--dataset_dir", + type=str, + help="Path for storing datasets.", + default="./datasets", + ) + + parser.add_argument( + "--hf_repo_id", + type=str, + # required=False, + help="Hugging Face model repository ID", + default="", + ) + + parser.add_argument( + "--hf_token", + type=str, + help="Hugging Face API token", + default="", + ) + + parser.add_argument( + "--clean_after_run", + action="store_true", + help="Whether to clean up (dataset, temporary files) after running", + default=False, + ) + + parser.add_argument( + "--code_directory", + type=str, + help="Path to code directory", + default=".", ) def add_validator_args(cls, parser): diff --git a/cancer_ai/validator/dataset_manager.py b/cancer_ai/validator/dataset_manager.py index f26d42b6..900ab59c 100644 --- a/cancer_ai/validator/dataset_manager.py +++ b/cancer_ai/validator/dataset_manager.py @@ -17,7 +17,7 @@ class DatasetManagerException(Exception): class DatasetManager(SerializableManager): def __init__( - self, config, competition_id: str, hf_repo_id: str, hf_filename: str, hf_repo_type: str + self, config, hf_repo_id: str, hf_filename: str, hf_repo_type: str ) -> None: """ Initializes a new instance of the DatasetManager class. @@ -32,7 +32,7 @@ def __init__( None """ self.config = config - self.competition_id = competition_id + self.competition_id = config.competition_id self.hf_repo_id = hf_repo_id self.hf_filename = hf_filename self.hf_repo_type = hf_repo_type diff --git a/neurons/miner.py b/neurons/miner.py deleted file mode 100644 index 9360a4af..00000000 --- a/neurons/miner.py +++ /dev/null @@ -1,175 +0,0 @@ - -import time -import typing -import bittensor as bt -import datetime as dt -import os -import datetime -import asyncio -import cancer_ai - -# import base miner class which takes care of most of the boilerplate -from cancer_ai.base.miner import BaseMinerNeuron -from cancer_ai.chain_models_store import ChainMinerModel, ChainModelMetadataStore - - -class Miner(BaseMinerNeuron): - """ - Your miner neuron class. You should use this class to define your miner's behavior. In particular, you should replace the forward function with your own logic. You may also want to override the blacklist and priority functions according to your needs. - - This class inherits from the BaseMinerNeuron class, which in turn inherits from BaseNeuron. The BaseNeuron class takes care of routine tasks such as setting up wallet, subtensor, metagraph, logging directory, parsing config, etc. You can override any of the methods in BaseNeuron if you need to customize the behavior. - - This class provides reasonable default behavior for a miner such as blacklisting unrecognized hotkeys, prioritizing requests based on stake, and forwarding requests to the forward function. If you need to define custom - """ - - def __init__(self, config=None): - super(Miner, self).__init__(config=config) - - self.metadata_store = ChainModelMetadataStore(subtensor=self.subtensor, subnet_uid=163, wallet=self.wallet) - - asyncio.run(self.store_and_retrieve_metadata_on_chain("mock_competition")) - - async def forward( - self, synapse: cancer_ai.protocol.Dummy - ) -> cancer_ai.protocol.Dummy: - """ - Processes the incoming 'Dummy' synapse by performing a predefined operation on the input data. - This method should be replaced with actual logic relevant to the miner's purpose. - - Args: - synapse (template.protocol.Dummy): The synapse object containing the 'dummy_input' data. - - Returns: - template.protocol.Dummy: The synapse object with the 'dummy_output' field set to twice the 'dummy_input' value. - - The 'forward' function is a placeholder and should be overridden with logic that is appropriate for - the miner's intended operation. This method demonstrates a basic transformation of input data. - """ - # TODO(developer): Replace with actual implementation logic. - synapse.dummy_output = synapse.dummy_input * 2 - return synapse - - async def blacklist( - self, synapse: cancer_ai.protocol.Dummy - ) -> typing.Tuple[bool, str]: - """ - Determines whether an incoming request should be blacklisted and thus ignored. Your implementation should - define the logic for blacklisting requests based on your needs and desired security parameters. - - Blacklist runs before the synapse data has been deserialized (i.e. before synapse.data is available). - The synapse is instead contracted via the headers of the request. It is important to blacklist - requests before they are deserialized to avoid wasting resources on requests that will be ignored. - - Args: - synapse (template.protocol.Dummy): A synapse object constructed from the headers of the incoming request. - - Returns: - Tuple[bool, str]: A tuple containing a boolean indicating whether the synapse's hotkey is blacklisted, - and a string providing the reason for the decision. - - This function is a security measure to prevent resource wastage on undesired requests. It should be enhanced - to include checks against the metagraph for entity registration, validator status, and sufficient stake - before deserialization of synapse data to minimize processing overhead. - - Example blacklist logic: - - Reject if the hotkey is not a registered entity within the metagraph. - - Consider blacklisting entities that are not validators or have insufficient stake. - - In practice it would be wise to blacklist requests from entities that are not validators, or do not have - enough stake. This can be checked via metagraph.S and metagraph.validator_permit. You can always attain - the uid of the sender via a metagraph.hotkeys.index( synapse.dendrite.hotkey ) call. - - Otherwise, allow the request to be processed further. - """ - - if synapse.dendrite is None or synapse.dendrite.hotkey is None: - bt.logging.warning("Received a request without a dendrite or hotkey.") - return True, "Missing dendrite or hotkey" - - # TODO(developer): Define how miners should blacklist requests. - uid = self.metagraph.hotkeys.index(synapse.dendrite.hotkey) - if ( - not self.config.blacklist.allow_non_registered - and synapse.dendrite.hotkey not in self.metagraph.hotkeys - ): - # Ignore requests from un-registered entities. - bt.logging.trace( - f"Blacklisting un-registered hotkey {synapse.dendrite.hotkey}" - ) - return True, "Unrecognized hotkey" - - if self.config.blacklist.force_validator_permit: - # If the config is set to force validator permit, then we should only allow requests from validators. - if not self.metagraph.validator_permit[uid]: - bt.logging.warning( - f"Blacklisting a request from non-validator hotkey {synapse.dendrite.hotkey}" - ) - return True, "Non-validator hotkey" - - bt.logging.trace( - f"Not Blacklisting recognized hotkey {synapse.dendrite.hotkey}" - ) - return False, "Hotkey recognized!" - - async def priority(self, synapse: cancer_ai.protocol.Dummy) -> float: - """ - The priority function determines the order in which requests are handled. More valuable or higher-priority - requests are processed before others. You should design your own priority mechanism with care. - - This implementation assigns priority to incoming requests based on the calling entity's stake in the metagraph. - - Args: - synapse (template.protocol.Dummy): The synapse object that contains metadata about the incoming request. - - Returns: - float: A priority score derived from the stake of the calling entity. - - Miners may receive messages from multiple entities at once. This function determines which request should be - processed first. Higher values indicate that the request should be processed first. Lower values indicate - that the request should be processed later. - - Example priority logic: - - A higher stake results in a higher priority value. - """ - if synapse.dendrite is None or synapse.dendrite.hotkey is None: - bt.logging.warning("Received a request without a dendrite or hotkey.") - return 0.0 - - # TODO(developer): Define how miners should prioritize requests. - caller_uid = self.metagraph.hotkeys.index( - synapse.dendrite.hotkey - ) # Get the caller index. - priority = float( - self.metagraph.S[caller_uid] - ) # Return the stake as the priority. - bt.logging.trace( - f"Prioritizing {synapse.dendrite.hotkey} with value: {priority}" - ) - return priority - - async def store_and_retrieve_metadata_on_chain(self, competition: str) -> None: - """ - PoC function to integrate with the structured business logic - """ - - model_id = ChainMinerModel(namespace=self.config.models.namespace, name=self.config.models.model_name, epoch=self.config.models.epoch_checkpoint, - date=datetime.datetime.now(), competition_id=competition, block=None) - - await self.metadata_store.store_model_metadata(model_id) - bt.logging.success(f"Model successfully pushed model metadata on chain. Model ID: {model_id}") - - time.sleep(10) - - model_metadata = await self.metadata_store.retrieve_model_metadata(self.wallet.hotkey.ss58_address) - - time.sleep(10) - print("Model Metadata name: ", model_metadata.id.name) - - -# This is the main function, which runs the miner. -if __name__ == "__main__": - with Miner() as miner: - while True: - bt.logging.info(f"Miner running... {time.time()}") - time.sleep(5) - diff --git a/neurons/miner3.py b/neurons/miner3.py index 169218c1..77568ca0 100644 --- a/neurons/miner3.py +++ b/neurons/miner3.py @@ -1,41 +1,60 @@ -import argparse -import sys import asyncio -from typing import Optional import bittensor as bt from dotenv import load_dotenv from huggingface_hub import HfApi +import huggingface_hub import onnx +import cancer_ai +import typing +import datetime -from neurons.miner_config import get_config, set_log_formatting from cancer_ai.validator.utils import ModelType, run_command from cancer_ai.validator.model_run_manager import ModelRunManager, ModelInfo from cancer_ai.validator.dataset_manager import DatasetManager from cancer_ai.validator.model_manager import ModelManager -from datetime import datetime +from cancer_ai.base.miner import BaseMinerNeuron +from cancer_ai.chain_models_store import ChainMinerModel, ChainModelMetadataStore -class MinerManagerCLI: - def __init__(self, config: bt.config): - self.config = config +class MinerManagerCLI(BaseMinerNeuron): + def __init__(self, config=None): + super(MinerManagerCLI, self).__init__(config=config) + self.metadata_store = ChainModelMetadataStore(subtensor=self.subtensor, + subnet_uid=self.config.netuid, wallet=self.wallet) self.hf_api = HfApi() + # TODO: Dive into BaseNeuron to switch off requirement to implement legacy methods, for now they are mocked. + async def forward( + self, synapse: cancer_ai.protocol.Dummy + ) -> cancer_ai.protocol.Dummy: + ... + + async def blacklist( + self, synapse: cancer_ai.protocol.Dummy + ) -> typing.Tuple[bool, str]: + ... + + async def priority(self, synapse: cancer_ai.protocol.Dummy) -> float: + ... + async def upload_to_hf(self) -> None: """Uploads model and code to Hugging Face.""" bt.logging.info("Uploading model to Hugging Face.") - now_str = datetime.now().strftime("%Y-%m-%d_%H-%M-%S") path = self.hf_api.upload_file( path_or_fileobj=self.config.model_path, - path_in_repo=f"{now_str}-{self.config.competition_id}.onnx", + path_in_repo=f"{self.config.competition_id}-{self.config.hf_model_name}.onnx", repo_id=self.config.hf_repo_id, repo_type="model", + token=self.config.hf_token, ) + bt.logging.info("Uploading code to Hugging Face.") path = self.hf_api.upload_file( path_or_fileobj=f"{self.config.code_directory}/code.zip", - path_in_repo=f"{now_str}-{self.config.competition_id}.zip", + path_in_repo=f"{self.config.competition_id}-{self.config.hf_model_name}.zip", repo_id=self.config.hf_repo_id, repo_type="model", + token=self.config.hf_token, ) bt.logging.info(f"Uploaded model to Hugging Face: {path}") @@ -57,7 +76,6 @@ async def evaluate_model(self) -> None: ) dataset_manager = DatasetManager( self.config, - self.config.competition_id, "safescanai/test_dataset", "skin_melanoma.zip", "dataset", @@ -68,9 +86,6 @@ async def evaluate_model(self) -> None: model_predictions = await run_manager.run(pred_x) - print(pred_y) - print(model_predictions) - if self.config.clean_after_run: dataset_manager.delete_dataset() @@ -80,16 +95,20 @@ async def compress_code(self) -> str: f"zip {self.config.code_directory}/code.zip {self.config.code_directory}/*" ) return f"{self.config.code_directory}/code.zip" - + async def submit_model(self) -> None: - bt.logging.info( - f"Initializing connection with Bittensor subnet {self.config.netuid} - Safe-Scan Project" - ) - bt.logging.info(f"Subtensor network: {self.config.subtensor.network}") - bt.logging.info(f"Wallet hotkey: {self.config.wallet.hotkey.ss58_address}") - wallet = self.wallet - subtensor = self.subtensor - metagraph = self.metagraph + # Check if the required model and files are present in hugging face repo + filenames = [self.config.hf_model_name + ".onnx", self.config.hf_model_name + ".zip"] + for file in filenames: + if not huggingface_hub.file_exists(repo_id=self.config.hf_repo_id, filename=file, token=self.config.hf_token): + bt.logging.error(f"{file} not found in Hugging Face repo") + return + bt.logging.info("Model and code found in Hugging Face repo") + + # Push model metadata to chain + model_id = ChainMinerModel(hf_repo_id=self.config.hf_repo_id, name=self.config.hf_model_name, date=datetime.datetime.now(), competition_id=self.config.competition_id, block=None) + await self.metadata_store.store_model_metadata(model_id) + bt.logging.success(f"Successfully pushed model metadata on chain. Model ID: {model_id}") async def main(self) -> None: bt.logging(config=self.config) @@ -111,8 +130,6 @@ async def main(self) -> None: if __name__ == "__main__": - config = get_config() - set_log_formatting() load_dotenv() - cli_manager = MinerManagerCLI(config) + cli_manager = MinerManagerCLI() asyncio.run(cli_manager.main()) diff --git a/neurons/miner_config.py b/neurons/miner_config.py deleted file mode 100644 index f82c3859..00000000 --- a/neurons/miner_config.py +++ /dev/null @@ -1,133 +0,0 @@ -import argparse - -from colorama import init, Fore, Back, Style -import bittensor as bt -from bittensor.btlogging import format - - -help = """ -How to run it: - -python3 neurons/miner2.py \ - evaluate \ - --logging.debug \ - --model_path /path/to/model \ - --competition_id "your_competition_id" - -python3 upload neurons/miner2.py \ - --model_path /path/to/model - --hf_repo_id "hf_org_id/your_hf_repo_id" - -python3 neurons/miner2.py \ - submit \ - --netuid 163 \ - --subtensor.network test \ - --wallet.name miner \ - --wallet.hotkey hot_validator \ - --model_path /path/to/model -""" -import argparse -import bittensor as bt - -import argparse - - -def set_log_formatting() -> None: - """Override bittensor logging formats.""" - - - format.LOG_TRACE_FORMATS = { - level: f"{Fore.BLUE}%(asctime)s{Fore.RESET}" - f" | {Style.BRIGHT}{color}%(levelname)s{Fore.RESET}{Back.RESET}{Style.RESET_ALL}" - f" |%(message)s" - for level, color in format.log_level_color_prefix.items() - } - - format.DEFAULT_LOG_FORMAT = ( - f"{Fore.BLUE}%(asctime)s{Fore.RESET} | " - f"{Style.BRIGHT}{Fore.WHITE}%(levelname)s{Style.RESET_ALL} | " - "%(message)s" - ) - - format.DEFAULT_TRACE_FORMAT = ( - f"{Fore.BLUE}%(asctime)s{Fore.RESET} | " - f"{Style.BRIGHT}{Fore.WHITE}%(levelname)s{Style.RESET_ALL} | " - f" %(message)s" - ) - - -def get_config() -> bt.config: - main_parser = argparse.ArgumentParser() - - main_parser.add_argument( - "--action", - choices=["submit", "evaluate", "upload"], - # required=True, - default="evaluate", - ) - main_parser.add_argument( - "--model_path", - type=str, - # required=True, - help="Path to ONNX model, used for evaluation", - default="neurons/simple_cnn_model.onnx", - ) - main_parser.add_argument( - "--competition_id", - type=str, - # required=True, - help="Competition ID", - default="melanoma-1", - ) - - main_parser.add_argument( - "--dataset_dir", - type=str, - help="Path for storing datasets.", - default="./datasets", - ) - # Subparser for upload command - - main_parser.add_argument( - "--hf_repo_id", - type=str, - required=False, - help="Hugging Face model repository ID", - default="eatcats/melanoma-test", - ) - - main_parser.add_argument( - "--clean-after-run", - action="store_true", - help="Whether to clean up (dataset, temporary files) after running", - default=False, - ) - main_parser.add_argument( - "--code-directory", - type=str, - help="Path to code directory", - default=".", - ) - - # Add additional args from bt modules - bt.wallet.add_args(main_parser) - bt.subtensor.add_args(main_parser) - bt.logging.add_args(main_parser) - - # Parse the arguments and return the config - # config = bt.config(main_parser) - # parsed = main_parser.parse_args() - # config = bt.config(main_parser) - - - config = main_parser.parse_args() - config.logging_dir = "./" - config.record_log = True - config.trace = True - config.debug = False - return config - - -if __name__ == "__main__": - config = get_config() - print(config) From 191643557f3390edf4fe24cffa534dd2ae932940 Mon Sep 17 00:00:00 2001 From: Konrad Date: Tue, 27 Aug 2024 21:46:48 +0200 Subject: [PATCH 081/227] some adjustments --- cancer_ai/utils/config.py | 1 - neurons/{miner3.py => miner.py} | 0 2 files changed, 1 deletion(-) rename neurons/{miner3.py => miner.py} (100%) diff --git a/cancer_ai/utils/config.py b/cancer_ai/utils/config.py index 8bc73d0f..c9f51f68 100644 --- a/cancer_ai/utils/config.py +++ b/cancer_ai/utils/config.py @@ -218,7 +218,6 @@ def add_miner_args(cls, parser): parser.add_argument( "--hf_repo_id", type=str, - # required=False, help="Hugging Face model repository ID", default="", ) diff --git a/neurons/miner3.py b/neurons/miner.py similarity index 100% rename from neurons/miner3.py rename to neurons/miner.py From c1555a8211fe028bbae3043419940f62d0795ebc Mon Sep 17 00:00:00 2001 From: Wojtek Jurkowlaniec Date: Wed, 28 Aug 2024 02:18:46 +0200 Subject: [PATCH 082/227] Update README.md --- README.md | 47 ----------------------------------------------- 1 file changed, 47 deletions(-) diff --git a/README.md b/README.md index 61df2ad7..a239564e 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,6 @@ - [💰 Tokenomy & Economy](#-tokenomy--economy) - [👨‍👨‍👦‍👦 Team Composition](#-team-composition) - [🛣️ Roadmap](#roadmap) -- [📊 SETUP WandB (HIGHLY RECOMMENDED - VALIDATORS PLEASE READ)](#-setup-wandb-highly-recommended---validators-please-read) - [👍 Running Validator](#-running-validator) - [⛏️ Running Miner](#-running-miner) - [🚀 Get invloved](#-get-involved) @@ -164,52 +163,6 @@ Given the complexity of creating a state-of-the-art roleplay LLM, we plan to div - [ ] Make competitions for breast cancer # **📊 SETUP WandB (HIGHLY RECOMMENDED - VALIDATORS PLEASE READ)** - -Before running your miner and validator, you may also choose to set up Weights & Biases (WANDB). It is a popular tool for tracking and visualizing machine learning experiments, and we use it for logging and tracking key metrics across miners and validators, all of which is available publicly [here](https://wandb.ai/shr1ftyy/sturdy-subnet/table?nw=nwusershr1ftyy). We ***highly recommend*** validators use WandB, as it allows subnet developers and miners to diagnose issues more quickly and effectively, say, in the event a validator were to be set abnormal weights. WandB logs are collected by default and done so in an anonymous fashion, but we recommend setting up an account to make it easier to differentiate between validators when searching for runs on our dashboard. If you would *not* like to run WandB, you can do so by adding the flag `--wandb.off` when running your miner/validator. - -Before getting started, as mentioned previously, you'll first need to [register](https://wandb.ai/login?signup=true) for a WANDB account, and then set your API key on your system. Here's a step-by-step guide on how to do this on Ubuntu: - -**Step 1: Installation of WANDB** - -Before logging in, make sure you have the WANDB Python package installed. If you haven't installed it yet, you can do so using pip: - - -``` -# Should already be installed with the sturdy repo -pip install wandb -``` - - -**Step 2: Obtain Your API Key** - -1. Log in to your Weights & Biases account through your web browser. -2. Go to your account settings, usually accessible from the top right corner under your profile. -3. Find the section labeled "API keys". -4. Copy your API key. It's a long string of characters unique to your account. - -**Step 3: Setting Up the API Key in Ubuntu** - -To configure your WANDB API key on your Ubuntu machine, follow these steps: - -1. **Log into WANDB**: Run the following command in the terminal: - - ``` - wandb login - ``` - -2. **Enter Your API Key**: When prompted, paste the API key you copied from your WANDB account settings. - - After pasting your API key, press `Enter`. - - WANDB should display a message confirming that you are logged in. -3. **Verifying the Login**: To verify that the API key was set correctly, you can start a small test script in Python that uses WANDB. If everything is set up correctly, the script should run without any authentication errors. -4. **Setting API Key Environment Variable (Optional)**: If you prefer not to log in every time, you can set your API key as an environment variable in your `~/.bashrc` or `~/.bash_profile` file: - - ``` - echo 'export WANDB_API_KEY=your_api_key' >> ~/.bashrc - source ~/.bashrc - ``` - - Replace `your_api_key` with the actual API key. This method automatically authenticates you with WandB every time you open a new terminal session. - # **👍 RUNNING VALIDATOR** ... From 8ce54302ed6f122ebfe6ae636492aefdd3ec6afd Mon Sep 17 00:00:00 2001 From: LEMSTUDI0 <105062843+LEMSTUDI0@users.noreply.github.com> Date: Wed, 28 Aug 2024 12:46:27 +0200 Subject: [PATCH 083/227] Update README.md --- README.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index a239564e..0154a031 100644 --- a/README.md +++ b/README.md @@ -109,10 +109,13 @@ However, brand recognition is just the beginning. Our marketing strategy will fo **🪙 UNIQUE TOKENOMY** Our tokenomics are uniquely designed to drive research and development of new algorithms while also supporting real-life applications. +**Competitions** Safe Scan organizes ongoing competitions focused on cancer detection using machine learning, providing a structured environment for participants to develop and test their models. -**Incentives**: Miners with the best-performing algorithms in our ongoing competitions are rewarded through our leaderboard system. The top-ranked miner receives significant incentives, promoting continuous improvement and innovation. +You can find comprehensive details about competition scheduling, dataset release, model submission, evaluation, configuration, and development tools here: [LINK] -**Royalties**: Miners whose algorithms are integrated into our app and software for real-life cancer detection applications earn additional 1% of emission royalties. This ensures ongoing motivation for developers to create cutting-edge solutions that contribute to our mission of saving lives. +**Incentives**: The winner of each competition receives the entire reward pool for that specific competition. The reward pool is determined by the total emission allocated for miners, divided by the number of competitions being held. + +If a miner stays at the top position for more than 30 days, their rewards start to decrease gradually. Every 7 days after the initial 30 days, their share of the rewards decreases by 10%. This reduction continues until their share reaches a minimum of 10% of the original reward. **📈 SELF-SUSTAINING ECONOMY** From 46743f5a605abe3ad0432a684189ae57acced8a9 Mon Sep 17 00:00:00 2001 From: Wojtek Jurkowlaniec Date: Wed, 28 Aug 2024 14:09:54 +0200 Subject: [PATCH 084/227] Competition readme and script for downloading dataset (#29) * Competition readme and script for downloading dataset * Update COMPETITIONS.md * Update COMPETITIONS.md * Update COMPETITIONS.md * Update COMPETITIONS.md --------- Co-authored-by: LEMSTUDI0 <105062843+LEMSTUDI0@users.noreply.github.com> --- cancer_ai/DOCS/COMPETITIONS.md | 75 +++++++++++++++++++++++ cancer_ai/DOCS/competitions/1-MELANOMA.md | 17 +++++ neurons/miner3.py | 5 ++ scripts/get_dataset.py | 32 ++++++++++ 4 files changed, 129 insertions(+) create mode 100644 cancer_ai/DOCS/COMPETITIONS.md create mode 100644 cancer_ai/DOCS/competitions/1-MELANOMA.md create mode 100644 scripts/get_dataset.py diff --git a/cancer_ai/DOCS/COMPETITIONS.md b/cancer_ai/DOCS/COMPETITIONS.md new file mode 100644 index 00000000..77093dc9 --- /dev/null +++ b/cancer_ai/DOCS/COMPETITIONS.md @@ -0,0 +1,75 @@ + + + + + +# Safe Scan: Machine Learning Competitions for Cancer Detection + +Welcome to **Safe Scan**, a platform dedicated to organizing machine learning competitions focused on cancer detection. Our goal is to foster innovation in developing accurate and efficient models for cancer detection using machine learning. Here, you can find all the details needed to participate, submit your models, and understand the evaluation process. + +## Table of Contents + +1. [Overview](#overview) +2. [Competition Schedule](#competition-schedule) +3. [Dataset and Model Submission](#dataset-and-model-submission) +4. [Evaluation and Scoring](#evaluation-and-scoring) +5. [Configuration and Development](#configuration-and-development) +6. [Command-Line Interface (CLI) Tools](#command-line-interface-cli-tools) +7. [Communication Channels](#communication-channels) +8. [Contribute](#contribute) + +## Overview + +Safe Scan organizes continuous competitions focused on cancer detection using machine learning. These competitions aim to advance the field by providing participants with the opportunity to develop and test their models in a structured environment. + +## Competition Schedule + +- **Frequency**: Competitions are held multiple times a day, at specific hours, continuously. This allows participants to join at different times that suit them best. +- **Timed Events**: Each competition starts with a dataset release 5 minutes before testing, providing a short window for participants to prepare. +- **Testing and Evaluation**: Models are evaluated immediately after each test, ensuring a quick feedback loop for participants. + +## Dataset and Model Submission + +- **Dataset Release**: A new dataset is provided for each competition, which is released exactly 5 minutes before testing begins. This dataset is used for training the models. +- **Model Submission**: Participants, referred to as "miners," are required to submit their trained models at the end of each competition. + - **Format**: All models must be in ONNX format. This ensures uniform testing and allows for broad deployment options, including on mobile and web platforms. + - **Training Code**: Each submission should include the code used for training the model to ensure transparency and reproducibility. + - **Upload Process**: Models are uploaded to Hugging Face at the end of each test. Miners then submit the Hugging Face repository link on the blockchain for evaluation by validators. + +## Evaluation and Scoring + +- **Independent Evaluation**: Each validator independently evaluates the submitted models according to predefined criteria. +- **Scoring Mechanism**: Detailed scoring mechanisms are outlined in the [competition guidelines](https://huggingface.co/spaces/safescanai/dashboard) and [DOCS](/DOCS/competitions). Validators run scheduled competitions and assess the models based on these criteria. +- **Winning Criteria**: The best-performing model, according to the evaluation metrics, is declared the winner of the competition. +- **Rewards**: The winner receives the full emission for that competition, divided by the number of competitions held. +- **Rewards time decay**: If a miner stays at the top position for more than 30 days, their rewards start to decrease gradually. Every 7 days after the initial 30 days, their share of the rewards decreases by 10%. This reduction continues until their share reaches a minimum of 10% of the original reward. + +## Configuration and Development + +- **Competition Configuration**: Each competition is configured through a `competition_config.json` file. This file defines all parameters and rules for the competition and is used by both miners and validators. +- **Tracking Changes**: Changes to the competition configuration are tracked via a GitHub issue tracker, ensuring transparency and allowing for community input. +- **Software Lifecycle**: The project follows a structured software lifecycle, including Git flow and integration testing. This ensures robust development practices and encourages community contributions. + +## Command-Line Interface (CLI) Tools + +- **Local Testing**: Miners are provided with an easy-to-use command-line interface (CLI) for local testing of their models. This tool helps streamline the process of testing models, uploading to Hugging Face, and submitting to the competition. +- **Automated Data Retrieval**: Code for automating the retrieval of training data for each competition is available to integrate with the model training process. The script is defined in [scripts/get_dataset.py](/scripts/get_dataset.py). + +## Communication Channels + +Stay connected and up-to-date with the latest news, discussions, and support: + +- **Discord**: Join our [Safe Scan Discord channel](https://discord.gg/rbBu7WuZ) and the Bittensor Discord in the #safescan channel for real-time updates and community interaction. +- **Dashboard**: Access the competition dashboard on [Hugging Face](https://huggingface.co/spaces/safescanai/dashboard). +- **Blog**: Visit our [blog](https://safe-scan.ai/news/) for news and updates. +- **Twitter/X**: Follow us on [Twitter/X](https://x.com/SAFESCAN_AI) for announcements and highlights. +- **Email**: Contact us directly at [info@safescanai.ai](mailto:info@safescanai.ai) for any inquiries or support. + +## Contribute + +We welcome contributions to this project! Whether you're interested in improving our codebase, adding new features, or enhancing documentation, your involvement is valued. To contribute: + +- Follow our software lifecycle and Git flow processes. +- Ensure all code changes pass integration testing. +- Contact us on our [Safe Scan Discord channel](https://discord.gg/rbBu7WuZ) for more details on how to get started. + diff --git a/cancer_ai/DOCS/competitions/1-MELANOMA.md b/cancer_ai/DOCS/competitions/1-MELANOMA.md new file mode 100644 index 00000000..33052f47 --- /dev/null +++ b/cancer_ai/DOCS/competitions/1-MELANOMA.md @@ -0,0 +1,17 @@ +# Melanoma competition + +Training starts on `ENTER DATE` + + +## Overview + +### Dataset + +## Evaluation criteria + +### recorded metrics + +### Scoring mechanism + +## Links to dataset + diff --git a/neurons/miner3.py b/neurons/miner3.py index 169218c1..a508cbdd 100644 --- a/neurons/miner3.py +++ b/neurons/miner3.py @@ -111,7 +111,12 @@ async def main(self) -> None: if __name__ == "__main__": + from types import SimpleNamespace config = get_config() + config = { + "dataset_dir": "./data", + } + config = SimpleNamespace( **config) set_log_formatting() load_dotenv() cli_manager = MinerManagerCLI(config) diff --git a/scripts/get_dataset.py b/scripts/get_dataset.py new file mode 100644 index 00000000..6324284b --- /dev/null +++ b/scripts/get_dataset.py @@ -0,0 +1,32 @@ +from cancer_ai.validator.dataset_manager import DatasetManager +from types import SimpleNamespace, Tuple, List +config = { + "dataset_dir": "./data", +} +config = SimpleNamespace( **config) + +NAME_OF_COMPETITION="melanoma-1" +HUGGINFACE_DATASET_ID="safescanai/test_dataset" +HUGGINFACE_FILEPATH="skin_melanoma.zip" +HUGINGFACE_REPO_TYPE="dataset" + +# can be also taken from competition_config.py + + + +async def get_training_data() -> Tuple[List, List]: + + dataset_manager = DatasetManager( + config, + NAME_OF_COMPETITION, + HUGGINFACE_DATASET_ID, + HUGGINFACE_FILEPATH, + HUGINGFACE_REPO_TYPE + ) + await dataset_manager.prepare_dataset() + + return await dataset_manager.get_data() + +if __name__ == "__main__": + import asyncio + pred_x, pred_y = asyncio.run(get_training_data()) From 468b45b6bff2649cad406ce9d06bd0ab6d75ef45 Mon Sep 17 00:00:00 2001 From: Wojtek Jurkowlaniec Date: Wed, 28 Aug 2024 14:28:44 +0200 Subject: [PATCH 085/227] fix readme (#38) --- {cancer_ai/DOCS/competitions => DOCS}/1-MELANOMA.md | 0 {cancer_ai/DOCS => DOCS}/COMPETITIONS.md | 0 README.md | 2 +- 3 files changed, 1 insertion(+), 1 deletion(-) rename {cancer_ai/DOCS/competitions => DOCS}/1-MELANOMA.md (100%) rename {cancer_ai/DOCS => DOCS}/COMPETITIONS.md (100%) diff --git a/cancer_ai/DOCS/competitions/1-MELANOMA.md b/DOCS/1-MELANOMA.md similarity index 100% rename from cancer_ai/DOCS/competitions/1-MELANOMA.md rename to DOCS/1-MELANOMA.md diff --git a/cancer_ai/DOCS/COMPETITIONS.md b/DOCS/COMPETITIONS.md similarity index 100% rename from cancer_ai/DOCS/COMPETITIONS.md rename to DOCS/COMPETITIONS.md diff --git a/README.md b/README.md index 0154a031..c3780baa 100644 --- a/README.md +++ b/README.md @@ -111,7 +111,7 @@ However, brand recognition is just the beginning. Our marketing strategy will fo Our tokenomics are uniquely designed to drive research and development of new algorithms while also supporting real-life applications. **Competitions** Safe Scan organizes ongoing competitions focused on cancer detection using machine learning, providing a structured environment for participants to develop and test their models. -You can find comprehensive details about competition scheduling, dataset release, model submission, evaluation, configuration, and development tools here: [LINK] +You can find comprehensive details about competition scheduling, dataset release, model submission, evaluation, configuration, and development tools here: [COMPETITION README](DOCS/COMPETITIONS.md) **Incentives**: The winner of each competition receives the entire reward pool for that specific competition. The reward pool is determined by the total emission allocated for miners, divided by the number of competitions being held. From 80f554384efc21c9e739e36c37d938f81bd7a4a8 Mon Sep 17 00:00:00 2001 From: Wojtek Jurkowlaniec Date: Wed, 28 Aug 2024 14:43:58 +0200 Subject: [PATCH 086/227] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index c3780baa..aca56b18 100644 --- a/README.md +++ b/README.md @@ -176,7 +176,7 @@ Given the complexity of creating a state-of-the-art roleplay LLM, we plan to div # **🚀 GET INVOLVED** -1. Visit our [![GitHub](https://img.shields.io/badge/GitHub-100000?style=for-the-badge&logo=github&logoColor=white)](https://github.com/safe-scan-ai/cancer-ai-3) to explore the code behind SAFE SCAN. +1. Visit our [![GitHub](https://img.shields.io/badge/GitHub-100000?style=for-the-badge&logo=github&logoColor=white)](https://github.com/safe-scan-ai/cancer-ai) to explore the code behind SAFE SCAN. 2. Join our [![Discord](https://img.shields.io/discord/308323056592486420.svg)](https://discord.com/channels/1259812760280236122/1262383307832823809) to stay updated and engage with the team. From 6b2b1cfece550a65f41d9e9d2e67a16ffd1b0bd6 Mon Sep 17 00:00:00 2001 From: LEMSTUDI0 <105062843+LEMSTUDI0@users.noreply.github.com> Date: Wed, 28 Aug 2024 17:13:43 +0200 Subject: [PATCH 087/227] Update README.md --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index aca56b18..ff949a65 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,12 @@ [![Discord Chat](https://img.shields.io/discord/308323056592486420.svg)]([https://discord.gg/bittensor](https://discord.com/channels/1259812760280236122/1262383307832823809)) [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) +[www.SAFE-SCAN.ai](https://www.safe-scan.ai)      [www.SKIN-SCAN.ai](https://www.skin-scan.ai)      [Follow us on X](https://x.com/SAFESCAN_AI) + + +# **📋 TABLE OF CONTENT** + + - [👋 Introduction](#-introduction) - [⚙️ Features](#features) - [👁️ Vision](#vision) From 4becee9639d1d2a21c64da53ef1d9202304e45a3 Mon Sep 17 00:00:00 2001 From: Konrad Date: Wed, 28 Aug 2024 20:21:27 +0200 Subject: [PATCH 088/227] initial rewarding model --- cancer_ai/rewards.py | 50 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 cancer_ai/rewards.py diff --git a/cancer_ai/rewards.py b/cancer_ai/rewards.py new file mode 100644 index 00000000..8bbbfa89 --- /dev/null +++ b/cancer_ai/rewards.py @@ -0,0 +1,50 @@ + +class Rewarder(): + def __init__(self, config=None): + # competition_id to (hotkey_uid, days_as_leader) + self.competitions_leaders = { + 1: (1, 21), + 2: (2, 3), + 3: (3, 28), + } + self.scores = {} + + def calculate_score_for_winner(self, competition_id: int, hotkey_uid: int) -> tuple[float, bool]: + num_competitions = len(self.competitions_leaders) + base_share = 1/num_competitions + + # check if current hotkey is already a leader + if self.competitions_leaders[competition_id][0] == hotkey_uid: + days_as_leader = self.competitions_leaders[competition_id][1] + else: + days_as_leader = 0 + self.competitions_leaders[competition_id] = (hotkey_uid, days_as_leader) + + if days_as_leader > 14: + periods = (days_as_leader - 14) // 7 + reduction_factor = max(0.1, 1 - 0.1 * periods) + return base_share * reduction_factor, periods > 0 + return base_share, False + + def update_scores(self): + scores = {} + for competition_id, (hotkey_uid, _) in self.competitions_leaders.items(): + score, is_reduced = self.calculate_score_for_winner(competition_id, hotkey_uid) + print(score, is_reduced) + scores[hotkey_uid] = (score, is_reduced) + + total_score_sum = sum(score for score, _ in scores.values()) + remaining_reward = 1 - total_score_sum + non_reduced_scores = [hotkey_uid for hotkey_uid, (_, is_reduced) in scores.items() if not is_reduced] + + if non_reduced_scores: + additional_reward_per_non_reduced = remaining_reward / len(non_reduced_scores) + for hotkey_uid in non_reduced_scores: + scores[hotkey_uid] = (scores[hotkey_uid][0] + additional_reward_per_non_reduced, scores[hotkey_uid][1]) + + self.scores = {uid: score for uid, (score, _) in scores.items()} + +rewarder = Rewarder() +rewarder.update_scores() +print(rewarder.scores) +print(rewarder.competitions_leaders) \ No newline at end of file From 16fd4caf5647447ef176aada4c48c7f4f338d1fa Mon Sep 17 00:00:00 2001 From: Konrad Date: Thu, 29 Aug 2024 12:33:07 +0200 Subject: [PATCH 089/227] rewards --- cancer_ai/base/validator.py | 27 +------------------ cancer_ai/rewards.py | 48 +++++++++++++++++++--------------- cancer_ai/utils/config.py | 8 +++--- cancer_ai/validator/forward.py | 11 -------- neurons/competition_runner.py | 6 ++--- neurons/validator.py | 32 +++++++++-------------- 6 files changed, 47 insertions(+), 85 deletions(-) diff --git a/cancer_ai/base/validator.py b/cancer_ai/base/validator.py index 1c924fc1..266f859b 100644 --- a/cancer_ai/base/validator.py +++ b/cancer_ai/base/validator.py @@ -107,32 +107,7 @@ def serve_axon(self): bt.logging.error(f"Failed to create Axon initialize with exception: {e}") pass - async def concurrent_forward(self): - coroutines = [ - self.forward() for _ in range(self.config.neuron.num_concurrent_forwards) - ] - await asyncio.gather(*coroutines) - def run(self): - """ - Initiates and manages the main loop for the miner on the Bittensor network. The main loop handles graceful shutdown on keyboard interrupts and logs unforeseen errors. - - This function performs the following primary tasks: - 1. Check for registration on the Bittensor network. - 2. Continuously forwards queries to the miners on the network, rewarding their responses and updating the scores accordingly. - 3. Periodically resynchronizes with the chain; updating the metagraph with the latest network state and setting weights. - - The essence of the validator's operations is in the forward function, which is called every step. The forward function is responsible for querying the network and scoring the responses. - - Note: - - The function leverages the global configurations set during the initialization of the miner. - - The miner's axon serves as its interface to the Bittensor network, handling incoming and outgoing requests. - - Raises: - KeyboardInterrupt: If the miner is stopped by a manual interruption. - Exception: For unforeseen errors during the miner's operation, which are logged for diagnosis. - """ - # Check that validator is registered on the network. self.sync() @@ -144,7 +119,7 @@ def run(self): bt.logging.info(f"step({self.step}) block({self.block})") # Run multiple forwards concurrently. - self.loop.run_until_complete(self.concurrent_forward()) + self.loop.run_until_complete(self.forward()) # Check if we should exit. if self.should_exit: diff --git a/cancer_ai/rewards.py b/cancer_ai/rewards.py index 8bbbfa89..07d1b794 100644 --- a/cancer_ai/rewards.py +++ b/cancer_ai/rewards.py @@ -3,13 +3,11 @@ class Rewarder(): def __init__(self, config=None): # competition_id to (hotkey_uid, days_as_leader) self.competitions_leaders = { - 1: (1, 21), - 2: (2, 3), - 3: (3, 28), + 1: (1, 1), } - self.scores = {} + self.scores = {1:0} - def calculate_score_for_winner(self, competition_id: int, hotkey_uid: int) -> tuple[float, bool]: + def calculate_score_for_winner(self, competition_id: int, hotkey_uid: int) -> tuple[float, float]: num_competitions = len(self.competitions_leaders) base_share = 1/num_competitions @@ -23,26 +21,34 @@ def calculate_score_for_winner(self, competition_id: int, hotkey_uid: int) -> tu if days_as_leader > 14: periods = (days_as_leader - 14) // 7 reduction_factor = max(0.1, 1 - 0.1 * periods) - return base_share * reduction_factor, periods > 0 - return base_share, False + final_share = base_share * reduction_factor + reduced_share = base_share - final_share + return final_share, reduced_share + return base_share, 0 def update_scores(self): - scores = {} - for competition_id, (hotkey_uid, _) in self.competitions_leaders.items(): - score, is_reduced = self.calculate_score_for_winner(competition_id, hotkey_uid) - print(score, is_reduced) - scores[hotkey_uid] = (score, is_reduced) + num_competitions = len(self.competitions_leaders) + reduced_shares_poll = {uid: 0.0 for uid in self.scores} + + # If there is only one competition, the winner takes all + if num_competitions == 1: + for curr_competition_id, (current_uid, _) in self.competitions_leaders.items(): + self.scores[current_uid] = 1.0 + return - total_score_sum = sum(score for score, _ in scores.values()) - remaining_reward = 1 - total_score_sum - non_reduced_scores = [hotkey_uid for hotkey_uid, (_, is_reduced) in scores.items() if not is_reduced] + for curr_competition_id, (current_uid, _) in self.competitions_leaders.items(): + score, reduced_share = self.calculate_score_for_winner(curr_competition_id, current_uid) + self.scores[current_uid] += score - if non_reduced_scores: - additional_reward_per_non_reduced = remaining_reward / len(non_reduced_scores) - for hotkey_uid in non_reduced_scores: - scores[hotkey_uid] = (scores[hotkey_uid][0] + additional_reward_per_non_reduced, scores[hotkey_uid][1]) - - self.scores = {uid: score for uid, (score, _) in scores.items()} + if reduced_share > 0: + # Distribute reduced share among all competitors (including the current winner if he wins another competition) + distributed_share = reduced_share / (num_competitions - 1) + for leader_competition_id, (uid, _) in self.competitions_leaders.items(): + if uid != current_uid or leader_competition_id != curr_competition_id: + reduced_shares_poll[uid] += distributed_share + + for uid, score in self.scores.items(): + self.scores[uid] += reduced_shares_poll[uid] rewarder = Rewarder() rewarder.update_scores() diff --git a/cancer_ai/utils/config.py b/cancer_ai/utils/config.py index c9f51f68..aa0fe296 100644 --- a/cancer_ai/utils/config.py +++ b/cancer_ai/utils/config.py @@ -175,10 +175,10 @@ def add_miner_args(cls, parser): ) parser.add_argument( - "--load_model_dir", + "--model_dir", type=str, - help="Path for for loading the starting model related to a training run.", - default="./models", + help="The directory where the models are stored.", + default="/tmp/models", ) parser.add_argument( @@ -212,7 +212,7 @@ def add_miner_args(cls, parser): "--dataset_dir", type=str, help="Path for storing datasets.", - default="./datasets", + default="/tmp/datasets", ) parser.add_argument( diff --git a/cancer_ai/validator/forward.py b/cancer_ai/validator/forward.py index 301664e9..d9c2002b 100644 --- a/cancer_ai/validator/forward.py +++ b/cancer_ai/validator/forward.py @@ -39,17 +39,6 @@ async def forward(self): # get_random_uids is an example method, but you can replace it with your own. miner_uids = get_random_uids(self, k=self.config.neuron.sample_size) - # The dendrite client queries the network. - responses = await self.dendrite( - # Send the query to selected miner axons in the network. - axons=[self.metagraph.axons[uid] for uid in miner_uids], - # Construct a dummy query. This simply contains a single integer. - synapse=Dummy(dummy_input=self.step), - # All responses have the deserialize function called on them before returning. - # You are encouraged to define your own deserialization function. - deserialize=True, - ) - # Log the results for monitoring purposes. bt.logging.info(f"Received responses: {responses}") diff --git a/neurons/competition_runner.py b/neurons/competition_runner.py index 94919e50..56b520c0 100644 --- a/neurons/competition_runner.py +++ b/neurons/competition_runner.py @@ -63,7 +63,7 @@ def log_results_to_wandb(project, entity, hotkey, evaluation_result: ModelEvalua return -async def schedule_competitions( +async def run_the_scheduler( competitions: CompetitionManager, path_config: str ) -> None: # Cache the next evaluation times for each competition @@ -106,7 +106,7 @@ async def schedule_competitions( competition_config["dataset_hf_repo_type"], ) print(f"Evaluating competition {competition_id} at {now_utc}") - await competition_manager.evaluate() + results = await competition_manager.evaluate() print( f"Results for competition {competition_id}: {competition_manager.results}" ) @@ -141,4 +141,4 @@ def run_all_competitions(path_config: str, competitions_cfg: List[dict]) -> None run_all_competitions(path_config, competitions_cfg) else: # Run the scheduling coroutine - asyncio.run(schedule_competitions(competitions, path_config)) + asyncio.run(run_the_scheduler(competitions, path_config)) diff --git a/neurons/validator.py b/neurons/validator.py index 07222c06..a3f7c589 100644 --- a/neurons/validator.py +++ b/neurons/validator.py @@ -19,14 +19,18 @@ import time - -# Bittensor +from typing import Any, List import bittensor as bt +import asyncio + +from numpy import ndarray -# import base validator class which takes care of most of the boilerplate from cancer_ai.base.validator import BaseValidatorNeuron -# Bittensor Validator Template: from cancer_ai.validator import forward +from types import SimpleNamespace +from datetime import datetime, timezone, timedelta +from cancer_ai.validator.competition_manager import CompetitionManager +from cancer_ai.validator.competition_handlers.base_handler import ModelEvaluationResult class Validator(BaseValidatorNeuron): @@ -40,24 +44,12 @@ class Validator(BaseValidatorNeuron): def __init__(self, config=None): super(Validator, self).__init__(config=config) - - bt.logging.info("load_state()") + # competition_id to (hotkey_uid, days_as_leader) + self.competitions_leaders = {} self.load_state() - - # TODO(developer): Anything specific to your use case you can do here - + async def forward(self): - """ - Validator forward pass. Consists of: - - Generating the query - - Querying the miners - - Getting the responses - - Rewarding the miners - - Updating the scores - """ - # TODO(developer): Rewrite this function based on your protocol definition. - return await forward(self) - + ... # The main function parses the configuration and runs the validator. if __name__ == "__main__": From ac660e46df091172fe6607abb03917b974ba4f93 Mon Sep 17 00:00:00 2001 From: Konrad Date: Thu, 29 Aug 2024 22:53:14 +0200 Subject: [PATCH 090/227] rewarder tests --- cancer_ai/rewarder.py | 60 ++++++++ cancer_ai/rewarder_test.py | 301 +++++++++++++++++++++++++++++++++++++ cancer_ai/rewards.py | 56 ------- 3 files changed, 361 insertions(+), 56 deletions(-) create mode 100644 cancer_ai/rewarder.py create mode 100644 cancer_ai/rewarder_test.py delete mode 100644 cancer_ai/rewards.py diff --git a/cancer_ai/rewarder.py b/cancer_ai/rewarder.py new file mode 100644 index 00000000..44dfbe79 --- /dev/null +++ b/cancer_ai/rewarder.py @@ -0,0 +1,60 @@ +from pydantic import BaseModel +from datetime import datetime + +class CompetitionLeader(BaseModel): + hotkey: str + leader_since: datetime + +class RewarderConfig(BaseModel): + competition_leader_mapping: dict[str, CompetitionLeader] + scores: dict[str, float] # hotkey -> score + +class Rewarder(): + def __init__(self, config: RewarderConfig): + self.competition_leader_mapping = config.competition_leader_mapping + self.scores = config.scores + + def calculate_score_for_winner(self, competition_id: str, hotkey: str) -> tuple[float, float]: + num_competitions = len(self.competition_leader_mapping) + base_share = 1/num_competitions + + # check if current hotkey is already a leader + if self.competition_leader_mapping[competition_id].hotkey == hotkey: + days_as_leader = (datetime.now() - self.competition_leader_mapping[competition_id].leader_since).days + else: + days_as_leader = 0 + self.competition_leader_mapping[competition_id] = CompetitionLeader(hotkey=hotkey, + leader_since=datetime.now()) + + if days_as_leader > 14: + periods = (days_as_leader - 14) // 7 + reduction_factor = max(0.1, 1 - 0.1 * periods) + final_share = base_share * reduction_factor + reduced_share = base_share - final_share + return final_share, reduced_share + return base_share, 0 + + def update_scores(self): + num_competitions = len(self.competition_leader_mapping) + reduced_shares_poll = {hotkey: 0.0 for hotkey in self.scores} + + # If there is only one competition, the winner takes it all + if num_competitions == 1: + single_key = next(iter(self.competition_leader_mapping)) + single_hotkey = self.competition_leader_mapping[single_key].hotkey + self.scores[single_hotkey] = 1.0 + return + + for curr_competition_id, comp_leader in self.competition_leader_mapping.items(): + score, reduced_share = self.calculate_score_for_winner(curr_competition_id, comp_leader.hotkey) + self.scores[comp_leader.hotkey] += score + + if reduced_share > 0: + # Distribute reduced share among all competitors (including the current winner if he wins another competition) + distributed_share = reduced_share / (num_competitions - 1) + for leader_competition_id, leader in self.competition_leader_mapping.items(): + if leader.hotkey != comp_leader.hotkey or leader_competition_id != curr_competition_id: + reduced_shares_poll[leader.hotkey] += distributed_share + + for hotkey, score in self.scores.items(): + self.scores[hotkey] += reduced_shares_poll[hotkey] \ No newline at end of file diff --git a/cancer_ai/rewarder_test.py b/cancer_ai/rewarder_test.py new file mode 100644 index 00000000..0dd72d90 --- /dev/null +++ b/cancer_ai/rewarder_test.py @@ -0,0 +1,301 @@ +import unittest +from datetime import datetime, timedelta +from rewarder import CompetitionLeader, RewarderConfig, Rewarder + +class TestRewarder(unittest.TestCase): + + def test_single_competition_single_leader(self): + """Test case 1: Only one competition with 1 leader -> leader takes it all""" + competitions_leaders = { + "competition-1": CompetitionLeader(hotkey="leader-1", leader_since=datetime.now()) + } + scores = {"leader-1": 0} + rewarder_config = RewarderConfig(competition_leader_mapping=competitions_leaders, scores=scores) + rewarder = Rewarder(config=rewarder_config) + + rewarder.update_scores() + + # Assert that the leader takes it all + self.assertAlmostEqual(rewarder.scores["leader-1"], 1.0) + + def test_three_competitions_three_leaders_no_reduction(self): + """Test case 2: 3 competitions with 3 different leaders, no reduction -> all have 33% of the shares""" + reward_split_by_three = 1 / 3 + competitions_leaders = { + "competition-1": CompetitionLeader(hotkey="leader-1", leader_since=datetime.now()), + "competition-2": CompetitionLeader(hotkey="leader-2", leader_since=datetime.now()), + "competition-3": CompetitionLeader(hotkey="leader-3", leader_since=datetime.now()) + } + scores = {"leader-1": 0, "leader-2": 0, "leader-3": 0} + rewarder_config = RewarderConfig(competition_leader_mapping=competitions_leaders, scores=scores) + rewarder = Rewarder(config=rewarder_config) + rewarder.update_scores() + + # Assert that all leaders have roughly 1/3 of the shares + self.assertAlmostEqual(rewarder.scores["leader-1"], reward_split_by_three, places=2) + self.assertAlmostEqual(rewarder.scores["leader-2"], reward_split_by_three, places=2) + self.assertAlmostEqual(rewarder.scores["leader-3"], reward_split_by_three, places=2) + + def test_three_competitions_three_leaders_with_reduction(self): + """Test case 3: 3 competitions with 3 different leaders, one has a reduced share by 10%""" + first_competion_leader_since = datetime.now() - timedelta(days=21) + + base_share = 1/3 + reduction_factor = 0.9 # 10% reduction + expected_share_leader_1 = base_share * reduction_factor + expected_reduction = base_share - expected_share_leader_1 + expected_share_leader_2_3 = base_share + (expected_reduction / 2) # Distributed reduction + + scores = {"leader-1": 0, "leader-2": 0, "leader-3": 0} + + competitions_leaders = { + "competition-1": CompetitionLeader(hotkey="leader-1", leader_since=first_competion_leader_since), + "competition-2": CompetitionLeader(hotkey="leader-2", leader_since=datetime.now()), + "competition-3": CompetitionLeader(hotkey="leader-3", leader_since=datetime.now()) + } + + rewarder_config = RewarderConfig(competition_leader_mapping=competitions_leaders, scores=scores) + rewarder = Rewarder(config=rewarder_config) + + rewarder.update_scores() + # Assert that leader-1 has the reduced share and others are higher + self.assertAlmostEqual(rewarder.scores["leader-1"], expected_share_leader_1, places=2) + self.assertAlmostEqual(rewarder.scores["leader-2"], expected_share_leader_2_3, places=2) + self.assertAlmostEqual(rewarder.scores["leader-3"], expected_share_leader_2_3, places=2) + + def test_three_competitions_three_leaders_two_reductions(self): + """Test case 4: 3 competitions with 3 different leaders, two with reduced shares""" + competitions_leaders = { + "competition-1": CompetitionLeader(hotkey="leader-1", leader_since=datetime.now() - timedelta(days=21)), + "competition-2": CompetitionLeader(hotkey="leader-2", leader_since=datetime.now() - timedelta(days=35)), + "competition-3": CompetitionLeader(hotkey="leader-3", leader_since=datetime.now()) + } + scores = {"leader-1": 0, "leader-2": 0, "leader-3": 0} + rewarder_config = RewarderConfig(competition_leader_mapping=competitions_leaders, scores=scores) + rewarder = Rewarder(config=rewarder_config) + + rewarder.update_scores() + + base_share = 1 / 3 + reduction_factor_leader_1 = 0.9 # 10% reduction + reduction_factor_leader_2 = 0.7 # 30% reduction + + expected_share_leader_1 = base_share * reduction_factor_leader_1 + expected_share_leader_2 = base_share * reduction_factor_leader_2 + expected_share_leader_3 = base_share + # Calculate distributed reduction + remaining_share_leader_1 = base_share - expected_share_leader_1 + remaining_share_leader_2 = base_share - expected_share_leader_2 + + # Leaders 2 and 3 gets their base share plus the distributed reduction from Leader 1 + expected_share_leader_2 += remaining_share_leader_1 / 2 + expected_share_leader_3 += remaining_share_leader_1 / 2 + + # Leaders 1 and 3 gets their base share plus the distributed reduction from Leader 2 + + expected_share_leader_1 += remaining_share_leader_2 / 2 + expected_share_leader_3 += remaining_share_leader_2 / 2 + + self.assertAlmostEqual(rewarder.scores["leader-1"], expected_share_leader_1, places=2) + self.assertAlmostEqual(rewarder.scores["leader-2"], expected_share_leader_2, places=2) + self.assertAlmostEqual(rewarder.scores["leader-3"], expected_share_leader_3, places=2) + + def test_three_competitions_three_leaders_all_different_reductions(self): + """Test case 5: All competitors have different degrees of reduced shares""" + competitions_leaders = { + "competition-1": CompetitionLeader(hotkey="leader-1", leader_since=datetime.now() - timedelta(days=21)), + "competition-2": CompetitionLeader(hotkey="leader-2", leader_since=datetime.now() - timedelta(days=35)), + "competition-3": CompetitionLeader(hotkey="leader-3", leader_since=datetime.now() - timedelta(days=49)) + } + scores = {"leader-1": 0, "leader-2": 0, "leader-3": 0} + rewarder_config = RewarderConfig(competition_leader_mapping=competitions_leaders, scores=scores) + rewarder = Rewarder(config=rewarder_config) + + rewarder.update_scores() + + base_share = 1 / 3 + reduction_factor_leader_1 = 0.9 # 10% reduction + reduction_factor_leader_2 = 0.7 # 30% reduction + reduction_factor_leader_3 = 0.5 # 50% reduction + + expected_share_leader_1 = base_share * reduction_factor_leader_1 + expected_share_leader_2 = base_share * reduction_factor_leader_2 + expected_share_leader_3 = base_share * reduction_factor_leader_3 + # Calculate distributed reduction + remaining_share_leader_1 = base_share - expected_share_leader_1 + remaining_share_leader_2 = base_share - expected_share_leader_2 + remaining_share_leader_3 = base_share - expected_share_leader_3 + + # Leaders 2 and 3 gets their base share plus the distributed reduction from Leader 1 + expected_share_leader_2 += remaining_share_leader_1 / 2 + expected_share_leader_3 += remaining_share_leader_1 / 2 + + # Leaders 1 and 3 gets their base share plus the distributed reduction from Leader 2 + expected_share_leader_1 += remaining_share_leader_2 / 2 + expected_share_leader_3 += remaining_share_leader_2 / 2 + + # Leaders 1 and 2 gets their base share plus the distributed reduction from Leader 3 + expected_share_leader_1 += remaining_share_leader_3 / 2 + expected_share_leader_2 += remaining_share_leader_3 / 2 + + self.assertAlmostEqual(rewarder.scores["leader-1"], expected_share_leader_1, places=2) + self.assertAlmostEqual(rewarder.scores["leader-2"], expected_share_leader_2, places=2) + self.assertAlmostEqual(rewarder.scores["leader-3"], expected_share_leader_3, places=2) + + def test_three_competitions_three_leaders_all_same_reductions(self): + """Test case 6: All competitors have the same amount of reduced shares""" + competitions_leaders = { + "competition-1": CompetitionLeader(hotkey="leader-1", leader_since=datetime.now() - timedelta(days=21)), + "competition-2": CompetitionLeader(hotkey="leader-2", leader_since=datetime.now() - timedelta(days=21)), + "competition-3": CompetitionLeader(hotkey="leader-3", leader_since=datetime.now() - timedelta(days=21)) + } + scores = {"leader-1": 0, "leader-2": 0, "leader-3": 0} + rewarder_config = RewarderConfig(competition_leader_mapping=competitions_leaders, scores=scores) + rewarder = Rewarder(config=rewarder_config) + + rewarder.update_scores() + + base_share = 1 / 3 + reduction_factor = 0.9 # 10% reduction for all + + expected_share_leader_1 = base_share * reduction_factor + expected_share_leader_2 = base_share * reduction_factor + expected_share_leader_3 = base_share * reduction_factor + + # Calculate distributed reduction + remaining_share_leader_1 = base_share - expected_share_leader_1 + remaining_share_leader_2 = base_share - expected_share_leader_2 + remaining_share_leader_3 = base_share - expected_share_leader_3 + + # Leaders 2 and 3 gets their base share plus the distributed reduction from Leader 1 + expected_share_leader_2 += remaining_share_leader_1 / 2 + expected_share_leader_3 += remaining_share_leader_1 / 2 + + # Leaders 1 and 3 gets their base share plus the distributed reduction from Leader 2 + expected_share_leader_1 += remaining_share_leader_2 / 2 + expected_share_leader_3 += remaining_share_leader_2 / 2 + + # Leaders 1 and 2 gets their base share plus the distributed reduction from Leader 3 + expected_share_leader_1 += remaining_share_leader_3 / 2 + expected_share_leader_2 += remaining_share_leader_3 / 2 + + # All should have the same shares + self.assertAlmostEqual(rewarder.scores["leader-1"], expected_share_leader_1, places=2) + self.assertAlmostEqual(rewarder.scores["leader-2"], expected_share_leader_2, places=2) + self.assertAlmostEqual(rewarder.scores["leader-3"], expected_share_leader_3, places=2) + + def test_three_competitions_three_leaders_all_maximum_reductions(self): + """Test case 7: All competitors have maximum reduced shares (90%)""" + competitions_leaders = { + "competition-1": CompetitionLeader(hotkey="leader-1", leader_since=datetime.now() - timedelta(days=91)), + "competition-2": CompetitionLeader(hotkey="leader-2", leader_since=datetime.now() - timedelta(days=91)), + "competition-3": CompetitionLeader(hotkey="leader-3", leader_since=datetime.now() - timedelta(days=91)) + } + scores = {"leader-1": 0, "leader-2": 0, "leader-3": 0} + rewarder_config = RewarderConfig(competition_leader_mapping=competitions_leaders, scores=scores) + rewarder = Rewarder(config=rewarder_config) + + rewarder.update_scores() + + base_share = 1 / 3 + reduction_factor = 0.1 # 90% reduction for all + + expected_share_leader_1 = base_share * reduction_factor + expected_share_leader_2 = base_share * reduction_factor + expected_share_leader_3 = base_share * reduction_factor + + # Calculate distributed reduction + remaining_share_leader_1 = base_share - expected_share_leader_1 + remaining_share_leader_2 = base_share - expected_share_leader_2 + remaining_share_leader_3 = base_share - expected_share_leader_3 + + # Leaders 2 and 3 gets their base share plus the distributed reduction from Leader 1 + expected_share_leader_2 += remaining_share_leader_1 / 2 + expected_share_leader_3 += remaining_share_leader_1 / 2 + + # Leaders 1 and 3 gets their base share plus the distributed reduction from Leader 2 + expected_share_leader_1 += remaining_share_leader_2 / 2 + expected_share_leader_3 += remaining_share_leader_2 / 2 + + # Leaders 1 and 2 gets their base share plus the distributed reduction from Leader 3 + expected_share_leader_1 += remaining_share_leader_3 / 2 + expected_share_leader_2 += remaining_share_leader_3 / 2 + + # All should have the same shares + self.assertAlmostEqual(rewarder.scores["leader-1"], expected_share_leader_1, places=2) + self.assertAlmostEqual(rewarder.scores["leader-2"], expected_share_leader_2, places=2) + self.assertAlmostEqual(rewarder.scores["leader-3"], expected_share_leader_3, places=2) + + def test_three_competitions_two_competitors(self): + """Test case 8: 3 competitions but only 2 competitors""" + competitions_leaders = { + "competition-1": CompetitionLeader(hotkey="leader-1", leader_since=datetime.now() - timedelta(days=21)), + "competition-2": CompetitionLeader(hotkey="leader-1", leader_since=datetime.now() - timedelta(days=10)), + "competition-3": CompetitionLeader(hotkey="leader-2", leader_since=datetime.now()) + } + scores = {"leader-1": 0, "leader-2": 0} + rewarder_config = RewarderConfig(competition_leader_mapping=competitions_leaders, scores=scores) + rewarder = Rewarder(config=rewarder_config) + + rewarder.update_scores() + + base_share = 1 / 3 + reduction_factor_leader_1_competition_1 = 0.9 # 10% reduction for 21 days + + # Calculate expected scores + expected_share_leader_1_competition_1 = base_share * reduction_factor_leader_1_competition_1 + expected_share_leader_1_competition_2 = base_share + expected_share_leader_2 = base_share + + remaining_share_leader_1_competition_1 = base_share - expected_share_leader_1_competition_1 + # The competitors of competition 2 and 3 (including leader-1) get the distributed reduction + expected_score_leader_1 = expected_share_leader_1_competition_1 + expected_share_leader_1_competition_2\ + + remaining_share_leader_1_competition_1 / 2 + expected_score_leader_2 = expected_share_leader_2 + remaining_share_leader_1_competition_1 / 2 + + self.assertAlmostEqual(rewarder.scores["leader-1"], expected_score_leader_1, places=2) + self.assertAlmostEqual(rewarder.scores["leader-2"], expected_score_leader_2, places=2) + + def test_five_competitions_three_competitors_two_repeating(self): + """Test case 9: 5 competitions with 3 competitors, 2 of them are repeating""" + competitions_leaders = { + "competition-1": CompetitionLeader(hotkey="leader-1", leader_since=datetime.now() - timedelta(days=21)), # 10% reduction + "competition-2": CompetitionLeader(hotkey="leader-2", leader_since=datetime.now() - timedelta(days=10)), # No reduction + "competition-3": CompetitionLeader(hotkey="leader-1", leader_since=datetime.now()), # No reduction + "competition-4": CompetitionLeader(hotkey="leader-3", leader_since=datetime.now() - timedelta(days=35)), # 30% reduction + "competition-5": CompetitionLeader(hotkey="leader-2", leader_since=datetime.now()) # No reduction + } + scores = {"leader-1": 0, "leader-2": 0, "leader-3": 0} + rewarder_config = RewarderConfig(competition_leader_mapping=competitions_leaders, scores=scores) + rewarder = Rewarder(config=rewarder_config) + + rewarder.update_scores() + + base_share = 1 / 5 + reduction_factor_leader_1_competition_1 = 0.9 # 10% reduction for 21 days + reduction_factor_leader_3_competition_4 = 0.7 # 30% reduction for 35 days + + # Calculate expected shares for each leader + expected_share_leader_1_competition_1 = base_share * reduction_factor_leader_1_competition_1 + expected_share_leader_1_competition_3 = base_share + expected_share_leader_2_competition_2 = base_share + expected_share_leader_2_competition_5 = base_share + expected_share_leader_3_competition_4 = base_share * reduction_factor_leader_3_competition_4 + + remaining_share_leader_1_competition_1 = base_share - expected_share_leader_1_competition_1 + remaining_share_leader_3_competition_4 = base_share - expected_share_leader_3_competition_4 + + # Calculate final scores with distributed reduction shares + expected_score_leader_1 = expected_share_leader_1_competition_1 + expected_share_leader_1_competition_3\ + + (remaining_share_leader_1_competition_1 / 3) + (remaining_share_leader_3_competition_4 / 2) + expected_score_leader_2 = expected_share_leader_2_competition_2 + expected_share_leader_2_competition_5\ + + (remaining_share_leader_1_competition_1 / 3) + (remaining_share_leader_3_competition_4 / 2) + expected_score_leader_3 = expected_share_leader_3_competition_4 + (remaining_share_leader_1_competition_1 / 3) + + self.assertAlmostEqual(rewarder.scores["leader-1"], expected_score_leader_1, places=2) + self.assertAlmostEqual(rewarder.scores["leader-2"], expected_score_leader_2, places=2) + self.assertAlmostEqual(rewarder.scores["leader-3"], expected_score_leader_3, places=2) + + +if __name__ == "__main__": + unittest.main() diff --git a/cancer_ai/rewards.py b/cancer_ai/rewards.py deleted file mode 100644 index 07d1b794..00000000 --- a/cancer_ai/rewards.py +++ /dev/null @@ -1,56 +0,0 @@ - -class Rewarder(): - def __init__(self, config=None): - # competition_id to (hotkey_uid, days_as_leader) - self.competitions_leaders = { - 1: (1, 1), - } - self.scores = {1:0} - - def calculate_score_for_winner(self, competition_id: int, hotkey_uid: int) -> tuple[float, float]: - num_competitions = len(self.competitions_leaders) - base_share = 1/num_competitions - - # check if current hotkey is already a leader - if self.competitions_leaders[competition_id][0] == hotkey_uid: - days_as_leader = self.competitions_leaders[competition_id][1] - else: - days_as_leader = 0 - self.competitions_leaders[competition_id] = (hotkey_uid, days_as_leader) - - if days_as_leader > 14: - periods = (days_as_leader - 14) // 7 - reduction_factor = max(0.1, 1 - 0.1 * periods) - final_share = base_share * reduction_factor - reduced_share = base_share - final_share - return final_share, reduced_share - return base_share, 0 - - def update_scores(self): - num_competitions = len(self.competitions_leaders) - reduced_shares_poll = {uid: 0.0 for uid in self.scores} - - # If there is only one competition, the winner takes all - if num_competitions == 1: - for curr_competition_id, (current_uid, _) in self.competitions_leaders.items(): - self.scores[current_uid] = 1.0 - return - - for curr_competition_id, (current_uid, _) in self.competitions_leaders.items(): - score, reduced_share = self.calculate_score_for_winner(curr_competition_id, current_uid) - self.scores[current_uid] += score - - if reduced_share > 0: - # Distribute reduced share among all competitors (including the current winner if he wins another competition) - distributed_share = reduced_share / (num_competitions - 1) - for leader_competition_id, (uid, _) in self.competitions_leaders.items(): - if uid != current_uid or leader_competition_id != curr_competition_id: - reduced_shares_poll[uid] += distributed_share - - for uid, score in self.scores.items(): - self.scores[uid] += reduced_shares_poll[uid] - -rewarder = Rewarder() -rewarder.update_scores() -print(rewarder.scores) -print(rewarder.competitions_leaders) \ No newline at end of file From 153f9b9aea2ab40889bae8c44d0fb9ab07d5834e Mon Sep 17 00:00:00 2001 From: Wojtek Jurkowlaniec Date: Thu, 29 Aug 2024 23:28:03 +0200 Subject: [PATCH 091/227] test discussed case --- cancer_ai/rewarder_test.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/cancer_ai/rewarder_test.py b/cancer_ai/rewarder_test.py index 0dd72d90..d9461a93 100644 --- a/cancer_ai/rewarder_test.py +++ b/cancer_ai/rewarder_test.py @@ -17,6 +17,28 @@ def test_single_competition_single_leader(self): # Assert that the leader takes it all self.assertAlmostEqual(rewarder.scores["leader-1"], 1.0) + + def test_4_2_reduction(self): + """Test case 2: 4 competitions with 2 different leaders, reduction -> 2 have 50% of the shares""" + leader_1_reward = 0.125 + leader_2_reward = 0.125 + leader_3_reward = leader_4_reward = 0.25 + 0.125 + reduction_50_percent_days = datetime.now() - timedelta(days=14 + 7*5) + competitions_leaders = { + "competition-1": CompetitionLeader(hotkey="leader-1", leader_since=reduction_50_percent_days), + "competition-2": CompetitionLeader(hotkey="leader-2", leader_since=reduction_50_percent_days), + "competition-3": CompetitionLeader(hotkey="leader-3", leader_since=reduction_50_percent_days), + "competition-4": CompetitionLeader(hotkey="leader-4", leader_since=reduction_50_percent_days), + } + scores = {"leader-1": 0, "leader-2": 0, "leader-3": 0, "leader-4": 0} + rewarder_config = RewarderConfig(competition_leader_mapping=competitions_leaders, scores=scores) + rewarder = Rewarder(config=rewarder_config) + rewarder.update_scores() + self.assertAlmostEqual(rewarder.scores["leader-1"], leader_1_reward, places=2) + self.assertAlmostEqual(rewarder.scores["leader-2"], leader_2_reward, places=2) + self.assertAlmostEqual(rewarder.scores["leader-3"], leader_3_reward, places=2) + self.assertAlmostEqual(rewarder.scores["leader-4"], leader_4_reward, places=2) + def test_three_competitions_three_leaders_no_reduction(self): """Test case 2: 3 competitions with 3 different leaders, no reduction -> all have 33% of the shares""" From 89e594e07cb809bf170b5d23cb45116819aa1da0 Mon Sep 17 00:00:00 2001 From: Wojtek Jurkowlaniec Date: Fri, 30 Aug 2024 11:14:29 +0200 Subject: [PATCH 092/227] fixed test --- cancer_ai/rewarder_test.py | 56 ++++++-------------------------------- 1 file changed, 8 insertions(+), 48 deletions(-) diff --git a/cancer_ai/rewarder_test.py b/cancer_ai/rewarder_test.py index d9461a93..0a194e9e 100644 --- a/cancer_ai/rewarder_test.py +++ b/cancer_ai/rewarder_test.py @@ -98,29 +98,11 @@ def test_three_competitions_three_leaders_two_reductions(self): rewarder.update_scores() - base_share = 1 / 3 - reduction_factor_leader_1 = 0.9 # 10% reduction - reduction_factor_leader_2 = 0.7 # 30% reduction - - expected_share_leader_1 = base_share * reduction_factor_leader_1 - expected_share_leader_2 = base_share * reduction_factor_leader_2 - expected_share_leader_3 = base_share - # Calculate distributed reduction - remaining_share_leader_1 = base_share - expected_share_leader_1 - remaining_share_leader_2 = base_share - expected_share_leader_2 + - # Leaders 2 and 3 gets their base share plus the distributed reduction from Leader 1 - expected_share_leader_2 += remaining_share_leader_1 / 2 - expected_share_leader_3 += remaining_share_leader_1 / 2 - - # Leaders 1 and 3 gets their base share plus the distributed reduction from Leader 2 - - expected_share_leader_1 += remaining_share_leader_2 / 2 - expected_share_leader_3 += remaining_share_leader_2 / 2 - - self.assertAlmostEqual(rewarder.scores["leader-1"], expected_share_leader_1, places=2) - self.assertAlmostEqual(rewarder.scores["leader-2"], expected_share_leader_2, places=2) - self.assertAlmostEqual(rewarder.scores["leader-3"], expected_share_leader_3, places=2) + self.assertAlmostEqual(rewarder.scores["leader-1"], 1/3, places=2) + self.assertAlmostEqual(rewarder.scores["leader-2"], 1/3, places=2) + self.assertAlmostEqual(rewarder.scores["leader-3"], 1/3, places=2) def test_three_competitions_three_leaders_all_different_reductions(self): """Test case 5: All competitors have different degrees of reduced shares""" @@ -178,33 +160,11 @@ def test_three_competitions_three_leaders_all_same_reductions(self): rewarder.update_scores() base_share = 1 / 3 - reduction_factor = 0.9 # 10% reduction for all - - expected_share_leader_1 = base_share * reduction_factor - expected_share_leader_2 = base_share * reduction_factor - expected_share_leader_3 = base_share * reduction_factor - - # Calculate distributed reduction - remaining_share_leader_1 = base_share - expected_share_leader_1 - remaining_share_leader_2 = base_share - expected_share_leader_2 - remaining_share_leader_3 = base_share - expected_share_leader_3 - - # Leaders 2 and 3 gets their base share plus the distributed reduction from Leader 1 - expected_share_leader_2 += remaining_share_leader_1 / 2 - expected_share_leader_3 += remaining_share_leader_1 / 2 - - # Leaders 1 and 3 gets their base share plus the distributed reduction from Leader 2 - expected_share_leader_1 += remaining_share_leader_2 / 2 - expected_share_leader_3 += remaining_share_leader_2 / 2 - - # Leaders 1 and 2 gets their base share plus the distributed reduction from Leader 3 - expected_share_leader_1 += remaining_share_leader_3 / 2 - expected_share_leader_2 += remaining_share_leader_3 / 2 - + # All should have the same shares - self.assertAlmostEqual(rewarder.scores["leader-1"], expected_share_leader_1, places=2) - self.assertAlmostEqual(rewarder.scores["leader-2"], expected_share_leader_2, places=2) - self.assertAlmostEqual(rewarder.scores["leader-3"], expected_share_leader_3, places=2) + self.assertAlmostEqual(rewarder.scores["leader-1"], base_share, places=2) + self.assertAlmostEqual(rewarder.scores["leader-2"], base_share, places=2) + self.assertAlmostEqual(rewarder.scores["leader-3"], base_share, places=2) def test_three_competitions_three_leaders_all_maximum_reductions(self): """Test case 7: All competitors have maximum reduced shares (90%)""" From 7b78ae904d4dfeb0440d3fe8f43891924fd9cb05 Mon Sep 17 00:00:00 2001 From: Konrad Date: Mon, 26 Aug 2024 16:03:19 +0200 Subject: [PATCH 093/227] plugged in config and integrated pushing model metadata on chain from miner --- cancer_ai/base/miner.py | 54 -------- cancer_ai/chain_models_store.py | 23 ++-- cancer_ai/utils/config.py | 64 +++++++-- cancer_ai/validator/dataset_manager.py | 4 +- neurons/miner.py | 175 ------------------------- neurons/miner3.py | 69 ++++++---- neurons/miner_config.py | 133 ------------------- 7 files changed, 107 insertions(+), 415 deletions(-) delete mode 100644 neurons/miner.py delete mode 100644 neurons/miner_config.py diff --git a/cancer_ai/base/miner.py b/cancer_ai/base/miner.py index 528ae34f..2990e77a 100644 --- a/cancer_ai/base/miner.py +++ b/cancer_ai/base/miner.py @@ -43,27 +43,6 @@ def add_args(cls, parser: argparse.ArgumentParser): def __init__(self, config=None): super().__init__(config=config) - # Warn if allowing incoming requests from anyone. - if not self.config.blacklist.force_validator_permit: - bt.logging.warning( - "You are allowing non-validators to send requests to your miner. This is a security risk." - ) - if self.config.blacklist.allow_non_registered: - bt.logging.warning( - "You are allowing non-registered entities to send requests to your miner. This is a security risk." - ) - # The axon handles request processing, allowing validators to send this miner requests. - self.axon = bt.axon(wallet=self.wallet, config=self.config() if callable(self.config) else self.config) - - # Attach determiners which functions are called when servicing a request. - bt.logging.info(f"Attaching forward function to miner axon.") - self.axon.attach( - forward_fn=self.forward, - blacklist_fn=self.blacklist, - priority_fn=self.priority, - ) - bt.logging.info(f"Axon created: {self.axon}") - # Instantiate runners self.should_exit: bool = False self.is_running: bool = False @@ -71,41 +50,8 @@ def __init__(self, config=None): self.lock = asyncio.Lock() def run(self): - """ - Initiates and manages the main loop for the miner on the Bittensor network. The main loop handles graceful shutdown on keyboard interrupts and logs unforeseen errors. - - This function performs the following primary tasks: - 1. Check for registration on the Bittensor network. - 2. Starts the miner's axon, making it active on the network. - 3. Periodically resynchronizes with the chain; updating the metagraph with the latest network state and setting weights. - - The miner continues its operations until `should_exit` is set to True or an external interruption occurs. - During each epoch of its operation, the miner waits for new blocks on the Bittensor network, updates its - knowledge of the network (metagraph), and sets its weights. This process ensures the miner remains active - and up-to-date with the network's latest state. - - Note: - - The function leverages the global configurations set during the initialization of the miner. - - The miner's axon serves as its interface to the Bittensor network, handling incoming and outgoing requests. - - Raises: - KeyboardInterrupt: If the miner is stopped by a manual interruption. - Exception: For unforeseen errors during the miner's operation, which are logged for diagnosis. - """ - # Check that miner is registered on the network. self.sync() - - # Serve passes the axon information to the network + netuid we are hosting on. - # This will auto-update if the axon port of external ip have changed. - bt.logging.info( - f"Serving miner axon {self.axon} on network: {self.config.subtensor.chain_endpoint} with netuid: {self.config.netuid}" - ) - self.axon.serve(netuid=self.config.netuid, subtensor=self.subtensor) - - # Start starts the miner's axon, making it active on the network. - self.axon.start() - bt.logging.info(f"Miner starting at block: {self.block}") # This loop maintains the miner's operations until intentionally stopped. diff --git a/cancer_ai/chain_models_store.py b/cancer_ai/chain_models_store.py index b20158e6..25b10062 100644 --- a/cancer_ai/chain_models_store.py +++ b/cancer_ai/chain_models_store.py @@ -10,15 +10,11 @@ class ChainMinerModel(BaseModel): """Uniquely identifies a trained model""" - namespace: str = Field( + hf_repo_id: str = Field( description="Namespace where the model can be found. ex. Hugging Face username/org." ) name: str = Field(description="Name of the model.") - epoch: int = Field( - description="The epoch number to submit as your checkpoint to evaluate e.g. 10" - ) - date: datetime.datetime = Field( description="The datetime at which model was pushed to hugging face" ) @@ -30,27 +26,25 @@ class ChainMinerModel(BaseModel): description="Block on which this model was claimed on the chain." ) - hf_repo_id: Optional[str] = Field(description="Hugging Face repo id.") - hf_filename: Optional[str] = Field(description="Hugging Face filename.") - hf_repo_type: Optional[str] = Field(description="Hugging Face repo type.") + # hf_filename: Optional[str] = Field(description="Hugging Face filename.") + # hf_repo_type: Optional[str] = Field(description="Hugging Face repo type.") class Config: arbitrary_types_allowed = True def to_compressed_str(self) -> str: """Returns a compressed string representation.""" - return f"{self.namespace}:{self.name}:{self.epoch}:{self.competition_id}:{self.date}" + return f"{self.hf_repo_id}:{self.name}:{self.date}:{self.competition_id}" @classmethod def from_compressed_str(cls, cs: str) -> Type["ChainMinerModel"]: """Returns an instance of this class from a compressed string representation""" tokens = cs.split(":") return cls( - namespace=tokens[0], + hf_repo_id=tokens[0], name=tokens[1], - epoch=tokens[2] if tokens[2] != "None" else None, - date=tokens[3] if tokens[3] != "None" else None, - competition_id=tokens[4] if tokens[4] != "None" else None, + date=tokens[2] if tokens[2] != "None" else None, + competition_id=tokens[3] if tokens[3] != "None" else None, ) @@ -94,7 +88,6 @@ async def retrieve_model_metadata(self, hotkey: str) -> Optional[ChainMinerModel metadata = run_in_subprocess(partial, 60) if not metadata: return None - print("piwo", metadata["info"]["fields"]) commitment = metadata["info"]["fields"][0] hex_data = commitment[list(commitment.keys())[0]][2:] @@ -110,4 +103,4 @@ async def retrieve_model_metadata(self, hotkey: str) -> Optional[ChainMinerModel return None # The block id at which the metadata is stored model.block = metadata["block"] - return model + return model \ No newline at end of file diff --git a/cancer_ai/utils/config.py b/cancer_ai/utils/config.py index 5967d6f3..8bc73d0f 100644 --- a/cancer_ai/utils/config.py +++ b/cancer_ai/utils/config.py @@ -175,31 +175,73 @@ def add_miner_args(cls, parser): ) parser.add_argument( - "--models.load_model_dir", + "--load_model_dir", type=str, help="Path for for loading the starting model related to a training run.", default="./models", ) parser.add_argument( - "--models.namespace", + "--hf_model_name", type=str, - help="Namespace where the model can be found.", - default="mock-namespace", + help="Name of the model to push to hugging face.", + default="", + ) + + parser.add_argument( + "--action", + choices=["submit", "evaluate", "upload"], + default="submit", ) parser.add_argument( - "--models.model_name", + "--model_path", type=str, - help="Name of the model to push to hugging face.", - default="mock-name", + help="Path to ONNX model, used for evaluation", + default="", ) parser.add_argument( - "--models.epoch_checkpoint", - type=int, - help="The epoch number to submit as your checkpoint to evaluate e.g. 10", - default=10, + "--competition_id", + type=str, + help="Competition ID", + default="melanoma-1", + ) + + parser.add_argument( + "--dataset_dir", + type=str, + help="Path for storing datasets.", + default="./datasets", + ) + + parser.add_argument( + "--hf_repo_id", + type=str, + # required=False, + help="Hugging Face model repository ID", + default="", + ) + + parser.add_argument( + "--hf_token", + type=str, + help="Hugging Face API token", + default="", + ) + + parser.add_argument( + "--clean_after_run", + action="store_true", + help="Whether to clean up (dataset, temporary files) after running", + default=False, + ) + + parser.add_argument( + "--code_directory", + type=str, + help="Path to code directory", + default=".", ) def add_validator_args(cls, parser): diff --git a/cancer_ai/validator/dataset_manager.py b/cancer_ai/validator/dataset_manager.py index f26d42b6..900ab59c 100644 --- a/cancer_ai/validator/dataset_manager.py +++ b/cancer_ai/validator/dataset_manager.py @@ -17,7 +17,7 @@ class DatasetManagerException(Exception): class DatasetManager(SerializableManager): def __init__( - self, config, competition_id: str, hf_repo_id: str, hf_filename: str, hf_repo_type: str + self, config, hf_repo_id: str, hf_filename: str, hf_repo_type: str ) -> None: """ Initializes a new instance of the DatasetManager class. @@ -32,7 +32,7 @@ def __init__( None """ self.config = config - self.competition_id = competition_id + self.competition_id = config.competition_id self.hf_repo_id = hf_repo_id self.hf_filename = hf_filename self.hf_repo_type = hf_repo_type diff --git a/neurons/miner.py b/neurons/miner.py deleted file mode 100644 index 9360a4af..00000000 --- a/neurons/miner.py +++ /dev/null @@ -1,175 +0,0 @@ - -import time -import typing -import bittensor as bt -import datetime as dt -import os -import datetime -import asyncio -import cancer_ai - -# import base miner class which takes care of most of the boilerplate -from cancer_ai.base.miner import BaseMinerNeuron -from cancer_ai.chain_models_store import ChainMinerModel, ChainModelMetadataStore - - -class Miner(BaseMinerNeuron): - """ - Your miner neuron class. You should use this class to define your miner's behavior. In particular, you should replace the forward function with your own logic. You may also want to override the blacklist and priority functions according to your needs. - - This class inherits from the BaseMinerNeuron class, which in turn inherits from BaseNeuron. The BaseNeuron class takes care of routine tasks such as setting up wallet, subtensor, metagraph, logging directory, parsing config, etc. You can override any of the methods in BaseNeuron if you need to customize the behavior. - - This class provides reasonable default behavior for a miner such as blacklisting unrecognized hotkeys, prioritizing requests based on stake, and forwarding requests to the forward function. If you need to define custom - """ - - def __init__(self, config=None): - super(Miner, self).__init__(config=config) - - self.metadata_store = ChainModelMetadataStore(subtensor=self.subtensor, subnet_uid=163, wallet=self.wallet) - - asyncio.run(self.store_and_retrieve_metadata_on_chain("mock_competition")) - - async def forward( - self, synapse: cancer_ai.protocol.Dummy - ) -> cancer_ai.protocol.Dummy: - """ - Processes the incoming 'Dummy' synapse by performing a predefined operation on the input data. - This method should be replaced with actual logic relevant to the miner's purpose. - - Args: - synapse (template.protocol.Dummy): The synapse object containing the 'dummy_input' data. - - Returns: - template.protocol.Dummy: The synapse object with the 'dummy_output' field set to twice the 'dummy_input' value. - - The 'forward' function is a placeholder and should be overridden with logic that is appropriate for - the miner's intended operation. This method demonstrates a basic transformation of input data. - """ - # TODO(developer): Replace with actual implementation logic. - synapse.dummy_output = synapse.dummy_input * 2 - return synapse - - async def blacklist( - self, synapse: cancer_ai.protocol.Dummy - ) -> typing.Tuple[bool, str]: - """ - Determines whether an incoming request should be blacklisted and thus ignored. Your implementation should - define the logic for blacklisting requests based on your needs and desired security parameters. - - Blacklist runs before the synapse data has been deserialized (i.e. before synapse.data is available). - The synapse is instead contracted via the headers of the request. It is important to blacklist - requests before they are deserialized to avoid wasting resources on requests that will be ignored. - - Args: - synapse (template.protocol.Dummy): A synapse object constructed from the headers of the incoming request. - - Returns: - Tuple[bool, str]: A tuple containing a boolean indicating whether the synapse's hotkey is blacklisted, - and a string providing the reason for the decision. - - This function is a security measure to prevent resource wastage on undesired requests. It should be enhanced - to include checks against the metagraph for entity registration, validator status, and sufficient stake - before deserialization of synapse data to minimize processing overhead. - - Example blacklist logic: - - Reject if the hotkey is not a registered entity within the metagraph. - - Consider blacklisting entities that are not validators or have insufficient stake. - - In practice it would be wise to blacklist requests from entities that are not validators, or do not have - enough stake. This can be checked via metagraph.S and metagraph.validator_permit. You can always attain - the uid of the sender via a metagraph.hotkeys.index( synapse.dendrite.hotkey ) call. - - Otherwise, allow the request to be processed further. - """ - - if synapse.dendrite is None or synapse.dendrite.hotkey is None: - bt.logging.warning("Received a request without a dendrite or hotkey.") - return True, "Missing dendrite or hotkey" - - # TODO(developer): Define how miners should blacklist requests. - uid = self.metagraph.hotkeys.index(synapse.dendrite.hotkey) - if ( - not self.config.blacklist.allow_non_registered - and synapse.dendrite.hotkey not in self.metagraph.hotkeys - ): - # Ignore requests from un-registered entities. - bt.logging.trace( - f"Blacklisting un-registered hotkey {synapse.dendrite.hotkey}" - ) - return True, "Unrecognized hotkey" - - if self.config.blacklist.force_validator_permit: - # If the config is set to force validator permit, then we should only allow requests from validators. - if not self.metagraph.validator_permit[uid]: - bt.logging.warning( - f"Blacklisting a request from non-validator hotkey {synapse.dendrite.hotkey}" - ) - return True, "Non-validator hotkey" - - bt.logging.trace( - f"Not Blacklisting recognized hotkey {synapse.dendrite.hotkey}" - ) - return False, "Hotkey recognized!" - - async def priority(self, synapse: cancer_ai.protocol.Dummy) -> float: - """ - The priority function determines the order in which requests are handled. More valuable or higher-priority - requests are processed before others. You should design your own priority mechanism with care. - - This implementation assigns priority to incoming requests based on the calling entity's stake in the metagraph. - - Args: - synapse (template.protocol.Dummy): The synapse object that contains metadata about the incoming request. - - Returns: - float: A priority score derived from the stake of the calling entity. - - Miners may receive messages from multiple entities at once. This function determines which request should be - processed first. Higher values indicate that the request should be processed first. Lower values indicate - that the request should be processed later. - - Example priority logic: - - A higher stake results in a higher priority value. - """ - if synapse.dendrite is None or synapse.dendrite.hotkey is None: - bt.logging.warning("Received a request without a dendrite or hotkey.") - return 0.0 - - # TODO(developer): Define how miners should prioritize requests. - caller_uid = self.metagraph.hotkeys.index( - synapse.dendrite.hotkey - ) # Get the caller index. - priority = float( - self.metagraph.S[caller_uid] - ) # Return the stake as the priority. - bt.logging.trace( - f"Prioritizing {synapse.dendrite.hotkey} with value: {priority}" - ) - return priority - - async def store_and_retrieve_metadata_on_chain(self, competition: str) -> None: - """ - PoC function to integrate with the structured business logic - """ - - model_id = ChainMinerModel(namespace=self.config.models.namespace, name=self.config.models.model_name, epoch=self.config.models.epoch_checkpoint, - date=datetime.datetime.now(), competition_id=competition, block=None) - - await self.metadata_store.store_model_metadata(model_id) - bt.logging.success(f"Model successfully pushed model metadata on chain. Model ID: {model_id}") - - time.sleep(10) - - model_metadata = await self.metadata_store.retrieve_model_metadata(self.wallet.hotkey.ss58_address) - - time.sleep(10) - print("Model Metadata name: ", model_metadata.id.name) - - -# This is the main function, which runs the miner. -if __name__ == "__main__": - with Miner() as miner: - while True: - bt.logging.info(f"Miner running... {time.time()}") - time.sleep(5) - diff --git a/neurons/miner3.py b/neurons/miner3.py index a508cbdd..0a6acb1f 100644 --- a/neurons/miner3.py +++ b/neurons/miner3.py @@ -1,41 +1,60 @@ -import argparse -import sys import asyncio -from typing import Optional import bittensor as bt from dotenv import load_dotenv from huggingface_hub import HfApi +import huggingface_hub import onnx +import cancer_ai +import typing +import datetime -from neurons.miner_config import get_config, set_log_formatting from cancer_ai.validator.utils import ModelType, run_command from cancer_ai.validator.model_run_manager import ModelRunManager, ModelInfo from cancer_ai.validator.dataset_manager import DatasetManager from cancer_ai.validator.model_manager import ModelManager -from datetime import datetime +from cancer_ai.base.miner import BaseMinerNeuron +from cancer_ai.chain_models_store import ChainMinerModel, ChainModelMetadataStore -class MinerManagerCLI: - def __init__(self, config: bt.config): - self.config = config +class MinerManagerCLI(BaseMinerNeuron): + def __init__(self, config=None): + super(MinerManagerCLI, self).__init__(config=config) + self.metadata_store = ChainModelMetadataStore(subtensor=self.subtensor, + subnet_uid=self.config.netuid, wallet=self.wallet) self.hf_api = HfApi() + # TODO: Dive into BaseNeuron to switch off requirement to implement legacy methods, for now they are mocked. + async def forward( + self, synapse: cancer_ai.protocol.Dummy + ) -> cancer_ai.protocol.Dummy: + ... + + async def blacklist( + self, synapse: cancer_ai.protocol.Dummy + ) -> typing.Tuple[bool, str]: + ... + + async def priority(self, synapse: cancer_ai.protocol.Dummy) -> float: + ... + async def upload_to_hf(self) -> None: """Uploads model and code to Hugging Face.""" bt.logging.info("Uploading model to Hugging Face.") - now_str = datetime.now().strftime("%Y-%m-%d_%H-%M-%S") path = self.hf_api.upload_file( path_or_fileobj=self.config.model_path, - path_in_repo=f"{now_str}-{self.config.competition_id}.onnx", + path_in_repo=f"{self.config.competition_id}-{self.config.hf_model_name}.onnx", repo_id=self.config.hf_repo_id, repo_type="model", + token=self.config.hf_token, ) + bt.logging.info("Uploading code to Hugging Face.") path = self.hf_api.upload_file( path_or_fileobj=f"{self.config.code_directory}/code.zip", - path_in_repo=f"{now_str}-{self.config.competition_id}.zip", + path_in_repo=f"{self.config.competition_id}-{self.config.hf_model_name}.zip", repo_id=self.config.hf_repo_id, repo_type="model", + token=self.config.hf_token, ) bt.logging.info(f"Uploaded model to Hugging Face: {path}") @@ -57,7 +76,6 @@ async def evaluate_model(self) -> None: ) dataset_manager = DatasetManager( self.config, - self.config.competition_id, "safescanai/test_dataset", "skin_melanoma.zip", "dataset", @@ -68,9 +86,6 @@ async def evaluate_model(self) -> None: model_predictions = await run_manager.run(pred_x) - print(pred_y) - print(model_predictions) - if self.config.clean_after_run: dataset_manager.delete_dataset() @@ -80,16 +95,20 @@ async def compress_code(self) -> str: f"zip {self.config.code_directory}/code.zip {self.config.code_directory}/*" ) return f"{self.config.code_directory}/code.zip" - + async def submit_model(self) -> None: - bt.logging.info( - f"Initializing connection with Bittensor subnet {self.config.netuid} - Safe-Scan Project" - ) - bt.logging.info(f"Subtensor network: {self.config.subtensor.network}") - bt.logging.info(f"Wallet hotkey: {self.config.wallet.hotkey.ss58_address}") - wallet = self.wallet - subtensor = self.subtensor - metagraph = self.metagraph + # Check if the required model and files are present in hugging face repo + filenames = [self.config.hf_model_name + ".onnx", self.config.hf_model_name + ".zip"] + for file in filenames: + if not huggingface_hub.file_exists(repo_id=self.config.hf_repo_id, filename=file, token=self.config.hf_token): + bt.logging.error(f"{file} not found in Hugging Face repo") + return + bt.logging.info("Model and code found in Hugging Face repo") + + # Push model metadata to chain + model_id = ChainMinerModel(hf_repo_id=self.config.hf_repo_id, name=self.config.hf_model_name, date=datetime.datetime.now(), competition_id=self.config.competition_id, block=None) + await self.metadata_store.store_model_metadata(model_id) + bt.logging.success(f"Successfully pushed model metadata on chain. Model ID: {model_id}") async def main(self) -> None: bt.logging(config=self.config) @@ -119,5 +138,5 @@ async def main(self) -> None: config = SimpleNamespace( **config) set_log_formatting() load_dotenv() - cli_manager = MinerManagerCLI(config) + cli_manager = MinerManagerCLI() asyncio.run(cli_manager.main()) diff --git a/neurons/miner_config.py b/neurons/miner_config.py deleted file mode 100644 index f82c3859..00000000 --- a/neurons/miner_config.py +++ /dev/null @@ -1,133 +0,0 @@ -import argparse - -from colorama import init, Fore, Back, Style -import bittensor as bt -from bittensor.btlogging import format - - -help = """ -How to run it: - -python3 neurons/miner2.py \ - evaluate \ - --logging.debug \ - --model_path /path/to/model \ - --competition_id "your_competition_id" - -python3 upload neurons/miner2.py \ - --model_path /path/to/model - --hf_repo_id "hf_org_id/your_hf_repo_id" - -python3 neurons/miner2.py \ - submit \ - --netuid 163 \ - --subtensor.network test \ - --wallet.name miner \ - --wallet.hotkey hot_validator \ - --model_path /path/to/model -""" -import argparse -import bittensor as bt - -import argparse - - -def set_log_formatting() -> None: - """Override bittensor logging formats.""" - - - format.LOG_TRACE_FORMATS = { - level: f"{Fore.BLUE}%(asctime)s{Fore.RESET}" - f" | {Style.BRIGHT}{color}%(levelname)s{Fore.RESET}{Back.RESET}{Style.RESET_ALL}" - f" |%(message)s" - for level, color in format.log_level_color_prefix.items() - } - - format.DEFAULT_LOG_FORMAT = ( - f"{Fore.BLUE}%(asctime)s{Fore.RESET} | " - f"{Style.BRIGHT}{Fore.WHITE}%(levelname)s{Style.RESET_ALL} | " - "%(message)s" - ) - - format.DEFAULT_TRACE_FORMAT = ( - f"{Fore.BLUE}%(asctime)s{Fore.RESET} | " - f"{Style.BRIGHT}{Fore.WHITE}%(levelname)s{Style.RESET_ALL} | " - f" %(message)s" - ) - - -def get_config() -> bt.config: - main_parser = argparse.ArgumentParser() - - main_parser.add_argument( - "--action", - choices=["submit", "evaluate", "upload"], - # required=True, - default="evaluate", - ) - main_parser.add_argument( - "--model_path", - type=str, - # required=True, - help="Path to ONNX model, used for evaluation", - default="neurons/simple_cnn_model.onnx", - ) - main_parser.add_argument( - "--competition_id", - type=str, - # required=True, - help="Competition ID", - default="melanoma-1", - ) - - main_parser.add_argument( - "--dataset_dir", - type=str, - help="Path for storing datasets.", - default="./datasets", - ) - # Subparser for upload command - - main_parser.add_argument( - "--hf_repo_id", - type=str, - required=False, - help="Hugging Face model repository ID", - default="eatcats/melanoma-test", - ) - - main_parser.add_argument( - "--clean-after-run", - action="store_true", - help="Whether to clean up (dataset, temporary files) after running", - default=False, - ) - main_parser.add_argument( - "--code-directory", - type=str, - help="Path to code directory", - default=".", - ) - - # Add additional args from bt modules - bt.wallet.add_args(main_parser) - bt.subtensor.add_args(main_parser) - bt.logging.add_args(main_parser) - - # Parse the arguments and return the config - # config = bt.config(main_parser) - # parsed = main_parser.parse_args() - # config = bt.config(main_parser) - - - config = main_parser.parse_args() - config.logging_dir = "./" - config.record_log = True - config.trace = True - config.debug = False - return config - - -if __name__ == "__main__": - config = get_config() - print(config) From 5d76dd69873b70a292bae5d41fe2b87824b91871 Mon Sep 17 00:00:00 2001 From: Konrad Date: Tue, 27 Aug 2024 21:46:48 +0200 Subject: [PATCH 094/227] some adjustments --- cancer_ai/utils/config.py | 1 - neurons/{miner3.py => miner.py} | 0 2 files changed, 1 deletion(-) rename neurons/{miner3.py => miner.py} (100%) diff --git a/cancer_ai/utils/config.py b/cancer_ai/utils/config.py index 8bc73d0f..c9f51f68 100644 --- a/cancer_ai/utils/config.py +++ b/cancer_ai/utils/config.py @@ -218,7 +218,6 @@ def add_miner_args(cls, parser): parser.add_argument( "--hf_repo_id", type=str, - # required=False, help="Hugging Face model repository ID", default="", ) diff --git a/neurons/miner3.py b/neurons/miner.py similarity index 100% rename from neurons/miner3.py rename to neurons/miner.py From 3f0517db7ac252c6eb5f4744311c7065da994c76 Mon Sep 17 00:00:00 2001 From: Wojtek Jurkowlaniec Date: Thu, 29 Aug 2024 22:46:28 +0200 Subject: [PATCH 095/227] fixes to chain model --- cancer_ai/chain_models_store.py | 32 +++++++++--------------- neurons/miner.py | 44 +++++++++++++++++++++------------ 2 files changed, 39 insertions(+), 37 deletions(-) diff --git a/cancer_ai/chain_models_store.py b/cancer_ai/chain_models_store.py index 25b10062..ac115a8f 100644 --- a/cancer_ai/chain_models_store.py +++ b/cancer_ai/chain_models_store.py @@ -10,31 +10,18 @@ class ChainMinerModel(BaseModel): """Uniquely identifies a trained model""" - hf_repo_id: str = Field( - description="Namespace where the model can be found. ex. Hugging Face username/org." - ) - name: str = Field(description="Name of the model.") - - date: datetime.datetime = Field( - description="The datetime at which model was pushed to hugging face" - ) - - # Identifier for competition competition_id: Optional[str] = Field(description="The competition id") - - block: Optional[str] = Field( - description="Block on which this model was claimed on the chain." - ) - - # hf_filename: Optional[str] = Field(description="Hugging Face filename.") - # hf_repo_type: Optional[str] = Field(description="Hugging Face repo type.") + hf_repo_id: str | None = None + hf_model_filename: str | None = None + hf_code_filename: str | None = None + hf_repo_type: str | None = None class Config: arbitrary_types_allowed = True def to_compressed_str(self) -> str: """Returns a compressed string representation.""" - return f"{self.hf_repo_id}:{self.name}:{self.date}:{self.competition_id}" + return f"{self.hf_repo_id}:{self.name}:{self.date}:{self.competition_id}:{self.hf_repo_type}:{self.hf_model_filename}:{self.hf_code_filename}" @classmethod def from_compressed_str(cls, cs: str) -> Type["ChainMinerModel"]: @@ -43,8 +30,11 @@ def from_compressed_str(cls, cs: str) -> Type["ChainMinerModel"]: return cls( hf_repo_id=tokens[0], name=tokens[1], - date=tokens[2] if tokens[2] != "None" else None, - competition_id=tokens[3] if tokens[3] != "None" else None, + date=tokens[2], + competition_id=tokens[3], + hf_repo_type=tokens[4], + hf_model_filename=tokens[5], + hf_code_filename=tokens[6], ) @@ -103,4 +93,4 @@ async def retrieve_model_metadata(self, hotkey: str) -> Optional[ChainMinerModel return None # The block id at which the metadata is stored model.block = metadata["block"] - return model \ No newline at end of file + return model diff --git a/neurons/miner.py b/neurons/miner.py index 0a6acb1f..1b91af08 100644 --- a/neurons/miner.py +++ b/neurons/miner.py @@ -7,12 +7,10 @@ import onnx import cancer_ai import typing -import datetime -from cancer_ai.validator.utils import ModelType, run_command +from cancer_ai.validator.utils import run_command from cancer_ai.validator.model_run_manager import ModelRunManager, ModelInfo from cancer_ai.validator.dataset_manager import DatasetManager -from cancer_ai.validator.model_manager import ModelManager from cancer_ai.base.miner import BaseMinerNeuron from cancer_ai.chain_models_store import ChainMinerModel, ChainModelMetadataStore @@ -20,23 +18,21 @@ class MinerManagerCLI(BaseMinerNeuron): def __init__(self, config=None): super(MinerManagerCLI, self).__init__(config=config) - self.metadata_store = ChainModelMetadataStore(subtensor=self.subtensor, - subnet_uid=self.config.netuid, wallet=self.wallet) + self.metadata_store = ChainModelMetadataStore( + subtensor=self.subtensor, subnet_uid=self.config.netuid, wallet=self.wallet + ) self.hf_api = HfApi() # TODO: Dive into BaseNeuron to switch off requirement to implement legacy methods, for now they are mocked. async def forward( self, synapse: cancer_ai.protocol.Dummy - ) -> cancer_ai.protocol.Dummy: - ... + ) -> cancer_ai.protocol.Dummy: ... async def blacklist( self, synapse: cancer_ai.protocol.Dummy - ) -> typing.Tuple[bool, str]: - ... + ) -> typing.Tuple[bool, str]: ... - async def priority(self, synapse: cancer_ai.protocol.Dummy) -> float: - ... + async def priority(self, synapse: cancer_ai.protocol.Dummy) -> float: ... async def upload_to_hf(self) -> None: """Uploads model and code to Hugging Face.""" @@ -95,20 +91,36 @@ async def compress_code(self) -> str: f"zip {self.config.code_directory}/code.zip {self.config.code_directory}/*" ) return f"{self.config.code_directory}/code.zip" - + async def submit_model(self) -> None: # Check if the required model and files are present in hugging face repo - filenames = [self.config.hf_model_name + ".onnx", self.config.hf_model_name + ".zip"] + filenames = [ + self.config.hf_model_name + ".onnx", + self.config.hf_model_name + ".zip", + ] for file in filenames: - if not huggingface_hub.file_exists(repo_id=self.config.hf_repo_id, filename=file, token=self.config.hf_token): + if not huggingface_hub.file_exists( + repo_id=self.config.hf_repo_id, + filename=file, + token=self.config.hf_token, + ): bt.logging.error(f"{file} not found in Hugging Face repo") return bt.logging.info("Model and code found in Hugging Face repo") # Push model metadata to chain - model_id = ChainMinerModel(hf_repo_id=self.config.hf_repo_id, name=self.config.hf_model_name, date=datetime.datetime.now(), competition_id=self.config.competition_id, block=None) + model_id = ChainMinerModel( + competition_id=self.config.competition_id, + hf_repo_id=self.config.hf_repo_id, + hf_repo_type=self.config.hf_repo_type, + hf_model_name=self.config.hf_model_name, + hf_model_filename=self.config.hf_model_name + ".onnx", + hf_code_filename=self.config.hf_model_name + ".zip", + ) await self.metadata_store.store_model_metadata(model_id) - bt.logging.success(f"Successfully pushed model metadata on chain. Model ID: {model_id}") + bt.logging.success( + f"Successfully pushed model metadata on chain. Model ID: {model_id}" + ) async def main(self) -> None: bt.logging(config=self.config) From b5c57da0f9c8570091e1150b0bedbd85ec8e44f7 Mon Sep 17 00:00:00 2001 From: Wojtek Jurkowlaniec Date: Fri, 30 Aug 2024 15:01:04 +0200 Subject: [PATCH 096/227] bittensor connection only on submitting models, fixes in configuration, fixes in competition logic --- DOCS/miner.md | 51 +++++++++ cancer_ai/utils/config.py | 76 +++---------- .../competition_handlers/melanoma_handler.py | 2 +- cancer_ai/validator/competition_manager.py | 3 +- cancer_ai/validator/dataset_manager.py | 2 +- .../validator/model_runners/onnx_runner.py | 2 +- .../model_runners/tensorflow_runner.py | 2 +- neurons/miner.py | 102 ++++++++++++------ 8 files changed, 142 insertions(+), 98 deletions(-) create mode 100644 DOCS/miner.md diff --git a/DOCS/miner.md b/DOCS/miner.md new file mode 100644 index 00000000..c8ef4282 --- /dev/null +++ b/DOCS/miner.md @@ -0,0 +1,51 @@ +# Miner + +## Installation + +- create virtualenv + +`virtualenv venv --python=3.10 + +- activate it + +`source venv/bin/activate` + +- install requirements + +`pip install -r requirements.txt` + +## Run + +Prerequirements + +- make sure you are in base directory of the project +- activate your virtualenv +- run `export PYTHONPATH="${PYTHONPATH}:./"` + + +### Evaluate model localy + +This mode will do following things +- download dataset +- load your model +- prepare data for executing +- prints evaluation results + + + +`python neurons/miner.py --action evaluate --competition-id melanoma-1 --model-path test_model.onnx ` + +If flag `--clean-after-run` is supplied, it will delete dataset after evaluating the model + +### Upload to HuggingFace + +- compresses code provided by --code-path +- uploads model and code to HuggingFace + +`python neurons/miner.py --action upload --competition-id melanoma-1 --model-path test_model.onnx --hf-model-name file_name --hf-repo-id repo/id --hf-token TOKEN` + + + +### Send model to validators + +- saves model information in metagraph \ No newline at end of file diff --git a/cancer_ai/utils/config.py b/cancer_ai/utils/config.py index c9f51f68..34727bd0 100644 --- a/cancer_ai/utils/config.py +++ b/cancer_ai/utils/config.py @@ -131,118 +131,72 @@ def add_args(cls, parser): def add_miner_args(cls, parser): """Add miner specific arguments to the parser.""" - - parser.add_argument( - "--neuron.name", - type=str, - help="Trials for this neuron go in neuron.root / (wallet_cold - wallet_hot) / neuron.name. ", - default="miner", - ) - - parser.add_argument( - "--blacklist.force_validator_permit", - action="store_true", - help="If set, we will force incoming requests to have a permit.", - default=False, - ) - parser.add_argument( - "--blacklist.allow_non_registered", - action="store_true", - help="If set, miners will accept queries from non registered entities. (Dangerous!)", - default=False, - ) - - parser.add_argument( - "--wandb.project_name", - type=str, - default="template-miners", - help="Wandb project to log to.", - ) - - parser.add_argument( - "--wandb.entity", + "--competition-id", type=str, - default="opentensor-dev", - help="Wandb entity to log to.", + help="Competition ID", ) parser.add_argument( - "--competition.entity", + "--model-dir", type=str, - default="opentensor-dev", - help="Wandb entity to log to.", + help="Path for for loading the starting model related to a training run.", + default="./models", ) parser.add_argument( - "--load_model_dir", + "--hf-repo-id", type=str, - help="Path for for loading the starting model related to a training run.", - default="./models", + help="Hugging Face model repository ID", + default="", ) parser.add_argument( - "--hf_model_name", + "--hf-model-name", type=str, help="Name of the model to push to hugging face.", - default="", ) parser.add_argument( "--action", choices=["submit", "evaluate", "upload"], - default="submit", ) parser.add_argument( - "--model_path", + "--model-path", type=str, help="Path to ONNX model, used for evaluation", - default="", ) parser.add_argument( - "--competition_id", - type=str, - help="Competition ID", - default="melanoma-1", - ) - - parser.add_argument( - "--dataset_dir", + "--dataset-dir", type=str, help="Path for storing datasets.", default="./datasets", ) parser.add_argument( - "--hf_repo_id", - type=str, - help="Hugging Face model repository ID", - default="", - ) - - parser.add_argument( - "--hf_token", + "--hf-token", type=str, help="Hugging Face API token", default="", ) parser.add_argument( - "--clean_after_run", + "--clean-after-run", action="store_true", help="Whether to clean up (dataset, temporary files) after running", default=False, ) parser.add_argument( - "--code_directory", + "--code-directory", type=str, help="Path to code directory", default=".", ) + def add_validator_args(cls, parser): """Add validator specific arguments to the parser.""" diff --git a/cancer_ai/validator/competition_handlers/melanoma_handler.py b/cancer_ai/validator/competition_handlers/melanoma_handler.py index f6ef85b5..311c6dd1 100644 --- a/cancer_ai/validator/competition_handlers/melanoma_handler.py +++ b/cancer_ai/validator/competition_handlers/melanoma_handler.py @@ -14,7 +14,7 @@ def __init__(self, X_test, y_test) -> None: def preprocess_data(self): new_X_test = [] - target_size=(224, 224) #TODO: Change this to the correct size + target_size=(224, 224) # TODO: Change this to the correct size for img in self.X_test: img = img.resize(target_size) diff --git a/cancer_ai/validator/competition_manager.py b/cancer_ai/validator/competition_manager.py index 3ee9a786..39c776dc 100644 --- a/cancer_ai/validator/competition_manager.py +++ b/cancer_ai/validator/competition_manager.py @@ -59,7 +59,7 @@ def __init__( self.results = [] self.model_manager = ModelManager(config) self.dataset_manager = DatasetManager( - config, competition_id, dataset_hf_repo, dataset_hf_id, dataset_hf_repo_type + config, dataset_hf_repo, dataset_hf_id, dataset_hf_repo_type ) self.chain_model_metadata_store = ChainModelMetadataStore(subtensor, subnet_uid) @@ -109,6 +109,7 @@ async def sync_chain_miners(self, hotkeys: list[str]): ) async def evaluate(self): + await self.dataset_manager.prepare_dataset() X_test, y_test = await self.dataset_manager.get_data() diff --git a/cancer_ai/validator/dataset_manager.py b/cancer_ai/validator/dataset_manager.py index 900ab59c..dbf3b9cd 100644 --- a/cancer_ai/validator/dataset_manager.py +++ b/cancer_ai/validator/dataset_manager.py @@ -68,7 +68,7 @@ def delete_dataset(self) -> None: bt.logging.info("Deleting dataset: ") try: - shutil.rmtree(self.local_compressed_path) + shutil.rmtree(self.local_extracted_dir) bt.logging.info("Dataset deleted") except OSError as e: bt.logging.error(f"Failed to delete dataset from disk: {e}") diff --git a/cancer_ai/validator/model_runners/onnx_runner.py b/cancer_ai/validator/model_runners/onnx_runner.py index 6159b82e..f394ba9f 100644 --- a/cancer_ai/validator/model_runners/onnx_runner.py +++ b/cancer_ai/validator/model_runners/onnx_runner.py @@ -2,7 +2,7 @@ from typing import List class OnnxRunnerHandler(BaseRunnerHandler): - def run(self, X_test: List) -> List: + async def run(self, X_test: List) -> List: import onnxruntime import numpy as np diff --git a/cancer_ai/validator/model_runners/tensorflow_runner.py b/cancer_ai/validator/model_runners/tensorflow_runner.py index eb6fea95..37397db8 100644 --- a/cancer_ai/validator/model_runners/tensorflow_runner.py +++ b/cancer_ai/validator/model_runners/tensorflow_runner.py @@ -3,7 +3,7 @@ class TensorflowRunnerHandler(BaseRunnerHandler): - def run(self, pred_x: List) -> List: + async def run(self, pred_x: List) -> List: import tensorflow as tf import numpy as np from tensorflow.keras.preprocessing.image import load_img diff --git a/neurons/miner.py b/neurons/miner.py index 1b91af08..63140271 100644 --- a/neurons/miner.py +++ b/neurons/miner.py @@ -1,53 +1,62 @@ import asyncio +import copy +import time import bittensor as bt from dotenv import load_dotenv -from huggingface_hub import HfApi +from huggingface_hub import HfApi, login as hf_login import huggingface_hub import onnx import cancer_ai import typing +import argparse from cancer_ai.validator.utils import run_command from cancer_ai.validator.model_run_manager import ModelRunManager, ModelInfo from cancer_ai.validator.dataset_manager import DatasetManager -from cancer_ai.base.miner import BaseMinerNeuron +from cancer_ai.validator.competition_manager import COMPETITION_HANDLER_MAPPING + +from cancer_ai.base.miner import BaseNeuron from cancer_ai.chain_models_store import ChainMinerModel, ChainModelMetadataStore +from cancer_ai.utils.config import path_config, add_miner_args -class MinerManagerCLI(BaseMinerNeuron): +class MinerManagerCLI: def __init__(self, config=None): - super(MinerManagerCLI, self).__init__(config=config) - self.metadata_store = ChainModelMetadataStore( - subtensor=self.subtensor, subnet_uid=self.config.netuid, wallet=self.wallet - ) - self.hf_api = HfApi() - - # TODO: Dive into BaseNeuron to switch off requirement to implement legacy methods, for now they are mocked. - async def forward( - self, synapse: cancer_ai.protocol.Dummy - ) -> cancer_ai.protocol.Dummy: ... - async def blacklist( - self, synapse: cancer_ai.protocol.Dummy - ) -> typing.Tuple[bool, str]: ... + # setting basic Bittensor objects + base_config = copy.deepcopy(config or BaseNeuron.config()) + self.config = path_config(self) + self.config.merge(base_config) + BaseNeuron.check_config(self.config) + bt.logging.set_config(config=self.config.logging) + bt.logging.info(self.config) - async def priority(self, synapse: cancer_ai.protocol.Dummy) -> float: ... + @classmethod + def add_args(cls, parser: argparse.ArgumentParser): + """Method for injecting miner arguments to the parser.""" + add_miner_args(cls, parser) async def upload_to_hf(self) -> None: """Uploads model and code to Hugging Face.""" bt.logging.info("Uploading model to Hugging Face.") - path = self.hf_api.upload_file( + hf_api = HfApi() + hf_login(token=self.config.hf_token) + + hf_model_path = f"{self.config.competition_id}-{self.config.hf_model_name}.onnx" + hf_code_path = f"{self.config.competition_id}-{self.config.hf_model_name}.zip" + + path = hf_api.upload_file( path_or_fileobj=self.config.model_path, - path_in_repo=f"{self.config.competition_id}-{self.config.hf_model_name}.onnx", + path_in_repo=hf_model_path, repo_id=self.config.hf_repo_id, repo_type="model", token=self.config.hf_token, ) bt.logging.info("Uploading code to Hugging Face.") - path = self.hf_api.upload_file( + path = hf_api.upload_file( path_or_fileobj=f"{self.config.code_directory}/code.zip", - path_in_repo=f"{self.config.competition_id}-{self.config.hf_model_name}.zip", + path_in_repo=hf_code_path, repo_id=self.config.hf_repo_id, repo_type="model", token=self.config.hf_token, @@ -67,6 +76,7 @@ def is_onnx_model(model_path: str) -> bool: async def evaluate_model(self) -> None: bt.logging.info("Evaluate model mode") + run_manager = ModelRunManager( config=self.config, model=ModelInfo(file_path=self.config.model_path) ) @@ -78,10 +88,19 @@ async def evaluate_model(self) -> None: ) await dataset_manager.prepare_dataset() - pred_x, pred_y = await dataset_manager.get_data() + X_test, y_test = await dataset_manager.get_data() - model_predictions = await run_manager.run(pred_x) + competition_handler = COMPETITION_HANDLER_MAPPING[self.config.competition_id]( + X_test=X_test, y_test=y_test + ) + X_test, y_test = competition_handler.preprocess_data() + + start_time = time.time() + y_pred = await run_manager.run(X_test) + run_time_s = time.time() - start_time + model_result = competition_handler.get_model_result(y_test, y_pred, run_time_s) + bt.logging.info(model_result) if self.config.clean_after_run: dataset_manager.delete_dataset() @@ -98,6 +117,24 @@ async def submit_model(self) -> None: self.config.hf_model_name + ".onnx", self.config.hf_model_name + ".zip", ] + self.wallet = bt.wallet(config=self.config) + self.subtensor = bt.subtensor(config=self.config) + self.metagraph = self.subtensor.metagraph(self.config.netuid) + bt.logging.info(f"Wallet: {self.wallet}") + bt.logging.info(f"Subtensor: {self.subtensor}") + bt.logging.info(f"Metagraph: {self.metagraph}") + if not self.subtensor.is_hotkey_registered( + netuid=self.config.netuid, + hotkey_ss58=self.wallet.hotkey.ss58_address, + ): + bt.logging.error( + f"Wallet: {self.wallet} is not registered on netuid {self.config.netuid}." + f" Please register the hotkey using `btcli subnets register` before trying again" + ) + exit() + self.metadata_store = ChainModelMetadataStore( + subtensor=self.subtensor, subnet_uid=self.config.netuid, wallet=self.wallet + ) for file in filenames: if not huggingface_hub.file_exists( repo_id=self.config.hf_repo_id, @@ -110,12 +147,11 @@ async def submit_model(self) -> None: # Push model metadata to chain model_id = ChainMinerModel( - competition_id=self.config.competition_id, hf_repo_id=self.config.hf_repo_id, - hf_repo_type=self.config.hf_repo_type, - hf_model_name=self.config.hf_model_name, - hf_model_filename=self.config.hf_model_name + ".onnx", - hf_code_filename=self.config.hf_model_name + ".zip", + name=self.config.hf_model_name, + date=datetime.datetime.now(), + competition_id=self.config.competition_id, + block=None, ) await self.metadata_store.store_model_metadata(model_id) bt.logging.success( @@ -123,10 +159,12 @@ async def submit_model(self) -> None: ) async def main(self) -> None: - bt.logging(config=self.config) - - if not self.is_onnx_model(self.config.model_path): - bt.logging.error("Provided model with --model_type is not in ONNX format") + # bt.logging(config=self.config) + if not self.config.model_path: + bt.logging.error("Missing --model-path argument") + return + if not MinerManagerCLI.is_onnx_model(self.config.model_path): + bt.logging.error("Provided model with is not in ONNX format") return match self.config.action: From 5cc3eef79b464eee29b3657e89c5d15adf750a1b Mon Sep 17 00:00:00 2001 From: Wojtek Jurkowlaniec Date: Sat, 31 Aug 2024 00:03:04 +0200 Subject: [PATCH 097/227] working uploading model info to chain --- DOCS/miner.md | 35 +++++++++++++++++++--- cancer_ai/chain_models_store.py | 8 ++--- cancer_ai/utils/config.py | 12 +++++++- neurons/miner.py | 53 ++++++++++++++++++++------------- 4 files changed, 77 insertions(+), 31 deletions(-) diff --git a/DOCS/miner.md b/DOCS/miner.md index c8ef4282..89f2a79f 100644 --- a/DOCS/miner.md +++ b/DOCS/miner.md @@ -33,7 +33,7 @@ This mode will do following things -`python neurons/miner.py --action evaluate --competition-id melanoma-1 --model-path test_model.onnx ` +`python neurons/miner.py --action evaluate --competition-id --model-path ` If flag `--clean-after-run` is supplied, it will delete dataset after evaluating the model @@ -42,10 +42,37 @@ If flag `--clean-after-run` is supplied, it will delete dataset after evaluating - compresses code provided by --code-path - uploads model and code to HuggingFace -`python neurons/miner.py --action upload --competition-id melanoma-1 --model-path test_model.onnx --hf-model-name file_name --hf-repo-id repo/id --hf-token TOKEN` - +`python neurons/miner.py --action upload --competition-id melanoma-1 --model-path test_model.onnx --hf-model-name file_name.zip --hf-repo-id repo/id --hf-token TOKEN` +```bash +python neurons/miner.py \ + --action upload \ + --competition-id \ + --model-path \ + --code-directory \ + --hf-model-name \ + --hf-repo-id \ + --hf-token \ + --logging.debug +``` ### Send model to validators -- saves model information in metagraph \ No newline at end of file +- saves model information in metagraph +- validator can get information about your model to test it + +```bash +python neurons/miner.py \ + --action submit \ + --model-path \ + --competition-id \ + --hf-code-filename "melanoma-1-piwo.zip" \ + --hf-model-name \ + --hf-repo-id \ + --hf-repo-type model \ + --wallet.name \ + --wallet.hotkey \ + --netuid \ + --subtensor.network \ + --logging.debug + ``` \ No newline at end of file diff --git a/cancer_ai/chain_models_store.py b/cancer_ai/chain_models_store.py index ac115a8f..6d1e3be4 100644 --- a/cancer_ai/chain_models_store.py +++ b/cancer_ai/chain_models_store.py @@ -21,7 +21,7 @@ class Config: def to_compressed_str(self) -> str: """Returns a compressed string representation.""" - return f"{self.hf_repo_id}:{self.name}:{self.date}:{self.competition_id}:{self.hf_repo_type}:{self.hf_model_filename}:{self.hf_code_filename}" + return f"{self.hf_repo_id}:{self.hf_model_filename}:{self.hf_code_filename}:{self.competition_id}:{self.hf_repo_type}" @classmethod def from_compressed_str(cls, cs: str) -> Type["ChainMinerModel"]: @@ -29,12 +29,10 @@ def from_compressed_str(cls, cs: str) -> Type["ChainMinerModel"]: tokens = cs.split(":") return cls( hf_repo_id=tokens[0], - name=tokens[1], - date=tokens[2], + hf_model_filename=tokens[1], + hf_code_filename=tokens[2], competition_id=tokens[3], hf_repo_type=tokens[4], - hf_model_filename=tokens[5], - hf_code_filename=tokens[6], ) diff --git a/cancer_ai/utils/config.py b/cancer_ai/utils/config.py index 34727bd0..c9fa16ed 100644 --- a/cancer_ai/utils/config.py +++ b/cancer_ai/utils/config.py @@ -154,7 +154,17 @@ def add_miner_args(cls, parser): parser.add_argument( "--hf-model-name", type=str, - help="Name of the model to push to hugging face.", + help="Filename of the model to push to hugging face.", + ) + parser.add_argument( + "--hf-code-filename", + type=str, + help="Filename of the code zip to push to hugging face.", + ) + parser.add_argument( + "--hf-repo-type", + type=str, + help="Type of hugging face repository.", ) parser.add_argument( diff --git a/neurons/miner.py b/neurons/miner.py index 63140271..f71f821b 100644 --- a/neurons/miner.py +++ b/neurons/miner.py @@ -43,8 +43,8 @@ async def upload_to_hf(self) -> None: hf_api = HfApi() hf_login(token=self.config.hf_token) - hf_model_path = f"{self.config.competition_id}-{self.config.hf_model_name}.onnx" - hf_code_path = f"{self.config.competition_id}-{self.config.hf_model_name}.zip" + hf_model_path = f"{self.config.competition_id}-{self.config.hf_model_name}" + hf_code_path = f"{self.config.competition_id}-{self.config.hf_model_name}" path = hf_api.upload_file( path_or_fileobj=self.config.model_path, @@ -55,7 +55,7 @@ async def upload_to_hf(self) -> None: ) bt.logging.info("Uploading code to Hugging Face.") path = hf_api.upload_file( - path_or_fileobj=f"{self.config.code_directory}/code.zip", + path_or_fileobj=f"{self.code_zip_path}", path_in_repo=hf_code_path, repo_id=self.config.hf_repo_id, repo_type="model", @@ -104,19 +104,18 @@ async def evaluate_model(self) -> None: if self.config.clean_after_run: dataset_manager.delete_dataset() - async def compress_code(self) -> str: + async def compress_code(self) -> None: bt.logging.info("Compressing code") + code_zip_path = f"{self.config.code_directory}/code.zip" out, err = await run_command( - f"zip {self.config.code_directory}/code.zip {self.config.code_directory}/*" + f"zip -r {code_zip_path} {self.config.code_directory}/*" ) - return f"{self.config.code_directory}/code.zip" + bt.logging.info(f"Code zip path: {code_zip_path}") + self.code_zip_path = code_zip_path async def submit_model(self) -> None: # Check if the required model and files are present in hugging face repo - filenames = [ - self.config.hf_model_name + ".onnx", - self.config.hf_model_name + ".zip", - ] + self.wallet = bt.wallet(config=self.config) self.subtensor = bt.subtensor(config=self.config) self.metagraph = self.subtensor.metagraph(self.config.netuid) @@ -135,23 +134,35 @@ async def submit_model(self) -> None: self.metadata_store = ChainModelMetadataStore( subtensor=self.subtensor, subnet_uid=self.config.netuid, wallet=self.wallet ) - for file in filenames: - if not huggingface_hub.file_exists( - repo_id=self.config.hf_repo_id, - filename=file, - token=self.config.hf_token, - ): - bt.logging.error(f"{file} not found in Hugging Face repo") - return + + if not huggingface_hub.file_exists( + repo_id=self.config.hf_repo_id, + filename=self.config.hf_model_name, + repo_type=self.config.hf_repo_type, + ): + bt.logging.error( + f"{self.config.hf_model_name} not found in Hugging Face repo" + ) + return + + if not huggingface_hub.file_exists( + repo_id=self.config.hf_repo_id, + filename=self.config.hf_code_filename, + repo_type=self.config.hf_repo_type, + ): + bt.logging.error( + f"{self.config.hf_model_name} not found in Hugging Face repo" + ) + return bt.logging.info("Model and code found in Hugging Face repo") # Push model metadata to chain model_id = ChainMinerModel( hf_repo_id=self.config.hf_repo_id, - name=self.config.hf_model_name, - date=datetime.datetime.now(), + hf_model_filename=self.config.hf_model_name, + hf_code_filename=self.config.hf_code_filename, competition_id=self.config.competition_id, - block=None, + hf_repo_type=self.config.hf_repo_type, ) await self.metadata_store.store_model_metadata(model_id) bt.logging.success( From 55d9cac17e4e59ad79dfba14fc72d5e97e01242e Mon Sep 17 00:00:00 2001 From: Wojtek Jurkowlaniec Date: Sat, 31 Aug 2024 01:45:03 +0200 Subject: [PATCH 098/227] Update DOCS/miner.md Co-authored-by: konrad0960 <71330299+konrad0960@users.noreply.github.com> --- DOCS/miner.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DOCS/miner.md b/DOCS/miner.md index 89f2a79f..62c9e210 100644 --- a/DOCS/miner.md +++ b/DOCS/miner.md @@ -29,7 +29,7 @@ This mode will do following things - download dataset - load your model - prepare data for executing -- prints evaluation results +- logs evaluation results From 7e65a986e60a4e982bb28a64721c0e57554c1353 Mon Sep 17 00:00:00 2001 From: Wojtek Jurkowlaniec Date: Sat, 31 Aug 2024 02:47:11 +0200 Subject: [PATCH 099/227] config --- cancer_ai/utils/config.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cancer_ai/utils/config.py b/cancer_ai/utils/config.py index c9fa16ed..7f42d131 100644 --- a/cancer_ai/utils/config.py +++ b/cancer_ai/utils/config.py @@ -135,6 +135,7 @@ def add_miner_args(cls, parser): "--competition-id", type=str, help="Competition ID", + required=True, ) parser.add_argument( @@ -148,7 +149,7 @@ def add_miner_args(cls, parser): "--hf-repo-id", type=str, help="Hugging Face model repository ID", - default="", + ) parser.add_argument( @@ -170,12 +171,14 @@ def add_miner_args(cls, parser): parser.add_argument( "--action", choices=["submit", "evaluate", "upload"], + required=True, ) parser.add_argument( "--model-path", type=str, help="Path to ONNX model, used for evaluation", + required=True, ) parser.add_argument( From c71d53f7ed98518643b137b9fd8fc8f18cc52da1 Mon Sep 17 00:00:00 2001 From: Wojtek Jurkowlaniec Date: Mon, 26 Aug 2024 16:59:35 +0200 Subject: [PATCH 100/227] in progress fixing competition scheduler --- neurons/competition_runner.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/neurons/competition_runner.py b/neurons/competition_runner.py index 94919e50..0c836f90 100644 --- a/neurons/competition_runner.py +++ b/neurons/competition_runner.py @@ -64,7 +64,7 @@ def log_results_to_wandb(project, entity, hotkey, evaluation_result: ModelEvalua async def schedule_competitions( - competitions: CompetitionManager, path_config: str + competitions: List[CompetitionManager], path_config: str ) -> None: # Cache the next evaluation times for each competition print("Initializing competitions") @@ -106,7 +106,7 @@ async def schedule_competitions( competition_config["dataset_hf_repo_type"], ) print(f"Evaluating competition {competition_id} at {now_utc}") - await competition_manager.evaluate() + results = await competition_manager.evaluate() print( f"Results for competition {competition_id}: {competition_manager.results}" ) From 9fe4a1a924981b8f834e83a6e86a8a68a15aa7e4 Mon Sep 17 00:00:00 2001 From: Wojtek Jurkowlaniec Date: Wed, 28 Aug 2024 17:12:19 +0200 Subject: [PATCH 101/227] competition scheduler --- cancer_ai/validator/competition_manager.py | 8 +- cancer_ai/validator/model_manager.py | 6 +- ...tion_config.py => competition_config.json} | 12 +- neurons/competition_runner.py | 116 ++++++++---------- 4 files changed, 69 insertions(+), 73 deletions(-) rename neurons/{competition_config.py => competition_config.json} (52%) diff --git a/cancer_ai/validator/competition_manager.py b/cancer_ai/validator/competition_manager.py index 39c776dc..157d0a7f 100644 --- a/cancer_ai/validator/competition_manager.py +++ b/cancer_ai/validator/competition_manager.py @@ -36,8 +36,8 @@ class CompetitionManager(SerializableManager): def __init__( self, config, - subtensor: bt.Subtensor, - subnet_uid: str, + subtensor: bt.Subtensor, # fetch from config, so not needed + subnet_uid: str, # fetch from config, so not needed competition_id: str, category: str, dataset_hf_repo: str, @@ -81,7 +81,7 @@ def set_state(self, state: dict): async def get_miner_model(self, chain_miner_model: ChainMinerModel): model_info = ModelInfo( hf_repo_id=chain_miner_model.hf_repo_id, - hf_filename=chain_miner_model.hf_filename, + hf_model_filename=chain_miner_model.hf_filename, hf_repo_type=chain_miner_model.hf_repo_type, ) return model_info @@ -116,7 +116,7 @@ async def evaluate(self): competition_handler = COMPETITION_HANDLER_MAPPING[self.competition_id]( X_test=X_test, y_test=y_test ) - + await self.sync_chain_miners([]) X_test, y_test = competition_handler.preprocess_data() for hotkey in self.model_manager.hotkey_store: diff --git a/cancer_ai/validator/model_manager.py b/cancer_ai/validator/model_manager.py index c1d7d43a..f2c45900 100644 --- a/cancer_ai/validator/model_manager.py +++ b/cancer_ai/validator/model_manager.py @@ -10,8 +10,10 @@ @dataclass class ModelInfo: hf_repo_id: str | None = None - hf_filename: str | None = None + hf_model_filename: str | None = None + hf_code_filename: str | None = None hf_repo_type: str | None = None + file_path: str | None = None model_type: str | None = None @@ -45,7 +47,7 @@ async def download_miner_model(self, hotkey) -> None: model_info = self.hotkey_store[hotkey] model_info.file_path = self.api.hf_hub_download( model_info.hf_repo_id, - model_info.hf_filename, + model_info.hf_model_filename, cache_dir=self.config.model_dir, repo_type=model_info.hf_repo_type, ) diff --git a/neurons/competition_config.py b/neurons/competition_config.json similarity index 52% rename from neurons/competition_config.py rename to neurons/competition_config.json index bae72d9d..f3d4695f 100644 --- a/neurons/competition_config.py +++ b/neurons/competition_config.json @@ -1,13 +1,15 @@ - -competitions = [ +[ { "competition_id": "melanoma-1", "category": "skin", - "evaluation_time": ["02:05", "15:30"], + "evaluation_time": [ + "02:05", + "15:30" + ], "dataset_hf_repo": "safescanai/test_dataset", "dataset_hf_filename": "skin_melanoma.zip", "dataset_hf_repo_type": "dataset", - "wandb_project": "testing_integration", - "wandb_entity": "urbaniak-bruno-safescanai", # TODO: Update this line to official entity + "wandb_project": "testing_integration", + "wandb_entity": "urbaniak-bruno-safescanai" } ] \ No newline at end of file diff --git a/neurons/competition_runner.py b/neurons/competition_runner.py index 0c836f90..e82c0567 100644 --- a/neurons/competition_runner.py +++ b/neurons/competition_runner.py @@ -10,17 +10,16 @@ import bittensor as bt from typing import List -from competition_config import competitions as competitions_cfg - import wandb # from cancer_ai.utils.config import config # TODO integrate with bt config path_config = SimpleNamespace( - **{"model_dir": "/tmp/models", "models_dataset_dir": "/tmp/datasets"} + **{"model_dir": "/tmp/models", "dataset_dir": "/tmp/datasets"} ) +competitions_cfg = json.load(open("neurons/competition_config.json", "r")) def calculate_next_evaluation_times(evaluation_times) -> List[datetime]: """Calculate the next evaluation times for a given list of times in UTC.""" @@ -62,65 +61,6 @@ def log_results_to_wandb(project, entity, hotkey, evaluation_result: ModelEvalua wandb.finish() return - -async def schedule_competitions( - competitions: List[CompetitionManager], path_config: str -) -> None: - # Cache the next evaluation times for each competition - print("Initializing competitions") - next_evaluation_times = {} - - # Calculate initial evaluation times - for competition_config in competitions: - competition_id = competition_config["competition_id"] - evaluation_times = competition_config["evaluation_time"] - next_evaluation_times[competition_id] = calculate_next_evaluation_times( - evaluation_times - ) - print( - f"Next evaluation times for competition {competition_id}: {next_evaluation_times[competition_id]}" - ) - - while True: - now_utc = datetime.now(timezone.utc) - - for competition_config in competitions: - competition_id = competition_config["competition_id"] - # Get the cached next evaluation times - next_times = next_evaluation_times[competition_id] - - for next_time in next_times: - if now_utc >= next_time: - print( - f"Next evaluation time for competition {competition_id} is {next_time}" - ) - # If it's time to run the competition - competition_manager = CompetitionManager( - path_config, - None, - 7, - competition_config["competition_id"], - competition_config["category"], - competition_config["dataset_hf_repo"], - competition_config["dataset_hf_filename"], - competition_config["dataset_hf_repo_type"], - ) - print(f"Evaluating competition {competition_id} at {now_utc}") - results = await competition_manager.evaluate() - print( - f"Results for competition {competition_id}: {competition_manager.results}" - ) - - # Calculate the next evaluation time for this specific time - next_times.remove(next_time) - next_times.append(next_time + timedelta(days=1)) - - # Update the cache with the next evaluation times - next_evaluation_times[competition_id] = next_times - if now_utc.minute % 5 == 0: - print("Waiting for next scheduled competition") - await asyncio.sleep(60) - def run_all_competitions(path_config: str, competitions_cfg: List[dict]) -> None: for competition_cfg in competitions_cfg: print("Starting competition: ", competition_cfg) @@ -136,9 +76,61 @@ def run_all_competitions(path_config: str, competitions_cfg: List[dict]) -> None ) asyncio.run(competition_manager.evaluate()) + +def get_competitions_time_arranged(path_config: str) -> List[CompetitionManager]: + """Return list of competitions arranged by launching time.""" + competitions_by_launching_time = [] + + for competition_config in competitions_cfg: + competition_manager = CompetitionManager( + path_config, + None, + 7, + competition_config["competition_id"], + competition_config["category"], + competition_config["dataset_hf_repo"], + competition_config["dataset_hf_filename"], + competition_config["dataset_hf_repo_type"], + ) + competitions_by_launching_time.append(competition_manager) + + return competitions_by_launching_time + + + +def config_for_scheduler() -> dict: + scheduler_config = {} + for competition_cfg in competitions_cfg: + for competition_time in competition_cfg["evaluation_time"]: + scheduler_config[competition_time] = CompetitionManager( + path_config, # TODO fetch bt config Konrad + None, + 7, + competition_cfg["competition_id"], + competition_cfg["category"], + competition_cfg["dataset_hf_repo"], + competition_cfg["dataset_hf_filename"], + competition_cfg["dataset_hf_repo_type"], + ) + return scheduler_config + +async def run_competitions(competition_times: List[CompetitionManager]) -> str | None: + """Checks if time is right and launches competition, returns winning hotkey""" + now_time = datetime.now() + now_time = f"{now_time.hour}:{now_time.minute}" + if now_time not in competition_times: + return None + for competition_time in competition_times: + if now_time == competition_time: + print(f"Running {competition_time.competition_id} at {now_time}") + return await competition_time.evaluate() + + + if __name__ == "__main__": if True: # run them right away run_all_competitions(path_config, competitions_cfg) else: # Run the scheduling coroutine + scheduler_config = config_for_scheduler() asyncio.run(schedule_competitions(competitions, path_config)) From 75c3808108458363785eb3c3baef570352c3c047 Mon Sep 17 00:00:00 2001 From: Wojtek Jurkowlaniec Date: Wed, 28 Aug 2024 18:43:34 +0200 Subject: [PATCH 102/227] fixes for running competitions --- cancer_ai/validator/competition_manager.py | 13 ++++++++++++- neurons/competition_runner.py | 19 ------------------- 2 files changed, 12 insertions(+), 20 deletions(-) diff --git a/cancer_ai/validator/competition_manager.py b/cancer_ai/validator/competition_manager.py index 157d0a7f..a28171e8 100644 --- a/cancer_ai/validator/competition_manager.py +++ b/cancer_ai/validator/competition_manager.py @@ -88,10 +88,21 @@ async def get_miner_model(self, chain_miner_model: ChainMinerModel): # return ModelInfo(hf_repo_id="safescanai/test_dataset", hf_filename="simple_cnn_model.onnx", hf_repo_type="dataset") + async def sync_chain_miners_test(self, hotkeys: list[str]): + hotkeys_with_models = { + "wojtek": ModelInfo(hf_repo_id="safescanai/test_dataset", hf_model_filename="model_dynamic.onnx", hf_repo_type="dataset"), + "bruno": ModelInfo(hf_repo_id="safescanai/test_dataset", hf_model_filename="best_model.onnx", hf_repo_type="dataset"), + } + self.model_manager.hotkey_store = hotkeys_with_models + + + async def sync_chain_miners(self, hotkeys: list[str]): """ Updates hotkeys and downloads information of models from the chain """ + + bt.logging.info("Synchronizing miners from the chain") self.hotkeys = hotkeys bt.logging.info(f"Amount of hotkeys: {len(hotkeys)}") @@ -116,7 +127,7 @@ async def evaluate(self): competition_handler = COMPETITION_HANDLER_MAPPING[self.competition_id]( X_test=X_test, y_test=y_test ) - await self.sync_chain_miners([]) + await self.sync_chain_miners_test([]) X_test, y_test = competition_handler.preprocess_data() for hotkey in self.model_manager.hotkey_store: diff --git a/neurons/competition_runner.py b/neurons/competition_runner.py index e82c0567..b9cbf0da 100644 --- a/neurons/competition_runner.py +++ b/neurons/competition_runner.py @@ -21,25 +21,6 @@ competitions_cfg = json.load(open("neurons/competition_config.json", "r")) -def calculate_next_evaluation_times(evaluation_times) -> List[datetime]: - """Calculate the next evaluation times for a given list of times in UTC.""" - now_utc = datetime.now(timezone.utc) - next_times = [] - - for time_str in evaluation_times: - # Parse the evaluation time to a datetime object in UTC - evaluation_time_utc = datetime.strptime(time_str, "%H:%M").replace( - tzinfo=timezone.utc, year=now_utc.year, month=now_utc.month, day=now_utc.day - ) - - # If the evaluation time has already passed today, schedule it for tomorrow - if evaluation_time_utc < now_utc: - evaluation_time_utc += timedelta(days=1) - - next_times.append(evaluation_time_utc) - - return next_times - def log_results_to_wandb(project, entity, hotkey, evaluation_result: ModelEvaluationResult): wandb.init(project=project, entity=entity) # TODO: Update this line as needed From 70e1eea4bcccc72a611acde05544695b4c5a22cf Mon Sep 17 00:00:00 2001 From: notbulubula Date: Wed, 28 Aug 2024 17:56:27 +0200 Subject: [PATCH 103/227] highhopes --- cancer_ai/validator/competition_manager.py | 8 +++- neurons/competition_runner.py | 45 +++++++++++++--------- 2 files changed, 33 insertions(+), 20 deletions(-) diff --git a/cancer_ai/validator/competition_manager.py b/cancer_ai/validator/competition_manager.py index a28171e8..6dbfbeba 100644 --- a/cancer_ai/validator/competition_manager.py +++ b/cancer_ai/validator/competition_manager.py @@ -13,6 +13,8 @@ from cancer_ai.chain_models_store import ChainModelMetadataStore, ChainMinerModel +from neurons.competition_runner import log_results_to_wandb + COMPETITION_HANDLER_MAPPING = { "melanoma-1": MelanomaCompetitionHandler, @@ -140,8 +142,10 @@ async def evaluate(self): start_time = time.time() y_pred = await model_manager.run(X_test) run_time_s = time.time() - start_time - print("Model prediction ", y_pred) - print("Ground truth: ", y_test) + # print("Model prediction ", y_pred) + # print("Ground truth: ", y_test) + log_results_to_wandb(y_test, y_pred, run_time_s, hotkey) + model_result = competition_handler.get_model_result( y_test, y_pred, run_time_s diff --git a/neurons/competition_runner.py b/neurons/competition_runner.py index b9cbf0da..6136ed67 100644 --- a/neurons/competition_runner.py +++ b/neurons/competition_runner.py @@ -22,24 +22,33 @@ competitions_cfg = json.load(open("neurons/competition_config.json", "r")) def log_results_to_wandb(project, entity, hotkey, evaluation_result: ModelEvaluationResult): - wandb.init(project=project, entity=entity) # TODO: Update this line as needed - - wandb.log({ - "hotkey": hotkey, - "tested_entries": evaluation_result.tested_entries, - "model_test_run_time": evaluation_result.run_time, - "accuracy": evaluation_result.accuracy, - "precision": evaluation_result.precision, - "recall": evaluation_result.recall, - "confusion_matrix": evaluation_result.confusion_matrix.tolist(), - "roc_curve": { - "fpr": evaluation_result.fpr.tolist(), - "tpr": evaluation_result.tpr.tolist() - }, - "roc_auc": evaluation_result.roc_auc - }) - - wandb.finish() + # wandb.init(project=project, entity=entity) # TODO: Update this line as needed + + # wandb.log({ + # "hotkey": hotkey, + # "tested_entries": evaluation_result.tested_entries, + # "model_test_run_time": evaluation_result.run_time, + # "accuracy": evaluation_result.accuracy, + # "precision": evaluation_result.precision, + # "recall": evaluation_result.recall, + # "confusion_matrix": evaluation_result.confusion_matrix.tolist(), + # "roc_curve": { + # "fpr": evaluation_result.fpr.tolist(), + # "tpr": evaluation_result.tpr.tolist() + # }, + # "roc_auc": evaluation_result.roc_auc + # }) + + # wandb.finish() + print("Logged results to wandb") + print("Hotkey: ", hotkey) + print("Tested entries: ", evaluation_result.tested_entries) + print("Model test run time: ", evaluation_result.run_time) + print("Accuracy: ", evaluation_result.accuracy) + print("Precision: ", evaluation_result.precision) + print("Recall: ", evaluation_result.recall) + print("roc_auc: ", evaluation_result.roc_auc) + return def run_all_competitions(path_config: str, competitions_cfg: List[dict]) -> None: From c9d7ce66bb535e568ca3f52d164e5dc7aba89eb5 Mon Sep 17 00:00:00 2001 From: Wojtek Jurkowlaniec Date: Wed, 28 Aug 2024 23:46:04 +0200 Subject: [PATCH 104/227] various fixes, wandb logging --- .env.example | 2 + cancer_ai/chain_models_store.py | 15 +++- cancer_ai/utils/config.py | 4 +- .../competition_handlers/base_handler.py | 2 +- .../competition_handlers/melanoma_handler.py | 36 +++++--- cancer_ai/validator/competition_manager.py | 86 ++++++++++++++---- neurons/competition_runner.py | 88 +++++++------------ 7 files changed, 139 insertions(+), 94 deletions(-) create mode 100644 .env.example diff --git a/.env.example b/.env.example new file mode 100644 index 00000000..95892173 --- /dev/null +++ b/.env.example @@ -0,0 +1,2 @@ +WANDB_API_KEY = +KAGGLE_AUTH_FILEPATH = \ No newline at end of file diff --git a/cancer_ai/chain_models_store.py b/cancer_ai/chain_models_store.py index 6d1e3be4..2fb5ccf6 100644 --- a/cancer_ai/chain_models_store.py +++ b/cancer_ai/chain_models_store.py @@ -11,10 +11,17 @@ class ChainMinerModel(BaseModel): """Uniquely identifies a trained model""" competition_id: Optional[str] = Field(description="The competition id") - hf_repo_id: str | None = None - hf_model_filename: str | None = None - hf_code_filename: str | None = None - hf_repo_type: str | None = None + + block: Optional[str] = Field( + description="Block on which this model was claimed on the chain." + ) + + hf_repo_id: Optional[str] = Field(description="Hugging Face repository id.") + hf_filename: Optional[str] = Field(description="Hugging Face model filename.") + hf_repo_type: Optional[str] = Field(description="Hugging Face repository type.") + hf_code_filename: Optional[str] = Field( + description="Hugging Face code zip filename." + ) class Config: arbitrary_types_allowed = True diff --git a/cancer_ai/utils/config.py b/cancer_ai/utils/config.py index 7f42d131..771f3a50 100644 --- a/cancer_ai/utils/config.py +++ b/cancer_ai/utils/config.py @@ -273,14 +273,14 @@ def add_validator_args(cls, parser): ) parser.add_argument( - "--wandb.project_name", + "--wandb_project_name", type=str, help="The name of the project where you are sending the new run.", default="template-validators", ) parser.add_argument( - "--wandb.entity", + "--wandb_entity", type=str, help="The name of the project where you are sending the new run.", default="opentensor-dev", diff --git a/cancer_ai/validator/competition_handlers/base_handler.py b/cancer_ai/validator/competition_handlers/base_handler.py index e19f76b9..aafdffa2 100644 --- a/cancer_ai/validator/competition_handlers/base_handler.py +++ b/cancer_ai/validator/competition_handlers/base_handler.py @@ -11,7 +11,7 @@ class ModelEvaluationResult: fpr: any tpr: any roc_auc: float - run_time: float + run_time_s: float tested_entries: int class BaseCompetitionHandler: diff --git a/cancer_ai/validator/competition_handlers/melanoma_handler.py b/cancer_ai/validator/competition_handlers/melanoma_handler.py index 311c6dd1..16068b49 100644 --- a/cancer_ai/validator/competition_handlers/melanoma_handler.py +++ b/cancer_ai/validator/competition_handlers/melanoma_handler.py @@ -4,11 +4,19 @@ from typing import List from PIL import Image import numpy as np -from sklearn.metrics import accuracy_score, precision_score, recall_score, confusion_matrix, roc_curve, auc +from sklearn.metrics import ( + accuracy_score, + precision_score, + recall_score, + confusion_matrix, + roc_curve, + auc, +) + class MelanomaCompetitionHandler(BaseCompetitionHandler): - """ - """ + """ """ + def __init__(self, X_test, y_test) -> None: super().__init__(X_test, y_test) @@ -19,11 +27,13 @@ def preprocess_data(self): for img in self.X_test: img = img.resize(target_size) img_array = np.array(img, dtype=np.float32) / 255.0 - img_array = np.array(img) - if img_array.shape[-1] != 3: # Handle grayscale images + img_array = np.array(img) + if img_array.shape[-1] != 3: # Handle grayscale images img_array = np.stack((img_array,) * 3, axis=-1) - - img_array = np.transpose(img_array, (2, 0, 1)) # Transpose image to (C, H, W) + + img_array = np.transpose( + img_array, (2, 0, 1) + ) # Transpose image to (C, H, W) new_X_test.append(img_array) @@ -33,8 +43,10 @@ def preprocess_data(self): new_y_test = [1 if y == "True" else 0 for y in self.y_test] return new_X_test, new_y_test - - def get_model_result(self, y_test: List[float], y_pred: np.ndarray, run_time_s: float) -> ModelEvaluationResult: + + def get_model_result( + self, y_test: List[float], y_pred: np.ndarray, run_time_s: float + ) -> ModelEvaluationResult: y_pred_binary = [1 if y > 0.5 else 0 for y in y_pred] tested_entries = len(y_test) accuracy = accuracy_score(y_test, y_pred_binary) @@ -44,8 +56,8 @@ def get_model_result(self, y_test: List[float], y_pred: np.ndarray, run_time_s: fpr, tpr, _ = roc_curve(y_test, y_pred) roc_auc = auc(fpr, tpr) return ModelEvaluationResult( - tested_entries=tested_entries, - run_time=run_time_s, + tested_entries=tested_entries, + run_time_s=run_time_s, accuracy=accuracy, precision=precision, recall=recall, @@ -53,4 +65,4 @@ def get_model_result(self, y_test: List[float], y_pred: np.ndarray, run_time_s: fpr=fpr, tpr=tpr, roc_auc=roc_auc, - ) \ No newline at end of file + ) diff --git a/cancer_ai/validator/competition_manager.py b/cancer_ai/validator/competition_manager.py index 6dbfbeba..499a271c 100644 --- a/cancer_ai/validator/competition_manager.py +++ b/cancer_ai/validator/competition_manager.py @@ -1,8 +1,16 @@ import time -import random from typing import List import bittensor as bt +from sklearn.metrics import ( + accuracy_score, + precision_score, + recall_score, + confusion_matrix, + roc_curve, + auc, +) +import wandb from .manager import SerializableManager from .model_manager import ModelManager, ModelInfo @@ -10,11 +18,12 @@ from .model_run_manager import ModelRunManager from .competition_handlers.melanoma_handler import MelanomaCompetitionHandler +from .competition_handlers.base_handler import ModelEvaluationResult from cancer_ai.chain_models_store import ChainModelMetadataStore, ChainMinerModel +from dotenv import load_dotenv -from neurons.competition_runner import log_results_to_wandb - +load_dotenv() COMPETITION_HANDLER_MAPPING = { "melanoma-1": MelanomaCompetitionHandler, @@ -38,8 +47,8 @@ class CompetitionManager(SerializableManager): def __init__( self, config, - subtensor: bt.Subtensor, # fetch from config, so not needed - subnet_uid: str, # fetch from config, so not needed + subtensor: bt.Subtensor, # fetch from config, so not needed + subnet_uid: str, # fetch from config, so not needed competition_id: str, category: str, dataset_hf_repo: str, @@ -68,6 +77,41 @@ def __init__( self.hotkeys = [] self.chain_miner_models = {} + def log_results_to_wandb( + self, hotkey: str, evaluation_result: ModelEvaluationResult + ): + # wandb.init(entity=self.config.wandb_entity, project=self.config.wandb_project_name) + wandb.init(project=self.config.wandb_project_name) + + wandb.log( + { + "hotkey": hotkey, + "tested_entries": evaluation_result.tested_entries, + # "model_test_run_time": evaluation_result.run_time, + "accuracy": evaluation_result.accuracy, + "precision": evaluation_result.precision, + "recall": evaluation_result.recall, + "confusion_matrix": evaluation_result.confusion_matrix.tolist(), + "roc_curve": { + "fpr": evaluation_result.fpr.tolist(), + "tpr": evaluation_result.tpr.tolist(), + }, + "roc_auc": evaluation_result.roc_auc, + } + ) + + wandb.finish() + print("Logged results to wandb") + print("Hotkey: ", hotkey) + print("Tested entries: ", evaluation_result.tested_entries) + print("Model test run time: ", evaluation_result.run_time) + print("Accuracy: ", evaluation_result.accuracy) + print("Precision: ", evaluation_result.precision) + print("Recall: ", evaluation_result.recall) + print("roc_auc: ", evaluation_result.roc_auc) + + return + def get_state(self): return { "competition_id": self.competition_id, @@ -84,27 +128,31 @@ async def get_miner_model(self, chain_miner_model: ChainMinerModel): model_info = ModelInfo( hf_repo_id=chain_miner_model.hf_repo_id, hf_model_filename=chain_miner_model.hf_filename, + hf_code_filename=chain_miner_model.hf_code_filename, hf_repo_type=chain_miner_model.hf_repo_type, ) return model_info - # return ModelInfo(hf_repo_id="safescanai/test_dataset", hf_filename="simple_cnn_model.onnx", hf_repo_type="dataset") - async def sync_chain_miners_test(self, hotkeys: list[str]): hotkeys_with_models = { - "wojtek": ModelInfo(hf_repo_id="safescanai/test_dataset", hf_model_filename="model_dynamic.onnx", hf_repo_type="dataset"), - "bruno": ModelInfo(hf_repo_id="safescanai/test_dataset", hf_model_filename="best_model.onnx", hf_repo_type="dataset"), + "wojtek": ModelInfo( + hf_repo_id="safescanai/test_dataset", + hf_model_filename="model_dynamic.onnx", + hf_repo_type="dataset", + ), + "bruno": ModelInfo( + hf_repo_id="safescanai/test_dataset", + hf_model_filename="best_model.onnx", + hf_repo_type="dataset", + ), } self.model_manager.hotkey_store = hotkeys_with_models - - async def sync_chain_miners(self, hotkeys: list[str]): """ Updates hotkeys and downloads information of models from the chain """ - bt.logging.info("Synchronizing miners from the chain") self.hotkeys = hotkeys bt.logging.info(f"Amount of hotkeys: {len(hotkeys)}") @@ -122,7 +170,7 @@ async def sync_chain_miners(self, hotkeys: list[str]): ) async def evaluate(self): - + """Returns hotkey of winning model miner and""" await self.dataset_manager.prepare_dataset() X_test, y_test = await self.dataset_manager.get_data() @@ -131,7 +179,7 @@ async def evaluate(self): ) await self.sync_chain_miners_test([]) X_test, y_test = competition_handler.preprocess_data() - + # print("Ground truth: ", y_test) for hotkey in self.model_manager.hotkey_store: bt.logging.info("Evaluating hotkey: ", hotkey) await self.model_manager.download_miner_model(hotkey) @@ -143,13 +191,15 @@ async def evaluate(self): y_pred = await model_manager.run(X_test) run_time_s = time.time() - start_time # print("Model prediction ", y_pred) - # print("Ground truth: ", y_test) - log_results_to_wandb(y_test, y_pred, run_time_s, hotkey) - model_result = competition_handler.get_model_result( y_test, y_pred, run_time_s ) + # log_results_to_wandb(y_test, y_pred, run_time_s, hotkey) self.results.append((hotkey, model_result)) + self.log_results_to_wandb(hotkey, model_result) - return self.results + winning_hotkey = sorted( + self.results, key=lambda x: x[1].accuracy, reverse=True + )[0][0] + return winning_hotkey diff --git a/neurons/competition_runner.py b/neurons/competition_runner.py index 6136ed67..a05616ad 100644 --- a/neurons/competition_runner.py +++ b/neurons/competition_runner.py @@ -1,5 +1,4 @@ from cancer_ai.validator.competition_manager import CompetitionManager -from cancer_ai.validator.competition_handlers.base_handler import ModelEvaluationResult from datetime import time, datetime import asyncio @@ -8,63 +7,37 @@ from types import SimpleNamespace from datetime import datetime, timezone, timedelta import bittensor as bt -from typing import List - -import wandb +from typing import List, Tuple # from cancer_ai.utils.config import config # TODO integrate with bt config path_config = SimpleNamespace( - **{"model_dir": "/tmp/models", "dataset_dir": "/tmp/datasets"} + **{ + "model_dir": "/tmp/models", + "dataset_dir": "/tmp/datasets", + "wandb_entity": "testnet", + "wandb_project_name": "melanoma-1", + } ) competitions_cfg = json.load(open("neurons/competition_config.json", "r")) -def log_results_to_wandb(project, entity, hotkey, evaluation_result: ModelEvaluationResult): - # wandb.init(project=project, entity=entity) # TODO: Update this line as needed - - # wandb.log({ - # "hotkey": hotkey, - # "tested_entries": evaluation_result.tested_entries, - # "model_test_run_time": evaluation_result.run_time, - # "accuracy": evaluation_result.accuracy, - # "precision": evaluation_result.precision, - # "recall": evaluation_result.recall, - # "confusion_matrix": evaluation_result.confusion_matrix.tolist(), - # "roc_curve": { - # "fpr": evaluation_result.fpr.tolist(), - # "tpr": evaluation_result.tpr.tolist() - # }, - # "roc_auc": evaluation_result.roc_auc - # }) - - # wandb.finish() - print("Logged results to wandb") - print("Hotkey: ", hotkey) - print("Tested entries: ", evaluation_result.tested_entries) - print("Model test run time: ", evaluation_result.run_time) - print("Accuracy: ", evaluation_result.accuracy) - print("Precision: ", evaluation_result.precision) - print("Recall: ", evaluation_result.recall) - print("roc_auc: ", evaluation_result.roc_auc) - - return def run_all_competitions(path_config: str, competitions_cfg: List[dict]) -> None: for competition_cfg in competitions_cfg: - print("Starting competition: ", competition_cfg) - competition_manager = CompetitionManager( - path_config, - None, - 7, - competition_cfg["competition_id"], - competition_cfg["category"], - competition_cfg["dataset_hf_repo"], - competition_cfg["dataset_hf_filename"], - competition_cfg["dataset_hf_repo_type"], - ) - asyncio.run(competition_manager.evaluate()) + print("Starting competition: ", competition_cfg) + competition_manager = CompetitionManager( + path_config, + None, + 7, + competition_cfg["competition_id"], + competition_cfg["category"], + competition_cfg["dataset_hf_repo"], + competition_cfg["dataset_hf_filename"], + competition_cfg["dataset_hf_repo_type"], + ) + print(asyncio.run(competition_manager.evaluate())) def get_competitions_time_arranged(path_config: str) -> List[CompetitionManager]: @@ -87,13 +60,12 @@ def get_competitions_time_arranged(path_config: str) -> List[CompetitionManager] return competitions_by_launching_time - def config_for_scheduler() -> dict: scheduler_config = {} for competition_cfg in competitions_cfg: for competition_time in competition_cfg["evaluation_time"]: scheduler_config[competition_time] = CompetitionManager( - path_config, # TODO fetch bt config Konrad + path_config, # TODO fetch bt config Konrad None, 7, competition_cfg["competition_id"], @@ -104,23 +76,25 @@ def config_for_scheduler() -> dict: ) return scheduler_config -async def run_competitions(competition_times: List[CompetitionManager]) -> str | None: - """Checks if time is right and launches competition, returns winning hotkey""" + +async def run_competitions_if_time_is_right( + competition_times: List[CompetitionManager], +) -> Tuple[str, str] | None: + """Checks if time is right and launches competition, returns winning hotkey and Competition ID""" now_time = datetime.now() now_time = f"{now_time.hour}:{now_time.minute}" if now_time not in competition_times: return None - for competition_time in competition_times: - if now_time == competition_time: - print(f"Running {competition_time.competition_id} at {now_time}") - return await competition_time.evaluate() - + for competition in competition_times: + if now_time == competition: + print(f"Running {competition.competition_id} at {now_time}") + return await (competition.evaluate(), competition.competition_id) if __name__ == "__main__": if True: # run them right away run_all_competitions(path_config, competitions_cfg) - - else: # Run the scheduling coroutine + + else: # Run the scheduling coroutine scheduler_config = config_for_scheduler() asyncio.run(schedule_competitions(competitions, path_config)) From 358cd6614c8d9110a1a9e6bfe802aaf84e62b8bb Mon Sep 17 00:00:00 2001 From: Wojtek Jurkowlaniec Date: Thu, 29 Aug 2024 21:42:50 +0200 Subject: [PATCH 105/227] finished scheduler --- .gitignore | 6 +- cancer_ai/validator/competition_manager.py | 15 ++--- neurons/competition_config.json | 8 +-- neurons/competition_runner.py | 76 +++++++++++----------- 4 files changed, 50 insertions(+), 55 deletions(-) diff --git a/.gitignore b/.gitignore index 861e8d0f..4d9b30d5 100644 --- a/.gitignore +++ b/.gitignore @@ -163,6 +163,6 @@ testing/ # Editors .vscode/settings.json - - -datasets \ No newline at end of file +datasets +data +wandb \ No newline at end of file diff --git a/cancer_ai/validator/competition_manager.py b/cancer_ai/validator/competition_manager.py index 499a271c..a7960c06 100644 --- a/cancer_ai/validator/competition_manager.py +++ b/cancer_ai/validator/competition_manager.py @@ -77,12 +77,11 @@ def __init__( self.hotkeys = [] self.chain_miner_models = {} - def log_results_to_wandb( - self, hotkey: str, evaluation_result: ModelEvaluationResult - ): - # wandb.init(entity=self.config.wandb_entity, project=self.config.wandb_project_name) wandb.init(project=self.config.wandb_project_name) + def log_results_to_wandb( + self, hotkey: str, evaluation_result: ModelEvaluationResult + ) -> None: wandb.log( { "hotkey": hotkey, @@ -104,14 +103,12 @@ def log_results_to_wandb( print("Logged results to wandb") print("Hotkey: ", hotkey) print("Tested entries: ", evaluation_result.tested_entries) - print("Model test run time: ", evaluation_result.run_time) + print("Model test run time: ", evaluation_result.run_time_s) print("Accuracy: ", evaluation_result.accuracy) print("Precision: ", evaluation_result.precision) print("Recall: ", evaluation_result.recall) print("roc_auc: ", evaluation_result.roc_auc) - return - def get_state(self): return { "competition_id": self.competition_id, @@ -169,8 +166,8 @@ async def sync_chain_miners(self, hotkeys: list[str]): f"Amount of chain miners with models: {len(self.chain_miner_models)}" ) - async def evaluate(self): - """Returns hotkey of winning model miner and""" + async def evaluate(self) -> str: + """Returns hotkey of winning model miner""" await self.dataset_manager.prepare_dataset() X_test, y_test = await self.dataset_manager.get_data() diff --git a/neurons/competition_config.json b/neurons/competition_config.json index f3d4695f..0acb097d 100644 --- a/neurons/competition_config.json +++ b/neurons/competition_config.json @@ -3,13 +3,11 @@ "competition_id": "melanoma-1", "category": "skin", "evaluation_time": [ - "02:05", - "15:30" + "21:31", + "21:32" ], "dataset_hf_repo": "safescanai/test_dataset", "dataset_hf_filename": "skin_melanoma.zip", - "dataset_hf_repo_type": "dataset", - "wandb_project": "testing_integration", - "wandb_entity": "urbaniak-bruno-safescanai" + "dataset_hf_repo_type": "dataset" } ] \ No newline at end of file diff --git a/neurons/competition_runner.py b/neurons/competition_runner.py index a05616ad..e4ab668c 100644 --- a/neurons/competition_runner.py +++ b/neurons/competition_runner.py @@ -7,12 +7,12 @@ from types import SimpleNamespace from datetime import datetime, timezone, timedelta import bittensor as bt -from typing import List, Tuple +from typing import List, Tuple, Dict # from cancer_ai.utils.config import config # TODO integrate with bt config -path_config = SimpleNamespace( +test_config = SimpleNamespace( **{ "model_dir": "/tmp/models", "dataset_dir": "/tmp/datasets", @@ -21,10 +21,11 @@ } ) -competitions_cfg = json.load(open("neurons/competition_config.json", "r")) +main_competitions_cfg = json.load(open("neurons/competition_config.json", "r")) def run_all_competitions(path_config: str, competitions_cfg: List[dict]) -> None: + """Run all competitions, for debug purposes""" for competition_cfg in competitions_cfg: print("Starting competition: ", competition_cfg) competition_manager = CompetitionManager( @@ -40,32 +41,13 @@ def run_all_competitions(path_config: str, competitions_cfg: List[dict]) -> None print(asyncio.run(competition_manager.evaluate())) -def get_competitions_time_arranged(path_config: str) -> List[CompetitionManager]: - """Return list of competitions arranged by launching time.""" - competitions_by_launching_time = [] - - for competition_config in competitions_cfg: - competition_manager = CompetitionManager( - path_config, - None, - 7, - competition_config["competition_id"], - competition_config["category"], - competition_config["dataset_hf_repo"], - competition_config["dataset_hf_filename"], - competition_config["dataset_hf_repo_type"], - ) - competitions_by_launching_time.append(competition_manager) - - return competitions_by_launching_time - - -def config_for_scheduler() -> dict: - scheduler_config = {} - for competition_cfg in competitions_cfg: +def config_for_scheduler() -> Dict[str, CompetitionManager]: + """Returns CompetitionManager instances arranged by competition time""" + time_arranged_competitions = {} + for competition_cfg in main_competitions_cfg: for competition_time in competition_cfg["evaluation_time"]: - scheduler_config[competition_time] = CompetitionManager( - path_config, # TODO fetch bt config Konrad + time_arranged_competitions[competition_time] = CompetitionManager( + test_config, # TODO fetch bt config Konrad None, 7, competition_cfg["competition_id"], @@ -74,27 +56,45 @@ def config_for_scheduler() -> dict: competition_cfg["dataset_hf_filename"], competition_cfg["dataset_hf_repo_type"], ) - return scheduler_config + return time_arranged_competitions -async def run_competitions_if_time_is_right( - competition_times: List[CompetitionManager], +async def run_competitions_tick( + competition_times: Dict[str, CompetitionManager], ) -> Tuple[str, str] | None: - """Checks if time is right and launches competition, returns winning hotkey and Competition ID""" + """Checks if time is right and launches competition, returns winning hotkey and Competition ID. Should be run each minute.""" now_time = datetime.now() now_time = f"{now_time.hour}:{now_time.minute}" + print(now_time) if now_time not in competition_times: return None - for competition in competition_times: - if now_time == competition: - print(f"Running {competition.competition_id} at {now_time}") - return await (competition.evaluate(), competition.competition_id) + for time_competition in competition_times: + if now_time == time_competition: + print( + f"Running {competition_times[time_competition].competition_id} at {now_time}" + ) + winning_evaluation_hotkey = await competition_times[ + time_competition + ].evaluate() + return ( + winning_evaluation_hotkey, + competition_times[time_competition].competition_id, + ) + + +async def competition_loop(scheduler_config: Dict[str, CompetitionManager]): + """Example of scheduling coroutine""" + while True: + print("run") + competition_result = await run_competitions_tick(scheduler_config) + print(competition_result) + await asyncio.sleep(60) if __name__ == "__main__": if True: # run them right away - run_all_competitions(path_config, competitions_cfg) + run_all_competitions(test_config, main_competitions_cfg) else: # Run the scheduling coroutine scheduler_config = config_for_scheduler() - asyncio.run(schedule_competitions(competitions, path_config)) + asyncio.run(competition_loop(scheduler_config)) From e8b462992639fdd8fcd2ca7e0910ca4cb920a17f Mon Sep 17 00:00:00 2001 From: Wojtek Jurkowlaniec Date: Thu, 29 Aug 2024 22:28:57 +0200 Subject: [PATCH 106/227] updated tests for model manager --- cancer_ai/validator/model_manager.py | 16 +++++++++++---- cancer_ai/validator/model_manager_test.py | 24 +++-------------------- 2 files changed, 15 insertions(+), 25 deletions(-) diff --git a/cancer_ai/validator/model_manager.py b/cancer_ai/validator/model_manager.py index f2c45900..cac0f568 100644 --- a/cancer_ai/validator/model_manager.py +++ b/cancer_ai/validator/model_manager.py @@ -39,7 +39,7 @@ def sync_hotkeys(self, hotkeys: list): if hotkey not in hotkeys: self.delete_model(hotkey) - async def download_miner_model(self, hotkey) -> None: + async def download_miner_model(self, hotkey) -> None: """Downloads the newest model from Hugging Face and saves it to disk. Returns: str: path to the downloaded model @@ -51,11 +51,19 @@ async def download_miner_model(self, hotkey) -> None: cache_dir=self.config.model_dir, repo_type=model_info.hf_repo_type, ) - - def add_model(self, hotkey, repo_id, filename) -> None: + def add_model( + self, + hotkey, + hf_repo_id, + hf_model_filename, + hf_code_filename=None, + hf_repo_type=None, + ) -> None: """Saves locally information about a new model.""" - self.hotkey_store[hotkey] = ModelInfo(repo_id, filename) + self.hotkey_store[hotkey] = ModelInfo( + hf_repo_id, hf_model_filename, hf_code_filename, hf_repo_type + ) def delete_model(self, hotkey) -> None: """Deletes locally information about a model and the corresponding file on disk.""" diff --git a/cancer_ai/validator/model_manager_test.py b/cancer_ai/validator/model_manager_test.py index efdd8509..fead7f45 100644 --- a/cancer_ai/validator/model_manager_test.py +++ b/cancer_ai/validator/model_manager_test.py @@ -14,11 +14,7 @@ @pytest.fixture def model_manager() -> ModelManager: - config = { - "models": SimpleNamespace(**{"model_dir": "/tmp/models"}), - } - config_obj = SimpleNamespace(**config) - print(config_obj.models.model_dir) + config_obj = SimpleNamespace(**{"model_dir": "/tmp/models"}) return ModelManager(config=config_obj) @@ -26,8 +22,8 @@ def test_add_model(model_manager: ModelManager) -> None: model_manager.add_model(hotkey, repo_id, filename) assert hotkey in model_manager.get_state() - assert model_manager.get_state()[hotkey]["repo_id"] == repo_id - assert model_manager.get_state()[hotkey]["filename"] == filename + assert model_manager.get_state()[hotkey]["hf_repo_id"] == repo_id + assert model_manager.get_state()[hotkey]["hf_model_filename"] == filename def test_delete_model(model_manager: ModelManager) -> None: @@ -44,20 +40,6 @@ def test_sync_hotkeys(model_manager: ModelManager): assert hotkey not in model_manager.get_state() -def test_load_save_model_state(model_manager: ModelManager) -> None: - # Create an instance of ModelManager - - # Create a dictionary of hotkey models - hotkey_models = { - "test_hotkey_1": {"repo_id": "repo_1", "filename": "file_1", "file_path": None}, - "test_hotkey_2": {"repo_id": "repo_2", "filename": "file_2", "file_path": None}, - } - - model_manager.set_state(hotkey_models) - - assert model_manager.get_state() == hotkey_models - - @pytest.mark.skip( reason="we don't want to test every time with downloading data from huggingface" ) From dec9fd1e226d0eef8f6ec37396401684d0e40c47 Mon Sep 17 00:00:00 2001 From: Wojtek Jurkowlaniec Date: Thu, 29 Aug 2024 22:32:01 +0200 Subject: [PATCH 107/227] fix for wandb sending --- cancer_ai/validator/competition_manager.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cancer_ai/validator/competition_manager.py b/cancer_ai/validator/competition_manager.py index a7960c06..bf7b92e9 100644 --- a/cancer_ai/validator/competition_manager.py +++ b/cancer_ai/validator/competition_manager.py @@ -77,11 +77,10 @@ def __init__( self.hotkeys = [] self.chain_miner_models = {} - wandb.init(project=self.config.wandb_project_name) - def log_results_to_wandb( self, hotkey: str, evaluation_result: ModelEvaluationResult ) -> None: + wandb.init(project=self.config.wandb_project_name) wandb.log( { "hotkey": hotkey, From a809097d169910c9d5fe619a08f95c13f42a8e56 Mon Sep 17 00:00:00 2001 From: Wojtek Jurkowlaniec Date: Sat, 31 Aug 2024 00:05:59 +0200 Subject: [PATCH 108/227] Update cancer_ai/validator/competition_manager.py Co-authored-by: konrad0960 <71330299+konrad0960@users.noreply.github.com> --- cancer_ai/validator/competition_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cancer_ai/validator/competition_manager.py b/cancer_ai/validator/competition_manager.py index bf7b92e9..d2dc99bc 100644 --- a/cancer_ai/validator/competition_manager.py +++ b/cancer_ai/validator/competition_manager.py @@ -166,7 +166,7 @@ async def sync_chain_miners(self, hotkeys: list[str]): ) async def evaluate(self) -> str: - """Returns hotkey of winning model miner""" + """Returns hotkey and competition id of winning model miner""" await self.dataset_manager.prepare_dataset() X_test, y_test = await self.dataset_manager.get_data() From 0d440f74c8071fe4f02e580c02c0d86b26d274e1 Mon Sep 17 00:00:00 2001 From: Wojtek Jurkowlaniec Date: Sat, 31 Aug 2024 01:05:26 +0200 Subject: [PATCH 109/227] CR fixes in progress --- .env.example | 2 +- cancer_ai/validator/competition_manager.py | 41 +++++++++++----------- 2 files changed, 22 insertions(+), 21 deletions(-) diff --git a/.env.example b/.env.example index 95892173..067c56f5 100644 --- a/.env.example +++ b/.env.example @@ -1,2 +1,2 @@ WANDB_API_KEY = -KAGGLE_AUTH_FILEPATH = \ No newline at end of file +WANDB_BASE_URL="https://api.wandb.ai" \ No newline at end of file diff --git a/cancer_ai/validator/competition_manager.py b/cancer_ai/validator/competition_manager.py index d2dc99bc..f5ea7ea5 100644 --- a/cancer_ai/validator/competition_manager.py +++ b/cancer_ai/validator/competition_manager.py @@ -47,8 +47,7 @@ class CompetitionManager(SerializableManager): def __init__( self, config, - subtensor: bt.Subtensor, # fetch from config, so not needed - subnet_uid: str, # fetch from config, so not needed + hotkeys: list[str], competition_id: str, category: str, dataset_hf_repo: str, @@ -72,9 +71,9 @@ def __init__( self.dataset_manager = DatasetManager( config, dataset_hf_repo, dataset_hf_id, dataset_hf_repo_type ) - self.chain_model_metadata_store = ChainModelMetadataStore(subtensor, subnet_uid) + self.chain_model_metadata_store = ChainModelMetadataStore(self.config.subtensor.network, self.config.netuid) - self.hotkeys = [] + self.hotkeys = hotkeys self.chain_miner_models = {} def log_results_to_wandb( @@ -99,14 +98,14 @@ def log_results_to_wandb( ) wandb.finish() - print("Logged results to wandb") - print("Hotkey: ", hotkey) - print("Tested entries: ", evaluation_result.tested_entries) - print("Model test run time: ", evaluation_result.run_time_s) - print("Accuracy: ", evaluation_result.accuracy) - print("Precision: ", evaluation_result.precision) - print("Recall: ", evaluation_result.recall) - print("roc_auc: ", evaluation_result.roc_auc) + bt.logging.info("Logged results to wandb") + bt.logging.info("Hotkey: ", hotkey) + bt.logging.info("Tested entries: ", evaluation_result.tested_entries) + bt.logging.info("Model test run time: ", evaluation_result.run_time_s) + bt.logging.info("Accuracy: ", evaluation_result.accuracy) + bt.logging.info("Precision: ", evaluation_result.precision) + bt.logging.info("Recall: ", evaluation_result.recall) + bt.logging.info("roc_auc: ", evaluation_result.roc_auc) def get_state(self): return { @@ -130,6 +129,7 @@ async def get_miner_model(self, chain_miner_model: ChainMinerModel): return model_info async def sync_chain_miners_test(self, hotkeys: list[str]): + """For testing purposes""" hotkeys_with_models = { "wojtek": ModelInfo( hf_repo_id="safescanai/test_dataset", @@ -144,15 +144,14 @@ async def sync_chain_miners_test(self, hotkeys: list[str]): } self.model_manager.hotkey_store = hotkeys_with_models - async def sync_chain_miners(self, hotkeys: list[str]): + async def sync_chain_miners(self): """ Updates hotkeys and downloads information of models from the chain """ - bt.logging.info("Synchronizing miners from the chain") - self.hotkeys = hotkeys - bt.logging.info(f"Amount of hotkeys: {len(hotkeys)}") - for hotkey in hotkeys: + + bt.logging.info(f"Amount of hotkeys: {len(self.hotkeys)}") + for hotkey in self.hotkeys: hotkey_metadata = ( await self.chain_model_metadata_store.retrieve_model_metadata(hotkey) ) @@ -173,9 +172,11 @@ async def evaluate(self) -> str: competition_handler = COMPETITION_HANDLER_MAPPING[self.competition_id]( X_test=X_test, y_test=y_test ) - await self.sync_chain_miners_test([]) + # test + # await self.sync_chain_miners_test() + await self.sync_chain_miners() X_test, y_test = competition_handler.preprocess_data() - # print("Ground truth: ", y_test) + # bt.logging.info("Ground truth: ", y_test) for hotkey in self.model_manager.hotkey_store: bt.logging.info("Evaluating hotkey: ", hotkey) await self.model_manager.download_miner_model(hotkey) @@ -186,7 +187,7 @@ async def evaluate(self) -> str: start_time = time.time() y_pred = await model_manager.run(X_test) run_time_s = time.time() - start_time - # print("Model prediction ", y_pred) + # bt.logging.info("Model prediction ", y_pred) model_result = competition_handler.get_model_result( y_test, y_pred, run_time_s From e220e6b07a6f6e0a35ebc99730f6ee3742998e30 Mon Sep 17 00:00:00 2001 From: Wojtek Jurkowlaniec Date: Sat, 31 Aug 2024 01:14:46 +0200 Subject: [PATCH 110/227] validator docs --- cancer_ai/DOCS/validator.md | 41 +++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 cancer_ai/DOCS/validator.md diff --git a/cancer_ai/DOCS/validator.md b/cancer_ai/DOCS/validator.md new file mode 100644 index 00000000..262888c2 --- /dev/null +++ b/cancer_ai/DOCS/validator.md @@ -0,0 +1,41 @@ +# Running validator + +## Server requirements + +### Minimal + - 32GB of RAM + - storage: 100GB, extendable + +### Recommended + - 64GB of RAM + - storage: 100GB0, extendable + - GPU - nVidia RTX, 12GB VRAM + +## System requirements + +- tested on Ubuntu 22.04 +- python 3.10 +- virtualenv + + +## Installation + +- create virtualenv + +`virtualenv venv --python=3.10` + +- activate it + +`source venv/bin/activate` + +- install requirements + +`pip install -r requirements.txt` + +## Running + +Prerequirements + +- make sure you are in base directory of the project +- activate your virtualenv +- run `export PYTHONPATH="${PYTHONPATH}:./"` \ No newline at end of file From 4e53cde4b35f496f9c09c9bbefaa82699a2fdbf6 Mon Sep 17 00:00:00 2001 From: Wojtek Jurkowlaniec Date: Sat, 31 Aug 2024 01:42:30 +0200 Subject: [PATCH 111/227] PR fixes --- cancer_ai/chain_models_store.py | 1 + cancer_ai/utils/config.py | 8 ++- cancer_ai/validator/competition_manager.py | 14 ++-- cancer_ai/validator/dataset_manager.py | 8 +-- cancer_ai/validator/model_manager.py | 3 +- .../model_runners/tensorflow_runner.py | 4 +- .../scripts/dataset_api_integration.py | 9 +-- cancer_ai/validator/utils.py | 2 +- neurons/competition_runner.py | 60 +++++----------- neurons/competition_runner_test.py | 68 +++++++++++++++++++ 10 files changed, 114 insertions(+), 63 deletions(-) create mode 100644 neurons/competition_runner_test.py diff --git a/cancer_ai/chain_models_store.py b/cancer_ai/chain_models_store.py index 2fb5ccf6..bc199856 100644 --- a/cancer_ai/chain_models_store.py +++ b/cancer_ai/chain_models_store.py @@ -83,6 +83,7 @@ async def retrieve_model_metadata(self, hotkey: str) -> Optional[ChainMinerModel metadata = run_in_subprocess(partial, 60) if not metadata: return None + bt.logging.info(f"Model metadata: {metadata["info"]["fields"]}") commitment = metadata["info"]["fields"][0] hex_data = commitment[list(commitment.keys())[0]][2:] diff --git a/cancer_ai/utils/config.py b/cancer_ai/utils/config.py index 771f3a50..c47d2f5f 100644 --- a/cancer_ai/utils/config.py +++ b/cancer_ai/utils/config.py @@ -52,7 +52,7 @@ def check_config(cls, config: "bt.Config"): config.neuron.name, ) ) - print("full path:", full_path) + print("Log path:", full_path) config.neuron.full_path = os.path.expanduser(full_path) if not os.path.exists(config.neuron.full_path): os.makedirs(config.neuron.full_path, exist_ok=True) @@ -297,6 +297,12 @@ def add_validator_args(cls, parser): help="Path for storing datasets.", default="./datasets", ) + parser.add_argument( + "--competition_config_path", + type=str, + help="Path with competition configuration .", + default="./neurons/competition_config.json", + ) def path_config(cls): diff --git a/cancer_ai/validator/competition_manager.py b/cancer_ai/validator/competition_manager.py index f5ea7ea5..b0118e2e 100644 --- a/cancer_ai/validator/competition_manager.py +++ b/cancer_ai/validator/competition_manager.py @@ -53,6 +53,7 @@ def __init__( dataset_hf_repo: str, dataset_hf_id: str, dataset_hf_repo_type: str, + test_mode: bool = False, ) -> None: """ Responsible for managing a competition. @@ -75,6 +76,7 @@ def __init__( self.hotkeys = hotkeys self.chain_miner_models = {} + self.test_mode = test_mode def log_results_to_wandb( self, hotkey: str, evaluation_result: ModelEvaluationResult @@ -84,7 +86,6 @@ def log_results_to_wandb( { "hotkey": hotkey, "tested_entries": evaluation_result.tested_entries, - # "model_test_run_time": evaluation_result.run_time, "accuracy": evaluation_result.accuracy, "precision": evaluation_result.precision, "recall": evaluation_result.recall, @@ -128,7 +129,7 @@ async def get_miner_model(self, chain_miner_model: ChainMinerModel): ) return model_info - async def sync_chain_miners_test(self, hotkeys: list[str]): + async def sync_chain_miners_test(self): """For testing purposes""" hotkeys_with_models = { "wojtek": ModelInfo( @@ -172,9 +173,11 @@ async def evaluate(self) -> str: competition_handler = COMPETITION_HANDLER_MAPPING[self.competition_id]( X_test=X_test, y_test=y_test ) - # test - # await self.sync_chain_miners_test() - await self.sync_chain_miners() + if self.test_mode: + await self.sync_chain_miners_test() + else: + await self.sync_chain_miners() + X_test, y_test = competition_handler.preprocess_data() # bt.logging.info("Ground truth: ", y_test) for hotkey in self.model_manager.hotkey_store: @@ -192,7 +195,6 @@ async def evaluate(self) -> str: model_result = competition_handler.get_model_result( y_test, y_pred, run_time_s ) - # log_results_to_wandb(y_test, y_pred, run_time_s, hotkey) self.results.append((hotkey, model_result)) self.log_results_to_wandb(hotkey, model_result) diff --git a/cancer_ai/validator/dataset_manager.py b/cancer_ai/validator/dataset_manager.py index dbf3b9cd..18f47793 100644 --- a/cancer_ai/validator/dataset_manager.py +++ b/cancer_ai/validator/dataset_manager.py @@ -37,7 +37,6 @@ def __init__( self.hf_filename = hf_filename self.hf_repo_type = hf_repo_type self.local_compressed_path = "" - print(self.config) self.local_extracted_dir = Path( self.config.dataset_dir, self.competition_id ) @@ -83,12 +82,12 @@ async def unzip_dataset(self) -> None: bt.logging.debug(f"Dataset extracted to: { self.local_compressed_path}") os.system(f"rm -R {self.local_extracted_dir}") - print(f"unzip {self.local_compressed_path} -d {self.local_extracted_dir}") + # TODO add error handling out, err = await run_command( f"unzip {self.local_compressed_path} -d {self.local_extracted_dir}" ) - print(err) - print("Dataset unzipped") + bt.logging.error(err) + bt.logging.info("Dataset unzipped") def set_dataset_handler(self) -> None: """Detect dataset type and set handler""" @@ -102,7 +101,6 @@ def set_dataset_handler(self) -> None: Path(self.local_extracted_dir, "labels.csv"), ) else: - #print("Files in dataset: ", os.listdir(self.local_extracted_dir)) raise NotImplementedError("Dataset handler not implemented") async def prepare_dataset(self) -> None: diff --git a/cancer_ai/validator/model_manager.py b/cancer_ai/validator/model_manager.py index cac0f568..940ac469 100644 --- a/cancer_ai/validator/model_manager.py +++ b/cancer_ai/validator/model_manager.py @@ -2,6 +2,7 @@ from datetime import datetime from time import sleep import os +import bittensor as bt from huggingface_hub import HfApi from .manager import SerializableManager @@ -68,7 +69,7 @@ def add_model( def delete_model(self, hotkey) -> None: """Deletes locally information about a model and the corresponding file on disk.""" - print("Deleting model: ", hotkey) + bt.logging.info(f"Deleting model: {hotkey}") if hotkey in self.hotkey_store and self.hotkey_store[hotkey].file_path: os.remove(self.hotkey_store[hotkey].file_path) self.hotkey_store[hotkey] = None diff --git a/cancer_ai/validator/model_runners/tensorflow_runner.py b/cancer_ai/validator/model_runners/tensorflow_runner.py index 37397db8..f89e0026 100644 --- a/cancer_ai/validator/model_runners/tensorflow_runner.py +++ b/cancer_ai/validator/model_runners/tensorflow_runner.py @@ -1,13 +1,13 @@ from . import BaseRunnerHandler from typing import List - +import bittensor as bt class TensorflowRunnerHandler(BaseRunnerHandler): async def run(self, pred_x: List) -> List: import tensorflow as tf import numpy as np from tensorflow.keras.preprocessing.image import load_img - print("imgs to test", len(pred_x)) + bt.logging.info(f"Images to test{len(pred_x)}") img_list = [load_img(img_path, target_size=(180, 180, 3)) for img_path in pred_x] img_list = [np.expand_dims(test_img, axis=0) for test_img in img_list] diff --git a/cancer_ai/validator/scripts/dataset_api_integration.py b/cancer_ai/validator/scripts/dataset_api_integration.py index 6b0979c5..8095b7f2 100644 --- a/cancer_ai/validator/scripts/dataset_api_integration.py +++ b/cancer_ai/validator/scripts/dataset_api_integration.py @@ -1,6 +1,7 @@ import os import csv import requests +import bittensor as bt # Base URL for downloading images (replace with actual base URL) @@ -24,17 +25,17 @@ # Define the local file path image_filename = f"images/{image_id}.jpg" - + # Download the image response = requests.get(image_url) if response.status_code == 200: with open(image_filename, "wb") as image_file: image_file.write(response.content) - print(f"Downloaded {image_filename}") + bt.logging.info(f"Downloaded {image_filename}") else: - print(f"Failed to download {image_filename}") + bt.logging.info(f"Failed to download {image_filename}") # Write the image path and label to the CSV file csv_writer.writerow([image_filename, is_melanoma]) -print("Process completed.") \ No newline at end of file +print("Process completed.") diff --git a/cancer_ai/validator/utils.py b/cancer_ai/validator/utils.py index b7aa3b69..c52d7e8d 100644 --- a/cancer_ai/validator/utils.py +++ b/cancer_ai/validator/utils.py @@ -64,7 +64,7 @@ async def run_command(cmd): process = await asyncio.create_subprocess_shell( cmd, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE ) - + bt.logging.debug(f"Running command: {cmd}") # Wait for the subprocess to finish and capture the output stdout, stderr = await process.communicate() diff --git a/neurons/competition_runner.py b/neurons/competition_runner.py index e4ab668c..fd269e92 100644 --- a/neurons/competition_runner.py +++ b/neurons/competition_runner.py @@ -11,45 +11,17 @@ # from cancer_ai.utils.config import config -# TODO integrate with bt config -test_config = SimpleNamespace( - **{ - "model_dir": "/tmp/models", - "dataset_dir": "/tmp/datasets", - "wandb_entity": "testnet", - "wandb_project_name": "melanoma-1", - } -) -main_competitions_cfg = json.load(open("neurons/competition_config.json", "r")) - - -def run_all_competitions(path_config: str, competitions_cfg: List[dict]) -> None: - """Run all competitions, for debug purposes""" - for competition_cfg in competitions_cfg: - print("Starting competition: ", competition_cfg) - competition_manager = CompetitionManager( - path_config, - None, - 7, - competition_cfg["competition_id"], - competition_cfg["category"], - competition_cfg["dataset_hf_repo"], - competition_cfg["dataset_hf_filename"], - competition_cfg["dataset_hf_repo_type"], - ) - print(asyncio.run(competition_manager.evaluate())) - - -def config_for_scheduler() -> Dict[str, CompetitionManager]: +def config_for_scheduler( + bt_config, hotkeys: List[str] +) -> Dict[str, CompetitionManager]: """Returns CompetitionManager instances arranged by competition time""" time_arranged_competitions = {} for competition_cfg in main_competitions_cfg: for competition_time in competition_cfg["evaluation_time"]: time_arranged_competitions[competition_time] = CompetitionManager( - test_config, # TODO fetch bt config Konrad - None, - 7, + bt_config, + hotkeys, competition_cfg["competition_id"], competition_cfg["category"], competition_cfg["dataset_hf_repo"], @@ -65,12 +37,12 @@ async def run_competitions_tick( """Checks if time is right and launches competition, returns winning hotkey and Competition ID. Should be run each minute.""" now_time = datetime.now() now_time = f"{now_time.hour}:{now_time.minute}" - print(now_time) + bt.logging.debug(now_time) if now_time not in competition_times: return None for time_competition in competition_times: if now_time == time_competition: - print( + bt.logging.info( f"Running {competition_times[time_competition].competition_id} at {now_time}" ) winning_evaluation_hotkey = await competition_times[ @@ -85,16 +57,18 @@ async def run_competitions_tick( async def competition_loop(scheduler_config: Dict[str, CompetitionManager]): """Example of scheduling coroutine""" while True: - print("run") competition_result = await run_competitions_tick(scheduler_config) - print(competition_result) + bt.logging.debug(f"Competition result: {competition_result}") await asyncio.sleep(60) if __name__ == "__main__": - if True: # run them right away - run_all_competitions(test_config, main_competitions_cfg) - - else: # Run the scheduling coroutine - scheduler_config = config_for_scheduler() - asyncio.run(competition_loop(scheduler_config)) + # fetch from config + competition_config_path = "neurons/competition_config.json" + main_competitions_cfg = json.load( + open(competition_config_path, "r") + ) # TODO fetch from config + hotkeys = [] + bt_config = {} # get from bt config + scheduler_config = config_for_scheduler(bt_config, hotkeys) + asyncio.run(competition_loop(scheduler_config)) diff --git a/neurons/competition_runner_test.py b/neurons/competition_runner_test.py new file mode 100644 index 00000000..ec8f770e --- /dev/null +++ b/neurons/competition_runner_test.py @@ -0,0 +1,68 @@ +from cancer_ai.validator.competition_manager import CompetitionManager +import asyncio +import json +from types import SimpleNamespace +import bittensor as bt +from typing import List, Dict +from competition_runner import run_competitions_tick, competition_loop + +# TODO integrate with bt config +test_config = SimpleNamespace( + **{ + "model_dir": "/tmp/models", + "dataset_dir": "/tmp/datasets", + "wandb_entity": "testnet", + "wandb_project_name": "melanoma-1", + "subtensor": SimpleNamespace(**{"network": "test"}), + "netuid": 163, + } +) + +main_competitions_cfg = json.load(open("neurons/competition_config.json", "r")) + + +def run_all_competitions(path_config: str, hotkeys: List[str], competitions_cfg: List[dict]) -> None: + """Run all competitions, for debug purposes""" + for competition_cfg in competitions_cfg: + bt.logging.info("Starting competition: ", competition_cfg) + competition_manager = CompetitionManager( + path_config, + hotkeys, + competition_cfg["competition_id"], + competition_cfg["category"], + competition_cfg["dataset_hf_repo"], + competition_cfg["dataset_hf_filename"], + competition_cfg["dataset_hf_repo_type"], + test_mode=True, + ) + bt.logging.info(asyncio.run(competition_manager.evaluate())) + + +def config_for_scheduler() -> Dict[str, CompetitionManager]: + """Returns CompetitionManager instances arranged by competition time""" + time_arranged_competitions = {} + for competition_cfg in main_competitions_cfg: + for competition_time in competition_cfg["evaluation_time"]: + time_arranged_competitions[competition_time] = CompetitionManager( + {}, # TODO fetch bt config Konrad + [], + competition_cfg["competition_id"], + competition_cfg["category"], + competition_cfg["dataset_hf_repo"], + competition_cfg["dataset_hf_filename"], + competition_cfg["dataset_hf_repo_type"], + test_mode=True, + ) + return time_arranged_competitions + + + + + +if __name__ == "__main__": + if True: # run them right away + run_all_competitions(test_config, [],main_competitions_cfg) + + else: # Run the scheduling coroutine + scheduler_config = config_for_scheduler() + asyncio.run(competition_loop(scheduler_config)) From ad9940eb23e7b5d5265ca8339729c120641d487f Mon Sep 17 00:00:00 2001 From: Konrad Date: Mon, 26 Aug 2024 16:03:19 +0200 Subject: [PATCH 112/227] plugged in config and integrated pushing model metadata on chain from miner --- cancer_ai/chain_models_store.py | 2 +- cancer_ai/utils/config.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/cancer_ai/chain_models_store.py b/cancer_ai/chain_models_store.py index bc199856..ab9df490 100644 --- a/cancer_ai/chain_models_store.py +++ b/cancer_ai/chain_models_store.py @@ -99,4 +99,4 @@ async def retrieve_model_metadata(self, hotkey: str) -> Optional[ChainMinerModel return None # The block id at which the metadata is stored model.block = metadata["block"] - return model + return model \ No newline at end of file diff --git a/cancer_ai/utils/config.py b/cancer_ai/utils/config.py index c47d2f5f..bda0afc8 100644 --- a/cancer_ai/utils/config.py +++ b/cancer_ai/utils/config.py @@ -210,6 +210,7 @@ def add_miner_args(cls, parser): ) + def add_validator_args(cls, parser): """Add validator specific arguments to the parser.""" From 282b2772e35ec6d2bb1b451b2570c61127977a00 Mon Sep 17 00:00:00 2001 From: Konrad Date: Tue, 27 Aug 2024 21:46:48 +0200 Subject: [PATCH 113/227] some adjustments --- cancer_ai/utils/config.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/cancer_ai/utils/config.py b/cancer_ai/utils/config.py index bda0afc8..2d9a586f 100644 --- a/cancer_ai/utils/config.py +++ b/cancer_ai/utils/config.py @@ -189,7 +189,14 @@ def add_miner_args(cls, parser): ) parser.add_argument( - "--hf-token", + "--hf_repo_id", + type=str, + help="Hugging Face model repository ID", + default="", + ) + + parser.add_argument( + "--hf_token", type=str, help="Hugging Face API token", default="", From 60650c268ff590b2d1d58a94e26938098cbd6277 Mon Sep 17 00:00:00 2001 From: Wojtek Jurkowlaniec Date: Thu, 29 Aug 2024 22:46:28 +0200 Subject: [PATCH 114/227] fixes to chain model --- cancer_ai/chain_models_store.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cancer_ai/chain_models_store.py b/cancer_ai/chain_models_store.py index ab9df490..bc199856 100644 --- a/cancer_ai/chain_models_store.py +++ b/cancer_ai/chain_models_store.py @@ -99,4 +99,4 @@ async def retrieve_model_metadata(self, hotkey: str) -> Optional[ChainMinerModel return None # The block id at which the metadata is stored model.block = metadata["block"] - return model \ No newline at end of file + return model From 9c5d9ff2e78d2724ed030a441c52aed06acfddb5 Mon Sep 17 00:00:00 2001 From: Wojtek Jurkowlaniec Date: Fri, 30 Aug 2024 15:01:04 +0200 Subject: [PATCH 115/227] bittensor connection only on submitting models, fixes in configuration, fixes in competition logic --- cancer_ai/utils/config.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/cancer_ai/utils/config.py b/cancer_ai/utils/config.py index 2d9a586f..ea5d09db 100644 --- a/cancer_ai/utils/config.py +++ b/cancer_ai/utils/config.py @@ -168,6 +168,12 @@ def add_miner_args(cls, parser): help="Type of hugging face repository.", ) + parser.add_argument( + "--hf-model-name", + type=str, + help="Name of the model to push to hugging face.", + ) + parser.add_argument( "--action", choices=["submit", "evaluate", "upload"], @@ -189,14 +195,7 @@ def add_miner_args(cls, parser): ) parser.add_argument( - "--hf_repo_id", - type=str, - help="Hugging Face model repository ID", - default="", - ) - - parser.add_argument( - "--hf_token", + "--hf-token", type=str, help="Hugging Face API token", default="", From 2260ab8b7b2a8b06e58c787fc1ce556d05553991 Mon Sep 17 00:00:00 2001 From: Wojtek Jurkowlaniec Date: Wed, 28 Aug 2024 18:43:34 +0200 Subject: [PATCH 116/227] fixes for running competitions --- cancer_ai/validator/competition_manager.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cancer_ai/validator/competition_manager.py b/cancer_ai/validator/competition_manager.py index b0118e2e..0f8ec456 100644 --- a/cancer_ai/validator/competition_manager.py +++ b/cancer_ai/validator/competition_manager.py @@ -149,6 +149,8 @@ async def sync_chain_miners(self): """ Updates hotkeys and downloads information of models from the chain """ + + bt.logging.info("Synchronizing miners from the chain") bt.logging.info(f"Amount of hotkeys: {len(self.hotkeys)}") From 22d3e4c81df1e451d11442aceb358ceef71d69f1 Mon Sep 17 00:00:00 2001 From: Wojtek Jurkowlaniec Date: Wed, 28 Aug 2024 23:46:04 +0200 Subject: [PATCH 117/227] various fixes, wandb logging --- cancer_ai/validator/competition_manager.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cancer_ai/validator/competition_manager.py b/cancer_ai/validator/competition_manager.py index 0f8ec456..2d05cd7b 100644 --- a/cancer_ai/validator/competition_manager.py +++ b/cancer_ai/validator/competition_manager.py @@ -108,6 +108,8 @@ def log_results_to_wandb( bt.logging.info("Recall: ", evaluation_result.recall) bt.logging.info("roc_auc: ", evaluation_result.roc_auc) + + def get_state(self): return { "competition_id": self.competition_id, @@ -150,7 +152,6 @@ async def sync_chain_miners(self): Updates hotkeys and downloads information of models from the chain """ - bt.logging.info("Synchronizing miners from the chain") bt.logging.info(f"Amount of hotkeys: {len(self.hotkeys)}") @@ -197,6 +198,7 @@ async def evaluate(self) -> str: model_result = competition_handler.get_model_result( y_test, y_pred, run_time_s ) + # log_results_to_wandb(y_test, y_pred, run_time_s, hotkey) self.results.append((hotkey, model_result)) self.log_results_to_wandb(hotkey, model_result) From dc3ceb88091168e95f69f466f531dcb6c7f43757 Mon Sep 17 00:00:00 2001 From: Wojtek Jurkowlaniec Date: Sat, 31 Aug 2024 01:05:26 +0200 Subject: [PATCH 118/227] CR fixes in progress --- cancer_ai/validator/competition_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cancer_ai/validator/competition_manager.py b/cancer_ai/validator/competition_manager.py index 2d05cd7b..d1e9b6fb 100644 --- a/cancer_ai/validator/competition_manager.py +++ b/cancer_ai/validator/competition_manager.py @@ -151,7 +151,6 @@ async def sync_chain_miners(self): """ Updates hotkeys and downloads information of models from the chain """ - bt.logging.info("Synchronizing miners from the chain") bt.logging.info(f"Amount of hotkeys: {len(self.hotkeys)}") @@ -176,6 +175,7 @@ async def evaluate(self) -> str: competition_handler = COMPETITION_HANDLER_MAPPING[self.competition_id]( X_test=X_test, y_test=y_test ) + # TODO get hotkeys if self.test_mode: await self.sync_chain_miners_test() else: From ead2946df73d3f49f79330608def63d6a602afb8 Mon Sep 17 00:00:00 2001 From: Wojtek Jurkowlaniec Date: Sat, 31 Aug 2024 01:42:30 +0200 Subject: [PATCH 119/227] PR fixes --- cancer_ai/validator/competition_manager.py | 1 - 1 file changed, 1 deletion(-) diff --git a/cancer_ai/validator/competition_manager.py b/cancer_ai/validator/competition_manager.py index d1e9b6fb..676ce196 100644 --- a/cancer_ai/validator/competition_manager.py +++ b/cancer_ai/validator/competition_manager.py @@ -198,7 +198,6 @@ async def evaluate(self) -> str: model_result = competition_handler.get_model_result( y_test, y_pred, run_time_s ) - # log_results_to_wandb(y_test, y_pred, run_time_s, hotkey) self.results.append((hotkey, model_result)) self.log_results_to_wandb(hotkey, model_result) From a1bf9bfb5a7a3cb61ac0fa64822d8a87d68ccec7 Mon Sep 17 00:00:00 2001 From: Wojtek Jurkowlaniec Date: Sat, 31 Aug 2024 03:32:27 +0200 Subject: [PATCH 120/227] fix --- neurons/competition_runner_test.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/neurons/competition_runner_test.py b/neurons/competition_runner_test.py index ec8f770e..edfcdb5e 100644 --- a/neurons/competition_runner_test.py +++ b/neurons/competition_runner_test.py @@ -13,6 +13,8 @@ "dataset_dir": "/tmp/datasets", "wandb_entity": "testnet", "wandb_project_name": "melanoma-1", + "competition_id": "melaonoma-1", + "hotkeys": [], "subtensor": SimpleNamespace(**{"network": "test"}), "netuid": 163, } From 6cfdacb1ce4a2fda2cd6dcc4bf5666b587888d8f Mon Sep 17 00:00:00 2001 From: Konrad Date: Sat, 31 Aug 2024 15:44:42 +0200 Subject: [PATCH 121/227] new rewarder with tests --- cancer_ai/rewarder.py | 61 ++-- cancer_ai/rewarder_test.py | 550 ++++++++++++++++++------------------- 2 files changed, 306 insertions(+), 305 deletions(-) diff --git a/cancer_ai/rewarder.py b/cancer_ai/rewarder.py index 44dfbe79..fbadbd8a 100644 --- a/cancer_ai/rewarder.py +++ b/cancer_ai/rewarder.py @@ -1,20 +1,22 @@ from pydantic import BaseModel -from datetime import datetime +from datetime import datetime, timedelta class CompetitionLeader(BaseModel): hotkey: str leader_since: datetime - +class Score(BaseModel): + score: float + reduction: float class RewarderConfig(BaseModel): - competition_leader_mapping: dict[str, CompetitionLeader] - scores: dict[str, float] # hotkey -> score + competitionID_to_leader_hotkey_map: dict[str, CompetitionLeader] # competition_id -> CompetitionLeader + hotkey_to_score_map: dict[str, Score] # hotkey -> Score class Rewarder(): - def __init__(self, config: RewarderConfig): - self.competition_leader_mapping = config.competition_leader_mapping - self.scores = config.scores + def __init__(self, rewarder_config: RewarderConfig): + self.competition_leader_mapping = rewarder_config.competitionID_to_leader_hotkey_map + self.scores = rewarder_config.hotkey_to_score_map - def calculate_score_for_winner(self, competition_id: str, hotkey: str) -> tuple[float, float]: + def get_scores_and_reduction(self, competition_id: str, hotkey: str) -> tuple[float, float]: num_competitions = len(self.competition_leader_mapping) base_share = 1/num_competitions @@ -25,7 +27,8 @@ def calculate_score_for_winner(self, competition_id: str, hotkey: str) -> tuple[ days_as_leader = 0 self.competition_leader_mapping[competition_id] = CompetitionLeader(hotkey=hotkey, leader_since=datetime.now()) - + + # Score degradation starts on 3rd week of leadership if days_as_leader > 14: periods = (days_as_leader - 14) // 7 reduction_factor = max(0.1, 1 - 0.1 * periods) @@ -36,25 +39,33 @@ def calculate_score_for_winner(self, competition_id: str, hotkey: str) -> tuple[ def update_scores(self): num_competitions = len(self.competition_leader_mapping) - reduced_shares_poll = {hotkey: 0.0 for hotkey in self.scores} # If there is only one competition, the winner takes it all if num_competitions == 1: - single_key = next(iter(self.competition_leader_mapping)) - single_hotkey = self.competition_leader_mapping[single_key].hotkey - self.scores[single_hotkey] = 1.0 + competition_id = next(iter(self.competition_leader_mapping)) + hotkey = self.competition_leader_mapping[competition_id].hotkey + self.scores[hotkey] = Score(score=1.0, reduction=0.0) return - - for curr_competition_id, comp_leader in self.competition_leader_mapping.items(): - score, reduced_share = self.calculate_score_for_winner(curr_competition_id, comp_leader.hotkey) - self.scores[comp_leader.hotkey] += score - if reduced_share > 0: - # Distribute reduced share among all competitors (including the current winner if he wins another competition) - distributed_share = reduced_share / (num_competitions - 1) - for leader_competition_id, leader in self.competition_leader_mapping.items(): - if leader.hotkey != comp_leader.hotkey or leader_competition_id != curr_competition_id: - reduced_shares_poll[leader.hotkey] += distributed_share + # gather reduced shares for all competitors + competitions_without_reduction = [] + for curr_competition_id, comp_leader in self.competition_leader_mapping.items(): + score, reduced_share = self.get_scores_and_reduction(curr_competition_id, comp_leader.hotkey) + self.scores[comp_leader.hotkey].score += score + self.scores[comp_leader.hotkey].reduction += reduced_share + if reduced_share == 0: + competitions_without_reduction.append(curr_competition_id) - for hotkey, score in self.scores.items(): - self.scores[hotkey] += reduced_shares_poll[hotkey] \ No newline at end of file + total_reduced_share = sum([score.reduction for score in self.scores.values()]) + + # if all competitions have reduced shares, distribute them among all competitors + if len(competitions_without_reduction) == 0: + # distribute the total reduced share among all competitors + for hotkey, score in self.scores.items(): + self.scores[hotkey].score += total_reduced_share / num_competitions + return + else: + # distribute the total reduced share among non-reduced competitons winners + for comp_id in competitions_without_reduction: + hotkey = self.competition_leader_mapping[comp_id].hotkey + self.scores[hotkey].score += total_reduced_share / len(competitions_without_reduction) \ No newline at end of file diff --git a/cancer_ai/rewarder_test.py b/cancer_ai/rewarder_test.py index 0a194e9e..7547a637 100644 --- a/cancer_ai/rewarder_test.py +++ b/cancer_ai/rewarder_test.py @@ -1,283 +1,273 @@ -import unittest +import pytest from datetime import datetime, timedelta -from rewarder import CompetitionLeader, RewarderConfig, Rewarder - -class TestRewarder(unittest.TestCase): - - def test_single_competition_single_leader(self): - """Test case 1: Only one competition with 1 leader -> leader takes it all""" - competitions_leaders = { - "competition-1": CompetitionLeader(hotkey="leader-1", leader_since=datetime.now()) - } - scores = {"leader-1": 0} - rewarder_config = RewarderConfig(competition_leader_mapping=competitions_leaders, scores=scores) - rewarder = Rewarder(config=rewarder_config) - - rewarder.update_scores() - - # Assert that the leader takes it all - self.assertAlmostEqual(rewarder.scores["leader-1"], 1.0) - - def test_4_2_reduction(self): - """Test case 2: 4 competitions with 2 different leaders, reduction -> 2 have 50% of the shares""" - leader_1_reward = 0.125 - leader_2_reward = 0.125 - leader_3_reward = leader_4_reward = 0.25 + 0.125 - reduction_50_percent_days = datetime.now() - timedelta(days=14 + 7*5) - competitions_leaders = { - "competition-1": CompetitionLeader(hotkey="leader-1", leader_since=reduction_50_percent_days), - "competition-2": CompetitionLeader(hotkey="leader-2", leader_since=reduction_50_percent_days), - "competition-3": CompetitionLeader(hotkey="leader-3", leader_since=reduction_50_percent_days), - "competition-4": CompetitionLeader(hotkey="leader-4", leader_since=reduction_50_percent_days), - } - scores = {"leader-1": 0, "leader-2": 0, "leader-3": 0, "leader-4": 0} - rewarder_config = RewarderConfig(competition_leader_mapping=competitions_leaders, scores=scores) - rewarder = Rewarder(config=rewarder_config) - rewarder.update_scores() - self.assertAlmostEqual(rewarder.scores["leader-1"], leader_1_reward, places=2) - self.assertAlmostEqual(rewarder.scores["leader-2"], leader_2_reward, places=2) - self.assertAlmostEqual(rewarder.scores["leader-3"], leader_3_reward, places=2) - self.assertAlmostEqual(rewarder.scores["leader-4"], leader_4_reward, places=2) - - - def test_three_competitions_three_leaders_no_reduction(self): - """Test case 2: 3 competitions with 3 different leaders, no reduction -> all have 33% of the shares""" - reward_split_by_three = 1 / 3 - competitions_leaders = { - "competition-1": CompetitionLeader(hotkey="leader-1", leader_since=datetime.now()), - "competition-2": CompetitionLeader(hotkey="leader-2", leader_since=datetime.now()), - "competition-3": CompetitionLeader(hotkey="leader-3", leader_since=datetime.now()) - } - scores = {"leader-1": 0, "leader-2": 0, "leader-3": 0} - rewarder_config = RewarderConfig(competition_leader_mapping=competitions_leaders, scores=scores) - rewarder = Rewarder(config=rewarder_config) - rewarder.update_scores() - - # Assert that all leaders have roughly 1/3 of the shares - self.assertAlmostEqual(rewarder.scores["leader-1"], reward_split_by_three, places=2) - self.assertAlmostEqual(rewarder.scores["leader-2"], reward_split_by_three, places=2) - self.assertAlmostEqual(rewarder.scores["leader-3"], reward_split_by_three, places=2) - - def test_three_competitions_three_leaders_with_reduction(self): - """Test case 3: 3 competitions with 3 different leaders, one has a reduced share by 10%""" - first_competion_leader_since = datetime.now() - timedelta(days=21) - - base_share = 1/3 - reduction_factor = 0.9 # 10% reduction - expected_share_leader_1 = base_share * reduction_factor - expected_reduction = base_share - expected_share_leader_1 - expected_share_leader_2_3 = base_share + (expected_reduction / 2) # Distributed reduction - - scores = {"leader-1": 0, "leader-2": 0, "leader-3": 0} - - competitions_leaders = { - "competition-1": CompetitionLeader(hotkey="leader-1", leader_since=first_competion_leader_since), - "competition-2": CompetitionLeader(hotkey="leader-2", leader_since=datetime.now()), - "competition-3": CompetitionLeader(hotkey="leader-3", leader_since=datetime.now()) - } - - rewarder_config = RewarderConfig(competition_leader_mapping=competitions_leaders, scores=scores) - rewarder = Rewarder(config=rewarder_config) - - rewarder.update_scores() - # Assert that leader-1 has the reduced share and others are higher - self.assertAlmostEqual(rewarder.scores["leader-1"], expected_share_leader_1, places=2) - self.assertAlmostEqual(rewarder.scores["leader-2"], expected_share_leader_2_3, places=2) - self.assertAlmostEqual(rewarder.scores["leader-3"], expected_share_leader_2_3, places=2) - - def test_three_competitions_three_leaders_two_reductions(self): - """Test case 4: 3 competitions with 3 different leaders, two with reduced shares""" - competitions_leaders = { - "competition-1": CompetitionLeader(hotkey="leader-1", leader_since=datetime.now() - timedelta(days=21)), - "competition-2": CompetitionLeader(hotkey="leader-2", leader_since=datetime.now() - timedelta(days=35)), - "competition-3": CompetitionLeader(hotkey="leader-3", leader_since=datetime.now()) - } - scores = {"leader-1": 0, "leader-2": 0, "leader-3": 0} - rewarder_config = RewarderConfig(competition_leader_mapping=competitions_leaders, scores=scores) - rewarder = Rewarder(config=rewarder_config) - - rewarder.update_scores() - - - - self.assertAlmostEqual(rewarder.scores["leader-1"], 1/3, places=2) - self.assertAlmostEqual(rewarder.scores["leader-2"], 1/3, places=2) - self.assertAlmostEqual(rewarder.scores["leader-3"], 1/3, places=2) - - def test_three_competitions_three_leaders_all_different_reductions(self): - """Test case 5: All competitors have different degrees of reduced shares""" - competitions_leaders = { - "competition-1": CompetitionLeader(hotkey="leader-1", leader_since=datetime.now() - timedelta(days=21)), - "competition-2": CompetitionLeader(hotkey="leader-2", leader_since=datetime.now() - timedelta(days=35)), - "competition-3": CompetitionLeader(hotkey="leader-3", leader_since=datetime.now() - timedelta(days=49)) - } - scores = {"leader-1": 0, "leader-2": 0, "leader-3": 0} - rewarder_config = RewarderConfig(competition_leader_mapping=competitions_leaders, scores=scores) - rewarder = Rewarder(config=rewarder_config) - - rewarder.update_scores() - - base_share = 1 / 3 - reduction_factor_leader_1 = 0.9 # 10% reduction - reduction_factor_leader_2 = 0.7 # 30% reduction - reduction_factor_leader_3 = 0.5 # 50% reduction - - expected_share_leader_1 = base_share * reduction_factor_leader_1 - expected_share_leader_2 = base_share * reduction_factor_leader_2 - expected_share_leader_3 = base_share * reduction_factor_leader_3 - # Calculate distributed reduction - remaining_share_leader_1 = base_share - expected_share_leader_1 - remaining_share_leader_2 = base_share - expected_share_leader_2 - remaining_share_leader_3 = base_share - expected_share_leader_3 - - # Leaders 2 and 3 gets their base share plus the distributed reduction from Leader 1 - expected_share_leader_2 += remaining_share_leader_1 / 2 - expected_share_leader_3 += remaining_share_leader_1 / 2 - - # Leaders 1 and 3 gets their base share plus the distributed reduction from Leader 2 - expected_share_leader_1 += remaining_share_leader_2 / 2 - expected_share_leader_3 += remaining_share_leader_2 / 2 - - # Leaders 1 and 2 gets their base share plus the distributed reduction from Leader 3 - expected_share_leader_1 += remaining_share_leader_3 / 2 - expected_share_leader_2 += remaining_share_leader_3 / 2 - - self.assertAlmostEqual(rewarder.scores["leader-1"], expected_share_leader_1, places=2) - self.assertAlmostEqual(rewarder.scores["leader-2"], expected_share_leader_2, places=2) - self.assertAlmostEqual(rewarder.scores["leader-3"], expected_share_leader_3, places=2) - - def test_three_competitions_three_leaders_all_same_reductions(self): - """Test case 6: All competitors have the same amount of reduced shares""" - competitions_leaders = { - "competition-1": CompetitionLeader(hotkey="leader-1", leader_since=datetime.now() - timedelta(days=21)), - "competition-2": CompetitionLeader(hotkey="leader-2", leader_since=datetime.now() - timedelta(days=21)), - "competition-3": CompetitionLeader(hotkey="leader-3", leader_since=datetime.now() - timedelta(days=21)) - } - scores = {"leader-1": 0, "leader-2": 0, "leader-3": 0} - rewarder_config = RewarderConfig(competition_leader_mapping=competitions_leaders, scores=scores) - rewarder = Rewarder(config=rewarder_config) - - rewarder.update_scores() - - base_share = 1 / 3 - - # All should have the same shares - self.assertAlmostEqual(rewarder.scores["leader-1"], base_share, places=2) - self.assertAlmostEqual(rewarder.scores["leader-2"], base_share, places=2) - self.assertAlmostEqual(rewarder.scores["leader-3"], base_share, places=2) - - def test_three_competitions_three_leaders_all_maximum_reductions(self): - """Test case 7: All competitors have maximum reduced shares (90%)""" - competitions_leaders = { - "competition-1": CompetitionLeader(hotkey="leader-1", leader_since=datetime.now() - timedelta(days=91)), - "competition-2": CompetitionLeader(hotkey="leader-2", leader_since=datetime.now() - timedelta(days=91)), - "competition-3": CompetitionLeader(hotkey="leader-3", leader_since=datetime.now() - timedelta(days=91)) - } - scores = {"leader-1": 0, "leader-2": 0, "leader-3": 0} - rewarder_config = RewarderConfig(competition_leader_mapping=competitions_leaders, scores=scores) - rewarder = Rewarder(config=rewarder_config) - - rewarder.update_scores() - - base_share = 1 / 3 - reduction_factor = 0.1 # 90% reduction for all - - expected_share_leader_1 = base_share * reduction_factor - expected_share_leader_2 = base_share * reduction_factor - expected_share_leader_3 = base_share * reduction_factor - - # Calculate distributed reduction - remaining_share_leader_1 = base_share - expected_share_leader_1 - remaining_share_leader_2 = base_share - expected_share_leader_2 - remaining_share_leader_3 = base_share - expected_share_leader_3 - - # Leaders 2 and 3 gets their base share plus the distributed reduction from Leader 1 - expected_share_leader_2 += remaining_share_leader_1 / 2 - expected_share_leader_3 += remaining_share_leader_1 / 2 - - # Leaders 1 and 3 gets their base share plus the distributed reduction from Leader 2 - expected_share_leader_1 += remaining_share_leader_2 / 2 - expected_share_leader_3 += remaining_share_leader_2 / 2 - - # Leaders 1 and 2 gets their base share plus the distributed reduction from Leader 3 - expected_share_leader_1 += remaining_share_leader_3 / 2 - expected_share_leader_2 += remaining_share_leader_3 / 2 - - # All should have the same shares - self.assertAlmostEqual(rewarder.scores["leader-1"], expected_share_leader_1, places=2) - self.assertAlmostEqual(rewarder.scores["leader-2"], expected_share_leader_2, places=2) - self.assertAlmostEqual(rewarder.scores["leader-3"], expected_share_leader_3, places=2) - - def test_three_competitions_two_competitors(self): - """Test case 8: 3 competitions but only 2 competitors""" - competitions_leaders = { - "competition-1": CompetitionLeader(hotkey="leader-1", leader_since=datetime.now() - timedelta(days=21)), - "competition-2": CompetitionLeader(hotkey="leader-1", leader_since=datetime.now() - timedelta(days=10)), - "competition-3": CompetitionLeader(hotkey="leader-2", leader_since=datetime.now()) - } - scores = {"leader-1": 0, "leader-2": 0} - rewarder_config = RewarderConfig(competition_leader_mapping=competitions_leaders, scores=scores) - rewarder = Rewarder(config=rewarder_config) - - rewarder.update_scores() - - base_share = 1 / 3 - reduction_factor_leader_1_competition_1 = 0.9 # 10% reduction for 21 days - - # Calculate expected scores - expected_share_leader_1_competition_1 = base_share * reduction_factor_leader_1_competition_1 - expected_share_leader_1_competition_2 = base_share - expected_share_leader_2 = base_share - - remaining_share_leader_1_competition_1 = base_share - expected_share_leader_1_competition_1 - # The competitors of competition 2 and 3 (including leader-1) get the distributed reduction - expected_score_leader_1 = expected_share_leader_1_competition_1 + expected_share_leader_1_competition_2\ - + remaining_share_leader_1_competition_1 / 2 - expected_score_leader_2 = expected_share_leader_2 + remaining_share_leader_1_competition_1 / 2 - - self.assertAlmostEqual(rewarder.scores["leader-1"], expected_score_leader_1, places=2) - self.assertAlmostEqual(rewarder.scores["leader-2"], expected_score_leader_2, places=2) - - def test_five_competitions_three_competitors_two_repeating(self): - """Test case 9: 5 competitions with 3 competitors, 2 of them are repeating""" - competitions_leaders = { - "competition-1": CompetitionLeader(hotkey="leader-1", leader_since=datetime.now() - timedelta(days=21)), # 10% reduction - "competition-2": CompetitionLeader(hotkey="leader-2", leader_since=datetime.now() - timedelta(days=10)), # No reduction - "competition-3": CompetitionLeader(hotkey="leader-1", leader_since=datetime.now()), # No reduction - "competition-4": CompetitionLeader(hotkey="leader-3", leader_since=datetime.now() - timedelta(days=35)), # 30% reduction - "competition-5": CompetitionLeader(hotkey="leader-2", leader_since=datetime.now()) # No reduction - } - scores = {"leader-1": 0, "leader-2": 0, "leader-3": 0} - rewarder_config = RewarderConfig(competition_leader_mapping=competitions_leaders, scores=scores) - rewarder = Rewarder(config=rewarder_config) - - rewarder.update_scores() - - base_share = 1 / 5 - reduction_factor_leader_1_competition_1 = 0.9 # 10% reduction for 21 days - reduction_factor_leader_3_competition_4 = 0.7 # 30% reduction for 35 days - - # Calculate expected shares for each leader - expected_share_leader_1_competition_1 = base_share * reduction_factor_leader_1_competition_1 - expected_share_leader_1_competition_3 = base_share - expected_share_leader_2_competition_2 = base_share - expected_share_leader_2_competition_5 = base_share - expected_share_leader_3_competition_4 = base_share * reduction_factor_leader_3_competition_4 - - remaining_share_leader_1_competition_1 = base_share - expected_share_leader_1_competition_1 - remaining_share_leader_3_competition_4 = base_share - expected_share_leader_3_competition_4 - - # Calculate final scores with distributed reduction shares - expected_score_leader_1 = expected_share_leader_1_competition_1 + expected_share_leader_1_competition_3\ - + (remaining_share_leader_1_competition_1 / 3) + (remaining_share_leader_3_competition_4 / 2) - expected_score_leader_2 = expected_share_leader_2_competition_2 + expected_share_leader_2_competition_5\ - + (remaining_share_leader_1_competition_1 / 3) + (remaining_share_leader_3_competition_4 / 2) - expected_score_leader_3 = expected_share_leader_3_competition_4 + (remaining_share_leader_1_competition_1 / 3) - - self.assertAlmostEqual(rewarder.scores["leader-1"], expected_score_leader_1, places=2) - self.assertAlmostEqual(rewarder.scores["leader-2"], expected_score_leader_2, places=2) - self.assertAlmostEqual(rewarder.scores["leader-3"], expected_score_leader_3, places=2) - +from pydantic import BaseModel +from .rewarder import CompetitionLeader, Score, RewarderConfig, Rewarder + +def test_update_scores_single_competitor(): + # Set up initial data for a single competitor + competition_leaders = { + "competition1": CompetitionLeader(hotkey="competitor1", leader_since=datetime.now() - timedelta(days=10)) + } + + scores = { + "competitor1": Score(score=0.0, reduction=0.0) + } + + # Set up the configuration with a single competition and a single competitor + rewarder_config = RewarderConfig( + competitionID_to_leader_hotkey_map=competition_leaders, + hotkey_to_score_map=scores + ) + + rewarder = Rewarder(rewarder_config) + rewarder.update_scores() + + # Check the updated scores and reductions for the single competitor + updated_score = rewarder.scores["competitor1"].score + updated_reduction = rewarder.scores["competitor1"].reduction + + # # With only one competitor, they should receive the full score of 1.0 + expected_score = 1.0 + expected_reduction = 0.0 + + assert updated_score == expected_score, f"Expected score: {expected_score}, got: {updated_score}" + assert updated_reduction == expected_reduction, f"Expected reduction: {expected_reduction}, got: {updated_reduction}" + +def test_update_scores_multiple_competitors_no_reduction(): + # Set up initial data for multiple competitors + competition_leaders = { + "competition1": CompetitionLeader(hotkey="competitor1", leader_since=datetime.now() - timedelta(days=10)), + "competition2": CompetitionLeader(hotkey="competitor2", leader_since=datetime.now() - timedelta(days=10)), + "competition3": CompetitionLeader(hotkey="competitor3", leader_since=datetime.now() - timedelta(days=10)) + } + + scores = { + "competitor1": Score(score=0.0, reduction=0.0), + "competitor2": Score(score=0.0, reduction=0.0), + "competitor3": Score(score=0.0, reduction=0.0) + } + + # Set up the configuration with multiple competitions and multiple competitors + rewarder_config = RewarderConfig( + competitionID_to_leader_hotkey_map=competition_leaders, + hotkey_to_score_map=scores + ) + + rewarder = Rewarder(rewarder_config) + rewarder.update_scores() + + # Check the updated scores and reductions for the multiple competitors + updated_scores = {hotkey: score.score for hotkey, score in rewarder.scores.items()} + updated_reductions = {hotkey: score.reduction for hotkey, score in rewarder.scores.items()} + + # With multiple competitors and no reductions, they should all receive the same score of 1/3 + expected_score = 1/3 + expected_reduction = 0.0 + + for _, score in updated_scores.items(): + assert score == expected_score, f"Expected score: {expected_score}, got: {score}" + + for _, reduction in updated_reductions.items(): + assert reduction == expected_reduction, f"Expected reduction: {expected_reduction}, got: {reduction}" + +def test_update_scores_multiple_competitors_with_some_reduced_shares(): + # Set up initial data for multiple competitors + competition_leaders = { + "competition1": CompetitionLeader(hotkey="competitor1", leader_since=datetime.now() - timedelta(days=14 + 3 * 7)), + "competition2": CompetitionLeader(hotkey="competitor2", leader_since=datetime.now() - timedelta(days=14 + 6 * 7)), + "competition3": CompetitionLeader(hotkey="competitor3", leader_since=datetime.now() - timedelta(days=14)), + "competition4": CompetitionLeader(hotkey="competitor4", leader_since=datetime.now() - timedelta(days=14)), + } + + scores = { + "competitor1": Score(score=0.0, reduction=0.0), + "competitor2": Score(score=0.0, reduction=0.0), + "competitor3": Score(score=0.0, reduction=0.0), + "competitor4": Score(score=0.0, reduction=0.0), + } + + # Set up the configuration with multiple competitions and multiple competitors + rewarder_config = RewarderConfig( + competitionID_to_leader_hotkey_map=competition_leaders, + hotkey_to_score_map=scores + ) + + rewarder = Rewarder(rewarder_config) + rewarder.update_scores() + + # Check the updated scores and reductions for the multiple competitors + updated_scores = {hotkey: score.score for hotkey, score in rewarder.scores.items()} + updated_reductions = {hotkey: score.reduction for hotkey, score in rewarder.scores.items()} + + # With multiple competitors and some reduced shares, they should receive different scores and reductions + expected_reductions = { + "competitor1": 1/4 * 0.3, + "competitor2": 1/4 * 0.6, + "competitor3": 0.0, + "competitor4": 0.0, + } + + expected_reductions_sum = sum(expected_reductions.values()) + expected_scores = { + "competitor1": 1/4 - expected_reductions["competitor1"], + "competitor2": 1/4 - expected_reductions["competitor2"], + "competitor3": 1/4 + expected_reductions_sum/2, + "competitor4": 1/4 + expected_reductions_sum/2, + } + + for hotkey, score in updated_scores.items(): + assert score == pytest.approx(expected_scores[hotkey], rel=1e-9), f"Expected score: {expected_scores[hotkey]}, got: {score}" + + for hotkey, reduction in updated_reductions.items(): + assert reduction == pytest.approx(expected_reductions[hotkey], rel=1e-9), f"Expected reduction: {expected_reductions[hotkey]}, got: {reduction}" + +def test_update_scores_all_competitors_with_reduced_shares(): + # Set up initial data for multiple competitors + competition_leaders = { + "competition1": CompetitionLeader(hotkey="competitor1", leader_since=datetime.now() - timedelta(days=14 + 3 * 7)), + "competition2": CompetitionLeader(hotkey="competitor2", leader_since=datetime.now() - timedelta(days=14 + 6 * 7)), + "competition3": CompetitionLeader(hotkey="competitor3", leader_since=datetime.now() - timedelta(days=14 + 9 * 7)) + } + + scores = { + "competitor1": Score(score=0.0, reduction=0.0), + "competitor2": Score(score=0.0, reduction=0.0), + "competitor3": Score(score=0.0, reduction=0.0) + } + + # Set up the configuration with multiple competitions and multiple competitors + rewarder_config = RewarderConfig( + competitionID_to_leader_hotkey_map=competition_leaders, + hotkey_to_score_map=scores + ) + + rewarder = Rewarder(rewarder_config) + rewarder.update_scores() + + # Check the updated scores and reductions for the multiple competitors + updated_scores = {hotkey: score.score for hotkey, score in rewarder.scores.items()} + updated_reductions = {hotkey: score.reduction for hotkey, score in rewarder.scores.items()} + + # With multiple competitors and reduced shares, they should receive different scores and reductions + expected_reductions = { + "competitor1": 0.1, + "competitor2": 0.2, + "competitor3": 0.3 + } + + expected_reductions_sum = sum(expected_reductions.values()) + expected_scores = { + "competitor1": 1/3 - expected_reductions["competitor1"] + expected_reductions_sum/3, + "competitor2": 1/3 - expected_reductions["competitor2"] + expected_reductions_sum/3, + "competitor3": 1/3 - expected_reductions["competitor3"] + expected_reductions_sum/3, + } + + for hotkey, score in updated_scores.items(): + assert score == expected_scores[hotkey], f"Expected score: {expected_scores[hotkey]}, got: {score}" + + for hotkey, reduction in updated_reductions.items(): + assert reduction == expected_reductions[hotkey], f"Expected reduction: {expected_reductions[hotkey]}, got: {reduction}" + +def test_update_scores_more_competitions_then_competitors(): + # Set up initial data for multiple competitors + competition_leaders = { + "competition1": CompetitionLeader(hotkey="competitor1", leader_since=datetime.now() - timedelta(days=14 + 3 * 7)), + "competition2": CompetitionLeader(hotkey="competitor2", leader_since=datetime.now() - timedelta(days=14)), + "competition3": CompetitionLeader(hotkey="competitor1", leader_since=datetime.now() - timedelta(days=14)), + "competition4": CompetitionLeader(hotkey="competitor3", leader_since=datetime.now() - timedelta(days=14)), + } + + scores = { + "competitor1": Score(score=0.0, reduction=0.0), + "competitor2": Score(score=0.0, reduction=0.0), + "competitor3": Score(score=0.0, reduction=0.0), + } + + # Set up the configuration with multiple competitions and multiple competitors + rewarder_config = RewarderConfig( + competitionID_to_leader_hotkey_map=competition_leaders, + hotkey_to_score_map=scores + ) + + rewarder = Rewarder(rewarder_config) + rewarder.update_scores() + + # Check the updated scores and reductions for the multiple competitors + updated_scores = {hotkey: score.score for hotkey, score in rewarder.scores.items()} + updated_reductions = {hotkey: score.reduction for hotkey, score in rewarder.scores.items()} + + # With multiple competitors and some reduced shares, they should receive different scores and reductions + expected_reductions = { + "competitor1": 1/4 * 0.3, + "competitor2": 0.0, + "competitor3": 0.0, + } + + expected_reductions_sum = sum(expected_reductions.values()) + expected_scores = { + "competitor1": 2/4 - expected_reductions["competitor1"] + expected_reductions_sum/3, + "competitor2": 1/4 + expected_reductions_sum/3, + "competitor3": 1/4 + expected_reductions_sum/3, + } + + for hotkey, score in updated_scores.items(): + assert score == pytest.approx(expected_scores[hotkey], rel=1e-9), f"Expected score: {expected_scores[hotkey]}, got: {score} for {hotkey}" + + for hotkey, reduction in updated_reductions.items(): + assert reduction == pytest.approx(expected_reductions[hotkey], rel=1e-9), f"Expected reduction: {expected_reductions[hotkey]}, got: {reduction} for {hotkey}" + +def test_update_scores_6_competitions_3_competitors(): + # Set up initial data for multiple competitors + competition_leaders = { + "competition1": CompetitionLeader(hotkey="competitor1", leader_since=datetime.now() - timedelta(days=14 + 3 * 7)), + "competition2": CompetitionLeader(hotkey="competitor2", leader_since=datetime.now() - timedelta(days=14 + 6 * 7)), + "competition3": CompetitionLeader(hotkey="competitor3", leader_since=datetime.now() - timedelta(days=14 + 9 * 7)), + "competition4": CompetitionLeader(hotkey="competitor4", leader_since=datetime.now() - timedelta(days=14)), + "competition5": CompetitionLeader(hotkey="competitor1", leader_since=datetime.now() - timedelta(days=14)), + "competition6": CompetitionLeader(hotkey="competitor2", leader_since=datetime.now() - timedelta(days=14 + 3 * 7)), + } + + scores = { + "competitor1": Score(score=0.0, reduction=0.0), + "competitor2": Score(score=0.0, reduction=0.0), + "competitor3": Score(score=0.0, reduction=0.0), + "competitor4": Score(score=0.0, reduction=0.0), + } + + # Set up the configuration with multiple competitions and multiple competitors + rewarder_config = RewarderConfig( + competitionID_to_leader_hotkey_map=competition_leaders, + hotkey_to_score_map=scores + ) + + rewarder = Rewarder(rewarder_config) + rewarder.update_scores() + + # Check the updated scores and reductions for the multiple competitors + updated_scores = {hotkey: score.score for hotkey, score in rewarder.scores.items()} + updated_reductions = {hotkey: score.reduction for hotkey, score in rewarder.scores.items()} + + # With multiple competitors and some reduced shares, they should receive different scores and reductions + expected_reductions = { + "competitor1": 1/6 * 0.3, + "competitor2": (1/6 * 0.6) + (1/6 * 0.3), + "competitor3": 1/6 * 0.9, + "competitor4": 0.0, + } + + expected_reductions_sum = sum(expected_reductions.values()) + expected_scores = { + "competitor1": (2/6 - expected_reductions["competitor1"]) + expected_reductions_sum/2, + "competitor2": (2/6 - expected_reductions["competitor2"]), + "competitor3": 1/6 - expected_reductions["competitor3"], + "competitor4": 1/6 + expected_reductions_sum/2, + } + + for hotkey, score in updated_scores.items(): + assert score == pytest.approx(expected_scores[hotkey], rel=1e-9), f"Expected score: {expected_scores[hotkey]}, got: {score} for {hotkey}" + + for hotkey, reduction in updated_reductions.items(): + assert reduction == pytest.approx(expected_reductions[hotkey], rel=1e-9), f"Expected reduction: {expected_reductions[hotkey]}, got: {reduction} for {hotkey}" if __name__ == "__main__": - unittest.main() + pytest.main() From 3da8b3328cd409e682a7b06366d9450623bf467b Mon Sep 17 00:00:00 2001 From: Wojtek Jurkowlaniec Date: Sat, 31 Aug 2024 03:44:26 +0200 Subject: [PATCH 122/227] miner config --- neurons/miner.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/neurons/miner.py b/neurons/miner.py index f71f821b..be17cd7f 100644 --- a/neurons/miner.py +++ b/neurons/miner.py @@ -191,13 +191,6 @@ async def main(self) -> None: if __name__ == "__main__": - from types import SimpleNamespace - config = get_config() - config = { - "dataset_dir": "./data", - } - config = SimpleNamespace( **config) - set_log_formatting() load_dotenv() cli_manager = MinerManagerCLI() asyncio.run(cli_manager.main()) From ba27bcc2f8b1bc81d941068e72b60ef78c3f991e Mon Sep 17 00:00:00 2001 From: Wojtek Jurkowlaniec Date: Sat, 31 Aug 2024 23:20:43 +0200 Subject: [PATCH 123/227] fixing feet --- DOCS/miner.md | 28 ++++++++++++++-------------- cancer_ai/utils/config.py | 31 +++++++++++-------------------- 2 files changed, 25 insertions(+), 34 deletions(-) diff --git a/DOCS/miner.md b/DOCS/miner.md index 62c9e210..de2df7e4 100644 --- a/DOCS/miner.md +++ b/DOCS/miner.md @@ -33,7 +33,7 @@ This mode will do following things -`python neurons/miner.py --action evaluate --competition-id --model-path ` +`python neurons/miner.py --action evaluate --competition_id --model_path ` If flag `--clean-after-run` is supplied, it will delete dataset after evaluating the model @@ -42,16 +42,16 @@ If flag `--clean-after-run` is supplied, it will delete dataset after evaluating - compresses code provided by --code-path - uploads model and code to HuggingFace -`python neurons/miner.py --action upload --competition-id melanoma-1 --model-path test_model.onnx --hf-model-name file_name.zip --hf-repo-id repo/id --hf-token TOKEN` +`python neurons/miner.py --action upload --competition_id melanoma-1 --model_path test_model.onnx --hf_model_name file_name.zip --hf_repo_id repo/id --hf_token TOKEN` ```bash python neurons/miner.py \ --action upload \ - --competition-id \ - --model-path \ - --code-directory \ - --hf-model-name \ - --hf-repo-id \ - --hf-token \ + --competition_id \ + --model_path \ + --code_directory \ + --hf_model_name \ + --hf_repo_id \ + --hf_token \ --logging.debug ``` @@ -64,12 +64,12 @@ python neurons/miner.py \ ```bash python neurons/miner.py \ --action submit \ - --model-path \ - --competition-id \ - --hf-code-filename "melanoma-1-piwo.zip" \ - --hf-model-name \ - --hf-repo-id \ - --hf-repo-type model \ + --model_path \ + --competition_id \ + --hf_code_filename "melanoma-1-piwo.zip" \ + --hf_model_name \ + --hf_repo_id \ + --hf_repo_type model \ --wallet.name \ --wallet.hotkey \ --netuid \ diff --git a/cancer_ai/utils/config.py b/cancer_ai/utils/config.py index 8cf80bdf..147ce476 100644 --- a/cancer_ai/utils/config.py +++ b/cancer_ai/utils/config.py @@ -132,84 +132,75 @@ def add_args(cls, parser): def add_miner_args(cls, parser): """Add miner specific arguments to the parser.""" parser.add_argument( - "--competition-id", + "--competition_id", type=str, help="Competition ID", - required=True, ) parser.add_argument( - "--model-dir", + "--model_dir", type=str, help="Path for for loading the starting model related to a training run.", default="./models", ) parser.add_argument( - "--hf-repo-id", + "--hf_repo_id", type=str, help="Hugging Face model repository ID", ) parser.add_argument( - "--hf-model-name", + "--hf_model_name", type=str, help="Filename of the model to push to hugging face.", ) parser.add_argument( - "--hf-code-filename", + "--hf_code_filename", type=str, help="Filename of the code zip to push to hugging face.", ) parser.add_argument( - "--hf-repo-type", + "--hf_repo_type", type=str, help="Type of hugging face repository.", ) - parser.add_argument( - "--hf-model-name", - type=str, - help="Name of the model to push to hugging face.", - ) - parser.add_argument( "--action", choices=["submit", "evaluate", "upload"], - required=True, ) parser.add_argument( - "--model-path", + "--model_path", type=str, help="Path to ONNX model, used for evaluation", - required=True, ) parser.add_argument( - "--dataset-dir", + "--dataset_dir", type=str, help="Path for storing datasets.", default="./datasets", ) parser.add_argument( - "--hf-token", + "--hf_token", type=str, help="Hugging Face API token", default="", ) parser.add_argument( - "--clean-after-run", + "--clean_after_run", action="store_true", help="Whether to clean up (dataset, temporary files) after running", default=False, ) parser.add_argument( - "--code-directory", + "--code_directory", type=str, help="Path to code directory", default=".", From 16ebbfe66ebbe473f6525fbbbfb69a4c24b10382 Mon Sep 17 00:00:00 2001 From: Konrad Date: Sun, 1 Sep 2024 03:17:46 +0200 Subject: [PATCH 124/227] WIP validator integration --- cancer_ai/base/neuron.py | 14 ------- cancer_ai/base/validator.py | 54 +------------------------ cancer_ai/validator/forward.py | 52 ------------------------ neurons/competition_runner.py | 22 ++++++---- neurons/competition_runner_test.py | 30 +++++++++++--- {cancer_ai => neurons}/rewarder.py | 45 +++++++++++++-------- {cancer_ai => neurons}/rewarder_test.py | 1 - neurons/validator.py | 37 +++++++++++++++-- 8 files changed, 102 insertions(+), 153 deletions(-) delete mode 100644 cancer_ai/validator/forward.py rename {cancer_ai => neurons}/rewarder.py (60%) rename {cancer_ai => neurons}/rewarder_test.py (99%) diff --git a/cancer_ai/base/neuron.py b/cancer_ai/base/neuron.py index 5b40199f..9878b0a5 100644 --- a/cancer_ai/base/neuron.py +++ b/cancer_ai/base/neuron.py @@ -108,10 +108,6 @@ def __init__(self, config=None): ) self.step = 0 - @abstractmethod - async def forward(self, synapse: bt.Synapse) -> bt.Synapse: - ... - @abstractmethod def run(self): ... @@ -167,13 +163,3 @@ def should_set_weights(self) -> bool: > self.config.neuron.epoch_length and self.neuron_type != "MinerNeuron" ) # don't set weights if you're a miner - - def save_state(self): - bt.logging.warning( - "save_state() not implemented for this neuron. You can implement this function to save model checkpoints or other useful data." - ) - - def load_state(self): - bt.logging.warning( - "load_state() not implemented for this neuron. You can implement this function to load model checkpoints or other useful data." - ) diff --git a/cancer_ai/base/validator.py b/cancer_ai/base/validator.py index 266f859b..eae717be 100644 --- a/cancer_ai/base/validator.py +++ b/cancer_ai/base/validator.py @@ -118,18 +118,13 @@ def run(self): while True: bt.logging.info(f"step({self.step}) block({self.block})") - # Run multiple forwards concurrently. - self.loop.run_until_complete(self.forward()) - # Check if we should exit. if self.should_exit: break # Sync metagraph and potentially set weights. self.sync() - self.step += 1 - # If someone intentionally stops the validator, it'll safely terminate operations. except KeyboardInterrupt: self.axon.stop() @@ -287,51 +282,6 @@ def resync_metagraph(self): # Update the hotkeys. self.hotkeys = copy.deepcopy(self.metagraph.hotkeys) - def update_scores(self, rewards: np.ndarray, uids: List[int]): - """Performs exponential moving average on the scores based on the rewards received from the miners.""" - - # Check if rewards contains NaN values. - if np.isnan(rewards).any(): - bt.logging.warning(f"NaN values detected in rewards: {rewards}") - # Replace any NaN values in rewards with 0. - rewards = np.nan_to_num(rewards, nan=0) - - # Ensure rewards is a numpy array. - rewards = np.asarray(rewards) - - # Check if `uids` is already a numpy array and copy it to avoid the warning. - if isinstance(uids, np.ndarray): - uids_array = uids.copy() - else: - uids_array = np.array(uids) - - # Handle edge case: If either rewards or uids_array is empty. - if rewards.size == 0 or uids_array.size == 0: - bt.logging.info(f"rewards: {rewards}, uids_array: {uids_array}") - bt.logging.warning( - "Either rewards or uids_array is empty. No updates will be performed." - ) - return - - # Check if sizes of rewards and uids_array match. - if rewards.size != uids_array.size: - raise ValueError( - f"Shape mismatch: rewards array of shape {rewards.shape} " - f"cannot be broadcast to uids array of shape {uids_array.shape}" - ) - - # Compute forward pass rewards, assumes uids are mutually exclusive. - # shape: [ metagraph.n ] - scattered_rewards: np.ndarray = np.zeros_like(self.scores) - scattered_rewards[uids_array] = rewards - bt.logging.debug(f"Scattered rewards: {rewards}") - - # Update scores with rewards produced by this step. - # shape: [ metagraph.n ] - alpha: float = self.config.neuron.moving_average_alpha - self.scores: np.ndarray = alpha * scattered_rewards + (1 - alpha) * self.scores - bt.logging.debug(f"Updated moving avg scores: {self.scores}") - def save_state(self): """Saves the state of the validator to a file.""" bt.logging.info("Saving validator state.") @@ -339,9 +289,9 @@ def save_state(self): # Save the state of the validator to file. np.savez( self.config.neuron.full_path + "/state.npz", - step=self.step, scores=self.scores, hotkeys=self.hotkeys, + rewarder_config=self.rewarder_config, ) def load_state(self): @@ -350,6 +300,6 @@ def load_state(self): # Load the state of the validator from file. state = np.load(self.config.neuron.full_path + "/state.npz") - self.step = state["step"] self.scores = state["scores"] self.hotkeys = state["hotkeys"] + self.rewarder_config = state["rewarder_config"] diff --git a/cancer_ai/validator/forward.py b/cancer_ai/validator/forward.py deleted file mode 100644 index d9c2002b..00000000 --- a/cancer_ai/validator/forward.py +++ /dev/null @@ -1,52 +0,0 @@ -# The MIT License (MIT) -# Copyright © 2023 Yuma Rao -# TODO(developer): Set your name -# Copyright © 2023 - -# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated -# documentation files (the “Software”), to deal in the Software without restriction, including without limitation -# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, -# and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all copies or substantial portions of -# the Software. - -# THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO -# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL -# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -# DEALINGS IN THE SOFTWARE. - -import time -import bittensor as bt - -from ..protocol import Dummy -from ..validator.reward import get_rewards -from ..utils.uids import get_random_uids - - -async def forward(self): - """ - The forward function is called by the validator every time step. - - It is responsible for querying the network and scoring the responses. - - Args: - self (:obj:`bittensor.neuron.Neuron`): The neuron object which contains all the necessary state for the validator. - - """ - # TODO(developer): Define how the validator selects a miner to query, how often, etc. - # get_random_uids is an example method, but you can replace it with your own. - miner_uids = get_random_uids(self, k=self.config.neuron.sample_size) - - # Log the results for monitoring purposes. - bt.logging.info(f"Received responses: {responses}") - - # TODO(developer): Define how the validator scores responses. - # Adjust the scores based on responses from miners. - rewards = get_rewards(self, query=self.step, responses=responses) - - bt.logging.info(f"Scored responses: {rewards}") - # Update the scores based on the rewards. You may want to define your own update_scores function for custom behavior. - self.update_scores(rewards, miner_uids) - time.sleep(5) diff --git a/neurons/competition_runner.py b/neurons/competition_runner.py index fd269e92..0696d45e 100644 --- a/neurons/competition_runner.py +++ b/neurons/competition_runner.py @@ -1,13 +1,12 @@ from cancer_ai.validator.competition_manager import CompetitionManager -from datetime import time, datetime +from datetime import datetime import asyncio import json -import timeit -from types import SimpleNamespace -from datetime import datetime, timezone, timedelta +from datetime import datetime, timezone import bittensor as bt from typing import List, Tuple, Dict +from rewarder import Rewarder, RewarderConfig, CompetitionLeader # from cancer_ai.utils.config import config @@ -35,7 +34,7 @@ async def run_competitions_tick( competition_times: Dict[str, CompetitionManager], ) -> Tuple[str, str] | None: """Checks if time is right and launches competition, returns winning hotkey and Competition ID. Should be run each minute.""" - now_time = datetime.now() + now_time = datetime.now(timezone.utc) now_time = f"{now_time.hour}:{now_time.minute}" bt.logging.debug(now_time) if now_time not in competition_times: @@ -54,11 +53,19 @@ async def run_competitions_tick( ) -async def competition_loop(scheduler_config: Dict[str, CompetitionManager]): +async def competition_loop(scheduler_config: Dict[str, CompetitionManager], rewarder_config: RewarderConfig): """Example of scheduling coroutine""" while True: competition_result = await run_competitions_tick(scheduler_config) bt.logging.debug(f"Competition result: {competition_result}") + if competition_result: + winning_evaluation_hotkey, competition_id = competition_result + rewarder = Rewarder(rewarder_config) + updated_rewarder_config = await rewarder.update_scores(winning_evaluation_hotkey, competition_id) + # save state of self.rewarder_config + # save state of self.score (map rewarder config to scores) + print(".....................Updated rewarder config:") + print(updated_rewarder_config) await asyncio.sleep(60) @@ -71,4 +78,5 @@ async def competition_loop(scheduler_config: Dict[str, CompetitionManager]): hotkeys = [] bt_config = {} # get from bt config scheduler_config = config_for_scheduler(bt_config, hotkeys) - asyncio.run(competition_loop(scheduler_config)) + rewarder_config = RewarderConfig({},{}) + asyncio.run(competition_loop(scheduler_config, rewarder_config)) diff --git a/neurons/competition_runner_test.py b/neurons/competition_runner_test.py index edfcdb5e..d2d933b4 100644 --- a/neurons/competition_runner_test.py +++ b/neurons/competition_runner_test.py @@ -5,6 +5,8 @@ import bittensor as bt from typing import List, Dict from competition_runner import run_competitions_tick, competition_loop +from rewarder import RewarderConfig, Rewarder +import time # TODO integrate with bt config test_config = SimpleNamespace( @@ -57,14 +59,30 @@ def config_for_scheduler() -> Dict[str, CompetitionManager]: ) return time_arranged_competitions +async def competition_loop(): + """Example of scheduling coroutine""" + while True: + test_cases = [ + ("hotkey1", "melanoma-1"), + ("hotkey2", "melanoma-1"), + ("hotkey1", "melanoma-2"), + ("hotkey1", "melanoma-1"), + ("hotkey2", "melanoma-3"), + ] + rewarder_config = RewarderConfig(competitionID_to_leader_hotkey_map={}, hotkey_to_score_map={}) + rewarder = Rewarder(rewarder_config) - + for winning_evaluation_hotkey, competition_id in test_cases: + rewarder.scores = {} + updated_rewarder_config = rewarder.update_scores(winning_evaluation_hotkey, competition_id) + print("Updated rewarder config:", updated_rewarder_config) + await asyncio.sleep(10) if __name__ == "__main__": - if True: # run them right away - run_all_competitions(test_config, [],main_competitions_cfg) + # if True: # run them right away + # run_all_competitions(test_config, [],main_competitions_cfg) - else: # Run the scheduling coroutine - scheduler_config = config_for_scheduler() - asyncio.run(competition_loop(scheduler_config)) + # else: # Run the scheduling coroutine + # scheduler_config = config_for_scheduler() + asyncio.run(competition_loop()) diff --git a/cancer_ai/rewarder.py b/neurons/rewarder.py similarity index 60% rename from cancer_ai/rewarder.py rename to neurons/rewarder.py index fbadbd8a..339ccefc 100644 --- a/cancer_ai/rewarder.py +++ b/neurons/rewarder.py @@ -1,5 +1,5 @@ from pydantic import BaseModel -from datetime import datetime, timedelta +from datetime import datetime, timezone class CompetitionLeader(BaseModel): hotkey: str @@ -16,19 +16,20 @@ def __init__(self, rewarder_config: RewarderConfig): self.competition_leader_mapping = rewarder_config.competitionID_to_leader_hotkey_map self.scores = rewarder_config.hotkey_to_score_map - def get_scores_and_reduction(self, competition_id: str, hotkey: str) -> tuple[float, float]: - num_competitions = len(self.competition_leader_mapping) - base_share = 1/num_competitions - + def get_score_and_reduction(self, competition_id: str, hotkey: str) -> tuple[float, float]: # check if current hotkey is already a leader - if self.competition_leader_mapping[competition_id].hotkey == hotkey: - days_as_leader = (datetime.now() - self.competition_leader_mapping[competition_id].leader_since).days + competition = self.competition_leader_mapping.get(competition_id) + if competition and competition.hotkey == hotkey: + days_as_leader = (datetime.now(timezone.utc) - self.competition_leader_mapping[competition_id].leader_since).days + else: days_as_leader = 0 self.competition_leader_mapping[competition_id] = CompetitionLeader(hotkey=hotkey, - leader_since=datetime.now()) + leader_since=datetime.now(timezone.utc)) + return # Score degradation starts on 3rd week of leadership + base_share = 1/len(self.competition_leader_mapping) if days_as_leader > 14: periods = (days_as_leader - 14) // 7 reduction_factor = max(0.1, 1 - 0.1 * periods) @@ -37,24 +38,32 @@ def get_scores_and_reduction(self, competition_id: str, hotkey: str) -> tuple[fl return final_share, reduced_share return base_share, 0 - def update_scores(self): - num_competitions = len(self.competition_leader_mapping) + def update_scores(self, new_winner_hotkey: str, new_winner_comp_id: str) -> RewarderConfig: + # get score and reduced share for the new winner + self.get_score_and_reduction(new_winner_comp_id, new_winner_hotkey) + num_competitions = len(self.competition_leader_mapping) # If there is only one competition, the winner takes it all if num_competitions == 1: competition_id = next(iter(self.competition_leader_mapping)) hotkey = self.competition_leader_mapping[competition_id].hotkey self.scores[hotkey] = Score(score=1.0, reduction=0.0) - return + return RewarderConfig(competitionID_to_leader_hotkey_map=self.competition_leader_mapping, hotkey_to_score_map=self.scores) # gather reduced shares for all competitors competitions_without_reduction = [] for curr_competition_id, comp_leader in self.competition_leader_mapping.items(): - score, reduced_share = self.get_scores_and_reduction(curr_competition_id, comp_leader.hotkey) - self.scores[comp_leader.hotkey].score += score - self.scores[comp_leader.hotkey].reduction += reduced_share - if reduced_share == 0: - competitions_without_reduction.append(curr_competition_id) + score, reduced_share = self.get_score_and_reduction(curr_competition_id, comp_leader.hotkey) + + if comp_leader.hotkey in self.scores: + self.scores[comp_leader.hotkey].score += score + self.scores[comp_leader.hotkey].reduction += reduced_share + if reduced_share == 0: + competitions_without_reduction.append(curr_competition_id) + else: + self.scores[comp_leader.hotkey] = Score(score=score, reduction=reduced_share) + if reduced_share == 0: + competitions_without_reduction.append(curr_competition_id) total_reduced_share = sum([score.reduction for score in self.scores.values()]) @@ -68,4 +77,6 @@ def update_scores(self): # distribute the total reduced share among non-reduced competitons winners for comp_id in competitions_without_reduction: hotkey = self.competition_leader_mapping[comp_id].hotkey - self.scores[hotkey].score += total_reduced_share / len(competitions_without_reduction) \ No newline at end of file + self.scores[hotkey].score += total_reduced_share / len(competitions_without_reduction) + + return RewarderConfig(competitionID_to_leader_hotkey_map=self.competition_leader_mapping, hotkey_to_score_map=self.scores) \ No newline at end of file diff --git a/cancer_ai/rewarder_test.py b/neurons/rewarder_test.py similarity index 99% rename from cancer_ai/rewarder_test.py rename to neurons/rewarder_test.py index 7547a637..75e8fc66 100644 --- a/cancer_ai/rewarder_test.py +++ b/neurons/rewarder_test.py @@ -1,6 +1,5 @@ import pytest from datetime import datetime, timedelta -from pydantic import BaseModel from .rewarder import CompetitionLeader, Score, RewarderConfig, Rewarder def test_update_scores_single_competitor(): diff --git a/neurons/validator.py b/neurons/validator.py index a3f7c589..063446f0 100644 --- a/neurons/validator.py +++ b/neurons/validator.py @@ -31,6 +31,8 @@ from datetime import datetime, timezone, timedelta from cancer_ai.validator.competition_manager import CompetitionManager from cancer_ai.validator.competition_handlers.base_handler import ModelEvaluationResult +from .competition_runner import competition_loop, config_for_scheduler, run_competitions_tick +from .rewarder import RewarderConfig, Rewarder class Validator(BaseValidatorNeuron): @@ -44,12 +46,39 @@ class Validator(BaseValidatorNeuron): def __init__(self, config=None): super(Validator, self).__init__(config=config) - # competition_id to (hotkey_uid, days_as_leader) - self.competitions_leaders = {} + + self.rewarder_config = RewarderConfig({},{}) self.load_state() + self.scheduler_config = config_for_scheduler(self.config, self.hotkeys) + + self.rewarder = Rewarder(self.rewarder_config) - async def forward(self): - ... + asyncio.run_coroutine_threadsafe(competition_loop(self.scheduler_config, self.rewarder_config), self.loop) + + + async def competition_loop(self, scheduler_config: dict[str, CompetitionManager], rewarder_config: RewarderConfig): + """Example of scheduling coroutine""" + while True: + competition_result = await run_competitions_tick(scheduler_config) + bt.logging.debug(f"Competition result: {competition_result}") + if competition_result: + winning_evaluation_hotkey, competition_id = competition_result + + # reset the scores before updating them + self.rewarder.scores = {} + + # update the scores + updated_rewarder_config = await self.rewarder.update_scores(winning_evaluation_hotkey, competition_id) + self.rewarder_config = updated_rewarder_config + self.save_state() + + hotkey_to_score_map = updated_rewarder_config.hotkey_to_score_map + + # get hotkeys to uid mapping + # save state of self.score (map rewarder config to scores) + print(".....................Updated rewarder config:") + print(updated_rewarder_config) + await asyncio.sleep(60) # The main function parses the configuration and runs the validator. if __name__ == "__main__": From 2d2a28ef2da920419d0254670b23c70fb88c1d45 Mon Sep 17 00:00:00 2001 From: Wojtek Jurkowlaniec Date: Sun, 1 Sep 2024 05:00:17 +0200 Subject: [PATCH 125/227] fixes for configuration --- cancer_ai/utils/config.py | 2 +- neurons/miner.py | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/cancer_ai/utils/config.py b/cancer_ai/utils/config.py index 147ce476..52ebd47c 100644 --- a/cancer_ai/utils/config.py +++ b/cancer_ai/utils/config.py @@ -317,4 +317,4 @@ def path_config(cls): bt.logging.add_args(parser) bt.axon.add_args(parser) cls.add_args(parser) - return bt.config(parser) + return bt.config(parser.parse_args()) diff --git a/neurons/miner.py b/neurons/miner.py index be17cd7f..f2b342a2 100644 --- a/neurons/miner.py +++ b/neurons/miner.py @@ -7,8 +7,6 @@ from huggingface_hub import HfApi, login as hf_login import huggingface_hub import onnx -import cancer_ai -import typing import argparse from cancer_ai.validator.utils import run_command @@ -35,6 +33,7 @@ def __init__(self, config=None): @classmethod def add_args(cls, parser: argparse.ArgumentParser): """Method for injecting miner arguments to the parser.""" + print("add") add_miner_args(cls, parser) async def upload_to_hf(self) -> None: From 2b2ae6fa6296a3ef58600f9d966465a2c8019d05 Mon Sep 17 00:00:00 2001 From: Konrad Date: Sun, 1 Sep 2024 14:00:36 +0200 Subject: [PATCH 126/227] rewarded logic adjustments --- neurons/competition_runner_test.py | 6 +++--- neurons/rewarder.py | 19 ++++++++++--------- neurons/validator.py | 21 ++++++++++----------- 3 files changed, 23 insertions(+), 23 deletions(-) diff --git a/neurons/competition_runner_test.py b/neurons/competition_runner_test.py index d2d933b4..46beb8fb 100644 --- a/neurons/competition_runner_test.py +++ b/neurons/competition_runner_test.py @@ -74,9 +74,9 @@ async def competition_loop(): rewarder = Rewarder(rewarder_config) for winning_evaluation_hotkey, competition_id in test_cases: - rewarder.scores = {} - updated_rewarder_config = rewarder.update_scores(winning_evaluation_hotkey, competition_id) - print("Updated rewarder config:", updated_rewarder_config) + rewarder.update_scores(winning_evaluation_hotkey, competition_id) + print("Updated rewarder competition leader map:", rewarder.competition_leader_mapping) + print("Updated rewarder scores:", rewarder.scores) await asyncio.sleep(10) if __name__ == "__main__": diff --git a/neurons/rewarder.py b/neurons/rewarder.py index 339ccefc..6f27591f 100644 --- a/neurons/rewarder.py +++ b/neurons/rewarder.py @@ -38,7 +38,10 @@ def get_score_and_reduction(self, competition_id: str, hotkey: str) -> tuple[flo return final_share, reduced_share return base_share, 0 - def update_scores(self, new_winner_hotkey: str, new_winner_comp_id: str) -> RewarderConfig: + def update_scores(self, new_winner_hotkey: str, new_winner_comp_id: str): + # reset the scores before updating them + self.scores = {} + # get score and reduced share for the new winner self.get_score_and_reduction(new_winner_comp_id, new_winner_hotkey) @@ -48,7 +51,7 @@ def update_scores(self, new_winner_hotkey: str, new_winner_comp_id: str) -> Rewa competition_id = next(iter(self.competition_leader_mapping)) hotkey = self.competition_leader_mapping[competition_id].hotkey self.scores[hotkey] = Score(score=1.0, reduction=0.0) - return RewarderConfig(competitionID_to_leader_hotkey_map=self.competition_leader_mapping, hotkey_to_score_map=self.scores) + return # gather reduced shares for all competitors competitions_without_reduction = [] @@ -73,10 +76,8 @@ def update_scores(self, new_winner_hotkey: str, new_winner_comp_id: str) -> Rewa for hotkey, score in self.scores.items(): self.scores[hotkey].score += total_reduced_share / num_competitions return - else: - # distribute the total reduced share among non-reduced competitons winners - for comp_id in competitions_without_reduction: - hotkey = self.competition_leader_mapping[comp_id].hotkey - self.scores[hotkey].score += total_reduced_share / len(competitions_without_reduction) - - return RewarderConfig(competitionID_to_leader_hotkey_map=self.competition_leader_mapping, hotkey_to_score_map=self.scores) \ No newline at end of file + + # distribute the total reduced share among non-reduced competitons winners + for comp_id in competitions_without_reduction: + hotkey = self.competition_leader_mapping[comp_id].hotkey + self.scores[hotkey].score += total_reduced_share / len(competitions_without_reduction) \ No newline at end of file diff --git a/neurons/validator.py b/neurons/validator.py index 063446f0..bf8a5a69 100644 --- a/neurons/validator.py +++ b/neurons/validator.py @@ -32,7 +32,7 @@ from cancer_ai.validator.competition_manager import CompetitionManager from cancer_ai.validator.competition_handlers.base_handler import ModelEvaluationResult from .competition_runner import competition_loop, config_for_scheduler, run_competitions_tick -from .rewarder import RewarderConfig, Rewarder +from .rewarder import RewarderConfig, Rewarder, Score class Validator(BaseValidatorNeuron): @@ -55,7 +55,6 @@ def __init__(self, config=None): asyncio.run_coroutine_threadsafe(competition_loop(self.scheduler_config, self.rewarder_config), self.loop) - async def competition_loop(self, scheduler_config: dict[str, CompetitionManager], rewarder_config: RewarderConfig): """Example of scheduling coroutine""" while True: @@ -64,20 +63,20 @@ async def competition_loop(self, scheduler_config: dict[str, CompetitionManager] if competition_result: winning_evaluation_hotkey, competition_id = competition_result - # reset the scores before updating them - self.rewarder.scores = {} - # update the scores - updated_rewarder_config = await self.rewarder.update_scores(winning_evaluation_hotkey, competition_id) - self.rewarder_config = updated_rewarder_config + await self.rewarder.update_scores(winning_evaluation_hotkey, competition_id) + self.rewarder_config = RewarderConfig(self.rewarder.competition_leader_mapping, self.rewarder.scores) self.save_state() - hotkey_to_score_map = updated_rewarder_config.hotkey_to_score_map + hotkey_to_score_map = self.rewarder_config.hotkey_to_score_map - # get hotkeys to uid mapping - # save state of self.score (map rewarder config to scores) + self.scores = [ + hotkey_to_score_map.get(hotkey, Score(score=0.0, reduction=0.0)).score + for hotkey in self.metagraph.hotkeys + ] + self.save_state() print(".....................Updated rewarder config:") - print(updated_rewarder_config) + print(self.rewarder_config) await asyncio.sleep(60) # The main function parses the configuration and runs the validator. From b7acdedd33dc4b4092b2c100961f908ce46ba50f Mon Sep 17 00:00:00 2001 From: Konrad Date: Sun, 1 Sep 2024 12:15:46 +0000 Subject: [PATCH 127/227] resolved Wojtek's comments --- neurons/rewarder.py | 12 +++++++----- neurons/rewarder_test.py | 34 +++++++++++++++++----------------- 2 files changed, 24 insertions(+), 22 deletions(-) diff --git a/neurons/rewarder.py b/neurons/rewarder.py index 6f27591f..9b2e1939 100644 --- a/neurons/rewarder.py +++ b/neurons/rewarder.py @@ -11,12 +11,14 @@ class RewarderConfig(BaseModel): competitionID_to_leader_hotkey_map: dict[str, CompetitionLeader] # competition_id -> CompetitionLeader hotkey_to_score_map: dict[str, Score] # hotkey -> Score +NON_REDUCTION_PERIOD = 30 + class Rewarder(): def __init__(self, rewarder_config: RewarderConfig): self.competition_leader_mapping = rewarder_config.competitionID_to_leader_hotkey_map self.scores = rewarder_config.hotkey_to_score_map - def get_score_and_reduction(self, competition_id: str, hotkey: str) -> tuple[float, float]: + def get_miner_score_and_reduction(self, competition_id: str, hotkey: str) -> tuple[float, float]: # check if current hotkey is already a leader competition = self.competition_leader_mapping.get(competition_id) if competition and competition.hotkey == hotkey: @@ -30,8 +32,8 @@ def get_score_and_reduction(self, competition_id: str, hotkey: str) -> tuple[flo # Score degradation starts on 3rd week of leadership base_share = 1/len(self.competition_leader_mapping) - if days_as_leader > 14: - periods = (days_as_leader - 14) // 7 + if days_as_leader > NON_REDUCTION_PERIOD: + periods = (days_as_leader - NON_REDUCTION_PERIOD) // 7 reduction_factor = max(0.1, 1 - 0.1 * periods) final_share = base_share * reduction_factor reduced_share = base_share - final_share @@ -43,7 +45,7 @@ def update_scores(self, new_winner_hotkey: str, new_winner_comp_id: str): self.scores = {} # get score and reduced share for the new winner - self.get_score_and_reduction(new_winner_comp_id, new_winner_hotkey) + self.get_miner_score_and_reduction(new_winner_comp_id, new_winner_hotkey) num_competitions = len(self.competition_leader_mapping) # If there is only one competition, the winner takes it all @@ -56,7 +58,7 @@ def update_scores(self, new_winner_hotkey: str, new_winner_comp_id: str): # gather reduced shares for all competitors competitions_without_reduction = [] for curr_competition_id, comp_leader in self.competition_leader_mapping.items(): - score, reduced_share = self.get_score_and_reduction(curr_competition_id, comp_leader.hotkey) + score, reduced_share = self.get_miner_score_and_reduction(curr_competition_id, comp_leader.hotkey) if comp_leader.hotkey in self.scores: self.scores[comp_leader.hotkey].score += score diff --git a/neurons/rewarder_test.py b/neurons/rewarder_test.py index 75e8fc66..6414a58f 100644 --- a/neurons/rewarder_test.py +++ b/neurons/rewarder_test.py @@ -72,10 +72,10 @@ def test_update_scores_multiple_competitors_no_reduction(): def test_update_scores_multiple_competitors_with_some_reduced_shares(): # Set up initial data for multiple competitors competition_leaders = { - "competition1": CompetitionLeader(hotkey="competitor1", leader_since=datetime.now() - timedelta(days=14 + 3 * 7)), - "competition2": CompetitionLeader(hotkey="competitor2", leader_since=datetime.now() - timedelta(days=14 + 6 * 7)), - "competition3": CompetitionLeader(hotkey="competitor3", leader_since=datetime.now() - timedelta(days=14)), - "competition4": CompetitionLeader(hotkey="competitor4", leader_since=datetime.now() - timedelta(days=14)), + "competition1": CompetitionLeader(hotkey="competitor1", leader_since=datetime.now() - timedelta(days=30 + 3 * 7)), + "competition2": CompetitionLeader(hotkey="competitor2", leader_since=datetime.now() - timedelta(days=30 + 6 * 7)), + "competition3": CompetitionLeader(hotkey="competitor3", leader_since=datetime.now() - timedelta(days=30)), + "competition4": CompetitionLeader(hotkey="competitor4", leader_since=datetime.now() - timedelta(days=30)), } scores = { @@ -123,9 +123,9 @@ def test_update_scores_multiple_competitors_with_some_reduced_shares(): def test_update_scores_all_competitors_with_reduced_shares(): # Set up initial data for multiple competitors competition_leaders = { - "competition1": CompetitionLeader(hotkey="competitor1", leader_since=datetime.now() - timedelta(days=14 + 3 * 7)), - "competition2": CompetitionLeader(hotkey="competitor2", leader_since=datetime.now() - timedelta(days=14 + 6 * 7)), - "competition3": CompetitionLeader(hotkey="competitor3", leader_since=datetime.now() - timedelta(days=14 + 9 * 7)) + "competition1": CompetitionLeader(hotkey="competitor1", leader_since=datetime.now() - timedelta(days=30 + 3 * 7)), + "competition2": CompetitionLeader(hotkey="competitor2", leader_since=datetime.now() - timedelta(days=30 + 6 * 7)), + "competition3": CompetitionLeader(hotkey="competitor3", leader_since=datetime.now() - timedelta(days=30 + 9 * 7)) } scores = { @@ -170,10 +170,10 @@ def test_update_scores_all_competitors_with_reduced_shares(): def test_update_scores_more_competitions_then_competitors(): # Set up initial data for multiple competitors competition_leaders = { - "competition1": CompetitionLeader(hotkey="competitor1", leader_since=datetime.now() - timedelta(days=14 + 3 * 7)), - "competition2": CompetitionLeader(hotkey="competitor2", leader_since=datetime.now() - timedelta(days=14)), - "competition3": CompetitionLeader(hotkey="competitor1", leader_since=datetime.now() - timedelta(days=14)), - "competition4": CompetitionLeader(hotkey="competitor3", leader_since=datetime.now() - timedelta(days=14)), + "competition1": CompetitionLeader(hotkey="competitor1", leader_since=datetime.now() - timedelta(days=30 + 3 * 7)), + "competition2": CompetitionLeader(hotkey="competitor2", leader_since=datetime.now() - timedelta(days=30)), + "competition3": CompetitionLeader(hotkey="competitor1", leader_since=datetime.now() - timedelta(days=30)), + "competition4": CompetitionLeader(hotkey="competitor3", leader_since=datetime.now() - timedelta(days=30)), } scores = { @@ -218,12 +218,12 @@ def test_update_scores_more_competitions_then_competitors(): def test_update_scores_6_competitions_3_competitors(): # Set up initial data for multiple competitors competition_leaders = { - "competition1": CompetitionLeader(hotkey="competitor1", leader_since=datetime.now() - timedelta(days=14 + 3 * 7)), - "competition2": CompetitionLeader(hotkey="competitor2", leader_since=datetime.now() - timedelta(days=14 + 6 * 7)), - "competition3": CompetitionLeader(hotkey="competitor3", leader_since=datetime.now() - timedelta(days=14 + 9 * 7)), - "competition4": CompetitionLeader(hotkey="competitor4", leader_since=datetime.now() - timedelta(days=14)), - "competition5": CompetitionLeader(hotkey="competitor1", leader_since=datetime.now() - timedelta(days=14)), - "competition6": CompetitionLeader(hotkey="competitor2", leader_since=datetime.now() - timedelta(days=14 + 3 * 7)), + "competition1": CompetitionLeader(hotkey="competitor1", leader_since=datetime.now() - timedelta(days=30 + 3 * 7)), + "competition2": CompetitionLeader(hotkey="competitor2", leader_since=datetime.now() - timedelta(days=30 + 6 * 7)), + "competition3": CompetitionLeader(hotkey="competitor3", leader_since=datetime.now() - timedelta(days=30 + 9 * 7)), + "competition4": CompetitionLeader(hotkey="competitor4", leader_since=datetime.now() - timedelta(days=30)), + "competition5": CompetitionLeader(hotkey="competitor1", leader_since=datetime.now() - timedelta(days=30)), + "competition6": CompetitionLeader(hotkey="competitor2", leader_since=datetime.now() - timedelta(days=30 + 3 * 7)), } scores = { From 3cbb8206626e506507febd516321cddbf0d3163a Mon Sep 17 00:00:00 2001 From: Konrad Date: Sun, 1 Sep 2024 17:39:07 +0200 Subject: [PATCH 128/227] adjustments --- neurons/competition_runner.py | 30 ++++++++++++++++-------------- neurons/miner.py | 2 -- neurons/validator.py | 14 +++++--------- 3 files changed, 21 insertions(+), 25 deletions(-) diff --git a/neurons/competition_runner.py b/neurons/competition_runner.py index 0696d45e..d2a87442 100644 --- a/neurons/competition_runner.py +++ b/neurons/competition_runner.py @@ -12,7 +12,7 @@ def config_for_scheduler( - bt_config, hotkeys: List[str] + bt_config, hotkeys: List[str], test_mode: bool = False ) -> Dict[str, CompetitionManager]: """Returns CompetitionManager instances arranged by competition time""" time_arranged_competitions = {} @@ -26,6 +26,7 @@ def config_for_scheduler( competition_cfg["dataset_hf_repo"], competition_cfg["dataset_hf_filename"], competition_cfg["dataset_hf_repo_type"], + test_mode=test_mode, ) return time_arranged_competitions @@ -37,20 +38,21 @@ async def run_competitions_tick( now_time = datetime.now(timezone.utc) now_time = f"{now_time.hour}:{now_time.minute}" bt.logging.debug(now_time) - if now_time not in competition_times: - return None + # if now_time not in competition_times: + # return None for time_competition in competition_times: - if now_time == time_competition: - bt.logging.info( - f"Running {competition_times[time_competition].competition_id} at {now_time}" - ) - winning_evaluation_hotkey = await competition_times[ - time_competition - ].evaluate() - return ( - winning_evaluation_hotkey, - competition_times[time_competition].competition_id, - ) + asyncio.sleep(60) + # if now_time == time_competition: + bt.logging.info( + f"Running {competition_times[time_competition].competition_id} at {now_time}" + ) + winning_evaluation_hotkey = await competition_times[ + time_competition + ].evaluate() + return ( + winning_evaluation_hotkey, + competition_times[time_competition].competition_id, + ) async def competition_loop(scheduler_config: Dict[str, CompetitionManager], rewarder_config: RewarderConfig): diff --git a/neurons/miner.py b/neurons/miner.py index f71f821b..a9db0e56 100644 --- a/neurons/miner.py +++ b/neurons/miner.py @@ -192,12 +192,10 @@ async def main(self) -> None: if __name__ == "__main__": from types import SimpleNamespace - config = get_config() config = { "dataset_dir": "./data", } config = SimpleNamespace( **config) - set_log_formatting() load_dotenv() cli_manager = MinerManagerCLI() asyncio.run(cli_manager.main()) diff --git a/neurons/validator.py b/neurons/validator.py index bf8a5a69..bfee1a6b 100644 --- a/neurons/validator.py +++ b/neurons/validator.py @@ -23,16 +23,12 @@ import bittensor as bt import asyncio -from numpy import ndarray +import numpy as np from cancer_ai.base.validator import BaseValidatorNeuron -from cancer_ai.validator import forward -from types import SimpleNamespace -from datetime import datetime, timezone, timedelta from cancer_ai.validator.competition_manager import CompetitionManager -from cancer_ai.validator.competition_handlers.base_handler import ModelEvaluationResult -from .competition_runner import competition_loop, config_for_scheduler, run_competitions_tick -from .rewarder import RewarderConfig, Rewarder, Score +from competition_runner import competition_loop, config_for_scheduler, run_competitions_tick +from rewarder import RewarderConfig, Rewarder, Score class Validator(BaseValidatorNeuron): @@ -49,7 +45,7 @@ def __init__(self, config=None): self.rewarder_config = RewarderConfig({},{}) self.load_state() - self.scheduler_config = config_for_scheduler(self.config, self.hotkeys) + self.scheduler_config = config_for_scheduler(self.config, self.hotkeys, test_mode=True) self.rewarder = Rewarder(self.rewarder_config) @@ -71,7 +67,7 @@ async def competition_loop(self, scheduler_config: dict[str, CompetitionManager] hotkey_to_score_map = self.rewarder_config.hotkey_to_score_map self.scores = [ - hotkey_to_score_map.get(hotkey, Score(score=0.0, reduction=0.0)).score + np.float32(hotkey_to_score_map.get(hotkey, Score(score=0.0, reduction=0.0)).score) for hotkey in self.metagraph.hotkeys ] self.save_state() From b5228d914e70a9eba75f940686dc12390d11164c Mon Sep 17 00:00:00 2001 From: Wojtek Jurkowlaniec Date: Sat, 31 Aug 2024 03:44:26 +0200 Subject: [PATCH 129/227] miner config --- neurons/miner.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/neurons/miner.py b/neurons/miner.py index a9db0e56..be17cd7f 100644 --- a/neurons/miner.py +++ b/neurons/miner.py @@ -191,11 +191,6 @@ async def main(self) -> None: if __name__ == "__main__": - from types import SimpleNamespace - config = { - "dataset_dir": "./data", - } - config = SimpleNamespace( **config) load_dotenv() cli_manager = MinerManagerCLI() asyncio.run(cli_manager.main()) From 1756e45ec68141a36ef13d73909297c223440797 Mon Sep 17 00:00:00 2001 From: Wojtek Jurkowlaniec Date: Sat, 31 Aug 2024 23:20:43 +0200 Subject: [PATCH 130/227] fixing feet --- DOCS/miner.md | 28 ++++++++++++++-------------- cancer_ai/utils/config.py | 31 +++++++++++-------------------- 2 files changed, 25 insertions(+), 34 deletions(-) diff --git a/DOCS/miner.md b/DOCS/miner.md index 62c9e210..de2df7e4 100644 --- a/DOCS/miner.md +++ b/DOCS/miner.md @@ -33,7 +33,7 @@ This mode will do following things -`python neurons/miner.py --action evaluate --competition-id --model-path ` +`python neurons/miner.py --action evaluate --competition_id --model_path ` If flag `--clean-after-run` is supplied, it will delete dataset after evaluating the model @@ -42,16 +42,16 @@ If flag `--clean-after-run` is supplied, it will delete dataset after evaluating - compresses code provided by --code-path - uploads model and code to HuggingFace -`python neurons/miner.py --action upload --competition-id melanoma-1 --model-path test_model.onnx --hf-model-name file_name.zip --hf-repo-id repo/id --hf-token TOKEN` +`python neurons/miner.py --action upload --competition_id melanoma-1 --model_path test_model.onnx --hf_model_name file_name.zip --hf_repo_id repo/id --hf_token TOKEN` ```bash python neurons/miner.py \ --action upload \ - --competition-id \ - --model-path \ - --code-directory \ - --hf-model-name \ - --hf-repo-id \ - --hf-token \ + --competition_id \ + --model_path \ + --code_directory \ + --hf_model_name \ + --hf_repo_id \ + --hf_token \ --logging.debug ``` @@ -64,12 +64,12 @@ python neurons/miner.py \ ```bash python neurons/miner.py \ --action submit \ - --model-path \ - --competition-id \ - --hf-code-filename "melanoma-1-piwo.zip" \ - --hf-model-name \ - --hf-repo-id \ - --hf-repo-type model \ + --model_path \ + --competition_id \ + --hf_code_filename "melanoma-1-piwo.zip" \ + --hf_model_name \ + --hf_repo_id \ + --hf_repo_type model \ --wallet.name \ --wallet.hotkey \ --netuid \ diff --git a/cancer_ai/utils/config.py b/cancer_ai/utils/config.py index 8cf80bdf..147ce476 100644 --- a/cancer_ai/utils/config.py +++ b/cancer_ai/utils/config.py @@ -132,84 +132,75 @@ def add_args(cls, parser): def add_miner_args(cls, parser): """Add miner specific arguments to the parser.""" parser.add_argument( - "--competition-id", + "--competition_id", type=str, help="Competition ID", - required=True, ) parser.add_argument( - "--model-dir", + "--model_dir", type=str, help="Path for for loading the starting model related to a training run.", default="./models", ) parser.add_argument( - "--hf-repo-id", + "--hf_repo_id", type=str, help="Hugging Face model repository ID", ) parser.add_argument( - "--hf-model-name", + "--hf_model_name", type=str, help="Filename of the model to push to hugging face.", ) parser.add_argument( - "--hf-code-filename", + "--hf_code_filename", type=str, help="Filename of the code zip to push to hugging face.", ) parser.add_argument( - "--hf-repo-type", + "--hf_repo_type", type=str, help="Type of hugging face repository.", ) - parser.add_argument( - "--hf-model-name", - type=str, - help="Name of the model to push to hugging face.", - ) - parser.add_argument( "--action", choices=["submit", "evaluate", "upload"], - required=True, ) parser.add_argument( - "--model-path", + "--model_path", type=str, help="Path to ONNX model, used for evaluation", - required=True, ) parser.add_argument( - "--dataset-dir", + "--dataset_dir", type=str, help="Path for storing datasets.", default="./datasets", ) parser.add_argument( - "--hf-token", + "--hf_token", type=str, help="Hugging Face API token", default="", ) parser.add_argument( - "--clean-after-run", + "--clean_after_run", action="store_true", help="Whether to clean up (dataset, temporary files) after running", default=False, ) parser.add_argument( - "--code-directory", + "--code_directory", type=str, help="Path to code directory", default=".", From 302bcc0590de2125d0034278aecd4fb086f42e43 Mon Sep 17 00:00:00 2001 From: Wojtek Jurkowlaniec Date: Sun, 1 Sep 2024 05:00:17 +0200 Subject: [PATCH 131/227] fixes for configuration --- cancer_ai/utils/config.py | 2 +- neurons/miner.py | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/cancer_ai/utils/config.py b/cancer_ai/utils/config.py index 147ce476..52ebd47c 100644 --- a/cancer_ai/utils/config.py +++ b/cancer_ai/utils/config.py @@ -317,4 +317,4 @@ def path_config(cls): bt.logging.add_args(parser) bt.axon.add_args(parser) cls.add_args(parser) - return bt.config(parser) + return bt.config(parser.parse_args()) diff --git a/neurons/miner.py b/neurons/miner.py index be17cd7f..f2b342a2 100644 --- a/neurons/miner.py +++ b/neurons/miner.py @@ -7,8 +7,6 @@ from huggingface_hub import HfApi, login as hf_login import huggingface_hub import onnx -import cancer_ai -import typing import argparse from cancer_ai.validator.utils import run_command @@ -35,6 +33,7 @@ def __init__(self, config=None): @classmethod def add_args(cls, parser: argparse.ArgumentParser): """Method for injecting miner arguments to the parser.""" + print("add") add_miner_args(cls, parser) async def upload_to_hf(self) -> None: From c1097f781945c7d6060824a49606b8aff6cad971 Mon Sep 17 00:00:00 2001 From: Konrad Date: Sun, 1 Sep 2024 23:47:10 +0200 Subject: [PATCH 132/227] WIP testing validator --- cancer_ai/base/{miner.py => base_miner.py} | 2 +- .../base/{validator.py => base_validator.py} | 4 +- cancer_ai/validator/competition_manager.py | 60 +++++++++---------- cancer_ai/validator/dataset_manager.py | 17 +++--- cancer_ai/validator/model_manager.py | 1 + cancer_ai/validator/reward.py | 55 ----------------- neurons/competition_runner.py | 1 - neurons/miner.py | 2 +- neurons/rewarder.py | 19 +++--- neurons/validator.py | 46 +++++++++++++- tests/test_template_validator.py | 2 +- 11 files changed, 99 insertions(+), 110 deletions(-) rename cancer_ai/base/{miner.py => base_miner.py} (99%) rename cancer_ai/base/{validator.py => base_validator.py} (99%) delete mode 100644 cancer_ai/validator/reward.py diff --git a/cancer_ai/base/miner.py b/cancer_ai/base/base_miner.py similarity index 99% rename from cancer_ai/base/miner.py rename to cancer_ai/base/base_miner.py index 2990e77a..66b8e6ec 100644 --- a/cancer_ai/base/miner.py +++ b/cancer_ai/base/base_miner.py @@ -23,7 +23,7 @@ import bittensor as bt -from ..base.neuron import BaseNeuron +from .neuron import BaseNeuron from ..utils.config import add_miner_args from typing import Union diff --git a/cancer_ai/base/validator.py b/cancer_ai/base/base_validator.py similarity index 99% rename from cancer_ai/base/validator.py rename to cancer_ai/base/base_validator.py index eae717be..f1881880 100644 --- a/cancer_ai/base/validator.py +++ b/cancer_ai/base/base_validator.py @@ -28,8 +28,8 @@ from typing import List, Union from traceback import print_exception -from ..base.neuron import BaseNeuron -from ..base.utils.weight_utils import ( +from .neuron import BaseNeuron +from .utils.weight_utils import ( process_weights_for_netuid, convert_weights_and_uids_for_emit, ) # TODO: Replace when bittensor switches to numpy diff --git a/cancer_ai/validator/competition_manager.py b/cancer_ai/validator/competition_manager.py index 676ce196..46cba5aa 100644 --- a/cancer_ai/validator/competition_manager.py +++ b/cancer_ai/validator/competition_manager.py @@ -78,35 +78,35 @@ def __init__( self.chain_miner_models = {} self.test_mode = test_mode - def log_results_to_wandb( - self, hotkey: str, evaluation_result: ModelEvaluationResult - ) -> None: - wandb.init(project=self.config.wandb_project_name) - wandb.log( - { - "hotkey": hotkey, - "tested_entries": evaluation_result.tested_entries, - "accuracy": evaluation_result.accuracy, - "precision": evaluation_result.precision, - "recall": evaluation_result.recall, - "confusion_matrix": evaluation_result.confusion_matrix.tolist(), - "roc_curve": { - "fpr": evaluation_result.fpr.tolist(), - "tpr": evaluation_result.tpr.tolist(), - }, - "roc_auc": evaluation_result.roc_auc, - } - ) - - wandb.finish() - bt.logging.info("Logged results to wandb") - bt.logging.info("Hotkey: ", hotkey) - bt.logging.info("Tested entries: ", evaluation_result.tested_entries) - bt.logging.info("Model test run time: ", evaluation_result.run_time_s) - bt.logging.info("Accuracy: ", evaluation_result.accuracy) - bt.logging.info("Precision: ", evaluation_result.precision) - bt.logging.info("Recall: ", evaluation_result.recall) - bt.logging.info("roc_auc: ", evaluation_result.roc_auc) + # def log_results_to_wandb( + # self, hotkey: str, evaluation_result: ModelEvaluationResult + # ) -> None: + # wandb.init(project=self.config.wandb_project_name) + # wandb.log( + # { + # "hotkey": hotkey, + # "tested_entries": evaluation_result.tested_entries, + # "accuracy": evaluation_result.accuracy, + # "precision": evaluation_result.precision, + # "recall": evaluation_result.recall, + # "confusion_matrix": evaluation_result.confusion_matrix.tolist(), + # "roc_curve": { + # "fpr": evaluation_result.fpr.tolist(), + # "tpr": evaluation_result.tpr.tolist(), + # }, + # "roc_auc": evaluation_result.roc_auc, + # } + # ) + + # wandb.finish() + # bt.logging.info("Logged results to wandb") + # bt.logging.info("Hotkey: ", hotkey) + # bt.logging.info("Tested entries: ", evaluation_result.tested_entries) + # bt.logging.info("Model test run time: ", evaluation_result.run_time_s) + # bt.logging.info("Accuracy: ", evaluation_result.accuracy) + # bt.logging.info("Precision: ", evaluation_result.precision) + # bt.logging.info("Recall: ", evaluation_result.recall) + # bt.logging.info("roc_auc: ", evaluation_result.roc_auc) @@ -199,7 +199,7 @@ async def evaluate(self) -> str: y_test, y_pred, run_time_s ) self.results.append((hotkey, model_result)) - self.log_results_to_wandb(hotkey, model_result) + # self.log_results_to_wandb(hotkey, model_result) winning_hotkey = sorted( self.results, key=lambda x: x[1].accuracy, reverse=True diff --git a/cancer_ai/validator/dataset_manager.py b/cancer_ai/validator/dataset_manager.py index 18f47793..309248f8 100644 --- a/cancer_ai/validator/dataset_manager.py +++ b/cancer_ai/validator/dataset_manager.py @@ -59,6 +59,7 @@ async def download_dataset(self): self.hf_filename, cache_dir=Path(self.config.dataset_dir), repo_type=self.hf_repo_type, + token=self.config.hf_token, ) def delete_dataset(self) -> None: @@ -77,7 +78,7 @@ async def unzip_dataset(self) -> None: """Unzip dataset""" self.local_extracted_dir = Path( - self.config.dataset_dir, self.competition_id + self.config.dataset_dir, self.config.competition_id ) bt.logging.debug(f"Dataset extracted to: { self.local_compressed_path}") @@ -92,7 +93,7 @@ async def unzip_dataset(self) -> None: def set_dataset_handler(self) -> None: """Detect dataset type and set handler""" if not self.local_compressed_path: - raise DatasetManagerException(f"Dataset '{self.competition_id}' not downloaded") + raise DatasetManagerException(f"Dataset '{self.config.competition_id}' not downloaded") # is csv in directory if os.path.exists(Path(self.local_extracted_dir, "labels.csv")): self.handler = DatasetImagesCSV( @@ -105,18 +106,18 @@ def set_dataset_handler(self) -> None: async def prepare_dataset(self) -> None: """Download dataset, unzip and set dataset handler""" - bt.logging.info(f"Preparing dataset '{self.competition_id}'") - bt.logging.info(f"Downloading dataset '{self.competition_id}'") + bt.logging.info(f"Preparing dataset '{self.config.competition_id}'") + bt.logging.info(f"Downloading dataset '{self.config.competition_id}'") await self.download_dataset() - bt.logging.info(f"Unzipping dataset '{self.competition_id}'") + bt.logging.info(f"Unzipping dataset '{self.config.competition_id}'") await self.unzip_dataset() - bt.logging.info(f"Setting dataset handler '{self.competition_id}'") + bt.logging.info(f"Setting dataset handler '{self.config.competition_id}'") self.set_dataset_handler() - bt.logging.info(f"Preprocessing dataset '{self.competition_id}'") + bt.logging.info(f"Preprocessing dataset '{self.config.competition_id}'") self.data = await self.handler.get_training_data() async def get_data(self) -> Tuple[List, List]: """Get data from dataset handler""" if not self.data: - raise DatasetManagerException(f"Dataset '{self.competition_id}' not initalized ") + raise DatasetManagerException(f"Dataset '{self.config.competition_id}' not initalized ") return self.data diff --git a/cancer_ai/validator/model_manager.py b/cancer_ai/validator/model_manager.py index 940ac469..62a74b09 100644 --- a/cancer_ai/validator/model_manager.py +++ b/cancer_ai/validator/model_manager.py @@ -51,6 +51,7 @@ async def download_miner_model(self, hotkey) -> None: model_info.hf_model_filename, cache_dir=self.config.model_dir, repo_type=model_info.hf_repo_type, + token=self.config.hf_token, ) def add_model( diff --git a/cancer_ai/validator/reward.py b/cancer_ai/validator/reward.py deleted file mode 100644 index 58492183..00000000 --- a/cancer_ai/validator/reward.py +++ /dev/null @@ -1,55 +0,0 @@ -# The MIT License (MIT) -# Copyright © 2023 Yuma Rao -# TODO(developer): Set your name -# Copyright © 2023 - -# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated -# documentation files (the “Software”), to deal in the Software without restriction, including without limitation -# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, -# and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all copies or substantial portions of -# the Software. - -# THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO -# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL -# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -# DEALINGS IN THE SOFTWARE. -import numpy as np -from typing import List -import bittensor as bt - - -def reward(query: int, response: int) -> float: - """ - Reward the miner response to the dummy request. This method returns a reward - value for the miner, which is used to update the miner's score. - - Returns: - - float: The reward value for the miner. - """ - bt.logging.info(f"In rewards, query val: {query}, response val: {response}, rewards val: {1.0 if response == query * 2 else 0}") - return 1.0 if response == query * 2 else 0 - - -def get_rewards( - self, - query: int, - responses: List[float], -) -> np.ndarray: - """ - Returns an array of rewards for the given query and responses. - - Args: - - query (int): The query sent to the miner. - - responses (List[float]): A list of responses from the miner. - - Returns: - - np.ndarray: An array of rewards for the given query and responses. - """ - # Get all the reward results by iteratively calling your reward() function. - - return np.array( - [reward(query, response) for response in responses] - ) diff --git a/neurons/competition_runner.py b/neurons/competition_runner.py index d2a87442..34c91d00 100644 --- a/neurons/competition_runner.py +++ b/neurons/competition_runner.py @@ -41,7 +41,6 @@ async def run_competitions_tick( # if now_time not in competition_times: # return None for time_competition in competition_times: - asyncio.sleep(60) # if now_time == time_competition: bt.logging.info( f"Running {competition_times[time_competition].competition_id} at {now_time}" diff --git a/neurons/miner.py b/neurons/miner.py index f2b342a2..3f842168 100644 --- a/neurons/miner.py +++ b/neurons/miner.py @@ -14,7 +14,7 @@ from cancer_ai.validator.dataset_manager import DatasetManager from cancer_ai.validator.competition_manager import COMPETITION_HANDLER_MAPPING -from cancer_ai.base.miner import BaseNeuron +from cancer_ai.base.base_miner import BaseNeuron from cancer_ai.chain_models_store import ChainMinerModel, ChainModelMetadataStore from cancer_ai.utils.config import path_config, add_miner_args diff --git a/neurons/rewarder.py b/neurons/rewarder.py index 9b2e1939..82af9c27 100644 --- a/neurons/rewarder.py +++ b/neurons/rewarder.py @@ -11,14 +11,16 @@ class RewarderConfig(BaseModel): competitionID_to_leader_hotkey_map: dict[str, CompetitionLeader] # competition_id -> CompetitionLeader hotkey_to_score_map: dict[str, Score] # hotkey -> Score -NON_REDUCTION_PERIOD = 30 +REWARD_REDUCTION_START_DAY = 30 +REWARD_REDUCTION_STEP = 0.1 +REWARD_REDUCTION_STEP_DAYS = 7 class Rewarder(): def __init__(self, rewarder_config: RewarderConfig): self.competition_leader_mapping = rewarder_config.competitionID_to_leader_hotkey_map self.scores = rewarder_config.hotkey_to_score_map - def get_miner_score_and_reduction(self, competition_id: str, hotkey: str) -> tuple[float, float]: + async def get_miner_score_and_reduction(self, competition_id: str, hotkey: str) -> tuple[float, float]: # check if current hotkey is already a leader competition = self.competition_leader_mapping.get(competition_id) if competition and competition.hotkey == hotkey: @@ -32,20 +34,20 @@ def get_miner_score_and_reduction(self, competition_id: str, hotkey: str) -> tup # Score degradation starts on 3rd week of leadership base_share = 1/len(self.competition_leader_mapping) - if days_as_leader > NON_REDUCTION_PERIOD: - periods = (days_as_leader - NON_REDUCTION_PERIOD) // 7 - reduction_factor = max(0.1, 1 - 0.1 * periods) + if days_as_leader > REWARD_REDUCTION_START_DAY: + periods = (days_as_leader - REWARD_REDUCTION_START_DAY) // REWARD_REDUCTION_STEP_DAYS + reduction_factor = max(REWARD_REDUCTION_STEP, 1 - REWARD_REDUCTION_STEP * periods) final_share = base_share * reduction_factor reduced_share = base_share - final_share return final_share, reduced_share return base_share, 0 - def update_scores(self, new_winner_hotkey: str, new_winner_comp_id: str): + async def update_scores(self, new_winner_hotkey: str, new_winner_comp_id: str): # reset the scores before updating them self.scores = {} # get score and reduced share for the new winner - self.get_miner_score_and_reduction(new_winner_comp_id, new_winner_hotkey) + await self.get_miner_score_and_reduction(new_winner_comp_id, new_winner_hotkey) num_competitions = len(self.competition_leader_mapping) # If there is only one competition, the winner takes it all @@ -53,12 +55,13 @@ def update_scores(self, new_winner_hotkey: str, new_winner_comp_id: str): competition_id = next(iter(self.competition_leader_mapping)) hotkey = self.competition_leader_mapping[competition_id].hotkey self.scores[hotkey] = Score(score=1.0, reduction=0.0) + print("BRUNO WYGRAL", self.scores, self.competition_leader_mapping) return # gather reduced shares for all competitors competitions_without_reduction = [] for curr_competition_id, comp_leader in self.competition_leader_mapping.items(): - score, reduced_share = self.get_miner_score_and_reduction(curr_competition_id, comp_leader.hotkey) + score, reduced_share = await self.get_miner_score_and_reduction(curr_competition_id, comp_leader.hotkey) if comp_leader.hotkey in self.scores: self.scores[comp_leader.hotkey].score += score diff --git a/neurons/validator.py b/neurons/validator.py index bfee1a6b..9d10a09b 100644 --- a/neurons/validator.py +++ b/neurons/validator.py @@ -25,7 +25,7 @@ import numpy as np -from cancer_ai.base.validator import BaseValidatorNeuron +from cancer_ai.base.base_validator import BaseValidatorNeuron from cancer_ai.validator.competition_manager import CompetitionManager from competition_runner import competition_loop, config_for_scheduler, run_competitions_tick from rewarder import RewarderConfig, Rewarder, Score @@ -49,7 +49,12 @@ def __init__(self, config=None): self.rewarder = Rewarder(self.rewarder_config) - asyncio.run_coroutine_threadsafe(competition_loop(self.scheduler_config, self.rewarder_config), self.loop) + self.loop.run_until_complete(self.competition_loop(self.scheduler_config, self.rewarder_config)) + + async def run_test_function(self): + print("Running test function") + await asyncio.sleep(5) + print("Test function done") async def competition_loop(self, scheduler_config: dict[str, CompetitionManager], rewarder_config: RewarderConfig): """Example of scheduling coroutine""" @@ -61,7 +66,9 @@ async def competition_loop(self, scheduler_config: dict[str, CompetitionManager] # update the scores await self.rewarder.update_scores(winning_evaluation_hotkey, competition_id) - self.rewarder_config = RewarderConfig(self.rewarder.competition_leader_mapping, self.rewarder.scores) + print("...,.,.,.,.,.,.,.,",self.rewarder.competition_leader_mapping, self.rewarder.scores) + self.rewarder_config = RewarderConfig(competitionID_to_leader_hotkey_map=self.rewarder.competition_leader_mapping, + hotkey_to_score_map=self.rewarder.scores) self.save_state() hotkey_to_score_map = self.rewarder_config.hotkey_to_score_map @@ -75,6 +82,39 @@ async def competition_loop(self, scheduler_config: dict[str, CompetitionManager] print(self.rewarder_config) await asyncio.sleep(60) + def save_state(self): + """Saves the state of the validator to a file.""" + bt.logging.info("Saving validator state.") + + # Save the state of the validator to file. + np.savez( + self.config.neuron.full_path + "/state.npz", + scores=self.scores, + hotkeys=self.hotkeys, + rewarder_config=self.rewarder_config.model_dump(), + ) + + def load_state(self): + """Loads the state of the validator from a file.""" + bt.logging.info("Loading validator state.") + + if not os.path.exists(self.config.neuron.full_path + "/state.npz"): + bt.logging.info("No state file found. Creating the file.") + np.savez( + self.config.neuron.full_path + "/state.npz", + scores=self.scores, + hotkeys=self.hotkeys, + rewarder_config=self.rewarder_config.model_dump(), + ) + return + + # Load the state of the validator from file. + state = np.load(self.config.neuron.full_path + "/state.npz", allow_pickle=True) + self.scores = state["scores"] + self.hotkeys = state["hotkeys"] + self.rewarder_config = RewarderConfig.model_validate(state["rewarder_config"].item()) + + # The main function parses the configuration and runs the validator. if __name__ == "__main__": with Validator() as validator: diff --git a/tests/test_template_validator.py b/tests/test_template_validator.py index 19b06b74..167be962 100644 --- a/tests/test_template_validator.py +++ b/tests/test_template_validator.py @@ -23,7 +23,7 @@ import torch from neurons.validator import Validator -from cancer_ai.base.validator import BaseValidatorNeuron +from cancer_ai.base.base_validator import BaseValidatorNeuron from cancer_ai.protocol import Dummy from cancer_ai.utils.uids import get_random_uids from cancer_ai.validator.reward import get_rewards From e9629423b8704f09fdd36e054d1352e9871fc4cf Mon Sep 17 00:00:00 2001 From: Wojtek Jurkowlaniec Date: Mon, 2 Sep 2024 03:24:07 +0200 Subject: [PATCH 133/227] config unification for miner and validator, renaming, attempt to do corutines --- cancer_ai/utils/config.py | 80 +++++++++++----------- cancer_ai/validator/competition_manager.py | 4 +- cancer_ai/validator/dataset_manager.py | 21 +++--- cancer_ai/validator/model_manager.py | 6 +- neurons/competition_runner.py | 8 ++- neurons/competition_runner_test.py | 4 +- neurons/miner.py | 8 +-- neurons/rewarder.py | 12 ++-- neurons/rewarder_test.py | 38 +++++----- neurons/validator.py | 74 +++++++++++--------- 10 files changed, 133 insertions(+), 122 deletions(-) diff --git a/cancer_ai/utils/config.py b/cancer_ai/utils/config.py index 52ebd47c..364cfb3f 100644 --- a/cancer_ai/utils/config.py +++ b/cancer_ai/utils/config.py @@ -131,18 +131,8 @@ def add_args(cls, parser): def add_miner_args(cls, parser): """Add miner specific arguments to the parser.""" - parser.add_argument( - "--competition_id", - type=str, - help="Competition ID", - ) - parser.add_argument( - "--model_dir", - type=str, - help="Path for for loading the starting model related to a training run.", - default="./models", - ) + parser.add_argument( "--hf_repo_id", @@ -176,36 +166,58 @@ def add_miner_args(cls, parser): "--model_path", type=str, help="Path to ONNX model, used for evaluation", + ) + + parser.add_argument( + "--clean_after_run", + action="store_true", + help="Whether to clean up (dataset, temporary files) after running", + default=False, ) parser.add_argument( - "--dataset_dir", + "--code_directory", type=str, - help="Path for storing datasets.", - default="./datasets", + help="Path to code directory", + default=".", ) + + +def add_common_args(cls, parser): + """Add validator and miner specific arguments to the parser.""" parser.add_argument( "--hf_token", type=str, help="Hugging Face API token", default="", ) + parser.add_argument( + "--competition.id", + type=str, + help="Path for storing competition participants models .", + ) parser.add_argument( - "--clean_after_run", - action="store_true", - help="Whether to clean up (dataset, temporary files) after running", - default=False, + "--models.model_dir", + type=str, + help="Path for storing competition participants models .", + default="/tmp/models", ) parser.add_argument( - "--code_directory", + "--models.dataset_dir", type=str, - help="Path to code directory", - default=".", + help="Path for storing datasets.", + default="/tmp/datasets", ) + parser.add_argument( + "--competition.config_path", + type=str, + help="Path with competition configuration .", + default="./neurons/competition_config.json", + ) @@ -285,24 +297,11 @@ def add_validator_args(cls, parser): help="The name of the project where you are sending the new run.", default="opentensor-dev", ) - parser.add_argument( - "--models.model_dir", - type=str, - help="Path for storing competition participants models .", - default="./models", - ) - parser.add_argument( - "--models.dataset_dir", - type=str, - help="Path for storing datasets.", - default="./datasets", - ) - parser.add_argument( - "--competition_config_path", - type=str, - help="Path with competition configuration .", - default="./neurons/competition_config.json", - ) + + + + + def path_config(cls): @@ -316,5 +315,6 @@ def path_config(cls): bt.subtensor.add_args(parser) bt.logging.add_args(parser) bt.axon.add_args(parser) + add_common_args(cls, parser) cls.add_args(parser) - return bt.config(parser.parse_args()) + return bt.config(parser) diff --git a/cancer_ai/validator/competition_manager.py b/cancer_ai/validator/competition_manager.py index 46cba5aa..cdf6c982 100644 --- a/cancer_ai/validator/competition_manager.py +++ b/cancer_ai/validator/competition_manager.py @@ -68,9 +68,9 @@ def __init__( self.competition_id = competition_id self.category = category self.results = [] - self.model_manager = ModelManager(config) + self.model_manager = ModelManager(self.config) self.dataset_manager = DatasetManager( - config, dataset_hf_repo, dataset_hf_id, dataset_hf_repo_type + self.config, dataset_hf_repo, dataset_hf_id, dataset_hf_repo_type ) self.chain_model_metadata_store = ChainModelMetadataStore(self.config.subtensor.network, self.config.netuid) diff --git a/cancer_ai/validator/dataset_manager.py b/cancer_ai/validator/dataset_manager.py index 309248f8..06bc04ba 100644 --- a/cancer_ai/validator/dataset_manager.py +++ b/cancer_ai/validator/dataset_manager.py @@ -32,13 +32,13 @@ def __init__( None """ self.config = config - self.competition_id = config.competition_id + self.hf_repo_id = hf_repo_id self.hf_filename = hf_filename self.hf_repo_type = hf_repo_type self.local_compressed_path = "" self.local_extracted_dir = Path( - self.config.dataset_dir, self.competition_id + self.config.models.dataset_dir, self.config.competition.id ) self.data: Tuple[List, List] = () self.handler = None @@ -57,7 +57,7 @@ async def download_dataset(self): self.local_compressed_path = HfApi().hf_hub_download( self.hf_repo_id, self.hf_filename, - cache_dir=Path(self.config.dataset_dir), + cache_dir=Path(self.config.models.dataset_dir), repo_type=self.hf_repo_type, token=self.config.hf_token, ) @@ -78,7 +78,7 @@ async def unzip_dataset(self) -> None: """Unzip dataset""" self.local_extracted_dir = Path( - self.config.dataset_dir, self.config.competition_id + self.config.models.dataset_dir, self.config.competition.id ) bt.logging.debug(f"Dataset extracted to: { self.local_compressed_path}") @@ -93,7 +93,7 @@ async def unzip_dataset(self) -> None: def set_dataset_handler(self) -> None: """Detect dataset type and set handler""" if not self.local_compressed_path: - raise DatasetManagerException(f"Dataset '{self.config.competition_id}' not downloaded") + raise DatasetManagerException(f"Dataset '{self.config.competition.id}' not downloaded") # is csv in directory if os.path.exists(Path(self.local_extracted_dir, "labels.csv")): self.handler = DatasetImagesCSV( @@ -106,18 +106,17 @@ def set_dataset_handler(self) -> None: async def prepare_dataset(self) -> None: """Download dataset, unzip and set dataset handler""" - bt.logging.info(f"Preparing dataset '{self.config.competition_id}'") - bt.logging.info(f"Downloading dataset '{self.config.competition_id}'") + bt.logging.info(f"Downloading dataset '{self.config.competition.id}'") await self.download_dataset() - bt.logging.info(f"Unzipping dataset '{self.config.competition_id}'") + bt.logging.info(f"Unzipping dataset '{self.config.competition.id}'") await self.unzip_dataset() - bt.logging.info(f"Setting dataset handler '{self.config.competition_id}'") + bt.logging.info(f"Setting dataset handler '{self.config.competition.id}'") self.set_dataset_handler() - bt.logging.info(f"Preprocessing dataset '{self.config.competition_id}'") + bt.logging.info(f"Preprocessing dataset '{self.config.competition.id}'") self.data = await self.handler.get_training_data() async def get_data(self) -> Tuple[List, List]: """Get data from dataset handler""" if not self.data: - raise DatasetManagerException(f"Dataset '{self.config.competition_id}' not initalized ") + raise DatasetManagerException(f"Dataset '{self.config.competition.id}' not initalized ") return self.data diff --git a/cancer_ai/validator/model_manager.py b/cancer_ai/validator/model_manager.py index 62a74b09..c63bc293 100644 --- a/cancer_ai/validator/model_manager.py +++ b/cancer_ai/validator/model_manager.py @@ -23,8 +23,8 @@ class ModelManager(SerializableManager): def __init__(self, config) -> None: self.config = config - if not os.path.exists(self.config.model_dir): - os.makedirs(self.config.model_dir) + if not os.path.exists(self.config.models.model_dir): + os.makedirs(self.config.models.model_dir) self.api = HfApi() self.hotkey_store = {} @@ -49,7 +49,7 @@ async def download_miner_model(self, hotkey) -> None: model_info.file_path = self.api.hf_hub_download( model_info.hf_repo_id, model_info.hf_model_filename, - cache_dir=self.config.model_dir, + cache_dir=self.config.models.model_dir, repo_type=model_info.hf_repo_type, token=self.config.hf_token, ) diff --git a/neurons/competition_runner.py b/neurons/competition_runner.py index 34c91d00..cb35550b 100644 --- a/neurons/competition_runner.py +++ b/neurons/competition_runner.py @@ -6,10 +6,12 @@ from datetime import datetime, timezone import bittensor as bt from typing import List, Tuple, Dict -from rewarder import Rewarder, RewarderConfig, CompetitionLeader +from rewarder import Rewarder, WinnersMapping, CompetitionLeader # from cancer_ai.utils.config import config +# TODO MOVE SOMEWHERE +main_competitions_cfg = json.load(open("neurons/competition_config.json", "r")) def config_for_scheduler( bt_config, hotkeys: List[str], test_mode: bool = False @@ -54,7 +56,7 @@ async def run_competitions_tick( ) -async def competition_loop(scheduler_config: Dict[str, CompetitionManager], rewarder_config: RewarderConfig): +async def competition_loop(scheduler_config: Dict[str, CompetitionManager], rewarder_config: WinnersMapping): """Example of scheduling coroutine""" while True: competition_result = await run_competitions_tick(scheduler_config) @@ -79,5 +81,5 @@ async def competition_loop(scheduler_config: Dict[str, CompetitionManager], rewa hotkeys = [] bt_config = {} # get from bt config scheduler_config = config_for_scheduler(bt_config, hotkeys) - rewarder_config = RewarderConfig({},{}) + rewarder_config = WinnersMapping({},{}) asyncio.run(competition_loop(scheduler_config, rewarder_config)) diff --git a/neurons/competition_runner_test.py b/neurons/competition_runner_test.py index 46beb8fb..f9d054ab 100644 --- a/neurons/competition_runner_test.py +++ b/neurons/competition_runner_test.py @@ -5,7 +5,7 @@ import bittensor as bt from typing import List, Dict from competition_runner import run_competitions_tick, competition_loop -from rewarder import RewarderConfig, Rewarder +from rewarder import WinnersMapping, Rewarder import time # TODO integrate with bt config @@ -70,7 +70,7 @@ async def competition_loop(): ("hotkey2", "melanoma-3"), ] - rewarder_config = RewarderConfig(competitionID_to_leader_hotkey_map={}, hotkey_to_score_map={}) + rewarder_config = WinnersMapping(competition_leader_map={}, hotkey_score_map={}) rewarder = Rewarder(rewarder_config) for winning_evaluation_hotkey, competition_id in test_cases: diff --git a/neurons/miner.py b/neurons/miner.py index 3f842168..03a16ecd 100644 --- a/neurons/miner.py +++ b/neurons/miner.py @@ -42,8 +42,8 @@ async def upload_to_hf(self) -> None: hf_api = HfApi() hf_login(token=self.config.hf_token) - hf_model_path = f"{self.config.competition_id}-{self.config.hf_model_name}" - hf_code_path = f"{self.config.competition_id}-{self.config.hf_model_name}" + hf_model_path = f"{self.config.competition.id}-{self.config.hf_model_name}" + hf_code_path = f"{self.config.competition.id}-{self.config.hf_model_name}" path = hf_api.upload_file( path_or_fileobj=self.config.model_path, @@ -89,7 +89,7 @@ async def evaluate_model(self) -> None: X_test, y_test = await dataset_manager.get_data() - competition_handler = COMPETITION_HANDLER_MAPPING[self.config.competition_id]( + competition_handler = COMPETITION_HANDLER_MAPPING[self.config.competition.id]( X_test=X_test, y_test=y_test ) @@ -160,7 +160,7 @@ async def submit_model(self) -> None: hf_repo_id=self.config.hf_repo_id, hf_model_filename=self.config.hf_model_name, hf_code_filename=self.config.hf_code_filename, - competition_id=self.config.competition_id, + competition_id=self.config.competition.id, hf_repo_type=self.config.hf_repo_type, ) await self.metadata_store.store_model_metadata(model_id) diff --git a/neurons/rewarder.py b/neurons/rewarder.py index 82af9c27..9a78dae9 100644 --- a/neurons/rewarder.py +++ b/neurons/rewarder.py @@ -7,18 +7,18 @@ class CompetitionLeader(BaseModel): class Score(BaseModel): score: float reduction: float -class RewarderConfig(BaseModel): - competitionID_to_leader_hotkey_map: dict[str, CompetitionLeader] # competition_id -> CompetitionLeader - hotkey_to_score_map: dict[str, Score] # hotkey -> Score +class WinnersMapping(BaseModel): + competition_leader_map: dict[str, CompetitionLeader] # competition_id -> CompetitionLeader + hotkey_score_map: dict[str, Score] # hotkey -> Score REWARD_REDUCTION_START_DAY = 30 REWARD_REDUCTION_STEP = 0.1 REWARD_REDUCTION_STEP_DAYS = 7 class Rewarder(): - def __init__(self, rewarder_config: RewarderConfig): - self.competition_leader_mapping = rewarder_config.competitionID_to_leader_hotkey_map - self.scores = rewarder_config.hotkey_to_score_map + def __init__(self, rewarder_config: WinnersMapping): + self.competition_leader_mapping = rewarder_config.competition_leader_map + self.scores = rewarder_config.hotkey_score_map async def get_miner_score_and_reduction(self, competition_id: str, hotkey: str) -> tuple[float, float]: # check if current hotkey is already a leader diff --git a/neurons/rewarder_test.py b/neurons/rewarder_test.py index 6414a58f..b00bc004 100644 --- a/neurons/rewarder_test.py +++ b/neurons/rewarder_test.py @@ -1,6 +1,6 @@ import pytest from datetime import datetime, timedelta -from .rewarder import CompetitionLeader, Score, RewarderConfig, Rewarder +from .rewarder import CompetitionLeader, Score, WinnersMapping, Rewarder def test_update_scores_single_competitor(): # Set up initial data for a single competitor @@ -13,9 +13,9 @@ def test_update_scores_single_competitor(): } # Set up the configuration with a single competition and a single competitor - rewarder_config = RewarderConfig( - competitionID_to_leader_hotkey_map=competition_leaders, - hotkey_to_score_map=scores + rewarder_config = WinnersMapping( + competition_leader_map=competition_leaders, + hotkey_score_map=scores ) rewarder = Rewarder(rewarder_config) @@ -47,9 +47,9 @@ def test_update_scores_multiple_competitors_no_reduction(): } # Set up the configuration with multiple competitions and multiple competitors - rewarder_config = RewarderConfig( - competitionID_to_leader_hotkey_map=competition_leaders, - hotkey_to_score_map=scores + rewarder_config = WinnersMapping( + competition_leader_map=competition_leaders, + hotkey_score_map=scores ) rewarder = Rewarder(rewarder_config) @@ -86,9 +86,9 @@ def test_update_scores_multiple_competitors_with_some_reduced_shares(): } # Set up the configuration with multiple competitions and multiple competitors - rewarder_config = RewarderConfig( - competitionID_to_leader_hotkey_map=competition_leaders, - hotkey_to_score_map=scores + rewarder_config = WinnersMapping( + competition_leader_map=competition_leaders, + hotkey_score_map=scores ) rewarder = Rewarder(rewarder_config) @@ -135,9 +135,9 @@ def test_update_scores_all_competitors_with_reduced_shares(): } # Set up the configuration with multiple competitions and multiple competitors - rewarder_config = RewarderConfig( - competitionID_to_leader_hotkey_map=competition_leaders, - hotkey_to_score_map=scores + rewarder_config = WinnersMapping( + competition_leader_map=competition_leaders, + hotkey_score_map=scores ) rewarder = Rewarder(rewarder_config) @@ -183,9 +183,9 @@ def test_update_scores_more_competitions_then_competitors(): } # Set up the configuration with multiple competitions and multiple competitors - rewarder_config = RewarderConfig( - competitionID_to_leader_hotkey_map=competition_leaders, - hotkey_to_score_map=scores + rewarder_config = WinnersMapping( + competition_leader_map=competition_leaders, + hotkey_score_map=scores ) rewarder = Rewarder(rewarder_config) @@ -234,9 +234,9 @@ def test_update_scores_6_competitions_3_competitors(): } # Set up the configuration with multiple competitions and multiple competitors - rewarder_config = RewarderConfig( - competitionID_to_leader_hotkey_map=competition_leaders, - hotkey_to_score_map=scores + rewarder_config = WinnersMapping( + competition_leader_map=competition_leaders, + hotkey_score_map=scores ) rewarder = Rewarder(rewarder_config) diff --git a/neurons/validator.py b/neurons/validator.py index 9d10a09b..d53a62cb 100644 --- a/neurons/validator.py +++ b/neurons/validator.py @@ -22,13 +22,13 @@ from typing import Any, List import bittensor as bt import asyncio - +import os import numpy as np from cancer_ai.base.base_validator import BaseValidatorNeuron from cancer_ai.validator.competition_manager import CompetitionManager from competition_runner import competition_loop, config_for_scheduler, run_competitions_tick -from rewarder import RewarderConfig, Rewarder, Score +from rewarder import WinnersMapping, Rewarder, Score class Validator(BaseValidatorNeuron): @@ -43,55 +43,65 @@ class Validator(BaseValidatorNeuron): def __init__(self, config=None): super(Validator, self).__init__(config=config) - self.rewarder_config = RewarderConfig({},{}) + self.winners_mapping = WinnersMapping(competition_leader_map={}, hotkey_score_map={}) self.load_state() self.scheduler_config = config_for_scheduler(self.config, self.hotkeys, test_mode=True) - self.rewarder = Rewarder(self.rewarder_config) + self.rewarder = Rewarder(self.winners_mapping) - self.loop.run_until_complete(self.competition_loop(self.scheduler_config, self.rewarder_config)) + self.loop.run_until_complete(self.competition_loop_tick(self.scheduler_config, self.winners_mapping)) + + async def concurrent_forward(self): + coroutines = [ + self.run_test_function(), + ] + await asyncio.gather(*coroutines) + async def run_test_function(self): print("Running test function") await asyncio.sleep(5) print("Test function done") - async def competition_loop(self, scheduler_config: dict[str, CompetitionManager], rewarder_config: RewarderConfig): + async def competition_loop_tick(self, scheduler_config: dict[str, CompetitionManager], rewarder_config: WinnersMapping): """Example of scheduling coroutine""" - while True: - competition_result = await run_competitions_tick(scheduler_config) - bt.logging.debug(f"Competition result: {competition_result}") - if competition_result: - winning_evaluation_hotkey, competition_id = competition_result - - # update the scores - await self.rewarder.update_scores(winning_evaluation_hotkey, competition_id) - print("...,.,.,.,.,.,.,.,",self.rewarder.competition_leader_mapping, self.rewarder.scores) - self.rewarder_config = RewarderConfig(competitionID_to_leader_hotkey_map=self.rewarder.competition_leader_mapping, - hotkey_to_score_map=self.rewarder.scores) - self.save_state() - - hotkey_to_score_map = self.rewarder_config.hotkey_to_score_map - - self.scores = [ - np.float32(hotkey_to_score_map.get(hotkey, Score(score=0.0, reduction=0.0)).score) - for hotkey in self.metagraph.hotkeys - ] - self.save_state() - print(".....................Updated rewarder config:") - print(self.rewarder_config) - await asyncio.sleep(60) + competition_result = await run_competitions_tick(scheduler_config) + bt.logging.debug(f"Competition result: {competition_result}") + if not competition_result: + return + + winning_evaluation_hotkey, competition_id = competition_result + + # update the scores + await self.rewarder.update_scores(winning_evaluation_hotkey, competition_id) + print("...,.,.,.,.,.,.,.,",self.rewarder.competition_leader_mapping, self.rewarder.scores) + self.winners_mapping = WinnersMapping(competition_leader_map=self.rewarder.competition_leader_mapping, + hotkey_score_map=self.rewarder.scores) + self.save_state() + + hotkey_to_score_map = self.winners_mapping.hotkey_score_map + + self.scores = [ + np.float32(hotkey_to_score_map.get(hotkey, Score(score=0.0, reduction=0.0)).score) + for hotkey in self.metagraph.hotkeys + ] + self.save_state() + print(".....................Updated rewarder config:") + print(self.winners_mapping) + # await asyncio.sleep(60) def save_state(self): """Saves the state of the validator to a file.""" bt.logging.info("Saving validator state.") # Save the state of the validator to file. + if not getattr(self, "winners_mapping", None): + self.winners_mapping = WinnersMapping(competition_leader_map={}, hotkey_score_map={}) np.savez( self.config.neuron.full_path + "/state.npz", scores=self.scores, hotkeys=self.hotkeys, - rewarder_config=self.rewarder_config.model_dump(), + rewarder_config=self.winners_mapping.model_dump(), ) def load_state(self): @@ -104,7 +114,7 @@ def load_state(self): self.config.neuron.full_path + "/state.npz", scores=self.scores, hotkeys=self.hotkeys, - rewarder_config=self.rewarder_config.model_dump(), + rewarder_config=self.winners_mapping.model_dump(), ) return @@ -112,7 +122,7 @@ def load_state(self): state = np.load(self.config.neuron.full_path + "/state.npz", allow_pickle=True) self.scores = state["scores"] self.hotkeys = state["hotkeys"] - self.rewarder_config = RewarderConfig.model_validate(state["rewarder_config"].item()) + self.winners_mapping = WinnersMapping.model_validate(state["rewarder_config"].item()) # The main function parses the configuration and runs the validator. From 3f442a18d516106f253a59d9ce257d8405594d6c Mon Sep 17 00:00:00 2001 From: Wojtek Jurkowlaniec Date: Mon, 2 Sep 2024 04:37:25 +0200 Subject: [PATCH 134/227] fix configs and make main loop run --- cancer_ai/base/base_validator.py | 26 +++++++++++++-- .../competition_handlers/base_handler.py | 16 ++++++---- cancer_ai/validator/competition_manager.py | 32 ++++++++++++++----- cancer_ai/validator/dataset_manager.py | 18 +++++------ cancer_ai/validator/model_manager.py | 1 + neurons/miner.py | 3 +- neurons/validator.py | 6 ++-- 7 files changed, 72 insertions(+), 30 deletions(-) diff --git a/cancer_ai/base/base_validator.py b/cancer_ai/base/base_validator.py index f1881880..b1c0ad69 100644 --- a/cancer_ai/base/base_validator.py +++ b/cancer_ai/base/base_validator.py @@ -32,7 +32,7 @@ from .utils.weight_utils import ( process_weights_for_netuid, convert_weights_and_uids_for_emit, -) # TODO: Replace when bittensor switches to numpy +) from ..mock import MockDendrite from ..utils.config import add_validator_args @@ -108,6 +108,25 @@ def serve_axon(self): pass def run(self): + """ + Initiates and manages the main loop for the miner on the Bittensor network. The main loop handles graceful shutdown on keyboard interrupts and logs unforeseen errors. + + This function performs the following primary tasks: + 1. Check for registration on the Bittensor network. + 2. Continuously forwards queries to the miners on the network, rewarding their responses and updating the scores accordingly. + 3. Periodically resynchronizes with the chain; updating the metagraph with the latest network state and setting weights. + + The essence of the validator's operations is in the forward function, which is called every step. The forward function is responsible for querying the network and scoring the responses. + + Note: + - The function leverages the global configurations set during the initialization of the miner. + - The miner's axon serves as its interface to the Bittensor network, handling incoming and outgoing requests. + + Raises: + KeyboardInterrupt: If the miner is stopped by a manual interruption. + Exception: For unforeseen errors during the miner's operation, which are logged for diagnosis. + """ + # Check that validator is registered on the network. self.sync() @@ -116,7 +135,8 @@ def run(self): # This loop maintains the validator's operations until intentionally stopped. try: while True: - bt.logging.info(f"step({self.step}) block({self.block})") + # Run multiple forwards concurrently. + self.loop.run_until_complete(self.concurrent_forward()) # Check if we should exit. if self.should_exit: @@ -291,7 +311,7 @@ def save_state(self): self.config.neuron.full_path + "/state.npz", scores=self.scores, hotkeys=self.hotkeys, - rewarder_config=self.rewarder_config, + rewarder_config = self.rewarder_config, ) def load_state(self): diff --git a/cancer_ai/validator/competition_handlers/base_handler.py b/cancer_ai/validator/competition_handlers/base_handler.py index aafdffa2..f2e87009 100644 --- a/cancer_ai/validator/competition_handlers/base_handler.py +++ b/cancer_ai/validator/competition_handlers/base_handler.py @@ -1,19 +1,23 @@ +from typing import Any from abc import abstractmethod -from dataclasses import dataclass +from numpy import ndarray +from pydantic import BaseModel -@dataclass -class ModelEvaluationResult: +class ModelEvaluationResult(BaseModel): accuracy: float precision: float recall: float - confusion_matrix: any - fpr: any - tpr: any + confusion_matrix: ndarray + fpr: ndarray + tpr: ndarray roc_auc: float run_time_s: float tested_entries: int + class Config: + arbitrary_types_allowed = True + class BaseCompetitionHandler: """ Base class for handling different competition types. diff --git a/cancer_ai/validator/competition_manager.py b/cancer_ai/validator/competition_manager.py index cdf6c982..f3be9636 100644 --- a/cancer_ai/validator/competition_manager.py +++ b/cancer_ai/validator/competition_manager.py @@ -70,9 +70,15 @@ def __init__( self.results = [] self.model_manager = ModelManager(self.config) self.dataset_manager = DatasetManager( - self.config, dataset_hf_repo, dataset_hf_id, dataset_hf_repo_type + self.config, + competition_id, + dataset_hf_repo, + dataset_hf_id, + dataset_hf_repo_type, + ) + self.chain_model_metadata_store = ChainModelMetadataStore( + self.config.subtensor.network, self.config.netuid ) - self.chain_model_metadata_store = ChainModelMetadataStore(self.config.subtensor.network, self.config.netuid) self.hotkeys = hotkeys self.chain_miner_models = {} @@ -108,8 +114,6 @@ def __init__( # bt.logging.info("Recall: ", evaluation_result.recall) # bt.logging.info("roc_auc: ", evaluation_result.roc_auc) - - def get_state(self): return { "competition_id": self.competition_id, @@ -123,11 +127,16 @@ def set_state(self, state: dict): self.category = state["category"] async def get_miner_model(self, chain_miner_model: ChainMinerModel): + if chain_miner_model.competition_id != self.competition_id: + raise ValueError( + f"Chain miner model {chain_miner_model.to_compressed_str()} does not belong to this competition" + ) model_info = ModelInfo( hf_repo_id=chain_miner_model.hf_repo_id, hf_model_filename=chain_miner_model.hf_filename, hf_code_filename=chain_miner_model.hf_code_filename, hf_repo_type=chain_miner_model.hf_repo_type, + competition_id=chain_miner_model.competition_id, ) return model_info @@ -152,16 +161,23 @@ async def sync_chain_miners(self): Updates hotkeys and downloads information of models from the chain """ bt.logging.info("Synchronizing miners from the chain") - bt.logging.info(f"Amount of hotkeys: {len(self.hotkeys)}") for hotkey in self.hotkeys: hotkey_metadata = ( await self.chain_model_metadata_store.retrieve_model_metadata(hotkey) ) - if hotkey_metadata: + if not hotkey_metadata: + bt.logging.warning( + f"Cannot get miner model for hotkey {hotkey} from the chain, skipping" + ) + continue + try: + miner_model = await self.get_miner_model(hotkey) self.chain_miner_models[hotkey] = hotkey_metadata - self.model_manager.hotkey_store[hotkey] = await self.get_miner_model( - hotkey + self.model_manager.hotkey_store[hotkey] = miner_model + except ValueError: + bt.logging.error( + f"Miner {hotkey} with data {hotkey_metadata.to_compressed_str()} does not belong to this competition, skipping" ) bt.logging.info( f"Amount of chain miners with models: {len(self.chain_miner_models)}" diff --git a/cancer_ai/validator/dataset_manager.py b/cancer_ai/validator/dataset_manager.py index 06bc04ba..461cda85 100644 --- a/cancer_ai/validator/dataset_manager.py +++ b/cancer_ai/validator/dataset_manager.py @@ -17,7 +17,7 @@ class DatasetManagerException(Exception): class DatasetManager(SerializableManager): def __init__( - self, config, hf_repo_id: str, hf_filename: str, hf_repo_type: str + self, config, competition_id: str, hf_repo_id: str, hf_filename: str, hf_repo_type: str ) -> None: """ Initializes a new instance of the DatasetManager class. @@ -36,9 +36,10 @@ def __init__( self.hf_repo_id = hf_repo_id self.hf_filename = hf_filename self.hf_repo_type = hf_repo_type + self.competition_id = competition_id self.local_compressed_path = "" self.local_extracted_dir = Path( - self.config.models.dataset_dir, self.config.competition.id + self.config.models.dataset_dir, competition_id ) self.data: Tuple[List, List] = () self.handler = None @@ -78,7 +79,7 @@ async def unzip_dataset(self) -> None: """Unzip dataset""" self.local_extracted_dir = Path( - self.config.models.dataset_dir, self.config.competition.id + self.config.models.dataset_dir, self.competition_id ) bt.logging.debug(f"Dataset extracted to: { self.local_compressed_path}") @@ -87,7 +88,6 @@ async def unzip_dataset(self) -> None: out, err = await run_command( f"unzip {self.local_compressed_path} -d {self.local_extracted_dir}" ) - bt.logging.error(err) bt.logging.info("Dataset unzipped") def set_dataset_handler(self) -> None: @@ -106,17 +106,17 @@ def set_dataset_handler(self) -> None: async def prepare_dataset(self) -> None: """Download dataset, unzip and set dataset handler""" - bt.logging.info(f"Downloading dataset '{self.config.competition.id}'") + bt.logging.info(f"Downloading dataset '{self.competition_id}'") await self.download_dataset() - bt.logging.info(f"Unzipping dataset '{self.config.competition.id}'") + bt.logging.info(f"Unzipping dataset '{self.competition_id}'") await self.unzip_dataset() - bt.logging.info(f"Setting dataset handler '{self.config.competition.id}'") + bt.logging.info(f"Setting dataset handler '{self.competition_id}'") self.set_dataset_handler() - bt.logging.info(f"Preprocessing dataset '{self.config.competition.id}'") + bt.logging.info(f"Preprocessing dataset '{self.competition_id}'") self.data = await self.handler.get_training_data() async def get_data(self) -> Tuple[List, List]: """Get data from dataset handler""" if not self.data: - raise DatasetManagerException(f"Dataset '{self.config.competition.id}' not initalized ") + raise DatasetManagerException(f"Dataset '{self.competition_id}' not initalized ") return self.data diff --git a/cancer_ai/validator/model_manager.py b/cancer_ai/validator/model_manager.py index c63bc293..01ae23af 100644 --- a/cancer_ai/validator/model_manager.py +++ b/cancer_ai/validator/model_manager.py @@ -15,6 +15,7 @@ class ModelInfo: hf_code_filename: str | None = None hf_repo_type: str | None = None + competition_id: str | None = None file_path: str | None = None model_type: str | None = None diff --git a/neurons/miner.py b/neurons/miner.py index 03a16ecd..f215c407 100644 --- a/neurons/miner.py +++ b/neurons/miner.py @@ -81,6 +81,7 @@ async def evaluate_model(self) -> None: ) dataset_manager = DatasetManager( self.config, + self.config.competition.id, "safescanai/test_dataset", "skin_melanoma.zip", "dataset", @@ -99,7 +100,7 @@ async def evaluate_model(self) -> None: y_pred = await run_manager.run(X_test) run_time_s = time.time() - start_time model_result = competition_handler.get_model_result(y_test, y_pred, run_time_s) - bt.logging.info(model_result) + bt.logging.info(f"\n {model_result}\n") if self.config.clean_after_run: dataset_manager.delete_dataset() diff --git a/neurons/validator.py b/neurons/validator.py index d53a62cb..723021ab 100644 --- a/neurons/validator.py +++ b/neurons/validator.py @@ -48,12 +48,12 @@ def __init__(self, config=None): self.scheduler_config = config_for_scheduler(self.config, self.hotkeys, test_mode=True) self.rewarder = Rewarder(self.winners_mapping) - - self.loop.run_until_complete(self.competition_loop_tick(self.scheduler_config, self.winners_mapping)) + async def concurrent_forward(self): coroutines = [ self.run_test_function(), + self.competition_loop_tick(self.scheduler_config, self.winners_mapping) ] await asyncio.gather(*coroutines) @@ -129,5 +129,5 @@ def load_state(self): if __name__ == "__main__": with Validator() as validator: while True: - bt.logging.info(f"Validator running... {time.time()}") + # bt.logging.info(f"Validator running... {time.time()}") time.sleep(5) From 89b812b955e2abd131754fc3a5f48bcc70f81eab Mon Sep 17 00:00:00 2001 From: Wojtek Jurkowlaniec Date: Mon, 2 Sep 2024 16:28:09 +0200 Subject: [PATCH 135/227] Update requirements.txt --- requirements.txt | 147 ++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 139 insertions(+), 8 deletions(-) diff --git a/requirements.txt b/requirements.txt index f44dfb74..25c35c3d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,139 @@ -bittensor>=7 -starlette>=0.30.0 -pydantic>=2 -rich>=13 -pytest>=8 -torch>=2 -numpy>=1 -setuptools>=68 \ No newline at end of file +absl-py==2.1.0 +aiofiles==24.1.0 +aiohappyeyeballs==2.3.5 +aiohttp==3.10.2 +aiosignal==1.3.1 +annotated-types==0.7.0 +ansible==6.7.0 +ansible-core==2.13.13 +ansible-vault==2.1.0 +anyio==4.4.0 +astunparse==1.6.3 +async-unzip==0.3.6 +attrs==24.2.0 +backoff==2.2.1 +base58==2.1.1 +bittensor==7.3.1 +black==24.8.0 +certifi==2024.2.2 +cffi==1.17.0 +charset-normalizer==3.3.2 +click==8.1.7 +colorama==0.4.6 +coloredlogs==15.0.1 +crontab==1.0.1 +cryptography==42.0.8 +cytoolz==0.12.3 +ddt==1.6.0 +decorator==5.1.1 +docker-pycreds==0.4.0 +ecdsa==0.19.0 +eth-hash==0.7.0 +eth-keys==0.5.1 +eth-typing==4.4.0 +eth-utils==2.2.2 +fastapi==0.110.3 +filelock==3.15.4 +flatbuffers==24.3.25 +frozenlist==1.4.1 +fsspec==2024.6.1 +fuzzywuzzy==0.18.0 +gast==0.6.0 +gitdb==4.0.11 +GitPython==3.1.43 +google-pasta==0.2.0 +grpcio==1.65.5 +h11==0.14.0 +h5py==3.11.0 +huggingface-hub==0.24.5 +humanfriendly==10.0 +idna==3.7 +iniconfig==2.0.0 +Jinja2==3.1.4 +joblib==1.4.2 +keras==3.5.0 +Levenshtein==0.25.1 +libclang==18.1.1 +Markdown==3.7 +markdown-it-py==3.0.0 +MarkupSafe==2.1.5 +mdurl==0.1.2 +ml-dtypes==0.4.0 +more-itertools==10.4.0 +mpmath==1.3.0 +msgpack==1.0.8 +msgpack-numpy-opentensor==0.5.0 +multidict==6.0.5 +munch==2.5.0 +mypy==1.11.1 +mypy-extensions==1.0.0 +namex==0.0.8 +nest-asyncio==1.6.0 +netaddr==1.3.0 +networkx==3.3 +numpy==1.26.4 +onnx==1.16.2 +onnxruntime==1.19.0 +opt-einsum==3.3.0 +optree==0.12.1 +packaging==24.1 +password-strength==0.0.3.post2 +pathspec==0.12.1 +pillow==10.4.0 +platformdirs==4.2.2 +pluggy==1.5.0 +protobuf==4.25.4 +psutil==6.0.0 +py==1.11.0 +py-bip39-bindings==0.1.11 +py-ed25519-zebra-bindings==1.0.1 +py-sr25519-bindings==0.2.0 +pycparser==2.22 +pycryptodome==3.20.0 +pydantic==2.8.2 +pydantic_core==2.20.1 +Pygments==2.18.0 +PyNaCl==1.5.0 +pytest==8.3.2 +python-dotenv==1.0.1 +python-Levenshtein==0.25.1 +python-statemachine==2.1.2 +PyYAML==6.0.2 +rapidfuzz==3.9.6 +redis==5.0.8 +requests==2.32.3 +resolvelib==0.8.1 +retry==0.9.2 +rich==13.7.1 +scalecodec==1.2.11 +schedule==1.2.2 +scikit-learn==1.5.1 +scipy==1.14.1 +sentry-sdk==2.13.0 +setproctitle==1.3.3 +setuptools==72.1.0 +shtab==1.6.5 +six==1.16.0 +smmap==5.0.1 +sniffio==1.3.1 +starlette==0.37.2 +substrate-interface==1.7.10 +sympy==1.13.1 +tensorboard==2.17.1 +tensorboard-data-server==0.7.2 +tensorflow==2.17.0 +termcolor==2.4.0 +threadpoolctl==3.5.0 +toolz==0.12.1 +torch==2.4.0 +tqdm==4.66.5 +typing_extensions==4.12.2 +urllib3==2.2.2 +uvicorn==0.30.0 +wandb==0.17.7 +websocket-client==1.8.0 +Werkzeug==3.0.3 +wheel==0.44.0 +wrapt==1.16.0 +xxhash==3.4.1 +yarl==1.9.4 From 2f13c99d3ae8c92595502b7dd39f3f60278ea505 Mon Sep 17 00:00:00 2001 From: Konrad Date: Mon, 2 Sep 2024 17:02:33 +0200 Subject: [PATCH 136/227] autovali test --- cancer_ai/chain_models_store.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cancer_ai/chain_models_store.py b/cancer_ai/chain_models_store.py index ab9df490..dc96635f 100644 --- a/cancer_ai/chain_models_store.py +++ b/cancer_ai/chain_models_store.py @@ -83,7 +83,7 @@ async def retrieve_model_metadata(self, hotkey: str) -> Optional[ChainMinerModel metadata = run_in_subprocess(partial, 60) if not metadata: return None - bt.logging.info(f"Model metadata: {metadata["info"]["fields"]}") + bt.logging.info(f"Model metadata: {metadata['info']['fields']}") commitment = metadata["info"]["fields"][0] hex_data = commitment[list(commitment.keys())[0]][2:] From d5618b129f4c37612f4bec80d92e0cfc3619292e Mon Sep 17 00:00:00 2001 From: Konrad Date: Mon, 2 Sep 2024 17:16:29 +0200 Subject: [PATCH 137/227] autovali --- scripts/start_validator.py | 193 +++++++++++++++++++++++++++++++++++++ 1 file changed, 193 insertions(+) create mode 100755 scripts/start_validator.py diff --git a/scripts/start_validator.py b/scripts/start_validator.py new file mode 100755 index 00000000..64c99407 --- /dev/null +++ b/scripts/start_validator.py @@ -0,0 +1,193 @@ +""" +This script runs a validator process and automatically updates it when a new version is released. +Command-line arguments will be forwarded to validator (`neurons/validator.py`), so you can pass +them like this: + python3 scripts/start_validator.py --wallet.name=my-wallet +Auto-updates are enabled by default and will make sure that the latest version is always running +by pulling the latest version from git and upgrading python packages. This is done periodically. +Local changes may prevent the update, but they will be preserved. + +The script will use the same virtual environment as the one used to run it. If you want to run +validator within virtual environment, run this auto-update script from the virtual environment. + +Pm2 is required for this script. This script will start a pm2 process using the name provided by +the --pm2_name argument. +""" +import argparse +import logging +import subprocess +import sys +import time +from datetime import timedelta +from shlex import split +from typing import List +from pathlib import Path + +log = logging.getLogger(__name__) +UPDATES_CHECK_TIME = timedelta(seconds=30) +CURRENT_WORKING_DIR = Path(__file__).parent.parent + + +def get_version() -> str: + """Extract the version as current git commit hash""" + result = subprocess.run( + split("git rev-parse HEAD"), + check=True, + capture_output=True, + cwd=CURRENT_WORKING_DIR, + ) + commit = result.stdout.decode().strip() + assert len(commit) == 40, f"Invalid commit hash: {commit}" + return commit[:8] + + +def start_validator_process(pm2_name: str, args: List[str]) -> subprocess.Popen: + """ + Spawn a new python process running neurons.validator. + `sys.executable` ensures thet the same python interpreter is used as the one + used to run this auto-updater. + """ + assert sys.executable, "Failed to get python executable" + + log.info("Starting validator process with pm2, name: %s", pm2_name) + process = subprocess.Popen( + ( + "pm2", + "start", + sys.executable, + "--name", + pm2_name, + "--", + "-m", + "neurons.validator", + *args, + ), + cwd=CURRENT_WORKING_DIR, + ) + process.pm2_name = pm2_name + + return process + + +def stop_validator_process(process: subprocess.Popen) -> None: + """Stop the validator process""" + subprocess.run( + ("pm2", "delete", process.pm2_name), cwd=CURRENT_WORKING_DIR, check=True + ) + + +def pull_latest_version() -> None: + """ + Pull the latest version from git. + This uses `git pull --rebase`, so if any changes were made to the local repository, + this will try to apply them on top of origin's changes. This is intentional, as we + don't want to overwrite any local changes. However, if there are any conflicts, + this will abort the rebase and return to the original state. + The conflicts are expected to happen rarely since validator is expected + to be used as-is. + """ + try: + subprocess.run( + split("git pull --rebase --autostash"), check=True, cwd=CURRENT_WORKING_DIR + ) + except subprocess.CalledProcessError as exc: + log.error("Failed to pull, reverting: %s", exc) + subprocess.run(split("git rebase --abort"), check=True, cwd=CURRENT_WORKING_DIR) + + +def upgrade_packages() -> None: + """ + Upgrade python packages by running `pip install --upgrade -r requirements.txt`. + Notice: this won't work if some package in `requirements.txt` is downgraded. + Ignored as this is unlikely to happen. + """ + + log.info("Upgrading packages") + try: + subprocess.run( + split(f"{sys.executable} -m pip install -e ."), + check=True, + cwd=CURRENT_WORKING_DIR, + ) + except subprocess.CalledProcessError as exc: + log.error("Failed to upgrade packages, proceeding anyway. %s", exc) + + +def main(pm2_name: str, args: List[str]) -> None: + """ + Run the validator process and automatically update it when a new version is released. + This will check for updates every `UPDATES_CHECK_TIME` and update the validator + if a new version is available. Update is performed as simple `git pull --rebase`. + """ + + validator = start_validator_process(pm2_name, args) + current_version = latest_version = get_version() + log.info("Current version: %s", current_version) + + try: + while True: + pull_latest_version() + latest_version = get_version() + log.info("Latest version: %s", latest_version) + + if latest_version != current_version: + log.info( + "Upgraded to latest version: %s -> %s", + current_version, + latest_version, + ) + upgrade_packages() + + stop_validator_process(validator) + validator = start_validator_process(pm2_name, args) + current_version = latest_version + + time.sleep(UPDATES_CHECK_TIME.total_seconds()) + + finally: + stop_validator_process(validator) + + +if __name__ == "__main__": + logging.basicConfig( + level=logging.INFO, + format="%(asctime)s %(levelname)s %(message)s", + handlers=[logging.StreamHandler(sys.stdout)], + ) + + parser = argparse.ArgumentParser( + description="Automatically update and restart the validator process when a new version is released.", + epilog="Example usage: python start_validator.py --pm2_name 'net9vali' --wallet_name 'wallet1' --wallet_hotkey 'key123'", + ) + + parser.add_argument( + "--pm2_name", default="cancer_ai_vali", help="Name of the PM2 process." + ) + + parser.add_argument( + "--wallet.name", default="validator", help="Name of the wallet." + ) + + parser.add_argument( + "--wallet.hotkey", default="default", help="Name of the hotkey." + ) + + parser.add_argument( + "--subtensor.network", default="test", help="Name of the network." + ) + + parser.add_argument( + "--netuid", default="163", help="Netuid of the network." + ) + + parser.add_argument( + "--logging.debug", default="", help="Enable debug logging." + ) + + parser.add_argument( + "--interpreter", default="python3", help="Python interpreter to use." + ) + + flags, extra_args = parser.parse_known_args() + + main(flags.pm2_name, extra_args) \ No newline at end of file From a9183b119b4cc789ab4f49651d03de7ac374c221 Mon Sep 17 00:00:00 2001 From: Konrad Date: Mon, 2 Sep 2024 17:35:28 +0200 Subject: [PATCH 138/227] delete setup.py --- setup.py | 96 -------------------------------------------------------- 1 file changed, 96 deletions(-) delete mode 100644 setup.py diff --git a/setup.py b/setup.py deleted file mode 100644 index f76ec9b2..00000000 --- a/setup.py +++ /dev/null @@ -1,96 +0,0 @@ -# The MIT License (MIT) -# Copyright © 2023 Yuma Rao -# TODO(developer): Set your name -# Copyright © 2023 - -# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated -# documentation files (the “Software”), to deal in the Software without restriction, including without limitation -# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, -# and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all copies or substantial portions of -# the Software. - -# THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO -# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL -# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -# DEALINGS IN THE SOFTWARE. - -import re -import os -import codecs -import pathlib -from os import path -from io import open -from setuptools import setup, find_packages -from pkg_resources import parse_requirements - - -def read_requirements(path): - with open(path, "r") as f: - requirements = f.read().splitlines() - processed_requirements = [] - - for req in requirements: - # For git or other VCS links - if req.startswith("git+") or "@" in req: - pkg_name = re.search(r"(#egg=)([\w\-_]+)", req) - if pkg_name: - processed_requirements.append(pkg_name.group(2)) - else: - # You may decide to raise an exception here, - # if you want to ensure every VCS link has an #egg= at the end - continue - else: - processed_requirements.append(req) - return processed_requirements - - -requirements = read_requirements("requirements.txt") -here = path.abspath(path.dirname(__file__)) - -with open(path.join(here, "README.md"), encoding="utf-8") as f: - long_description = f.read() - -# loading version from setup.py -with codecs.open( - os.path.join(here, "template/__init__.py"), encoding="utf-8" -) as init_file: - version_match = re.search( - r"^__version__ = ['\"]([^'\"]*)['\"]", init_file.read(), re.M - ) - version_string = version_match.group(1) - -setup( - name="bittensor_subnet_template", # TODO(developer): Change this value to your module subnet name. - version=version_string, - description="bittensor_subnet_template", # TODO(developer): Change this value to your module subnet description. - long_description=long_description, - long_description_content_type="text/markdown", - url="https://github.com/opentensor/bittensor-subnet-template", # TODO(developer): Change this url to your module subnet github url. - author="bittensor.com", # TODO(developer): Change this value to your module subnet author name. - packages=find_packages(), - include_package_data=True, - author_email="", # TODO(developer): Change this value to your module subnet author email. - license="MIT", - python_requires=">=3.8", - install_requires=requirements, - classifiers=[ - "Development Status :: 3 - Alpha", - "Intended Audience :: Developers", - "Topic :: Software Development :: Build Tools", - # Pick your license as you wish - "License :: OSI Approved :: MIT License", - "Programming Language :: Python :: 3 :: Only", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - "Topic :: Scientific/Engineering", - "Topic :: Scientific/Engineering :: Mathematics", - "Topic :: Scientific/Engineering :: Artificial Intelligence", - "Topic :: Software Development", - "Topic :: Software Development :: Libraries", - "Topic :: Software Development :: Libraries :: Python Modules", - ], -) From 9c9dfc1ba2135c52a60b6a41e48825bd92f3d717 Mon Sep 17 00:00:00 2001 From: Konrad Date: Mon, 2 Sep 2024 17:41:59 +0200 Subject: [PATCH 139/227] autovalidator-test --- scripts/start_validator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/start_validator.py b/scripts/start_validator.py index 64c99407..7e4fd081 100755 --- a/scripts/start_validator.py +++ b/scripts/start_validator.py @@ -105,7 +105,7 @@ def upgrade_packages() -> None: log.info("Upgrading packages") try: subprocess.run( - split(f"{sys.executable} -m pip install -e ."), + split(f"{sys.executable} -m pip install --upgrade -r requirements.txt"), check=True, cwd=CURRENT_WORKING_DIR, ) From 6884deb7175d309a1a89ee94d48c70cb5a671df3 Mon Sep 17 00:00:00 2001 From: Konrad Date: Mon, 2 Sep 2024 17:51:01 +0200 Subject: [PATCH 140/227] further autovalidator fixes --- scripts/start_validator.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/scripts/start_validator.py b/scripts/start_validator.py index 7e4fd081..47df4810 100755 --- a/scripts/start_validator.py +++ b/scripts/start_validator.py @@ -184,10 +184,6 @@ def main(pm2_name: str, args: List[str]) -> None: "--logging.debug", default="", help="Enable debug logging." ) - parser.add_argument( - "--interpreter", default="python3", help="Python interpreter to use." - ) - flags, extra_args = parser.parse_known_args() main(flags.pm2_name, extra_args) \ No newline at end of file From 90c8407c251925739acb02ad111bf012be0b6129 Mon Sep 17 00:00:00 2001 From: Konrad Date: Mon, 2 Sep 2024 17:53:18 +0200 Subject: [PATCH 141/227] furthe autovali testing --- scripts/start_validator.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/start_validator.py b/scripts/start_validator.py index 47df4810..77492148 100755 --- a/scripts/start_validator.py +++ b/scripts/start_validator.py @@ -23,6 +23,8 @@ from typing import List from pathlib import Path +sys.path.append(str(Path(__file__).resolve().parent.parent)) + log = logging.getLogger(__name__) UPDATES_CHECK_TIME = timedelta(seconds=30) CURRENT_WORKING_DIR = Path(__file__).parent.parent From fa173679ea509b79bda8365ff7fb302a2f22c554 Mon Sep 17 00:00:00 2001 From: Konrad Date: Mon, 2 Sep 2024 21:14:27 +0200 Subject: [PATCH 142/227] working autovalidator --- neurons/validator.py | 3 +- scripts/start_validator.py | 76 +++++++++++++++++++++----------------- 2 files changed, 43 insertions(+), 36 deletions(-) diff --git a/neurons/validator.py b/neurons/validator.py index 723021ab..894e26bc 100644 --- a/neurons/validator.py +++ b/neurons/validator.py @@ -19,7 +19,6 @@ import time -from typing import Any, List import bittensor as bt import asyncio import os @@ -27,7 +26,7 @@ from cancer_ai.base.base_validator import BaseValidatorNeuron from cancer_ai.validator.competition_manager import CompetitionManager -from competition_runner import competition_loop, config_for_scheduler, run_competitions_tick +from competition_runner import config_for_scheduler, run_competitions_tick from rewarder import WinnersMapping, Rewarder, Score diff --git a/scripts/start_validator.py b/scripts/start_validator.py index 77492148..98fbe559 100755 --- a/scripts/start_validator.py +++ b/scripts/start_validator.py @@ -1,34 +1,20 @@ -""" -This script runs a validator process and automatically updates it when a new version is released. -Command-line arguments will be forwarded to validator (`neurons/validator.py`), so you can pass -them like this: - python3 scripts/start_validator.py --wallet.name=my-wallet -Auto-updates are enabled by default and will make sure that the latest version is always running -by pulling the latest version from git and upgrading python packages. This is done periodically. -Local changes may prevent the update, but they will be preserved. - -The script will use the same virtual environment as the one used to run it. If you want to run -validator within virtual environment, run this auto-update script from the virtual environment. - -Pm2 is required for this script. This script will start a pm2 process using the name provided by -the --pm2_name argument. -""" import argparse import logging import subprocess import sys import time +import os from datetime import timedelta from shlex import split from typing import List +from argparse import Namespace from pathlib import Path -sys.path.append(str(Path(__file__).resolve().parent.parent)) - log = logging.getLogger(__name__) UPDATES_CHECK_TIME = timedelta(seconds=30) CURRENT_WORKING_DIR = Path(__file__).parent.parent +ECOSYSTEM_CONFIG_PATH = CURRENT_WORKING_DIR / "ecosystem.config.js" # Path to the pm2 ecosystem config file def get_version() -> str: """Extract the version as current git commit hash""" @@ -43,27 +29,44 @@ def get_version() -> str: return commit[:8] +def generate_pm2_config(pm2_name: str, args: List[str]) -> None: + """ + Generate a pm2 ecosystem config file to run the validator. + """ + config_content = f""" + module.exports = {{ + apps: [ + {{ + name: '{pm2_name}', + script: 'neurons/validator.py', + interpreter: '{sys.executable}', + env: {{ + PYTHONPATH: '{os.environ.get('PYTHONPATH', '')}:./', + }}, + args: '{' '.join(args)}' + }} + ] + }}; + """ + with open(ECOSYSTEM_CONFIG_PATH, "w") as f: + f.write(config_content) + log.info("Generated pm2 ecosystem config at: %s", ECOSYSTEM_CONFIG_PATH) + + def start_validator_process(pm2_name: str, args: List[str]) -> subprocess.Popen: """ - Spawn a new python process running neurons.validator. - `sys.executable` ensures thet the same python interpreter is used as the one - used to run this auto-updater. + Spawn a new python process running neurons.validator using pm2. """ assert sys.executable, "Failed to get python executable" + generate_pm2_config(pm2_name, args) # Generate the pm2 config file log.info("Starting validator process with pm2, name: %s", pm2_name) process = subprocess.Popen( - ( + [ "pm2", "start", - sys.executable, - "--name", - pm2_name, - "--", - "-m", - "neurons.validator", - *args, - ), + str(ECOSYSTEM_CONFIG_PATH) + ], cwd=CURRENT_WORKING_DIR, ) process.pm2_name = pm2_name @@ -103,7 +106,6 @@ def upgrade_packages() -> None: Notice: this won't work if some package in `requirements.txt` is downgraded. Ignored as this is unlikely to happen. """ - log.info("Upgrading packages") try: subprocess.run( @@ -115,14 +117,21 @@ def upgrade_packages() -> None: log.error("Failed to upgrade packages, proceeding anyway. %s", exc) -def main(pm2_name: str, args: List[str]) -> None: +def main(pm2_name: str, args_namespace: Namespace) -> None: """ Run the validator process and automatically update it when a new version is released. This will check for updates every `UPDATES_CHECK_TIME` and update the validator if a new version is available. Update is performed as simple `git pull --rebase`. """ - validator = start_validator_process(pm2_name, args) + args_list = [] + for key, value in vars(args_namespace).items(): + if value != '' and value is not None: + args_list.append(f"--{key}") + if not isinstance(value, bool): + args_list.append(str(value)) + + validator = start_validator_process(pm2_name, args_list) current_version = latest_version = get_version() log.info("Current version: %s", current_version) @@ -187,5 +196,4 @@ def main(pm2_name: str, args: List[str]) -> None: ) flags, extra_args = parser.parse_known_args() - - main(flags.pm2_name, extra_args) \ No newline at end of file + main(flags.pm2_name, flags) From 056ce99461c3c81eb0866ac40db6e34f3d6c7e9f Mon Sep 17 00:00:00 2001 From: Konrad Date: Mon, 2 Sep 2024 21:20:57 +0200 Subject: [PATCH 143/227] autovalidator fix --- cancer_ai/utils/config.py | 6 +++--- scripts/start_validator.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cancer_ai/utils/config.py b/cancer_ai/utils/config.py index 364cfb3f..9eca520f 100644 --- a/cancer_ai/utils/config.py +++ b/cancer_ai/utils/config.py @@ -193,9 +193,9 @@ def add_common_args(cls, parser): default="", ) parser.add_argument( - "--competition.id", - type=str, - help="Path for storing competition participants models .", + "--competition.id", + type=str, + help="Path for storing competition participants models .", ) parser.add_argument( diff --git a/scripts/start_validator.py b/scripts/start_validator.py index 98fbe559..d341c47e 100755 --- a/scripts/start_validator.py +++ b/scripts/start_validator.py @@ -150,7 +150,7 @@ def main(pm2_name: str, args_namespace: Namespace) -> None: upgrade_packages() stop_validator_process(validator) - validator = start_validator_process(pm2_name, args) + validator = start_validator_process(pm2_name, args_list) current_version = latest_version time.sleep(UPDATES_CHECK_TIME.total_seconds()) From 6440dbc6c1fcfff1bce005c2553dc21fb9551dff Mon Sep 17 00:00:00 2001 From: Konrad Date: Tue, 3 Sep 2024 01:12:41 +0200 Subject: [PATCH 144/227] weight utils fix --- cancer_ai/base/utils/weight_utils.py | 12 ++++++++---- scripts/start_validator.py | 2 +- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/cancer_ai/base/utils/weight_utils.py b/cancer_ai/base/utils/weight_utils.py index 26133efd..89f260e2 100644 --- a/cancer_ai/base/utils/weight_utils.py +++ b/cancer_ai/base/utils/weight_utils.py @@ -159,15 +159,21 @@ def process_weights_for_netuid( non_zero_weight_idx = np.argwhere(weights > 0).squeeze() non_zero_weight_uids = uids[non_zero_weight_idx] non_zero_weights = weights[non_zero_weight_idx] + + if not isinstance(non_zero_weights, np.ndarray): + non_zero_weights = np.array(non_zero_weights) + bittensor.logging.debug("Converting non_zero_weights to numpy array") + if non_zero_weights.size == 0 or metagraph.n < min_allowed_weights: bittensor.logging.warning("No non-zero weights returning all ones.") final_weights = np.ones(metagraph.n) / metagraph.n bittensor.logging.debug("final_weights", final_weights) return np.arange(len(final_weights)), final_weights + elif non_zero_weights.size < min_allowed_weights: bittensor.logging.warning( - "No non-zero weights less then min allowed weight, returning all ones." + "No non-zero weights less than min allowed weight, returning all ones." ) weights = ( np.ones(metagraph.n) * 1e-5 @@ -182,9 +188,7 @@ def process_weights_for_netuid( bittensor.logging.debug("non_zero_weights", non_zero_weights) # Compute the exclude quantile and find the weights in the lowest quantile - max_exclude = max(0, len(non_zero_weights) - min_allowed_weights) / len( - non_zero_weights - ) + max_exclude = max(0, non_zero_weights.size - min_allowed_weights) / non_zero_weights.size exclude_quantile = min([quantile, max_exclude]) lowest_quantile = np.quantile(non_zero_weights, exclude_quantile) bittensor.logging.debug("max_exclude", max_exclude) diff --git a/scripts/start_validator.py b/scripts/start_validator.py index d341c47e..4e9bf235 100755 --- a/scripts/start_validator.py +++ b/scripts/start_validator.py @@ -192,7 +192,7 @@ def main(pm2_name: str, args_namespace: Namespace) -> None: ) parser.add_argument( - "--logging.debug", default="", help="Enable debug logging." + "--logging.debug", default=1, help="Enable debug logging." ) flags, extra_args = parser.parse_known_args() From 4e13ac64b934db35db3093d63e31185ec4389cd9 Mon Sep 17 00:00:00 2001 From: Wojtek Jurkowlaniec Date: Tue, 3 Sep 2024 01:32:06 +0200 Subject: [PATCH 145/227] check back 15 minutes of there are any due competitions to run --- cancer_ai/base/base_validator.py | 7 +- cancer_ai/utils/config.py | 2 +- cancer_ai/validator/competition_manager.py | 8 +- {neurons => cancer_ai/validator}/rewarder.py | 0 .../validator}/rewarder_test.py | 0 neurons/competition_config.json | 15 +++- neurons/competition_runner.py | 26 +++--- neurons/competition_runner_test.py | 2 +- neurons/validator.py | 82 ++++++++++++------- 9 files changed, 90 insertions(+), 52 deletions(-) rename {neurons => cancer_ai/validator}/rewarder.py (100%) rename {neurons => cancer_ai/validator}/rewarder_test.py (100%) diff --git a/cancer_ai/base/base_validator.py b/cancer_ai/base/base_validator.py index b1c0ad69..a9d0f098 100644 --- a/cancer_ai/base/base_validator.py +++ b/cancer_ai/base/base_validator.py @@ -17,6 +17,7 @@ # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER # DEALINGS IN THE SOFTWARE. +from abc import abstractmethod import copy import numpy as np @@ -107,6 +108,10 @@ def serve_axon(self): bt.logging.error(f"Failed to create Axon initialize with exception: {e}") pass + @abstractmethod + def concurrent_forward(self): + pass + def run(self): """ Initiates and manages the main loop for the miner on the Bittensor network. The main loop handles graceful shutdown on keyboard interrupts and logs unforeseen errors. @@ -311,7 +316,7 @@ def save_state(self): self.config.neuron.full_path + "/state.npz", scores=self.scores, hotkeys=self.hotkeys, - rewarder_config = self.rewarder_config, + rewarder_config=self.rewarder_config, ) def load_state(self): diff --git a/cancer_ai/utils/config.py b/cancer_ai/utils/config.py index 9eca520f..afe52cb2 100644 --- a/cancer_ai/utils/config.py +++ b/cancer_ai/utils/config.py @@ -274,7 +274,7 @@ def add_validator_args(cls, parser): # Note: the validator needs to serve an Axon with their IP or they may # be blacklisted by the firewall of serving peers on the network. help="Set this flag to not attempt to serve an Axon.", - default=False, + default=True, ) parser.add_argument( diff --git a/cancer_ai/validator/competition_manager.py b/cancer_ai/validator/competition_manager.py index f3be9636..78409014 100644 --- a/cancer_ai/validator/competition_manager.py +++ b/cancer_ai/validator/competition_manager.py @@ -27,6 +27,7 @@ COMPETITION_HANDLER_MAPPING = { "melanoma-1": MelanomaCompetitionHandler, + "melanoma-testnet": MelanomaCompetitionHandler, } @@ -141,14 +142,15 @@ async def get_miner_model(self, chain_miner_model: ChainMinerModel): return model_info async def sync_chain_miners_test(self): - """For testing purposes""" + """Get registered mineres from testnet subnet 163""" + hotkeys_with_models = { - "wojtek": ModelInfo( + "5Fo2fenxPY1D7hgTHc88g1zrX2ZX17g8DvE5KnazueYefjN5": ModelInfo( hf_repo_id="safescanai/test_dataset", hf_model_filename="model_dynamic.onnx", hf_repo_type="dataset", ), - "bruno": ModelInfo( + "5DZZnwU2LapwmZfYL9AEAWpUR6FoFvqHnzQ5F71Mhwotxujq": ModelInfo( hf_repo_id="safescanai/test_dataset", hf_model_filename="best_model.onnx", hf_repo_type="dataset", diff --git a/neurons/rewarder.py b/cancer_ai/validator/rewarder.py similarity index 100% rename from neurons/rewarder.py rename to cancer_ai/validator/rewarder.py diff --git a/neurons/rewarder_test.py b/cancer_ai/validator/rewarder_test.py similarity index 100% rename from neurons/rewarder_test.py rename to cancer_ai/validator/rewarder_test.py diff --git a/neurons/competition_config.json b/neurons/competition_config.json index 0acb097d..febe149b 100644 --- a/neurons/competition_config.json +++ b/neurons/competition_config.json @@ -2,9 +2,18 @@ { "competition_id": "melanoma-1", "category": "skin", - "evaluation_time": [ - "21:31", - "21:32" + "evaluation_times": [ + "23:12" + ], + "dataset_hf_repo": "safescanai/test_dataset", + "dataset_hf_filename": "skin_melanoma.zip", + "dataset_hf_repo_type": "dataset" + }, + { + "competition_id": "melanoma-testnet", + "category": "skin", + "evaluation_times": [ + "22:10" ], "dataset_hf_repo": "safescanai/test_dataset", "dataset_hf_filename": "skin_melanoma.zip", diff --git a/neurons/competition_runner.py b/neurons/competition_runner.py index cb35550b..fd7fb792 100644 --- a/neurons/competition_runner.py +++ b/neurons/competition_runner.py @@ -40,20 +40,18 @@ async def run_competitions_tick( now_time = datetime.now(timezone.utc) now_time = f"{now_time.hour}:{now_time.minute}" bt.logging.debug(now_time) - # if now_time not in competition_times: - # return None - for time_competition in competition_times: - # if now_time == time_competition: - bt.logging.info( - f"Running {competition_times[time_competition].competition_id} at {now_time}" - ) - winning_evaluation_hotkey = await competition_times[ - time_competition - ].evaluate() - return ( - winning_evaluation_hotkey, - competition_times[time_competition].competition_id, - ) + competition_to_run = competition_times.get(now_time) + if not competition_to_run: + bt.logging.info("No competitions to run") + return + bt.logging.info( + f"Running {competition_to_run.competition_id} at {now_time}" + ) + winning_evaluation_hotkey = await competition_to_run.evaluate() + return ( + winning_evaluation_hotkey, + competition_to_run.competition_id, + ) async def competition_loop(scheduler_config: Dict[str, CompetitionManager], rewarder_config: WinnersMapping): diff --git a/neurons/competition_runner_test.py b/neurons/competition_runner_test.py index f9d054ab..5f14e0a0 100644 --- a/neurons/competition_runner_test.py +++ b/neurons/competition_runner_test.py @@ -5,7 +5,7 @@ import bittensor as bt from typing import List, Dict from competition_runner import run_competitions_tick, competition_loop -from rewarder import WinnersMapping, Rewarder +from cancer_ai.validator.rewarder import WinnersMapping, Rewarder import time # TODO integrate with bt config diff --git a/neurons/validator.py b/neurons/validator.py index 894e26bc..0c669235 100644 --- a/neurons/validator.py +++ b/neurons/validator.py @@ -21,73 +21,87 @@ import time import bittensor as bt import asyncio -import os +import os import numpy as np +from cancer_ai.validator.rewarder import WinnersMapping, Rewarder, Score from cancer_ai.base.base_validator import BaseValidatorNeuron from cancer_ai.validator.competition_manager import CompetitionManager -from competition_runner import config_for_scheduler, run_competitions_tick -from rewarder import WinnersMapping, Rewarder, Score +from competition_runner import ( + config_for_scheduler, + run_competitions_tick, + CompetitionRunLog, +) class Validator(BaseValidatorNeuron): - """ - Your validator neuron class. You should use this class to define your validator's behavior. In particular, you should replace the forward function with your own logic. - - This class inherits from the BaseValidatorNeuron class, which in turn inherits from BaseNeuron. The BaseNeuron class takes care of routine tasks such as setting up wallet, subtensor, metagraph, logging directory, parsing config, etc. You can override any of the methods in BaseNeuron if you need to customize the behavior. - - This class provides reasonable default behavior for a validator such as keeping a moving average of the scores of the miners and using them to set weights at the end of each epoch. Additionally, the scores are reset for new hotkeys at the end of each epoch. - """ - def __init__(self, config=None): super(Validator, self).__init__(config=config) - self.winners_mapping = WinnersMapping(competition_leader_map={}, hotkey_score_map={}) + self.run_log = CompetitionRunLog(runs=[]) + self.winners_mapping = WinnersMapping( + competition_leader_map={}, hotkey_score_map={} + ) + self.load_state() - self.scheduler_config = config_for_scheduler(self.config, self.hotkeys, test_mode=True) - self.rewarder = Rewarder(self.winners_mapping) + self.competition_scheduler = config_for_scheduler( + self.config, self.hotkeys, test_mode=True + ) + bt.logging.info(f"Scheduler config: {self.competition_scheduler}") + self.rewarder = Rewarder(self.winners_mapping) async def concurrent_forward(self): coroutines = [ self.run_test_function(), - self.competition_loop_tick(self.scheduler_config, self.winners_mapping) + self.competition_loop_tick(self.competition_scheduler), ] await asyncio.gather(*coroutines) - async def run_test_function(self): print("Running test function") await asyncio.sleep(5) print("Test function done") - async def competition_loop_tick(self, scheduler_config: dict[str, CompetitionManager], rewarder_config: WinnersMapping): - """Example of scheduling coroutine""" - competition_result = await run_competitions_tick(scheduler_config) - bt.logging.debug(f"Competition result: {competition_result}") + async def competition_loop_tick( + self, scheduler_config: dict[str, CompetitionManager] + ): + + bt.logging.debug("Run log", self.run_log) + competition_result = await run_competitions_tick(scheduler_config, self.run_log) + if not competition_result: - return - + return + + bt.logging.debug(f"Competition result: {competition_result}") + winning_evaluation_hotkey, competition_id = competition_result # update the scores await self.rewarder.update_scores(winning_evaluation_hotkey, competition_id) - print("...,.,.,.,.,.,.,.,",self.rewarder.competition_leader_mapping, self.rewarder.scores) - self.winners_mapping = WinnersMapping(competition_leader_map=self.rewarder.competition_leader_mapping, - hotkey_score_map=self.rewarder.scores) + print( + "...,.,.,.,.,.,.,.,", + self.rewarder.competition_leader_mapping, + self.rewarder.scores, + ) + self.winners_mapping = WinnersMapping( + competition_leader_map=self.rewarder.competition_leader_mapping, + hotkey_score_map=self.rewarder.scores, + ) self.save_state() hotkey_to_score_map = self.winners_mapping.hotkey_score_map self.scores = [ - np.float32(hotkey_to_score_map.get(hotkey, Score(score=0.0, reduction=0.0)).score) + np.float32( + hotkey_to_score_map.get(hotkey, Score(score=0.0, reduction=0.0)).score + ) for hotkey in self.metagraph.hotkeys ] self.save_state() print(".....................Updated rewarder config:") print(self.winners_mapping) - # await asyncio.sleep(60) def save_state(self): """Saves the state of the validator to a file.""" @@ -95,12 +109,18 @@ def save_state(self): # Save the state of the validator to file. if not getattr(self, "winners_mapping", None): - self.winners_mapping = WinnersMapping(competition_leader_map={}, hotkey_score_map={}) + self.winners_mapping = WinnersMapping( + competition_leader_map={}, hotkey_score_map={} + ) + if not getattr(self, "run_log", None): + self.run_log = CompetitionRunLog(runs=[]) + np.savez( self.config.neuron.full_path + "/state.npz", scores=self.scores, hotkeys=self.hotkeys, rewarder_config=self.winners_mapping.model_dump(), + run_log=self.run_log.model_dump(), ) def load_state(self): @@ -114,6 +134,7 @@ def load_state(self): scores=self.scores, hotkeys=self.hotkeys, rewarder_config=self.winners_mapping.model_dump(), + run_log=self.run_log.model_dump(), ) return @@ -121,7 +142,10 @@ def load_state(self): state = np.load(self.config.neuron.full_path + "/state.npz", allow_pickle=True) self.scores = state["scores"] self.hotkeys = state["hotkeys"] - self.winners_mapping = WinnersMapping.model_validate(state["rewarder_config"].item()) + self.winners_mapping = WinnersMapping.model_validate( + state["rewarder_config"].item() + ) + self.run_log = CompetitionRunLog.model_validate(state["run_log"].item()) # The main function parses the configuration and runs the validator. From 9ba8bf694b8487117ca5a4f972ffbba4d5aeede0 Mon Sep 17 00:00:00 2001 From: Wojtek Jurkowlaniec Date: Tue, 3 Sep 2024 19:36:31 +0200 Subject: [PATCH 146/227] competittion runner --- neurons/competition_config.json | 4 +- neurons/competition_runner.py | 138 +++++++++++++++++++++++++------- 2 files changed, 112 insertions(+), 30 deletions(-) diff --git a/neurons/competition_config.json b/neurons/competition_config.json index febe149b..9a9719cc 100644 --- a/neurons/competition_config.json +++ b/neurons/competition_config.json @@ -3,7 +3,7 @@ "competition_id": "melanoma-1", "category": "skin", "evaluation_times": [ - "23:12" + "23:42" ], "dataset_hf_repo": "safescanai/test_dataset", "dataset_hf_filename": "skin_melanoma.zip", @@ -13,7 +13,7 @@ "competition_id": "melanoma-testnet", "category": "skin", "evaluation_times": [ - "22:10" + "23:44" ], "dataset_hf_repo": "safescanai/test_dataset", "dataset_hf_filename": "skin_melanoma.zip", diff --git a/neurons/competition_runner.py b/neurons/competition_runner.py index fd7fb792..5611c07b 100644 --- a/neurons/competition_runner.py +++ b/neurons/competition_runner.py @@ -1,26 +1,71 @@ from cancer_ai.validator.competition_manager import CompetitionManager -from datetime import datetime - +from datetime import datetime, time, timedelta +from pydantic import BaseModel import asyncio import json -from datetime import datetime, timezone +from datetime import datetime, timezone, timedelta import bittensor as bt from typing import List, Tuple, Dict -from rewarder import Rewarder, WinnersMapping, CompetitionLeader +from cancer_ai.validator.rewarder import Rewarder, WinnersMapping, CompetitionLeader # from cancer_ai.utils.config import config # TODO MOVE SOMEWHERE main_competitions_cfg = json.load(open("neurons/competition_config.json", "r")) +MINUTES_BACK = 15 + + +class CompetitionRun(BaseModel): + competition_id: str + start_time: datetime + end_time: datetime | None = None + + +class CompetitionRunLog(BaseModel): + runs: list[CompetitionRun] + + def add_run(self, new_run: CompetitionRun): + """Add a new run and rotate the list if it exceeds 20 entries.""" + self.runs.append(new_run) + if len(self.runs) > 20: + self.runs = self.runs[-20:] + + def finish_run(self, competition_id: str): + """Finish the run with the given competition_id""" + for run in self.runs: + if run.competition_id == competition_id: + run.end_time = datetime.now(timezone.utc) + + def was_competition_already_executed( + self, last_minutes: int = 5, competition_id: str = None + ): + """Check if competition was executed in last minutes""" + now_time = datetime.now(timezone.utc) + for run in self.runs: + if competition_id and run.competition_id != competition_id: + continue + if run.end_time and (now_time - run.end_time).seconds < last_minutes * 60: + return True + return False + + +class CompetitionSchedulerConfig(BaseModel): + config: dict[datetime.time, CompetitionManager] + + class Config: + arbitrary_types_allowed = True + + def config_for_scheduler( bt_config, hotkeys: List[str], test_mode: bool = False -) -> Dict[str, CompetitionManager]: +) -> CompetitionSchedulerConfig: """Returns CompetitionManager instances arranged by competition time""" - time_arranged_competitions = {} + scheduler_config = {} for competition_cfg in main_competitions_cfg: - for competition_time in competition_cfg["evaluation_time"]: - time_arranged_competitions[competition_time] = CompetitionManager( + for competition_time in competition_cfg["evaluation_times"]: + parsed_time = datetime.strptime(competition_time, "%H:%M").time() + scheduler_config[parsed_time] = CompetitionManager( bt_config, hotkeys, competition_cfg["competition_id"], @@ -30,31 +75,66 @@ def config_for_scheduler( competition_cfg["dataset_hf_repo_type"], test_mode=test_mode, ) - return time_arranged_competitions + + return scheduler_config async def run_competitions_tick( - competition_times: Dict[str, CompetitionManager], + competition_times: CompetitionSchedulerConfig, + run_log: CompetitionRunLog, ) -> Tuple[str, str] | None: """Checks if time is right and launches competition, returns winning hotkey and Competition ID. Should be run each minute.""" - now_time = datetime.now(timezone.utc) - now_time = f"{now_time.hour}:{now_time.minute}" - bt.logging.debug(now_time) - competition_to_run = competition_times.get(now_time) - if not competition_to_run: - bt.logging.info("No competitions to run") - return - bt.logging.info( - f"Running {competition_to_run.competition_id} at {now_time}" - ) - winning_evaluation_hotkey = await competition_to_run.evaluate() - return ( - winning_evaluation_hotkey, - competition_to_run.competition_id, + + # getting current time + now = datetime.now(timezone.utc) + now_time = time(now.hour, now.minute) + bt.logging.info(f"Checking competitions at {now_time}") + + for i in range(0, MINUTES_BACK): + # getting current time minus X minutes + check_time = ( + datetime.combine(datetime.today(), now_time) - timedelta(minutes=i) + ).time() + + # bt.logging.debug(f"Checking competitions at {check_time}") + if competition_manager := competition_times.get(check_time): + bt.logging.debug( + f"Found competition {competition_manager.competition_id} at {check_time}" + ) + else: + continue + + if run_log.was_competition_already_executed( + competition_id=competition_manager.competition_id + ): + bt.logging.info( + f"Competition {competition_manager.competition_id} already executed, skipping" + ) + continue + + bt.logging.info(f"Running {competition_manager.competition_id} at {now_time}") + run_log.add_run( + CompetitionRun( + competition_id=competition_manager.competition_id, + start_time=datetime.now(timezone.utc), + ) + ) + winning_evaluation_hotkey = await competition_manager.evaluate() + run_log.finish_run(competition_manager.competition_id) + # TODO log last run to WANDB + return ( + winning_evaluation_hotkey, + competition_manager.competition_id, + ) + + bt.logging.debug( + f"Did not find any competitions to run for past {MINUTES_BACK} minutes" ) -async def competition_loop(scheduler_config: Dict[str, CompetitionManager], rewarder_config: WinnersMapping): +async def competition_loop_not_used( + scheduler_config: CompetitionSchedulerConfig, rewarder_config: WinnersMapping +): """Example of scheduling coroutine""" while True: competition_result = await run_competitions_tick(scheduler_config) @@ -62,7 +142,9 @@ async def competition_loop(scheduler_config: Dict[str, CompetitionManager], rewa if competition_result: winning_evaluation_hotkey, competition_id = competition_result rewarder = Rewarder(rewarder_config) - updated_rewarder_config = await rewarder.update_scores(winning_evaluation_hotkey, competition_id) + updated_rewarder_config = await rewarder.update_scores( + winning_evaluation_hotkey, competition_id + ) # save state of self.rewarder_config # save state of self.score (map rewarder config to scores) print(".....................Updated rewarder config:") @@ -79,5 +161,5 @@ async def competition_loop(scheduler_config: Dict[str, CompetitionManager], rewa hotkeys = [] bt_config = {} # get from bt config scheduler_config = config_for_scheduler(bt_config, hotkeys) - rewarder_config = WinnersMapping({},{}) - asyncio.run(competition_loop(scheduler_config, rewarder_config)) + rewarder_config = WinnersMapping(competition_leader_map={}, hotkey_score_map={}) + asyncio.run(competition_loop_not_used(scheduler_config, rewarder_config)) From 8894650f36d7b8f7d88e87b1c88130eb4c4be805 Mon Sep 17 00:00:00 2001 From: Konrad Date: Tue, 3 Sep 2024 20:57:13 +0200 Subject: [PATCH 147/227] fixed states saving/loading --- cancer_ai/base/base_validator.py | 11 +++++++++++ neurons/validator.py | 14 -------------- 2 files changed, 11 insertions(+), 14 deletions(-) diff --git a/cancer_ai/base/base_validator.py b/cancer_ai/base/base_validator.py index a9d0f098..e77b7a3e 100644 --- a/cancer_ai/base/base_validator.py +++ b/cancer_ai/base/base_validator.py @@ -37,6 +37,12 @@ from ..mock import MockDendrite from ..utils.config import add_validator_args +from neurons.competition_runner import ( + CompetitionRunLog, +) + +from cancer_ai.validator.rewarder import WinnersMapping + class BaseValidatorNeuron(BaseNeuron): """ @@ -66,6 +72,11 @@ def __init__(self, config=None): # Set up initial scoring weights for validation bt.logging.info("Building validation weights.") self.scores = np.zeros(self.metagraph.n, dtype=np.float32) + self.run_log = CompetitionRunLog(runs=[]) + self.winners_mapping = WinnersMapping( + competition_leader_map={}, hotkey_score_map={} + ) + self.load_state() # Init sync with the network. Updates the metagraph. self.sync() diff --git a/neurons/validator.py b/neurons/validator.py index 0c669235..897dfedc 100644 --- a/neurons/validator.py +++ b/neurons/validator.py @@ -38,13 +38,6 @@ class Validator(BaseValidatorNeuron): def __init__(self, config=None): super(Validator, self).__init__(config=config) - self.run_log = CompetitionRunLog(runs=[]) - self.winners_mapping = WinnersMapping( - competition_leader_map={}, hotkey_score_map={} - ) - - self.load_state() - self.competition_scheduler = config_for_scheduler( self.config, self.hotkeys, test_mode=True ) @@ -80,11 +73,6 @@ async def competition_loop_tick( # update the scores await self.rewarder.update_scores(winning_evaluation_hotkey, competition_id) - print( - "...,.,.,.,.,.,.,.,", - self.rewarder.competition_leader_mapping, - self.rewarder.scores, - ) self.winners_mapping = WinnersMapping( competition_leader_map=self.rewarder.competition_leader_mapping, hotkey_score_map=self.rewarder.scores, @@ -100,8 +88,6 @@ async def competition_loop_tick( for hotkey in self.metagraph.hotkeys ] self.save_state() - print(".....................Updated rewarder config:") - print(self.winners_mapping) def save_state(self): """Saves the state of the validator to a file.""" From d39772d2fb3c8c15652d4179922e24183364a9d3 Mon Sep 17 00:00:00 2001 From: Konrad Date: Tue, 3 Sep 2024 21:08:16 +0200 Subject: [PATCH 148/227] deleted unecessary test process --- neurons/validator.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/neurons/validator.py b/neurons/validator.py index 897dfedc..c400aed9 100644 --- a/neurons/validator.py +++ b/neurons/validator.py @@ -47,16 +47,10 @@ def __init__(self, config=None): async def concurrent_forward(self): coroutines = [ - self.run_test_function(), self.competition_loop_tick(self.competition_scheduler), ] await asyncio.gather(*coroutines) - async def run_test_function(self): - print("Running test function") - await asyncio.sleep(5) - print("Test function done") - async def competition_loop_tick( self, scheduler_config: dict[str, CompetitionManager] ): From 433d46a3ff34fa34c6f4fd27414c601a73295706 Mon Sep 17 00:00:00 2001 From: Wojtek Jurkowlaniec Date: Wed, 4 Sep 2024 00:17:34 +0200 Subject: [PATCH 149/227] optimized not optimized code --- cancer_ai/utils/config.py | 18 ++----- cancer_ai/validator/competition_manager.py | 58 +++++++++++----------- cancer_ai/validator/dataset_manager.py | 25 ++++++---- cancer_ai/validator/model_manager.py | 2 +- neurons/competition_runner_test.py | 40 ++++++++++----- 5 files changed, 78 insertions(+), 65 deletions(-) diff --git a/cancer_ai/utils/config.py b/cancer_ai/utils/config.py index afe52cb2..b28cb42d 100644 --- a/cancer_ai/utils/config.py +++ b/cancer_ai/utils/config.py @@ -132,13 +132,10 @@ def add_args(cls, parser): def add_miner_args(cls, parser): """Add miner specific arguments to the parser.""" - - parser.add_argument( "--hf_repo_id", type=str, help="Hugging Face model repository ID", - ) parser.add_argument( @@ -166,7 +163,7 @@ def add_miner_args(cls, parser): "--model_path", type=str, help="Path to ONNX model, used for evaluation", - ) + ) parser.add_argument( "--clean_after_run", @@ -183,7 +180,6 @@ def add_miner_args(cls, parser): ) - def add_common_args(cls, parser): """Add validator and miner specific arguments to the parser.""" parser.add_argument( @@ -220,8 +216,6 @@ def add_common_args(cls, parser): ) - - def add_validator_args(cls, parser): """Add validator specific arguments to the parser.""" @@ -299,12 +293,7 @@ def add_validator_args(cls, parser): ) - - - - - -def path_config(cls): +def path_config(cls=None): """ Returns the configuration object specific to this miner or validator after adding relevant arguments. """ @@ -316,5 +305,6 @@ def path_config(cls): bt.logging.add_args(parser) bt.axon.add_args(parser) add_common_args(cls, parser) - cls.add_args(parser) + if cls: + cls.add_args(parser) return bt.config(parser) diff --git a/cancer_ai/validator/competition_manager.py b/cancer_ai/validator/competition_manager.py index 78409014..54caa04f 100644 --- a/cancer_ai/validator/competition_manager.py +++ b/cancer_ai/validator/competition_manager.py @@ -85,35 +85,35 @@ def __init__( self.chain_miner_models = {} self.test_mode = test_mode - # def log_results_to_wandb( - # self, hotkey: str, evaluation_result: ModelEvaluationResult - # ) -> None: - # wandb.init(project=self.config.wandb_project_name) - # wandb.log( - # { - # "hotkey": hotkey, - # "tested_entries": evaluation_result.tested_entries, - # "accuracy": evaluation_result.accuracy, - # "precision": evaluation_result.precision, - # "recall": evaluation_result.recall, - # "confusion_matrix": evaluation_result.confusion_matrix.tolist(), - # "roc_curve": { - # "fpr": evaluation_result.fpr.tolist(), - # "tpr": evaluation_result.tpr.tolist(), - # }, - # "roc_auc": evaluation_result.roc_auc, - # } - # ) - - # wandb.finish() - # bt.logging.info("Logged results to wandb") - # bt.logging.info("Hotkey: ", hotkey) - # bt.logging.info("Tested entries: ", evaluation_result.tested_entries) - # bt.logging.info("Model test run time: ", evaluation_result.run_time_s) - # bt.logging.info("Accuracy: ", evaluation_result.accuracy) - # bt.logging.info("Precision: ", evaluation_result.precision) - # bt.logging.info("Recall: ", evaluation_result.recall) - # bt.logging.info("roc_auc: ", evaluation_result.roc_auc) + def log_results_to_wandb( + self, hotkey: str, evaluation_result: ModelEvaluationResult + ) -> None: + wandb.init(project=self.config.wandb_project_name) + wandb.log( + { + "hotkey": hotkey, + "tested_entries": evaluation_result.tested_entries, + "accuracy": evaluation_result.accuracy, + "precision": evaluation_result.precision, + "recall": evaluation_result.recall, + "confusion_matrix": evaluation_result.confusion_matrix.tolist(), + "roc_curve": { + "fpr": evaluation_result.fpr.tolist(), + "tpr": evaluation_result.tpr.tolist(), + }, + "roc_auc": evaluation_result.roc_auc, + } + ) + + wandb.finish() + bt.logging.info("Logged results to wandb") + bt.logging.info("Hotkey: ", hotkey) + bt.logging.info("Tested entries: ", evaluation_result.tested_entries) + bt.logging.info("Model test run time: ", evaluation_result.run_time_s) + bt.logging.info("Accuracy: ", evaluation_result.accuracy) + bt.logging.info("Precision: ", evaluation_result.precision) + bt.logging.info("Recall: ", evaluation_result.recall) + bt.logging.info("roc_auc: ", evaluation_result.roc_auc) def get_state(self): return { diff --git a/cancer_ai/validator/dataset_manager.py b/cancer_ai/validator/dataset_manager.py index 461cda85..29b9909d 100644 --- a/cancer_ai/validator/dataset_manager.py +++ b/cancer_ai/validator/dataset_manager.py @@ -17,7 +17,12 @@ class DatasetManagerException(Exception): class DatasetManager(SerializableManager): def __init__( - self, config, competition_id: str, hf_repo_id: str, hf_filename: str, hf_repo_type: str + self, + config, + competition_id: str, + hf_repo_id: str, + hf_filename: str, + hf_repo_type: str, ) -> None: """ Initializes a new instance of the DatasetManager class. @@ -32,15 +37,13 @@ def __init__( None """ self.config = config - + self.hf_repo_id = hf_repo_id self.hf_filename = hf_filename self.hf_repo_type = hf_repo_type self.competition_id = competition_id self.local_compressed_path = "" - self.local_extracted_dir = Path( - self.config.models.dataset_dir, competition_id - ) + self.local_extracted_dir = Path(self.config.models.dataset_dir, competition_id) self.data: Tuple[List, List] = () self.handler = None @@ -54,13 +57,13 @@ def set_state(self, state: dict): async def download_dataset(self): if not os.path.exists(self.local_extracted_dir): os.makedirs(self.local_extracted_dir) - + self.local_compressed_path = HfApi().hf_hub_download( self.hf_repo_id, self.hf_filename, cache_dir=Path(self.config.models.dataset_dir), repo_type=self.hf_repo_type, - token=self.config.hf_token, + token=self.config.hf_token if hasattr(self.config, "hf_token") else None, ) def delete_dataset(self) -> None: @@ -93,7 +96,9 @@ async def unzip_dataset(self) -> None: def set_dataset_handler(self) -> None: """Detect dataset type and set handler""" if not self.local_compressed_path: - raise DatasetManagerException(f"Dataset '{self.config.competition.id}' not downloaded") + raise DatasetManagerException( + f"Dataset '{self.config.competition.id}' not downloaded" + ) # is csv in directory if os.path.exists(Path(self.local_extracted_dir, "labels.csv")): self.handler = DatasetImagesCSV( @@ -118,5 +123,7 @@ async def prepare_dataset(self) -> None: async def get_data(self) -> Tuple[List, List]: """Get data from dataset handler""" if not self.data: - raise DatasetManagerException(f"Dataset '{self.competition_id}' not initalized ") + raise DatasetManagerException( + f"Dataset '{self.competition_id}' not initalized " + ) return self.data diff --git a/cancer_ai/validator/model_manager.py b/cancer_ai/validator/model_manager.py index 01ae23af..a84a8bc4 100644 --- a/cancer_ai/validator/model_manager.py +++ b/cancer_ai/validator/model_manager.py @@ -52,7 +52,7 @@ async def download_miner_model(self, hotkey) -> None: model_info.hf_model_filename, cache_dir=self.config.models.model_dir, repo_type=model_info.hf_repo_type, - token=self.config.hf_token, + token=self.config.hf_token if hasattr(self.config, "hf_token") else None, ) def add_model( diff --git a/neurons/competition_runner_test.py b/neurons/competition_runner_test.py index 5f14e0a0..6575a7d4 100644 --- a/neurons/competition_runner_test.py +++ b/neurons/competition_runner_test.py @@ -4,28 +4,36 @@ from types import SimpleNamespace import bittensor as bt from typing import List, Dict -from competition_runner import run_competitions_tick, competition_loop from cancer_ai.validator.rewarder import WinnersMapping, Rewarder import time +from cancer_ai.base.base_miner import BaseNeuron +from cancer_ai.utils.config import path_config, add_miner_args +import copy # TODO integrate with bt config test_config = SimpleNamespace( **{ - "model_dir": "/tmp/models", - "dataset_dir": "/tmp/datasets", "wandb_entity": "testnet", "wandb_project_name": "melanoma-1", "competition_id": "melaonoma-1", "hotkeys": [], "subtensor": SimpleNamespace(**{"network": "test"}), "netuid": 163, + "models": SimpleNamespace( + **{ + "model_dir": "/tmp/models", + "dataset_dir": "/tmp/datasets", + } + ), } ) main_competitions_cfg = json.load(open("neurons/competition_config.json", "r")) -def run_all_competitions(path_config: str, hotkeys: List[str], competitions_cfg: List[dict]) -> None: +async def run_all_competitions( + path_config: str, hotkeys: List[str], competitions_cfg: List[dict] +) -> None: """Run all competitions, for debug purposes""" for competition_cfg in competitions_cfg: bt.logging.info("Starting competition: ", competition_cfg) @@ -39,7 +47,7 @@ def run_all_competitions(path_config: str, hotkeys: List[str], competitions_cfg: competition_cfg["dataset_hf_repo_type"], test_mode=True, ) - bt.logging.info(asyncio.run(competition_manager.evaluate())) + bt.logging.info(await competition_manager.evaluate()) def config_for_scheduler() -> Dict[str, CompetitionManager]: @@ -59,6 +67,7 @@ def config_for_scheduler() -> Dict[str, CompetitionManager]: ) return time_arranged_competitions + async def competition_loop(): """Example of scheduling coroutine""" while True: @@ -74,15 +83,22 @@ async def competition_loop(): rewarder = Rewarder(rewarder_config) for winning_evaluation_hotkey, competition_id in test_cases: - rewarder.update_scores(winning_evaluation_hotkey, competition_id) - print("Updated rewarder competition leader map:", rewarder.competition_leader_mapping) + await rewarder.update_scores(winning_evaluation_hotkey, competition_id) + print( + "Updated rewarder competition leader map:", + rewarder.competition_leader_mapping, + ) print("Updated rewarder scores:", rewarder.scores) await asyncio.sleep(10) + if __name__ == "__main__": + config = BaseNeuron.config() + bt.logging.set_config(config=config) # if True: # run them right away - # run_all_competitions(test_config, [],main_competitions_cfg) - - # else: # Run the scheduling coroutine - # scheduler_config = config_for_scheduler() - asyncio.run(competition_loop()) + path_config = path_config(None) + # config = config.merge(path_config) + # BaseNeuron.check_config(config) + bt.logging.set_config(config=config.logging) + bt.logging.info(config) + asyncio.run(run_all_competitions(test_config, [], main_competitions_cfg)) From 15abb623848b90723a8c1da2fbd2ec77200eb1cd Mon Sep 17 00:00:00 2001 From: Wojtek Jurkowlaniec Date: Wed, 4 Sep 2024 00:32:25 +0200 Subject: [PATCH 150/227] Update neurons/competition_runner.py --- neurons/competition_runner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/neurons/competition_runner.py b/neurons/competition_runner.py index 5611c07b..feb2fed9 100644 --- a/neurons/competition_runner.py +++ b/neurons/competition_runner.py @@ -38,7 +38,7 @@ def finish_run(self, competition_id: str): run.end_time = datetime.now(timezone.utc) def was_competition_already_executed( - self, last_minutes: int = 5, competition_id: str = None + self, last_minutes: int = 5, competition_id: str ): """Check if competition was executed in last minutes""" now_time = datetime.now(timezone.utc) From 5ff0f0088896307ceaf56bd6e281183ea01e4a7b Mon Sep 17 00:00:00 2001 From: Wojtek Jurkowlaniec Date: Wed, 4 Sep 2024 01:53:15 +0200 Subject: [PATCH 151/227] Update miner.md --- DOCS/miner.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/DOCS/miner.md b/DOCS/miner.md index de2df7e4..4b1ae8e2 100644 --- a/DOCS/miner.md +++ b/DOCS/miner.md @@ -2,9 +2,11 @@ ## Installation -- create virtualenv +- `git clone https://github.com/safe-scan-ai/cancer-ai` -`virtualenv venv --python=3.10 +- create python virtualenv + +`virtualenv venv --python=3.10` - activate it @@ -75,4 +77,4 @@ python neurons/miner.py \ --netuid \ --subtensor.network \ --logging.debug - ``` \ No newline at end of file + ``` From a5d377ec8c8044b0b501705553bd441c54874ab8 Mon Sep 17 00:00:00 2001 From: Wojtek Jurkowlaniec Date: Wed, 4 Sep 2024 01:54:40 +0200 Subject: [PATCH 152/227] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 0154a031..7442a0e3 100644 --- a/README.md +++ b/README.md @@ -172,7 +172,7 @@ Given the complexity of creating a state-of-the-art roleplay LLM, we plan to div # **⛏️ RUNNING MINER** -... +[Miner](DOCS/miner.md) # **🚀 GET INVOLVED** From 98b1876517083c94a542330e535bedc514f54727 Mon Sep 17 00:00:00 2001 From: LEMSTUDI0 <105062843+LEMSTUDI0@users.noreply.github.com> Date: Wed, 4 Sep 2024 01:57:30 +0200 Subject: [PATCH 153/227] Update README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 7442a0e3..1410d44f 100644 --- a/README.md +++ b/README.md @@ -171,8 +171,8 @@ Given the complexity of creating a state-of-the-art roleplay LLM, we plan to div ... # **⛏️ RUNNING MINER** - -[Miner](DOCS/miner.md) +To run a miner follow instructions in this link +[Running miner](DOCS/miner.md) # **🚀 GET INVOLVED** From 3c098ae430204c4aaa69313c4390f549054fd3bd Mon Sep 17 00:00:00 2001 From: LEMSTUDI0 <105062843+LEMSTUDI0@users.noreply.github.com> Date: Wed, 4 Sep 2024 01:58:05 +0200 Subject: [PATCH 154/227] Update README.md --- README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 1410d44f..82334441 100644 --- a/README.md +++ b/README.md @@ -171,8 +171,9 @@ Given the complexity of creating a state-of-the-art roleplay LLM, we plan to div ... # **⛏️ RUNNING MINER** -To run a miner follow instructions in this link -[Running miner](DOCS/miner.md) +To run a miner follow instructions in this link: + +[RUNNING MINER](DOCS/miner.md) # **🚀 GET INVOLVED** From 124844b500ba0f457f421163f949902bbfc2dc82 Mon Sep 17 00:00:00 2001 From: LEMSTUDI0 <105062843+LEMSTUDI0@users.noreply.github.com> Date: Wed, 4 Sep 2024 01:58:53 +0200 Subject: [PATCH 155/227] Update miner.md --- DOCS/miner.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DOCS/miner.md b/DOCS/miner.md index 4b1ae8e2..bebac256 100644 --- a/DOCS/miner.md +++ b/DOCS/miner.md @@ -1,4 +1,4 @@ -# Miner +# ⛏️ RUNNING MINER ## Installation From 1f0b8d65582d9d691e54e164aa922c65f551c283 Mon Sep 17 00:00:00 2001 From: Wojtek Jurkowlaniec Date: Wed, 4 Sep 2024 12:06:08 +0200 Subject: [PATCH 156/227] komit --- neurons/competition_runner.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/neurons/competition_runner.py b/neurons/competition_runner.py index feb2fed9..d6ad51a0 100644 --- a/neurons/competition_runner.py +++ b/neurons/competition_runner.py @@ -38,7 +38,7 @@ def finish_run(self, competition_id: str): run.end_time = datetime.now(timezone.utc) def was_competition_already_executed( - self, last_minutes: int = 5, competition_id: str + self, competition_id: str, last_minutes: int = 15 ): """Check if competition was executed in last minutes""" now_time = datetime.now(timezone.utc) @@ -105,7 +105,7 @@ async def run_competitions_tick( continue if run_log.was_competition_already_executed( - competition_id=competition_manager.competition_id + competition_id=competition_manager.competition_id, last_minutes=MINUTES_BACK ): bt.logging.info( f"Competition {competition_manager.competition_id} already executed, skipping" From 0c8e00e728163362ae984390c1df1b70b8dd5c6f Mon Sep 17 00:00:00 2001 From: LEMSTUDI0 <105062843+LEMSTUDI0@users.noreply.github.com> Date: Wed, 4 Sep 2024 13:50:21 +0200 Subject: [PATCH 157/227] Create prerequirements.md --- DOCS/prerequirements.md | 147 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 147 insertions(+) create mode 100644 DOCS/prerequirements.md diff --git a/DOCS/prerequirements.md b/DOCS/prerequirements.md new file mode 100644 index 00000000..39585623 --- /dev/null +++ b/DOCS/prerequirements.md @@ -0,0 +1,147 @@ +Bittensor +Bittensor is a mining network, similar to Bitcoin, that includes built-in incentives designed to encourage computers to provide access to machine learning models in an efficient and censorship-resistant manner. These models can be queried by users seeking outputs from the network, for instance; generating text, audio, and images, or for extracting numerical representations of these input types. Under the hood, Bittensor’s economic market, is facilitated by a blockchain token mechanism, through which producers (miners) and the verification of the work done by those miners (validators) are rewarded. Miners host, train or otherwise procure machine learning systems into the network as a means of fulfilling the verification problems defined by the validators, like the ability to generate responses from prompts i.e. “What is the capital of Texas?. + +The token based mechanism under which the miners are incentivized ensures that they are constantly driven to make their knowledge output more useful, in terms of speed, intelligence and diversity. The value generated by the network is distributed directly to the individuals producing that value, without intermediaries. Anyone can participate in this endeavour, extract value from the network, and govern Bittensor. The network is open to all participants, and no individual or group has full control over what is learned, who can profit from it, or who can access it. + +To learn more about Bittensor, please read our paper. + +Install +There are three ways to install Bittensor + +Through the installer: +$ /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/opentensor/bittensor/master/scripts/install.sh)" +With pip: +$ pip3 install bittensor +From source: +$ git clone https://github.com/opentensor/bittensor.git +$ python3 -m pip install -e bittensor/ +Using Conda (recommended for Apple M1): +$ conda env create -f ~/.bittensor/bittensor/scripts/environments/apple_m1_environment.yml +$ conda activate bittensor +To test your installation, type: + +$ btcli --help +or using python + +import bittensor +CUDA +If you anticipate using PoW registration for subnets or the faucet (only available on staging), please install cubit as well for your version of python. You can find the Opentensor cubit implementation and instructions here. + +For example with python 3.10: + +pip install https://github.com/opentensor/cubit/releases/download/v1.1.2/cubit-1.1.2-cp310-cp310-linux_x86_64.whl +Wallets +Wallets are the core ownership and identity technology around which all functions on Bittensor are carried out. Bittensor wallets consists of a coldkey and hotkey where the coldkey may contain many hotkeys, while each hotkey can only belong to a single coldkey. Coldkeys store funds securely, and operate functions such as transfers and staking, while hotkeys are used for all online operations such as signing queries, running miners and validating. + +Wallets can be created in two ways. + +Using the python-api +import bittensor +wallet = bittensor.wallet() +wallet.create_new_coldkey() +wallet.create_new_hotkey() +print (wallet) +"Wallet (default, default, ~/.bittensor/wallets/)" +Or using btcli +Use the subcommand wallet or it's alias w: + +$ btcli wallet new_coldkey + Enter wallet name (default): + + IMPORTANT: Store this mnemonic in a secure (preferably offline place), as anyone who has possession of this mnemonic can use it to regenerate the key and access your tokens. + The mnemonic to the new coldkey is: + **** *** **** **** ***** **** *** **** **** **** ***** ***** + You can use the mnemonic to recreate the key in case it gets lost. The command to use to regenerate the key using this mnemonic is: + btcli w regen_coldkey --mnemonic post maid erode shy captain verify scan shoulder brisk mountain pelican elbow + +$ btcli wallet new_hotkey + Enter wallet name (default): d1 + Enter hotkey name (default): + + IMPORTANT: Store this mnemonic in a secure (preferably offline place), as anyone who has possession of this mnemonic can use it to regenerate the key and access your tokens. + The mnemonic to the new hotkey is: + **** *** **** **** ***** **** *** **** **** **** ***** ***** + You can use the mnemonic to recreate the key in case it gets lost. The command to use to regenerate the key using this mnemonic is: + btcli w regen_hotkey --mnemonic total steak hour bird hedgehog trim timber can friend dry worry text +In both cases you should be able to view your keys by navigating to ~/.bittensor/wallets or viewed by running btcli wallet list + +$ tree ~/.bittensor/ + .bittensor/ # Bittensor, root directory. + wallets/ # The folder containing all bittensor wallets. + default/ # The name of your wallet, "default" + coldkey # You encrypted coldkey. + coldkeypub.txt # Your coldkey public address + hotkeys/ # The folder containing all of your hotkeys. + default # You unencrypted hotkey information. +Your default wallet Wallet (default, default, ~/.bittensor/wallets/) is always used unless you specify otherwise. Be sure to store your mnemonics safely. If you lose your password to your wallet, or the access to the machine where the wallet is stored, you can always regenerate the coldkey using the mnemonic you saved from above. + +$ btcli wallet regen_coldkey --mnemonic **** *** **** **** ***** **** *** **** **** **** ***** ***** +Using the cli +The Bittensor command line interface (btcli) is the primary command line tool for interacting with the Bittensor network. It can be used to deploy nodes, manage wallets, stake/unstake, nominate, transfer tokens, and more. + +Basic Usage +To get the list of all the available commands and their descriptions, you can use: + +btcli --help + +usage: btcli + +bittensor cli v{bittensor.__version__} + +commands: + subnets (s, subnet) - Commands for managing and viewing subnetworks. + root (r, roots) - Commands for managing and viewing the root network. + wallet (w, wallets) - Commands for managing and viewing wallets. + stake (st, stakes) - Commands for staking and removing stake from hotkey accounts. + sudo (su, sudos) - Commands for subnet management. + legacy (l) - Miscellaneous commands. +Example Commands +Viewing Senate Proposals +btcli root proposals +Viewing Senate Members +btcli root list_delegates +Viewing Proposal Votes +btcli root senate_vote --proposal=[PROPOSAL_HASH] +Registering for Senate +btcli root register +Leaving Senate +btcli root undelegate +Voting in Senate +btcli root senate_vote --proposal=[PROPOSAL_HASH] +Miscellaneous Commands +btcli legacy update +btcli legacy faucet +Managing Subnets +btcli subnets list +btcli subnets create +Managing Wallets +btcli wallet list +btcli wallet transfer +Note +Please replace the subcommands and arguments as necessary to suit your needs, and always refer to btcli --help or btcli --help for the most up-to-date and accurate information. + +For example: + +btcli subnets --help + +usage: btcli subnets [-h] {list,metagraph,lock_cost,create,register,pow_register,hyperparameters} ... + +positional arguments: + {list,metagraph,lock_cost,create,register,pow_register,hyperparameters} + Commands for managing and viewing subnetworks. + list List all subnets on the network. + metagraph View a subnet metagraph information. + lock_cost Return the lock cost to register a subnet. + create Create a new bittensor subnetwork on this chain. + register Register a wallet to a network. + pow_register Register a wallet to a network using PoW. + hyperparameters View subnet hyperparameters. + +options: + -h, --help show this help message and exit +Post-Installation Steps +To enable autocompletion for Bittensor CLI, run the following commands: + +btcli --print-completion bash >> ~/.bashrc # For Bash +btcli --print-completion zsh >> ~/.zshrc # For Zsh +source ~/.bashrc # Reload Bash configuration to take effect From 4367da97813fbb690b6ce9deddf2c5e196b6ced0 Mon Sep 17 00:00:00 2001 From: LEMSTUDI0 <105062843+LEMSTUDI0@users.noreply.github.com> Date: Wed, 4 Sep 2024 13:55:12 +0200 Subject: [PATCH 158/227] Update prerequirements.md --- DOCS/prerequirements.md | 167 +++++++++++++++++++++++++++++++--------- 1 file changed, 131 insertions(+), 36 deletions(-) diff --git a/DOCS/prerequirements.md b/DOCS/prerequirements.md index 39585623..ef22db8b 100644 --- a/DOCS/prerequirements.md +++ b/DOCS/prerequirements.md @@ -1,54 +1,90 @@ -Bittensor -Bittensor is a mining network, similar to Bitcoin, that includes built-in incentives designed to encourage computers to provide access to machine learning models in an efficient and censorship-resistant manner. These models can be queried by users seeking outputs from the network, for instance; generating text, audio, and images, or for extracting numerical representations of these input types. Under the hood, Bittensor’s economic market, is facilitated by a blockchain token mechanism, through which producers (miners) and the verification of the work done by those miners (validators) are rewarded. Miners host, train or otherwise procure machine learning systems into the network as a means of fulfilling the verification problems defined by the validators, like the ability to generate responses from prompts i.e. “What is the capital of Texas?. +# 💡BITTENSOR + +Bittensor is a mining network, similar to Bitcoin, that includes built-in incentives designed to encourage computers to provide access to machine learning models in an efficient and censorship-resistant manner. These models can be queried by users seeking outputs from the network, for instance; generating text, audio, and images, or for extracting numerical representations of these input types. Under the hood, Bittensor’s *economic market*, is facilitated by a blockchain token mechanism, through which producers (***miners***) and the verification of the work done by those miners (***validators***) are rewarded. Miners host, train or otherwise procure machine learning systems into the network as a means of fulfilling the verification problems defined by the validators, like the ability to generate responses from prompts i.e. “What is the capital of Texas?. The token based mechanism under which the miners are incentivized ensures that they are constantly driven to make their knowledge output more useful, in terms of speed, intelligence and diversity. The value generated by the network is distributed directly to the individuals producing that value, without intermediaries. Anyone can participate in this endeavour, extract value from the network, and govern Bittensor. The network is open to all participants, and no individual or group has full control over what is learned, who can profit from it, or who can access it. -To learn more about Bittensor, please read our paper. +To learn more about Bittensor, please read our [paper](https://bittensor.com/whitepaper). + +# **🛠️ INSTALL** -Install There are three ways to install Bittensor -Through the installer: +1. Through the installer: + +``` $ /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/opentensor/bittensor/master/scripts/install.sh)" -With pip: +``` + +1. With pip: + +``` $ pip3 install bittensor -From source: +``` + +1. From source: + +``` $ git clone https://github.com/opentensor/bittensor.git $ python3 -m pip install -e bittensor/ -Using Conda (recommended for Apple M1): +``` + +1. Using Conda (recommended for **Apple M1**): + +``` $ conda env create -f ~/.bittensor/bittensor/scripts/environments/apple_m1_environment.yml $ conda activate bittensor +``` + To test your installation, type: +``` $ btcli --help +``` + or using python +``` import bittensor -CUDA -If you anticipate using PoW registration for subnets or the faucet (only available on staging), please install cubit as well for your version of python. You can find the Opentensor cubit implementation and instructions here. +``` + +**CUDA** + +If you anticipate using PoW registration for subnets or the faucet (only available on staging), please install `cubit` as well for your version of python. You can find the Opentensor cubit implementation and instructions [here](https://github.com/opentensor/cubit). For example with python 3.10: +``` pip install https://github.com/opentensor/cubit/releases/download/v1.1.2/cubit-1.1.2-cp310-cp310-linux_x86_64.whl -Wallets +``` + +# **👛 WALLETS** + Wallets are the core ownership and identity technology around which all functions on Bittensor are carried out. Bittensor wallets consists of a coldkey and hotkey where the coldkey may contain many hotkeys, while each hotkey can only belong to a single coldkey. Coldkeys store funds securely, and operate functions such as transfers and staking, while hotkeys are used for all online operations such as signing queries, running miners and validating. Wallets can be created in two ways. -Using the python-api +1. Using the python-api + +``` import bittensor wallet = bittensor.wallet() wallet.create_new_coldkey() wallet.create_new_hotkey() print (wallet) "Wallet (default, default, ~/.bittensor/wallets/)" -Or using btcli -Use the subcommand wallet or it's alias w: +``` + +1. Or using btcli + +> Use the subcommand wallet or it's alias w: +> +``` $ btcli wallet new_coldkey - Enter wallet name (default): + Enter wallet name (default): - IMPORTANT: Store this mnemonic in a secure (preferably offline place), as anyone who has possession of this mnemonic can use it to regenerate the key and access your tokens. + IMPORTANT: Store this mnemonic in a secure (preferably offline place), as anyone who has possession of this mnemonic can use it to regenerate the key and access your tokens. The mnemonic to the new coldkey is: **** *** **** **** ***** **** *** **** **** **** ***** ***** You can use the mnemonic to recreate the key in case it gets lost. The command to use to regenerate the key using this mnemonic is: @@ -56,15 +92,18 @@ $ btcli wallet new_coldkey $ btcli wallet new_hotkey Enter wallet name (default): d1 - Enter hotkey name (default): + Enter hotkey name (default): - IMPORTANT: Store this mnemonic in a secure (preferably offline place), as anyone who has possession of this mnemonic can use it to regenerate the key and access your tokens. + IMPORTANT: Store this mnemonic in a secure (preferably offline place), as anyone who has possession of this mnemonic can use it to regenerate the key and access your tokens. The mnemonic to the new hotkey is: **** *** **** **** ***** **** *** **** **** **** ***** ***** You can use the mnemonic to recreate the key in case it gets lost. The command to use to regenerate the key using this mnemonic is: btcli w regen_hotkey --mnemonic total steak hour bird hedgehog trim timber can friend dry worry text -In both cases you should be able to view your keys by navigating to ~/.bittensor/wallets or viewed by running btcli wallet list +``` +In both cases you should be able to view your keys by navigating to ~/.bittensor/wallets or viewed by running `btcli wallet list` + +``` $ tree ~/.bittensor/ .bittensor/ # Bittensor, root directory. wallets/ # The folder containing all bittensor wallets. @@ -73,15 +112,23 @@ $ tree ~/.bittensor/ coldkeypub.txt # Your coldkey public address hotkeys/ # The folder containing all of your hotkeys. default # You unencrypted hotkey information. -Your default wallet Wallet (default, default, ~/.bittensor/wallets/) is always used unless you specify otherwise. Be sure to store your mnemonics safely. If you lose your password to your wallet, or the access to the machine where the wallet is stored, you can always regenerate the coldkey using the mnemonic you saved from above. +``` + +Your default wallet `Wallet (default, default, ~/.bittensor/wallets/)` is always used unless you specify otherwise. Be sure to store your mnemonics safely. If you lose your password to your wallet, or the access to the machine where the wallet is stored, you can always regenerate the coldkey using the mnemonic you saved from above. +``` $ btcli wallet regen_coldkey --mnemonic **** *** **** **** ***** **** *** **** **** **** ***** ***** -Using the cli -The Bittensor command line interface (btcli) is the primary command line tool for interacting with the Bittensor network. It can be used to deploy nodes, manage wallets, stake/unstake, nominate, transfer tokens, and more. +``` + +**Using the cli** + +The Bittensor command line interface (`btcli`) is the primary command line tool for interacting with the Bittensor network. It can be used to deploy nodes, manage wallets, stake/unstake, nominate, transfer tokens, and more. + +**Basic Usage** -Basic Usage To get the list of all the available commands and their descriptions, you can use: +``` btcli --help usage: btcli @@ -95,33 +142,74 @@ commands: stake (st, stakes) - Commands for staking and removing stake from hotkey accounts. sudo (su, sudos) - Commands for subnet management. legacy (l) - Miscellaneous commands. -Example Commands -Viewing Senate Proposals +``` + +**Example Commands** + +**Viewing Senate Proposals** + +``` btcli root proposals -Viewing Senate Members +``` + +**Viewing Senate Members** + +``` btcli root list_delegates -Viewing Proposal Votes +``` + +**Viewing Proposal Votes** + +``` btcli root senate_vote --proposal=[PROPOSAL_HASH] -Registering for Senate +``` + +**Registering for Senate** + +``` btcli root register -Leaving Senate +``` + +**Leaving Senate** + +``` btcli root undelegate -Voting in Senate +``` + +**Voting in Senate** + +``` btcli root senate_vote --proposal=[PROPOSAL_HASH] -Miscellaneous Commands +``` + +**Miscellaneous Commands** + +``` btcli legacy update btcli legacy faucet -Managing Subnets +``` + +**Managing Subnets** + +``` btcli subnets list btcli subnets create -Managing Wallets +``` + +**Managing Wallets** + +``` btcli wallet list btcli wallet transfer -Note -Please replace the subcommands and arguments as necessary to suit your needs, and always refer to btcli --help or btcli --help for the most up-to-date and accurate information. +``` + +**Note** + +Please replace the subcommands and arguments as necessary to suit your needs, and always refer to `btcli --help` or `btcli --help` for the most up-to-date and accurate information. For example: +``` btcli subnets --help usage: btcli subnets [-h] {list,metagraph,lock_cost,create,register,pow_register,hyperparameters} ... @@ -139,9 +227,16 @@ positional arguments: options: -h, --help show this help message and exit -Post-Installation Steps +``` + +**Post-Installation Steps** + To enable autocompletion for Bittensor CLI, run the following commands: +``` btcli --print-completion bash >> ~/.bashrc # For Bash btcli --print-completion zsh >> ~/.zshrc # For Zsh source ~/.bashrc # Reload Bash configuration to take effect +``` + +# From c253ee7c15d804bb0418a8dff9fda0b2896e7be9 Mon Sep 17 00:00:00 2001 From: notbulubula Date: Wed, 4 Sep 2024 12:49:41 +0200 Subject: [PATCH 159/227] Adding WANDB setup --- README.md | 67 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) diff --git a/README.md b/README.md index ff949a65..f59b5947 100644 --- a/README.md +++ b/README.md @@ -172,6 +172,73 @@ Given the complexity of creating a state-of-the-art roleplay LLM, we plan to div - [ ] Make competitions for breast cancer # **📊 SETUP WandB (HIGHLY RECOMMENDED - VALIDATORS PLEASE READ)** + +WandB is a valuable tool for tracking and visualizing machine learning experiments, and it helps log and monitor key metrics for miners and validators. + +Here’s a quick guide to setting up your WandB + +## **Instaliation** +To get started with WandB, you need to install the WandB Python package. + +``` +pip install wandb +``` + +## **Obtaining API key** + +1. Log into your **Weights & Biases** account in a browser. +2. Go to user settings and scroll down to **API keys** section. +3. Copy your API key to procede with next steps. + +## **Setting up the API key** + +After obtaining your API key, you need to set it up in your environment so that WandB can authenticate your account. + +1. Log into WANDB by running following command in your terminal: +``` +wandb login +``` +2. Enter your API key and press Enter + +## **Set API Key as Environment Variable (OPTIONAL)** +If you prefer not to log in every time, you can set your API key as an environment variable. + +### **Linux** + +To set the WANDB_API_KEY environment variable permanently on Linux, you’ll need to add it to your .bashrc (or .bash_profile, .profile, depending on your distribution and shell). +``` +echo 'export WANDB_API_KEY=your_api_key' >> ~/.bashrc +source ~/.bashrc +``` +Replace your_api_key with API key copied from Weights & Biases + +**Verification** + +``` +echo $WANDB_API_KEY +``` + +### Windows + +To set it permanently (system-wide), use the following steps: + + - Open Environment Variables Dialog: + + - Right-click on This PC or Computer on the Desktop or in File Explorer and select Properties. + - Click on Advanced system settings. + - In the System Properties window, click on the Environment Variables button. +- Add New System Variable: + + - In the Environment Variables window, click on New under the System variables section. + - Set the Variable name to WANDB_API_KEY and Variable value to your API key. + - Click OK to close all dialogs. + +**Verification** +``` +echo %WANDB_API_KEY% +``` + + # **👍 RUNNING VALIDATOR** ... From fd7667a6b3d451b96274b6331ff6cb1e6b44cc93 Mon Sep 17 00:00:00 2001 From: Konrad Date: Wed, 4 Sep 2024 13:58:52 +0200 Subject: [PATCH 160/227] some timing adjustments --- cancer_ai/base/base_validator.py | 1 - neurons/competition_runner.py | 1 + neurons/validator.py | 2 ++ 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/cancer_ai/base/base_validator.py b/cancer_ai/base/base_validator.py index e77b7a3e..5aa3761b 100644 --- a/cancer_ai/base/base_validator.py +++ b/cancer_ai/base/base_validator.py @@ -77,7 +77,6 @@ def __init__(self, config=None): competition_leader_map={}, hotkey_score_map={} ) self.load_state() - # Init sync with the network. Updates the metagraph. self.sync() diff --git a/neurons/competition_runner.py b/neurons/competition_runner.py index d6ad51a0..dba77d1e 100644 --- a/neurons/competition_runner.py +++ b/neurons/competition_runner.py @@ -130,6 +130,7 @@ async def run_competitions_tick( bt.logging.debug( f"Did not find any competitions to run for past {MINUTES_BACK} minutes" ) + asyncio.sleep(60) async def competition_loop_not_used( diff --git a/neurons/validator.py b/neurons/validator.py index c400aed9..c592fd87 100644 --- a/neurons/validator.py +++ b/neurons/validator.py @@ -83,6 +83,8 @@ async def competition_loop_tick( ] self.save_state() + asyncio.sleep(60) + def save_state(self): """Saves the state of the validator to a file.""" bt.logging.info("Saving validator state.") From f8caf35da29f8c1277f9cdaf8092ef2b905319b8 Mon Sep 17 00:00:00 2001 From: LEMSTUDI0 <105062843+LEMSTUDI0@users.noreply.github.com> Date: Wed, 4 Sep 2024 14:00:28 +0200 Subject: [PATCH 161/227] Update README.md --- README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.md b/README.md index 82334441..0cf9eb13 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,7 @@ - [💰 Tokenomy & Economy](#-tokenomy--economy) - [👨‍👨‍👦‍👦 Team Composition](#-team-composition) - [🛣️ Roadmap](#roadmap) +- [✅ PRE REQUIRMENTS](#pre-prerequirments) - [👍 Running Validator](#-running-validator) - [⛏️ Running Miner](#-running-miner) - [🚀 Get invloved](#-get-involved) @@ -164,6 +165,10 @@ Given the complexity of creating a state-of-the-art roleplay LLM, we plan to div - [ ] Optimize skin cancer detection models to create one mixture-of-experts model which will run on mobile devices - [ ] Start the process for certifying models - FDA approval - [ ] Make competitions for breast cancer +# **✅ PRE REQUIRMENTS +To install BITTENSOR and set up a wallet follow instructions in this link: + +[PRE REQUIRMENTS](DOCS/prerequirments.md) # **📊 SETUP WandB (HIGHLY RECOMMENDED - VALIDATORS PLEASE READ)** # **👍 RUNNING VALIDATOR** From 42a094b52ed5f58e4e9f77863208425b634725d3 Mon Sep 17 00:00:00 2001 From: LEMSTUDI0 <105062843+LEMSTUDI0@users.noreply.github.com> Date: Wed, 4 Sep 2024 14:01:22 +0200 Subject: [PATCH 162/227] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 0cf9eb13..f2b1da3f 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ - [💰 Tokenomy & Economy](#-tokenomy--economy) - [👨‍👨‍👦‍👦 Team Composition](#-team-composition) - [🛣️ Roadmap](#roadmap) -- [✅ PRE REQUIRMENTS](#pre-prerequirments) +- [✅ Pre requirments](#-pre-requirments) - [👍 Running Validator](#-running-validator) - [⛏️ Running Miner](#-running-miner) - [🚀 Get invloved](#-get-involved) From ed41d3cc0cfe58578d44e64851f76ee91bbac5e6 Mon Sep 17 00:00:00 2001 From: Konrad Date: Wed, 4 Sep 2024 14:07:47 +0200 Subject: [PATCH 163/227] auto-validator docs --- scripts/start_validator.py | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/scripts/start_validator.py b/scripts/start_validator.py index 4e9bf235..f87617ef 100755 --- a/scripts/start_validator.py +++ b/scripts/start_validator.py @@ -1,3 +1,21 @@ +""" +The script was based on the original script from the Pretraining Subnet repository. + +This script runs a validator process and automatically updates it when a new version is released. +Command-line arguments will be forwarded to validator (`neurons/validator.py`), so you can pass +them like this: + python3 scripts/start_validator.py --wallet.name=my-wallet +Auto-updates are enabled by default and will make sure that the latest version is always running +by pulling the latest version from git and upgrading python packages. This is done periodically. +Local changes may prevent the update, but they will be preserved. + +The script will use the same virtual environment as the one used to run it. If you want to run +validator within virtual environment, run this auto-update script from the virtual environment. + +Pm2 is required for this script. This script will start a pm2 process using the name provided by +the --pm2_name argument. +""" + import argparse import logging import subprocess @@ -176,11 +194,11 @@ def main(pm2_name: str, args_namespace: Namespace) -> None: ) parser.add_argument( - "--wallet.name", default="validator", help="Name of the wallet." + "--wallet.name", default="", help="Name of the wallet." ) parser.add_argument( - "--wallet.hotkey", default="default", help="Name of the hotkey." + "--wallet.hotkey", default="", help="Name of the hotkey." ) parser.add_argument( From 61af1bd897a1401191f1dff2c9bbf6566a1d4276 Mon Sep 17 00:00:00 2001 From: Konrad Date: Wed, 4 Sep 2024 14:34:12 +0200 Subject: [PATCH 164/227] validator docs --- Validator.md | 71 ++++++++++++++++++++++++++++++++++++++ scripts/start_validator.py | 12 ++++--- 2 files changed, 78 insertions(+), 5 deletions(-) create mode 100644 Validator.md diff --git a/Validator.md b/Validator.md new file mode 100644 index 00000000..59e95cb9 --- /dev/null +++ b/Validator.md @@ -0,0 +1,71 @@ +# Validator Script Documentation + +This documentation provides an overview of the validator script, its functionality, requirements, and usage instructions. + +## Overview + +The validator script is designed to run a validator process and automatically update it whenever a new version is released. This script was adapted from the original script in the Pretraining Subnet repository. + +Key features of the script include: +- **Automatic Updates**: The script checks for updates periodically and ensures that the latest version of the validator is running by pulling the latest code from the repository and upgrading necessary Python packages. +- **Command-Line Argument Compatibility**: The script now properly handles custom command-line arguments and forwards them to the validator (`neurons/validator.py`). +- **Virtual Environment Support**: The script runs within the same virtual environment that it is executed in, ensuring compatibility and ease of use. +- **PM2 Process Management**: The script uses PM2, a process manager, to manage the validator process. + +## Prerequisites + +- **Python 3.x**: The script is written in Python and requires Python 3.x to run. +- **PM2**: PM2 must be installed and available on your system. It is used to manage the validator process. + +## Installation and Setup + +1. **Clone the Repository**: Make sure you have cloned the repository containing this script and have navigated to the correct directory. + +2. **Install PM2**: Ensure PM2 is installed globally on your system. If it isn't, you can install it using npm: + +``` + npm install -g pm2 +``` + +3. **Set Up Virtual Environment**: If you wish to run the script within a virtual environment, create and activate the environment before running the script: +``` + python3 -m venv venv + source venv/bin/activate # On Windows use `venv\Scripts\activate` +``` + +4. **Install Required Python Packages**: Install any required Python packages listed in requirements.txt: +``` +pip install -r requirements.txt +``` + + +## Usage +To run the validator script, use the following command: +**TODO(DEV): CHANGE THESE VALUES BEFORE THE RELEASE TO MAINNET NETUID!!!** + +``` +python3 scripts/start_validator.py --wallet.name=my-wallet --wallet.hotkey=my-hotkey --netuid=163 + +``` + +## Command-Line Arguments + +- `--pm2_name`: Specifies the name of the PM2 process. Default is `"cancer_ai_vali"`. +- `--wallet.name`: Specifies the wallet name to be used by the validator. +- `--wallet.hotkey`: Specifies the hotkey associated with the wallet. +- `--subtensor.network`: Specifies the network name. Default is `"test"`. +- `--netuid`: Specifies the Netuid of the network. Default is `"163"`. +- `--logging.debug`: Enables debug logging if set to `1`. Default is `1`. + + +## How It Works + +1. **Start Validator Process**: The script starts the validator process using PM2, based on the provided PM2 process name. +2. **Periodic Updates**: The script periodically checks for updates (every 5 minutes by default) by fetching the latest code from the git repository. +3. **Handle Updates**: If a new version is detected, the script pulls the latest changes, upgrades the Python packages, stops the current validator process, and restarts it with the updated code. +4. **Local Changes**: If there are local changes in the repository that conflict with the updates, the script attempts to rebase them. If conflicts persist, the rebase is aborted to preserve the local changes. + +## Notes + +- **Local Changes**: If you have made local changes to the codebase, the auto-update feature will attempt to preserve them. However, conflicts might require manual resolution. +- **Environment**: The script uses the environment from which it is executed, so ensure all necessary environment variables and dependencies are correctly configured. diff --git a/scripts/start_validator.py b/scripts/start_validator.py index f87617ef..90bd084e 100755 --- a/scripts/start_validator.py +++ b/scripts/start_validator.py @@ -29,7 +29,7 @@ from pathlib import Path log = logging.getLogger(__name__) -UPDATES_CHECK_TIME = timedelta(seconds=30) +UPDATES_CHECK_TIME = timedelta(minutes=5) CURRENT_WORKING_DIR = Path(__file__).parent.parent ECOSYSTEM_CONFIG_PATH = CURRENT_WORKING_DIR / "ecosystem.config.js" # Path to the pm2 ecosystem config file @@ -135,7 +135,7 @@ def upgrade_packages() -> None: log.error("Failed to upgrade packages, proceeding anyway. %s", exc) -def main(pm2_name: str, args_namespace: Namespace) -> None: +def main(pm2_name: str, args_namespace: Namespace, extra_args: List[str]) -> None: """ Run the validator process and automatically update it when a new version is released. This will check for updates every `UPDATES_CHECK_TIME` and update the validator @@ -149,6 +149,8 @@ def main(pm2_name: str, args_namespace: Namespace) -> None: if not isinstance(value, bool): args_list.append(str(value)) + args_list.extend(extra_args) + validator = start_validator_process(pm2_name, args_list) current_version = latest_version = get_version() log.info("Current version: %s", current_version) @@ -194,11 +196,11 @@ def main(pm2_name: str, args_namespace: Namespace) -> None: ) parser.add_argument( - "--wallet.name", default="", help="Name of the wallet." + "--wallet.name", default="validator", help="Name of the wallet." ) parser.add_argument( - "--wallet.hotkey", default="", help="Name of the hotkey." + "--wallet.hotkey", default="default", help="Name of the hotkey." ) parser.add_argument( @@ -214,4 +216,4 @@ def main(pm2_name: str, args_namespace: Namespace) -> None: ) flags, extra_args = parser.parse_known_args() - main(flags.pm2_name, flags) + main(flags.pm2_name, flags, extra_args) From 6cb4396a226fcbf3b3d110652365b62972577c53 Mon Sep 17 00:00:00 2001 From: Konrad Date: Wed, 4 Sep 2024 14:37:05 +0200 Subject: [PATCH 165/227] adjusted to Wojtek's comments --- Validator.md | 2 +- scripts/start_validator.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/Validator.md b/Validator.md index 59e95cb9..0cb8225b 100644 --- a/Validator.md +++ b/Validator.md @@ -4,7 +4,7 @@ This documentation provides an overview of the validator script, its functionali ## Overview -The validator script is designed to run a validator process and automatically update it whenever a new version is released. This script was adapted from the original script in the Pretraining Subnet repository. +The validator script is designed to run a validator process and automatically update it whenever a new version is released. This script was adapted from the [original script](https://github.com/macrocosm-os/pretraining/blob/main/scripts/start_validator.py) in the Pretraining Subnet repository. Key features of the script include: - **Automatic Updates**: The script checks for updates periodically and ensures that the latest version of the validator is running by pulling the latest code from the repository and upgrading necessary Python packages. diff --git a/scripts/start_validator.py b/scripts/start_validator.py index 90bd084e..8d669122 100755 --- a/scripts/start_validator.py +++ b/scripts/start_validator.py @@ -1,5 +1,6 @@ """ The script was based on the original script from the Pretraining Subnet repository. +https://github.com/macrocosm-os/pretraining/blob/main/scripts/start_validator.py This script runs a validator process and automatically updates it when a new version is released. Command-line arguments will be forwarded to validator (`neurons/validator.py`), so you can pass From f00ad3d51bf724e1958e416498d034755fe20d4d Mon Sep 17 00:00:00 2001 From: Konrad Date: Wed, 4 Sep 2024 15:06:36 +0200 Subject: [PATCH 166/227] Adjusted miner readme --- DOCS/miner.md | 90 ++++++++++++++++++++++++++++++++++----------------- 1 file changed, 60 insertions(+), 30 deletions(-) diff --git a/DOCS/miner.md b/DOCS/miner.md index de2df7e4..1eee1523 100644 --- a/DOCS/miner.md +++ b/DOCS/miner.md @@ -1,49 +1,75 @@ -# Miner +# Miner Script Documentation -## Installation +This documentation provides an overview of the miner script, its functionality, requirements, and usage instructions. -- create virtualenv +## Overview -`virtualenv venv --python=3.10 +The miner script is designed to manage models, evaluate them locally, and upload them to HuggingFace, as well as submit models to validators within a specified network. -- activate it +Key features of the script include: +- **Local Model Evaluation**: Allows you to evaluate models against a dataset locally. +- **HuggingFace Upload**: Compresses and uploads models and code to HuggingFace. +- **Model Submission to Validators**: Saves model information in the metagraph, enabling validators to test the models. -`source venv/bin/activate` +## Prerequisites -- install requirements +- **Python 3.10**: The script is written in Python and requires Python 3.10 to run. +- **Virtual Environment**: It's recommended to run the script within a virtual environment to manage dependencies. -`pip install -r requirements.txt` +## Installation -## Run +1. **Create a Virtual Environment** +Set up a virtual environment for the project: -Prerequirements + ```bash + virtualenv venv --python=3.10 + source venv/bin/activate + ``` -- make sure you are in base directory of the project -- activate your virtualenv -- run `export PYTHONPATH="${PYTHONPATH}:./"` +## Install Required Python Packages +Install any required Python packages listed in `requirements.txt`: +``` +pip install -r requirements.txt +``` + +## Usage + +### Prerequisites -### Evaluate model localy +Before running the script, ensure the following: + +- You are in the base directory of the project. +- Your virtual environment is activated. +- Run the following command to set the `PYTHONPATH`: + +``` +export PYTHONPATH="${PYTHONPATH}:./" +``` -This mode will do following things -- download dataset -- load your model -- prepare data for executing -- logs evaluation results +### Evaluate Model Locally +This mode performs the following tasks: +- Downloads the dataset. +- Loads your model. +- Prepares data for execution. +- Logs evaluation results. +To evaluate a model locally, use the following command: -`python neurons/miner.py --action evaluate --competition_id --model_path ` +``` +python neurons/miner.py --action evaluate --competition_id --model_path +``` If flag `--clean-after-run` is supplied, it will delete dataset after evaluating the model ### Upload to HuggingFace -- compresses code provided by --code-path -- uploads model and code to HuggingFace +This mode compresses the code provided by `--code-path` and uploads the model and code to HuggingFace. -`python neurons/miner.py --action upload --competition_id melanoma-1 --model_path test_model.onnx --hf_model_name file_name.zip --hf_repo_id repo/id --hf_token TOKEN` -```bash +To upload to HuggingFace, use the following command: + +``` python neurons/miner.py \ --action upload \ --competition_id \ @@ -55,13 +81,13 @@ python neurons/miner.py \ --logging.debug ``` +### Submit Model to Validators -### Send model to validators +This mode saves model information in the metagraph, allowing validators to retrieve information about your model for testing. -- saves model information in metagraph -- validator can get information about your model to test it +To submit a model to validators, use the following command: -```bash +``` python neurons/miner.py \ --action submit \ --model_path \ @@ -74,5 +100,9 @@ python neurons/miner.py \ --wallet.hotkey \ --netuid \ --subtensor.network \ - --logging.debug - ``` \ No newline at end of file + --logging.debug +``` + +## Notes +- **Environment**: The script uses the environment from which it is executed, so ensure all necessary environment variables and dependencies are correctly configured. +- **Model Evaluation**: The `evaluate` action downloads necessary datasets and runs the model locally; ensure that your local environment has sufficient resources. From 28268a5308c8b1dd2605fe2f3c2cfd197997163a Mon Sep 17 00:00:00 2001 From: Konrad Date: Wed, 4 Sep 2024 15:33:39 +0200 Subject: [PATCH 167/227] some time adjustments --- neurons/competition_runner.py | 2 +- neurons/validator.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/neurons/competition_runner.py b/neurons/competition_runner.py index dba77d1e..280463fb 100644 --- a/neurons/competition_runner.py +++ b/neurons/competition_runner.py @@ -130,7 +130,7 @@ async def run_competitions_tick( bt.logging.debug( f"Did not find any competitions to run for past {MINUTES_BACK} minutes" ) - asyncio.sleep(60) + await asyncio.sleep(60) async def competition_loop_not_used( diff --git a/neurons/validator.py b/neurons/validator.py index c592fd87..5d5c3657 100644 --- a/neurons/validator.py +++ b/neurons/validator.py @@ -83,7 +83,7 @@ async def competition_loop_tick( ] self.save_state() - asyncio.sleep(60) + await asyncio.sleep(60) def save_state(self): """Saves the state of the validator to a file.""" From 23baf3dbe027e1b8be00bd354567822c9f5c6851 Mon Sep 17 00:00:00 2001 From: Wojtek Jurkowlaniec Date: Wed, 4 Sep 2024 16:13:17 +0200 Subject: [PATCH 168/227] more wandb logging --- cancer_ai/validator/competition_manager.py | 11 +---- neurons/competition_config.json | 4 +- neurons/competition_runner.py | 23 +++++----- neurons/validator.py | 49 +++++++++++++++++----- 4 files changed, 54 insertions(+), 33 deletions(-) diff --git a/cancer_ai/validator/competition_manager.py b/cancer_ai/validator/competition_manager.py index 54caa04f..0bdd07f7 100644 --- a/cancer_ai/validator/competition_manager.py +++ b/cancer_ai/validator/competition_manager.py @@ -88,7 +88,7 @@ def __init__( def log_results_to_wandb( self, hotkey: str, evaluation_result: ModelEvaluationResult ) -> None: - wandb.init(project=self.config.wandb_project_name) + wandb.init(project=self.competition_id, group="model_evaluation") wandb.log( { "hotkey": hotkey, @@ -106,14 +106,7 @@ def log_results_to_wandb( ) wandb.finish() - bt.logging.info("Logged results to wandb") - bt.logging.info("Hotkey: ", hotkey) - bt.logging.info("Tested entries: ", evaluation_result.tested_entries) - bt.logging.info("Model test run time: ", evaluation_result.run_time_s) - bt.logging.info("Accuracy: ", evaluation_result.accuracy) - bt.logging.info("Precision: ", evaluation_result.precision) - bt.logging.info("Recall: ", evaluation_result.recall) - bt.logging.info("roc_auc: ", evaluation_result.roc_auc) + bt.logging.info("Results: ", evaluation_result) def get_state(self): return { diff --git a/neurons/competition_config.json b/neurons/competition_config.json index 9a9719cc..739e1e2c 100644 --- a/neurons/competition_config.json +++ b/neurons/competition_config.json @@ -3,7 +3,7 @@ "competition_id": "melanoma-1", "category": "skin", "evaluation_times": [ - "23:42" + "14:11" ], "dataset_hf_repo": "safescanai/test_dataset", "dataset_hf_filename": "skin_melanoma.zip", @@ -13,7 +13,7 @@ "competition_id": "melanoma-testnet", "category": "skin", "evaluation_times": [ - "23:44" + "13:53" ], "dataset_hf_repo": "safescanai/test_dataset", "dataset_hf_filename": "skin_melanoma.zip", diff --git a/neurons/competition_runner.py b/neurons/competition_runner.py index 280463fb..849dad6e 100644 --- a/neurons/competition_runner.py +++ b/neurons/competition_runner.py @@ -7,6 +7,7 @@ import bittensor as bt from typing import List, Tuple, Dict from cancer_ai.validator.rewarder import Rewarder, WinnersMapping, CompetitionLeader +import wandb # from cancer_ai.utils.config import config @@ -80,9 +81,9 @@ def config_for_scheduler( async def run_competitions_tick( - competition_times: CompetitionSchedulerConfig, + competition_scheduler: CompetitionSchedulerConfig, run_log: CompetitionRunLog, -) -> Tuple[str, str] | None: +) -> Tuple[str, str] | Tuple[None, None]: """Checks if time is right and launches competition, returns winning hotkey and Competition ID. Should be run each minute.""" # getting current time @@ -95,15 +96,13 @@ async def run_competitions_tick( check_time = ( datetime.combine(datetime.today(), now_time) - timedelta(minutes=i) ).time() + competition_manager = competition_scheduler.get(check_time) + if not competition_manager: + return (None, None) - # bt.logging.debug(f"Checking competitions at {check_time}") - if competition_manager := competition_times.get(check_time): - bt.logging.debug( - f"Found competition {competition_manager.competition_id} at {check_time}" - ) - else: - continue - + bt.logging.debug( + f"Found competition {competition_manager.competition_id} at {check_time}" + ) if run_log.was_competition_already_executed( competition_id=competition_manager.competition_id, last_minutes=MINUTES_BACK ): @@ -113,6 +112,8 @@ async def run_competitions_tick( continue bt.logging.info(f"Running {competition_manager.competition_id} at {now_time}") + + run_log.add_run( CompetitionRun( competition_id=competition_manager.competition_id, @@ -121,7 +122,6 @@ async def run_competitions_tick( ) winning_evaluation_hotkey = await competition_manager.evaluate() run_log.finish_run(competition_manager.competition_id) - # TODO log last run to WANDB return ( winning_evaluation_hotkey, competition_manager.competition_id, @@ -131,6 +131,7 @@ async def run_competitions_tick( f"Did not find any competitions to run for past {MINUTES_BACK} minutes" ) await asyncio.sleep(60) + return (None, None) async def competition_loop_not_used( diff --git a/neurons/validator.py b/neurons/validator.py index 5d5c3657..725cc527 100644 --- a/neurons/validator.py +++ b/neurons/validator.py @@ -19,10 +19,13 @@ import time -import bittensor as bt import asyncio import os +import traceback + +import bittensor as bt import numpy as np +import wandb from cancer_ai.validator.rewarder import WinnersMapping, Rewarder, Score from cancer_ai.base.base_validator import BaseValidatorNeuron @@ -47,26 +50,51 @@ def __init__(self, config=None): async def concurrent_forward(self): coroutines = [ - self.competition_loop_tick(self.competition_scheduler), + self.competition_loop_tick(), ] await asyncio.gather(*coroutines) - async def competition_loop_tick( - self, scheduler_config: dict[str, CompetitionManager] - ): + async def competition_loop_tick(self): bt.logging.debug("Run log", self.run_log) - competition_result = await run_competitions_tick(scheduler_config, self.run_log) + try: + winning_hotkey, competition_id = await run_competitions_tick( + self.competition_scheduler, self.run_log + ) + except Exception as e: + formatted_traceback = traceback.format_exc() + bt.logging.error(f"Error running competition: {formatted_traceback}") + wandb.init(project="competition_id", group="competition_evaluation") + wandb.log( + { + "winning_evaluation_hotkey": "", + "run_time": "", + "validator_id": self.wallet.hotkey.ss58_address, + "errors": str(formatted_traceback), + } + ) + wandb.finish() + return - if not competition_result: + if not winning_hotkey: return - bt.logging.debug(f"Competition result: {competition_result}") + wandb.init(project=competition_id, group="competition_evaluation") + wandb.log( + { + "winning_hotkey": winning_hotkey, + "run_time": self.run_log.runs[-1].end_time + - self.run_log.runs[-1].start_time, + "validator_id": self.wallet.hotkey.ss58_address, + "errors": "", + } + ) + wandb.finish() - winning_evaluation_hotkey, competition_id = competition_result + bt.logging.info(f"Competition result for {competition_id}: {winning_hotkey}") # update the scores - await self.rewarder.update_scores(winning_evaluation_hotkey, competition_id) + await self.rewarder.update_scores(winning_hotkey, competition_id) self.winners_mapping = WinnersMapping( competition_leader_map=self.rewarder.competition_leader_mapping, hotkey_score_map=self.rewarder.scores, @@ -83,7 +111,6 @@ async def competition_loop_tick( ] self.save_state() - await asyncio.sleep(60) def save_state(self): """Saves the state of the validator to a file.""" From 192d9e03170c391e1e6ea6f15c11f3c6bea398ac Mon Sep 17 00:00:00 2001 From: Wojtek Jurkowlaniec Date: Wed, 4 Sep 2024 16:20:56 +0200 Subject: [PATCH 169/227] small fix --- cancer_ai/validator/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cancer_ai/validator/utils.py b/cancer_ai/validator/utils.py index c52d7e8d..bf4f285c 100644 --- a/cancer_ai/validator/utils.py +++ b/cancer_ai/validator/utils.py @@ -24,7 +24,7 @@ async def wrapper(*args, **kwargs): result = await func(*args, **kwargs) end_time = time.time() module_name = func.__module__ - bt.logging.debug(f"'{module_name}.{func.__name__}' took {end_time - start_time:.4f}s") + bt.logging.trace(f"'{module_name}.{func.__name__}' took {end_time - start_time:.4f}s") return result return wrapper From 9da512342b5d07823dd597757d799eb955db6901 Mon Sep 17 00:00:00 2001 From: Wojtek Jurkowlaniec Date: Wed, 4 Sep 2024 18:44:19 +0200 Subject: [PATCH 170/227] logging errors and whole runs --- cancer_ai/validator/competition_manager.py | 5 +++-- neurons/competition_runner.py | 6 ++---- neurons/validator.py | 15 +++++++++------ 3 files changed, 14 insertions(+), 12 deletions(-) diff --git a/cancer_ai/validator/competition_manager.py b/cancer_ai/validator/competition_manager.py index 0bdd07f7..1c625283 100644 --- a/cancer_ai/validator/competition_manager.py +++ b/cancer_ai/validator/competition_manager.py @@ -28,6 +28,7 @@ COMPETITION_HANDLER_MAPPING = { "melanoma-1": MelanomaCompetitionHandler, "melanoma-testnet": MelanomaCompetitionHandler, + "melanoma-7": MelanomaCompetitionHandler, } @@ -88,7 +89,7 @@ def __init__( def log_results_to_wandb( self, hotkey: str, evaluation_result: ModelEvaluationResult ) -> None: - wandb.init(project=self.competition_id, group="model_evaluation") + wandb.init(reinit=True, project=self.competition_id, group="model_evaluation") wandb.log( { "hotkey": hotkey, @@ -210,7 +211,7 @@ async def evaluate(self) -> str: y_test, y_pred, run_time_s ) self.results.append((hotkey, model_result)) - # self.log_results_to_wandb(hotkey, model_result) + self.log_results_to_wandb(hotkey, model_result) winning_hotkey = sorted( self.results, key=lambda x: x[1].accuracy, reverse=True diff --git a/neurons/competition_runner.py b/neurons/competition_runner.py index 849dad6e..d48d4ae5 100644 --- a/neurons/competition_runner.py +++ b/neurons/competition_runner.py @@ -44,7 +44,7 @@ def was_competition_already_executed( """Check if competition was executed in last minutes""" now_time = datetime.now(timezone.utc) for run in self.runs: - if competition_id and run.competition_id != competition_id: + if run.competition_id != competition_id: continue if run.end_time and (now_time - run.end_time).seconds < last_minutes * 60: return True @@ -76,7 +76,6 @@ def config_for_scheduler( competition_cfg["dataset_hf_repo_type"], test_mode=test_mode, ) - return scheduler_config @@ -98,7 +97,7 @@ async def run_competitions_tick( ).time() competition_manager = competition_scheduler.get(check_time) if not competition_manager: - return (None, None) + continue bt.logging.debug( f"Found competition {competition_manager.competition_id} at {check_time}" @@ -112,7 +111,6 @@ async def run_competitions_tick( continue bt.logging.info(f"Running {competition_manager.competition_id} at {now_time}") - run_log.add_run( CompetitionRun( diff --git a/neurons/validator.py b/neurons/validator.py index 725cc527..cf081ecc 100644 --- a/neurons/validator.py +++ b/neurons/validator.py @@ -61,10 +61,12 @@ async def competition_loop_tick(self): winning_hotkey, competition_id = await run_competitions_tick( self.competition_scheduler, self.run_log ) - except Exception as e: + except Exception: formatted_traceback = traceback.format_exc() bt.logging.error(f"Error running competition: {formatted_traceback}") - wandb.init(project="competition_id", group="competition_evaluation") + wandb.init( + reinit=True, project="competition_id", group="competition_evaluation" + ) wandb.log( { "winning_evaluation_hotkey": "", @@ -79,12 +81,14 @@ async def competition_loop_tick(self): if not winning_hotkey: return - wandb.init(project=competition_id, group="competition_evaluation") + wandb.init(reinit=True, project=competition_id, group="competition_evaluation") + run_time_s = ( + self.run_log.runs[-1].end_time - self.run_log.runs[-1].start_time + ).seconds wandb.log( { "winning_hotkey": winning_hotkey, - "run_time": self.run_log.runs[-1].end_time - - self.run_log.runs[-1].start_time, + "run_time_s": run_time_s, "validator_id": self.wallet.hotkey.ss58_address, "errors": "", } @@ -111,7 +115,6 @@ async def competition_loop_tick(self): ] self.save_state() - def save_state(self): """Saves the state of the validator to a file.""" bt.logging.info("Saving validator state.") From 17d24ce4c06e0616e38e4a40806652850709ac46 Mon Sep 17 00:00:00 2001 From: Wojtek Jurkowlaniec Date: Wed, 4 Sep 2024 18:57:42 +0200 Subject: [PATCH 171/227] initial changelog --- CHANGELOG.md | 3 +++ neurons/competition_config.json | 20 +++++++++++++++----- 2 files changed, 18 insertions(+), 5 deletions(-) create mode 100644 CHANGELOG.md diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 00000000..90f579a6 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,3 @@ +# 0.8 + +- Initial release of the subnet code \ No newline at end of file diff --git a/neurons/competition_config.json b/neurons/competition_config.json index 9a9719cc..c85d6112 100644 --- a/neurons/competition_config.json +++ b/neurons/competition_config.json @@ -3,20 +3,30 @@ "competition_id": "melanoma-1", "category": "skin", "evaluation_times": [ - "23:42" + "14:38" ], "dataset_hf_repo": "safescanai/test_dataset", - "dataset_hf_filename": "skin_melanoma.zip", + "dataset_hf_filename": "test_dataset.zip", "dataset_hf_repo_type": "dataset" }, { "competition_id": "melanoma-testnet", "category": "skin", "evaluation_times": [ - "23:44" + "16:39" ], "dataset_hf_repo": "safescanai/test_dataset", - "dataset_hf_filename": "skin_melanoma.zip", + "dataset_hf_filename": "test_dataset.zip", + "dataset_hf_repo_type": "dataset" + }, + { + "competition_id": "melanoma-7", + "category": "skin", + "evaluation_times": [ + "16:33" + ], + "dataset_hf_repo": "safescanai/test_dataset", + "dataset_hf_filename": "test_dataset.zip", "dataset_hf_repo_type": "dataset" } -] \ No newline at end of file +] From c68ac83befda155a2aa48dabd77de1548f1c8ccc Mon Sep 17 00:00:00 2001 From: LEMSTUDI0 <105062843+LEMSTUDI0@users.noreply.github.com> Date: Wed, 4 Sep 2024 19:47:15 +0200 Subject: [PATCH 172/227] Update README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index f59b5947..7c407e08 100644 --- a/README.md +++ b/README.md @@ -23,6 +23,7 @@ - [💰 Tokenomy & Economy](#-tokenomy--economy) - [👨‍👨‍👦‍👦 Team Composition](#-team-composition) - [🛣️ Roadmap](#roadmap) +- [📊 SETUP WandB (HIGHLY RECOMMENDED - VALIDATORS PLEASE READ](#-SETUP-WandB-(HIGHLY-RECOMMENDED-VALIDATORS-PLEASE-READ) - [👍 Running Validator](#-running-validator) - [⛏️ Running Miner](#-running-miner) - [🚀 Get invloved](#-get-involved) From 75bca969e644eed6df62ff5e7e78bf44197c7f9e Mon Sep 17 00:00:00 2001 From: LEMSTUDI0 <105062843+LEMSTUDI0@users.noreply.github.com> Date: Wed, 4 Sep 2024 19:47:34 +0200 Subject: [PATCH 173/227] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 7c407e08..7d634462 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ - [💰 Tokenomy & Economy](#-tokenomy--economy) - [👨‍👨‍👦‍👦 Team Composition](#-team-composition) - [🛣️ Roadmap](#roadmap) -- [📊 SETUP WandB (HIGHLY RECOMMENDED - VALIDATORS PLEASE READ](#-SETUP-WandB-(HIGHLY-RECOMMENDED-VALIDATORS-PLEASE-READ) +- [📊 SETUP WandB (HIGHLY RECOMMENDED - VALIDATORS PLEASE READ)](#-SETUP-WandB-(HIGHLY-RECOMMENDED-VALIDATORS-PLEASE-READ) - [👍 Running Validator](#-running-validator) - [⛏️ Running Miner](#-running-miner) - [🚀 Get invloved](#-get-involved) From fa1f4970e429d5667b2d32bacfa544ea29970556 Mon Sep 17 00:00:00 2001 From: LEMSTUDI0 <105062843+LEMSTUDI0@users.noreply.github.com> Date: Wed, 4 Sep 2024 19:48:06 +0200 Subject: [PATCH 174/227] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 7d634462..3c714061 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ - [💰 Tokenomy & Economy](#-tokenomy--economy) - [👨‍👨‍👦‍👦 Team Composition](#-team-composition) - [🛣️ Roadmap](#roadmap) -- [📊 SETUP WandB (HIGHLY RECOMMENDED - VALIDATORS PLEASE READ)](#-SETUP-WandB-(HIGHLY-RECOMMENDED-VALIDATORS-PLEASE-READ) +- [📊 SETUP WandB](#-SETUP-WandB) - [👍 Running Validator](#-running-validator) - [⛏️ Running Miner](#-running-miner) - [🚀 Get invloved](#-get-involved) From 09aae1a4f6fd09724d30aba29e731c5b29a63f7f Mon Sep 17 00:00:00 2001 From: LEMSTUDI0 <105062843+LEMSTUDI0@users.noreply.github.com> Date: Wed, 4 Sep 2024 19:48:37 +0200 Subject: [PATCH 175/227] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 3c714061..f8779b12 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ - [💰 Tokenomy & Economy](#-tokenomy--economy) - [👨‍👨‍👦‍👦 Team Composition](#-team-composition) - [🛣️ Roadmap](#roadmap) -- [📊 SETUP WandB](#-SETUP-WandB) +- [📊 Setup WandB](#-setup-WandB) - [👍 Running Validator](#-running-validator) - [⛏️ Running Miner](#-running-miner) - [🚀 Get invloved](#-get-involved) From 68f5e6f34d2301d992eb026c93fdf757b6f2b8e3 Mon Sep 17 00:00:00 2001 From: LEMSTUDI0 <105062843+LEMSTUDI0@users.noreply.github.com> Date: Wed, 4 Sep 2024 23:39:13 +0200 Subject: [PATCH 176/227] Update README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 40f2ca84..a0cf495e 100644 --- a/README.md +++ b/README.md @@ -171,10 +171,10 @@ Given the complexity of creating a state-of-the-art roleplay LLM, we plan to div - [ ] Optimize skin cancer detection models to create one mixture-of-experts model which will run on mobile devices - [ ] Start the process for certifying models - FDA approval - [ ] Make competitions for breast cancer -# **✅ PRE REQUIRMENTS +# **✅ PRE REQUIRMENTS** To install BITTENSOR and set up a wallet follow instructions in this link: -[PRE REQUIRMENTS](DOCS/prerequirments.md) +[PRE REQUIRMENTS](DOCS/prerequirements.md) # **📊 SETUP WandB (HIGHLY RECOMMENDED - VALIDATORS PLEASE READ)** From e0cd4f0ca2faa07fe726da5dcf88d2de4e134c23 Mon Sep 17 00:00:00 2001 From: LEMSTUDI0 <105062843+LEMSTUDI0@users.noreply.github.com> Date: Wed, 4 Sep 2024 23:46:13 +0200 Subject: [PATCH 177/227] Update README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index a0cf495e..e06ccfbc 100644 --- a/README.md +++ b/README.md @@ -24,6 +24,7 @@ - [👨‍👨‍👦‍👦 Team Composition](#-team-composition) - [🛣️ Roadmap](#roadmap) - [✅ Pre requirments](#-pre-requirments) +- [📊 SETUP WandB (HIGHLY RECOMMENDED - VALIDATORS PLEASE READ)](#setup-wandb-highly-recommended---validators-please-read) - [👍 Running Validator](#-running-validator) - [⛏️ Running Miner](#-running-miner) - [🚀 Get invloved](#-get-involved) From e3e136c46736b121a746dad951bb53f82abaf59a Mon Sep 17 00:00:00 2001 From: LEMSTUDI0 <105062843+LEMSTUDI0@users.noreply.github.com> Date: Wed, 4 Sep 2024 23:47:31 +0200 Subject: [PATCH 178/227] Update README.md --- README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index e06ccfbc..14e4245d 100644 --- a/README.md +++ b/README.md @@ -246,13 +246,14 @@ echo %WANDB_API_KEY% # **👍 RUNNING VALIDATOR** +To run a validator follow instructions in this link: -... +[RUNNING VALIDATOR](DOCS/miner.md) # **⛏️ RUNNING MINER** To run a miner follow instructions in this link: -[RUNNING MINER](DOCS/miner.md) +[RUNNING MINER](DOCS/validator.md) # **🚀 GET INVOLVED** From 758bde5513c2c393b26a7fe6b4a41d9f1df89bf6 Mon Sep 17 00:00:00 2001 From: LEMSTUDI0 <105062843+LEMSTUDI0@users.noreply.github.com> Date: Wed, 4 Sep 2024 23:49:14 +0200 Subject: [PATCH 179/227] Update README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 14e4245d..29bd0d61 100644 --- a/README.md +++ b/README.md @@ -248,12 +248,12 @@ echo %WANDB_API_KEY% # **👍 RUNNING VALIDATOR** To run a validator follow instructions in this link: -[RUNNING VALIDATOR](DOCS/miner.md) +[RUNNING VALIDATOR](DOCS/validator.md) # **⛏️ RUNNING MINER** To run a miner follow instructions in this link: -[RUNNING MINER](DOCS/validator.md) +[RUNNING MINER](DOCS/miner.md) # **🚀 GET INVOLVED** From 7921f7191d9078516d2c6ef3956275406a865e51 Mon Sep 17 00:00:00 2001 From: Wojtek Jurkowlaniec Date: Thu, 5 Sep 2024 01:29:55 +0200 Subject: [PATCH 180/227] installation in progress --- DOCS/validator.md | 48 ++++++++++++++++++++++++++ cancer_ai/DOCS/validator.md | 41 ---------------------- cancer_ai/validator/dataset_manager.py | 3 ++ 3 files changed, 51 insertions(+), 41 deletions(-) delete mode 100644 cancer_ai/DOCS/validator.md diff --git a/DOCS/validator.md b/DOCS/validator.md index e69de29b..6819cf01 100644 --- a/DOCS/validator.md +++ b/DOCS/validator.md @@ -0,0 +1,48 @@ +# Running validator + +## Server requirements + +### Minimal + - 32GB of RAM + - storage: 100GB, extendable + +### Recommended + - 64GB of RAM + - storage: 100GB0, extendable + - GPU - nVidia RTX, 12GB VRAM + +## System requirements + +- tested on Ubuntu 22.04 +- python 3.10 +- virtualenv + + +## Installation + +- install `unzip` command + +- create virtualenv + +`virtualenv venv --python=3.10` + +- activate it + +`source venv/bin/activate` + +- install requirements + +`pip install -r requirements.txt` + +## Running + +Prerequirements + +- make sure you are in base directory of the project +- activate your virtualenv +- run `export PYTHONPATH="${PYTHONPATH}:./"` + +Main command + + +python neurons/validator.py --netuid 163 --subtensor.network test --wallet.name validator_testnet --wallet.hotkey hotkey1 --logging.debug \ No newline at end of file diff --git a/cancer_ai/DOCS/validator.md b/cancer_ai/DOCS/validator.md deleted file mode 100644 index 262888c2..00000000 --- a/cancer_ai/DOCS/validator.md +++ /dev/null @@ -1,41 +0,0 @@ -# Running validator - -## Server requirements - -### Minimal - - 32GB of RAM - - storage: 100GB, extendable - -### Recommended - - 64GB of RAM - - storage: 100GB0, extendable - - GPU - nVidia RTX, 12GB VRAM - -## System requirements - -- tested on Ubuntu 22.04 -- python 3.10 -- virtualenv - - -## Installation - -- create virtualenv - -`virtualenv venv --python=3.10` - -- activate it - -`source venv/bin/activate` - -- install requirements - -`pip install -r requirements.txt` - -## Running - -Prerequirements - -- make sure you are in base directory of the project -- activate your virtualenv -- run `export PYTHONPATH="${PYTHONPATH}:./"` \ No newline at end of file diff --git a/cancer_ai/validator/dataset_manager.py b/cancer_ai/validator/dataset_manager.py index 29b9909d..af86d0ef 100644 --- a/cancer_ai/validator/dataset_manager.py +++ b/cancer_ai/validator/dataset_manager.py @@ -91,6 +91,9 @@ async def unzip_dataset(self) -> None: out, err = await run_command( f"unzip {self.local_compressed_path} -d {self.local_extracted_dir}" ) + if err: + bt.logging.error(f"Error unzipping dataset: {err}") + raise DatasetManagerException(f"Error unzipping dataset: {err}") bt.logging.info("Dataset unzipped") def set_dataset_handler(self) -> None: From ffcbe85970900005f0b0a0b9fa0851b22ab963d6 Mon Sep 17 00:00:00 2001 From: Wojtek Jurkowlaniec Date: Thu, 5 Sep 2024 02:00:13 +0200 Subject: [PATCH 181/227] Update validator.md --- DOCS/validator.md | 74 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 74 insertions(+) diff --git a/DOCS/validator.md b/DOCS/validator.md index e69de29b..412ccfc8 100644 --- a/DOCS/validator.md +++ b/DOCS/validator.md @@ -0,0 +1,74 @@ +# Running validator + +## Server requirements + + - 64GB of RAM + - storage: 100GB, extendable + - GPU - nVidia RTX, 12GB VRAM + +## System requirements + +- tested on Ubuntu 22.04 +- python 3.10 +- virtualenv +- unzip and zip commands + + +## Installation + +- create virtualenv + +`virtualenv venv --python=3.10` + +- activate it + +`source venv/bin/activate` + +- install requirements + +`pip install -r requirements.txt` + +## Running + +Prerequirements + +- make sure you are in base directory of the project +- activate your virtualenv +- run `export PYTHONPATH="${PYTHONPATH}:./"` + +Main command + +```bash +python neurons/validator.py \ + --netuid \ + --wallet.name \ + --wallet.hotkey \ + --subtensor.network \ + --logging.debug +``` + +Example for testnet + +```bash +python neurons/validator.py \ + --netuid 163 \ + --subtensor.network test \ + --wallet.name validator_testnet \ + --wallet.hotkey hotkey1 \ + --logging.debug +``` + + +You can also run validator using auto-restart script, which does the following: + + - detects changes from git + - automatically pulls new code or configuration + - installs new packages if required + - restarts validator process + +You can use the same configuration switches as above, with a twist + +```bash +python scripts/start_validator.py --pm2_name +``` + From e3e8c420cb865fc0fedaab28c6efc1108e4b42b8 Mon Sep 17 00:00:00 2001 From: Wojtek Jurkowlaniec Date: Thu, 5 Sep 2024 13:33:11 +0200 Subject: [PATCH 182/227] split dataset chunks --- DOCS/validator.md | 37 +++++++++++- cancer_ai/utils/config.py | 6 -- .../competition_handlers/base_handler.py | 10 ++-- .../competition_handlers/melanoma_handler.py | 27 +-------- cancer_ai/validator/competition_manager.py | 5 +- .../validator/dataset_handlers/image_csv.py | 6 +- .../validator/model_runners/onnx_runner.py | 56 +++++++++++++++---- neurons/miner.py | 2 +- neurons/validator.py | 2 - 9 files changed, 94 insertions(+), 57 deletions(-) diff --git a/DOCS/validator.md b/DOCS/validator.md index 6819cf01..674a210e 100644 --- a/DOCS/validator.md +++ b/DOCS/validator.md @@ -20,7 +20,9 @@ ## Installation -- install `unzip` command +- install `unzip` and `zip` commands + +`sudo apt install zip unzip` - create virtualenv @@ -44,5 +46,36 @@ Prerequirements Main command +```bash +python neurons/validator.py \ + --netuid \ + --wallet.name \ + --wallet.hotkey \ + --subtensor.network \ + --logging.debug +``` + +Example for testnet + +```bash +python neurons/validator.py \ + --netuid 163 \ + --subtensor.network test \ + --wallet.name validator_testnet \ + --wallet.hotkey hotkey1 \ + --logging.debug +``` + + +You can also run validator using auto-restart script, which does the following: + - detects changes from git + - automatically pulls new code or configuration + - installs new packages if required + - restarts validator process + +You can use the same configuration switches as above, with one extra command - `--pm2_name` + +```bash +python scripts/start_validator.py --pm2_name +``` -python neurons/validator.py --netuid 163 --subtensor.network test --wallet.name validator_testnet --wallet.hotkey hotkey1 --logging.debug \ No newline at end of file diff --git a/cancer_ai/utils/config.py b/cancer_ai/utils/config.py index 83497502..3255d9e3 100644 --- a/cancer_ai/utils/config.py +++ b/cancer_ai/utils/config.py @@ -185,12 +185,6 @@ def add_miner_args(cls, parser): default="./datasets", ) - parser.add_argument( - "--hf_token", - type=str, - help="Hugging Face API token", - default="", - ) parser.add_argument( "--clean_after_run", diff --git a/cancer_ai/validator/competition_handlers/base_handler.py b/cancer_ai/validator/competition_handlers/base_handler.py index f2e87009..50927490 100644 --- a/cancer_ai/validator/competition_handlers/base_handler.py +++ b/cancer_ai/validator/competition_handlers/base_handler.py @@ -4,6 +4,7 @@ from numpy import ndarray from pydantic import BaseModel + class ModelEvaluationResult(BaseModel): accuracy: float precision: float @@ -18,6 +19,7 @@ class ModelEvaluationResult(BaseModel): class Config: arbitrary_types_allowed = True + class BaseCompetitionHandler: """ Base class for handling different competition types. @@ -25,13 +27,9 @@ class BaseCompetitionHandler: This class initializes the config and competition_id attributes. """ - def __init__(self, X_test, y_test) -> None: + def __init__(self, X_test: list, y_test: list) -> None: """ Initializes the BaseCompetitionHandler object. - - Args: - X_test (list): List of test images. - y_test (list): List of test labels. """ self.X_test = X_test self.y_test = y_test @@ -50,4 +48,4 @@ def get_model_result(self) -> ModelEvaluationResult: Abstract method to evaluate the competition. This method is responsible for evaluating the competition. - """ \ No newline at end of file + """ diff --git a/cancer_ai/validator/competition_handlers/melanoma_handler.py b/cancer_ai/validator/competition_handlers/melanoma_handler.py index 16068b49..4d77347a 100644 --- a/cancer_ai/validator/competition_handlers/melanoma_handler.py +++ b/cancer_ai/validator/competition_handlers/melanoma_handler.py @@ -15,34 +15,13 @@ class MelanomaCompetitionHandler(BaseCompetitionHandler): - """ """ + """Handler for melanoma competition""" def __init__(self, X_test, y_test) -> None: super().__init__(X_test, y_test) - def preprocess_data(self): - new_X_test = [] - target_size=(224, 224) # TODO: Change this to the correct size - - for img in self.X_test: - img = img.resize(target_size) - img_array = np.array(img, dtype=np.float32) / 255.0 - img_array = np.array(img) - if img_array.shape[-1] != 3: # Handle grayscale images - img_array = np.stack((img_array,) * 3, axis=-1) - - img_array = np.transpose( - img_array, (2, 0, 1) - ) # Transpose image to (C, H, W) - - new_X_test.append(img_array) - - new_X_test = np.array(new_X_test, dtype=np.float32) - - # Map y_test to 0, 1 - new_y_test = [1 if y == "True" else 0 for y in self.y_test] - - return new_X_test, new_y_test + def prepare_y_pred(self, y_pred: np.ndarray) -> np.ndarray: + return [1 if y == "True" else 0 for y in self.y_test] def get_model_result( self, y_test: List[float], y_pred: np.ndarray, run_time_s: float diff --git a/cancer_ai/validator/competition_manager.py b/cancer_ai/validator/competition_manager.py index 1c625283..dd266d24 100644 --- a/cancer_ai/validator/competition_manager.py +++ b/cancer_ai/validator/competition_manager.py @@ -86,6 +86,9 @@ def __init__( self.chain_miner_models = {} self.test_mode = test_mode + def __repr__(self) -> str: + return f"CompetitionManager<{self.competition_id}>" + def log_results_to_wandb( self, hotkey: str, evaluation_result: ModelEvaluationResult ) -> None: @@ -193,7 +196,7 @@ async def evaluate(self) -> str: else: await self.sync_chain_miners() - X_test, y_test = competition_handler.preprocess_data() + y_test = competition_handler.prepare_y_pred(y_test) # bt.logging.info("Ground truth: ", y_test) for hotkey in self.model_manager.hotkey_store: bt.logging.info("Evaluating hotkey: ", hotkey) diff --git a/cancer_ai/validator/dataset_handlers/image_csv.py b/cancer_ai/validator/dataset_handlers/image_csv.py index 854c5719..0cfbbd24 100644 --- a/cancer_ai/validator/dataset_handlers/image_csv.py +++ b/cancer_ai/validator/dataset_handlers/image_csv.py @@ -52,11 +52,7 @@ async def get_training_data(self) -> Tuple[List, List]: """ await self.sync_training_data() pred_x = [ - Image.open( - str( - Path(self.dataset_path, entry.relative_path).resolve(), - ), - ) + Path(self.dataset_path, entry.relative_path).resolve() for entry in self.entries ] pred_y = [entry.is_melanoma for entry in self.entries] diff --git a/cancer_ai/validator/model_runners/onnx_runner.py b/cancer_ai/validator/model_runners/onnx_runner.py index f394ba9f..086f2841 100644 --- a/cancer_ai/validator/model_runners/onnx_runner.py +++ b/cancer_ai/validator/model_runners/onnx_runner.py @@ -1,22 +1,58 @@ from . import BaseRunnerHandler -from typing import List +from typing import List, AsyncGenerator +import numpy as np + class OnnxRunnerHandler(BaseRunnerHandler): + async def get_chunk_of_data( + self, X_test: List, chunk_size: int + ) -> AsyncGenerator[List]: + """Opens images using PIL and yields a chunk of them""" + import PIL.Image as Image + + for i in range(0, len(X_test), chunk_size): + print(f"Processing chunk {i} to {i + chunk_size}") + chunk = [] + for img_path in X_test[i : i + chunk_size]: + img = Image.open(img_path) + chunk.append(img) + chunk = self.preprocess_data(chunk) + yield chunk + + def preprocess_data(self, X_test: List) -> List: + new_X_test = [] + target_size = (224, 224) # TODO: Change this to the correct size + + for img in X_test: + img = img.resize(target_size) + img_array = np.array(img, dtype=np.float32) / 255.0 + img_array = np.array(img) + if img_array.shape[-1] != 3: # Handle grayscale images + img_array = np.stack((img_array,) * 3, axis=-1) + + img_array = np.transpose( + img_array, (2, 0, 1) + ) # Transpose image to (C, H, W) + + new_X_test.append(img_array) + + new_X_test = np.array(new_X_test, dtype=np.float32) + + return new_X_test + async def run(self, X_test: List) -> List: import onnxruntime - import numpy as np - # Load the ONNX model session = onnxruntime.InferenceSession(self.model_path) - # Stack input images into a single batch - input_batch = np.stack(X_test) + results = [] - # Prepare input for ONNX model - input_name = session.get_inputs()[0].name - input_data = {input_name: input_batch} + async for chunk in self.get_chunk_of_data(X_test, chunk_size=200): + input_batch = np.stack(chunk, axis=0) + input_name = session.get_inputs()[0].name + input_data = {input_name: input_batch} - # Perform inference on the batch - results = session.run(None, input_data)[0] + chunk_results = session.run(None, input_data)[0] + results.extend(chunk_results) return results diff --git a/neurons/miner.py b/neurons/miner.py index f215c407..79b418f1 100644 --- a/neurons/miner.py +++ b/neurons/miner.py @@ -94,7 +94,7 @@ async def evaluate_model(self) -> None: X_test=X_test, y_test=y_test ) - X_test, y_test = competition_handler.preprocess_data() + y_test = competition_handler.preprocess_data() start_time = time.time() y_pred = await run_manager.run(X_test) diff --git a/neurons/validator.py b/neurons/validator.py index cf081ecc..534b958c 100644 --- a/neurons/validator.py +++ b/neurons/validator.py @@ -55,8 +55,6 @@ async def concurrent_forward(self): await asyncio.gather(*coroutines) async def competition_loop_tick(self): - - bt.logging.debug("Run log", self.run_log) try: winning_hotkey, competition_id = await run_competitions_tick( self.competition_scheduler, self.run_log From 2ef2b9af68b258ff2f721ca63b10ce77183c3492 Mon Sep 17 00:00:00 2001 From: Wojtek Jurkowlaniec Date: Thu, 5 Sep 2024 15:41:46 +0200 Subject: [PATCH 183/227] Update validator.md --- DOCS/validator.md | 102 ++++++++++++++++++++++++---------------------- 1 file changed, 54 insertions(+), 48 deletions(-) diff --git a/DOCS/validator.md b/DOCS/validator.md index 412ccfc8..51082036 100644 --- a/DOCS/validator.md +++ b/DOCS/validator.md @@ -1,74 +1,80 @@ -# Running validator +# Validator Script Documentation -## Server requirements +This documentation provides an overview of the validator script, its functionality, requirements, and usage instructions. - - 64GB of RAM - - storage: 100GB, extendable - - GPU - nVidia RTX, 12GB VRAM +## Overview + +The validator script is designed to run a validator process and automatically update it whenever a new version is released. This script was adapted from the [original script](https://github.com/macrocosm-os/pretraining/blob/main/scripts/start_validator.py) in the Pretraining Subnet repository. -## System requirements +Key features of the script include: +- **Automatic Updates**: The script checks for updates periodically and ensures that the latest version of the validator is running by pulling the latest code from the repository and upgrading necessary Python packages. +- **Command-Line Argument Compatibility**: The script now properly handles custom command-line arguments and forwards them to the validator (`neurons/validator.py`). +- **Virtual Environment Support**: The script runs within the same virtual environment that it is executed in, ensuring compatibility and ease of use. +- **PM2 Process Management**: The script uses PM2, a process manager, to manage the validator process. -- tested on Ubuntu 22.04 -- python 3.10 -- virtualenv -- unzip and zip commands +## Prerequisites +### Server requirements -## Installation + - 64GB of RAM + - storage: 500GB, extendable + - GPU - nVidia RTX, 12GB VRAM (will work without GPU, but slower) -- create virtualenv +### System requirements -`virtualenv venv --python=3.10` +- **Python 3.10 and virtualenv **: The script is written in Python and requires Python 3.10 to run. +- **PM2**: PM2 must be installed and available on your system. It is used to manage the validator process. +- **zip and unzip** -- activate it +## Installation and Setup -`source venv/bin/activate` +1. **Clone the Repository**: Make sure you have cloned the repository containing this script and have navigated to the correct directory. -- install requirements +2. **Install PM2**: Ensure PM2 is installed globally on your system. If it isn't, you can install it using npm: -`pip install -r requirements.txt` +``` + npm install -g pm2 +``` -## Running +3. **Set Up Virtual Environment**: If you wish to run the script within a virtual environment, create and activate the environment before running the script: +``` + python3 -m venv venv + source venv/bin/activate # On Windows use `venv\Scripts\activate` +``` -Prerequirements +4. **Install Required Python Packages**: Install any required Python packages listed in requirements.txt: +``` +pip install -r requirements.txt +``` -- make sure you are in base directory of the project -- activate your virtualenv -- run `export PYTHONPATH="${PYTHONPATH}:./"` -Main command +## Usage +To run the validator script, use the following command: +**TODO(DEV): CHANGE THESE VALUES BEFORE THE RELEASE TO MAINNET NETUID!!!** -```bash -python neurons/validator.py \ - --netuid \ - --wallet.name \ - --wallet.hotkey \ - --subtensor.network \ - --logging.debug ``` +python3 scripts/start_validator.py --wallet.name=my-wallet --wallet.hotkey=my-hotkey --netuid=163 -Example for testnet - -```bash -python neurons/validator.py \ - --netuid 163 \ - --subtensor.network test \ - --wallet.name validator_testnet \ - --wallet.hotkey hotkey1 \ - --logging.debug ``` +## Command-Line Arguments -You can also run validator using auto-restart script, which does the following: +- `--pm2_name`: Specifies the name of the PM2 process. Default is `"cancer_ai_vali"`. +- `--wallet.name`: Specifies the wallet name to be used by the validator. +- `--wallet.hotkey`: Specifies the hotkey associated with the wallet. +- `--subtensor.network`: Specifies the network name. Default is `"test"`. +- `--netuid`: Specifies the Netuid of the network. Default is `"163"`. +- `--logging.debug`: Enables debug logging if set to `1`. Default is `1`. - - detects changes from git - - automatically pulls new code or configuration - - installs new packages if required - - restarts validator process -You can use the same configuration switches as above, with a twist +## How It Works -```bash -python scripts/start_validator.py --pm2_name -``` +1. **Start Validator Process**: The script starts the validator process using PM2, based on the provided PM2 process name. +2. **Periodic Updates**: The script periodically checks for updates (every 5 minutes by default) by fetching the latest code from the git repository. +3. **Handle Updates**: If a new version is detected, the script pulls the latest changes, upgrades the Python packages, stops the current validator process, and restarts it with the updated code. +4. **Local Changes**: If there are local changes in the repository that conflict with the updates, the script attempts to rebase them. If conflicts persist, the rebase is aborted to preserve the local changes. + +## Notes +- **Local Changes**: If you have made local changes to the codebase, the auto-update feature will attempt to preserve them. However, conflicts might require manual resolution. +- **Environment**: The script uses the environment from which it is executed, so ensure all necessary environment variables and dependencies are correctly configured. From eec2fb00d5f7747207670f6dfe2560fa4c6ee6e8 Mon Sep 17 00:00:00 2001 From: Wojtek Jurkowlaniec Date: Thu, 5 Sep 2024 15:42:33 +0200 Subject: [PATCH 184/227] Delete Validator.md --- Validator.md | 71 ---------------------------------------------------- 1 file changed, 71 deletions(-) delete mode 100644 Validator.md diff --git a/Validator.md b/Validator.md deleted file mode 100644 index 0cb8225b..00000000 --- a/Validator.md +++ /dev/null @@ -1,71 +0,0 @@ -# Validator Script Documentation - -This documentation provides an overview of the validator script, its functionality, requirements, and usage instructions. - -## Overview - -The validator script is designed to run a validator process and automatically update it whenever a new version is released. This script was adapted from the [original script](https://github.com/macrocosm-os/pretraining/blob/main/scripts/start_validator.py) in the Pretraining Subnet repository. - -Key features of the script include: -- **Automatic Updates**: The script checks for updates periodically and ensures that the latest version of the validator is running by pulling the latest code from the repository and upgrading necessary Python packages. -- **Command-Line Argument Compatibility**: The script now properly handles custom command-line arguments and forwards them to the validator (`neurons/validator.py`). -- **Virtual Environment Support**: The script runs within the same virtual environment that it is executed in, ensuring compatibility and ease of use. -- **PM2 Process Management**: The script uses PM2, a process manager, to manage the validator process. - -## Prerequisites - -- **Python 3.x**: The script is written in Python and requires Python 3.x to run. -- **PM2**: PM2 must be installed and available on your system. It is used to manage the validator process. - -## Installation and Setup - -1. **Clone the Repository**: Make sure you have cloned the repository containing this script and have navigated to the correct directory. - -2. **Install PM2**: Ensure PM2 is installed globally on your system. If it isn't, you can install it using npm: - -``` - npm install -g pm2 -``` - -3. **Set Up Virtual Environment**: If you wish to run the script within a virtual environment, create and activate the environment before running the script: -``` - python3 -m venv venv - source venv/bin/activate # On Windows use `venv\Scripts\activate` -``` - -4. **Install Required Python Packages**: Install any required Python packages listed in requirements.txt: -``` -pip install -r requirements.txt -``` - - -## Usage -To run the validator script, use the following command: -**TODO(DEV): CHANGE THESE VALUES BEFORE THE RELEASE TO MAINNET NETUID!!!** - -``` -python3 scripts/start_validator.py --wallet.name=my-wallet --wallet.hotkey=my-hotkey --netuid=163 - -``` - -## Command-Line Arguments - -- `--pm2_name`: Specifies the name of the PM2 process. Default is `"cancer_ai_vali"`. -- `--wallet.name`: Specifies the wallet name to be used by the validator. -- `--wallet.hotkey`: Specifies the hotkey associated with the wallet. -- `--subtensor.network`: Specifies the network name. Default is `"test"`. -- `--netuid`: Specifies the Netuid of the network. Default is `"163"`. -- `--logging.debug`: Enables debug logging if set to `1`. Default is `1`. - - -## How It Works - -1. **Start Validator Process**: The script starts the validator process using PM2, based on the provided PM2 process name. -2. **Periodic Updates**: The script periodically checks for updates (every 5 minutes by default) by fetching the latest code from the git repository. -3. **Handle Updates**: If a new version is detected, the script pulls the latest changes, upgrades the Python packages, stops the current validator process, and restarts it with the updated code. -4. **Local Changes**: If there are local changes in the repository that conflict with the updates, the script attempts to rebase them. If conflicts persist, the rebase is aborted to preserve the local changes. - -## Notes - -- **Local Changes**: If you have made local changes to the codebase, the auto-update feature will attempt to preserve them. However, conflicts might require manual resolution. -- **Environment**: The script uses the environment from which it is executed, so ensure all necessary environment variables and dependencies are correctly configured. From e9d84f01ddca268156ccc993cb53e1badd2d245a Mon Sep 17 00:00:00 2001 From: konrad0960 <71330299+konrad0960@users.noreply.github.com> Date: Thu, 5 Sep 2024 23:38:53 +0200 Subject: [PATCH 185/227] miner fixes (#59) * fixes for upload * docs * fix config flags * chain metadata model * add flag debug by defailt to show logs * changelog --- CHANGELOG.md | 10 ++- DOCS/miner.md | 82 ++++++++++++------- cancer_ai/chain_models_store.py | 10 +-- cancer_ai/utils/config.py | 2 - .../competition_handlers/base_handler.py | 6 +- cancer_ai/validator/dataset_manager.py | 7 +- .../validator/model_runners/onnx_runner.py | 9 +- neurons/miner.py | 28 ++++--- requirements.txt | 6 +- 9 files changed, 101 insertions(+), 59 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 90f579a6..b4ac9b4a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ -# 0.8 +# Changelog -- Initial release of the subnet code \ No newline at end of file +## 0.5.1 + +- Various fixes for miner script + +## 0.5 + +- Initial release of the subnet code diff --git a/DOCS/miner.md b/DOCS/miner.md index eaab2a17..9b7ea52e 100644 --- a/DOCS/miner.md +++ b/DOCS/miner.md @@ -7,6 +7,7 @@ This documentation provides an overview of the miner script, its functionality, The miner script is designed to manage models, evaluate them locally, and upload them to HuggingFace, as well as submit models to validators within a specified network. Key features of the script include: + - **Local Model Evaluation**: Allows you to evaluate models against a dataset locally. - **HuggingFace Upload**: Compresses and uploads models and code to HuggingFace. - **Model Submission to Validators**: Saves model information in the metagraph, enabling validators to test the models. @@ -15,23 +16,31 @@ Key features of the script include: - **Python 3.10**: The script is written in Python and requires Python 3.10 to run. - **Virtual Environment**: It's recommended to run the script within a virtual environment to manage dependencies. +- **8GB RAM**: minimum required operating memory for testing (evaluate) machine learning model locally ## Installation +1. **Clone repository** + + ```bash + git clone git@github.com:safe-scan-ai/cancer-ai.git + cd cancer-ai + ``` + 1. **Create a Virtual Environment** -Set up a virtual environment for the project: - ```bash - virtualenv venv --python=3.10 - source venv/bin/activate - ``` + ```bash + virtualenv venv --python=3.10 + source venv/bin/activate + ``` -## Install Required Python Packages -Install any required Python packages listed in `requirements.txt`: +1. **Install Required Python Packages** -``` -pip install -r requirements.txt -``` + Install any required Python packages listed in `requirements.txt`: + + ```bash + pip install -r requirements.txt + ``` ## Usage @@ -48,6 +57,7 @@ export PYTHONPATH="${PYTHONPATH}:./" ``` ### Evaluate Model Locally + This mode performs the following tasks: - Downloads the dataset. @@ -58,10 +68,15 @@ This mode performs the following tasks: To evaluate a model locally, use the following command: ``` -python neurons/miner.py --action evaluate --competition_id --model_path +python neurons/miner.py --action evaluate --competition.id --model_path ``` -If flag `--clean-after-run` is supplied, it will delete dataset after evaluating the model +Command line argument explanation + +- `--action` - action to perform , choices are "upload", "evaluate", "submit" +- `--model_path` - local path of ONNX model +- `--competition.id` - ID of competition. List of current competitions are in [competition_config.json](neurons/competition_config.json) +- `--clean-after-run` - it will delete dataset after evaluating the model ### Upload to HuggingFace @@ -69,24 +84,25 @@ This mode compresses the code provided by `--code-path` and uploads the model an To upload to HuggingFace, use the following command: -``` +```bash python neurons/miner.py \ - --action upload \ - --competition_id \ - --model_path \ - --code_directory \ - --hf_model_name \ - --hf_repo_id \ - --hf_token \ - --competition_id \ - --model_path \ + --action upload \ + --competition.id \ + --model_path \ --code_directory \ --hf_model_name \ --hf_repo_id \ - --hf_token \ - --logging.debug + --hf_repo_type model \ + --hf_token ``` +Command line argument explanation + +- `--code_directory` - local directory of code +- `--hf_repo_id` - hugging face repository ID - ex. "username/repo" +- `--hf_repo_type` - hugging face type of repository - "model" or "dataset" +- `--hf_token` - hugging face authentication token + ### Submit Model to Validators This mode saves model information in the metagraph, allowing validators to retrieve information about your model for testing. @@ -97,13 +113,7 @@ To submit a model to validators, use the following command: python neurons/miner.py \ --action submit \ --model_path \ - --competition_id \ - --hf_code_filename "melanoma-1-piwo.zip" \ - --hf_model_name \ - --hf_repo_id \ - --hf_repo_type model \ - --model_path \ - --competition_id \ + --competition.id \ --hf_code_filename "melanoma-1-piwo.zip" \ --hf_model_name \ --hf_repo_id \ @@ -115,6 +125,16 @@ python neurons/miner.py \ --logging.debug ``` +Command line argument explanation + +- `--hf_code_filename` - name of file in hugging face repository containing zipped code +- `--hf_model_name` - name of file in hugging face repository containing model +- `--wallet.name` - name of wallet coldkey used for authentication with Bittensor network +- `--wallet.hotkey` - name of wallet hotkey used for authentication with Bittensor network +- `--netuid` - subnet number +- `--subtensor.network` - Bittensor network to connect to - + ## Notes + - **Environment**: The script uses the environment from which it is executed, so ensure all necessary environment variables and dependencies are correctly configured. - **Model Evaluation**: The `evaluate` action downloads necessary datasets and runs the model locally; ensure that your local environment has sufficient resources. diff --git a/cancer_ai/chain_models_store.py b/cancer_ai/chain_models_store.py index dc96635f..e8d8ee25 100644 --- a/cancer_ai/chain_models_store.py +++ b/cancer_ai/chain_models_store.py @@ -11,17 +11,15 @@ class ChainMinerModel(BaseModel): """Uniquely identifies a trained model""" competition_id: Optional[str] = Field(description="The competition id") - - block: Optional[str] = Field( - description="Block on which this model was claimed on the chain." - ) - hf_repo_id: Optional[str] = Field(description="Hugging Face repository id.") - hf_filename: Optional[str] = Field(description="Hugging Face model filename.") + hf_model_filename: Optional[str] = Field(description="Hugging Face model filename.") hf_repo_type: Optional[str] = Field(description="Hugging Face repository type.") hf_code_filename: Optional[str] = Field( description="Hugging Face code zip filename." ) + block: Optional[str] = Field( + description="Block on which this model was claimed on the chain." + ) class Config: arbitrary_types_allowed = True diff --git a/cancer_ai/utils/config.py b/cancer_ai/utils/config.py index 3255d9e3..e972096b 100644 --- a/cancer_ai/utils/config.py +++ b/cancer_ai/utils/config.py @@ -172,7 +172,6 @@ def add_miner_args(cls, parser): ) parser.add_argument( - "--model_path", "--model_path", type=str, help="Path to ONNX model, used for evaluation", @@ -185,7 +184,6 @@ def add_miner_args(cls, parser): default="./datasets", ) - parser.add_argument( "--clean_after_run", action="store_true", diff --git a/cancer_ai/validator/competition_handlers/base_handler.py b/cancer_ai/validator/competition_handlers/base_handler.py index 50927490..46c72b3f 100644 --- a/cancer_ai/validator/competition_handlers/base_handler.py +++ b/cancer_ai/validator/competition_handlers/base_handler.py @@ -2,7 +2,7 @@ from abc import abstractmethod from numpy import ndarray -from pydantic import BaseModel +from pydantic import BaseModel, field_serializer class ModelEvaluationResult(BaseModel): @@ -16,6 +16,10 @@ class ModelEvaluationResult(BaseModel): run_time_s: float tested_entries: int + @field_serializer("confusion_matrix", "fpr", "tpr") + def serialize_numpy(self, value: Any) -> Any: + return value.tolist() + class Config: arbitrary_types_allowed = True diff --git a/cancer_ai/validator/dataset_manager.py b/cancer_ai/validator/dataset_manager.py index af86d0ef..e7cf0aa1 100644 --- a/cancer_ai/validator/dataset_manager.py +++ b/cancer_ai/validator/dataset_manager.py @@ -72,7 +72,12 @@ def delete_dataset(self) -> None: bt.logging.info("Deleting dataset: ") try: - shutil.rmtree(self.local_extracted_dir) + if not os.access(self.config.models.dataset_dir, os.W_OK): + bt.logging.error(f"No write permissions for: {self.local_extracted_dir}") + return + + # Optional: Check if any files are open or being used. + shutil.rmtree(self.config.models.dataset_dir) bt.logging.info("Dataset deleted") except OSError as e: bt.logging.error(f"Failed to delete dataset from disk: {e}") diff --git a/cancer_ai/validator/model_runners/onnx_runner.py b/cancer_ai/validator/model_runners/onnx_runner.py index 086f2841..0f529a5f 100644 --- a/cancer_ai/validator/model_runners/onnx_runner.py +++ b/cancer_ai/validator/model_runners/onnx_runner.py @@ -1,17 +1,20 @@ -from . import BaseRunnerHandler from typing import List, AsyncGenerator + import numpy as np +import onnxruntime +import bittensor as bt +from . import BaseRunnerHandler class OnnxRunnerHandler(BaseRunnerHandler): async def get_chunk_of_data( self, X_test: List, chunk_size: int - ) -> AsyncGenerator[List]: + ) -> AsyncGenerator[List, None]: """Opens images using PIL and yields a chunk of them""" import PIL.Image as Image for i in range(0, len(X_test), chunk_size): - print(f"Processing chunk {i} to {i + chunk_size}") + bt.logging.debug(f"Processing chunk {i} to {i + chunk_size}") chunk = [] for img_path in X_test[i : i + chunk_size]: img = Image.open(img_path) diff --git a/neurons/miner.py b/neurons/miner.py index 79b418f1..b1181fed 100644 --- a/neurons/miner.py +++ b/neurons/miner.py @@ -26,14 +26,13 @@ def __init__(self, config=None): base_config = copy.deepcopy(config or BaseNeuron.config()) self.config = path_config(self) self.config.merge(base_config) + self.config.logging.debug = True BaseNeuron.check_config(self.config) bt.logging.set_config(config=self.config.logging) - bt.logging.info(self.config) @classmethod def add_args(cls, parser: argparse.ArgumentParser): """Method for injecting miner arguments to the parser.""" - print("add") add_miner_args(cls, parser) async def upload_to_hf(self) -> None: @@ -42,8 +41,8 @@ async def upload_to_hf(self) -> None: hf_api = HfApi() hf_login(token=self.config.hf_token) - hf_model_path = f"{self.config.competition.id}-{self.config.hf_model_name}" - hf_code_path = f"{self.config.competition.id}-{self.config.hf_model_name}" + hf_model_path = f"{self.config.competition.id}-{self.config.hf_model_name}.onnx" + hf_code_path = f"{self.config.competition.id}-{self.config.hf_model_name}.zip" path = hf_api.upload_file( path_or_fileobj=self.config.model_path, @@ -54,7 +53,7 @@ async def upload_to_hf(self) -> None: ) bt.logging.info("Uploading code to Hugging Face.") path = hf_api.upload_file( - path_or_fileobj=f"{self.code_zip_path}", + path_or_fileobj=self.code_zip_path, path_in_repo=hf_code_path, repo_id=self.config.hf_repo_id, repo_type="model", @@ -83,7 +82,7 @@ async def evaluate_model(self) -> None: self.config, self.config.competition.id, "safescanai/test_dataset", - "skin_melanoma.zip", + "test_dataset.zip", "dataset", ) await dataset_manager.prepare_dataset() @@ -94,13 +93,17 @@ async def evaluate_model(self) -> None: X_test=X_test, y_test=y_test ) - y_test = competition_handler.preprocess_data() + y_test = competition_handler.prepare_y_pred(y_test) start_time = time.time() y_pred = await run_manager.run(X_test) run_time_s = time.time() - start_time + + # print(y_pred) model_result = competition_handler.get_model_result(y_test, y_pred, run_time_s) - bt.logging.info(f"\n {model_result}\n") + bt.logging.info( + f"Evalutaion results:\n{model_result.model_dump_json(indent=4)}" + ) if self.config.clean_after_run: dataset_manager.delete_dataset() @@ -110,6 +113,10 @@ async def compress_code(self) -> None: out, err = await run_command( f"zip -r {code_zip_path} {self.config.code_directory}/*" ) + if err: + "Error zipping code" + bt.logging.error(err) + return bt.logging.info(f"Code zip path: {code_zip_path}") self.code_zip_path = code_zip_path @@ -158,11 +165,12 @@ async def submit_model(self) -> None: # Push model metadata to chain model_id = ChainMinerModel( + competition_id=self.config.competition.id, hf_repo_id=self.config.hf_repo_id, hf_model_filename=self.config.hf_model_name, - hf_code_filename=self.config.hf_code_filename, - competition_id=self.config.competition.id, hf_repo_type=self.config.hf_repo_type, + hf_code_filename=self.config.hf_code_filename, + block=None, ) await self.metadata_store.store_model_metadata(model_id) bt.logging.success( diff --git a/requirements.txt b/requirements.txt index 25c35c3d..5c0a25d9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -71,7 +71,7 @@ namex==0.0.8 nest-asyncio==1.6.0 netaddr==1.3.0 networkx==3.3 -numpy==1.26.4 +numpy==2.1.1 onnx==1.16.2 onnxruntime==1.19.0 opt-einsum==3.3.0 @@ -104,14 +104,14 @@ redis==5.0.8 requests==2.32.3 resolvelib==0.8.1 retry==0.9.2 -rich==13.7.1 +rich==13.8.0 scalecodec==1.2.11 schedule==1.2.2 scikit-learn==1.5.1 scipy==1.14.1 sentry-sdk==2.13.0 setproctitle==1.3.3 -setuptools==72.1.0 +setuptools==74.1.1 shtab==1.6.5 six==1.16.0 smmap==5.0.1 From cdb3aa651e6418d8b797697ddf637062063adec7 Mon Sep 17 00:00:00 2001 From: notbulubula <101974829+notbulubula@users.noreply.github.com> Date: Thu, 5 Sep 2024 23:48:01 +0200 Subject: [PATCH 186/227] Adding Melanoma.md (#62) * Add Melanoma.md --- Melanoma.md | 45 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 Melanoma.md diff --git a/Melanoma.md b/Melanoma.md new file mode 100644 index 00000000..299c3cc7 --- /dev/null +++ b/Melanoma.md @@ -0,0 +1,45 @@ +# Description of Melanoma Competition + +## Overview +This competition invites participants to develop a machine learning model that **aids in detecting the possibility of melanoma**. The goal is to create a model that can identify patterns in data that are associated with an increased likelihood of melanoma in visual recognition. + +### Objective +The primary objective is to develop a model that can analyze photos taken by users of their skin lesions or areas of concern. +The model should **assist users** by providing a risk assessment or likelihood score that helps them decide if they should seek further medical advice. +As a result, best model will be released in Skin Scan mobile app to run locally on the phone, and a website that will host it, free for anyone to use. + +## Evaluation Criteria +Models will be evaluated based on described **performance metrics** of the model. +The evaluation will be calculaded on following metrics with described weights. + +### Performance Metrics + + The models will be assessed on the following metrics with the corresponding weights: + +| **Metric** | **Description** | **Weight** | +|-------------|-------------------------------------------------------|------------| +| **F-beta** | Prioritizes recall, with a high beta to emphasize it. | 0.50 | +| **Recall** | Measures the ability to correctly identify positives. | 0.20 | +| **AUC** | Evaluates the model's ability to distinguish classes. | 0.15 | +| **Accuracy**| Measures the overall correctness of predictions. | 0.15 | + + +## Model Inputs and Outputs + +### Inputs +- **Input Format**: Multiple images in JPEG or PNG format. +- **Input Features**: During preprocessing, images are resized to 224x224 pixels. Images are converted to numpy arrays with a datatype of `np.float32`, normalized to the range [0, 1]. + +### Outputs +- **Output Format**: A numerical value between 0 and 1, represented as a `float`. This value indicates the likelihood or risk score of the area of concern warranting further investigation. + +### Submission Requirements +- **Model Submission**: Models must be submitted in ONNX format. They should be capable of handling dynamic batch sizes and accept inputs with the shape `(None, 244, 244, 3)`, where `None` represents the batch dimension. This ensures that the model can process a variable number of images in a single batch. + + +## Rules and Guidelines + +- **Timeline**: + - every day competition will be run one or more times a day. Timings are defined in [competition_config.json](neurons/competition_config.json) + - couple of minutes before start of competition, new part of dataset will be published for testing. +- Results of competition will be available on the dashboard From 96740a7a84e1be0f84a2e8d8119ad2fe18e2d5c5 Mon Sep 17 00:00:00 2001 From: Wojtek Jurkowlaniec Date: Thu, 12 Sep 2024 01:12:50 +0200 Subject: [PATCH 187/227] Release production (#86) * Fix hotkeys not updating Refresh hotkeys while validator is running * Adding fbeta metric (#63) * new way of calculating models * documentation * fix against wrong model or wrong dataset (#69) * fix against wrong model or wrong dataset * Chain models sync and cache, working competitions (#72) * improved documentation * fixed a lot of bugs * introduced cache for miner models from chain * docs adjustments (#68) * miner docs * updated rewarder * logging change * WIP * updated test, still not working * tests for rewarder * async runner for pytest * new tests all passing * fix and format * deleting pm2 method for starting validator from docs (#81) * various fixes in logic (#82) preparing for mainnet * Making model path not required for submit mode (#84) Making model path not required for submit mode * Blackist miners (#85) * moved config files * blackisting hotkeys * check if miner model exists * changelog * env example * Wandb API key docs update * final competition config * adjusted docs to finney netuid * log validator hotkey to wandb (#87) * log validator hotkey to wandb * move mock data from competitionmanager --------- Co-authored-by: konrad0960 <71330299+konrad0960@users.noreply.github.com> Co-authored-by: notbulubula <101974829+notbulubula@users.noreply.github.com> Co-authored-by: Kabalisticus <127131450+Kabalisticus@users.noreply.github.com> Co-authored-by: Konrad --- .env.example | 3 +- .gitignore | 3 +- CHANGELOG.md | 9 + DOCS/miner.md | 36 +- DOCS/validator.md | 13 +- Melanoma.md | 35 +- cancer_ai/base/base_validator.py | 34 +- cancer_ai/base/neuron.py | 57 +- cancer_ai/chain_models_store.py | 52 +- cancer_ai/utils/config.py | 9 +- cancer_ai/utils/models_storage_utils.py | 3 +- .../competition_handlers/base_handler.py | 26 +- .../competition_handlers/melanoma_handler.py | 26 +- cancer_ai/validator/competition_manager.py | 172 +++--- cancer_ai/validator/dataset_manager.py | 6 +- cancer_ai/validator/exceptions.py | 7 + cancer_ai/validator/model_manager.py | 40 +- cancer_ai/validator/model_run_manager.py | 11 +- .../validator/model_runners/onnx_runner.py | 12 +- cancer_ai/validator/rewarder.py | 119 +++- cancer_ai/validator/rewarder_test.py | 568 +++++++++++++----- cancer_ai/validator/tests/mock_data.py | 29 + config/competition_config.json | 13 + config/competition_config_testnet.json | 12 + config/hotkey_blacklist.json | 1 + config/hotkey_blacklist_testnet.json | 22 + min_compute.yml | 57 +- neurons/competition_config.json | 32 - neurons/competition_runner.py | 105 ++-- neurons/competition_runner_test.py | 47 +- neurons/miner.py | 14 +- neurons/tests/competition_runner_test.py | 136 +++++ neurons/validator.py | 188 ++++-- requirements.txt | 27 +- scripts/start_validator.py | 6 +- 35 files changed, 1356 insertions(+), 574 deletions(-) create mode 100644 cancer_ai/validator/exceptions.py create mode 100644 cancer_ai/validator/tests/mock_data.py create mode 100644 config/competition_config.json create mode 100644 config/competition_config_testnet.json create mode 100644 config/hotkey_blacklist.json create mode 100644 config/hotkey_blacklist_testnet.json delete mode 100644 neurons/competition_config.json create mode 100644 neurons/tests/competition_runner_test.py diff --git a/.env.example b/.env.example index 067c56f5..dbaaffa1 100644 --- a/.env.example +++ b/.env.example @@ -1,2 +1,3 @@ WANDB_API_KEY = -WANDB_BASE_URL="https://api.wandb.ai" \ No newline at end of file +WANDB_BASE_URL="https://api.wandb.ai" +WANDB_SILENT="true" \ No newline at end of file diff --git a/.gitignore b/.gitignore index 4d9b30d5..4b2aaf36 100644 --- a/.gitignore +++ b/.gitignore @@ -165,4 +165,5 @@ testing/ .vscode/settings.json datasets data -wandb \ No newline at end of file +wandb +ecosystem.config.js diff --git a/CHANGELOG.md b/CHANGELOG.md index b4ac9b4a..451c35a9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,14 @@ # Changelog +## 0.6 + +- added blacklisting of miners +- various logic fixes +- protection from model copying +- chain synchtonization fixes +- fixes for miner CLI +- stability improvements + ## 0.5.1 - Various fixes for miner script diff --git a/DOCS/miner.md b/DOCS/miner.md index 9b7ea52e..4279eab1 100644 --- a/DOCS/miner.md +++ b/DOCS/miner.md @@ -42,6 +42,31 @@ Key features of the script include: pip install -r requirements.txt ``` +## Registering miner on the subnet + +If you haven't yet created a miner wallet and registered on our subnet here is the set of commands to run: + +Create a miner coldkey: + +``` +btcli wallet new_coldkey --wallet.name miner +``` + +Create a hotkey for the miner: +``` +btcli wallet new_hotkey --wallet.name miner --wallet.hotkey default +``` + +Register miner on the CancerAI subnet: +``` +btcli subnet recycle_register --netuid --subtensor.network finney --wallet.name miner --wallet.hotkey default +``` + +Check that your key was registered: +``` +btcli wallet overview --wallet.name miner +``` + ## Usage ### Prerequisites @@ -75,7 +100,7 @@ Command line argument explanation - `--action` - action to perform , choices are "upload", "evaluate", "submit" - `--model_path` - local path of ONNX model -- `--competition.id` - ID of competition. List of current competitions are in [competition_config.json](neurons/competition_config.json) +- `--competition.id` - ID of competition. List of current competitions are in [competition_config.json](config/competition_config.json) - `--clean-after-run` - it will delete dataset after evaluating the model ### Upload to HuggingFace @@ -86,9 +111,9 @@ To upload to HuggingFace, use the following command: ```bash python neurons/miner.py \ - --action upload \ + --action upload \ --competition.id \ - --model_path \ + --model_path \ --code_directory \ --hf_model_name \ --hf_repo_id \ @@ -112,10 +137,9 @@ To submit a model to validators, use the following command: ``` python neurons/miner.py \ --action submit \ - --model_path \ --competition.id \ - --hf_code_filename "melanoma-1-piwo.zip" \ - --hf_model_name \ + --hf_code_filename \ + --hf_model_name \ --hf_repo_id \ --hf_repo_type model \ --wallet.name \ diff --git a/DOCS/validator.md b/DOCS/validator.md index ffdf9b7a..6081462a 100644 --- a/DOCS/validator.md +++ b/DOCS/validator.md @@ -26,6 +26,10 @@ Key features of the script include: - **PM2**: PM2 must be installed and available on your system. It is used to manage the validator process. - **zip and unzip** +### Wandb API key requirement +- Contact us [https://discord.com/channels/1259812760280236122/1262734148020338780](on discord) to get Wandb API key +- Put your key in .env.example file + ## Installation and Setup 1. **Clone the Repository**: Make sure you have cloned the repository containing this script and have navigated to the correct directory. @@ -50,10 +54,9 @@ pip install -r requirements.txt ## Usage To run the validator script, use the following command: -**TODO(DEV): CHANGE THESE VALUES BEFORE THE RELEASE TO MAINNET NETUID!!!** -``` -python3 scripts/start_validator.py --wallet.name=my-wallet --wallet.hotkey=my-hotkey --netuid=163 +```bash +python3 scripts/start_validator.py --wallet.name=my-wallet --wallet.hotkey=my-hotkey --netuid=46 ``` @@ -62,8 +65,8 @@ python3 scripts/start_validator.py --wallet.name=my-wallet --wallet.hotkey=my-ho - `--pm2_name`: Specifies the name of the PM2 process. Default is `"cancer_ai_vali"`. - `--wallet.name`: Specifies the wallet name to be used by the validator. - `--wallet.hotkey`: Specifies the hotkey associated with the wallet. -- `--subtensor.network`: Specifies the network name. Default is `"test"`. -- `--netuid`: Specifies the Netuid of the network. Default is `"163"`. +- `--subtensor.network`: Specifies the network name. Default is `"finney"`. +- `--netuid`: Specifies the Netuid of the network. Default is `"46"`. - `--logging.debug`: Enables debug logging if set to `1`. Default is `1`. diff --git a/Melanoma.md b/Melanoma.md index 299c3cc7..b1186fb9 100644 --- a/Melanoma.md +++ b/Melanoma.md @@ -18,10 +18,35 @@ The evaluation will be calculaded on following metrics with described weights. | **Metric** | **Description** | **Weight** | |-------------|-------------------------------------------------------|------------| -| **F-beta** | Prioritizes recall, with a high beta to emphasize it. | 0.50 | -| **Recall** | Measures the ability to correctly identify positives. | 0.20 | -| **AUC** | Evaluates the model's ability to distinguish classes. | 0.15 | -| **Accuracy**| Measures the overall correctness of predictions. | 0.15 | +| **F-beta** | Prioritizes recall, with a high beta to emphasize it. $\beta = 2$ | 0.60 | +| **Accuracy**| Measures the overall correctness of predictions. | 0.30 | +| **AUC** | Evaluates the model's ability to distinguish classes. | 0.10 | + +### Mathematical Formulas + +1. **F-beta Score $F\_\beta\$** + + + $$F_\beta = \left(1 + \beta^2\right) \cdot \frac{\text{Precision} \cdot \text{Recall}}{\left(\beta^2 \cdot \text{Precision}\right) + \text{Recall}}$$ + + + Where: + - **$\beta$** is the weight of recall in the combined score + - in our case $\beta = 2$ for higher recall importance + +2. **Accuracy** + + $$\text{Accuracy} = \frac{\text{True Positives} + \text{True Negatives}}{\text{Total Number of Samples}}$$ + +3. **Area Under the Curve (AUC)** + + AUC is the area under the Receiver Operating Characteristic (ROC) curve. It is calculated using the trapezoidal rule: + + $$\text{AUC} = \int_0^1 \text{TPR} \, d(\text{FPR})$$ + + Where: + - **TPR** = True Positive Rate + - **FPR** = False Positive Rate ## Model Inputs and Outputs @@ -40,6 +65,6 @@ The evaluation will be calculaded on following metrics with described weights. ## Rules and Guidelines - **Timeline**: - - every day competition will be run one or more times a day. Timings are defined in [competition_config.json](neurons/competition_config.json) + - every day competition will be run one or more times a day. Timings are defined in [competition_config.json](config/competition_config.json) - couple of minutes before start of competition, new part of dataset will be published for testing. - Results of competition will be available on the dashboard diff --git a/cancer_ai/base/base_validator.py b/cancer_ai/base/base_validator.py index 5aa3761b..2e5e5b6f 100644 --- a/cancer_ai/base/base_validator.py +++ b/cancer_ai/base/base_validator.py @@ -26,7 +26,7 @@ import threading import bittensor as bt -from typing import List, Union +from typing import Union from traceback import print_exception from .neuron import BaseNeuron @@ -37,11 +37,10 @@ from ..mock import MockDendrite from ..utils.config import add_validator_args -from neurons.competition_runner import ( - CompetitionRunLog, -) +from neurons.competition_runner import CompetitionRunStore +from cancer_ai.chain_models_store import ChainMinerModelStore -from cancer_ai.validator.rewarder import WinnersMapping +from cancer_ai.validator.rewarder import CompetitionWinnersStore class BaseValidatorNeuron(BaseNeuron): @@ -72,10 +71,11 @@ def __init__(self, config=None): # Set up initial scoring weights for validation bt.logging.info("Building validation weights.") self.scores = np.zeros(self.metagraph.n, dtype=np.float32) - self.run_log = CompetitionRunLog(runs=[]) - self.winners_mapping = WinnersMapping( + self.run_log = CompetitionRunStore(runs=[]) + self.winners_store = CompetitionWinnersStore( competition_leader_map={}, hotkey_score_map={} ) + self.chain_models_store = ChainMinerModelStore(hotkeys={}) self.load_state() # Init sync with the network. Updates the metagraph. self.sync() @@ -317,24 +317,10 @@ def resync_metagraph(self): # Update the hotkeys. self.hotkeys = copy.deepcopy(self.metagraph.hotkeys) + @abstractmethod def save_state(self): """Saves the state of the validator to a file.""" - bt.logging.info("Saving validator state.") - - # Save the state of the validator to file. - np.savez( - self.config.neuron.full_path + "/state.npz", - scores=self.scores, - hotkeys=self.hotkeys, - rewarder_config=self.rewarder_config, - ) - + + @abstractmethod def load_state(self): """Loads the state of the validator from a file.""" - bt.logging.info("Loading validator state.") - - # Load the state of the validator from file. - state = np.load(self.config.neuron.full_path + "/state.npz") - self.scores = state["scores"] - self.hotkeys = state["hotkeys"] - self.rewarder_config = state["rewarder_config"] diff --git a/cancer_ai/base/neuron.py b/cancer_ai/base/neuron.py index 9878b0a5..0e52feec 100644 --- a/cancer_ai/base/neuron.py +++ b/cancer_ai/base/neuron.py @@ -16,7 +16,7 @@ # DEALINGS IN THE SOFTWARE. import copy -import typing +import sys import bittensor as bt @@ -81,12 +81,8 @@ def __init__(self, config=None): # The wallet holds the cryptographic key pairs for the miner. if self.config.mock: self.wallet = bt.MockWallet(config=self.config) - self.subtensor = MockSubtensor( - self.config.netuid, wallet=self.wallet - ) - self.metagraph = MockMetagraph( - self.config.netuid, subtensor=self.subtensor - ) + self.subtensor = MockSubtensor(self.config.netuid, wallet=self.wallet) + self.metagraph = MockMetagraph(self.config.netuid, subtensor=self.subtensor) else: self.wallet = bt.wallet(config=self.config) self.subtensor = bt.subtensor(config=self.config) @@ -100,17 +96,14 @@ def __init__(self, config=None): self.check_registered() # Each miner gets a unique identity (UID) in the network for differentiation. - self.uid = self.metagraph.hotkeys.index( - self.wallet.hotkey.ss58_address - ) + self.uid = self.metagraph.hotkeys.index(self.wallet.hotkey.ss58_address) bt.logging.info( f"Running neuron on subnet: {self.config.netuid} with uid {self.uid} using network: {self.subtensor.chain_endpoint}" ) self.step = 0 @abstractmethod - def run(self): - ... + def run(self): ... def sync(self): """ @@ -129,16 +122,30 @@ def sync(self): self.save_state() def check_registered(self): - # --- Check for registration. - if not self.subtensor.is_hotkey_registered( - netuid=self.config.netuid, - hotkey_ss58=self.wallet.hotkey.ss58_address, - ): - bt.logging.error( - f"Wallet: {self.wallet} is not registered on netuid {self.config.netuid}." - f" Please register the hotkey using `btcli subnets register` before trying again" - ) - exit() + retries = 3 + while retries > 0: + try: + if not hasattr(self, "is_registered"): + self.is_registered = self.subtensor.is_hotkey_registered( + netuid=self.config.netuid, + hotkey_ss58=self.wallet.hotkey.ss58_address, + ) + if not self.is_registered: + bt.logging.error( + f"Wallet: {self.wallet} is not registered on netuid {self.config.netuid}." + f" Please register the hotkey using `btcli subnets register` before trying again" + ) + sys.exit() + + return self.is_registered + + except Exception as e: + bt.logging.error(f"Error checking validator's hotkey registration: {e}") + retries -= 1 + if retries == 0: + sys.exit() + else: + bt.logging.info(f"Retrying... {retries} retries left.") def should_sync_metagraph(self): """ @@ -159,7 +166,5 @@ def should_set_weights(self) -> bool: # Define appropriate logic for when set weights. return ( - (self.block - self.metagraph.last_update[self.uid]) - > self.config.neuron.epoch_length - and self.neuron_type != "MinerNeuron" - ) # don't set weights if you're a miner + self.block - self.metagraph.last_update[self.uid] + ) > self.config.neuron.epoch_length and self.neuron_type != "MinerNeuron" # don't set weights if you're a miner diff --git a/cancer_ai/chain_models_store.py b/cancer_ai/chain_models_store.py index e8d8ee25..3f603b27 100644 --- a/cancer_ai/chain_models_store.py +++ b/cancer_ai/chain_models_store.py @@ -1,10 +1,10 @@ import functools +from typing import Optional, Type + import bittensor as bt -import datetime -from typing import ClassVar, Optional, Type +from pydantic import BaseModel, Field from .utils.models_storage_utils import run_in_subprocess -from pydantic import BaseModel, Field, PositiveInt class ChainMinerModel(BaseModel): @@ -13,11 +13,13 @@ class ChainMinerModel(BaseModel): competition_id: Optional[str] = Field(description="The competition id") hf_repo_id: Optional[str] = Field(description="Hugging Face repository id.") hf_model_filename: Optional[str] = Field(description="Hugging Face model filename.") - hf_repo_type: Optional[str] = Field(description="Hugging Face repository type.") + hf_repo_type: Optional[str] = Field( + description="Hugging Face repository type.", default="model" + ) hf_code_filename: Optional[str] = Field( description="Hugging Face code zip filename." ) - block: Optional[str] = Field( + block: Optional[int] = Field( description="Block on which this model was claimed on the chain." ) @@ -32,29 +34,37 @@ def to_compressed_str(self) -> str: def from_compressed_str(cls, cs: str) -> Type["ChainMinerModel"]: """Returns an instance of this class from a compressed string representation""" tokens = cs.split(":") + if len(tokens) != 5: + return None return cls( hf_repo_id=tokens[0], hf_model_filename=tokens[1], hf_code_filename=tokens[2], competition_id=tokens[3], hf_repo_type=tokens[4], + block=None, ) -class ChainModelMetadataStore: +class ChainMinerModelStore(BaseModel): + hotkeys: dict[str, ChainMinerModel | None] + last_updated: float | None = None + + +class ChainModelMetadata: """Chain based implementation for storing and retrieving metadata about a model.""" def __init__( self, subtensor: bt.subtensor, - subnet_uid: int, + netuid: int, wallet: Optional[bt.wallet] = None, ): self.subtensor = subtensor self.wallet = ( wallet # Wallet is only needed to write to the chain, not to read. ) - self.subnet_uid = subnet_uid + self.netuid = netuid async def store_model_metadata(self, model_id: ChainMinerModel): """Stores model metadata on this subnet for a specific wallet.""" @@ -65,30 +75,36 @@ async def store_model_metadata(self, model_id: ChainMinerModel): partial = functools.partial( self.subtensor.commit, self.wallet, - self.subnet_uid, + self.netuid, model_id.to_compressed_str(), ) run_in_subprocess(partial, 60) async def retrieve_model_metadata(self, hotkey: str) -> Optional[ChainMinerModel]: """Retrieves model metadata on this subnet for specific hotkey""" - # Wrap calls to the subtensor in a subprocess with a timeout to handle potential hangs. - partial = functools.partial( - bt.extrinsics.serving.get_metadata, self.subtensor, self.subnet_uid, hotkey - ) - - metadata = run_in_subprocess(partial, 60) + try: + metadata = bt.extrinsics.serving.get_metadata( + self.subtensor, self.netuid, hotkey + ) + except Exception as e: + bt.logging.error(f"Error retrieving metadata for hotkey {hotkey}: {e}") + return None if not metadata: return None - bt.logging.info(f"Model metadata: {metadata['info']['fields']}") + bt.logging.trace(f"Model metadata: {metadata['info']['fields']}") commitment = metadata["info"]["fields"][0] hex_data = commitment[list(commitment.keys())[0]][2:] chain_str = bytes.fromhex(hex_data).decode() - try: model = ChainMinerModel.from_compressed_str(chain_str) + bt.logging.debug(f"Model: {model}") + if model is None: + bt.logging.error( + f"Metadata might be in old format on the chain for hotkey {hotkey}. Raw value: {chain_str}" + ) + return None except: # If the metadata format is not correct on the chain then we return None. bt.logging.error( @@ -97,4 +113,4 @@ async def retrieve_model_metadata(self, hotkey: str) -> Optional[ChainMinerModel return None # The block id at which the metadata is stored model.block = metadata["block"] - return model \ No newline at end of file + return model diff --git a/cancer_ai/utils/config.py b/cancer_ai/utils/config.py index e972096b..71f52bfd 100644 --- a/cancer_ai/utils/config.py +++ b/cancer_ai/utils/config.py @@ -232,7 +232,7 @@ def add_common_args(cls, parser): "--competition.config_path", type=str, help="Path with competition configuration .", - default="./neurons/competition_config.json", + default="./config/competition_config.json", ) @@ -312,6 +312,13 @@ def add_validator_args(cls, parser): default="opentensor-dev", ) + parser.add_argument( + "--test_mode", + action="store_true", + help="Test(net) mode", + default=False, + ) + def path_config(cls=None): """ diff --git a/cancer_ai/utils/models_storage_utils.py b/cancer_ai/utils/models_storage_utils.py index 68e6ad2e..54d147c2 100644 --- a/cancer_ai/utils/models_storage_utils.py +++ b/cancer_ai/utils/models_storage_utils.py @@ -2,6 +2,7 @@ import multiprocessing from typing import Any + def run_in_subprocess(func: functools.partial, ttl: int) -> Any: """Runs the provided function on a subprocess with 'ttl' seconds to complete. @@ -45,4 +46,4 @@ def wrapped_func(func: functools.partial, queue: multiprocessing.Queue): if isinstance(result, BaseException): raise Exception(f"BaseException raised in subprocess: {str(result)}") - return result \ No newline at end of file + return result diff --git a/cancer_ai/validator/competition_handlers/base_handler.py b/cancer_ai/validator/competition_handlers/base_handler.py index 46c72b3f..c5d069ca 100644 --- a/cancer_ai/validator/competition_handlers/base_handler.py +++ b/cancer_ai/validator/competition_handlers/base_handler.py @@ -2,23 +2,23 @@ from abc import abstractmethod from numpy import ndarray +import numpy as np from pydantic import BaseModel, field_serializer class ModelEvaluationResult(BaseModel): - accuracy: float - precision: float - recall: float - confusion_matrix: ndarray - fpr: ndarray - tpr: ndarray - roc_auc: float - run_time_s: float - tested_entries: int - - @field_serializer("confusion_matrix", "fpr", "tpr") - def serialize_numpy(self, value: Any) -> Any: - return value.tolist() + accuracy: float = 0.0 + precision: float = 0.0 + recall: float = 0.0 + fbeta: float = 0.0 + confusion_matrix: list = [[0, 0], [0, 0]] + fpr: list = [] + tpr: list = [] + roc_auc: float = 0.0 + run_time_s: float = 0.0 + tested_entries: int = 0 + + score: float = 0.0 class Config: arbitrary_types_allowed = True diff --git a/cancer_ai/validator/competition_handlers/melanoma_handler.py b/cancer_ai/validator/competition_handlers/melanoma_handler.py index 4d77347a..d5bb86ef 100644 --- a/cancer_ai/validator/competition_handlers/melanoma_handler.py +++ b/cancer_ai/validator/competition_handlers/melanoma_handler.py @@ -7,12 +7,19 @@ from sklearn.metrics import ( accuracy_score, precision_score, + fbeta_score, recall_score, confusion_matrix, roc_curve, auc, ) +# Weights for the competition, for calcualting model score + +WEIGHT_FBETA = 0.6 +WEIGHT_ACCURACY = 0.3 +WEIGHT_AUC = 0.1 + class MelanomaCompetitionHandler(BaseCompetitionHandler): """Handler for melanoma competition""" @@ -23,25 +30,34 @@ def __init__(self, X_test, y_test) -> None: def prepare_y_pred(self, y_pred: np.ndarray) -> np.ndarray: return [1 if y == "True" else 0 for y in self.y_test] + def calculate_score(self, fbeta: float, accuracy: float, roc_auc: float) -> float: + return fbeta * WEIGHT_FBETA + accuracy * WEIGHT_ACCURACY + roc_auc * WEIGHT_AUC + def get_model_result( self, y_test: List[float], y_pred: np.ndarray, run_time_s: float ) -> ModelEvaluationResult: y_pred_binary = [1 if y > 0.5 else 0 for y in y_pred] tested_entries = len(y_test) accuracy = accuracy_score(y_test, y_pred_binary) - precision = precision_score(y_test, y_pred_binary) - recall = recall_score(y_test, y_pred_binary) + precision = precision_score(y_test, y_pred_binary, zero_division=0) + fbeta = fbeta_score(y_test, y_pred_binary, beta=2, zero_division=0) + recall = recall_score(y_test, y_pred_binary, zero_division=0) conf_matrix = confusion_matrix(y_test, y_pred_binary) fpr, tpr, _ = roc_curve(y_test, y_pred) roc_auc = auc(fpr, tpr) + + score = self.calculate_score(fbeta, accuracy, roc_auc) + return ModelEvaluationResult( tested_entries=tested_entries, run_time_s=run_time_s, accuracy=accuracy, precision=precision, + fbeta=fbeta, recall=recall, - confusion_matrix=conf_matrix, - fpr=fpr, - tpr=tpr, + confusion_matrix=conf_matrix.tolist(), + fpr=fpr.tolist(), + tpr=tpr.tolist(), roc_auc=roc_auc, + score=score, ) diff --git a/cancer_ai/validator/competition_manager.py b/cancer_ai/validator/competition_manager.py index dd266d24..bb2efddc 100644 --- a/cancer_ai/validator/competition_manager.py +++ b/cancer_ai/validator/competition_manager.py @@ -1,27 +1,24 @@ import time -from typing import List +from typing import List, Tuple import bittensor as bt -from sklearn.metrics import ( - accuracy_score, - precision_score, - recall_score, - confusion_matrix, - roc_curve, - auc, -) import wandb +from dotenv import load_dotenv from .manager import SerializableManager from .model_manager import ModelManager, ModelInfo from .dataset_manager import DatasetManager from .model_run_manager import ModelRunManager +from .exceptions import ModelRunException from .competition_handlers.melanoma_handler import MelanomaCompetitionHandler from .competition_handlers.base_handler import ModelEvaluationResult - -from cancer_ai.chain_models_store import ChainModelMetadataStore, ChainMinerModel -from dotenv import load_dotenv +from .tests.mock_data import get_mock_hotkeys_with_models +from cancer_ai.chain_models_store import ( + ChainModelMetadata, + ChainMinerModel, + ChainMinerModelStore, +) load_dotenv() @@ -49,7 +46,10 @@ class CompetitionManager(SerializableManager): def __init__( self, config, + subtensor: bt.subtensor, hotkeys: list[str], + validator_hotkey: str, + chain_miners_store: ChainMinerModelStore, competition_id: str, category: str, dataset_hf_repo: str, @@ -65,8 +65,9 @@ def __init__( competition_id (str): Unique identifier for the competition. category (str): Category of the competition. """ - bt.logging.info(f"Initializing Competition: {competition_id}") + bt.logging.trace(f"Initializing Competition: {competition_id}") self.config = config + self.subtensor = subtensor self.competition_id = competition_id self.category = category self.results = [] @@ -78,34 +79,38 @@ def __init__( dataset_hf_id, dataset_hf_repo_type, ) - self.chain_model_metadata_store = ChainModelMetadataStore( - self.config.subtensor.network, self.config.netuid + self.chain_model_metadata_store = ChainModelMetadata( + self.subtensor, self.config.netuid ) self.hotkeys = hotkeys - self.chain_miner_models = {} + self.validator_hotkey = validator_hotkey + self.chain_miners_store = chain_miners_store self.test_mode = test_mode def __repr__(self) -> str: return f"CompetitionManager<{self.competition_id}>" def log_results_to_wandb( - self, hotkey: str, evaluation_result: ModelEvaluationResult + self, miner_hotkey: str, validator_hotkey: str, evaluation_result: ModelEvaluationResult ) -> None: - wandb.init(reinit=True, project=self.competition_id, group="model_evaluation") + wandb.init(project=self.competition_id, group="model_evaluation") wandb.log( { - "hotkey": hotkey, + "miner_hotkey": miner_hotkey, + "validator_hotkey": validator_hotkey, "tested_entries": evaluation_result.tested_entries, "accuracy": evaluation_result.accuracy, "precision": evaluation_result.precision, + "fbeta": evaluation_result.fbeta, "recall": evaluation_result.recall, - "confusion_matrix": evaluation_result.confusion_matrix.tolist(), + "confusion_matrix": evaluation_result.confusion_matrix, "roc_curve": { - "fpr": evaluation_result.fpr.tolist(), - "tpr": evaluation_result.tpr.tolist(), + "fpr": evaluation_result.fpr, + "tpr": evaluation_result.tpr, }, "roc_auc": evaluation_result.roc_auc, + "score": evaluation_result.score, } ) @@ -124,14 +129,18 @@ def set_state(self, state: dict): self.model_manager.set_state(state["model_manager"]) self.category = state["category"] - async def get_miner_model(self, chain_miner_model: ChainMinerModel): + async def chain_miner_to_model_info( + self, chain_miner_model: ChainMinerModel + ) -> ModelInfo | None: + bt.logging.warning(f"Chain miner model: {chain_miner_model.model_dump()}") if chain_miner_model.competition_id != self.competition_id: - raise ValueError( + bt.logging.debug( f"Chain miner model {chain_miner_model.to_compressed_str()} does not belong to this competition" ) + raise ValueError("Chain miner model does not belong to this competition") model_info = ModelInfo( hf_repo_id=chain_miner_model.hf_repo_id, - hf_model_filename=chain_miner_model.hf_filename, + hf_model_filename=chain_miner_model.hf_model_filename, hf_code_filename=chain_miner_model.hf_code_filename, hf_repo_type=chain_miner_model.hf_repo_type, competition_id=chain_miner_model.competition_id, @@ -140,83 +149,94 @@ async def get_miner_model(self, chain_miner_model: ChainMinerModel): async def sync_chain_miners_test(self): """Get registered mineres from testnet subnet 163""" - - hotkeys_with_models = { - "5Fo2fenxPY1D7hgTHc88g1zrX2ZX17g8DvE5KnazueYefjN5": ModelInfo( - hf_repo_id="safescanai/test_dataset", - hf_model_filename="model_dynamic.onnx", - hf_repo_type="dataset", - ), - "5DZZnwU2LapwmZfYL9AEAWpUR6FoFvqHnzQ5F71Mhwotxujq": ModelInfo( - hf_repo_id="safescanai/test_dataset", - hf_model_filename="best_model.onnx", - hf_repo_type="dataset", - ), - } - self.model_manager.hotkey_store = hotkeys_with_models + self.model_manager.hotkey_store = get_mock_hotkeys_with_models() async def sync_chain_miners(self): """ Updates hotkeys and downloads information of models from the chain """ - bt.logging.info("Synchronizing miners from the chain") + bt.logging.info("Selecting models for competition") bt.logging.info(f"Amount of hotkeys: {len(self.hotkeys)}") - for hotkey in self.hotkeys: - hotkey_metadata = ( - await self.chain_model_metadata_store.retrieve_model_metadata(hotkey) - ) - if not hotkey_metadata: - bt.logging.warning( - f"Cannot get miner model for hotkey {hotkey} from the chain, skipping" - ) + for hotkey in self.chain_miners_store.hotkeys: + if self.chain_miners_store.hotkeys[hotkey] is None: continue try: - miner_model = await self.get_miner_model(hotkey) - self.chain_miner_models[hotkey] = hotkey_metadata - self.model_manager.hotkey_store[hotkey] = miner_model + model_info = await self.chain_miner_to_model_info( + self.chain_miners_store.hotkeys[hotkey] + ) except ValueError: - bt.logging.error( - f"Miner {hotkey} with data {hotkey_metadata.to_compressed_str()} does not belong to this competition, skipping" + bt.logging.warning( + f"Miner {hotkey} does not belong to this competition, skipping" ) + continue + self.model_manager.hotkey_store[hotkey] = model_info + bt.logging.info( - f"Amount of chain miners with models: {len(self.chain_miner_models)}" + f"Amount of hotkeys with valid models: {len(self.model_manager.hotkey_store)}" ) - async def evaluate(self) -> str: + async def evaluate(self) -> Tuple[str | None, ModelEvaluationResult | None]: """Returns hotkey and competition id of winning model miner""" + bt.logging.info(f"Start of evaluation of {self.competition_id}") + if self.test_mode: + await self.sync_chain_miners_test() + else: + await self.sync_chain_miners() + if len(self.model_manager.hotkey_store) == 0: + bt.logging.error("No models to evaluate") + return None, None await self.dataset_manager.prepare_dataset() X_test, y_test = await self.dataset_manager.get_data() competition_handler = COMPETITION_HANDLER_MAPPING[self.competition_id]( X_test=X_test, y_test=y_test ) - # TODO get hotkeys - if self.test_mode: - await self.sync_chain_miners_test() - else: - await self.sync_chain_miners() y_test = competition_handler.prepare_y_pred(y_test) - # bt.logging.info("Ground truth: ", y_test) - for hotkey in self.model_manager.hotkey_store: - bt.logging.info("Evaluating hotkey: ", hotkey) - await self.model_manager.download_miner_model(hotkey) - - model_manager = ModelRunManager( - self.config, self.model_manager.hotkey_store[hotkey] - ) + for miner_hotkey in self.model_manager.hotkey_store: + bt.logging.info(f"Evaluating hotkey: {miner_hotkey}") + model_or_none = await self.model_manager.download_miner_model(miner_hotkey) + if not model_or_none: + bt.logging.error( + f"Failed to download model for hotkey {miner_hotkey} Skipping." + ) + continue + try: + model_manager = ModelRunManager( + self.config, self.model_manager.hotkey_store[miner_hotkey] + ) + except ModelRunException: + bt.logging.error( + f"Model hotkey: {miner_hotkey} failed to initialize. Skipping" + ) + continue start_time = time.time() - y_pred = await model_manager.run(X_test) + + try: + y_pred = await model_manager.run(X_test) + except ModelRunException: + bt.logging.error(f"Model hotkey: {miner_hotkey} failed to run. Skipping") + continue run_time_s = time.time() - start_time - # bt.logging.info("Model prediction ", y_pred) model_result = competition_handler.get_model_result( y_test, y_pred, run_time_s ) - self.results.append((hotkey, model_result)) - self.log_results_to_wandb(hotkey, model_result) + self.results.append((miner_hotkey, model_result)) + if not self.test_mode: + self.log_results_to_wandb(miner_hotkey, self.validator_hotkey, model_result) + if len(self.results) == 0: + bt.logging.error("No models were able to run") + return None, None + winning_hotkey, winning_model_result = sorted( + self.results, key=lambda x: x[1].score, reverse=True + )[0] + for miner_hotkey, model_result in self.results: + bt.logging.debug( + f"Model result for {miner_hotkey}:\n {model_result.model_dump_json(indent=4)} \n" + ) - winning_hotkey = sorted( - self.results, key=lambda x: x[1].accuracy, reverse=True - )[0][0] - return winning_hotkey + bt.logging.info( + f"Winning hotkey for competition {self.competition_id}: {winning_hotkey}" + ) + return winning_hotkey, winning_model_result diff --git a/cancer_ai/validator/dataset_manager.py b/cancer_ai/validator/dataset_manager.py index e7cf0aa1..bbd87350 100644 --- a/cancer_ai/validator/dataset_manager.py +++ b/cancer_ai/validator/dataset_manager.py @@ -9,11 +9,7 @@ from .manager import SerializableManager from .utils import run_command, log_time from .dataset_handlers.image_csv import DatasetImagesCSV - - -class DatasetManagerException(Exception): - pass - +from .exceptions import DatasetManagerException class DatasetManager(SerializableManager): def __init__( diff --git a/cancer_ai/validator/exceptions.py b/cancer_ai/validator/exceptions.py new file mode 100644 index 00000000..e34e06c3 --- /dev/null +++ b/cancer_ai/validator/exceptions.py @@ -0,0 +1,7 @@ + + +class ModelRunException(Exception): + pass + +class DatasetManagerException(Exception): + pass \ No newline at end of file diff --git a/cancer_ai/validator/model_manager.py b/cancer_ai/validator/model_manager.py index a84a8bc4..4b53024e 100644 --- a/cancer_ai/validator/model_manager.py +++ b/cancer_ai/validator/model_manager.py @@ -1,11 +1,11 @@ -from dataclasses import dataclass, asdict, is_dataclass -from datetime import datetime -from time import sleep import os +from dataclasses import dataclass, asdict, is_dataclass + import bittensor as bt from huggingface_hub import HfApi from .manager import SerializableManager +from .exceptions import ModelRunException @dataclass @@ -27,7 +27,7 @@ def __init__(self, config) -> None: if not os.path.exists(self.config.models.model_dir): os.makedirs(self.config.models.model_dir) self.api = HfApi() - self.hotkey_store = {} + self.hotkey_store: dict[str, ModelInfo] = {} def get_state(self): return {k: asdict(v) for k, v in self.hotkey_store.items() if is_dataclass(v)} @@ -35,25 +35,27 @@ def get_state(self): def set_state(self, hotkey_models: dict): self.hotkey_store = {k: ModelInfo(**v) for k, v in hotkey_models.items()} - def sync_hotkeys(self, hotkeys: list): - hotkey_copy = list(self.hotkey_store.keys()) - for hotkey in hotkey_copy: - if hotkey not in hotkeys: - self.delete_model(hotkey) - - async def download_miner_model(self, hotkey) -> None: + async def download_miner_model(self, hotkey) -> bool: """Downloads the newest model from Hugging Face and saves it to disk. Returns: - str: path to the downloaded model + bool: True if the model was downloaded successfully, False otherwise. """ model_info = self.hotkey_store[hotkey] - model_info.file_path = self.api.hf_hub_download( - model_info.hf_repo_id, - model_info.hf_model_filename, - cache_dir=self.config.models.model_dir, - repo_type=model_info.hf_repo_type, - token=self.config.hf_token if hasattr(self.config, "hf_token") else None, - ) + + try: + model_info.file_path = self.api.hf_hub_download( + model_info.hf_repo_id, + model_info.hf_model_filename, + cache_dir=self.config.models.model_dir, + repo_type=model_info.hf_repo_type, + token=( + self.config.hf_token if hasattr(self.config, "hf_token") else None + ), + ) + except Exception as e: + bt.logging.error(f"Failed to download model {e}") + return False + return True def add_model( self, diff --git a/cancer_ai/validator/model_run_manager.py b/cancer_ai/validator/model_run_manager.py index dce6c6a9..76bf1812 100644 --- a/cancer_ai/validator/model_run_manager.py +++ b/cancer_ai/validator/model_run_manager.py @@ -1,3 +1,4 @@ +import bittensor as bt from typing import List from .manager import SerializableManager @@ -6,6 +7,7 @@ from .model_runners.pytorch_runner import PytorchRunnerHandler from .model_runners.tensorflow_runner import TensorflowRunnerHandler from .model_runners.onnx_runner import OnnxRunnerHandler +from .exceptions import ModelRunException MODEL_TYPE_HANDLERS = { @@ -15,6 +17,7 @@ } + class ModelRunManager(SerializableManager): def __init__(self, config, model: ModelInfo) -> None: self.config = config @@ -32,7 +35,13 @@ def set_runner_handler(self) -> None: model_type = detect_model_format(self.model.file_path) # initializing ml model handler object - model_handler = MODEL_TYPE_HANDLERS[model_type] + + model_handler = MODEL_TYPE_HANDLERS.get(model_type) + if model_handler == None: + bt.logging.error (f"Unknown model format {self.model.hf_repo_id} {self.model.hf_repo_id}") + raise ModelRunException("Unknown model format") + + self.handler = model_handler(self.config, self.model.file_path) async def run(self, pred_x: List) -> List: diff --git a/cancer_ai/validator/model_runners/onnx_runner.py b/cancer_ai/validator/model_runners/onnx_runner.py index 0f529a5f..f65e6386 100644 --- a/cancer_ai/validator/model_runners/onnx_runner.py +++ b/cancer_ai/validator/model_runners/onnx_runner.py @@ -1,7 +1,9 @@ from typing import List, AsyncGenerator import numpy as np -import onnxruntime +import bittensor as bt +from ..exceptions import ModelRunException + import bittensor as bt from . import BaseRunnerHandler @@ -45,8 +47,14 @@ def preprocess_data(self, X_test: List) -> List: async def run(self, X_test: List) -> List: import onnxruntime + try: + session = onnxruntime.InferenceSession(self.model_path) + except Exception as e: + bt.logging.error(f"Failed to run model {e}") + raise ModelRunException("Failed to run model") + + - session = onnxruntime.InferenceSession(self.model_path) results = [] diff --git a/cancer_ai/validator/rewarder.py b/cancer_ai/validator/rewarder.py index 9a78dae9..a6e6b31a 100644 --- a/cancer_ai/validator/rewarder.py +++ b/cancer_ai/validator/rewarder.py @@ -1,53 +1,119 @@ from pydantic import BaseModel from datetime import datetime, timezone +from cancer_ai.validator.competition_handlers.base_handler import ModelEvaluationResult + class CompetitionLeader(BaseModel): hotkey: str leader_since: datetime + model_result: ModelEvaluationResult + + class Score(BaseModel): score: float reduction: float -class WinnersMapping(BaseModel): - competition_leader_map: dict[str, CompetitionLeader] # competition_id -> CompetitionLeader - hotkey_score_map: dict[str, Score] # hotkey -> Score + + +class CompetitionWinnersStore(BaseModel): + competition_leader_map: dict[ + str, CompetitionLeader + ] # competition_id -> CompetitionLeader + hotkey_score_map: dict[str, Score] # hotkey -> Score + REWARD_REDUCTION_START_DAY = 30 REWARD_REDUCTION_STEP = 0.1 REWARD_REDUCTION_STEP_DAYS = 7 -class Rewarder(): - def __init__(self, rewarder_config: WinnersMapping): + +class Rewarder: + def __init__(self, rewarder_config: CompetitionWinnersStore): self.competition_leader_mapping = rewarder_config.competition_leader_map self.scores = rewarder_config.hotkey_score_map - async def get_miner_score_and_reduction(self, competition_id: str, hotkey: str) -> tuple[float, float]: + async def get_miner_score_and_reduction( + self, + competition_id: str, + hotkey: str, + winner_model_result: ModelEvaluationResult, + result_improved: bool = False, + ) -> tuple[float, float]: # check if current hotkey is already a leader competition = self.competition_leader_mapping.get(competition_id) if competition and competition.hotkey == hotkey: - days_as_leader = (datetime.now(timezone.utc) - self.competition_leader_mapping[competition_id].leader_since).days + if result_improved: + self.competition_leader_mapping[competition_id].model_result = ( + winner_model_result + ) + days_as_leader = 0 + else: + days_as_leader = ( + datetime.now(timezone.utc) + - self.competition_leader_mapping[competition_id].leader_since + ).days else: days_as_leader = 0 - self.competition_leader_mapping[competition_id] = CompetitionLeader(hotkey=hotkey, - leader_since=datetime.now(timezone.utc)) + self.competition_leader_mapping[competition_id] = CompetitionLeader( + hotkey=hotkey, + leader_since=datetime.now(timezone.utc), + model_result=winner_model_result, + ) return - - # Score degradation starts on 3rd week of leadership - base_share = 1/len(self.competition_leader_mapping) + + # Score degradation starts on 3rd week of leadership + base_share = 1 / len(self.competition_leader_mapping) if days_as_leader > REWARD_REDUCTION_START_DAY: - periods = (days_as_leader - REWARD_REDUCTION_START_DAY) // REWARD_REDUCTION_STEP_DAYS - reduction_factor = max(REWARD_REDUCTION_STEP, 1 - REWARD_REDUCTION_STEP * periods) + periods = ( + days_as_leader - REWARD_REDUCTION_START_DAY + ) // REWARD_REDUCTION_STEP_DAYS + reduction_factor = max( + REWARD_REDUCTION_STEP, 1 - REWARD_REDUCTION_STEP * periods + ) final_share = base_share * reduction_factor reduced_share = base_share - final_share return final_share, reduced_share return base_share, 0 - - async def update_scores(self, new_winner_hotkey: str, new_winner_comp_id: str): + + async def update_scores( + self, + winner_hotkey: str, + competition_id: str, + winner_model_result: ModelEvaluationResult, + ): + """ + Update the scores of the competitors based on the winner of the competition. + + Args: + winner_hotkey: Competition winner's hotkey. + competition_id: Competition ID. + winner_model_result: Information about the winner's model. + + """ + result_improved = False + # Logic to check if new winner's model made any improvement. If not, keep current winner + if ( + len(self.competition_leader_mapping) > 0 + and competition_id in self.competition_leader_mapping + ): + current_leader_model_result = self.competition_leader_mapping[ + competition_id + ].model_result + + result_improved = ( + winner_model_result.score - current_leader_model_result.score > 0.001 + ) + + if not result_improved: + winner_hotkey = self.competition_leader_mapping[competition_id].hotkey + # reset the scores before updating them self.scores = {} - + # get score and reduced share for the new winner - await self.get_miner_score_and_reduction(new_winner_comp_id, new_winner_hotkey) + await self.get_miner_score_and_reduction( + competition_id, winner_hotkey, winner_model_result, result_improved + ) num_competitions = len(self.competition_leader_mapping) # If there is only one competition, the winner takes it all @@ -55,13 +121,14 @@ async def update_scores(self, new_winner_hotkey: str, new_winner_comp_id: str): competition_id = next(iter(self.competition_leader_mapping)) hotkey = self.competition_leader_mapping[competition_id].hotkey self.scores[hotkey] = Score(score=1.0, reduction=0.0) - print("BRUNO WYGRAL", self.scores, self.competition_leader_mapping) return # gather reduced shares for all competitors competitions_without_reduction = [] for curr_competition_id, comp_leader in self.competition_leader_mapping.items(): - score, reduced_share = await self.get_miner_score_and_reduction(curr_competition_id, comp_leader.hotkey) + score, reduced_share = await self.get_miner_score_and_reduction( + curr_competition_id, comp_leader.hotkey, winner_model_result + ) if comp_leader.hotkey in self.scores: self.scores[comp_leader.hotkey].score += score @@ -69,10 +136,12 @@ async def update_scores(self, new_winner_hotkey: str, new_winner_comp_id: str): if reduced_share == 0: competitions_without_reduction.append(curr_competition_id) else: - self.scores[comp_leader.hotkey] = Score(score=score, reduction=reduced_share) + self.scores[comp_leader.hotkey] = Score( + score=score, reduction=reduced_share + ) if reduced_share == 0: competitions_without_reduction.append(curr_competition_id) - + total_reduced_share = sum([score.reduction for score in self.scores.values()]) # if all competitions have reduced shares, distribute them among all competitors @@ -81,8 +150,10 @@ async def update_scores(self, new_winner_hotkey: str, new_winner_comp_id: str): for hotkey, score in self.scores.items(): self.scores[hotkey].score += total_reduced_share / num_competitions return - + # distribute the total reduced share among non-reduced competitons winners for comp_id in competitions_without_reduction: hotkey = self.competition_leader_mapping[comp_id].hotkey - self.scores[hotkey].score += total_reduced_share / len(competitions_without_reduction) \ No newline at end of file + self.scores[hotkey].score += total_reduced_share / len( + competitions_without_reduction + ) diff --git a/cancer_ai/validator/rewarder_test.py b/cancer_ai/validator/rewarder_test.py index b00bc004..ebe79a61 100644 --- a/cancer_ai/validator/rewarder_test.py +++ b/cancer_ai/validator/rewarder_test.py @@ -1,272 +1,560 @@ import pytest -from datetime import datetime, timedelta -from .rewarder import CompetitionLeader, Score, WinnersMapping, Rewarder +from datetime import datetime, timedelta, timezone +from .rewarder import CompetitionLeader, Score, CompetitionWinnersStore, Rewarder +from cancer_ai.validator.competition_handlers.base_handler import ModelEvaluationResult +import numpy as np + + +@pytest.mark.asyncio +async def test_winner_results_model_improved(): + """ + Set new leader if winner's model has better scores + """ + current_model_results = ModelEvaluationResult( + score=0.90, + ) + + new_model_results = ModelEvaluationResult( + score=0.99, + ) + + competition_leaders = { + "competition1": CompetitionLeader( + hotkey="player_1", + leader_since=datetime.now() - timedelta(days=30 + 3 * 7), + model_result=current_model_results, + ), + } + + scores = { + "player_1": Score(score=1.0, reduction=0.0), + } + + winners_store = CompetitionWinnersStore( + competition_leader_map=competition_leaders, hotkey_score_map=scores + ) + + rewarder = Rewarder(winners_store) + await rewarder.update_scores( + winner_hotkey="player_2", + competition_id="competition1", + winner_model_result=new_model_results, + ) + assert ( + winners_store.competition_leader_map["competition1"].model_result + == new_model_results + ) + assert winners_store.competition_leader_map["competition1"].hotkey == "player_2" + + +@pytest.mark.asyncio +async def test_winner_empty_store(): + """ + Test rewards if store is empty + """ + model_results = ModelEvaluationResult( + score=0.9, + ) + competition_leaders = {} + scores = {} + + winners_store = CompetitionWinnersStore( + competition_leader_map=competition_leaders, hotkey_score_map=scores + ) + rewarder = Rewarder(winners_store) + await rewarder.update_scores( + winner_hotkey="player_1", + competition_id="competition1", + winner_model_result=model_results, + ) + assert ( + winners_store.competition_leader_map["competition1"].model_result + == model_results + ) + + +@pytest.mark.asyncio +async def test_winner_results_model_copying(): + """ + Set new leader if winner's model has better scores + """ + current_model_results = ModelEvaluationResult( + score=0.9, + ) + + new_model_results = ModelEvaluationResult( + score=0.9002, + ) + + competition_leaders = { + "competition1": CompetitionLeader( + hotkey="player_1", + leader_since=datetime.now(timezone.utc) - timedelta(days=30 + 3 * 7), + model_result=current_model_results, + ), + } + + scores = { + "player_1": Score(score=1.0, reduction=0.0), + } + + winners_store = CompetitionWinnersStore( + competition_leader_map=competition_leaders, hotkey_score_map=scores + ) + + rewarder = Rewarder(winners_store) + await rewarder.update_scores( + winner_hotkey="player_2", + competition_id="competition1", + winner_model_result=new_model_results, + ) + assert ( + winners_store.competition_leader_map["competition1"].model_result.score + == current_model_results.score + ) + assert winners_store.competition_leader_map["competition1"].hotkey == "player_1" -def test_update_scores_single_competitor(): +@pytest.mark.asyncio +async def test_update_scores_single_competitor(): # Set up initial data for a single competitor competition_leaders = { - "competition1": CompetitionLeader(hotkey="competitor1", leader_since=datetime.now() - timedelta(days=10)) + "competition_1": CompetitionLeader( + hotkey="competitor_1", + leader_since=datetime.now(timezone.utc) - timedelta(days=10), + model_result=ModelEvaluationResult(score=0.9), + ) } scores = { - "competitor1": Score(score=0.0, reduction=0.0) + "competitor_1": Score(score=1.0, reduction=0.0), } # Set up the configuration with a single competition and a single competitor - rewarder_config = WinnersMapping( - competition_leader_map=competition_leaders, - hotkey_score_map=scores - ) + winners_store = CompetitionWinnersStore( + competition_leader_map=competition_leaders, hotkey_score_map=scores) + + rewarder = Rewarder(winners_store) + expected_winner_model_score = 0.95 + await rewarder.update_scores(winner_hotkey="competitor_1", competition_id="competition_1", + winner_model_result=ModelEvaluationResult(score=expected_winner_model_score)) - rewarder = Rewarder(rewarder_config) - rewarder.update_scores() # Check the updated scores and reductions for the single competitor - updated_score = rewarder.scores["competitor1"].score - updated_reduction = rewarder.scores["competitor1"].reduction + updated_score = rewarder.scores["competitor_1"].score + updated_reduction = rewarder.scores["competitor_1"].reduction # # With only one competitor, they should receive the full score of 1.0 expected_score = 1.0 expected_reduction = 0.0 - assert updated_score == expected_score, f"Expected score: {expected_score}, got: {updated_score}" - assert updated_reduction == expected_reduction, f"Expected reduction: {expected_reduction}, got: {updated_reduction}" - -def test_update_scores_multiple_competitors_no_reduction(): - # Set up initial data for multiple competitors + assert ( + winners_store.competition_leader_map["competition_1"].model_result.score + == expected_winner_model_score + ) + assert winners_store.competition_leader_map["competition_1"].hotkey == "competitor_1" + assert ( + updated_score == expected_score + ), f"Expected score: {expected_score}, got: {updated_score}" + assert ( + updated_reduction == expected_reduction + ), f"Expected reduction: {expected_reduction}, got: {updated_reduction}" + +@pytest.mark.asyncio +async def test_update_scores_multiple_competitors_no_reduction(): + # Set up initial data for a multiple competitors competition_leaders = { - "competition1": CompetitionLeader(hotkey="competitor1", leader_since=datetime.now() - timedelta(days=10)), - "competition2": CompetitionLeader(hotkey="competitor2", leader_since=datetime.now() - timedelta(days=10)), - "competition3": CompetitionLeader(hotkey="competitor3", leader_since=datetime.now() - timedelta(days=10)) + "competition_1": CompetitionLeader( + hotkey="competitor_1", + leader_since=datetime.now(timezone.utc) - timedelta(days=10), + model_result=ModelEvaluationResult(score=0.9), + ), + "competition_2": CompetitionLeader( + hotkey="competitor_2", + leader_since=datetime.now(timezone.utc) - timedelta(days=10), + model_result=ModelEvaluationResult(score=0.9), + ), + "competition_3": CompetitionLeader( + hotkey="competitor_3", + leader_since=datetime.now(timezone.utc) - timedelta(days=10), + model_result=ModelEvaluationResult(score=0.9), + ), } scores = { - "competitor1": Score(score=0.0, reduction=0.0), - "competitor2": Score(score=0.0, reduction=0.0), - "competitor3": Score(score=0.0, reduction=0.0) + "competitor_1": Score(score=0.0, reduction=0.0), + "competitor_2": Score(score=0.0, reduction=0.0), + "competitor_3": Score(score=0.0, reduction=0.0), } # Set up the configuration with multiple competitions and multiple competitors - rewarder_config = WinnersMapping( - competition_leader_map=competition_leaders, - hotkey_score_map=scores - ) + winners_store = CompetitionWinnersStore( + competition_leader_map=competition_leaders, hotkey_score_map=scores) + + rewarder = Rewarder(winners_store) + await rewarder.update_scores(winner_hotkey="competitor_1", competition_id="competition_1", + winner_model_result=ModelEvaluationResult(score=0.9)) - rewarder = Rewarder(rewarder_config) - rewarder.update_scores() # Check the updated scores and reductions for the multiple competitors updated_scores = {hotkey: score.score for hotkey, score in rewarder.scores.items()} - updated_reductions = {hotkey: score.reduction for hotkey, score in rewarder.scores.items()} + updated_reductions = { + hotkey: score.reduction for hotkey, score in rewarder.scores.items() + } + updated_model_scores = {competition_id: leader.model_result.score for competition_id, leader in winners_store.competition_leader_map.items()} # With multiple competitors and no reductions, they should all receive the same score of 1/3 - expected_score = 1/3 + expected_score = 1 / 3 expected_reduction = 0.0 + expected_model_score = 0.9 + + + for _, score in updated_model_scores.items(): + assert ( + score == expected_model_score + ), f"Expected score: {expected_model_score}, got: {score}" for _, score in updated_scores.items(): - assert score == expected_score, f"Expected score: {expected_score}, got: {score}" + assert ( + score == expected_score + ), f"Expected score: {expected_score}, got: {score}" for _, reduction in updated_reductions.items(): - assert reduction == expected_reduction, f"Expected reduction: {expected_reduction}, got: {reduction}" + assert ( + reduction == expected_reduction + ), f"Expected reduction: {expected_reduction}, got: {reduction}" -def test_update_scores_multiple_competitors_with_some_reduced_shares(): - # Set up initial data for multiple competitors +@pytest.mark.asyncio +async def test_update_scores_multiple_competitors_with_some_reduced_shares(): + # Set up initial data for a multiple competitors competition_leaders = { - "competition1": CompetitionLeader(hotkey="competitor1", leader_since=datetime.now() - timedelta(days=30 + 3 * 7)), - "competition2": CompetitionLeader(hotkey="competitor2", leader_since=datetime.now() - timedelta(days=30 + 6 * 7)), - "competition3": CompetitionLeader(hotkey="competitor3", leader_since=datetime.now() - timedelta(days=30)), - "competition4": CompetitionLeader(hotkey="competitor4", leader_since=datetime.now() - timedelta(days=30)), + "competition_1": CompetitionLeader( + hotkey="competitor_1", + leader_since=datetime.now(timezone.utc) - timedelta(days=30 + 3 * 7), + model_result=ModelEvaluationResult(score=0.9), + ), + "competition_2": CompetitionLeader( + hotkey="competitor_2", + leader_since=datetime.now(timezone.utc) - timedelta(days=30 + 6 * 7), + model_result=ModelEvaluationResult(score=0.9), + ), + "competition_3": CompetitionLeader( + hotkey="competitor_3", + leader_since=datetime.now(timezone.utc) - timedelta(days=30), + model_result=ModelEvaluationResult(score=0.9), + ), + "competition_4": CompetitionLeader( + hotkey="competitor_4", + leader_since=datetime.now(timezone.utc) - timedelta(days=30), + model_result=ModelEvaluationResult(score=0.9), + ), } scores = { - "competitor1": Score(score=0.0, reduction=0.0), - "competitor2": Score(score=0.0, reduction=0.0), - "competitor3": Score(score=0.0, reduction=0.0), - "competitor4": Score(score=0.0, reduction=0.0), + "competitor_1": Score(score=0.0, reduction=0.0), + "competitor_2": Score(score=0.0, reduction=0.0), + "competitor_3": Score(score=0.0, reduction=0.0), + "competitor_4": Score(score=0.0, reduction=0.0), } # Set up the configuration with multiple competitions and multiple competitors - rewarder_config = WinnersMapping( - competition_leader_map=competition_leaders, - hotkey_score_map=scores - ) + winners_store = CompetitionWinnersStore( + competition_leader_map=competition_leaders, hotkey_score_map=scores) + + rewarder = Rewarder(winners_store) + await rewarder.update_scores(winner_hotkey="competitor_1", competition_id="competition_1", + winner_model_result=ModelEvaluationResult(score=0.9)) - rewarder = Rewarder(rewarder_config) - rewarder.update_scores() # Check the updated scores and reductions for the multiple competitors updated_scores = {hotkey: score.score for hotkey, score in rewarder.scores.items()} - updated_reductions = {hotkey: score.reduction for hotkey, score in rewarder.scores.items()} + updated_reductions = { + hotkey: score.reduction for hotkey, score in rewarder.scores.items() + } + updated_model_scores = {competition_id: leader.model_result.score for competition_id, leader in winners_store.competition_leader_map.items()} # With multiple competitors and some reduced shares, they should receive different scores and reductions expected_reductions = { - "competitor1": 1/4 * 0.3, - "competitor2": 1/4 * 0.6, - "competitor3": 0.0, - "competitor4": 0.0, + "competitor_1": 1 / 4 * 0.3, + "competitor_2": 1 / 4 * 0.6, + "competitor_3": 0.0, + "competitor_4": 0.0, } expected_reductions_sum = sum(expected_reductions.values()) expected_scores = { - "competitor1": 1/4 - expected_reductions["competitor1"], - "competitor2": 1/4 - expected_reductions["competitor2"], - "competitor3": 1/4 + expected_reductions_sum/2, - "competitor4": 1/4 + expected_reductions_sum/2, + "competitor_1": 1 / 4 - expected_reductions["competitor_1"], + "competitor_2": 1 / 4 - expected_reductions["competitor_2"], + "competitor_3": 1 / 4 + expected_reductions_sum / 2, + "competitor_4": 1 / 4 + expected_reductions_sum / 2, } + expected_model_score = 0.9 + + + for _, score in updated_model_scores.items(): + assert ( + score == expected_model_score + ), f"Expected score: {expected_model_score}, got: {score}" for hotkey, score in updated_scores.items(): - assert score == pytest.approx(expected_scores[hotkey], rel=1e-9), f"Expected score: {expected_scores[hotkey]}, got: {score}" + assert score == pytest.approx( + expected_scores[hotkey], rel=1e-9 + ), f"Expected score: {expected_scores[hotkey]}, got: {score}" for hotkey, reduction in updated_reductions.items(): - assert reduction == pytest.approx(expected_reductions[hotkey], rel=1e-9), f"Expected reduction: {expected_reductions[hotkey]}, got: {reduction}" + assert reduction == pytest.approx( + expected_reductions[hotkey], rel=1e-9 + ), f"Expected reduction: {expected_reductions[hotkey]}, got: {reduction}" -def test_update_scores_all_competitors_with_reduced_shares(): - # Set up initial data for multiple competitors +@pytest.mark.asyncio +async def test_update_scores_all_competitors_with_reduced_shares(): + # Set up initial data for a multiple competitors competition_leaders = { - "competition1": CompetitionLeader(hotkey="competitor1", leader_since=datetime.now() - timedelta(days=30 + 3 * 7)), - "competition2": CompetitionLeader(hotkey="competitor2", leader_since=datetime.now() - timedelta(days=30 + 6 * 7)), - "competition3": CompetitionLeader(hotkey="competitor3", leader_since=datetime.now() - timedelta(days=30 + 9 * 7)) + "competition_1": CompetitionLeader( + hotkey="competitor_1", + leader_since=datetime.now(timezone.utc) - timedelta(days=30 + 3 * 7), + model_result=ModelEvaluationResult(score=0.9), + ), + "competition_2": CompetitionLeader( + hotkey="competitor_2", + leader_since=datetime.now(timezone.utc) - timedelta(days=30 + 6 * 7), + model_result=ModelEvaluationResult(score=0.9), + ), + "competition_3": CompetitionLeader( + hotkey="competitor_3", + leader_since=datetime.now(timezone.utc) - timedelta(days=30 + 9 * 7), + model_result=ModelEvaluationResult(score=0.9), + ), } scores = { - "competitor1": Score(score=0.0, reduction=0.0), - "competitor2": Score(score=0.0, reduction=0.0), - "competitor3": Score(score=0.0, reduction=0.0) + "competitor_1": Score(score=0.0, reduction=0.0), + "competitor_2": Score(score=0.0, reduction=0.0), + "competitor_3": Score(score=0.0, reduction=0.0), } # Set up the configuration with multiple competitions and multiple competitors - rewarder_config = WinnersMapping( - competition_leader_map=competition_leaders, - hotkey_score_map=scores - ) + winners_store = CompetitionWinnersStore( + competition_leader_map=competition_leaders, hotkey_score_map=scores) + + rewarder = Rewarder(winners_store) + await rewarder.update_scores(winner_hotkey="competitor_1", competition_id="competition_1", + winner_model_result=ModelEvaluationResult(score=0.9)) - rewarder = Rewarder(rewarder_config) - rewarder.update_scores() # Check the updated scores and reductions for the multiple competitors updated_scores = {hotkey: score.score for hotkey, score in rewarder.scores.items()} - updated_reductions = {hotkey: score.reduction for hotkey, score in rewarder.scores.items()} - - # With multiple competitors and reduced shares, they should receive different scores and reductions - expected_reductions = { - "competitor1": 0.1, - "competitor2": 0.2, - "competitor3": 0.3 + updated_reductions = { + hotkey: score.reduction for hotkey, score in rewarder.scores.items() } + updated_model_scores = {competition_id: leader.model_result.score for competition_id, leader in winners_store.competition_leader_map.items()} + + # With multiple competitors and some reduced shares, they should receive different scores and reductions + expected_reductions = {"competitor_1": 0.1, "competitor_2": 0.2, "competitor_3": 0.3} expected_reductions_sum = sum(expected_reductions.values()) expected_scores = { - "competitor1": 1/3 - expected_reductions["competitor1"] + expected_reductions_sum/3, - "competitor2": 1/3 - expected_reductions["competitor2"] + expected_reductions_sum/3, - "competitor3": 1/3 - expected_reductions["competitor3"] + expected_reductions_sum/3, + "competitor_1": 1 / 3 + - expected_reductions["competitor_1"] + + expected_reductions_sum / 3, + "competitor_2": 1 / 3 + - expected_reductions["competitor_2"] + + expected_reductions_sum / 3, + "competitor_3": 1 / 3 + - expected_reductions["competitor_3"] + + expected_reductions_sum / 3, } + expected_model_score = 0.9 + + + for _, score in updated_model_scores.items(): + assert ( + score == expected_model_score + ), f"Expected score: {expected_model_score}, got: {score}" for hotkey, score in updated_scores.items(): - assert score == expected_scores[hotkey], f"Expected score: {expected_scores[hotkey]}, got: {score}" + assert score == pytest.approx( + expected_scores[hotkey], rel=1e-9 + ), f"Expected score: {expected_scores[hotkey]}, got: {score}" for hotkey, reduction in updated_reductions.items(): - assert reduction == expected_reductions[hotkey], f"Expected reduction: {expected_reductions[hotkey]}, got: {reduction}" + assert reduction == pytest.approx( + expected_reductions[hotkey], rel=1e-9 + ), f"Expected reduction: {expected_reductions[hotkey]}, got: {reduction}" -def test_update_scores_more_competitions_then_competitors(): - # Set up initial data for multiple competitors +@pytest.mark.asyncio +async def test_update_scores_more_competitions_then_competitors(): + # Set up initial data for a multiple competitors competition_leaders = { - "competition1": CompetitionLeader(hotkey="competitor1", leader_since=datetime.now() - timedelta(days=30 + 3 * 7)), - "competition2": CompetitionLeader(hotkey="competitor2", leader_since=datetime.now() - timedelta(days=30)), - "competition3": CompetitionLeader(hotkey="competitor1", leader_since=datetime.now() - timedelta(days=30)), - "competition4": CompetitionLeader(hotkey="competitor3", leader_since=datetime.now() - timedelta(days=30)), + "competition_1": CompetitionLeader( + hotkey="competitor_1", + leader_since=datetime.now(timezone.utc) - timedelta(days=30 + 3 * 7), + model_result=ModelEvaluationResult(score=0.9), + ), + "competition_2": CompetitionLeader( + hotkey="competitor_2", + leader_since=datetime.now(timezone.utc) - timedelta(days=30), + model_result=ModelEvaluationResult(score=0.9), + ), + "competition_3": CompetitionLeader( + hotkey="competitor_1", + leader_since=datetime.now(timezone.utc) - timedelta(days=30), + model_result=ModelEvaluationResult(score=0.9), + ), + "competition_4": CompetitionLeader( + hotkey="competitor_3", + leader_since=datetime.now(timezone.utc) - timedelta(days=30), + model_result=ModelEvaluationResult(score=0.9), + ), } scores = { - "competitor1": Score(score=0.0, reduction=0.0), - "competitor2": Score(score=0.0, reduction=0.0), - "competitor3": Score(score=0.0, reduction=0.0), + "competitor_1": Score(score=0.0, reduction=0.0), + "competitor_2": Score(score=0.0, reduction=0.0), + "competitor_3": Score(score=0.0, reduction=0.0), } # Set up the configuration with multiple competitions and multiple competitors - rewarder_config = WinnersMapping( - competition_leader_map=competition_leaders, - hotkey_score_map=scores - ) + winners_store = CompetitionWinnersStore( + competition_leader_map=competition_leaders, hotkey_score_map=scores) + + rewarder = Rewarder(winners_store) + await rewarder.update_scores(winner_hotkey="competitor_1", competition_id="competition_1", + winner_model_result=ModelEvaluationResult(score=0.9)) - rewarder = Rewarder(rewarder_config) - rewarder.update_scores() # Check the updated scores and reductions for the multiple competitors updated_scores = {hotkey: score.score for hotkey, score in rewarder.scores.items()} - updated_reductions = {hotkey: score.reduction for hotkey, score in rewarder.scores.items()} + updated_reductions = { + hotkey: score.reduction for hotkey, score in rewarder.scores.items() + } + updated_model_scores = {competition_id: leader.model_result.score for competition_id, leader in winners_store.competition_leader_map.items()} # With multiple competitors and some reduced shares, they should receive different scores and reductions expected_reductions = { - "competitor1": 1/4 * 0.3, - "competitor2": 0.0, - "competitor3": 0.0, + "competitor_1": 1 / 4 * 0.3, + "competitor_2": 0.0, + "competitor_3": 0.0, } expected_reductions_sum = sum(expected_reductions.values()) expected_scores = { - "competitor1": 2/4 - expected_reductions["competitor1"] + expected_reductions_sum/3, - "competitor2": 1/4 + expected_reductions_sum/3, - "competitor3": 1/4 + expected_reductions_sum/3, + "competitor_1": 2 / 4 + - expected_reductions["competitor_1"] + + expected_reductions_sum / 3, + "competitor_2": 1 / 4 + expected_reductions_sum / 3, + "competitor_3": 1 / 4 + expected_reductions_sum / 3, } + expected_model_score = 0.9 + + + for _, score in updated_model_scores.items(): + assert ( + score == expected_model_score + ), f"Expected score: {expected_model_score}, got: {score}" for hotkey, score in updated_scores.items(): - assert score == pytest.approx(expected_scores[hotkey], rel=1e-9), f"Expected score: {expected_scores[hotkey]}, got: {score} for {hotkey}" + assert score == pytest.approx( + expected_scores[hotkey], rel=1e-9 + ), f"Expected score: {expected_scores[hotkey]}, got: {score}" for hotkey, reduction in updated_reductions.items(): - assert reduction == pytest.approx(expected_reductions[hotkey], rel=1e-9), f"Expected reduction: {expected_reductions[hotkey]}, got: {reduction} for {hotkey}" + assert reduction == pytest.approx( + expected_reductions[hotkey], rel=1e-9 + ), f"Expected reduction: {expected_reductions[hotkey]}, got: {reduction}" -def test_update_scores_6_competitions_3_competitors(): - # Set up initial data for multiple competitors +@pytest.mark.asyncio +async def test_update_scores_6_competitions_4_competitors(): + # Set up initial data for a multiple competitors competition_leaders = { - "competition1": CompetitionLeader(hotkey="competitor1", leader_since=datetime.now() - timedelta(days=30 + 3 * 7)), - "competition2": CompetitionLeader(hotkey="competitor2", leader_since=datetime.now() - timedelta(days=30 + 6 * 7)), - "competition3": CompetitionLeader(hotkey="competitor3", leader_since=datetime.now() - timedelta(days=30 + 9 * 7)), - "competition4": CompetitionLeader(hotkey="competitor4", leader_since=datetime.now() - timedelta(days=30)), - "competition5": CompetitionLeader(hotkey="competitor1", leader_since=datetime.now() - timedelta(days=30)), - "competition6": CompetitionLeader(hotkey="competitor2", leader_since=datetime.now() - timedelta(days=30 + 3 * 7)), + "competition_1": CompetitionLeader( + hotkey="competitor_1", + leader_since=datetime.now(timezone.utc) - timedelta(days=30 + 3 * 7), + model_result=ModelEvaluationResult(score=0.9), + ), + "competition_2": CompetitionLeader( + hotkey="competitor_2", + leader_since=datetime.now(timezone.utc) - timedelta(days=30 + 6 * 7), + model_result=ModelEvaluationResult(score=0.9), + ), + "competition_3": CompetitionLeader( + hotkey="competitor_3", + leader_since=datetime.now(timezone.utc) - timedelta(days=30 + 9 * 7), + model_result=ModelEvaluationResult(score=0.9), + ), + "competition_4": CompetitionLeader( + hotkey="competitor_4", + leader_since=datetime.now(timezone.utc) - timedelta(days=30), + model_result=ModelEvaluationResult(score=0.9), + ), + "competition_5": CompetitionLeader( + hotkey="competitor_1", + leader_since=datetime.now(timezone.utc) - timedelta(days=30), + model_result=ModelEvaluationResult(score=0.9), + ), + "competition_6": CompetitionLeader( + hotkey="competitor_2", + leader_since=datetime.now(timezone.utc) - timedelta(days=30 + 3 * 7), + model_result=ModelEvaluationResult(score=0.9), + ), } scores = { - "competitor1": Score(score=0.0, reduction=0.0), - "competitor2": Score(score=0.0, reduction=0.0), - "competitor3": Score(score=0.0, reduction=0.0), - "competitor4": Score(score=0.0, reduction=0.0), + "competitor_1": Score(score=0.0, reduction=0.0), + "competitor_2": Score(score=0.0, reduction=0.0), + "competitor_3": Score(score=0.0, reduction=0.0), + "competitor_4": Score(score=0.0, reduction=0.0), } # Set up the configuration with multiple competitions and multiple competitors - rewarder_config = WinnersMapping( - competition_leader_map=competition_leaders, - hotkey_score_map=scores - ) + winners_store = CompetitionWinnersStore( + competition_leader_map=competition_leaders, hotkey_score_map=scores) + + rewarder = Rewarder(winners_store) + await rewarder.update_scores(winner_hotkey="competitor_1", competition_id="competition_1", + winner_model_result=ModelEvaluationResult(score=0.9)) - rewarder = Rewarder(rewarder_config) - rewarder.update_scores() # Check the updated scores and reductions for the multiple competitors updated_scores = {hotkey: score.score for hotkey, score in rewarder.scores.items()} - updated_reductions = {hotkey: score.reduction for hotkey, score in rewarder.scores.items()} + updated_reductions = { + hotkey: score.reduction for hotkey, score in rewarder.scores.items() + } + updated_model_scores = {competition_id: leader.model_result.score for competition_id, leader in winners_store.competition_leader_map.items()} # With multiple competitors and some reduced shares, they should receive different scores and reductions expected_reductions = { - "competitor1": 1/6 * 0.3, - "competitor2": (1/6 * 0.6) + (1/6 * 0.3), - "competitor3": 1/6 * 0.9, - "competitor4": 0.0, + "competitor_1": 1 / 6 * 0.3, + "competitor_2": (1 / 6 * 0.6) + (1 / 6 * 0.3), + "competitor_3": 1 / 6 * 0.9, + "competitor_4": 0.0, } expected_reductions_sum = sum(expected_reductions.values()) expected_scores = { - "competitor1": (2/6 - expected_reductions["competitor1"]) + expected_reductions_sum/2, - "competitor2": (2/6 - expected_reductions["competitor2"]), - "competitor3": 1/6 - expected_reductions["competitor3"], - "competitor4": 1/6 + expected_reductions_sum/2, + "competitor_1": (2 / 6 - expected_reductions["competitor_1"]) + + expected_reductions_sum / 2, + "competitor_2": (2 / 6 - expected_reductions["competitor_2"]), + "competitor_3": 1 / 6 - expected_reductions["competitor_3"], + "competitor_4": 1 / 6 + expected_reductions_sum / 2, } + expected_model_score = 0.9 + + + for _, score in updated_model_scores.items(): + assert ( + score == expected_model_score + ), f"Expected score: {expected_model_score}, got: {score}" for hotkey, score in updated_scores.items(): - assert score == pytest.approx(expected_scores[hotkey], rel=1e-9), f"Expected score: {expected_scores[hotkey]}, got: {score} for {hotkey}" + assert score == pytest.approx( + expected_scores[hotkey], rel=1e-9 + ), f"Expected score: {expected_scores[hotkey]}, got: {score}" for hotkey, reduction in updated_reductions.items(): - assert reduction == pytest.approx(expected_reductions[hotkey], rel=1e-9), f"Expected reduction: {expected_reductions[hotkey]}, got: {reduction} for {hotkey}" + assert reduction == pytest.approx( + expected_reductions[hotkey], rel=1e-9 + ), f"Expected reduction: {expected_reductions[hotkey]}, got: {reduction}" + if __name__ == "__main__": pytest.main() diff --git a/cancer_ai/validator/tests/mock_data.py b/cancer_ai/validator/tests/mock_data.py new file mode 100644 index 00000000..bd9cccc9 --- /dev/null +++ b/cancer_ai/validator/tests/mock_data.py @@ -0,0 +1,29 @@ +from cancer_ai.validator.model_manager import ModelInfo + +def get_mock_hotkeys_with_models(): + return { + # good model + "hfsss_OgEeYLdTgrRIlWIdmbcPQZWTdafatdKfSwwddsavDfO": ModelInfo( + hf_repo_id="Kabalisticus/test_bs_model", + hf_model_filename="good_test_model.onnx", + hf_repo_type="model", + ), + # Model made from image, extension changed + "hfddd_OgEeYLdTgrRIlWIdmbcPQZWTfsafasftdKfSwwvDf": ModelInfo( + hf_repo_id="Kabalisticus/test_bs_model", + hf_model_filename="false_from_image_model.onnx", + hf_repo_type="model", + ), + # Good model with wrong extension + "hf_OgEeYLdTslgrRfasftdKfSwwvDf": ModelInfo( + hf_repo_id="Kabalisticus/test_bs_model", + hf_model_filename="wrong_extension_model.onx", + hf_repo_type="model", + ), + # good model on safescan + "wU2LapwmZfYL9AEAWpUR6sasfsaFoFvqHnzQ5F71Mhwotxujq": ModelInfo( + hf_repo_id="safescanai/test_dataset", + hf_model_filename="best_model.onnx", + hf_repo_type="dataset", + ), + } \ No newline at end of file diff --git a/config/competition_config.json b/config/competition_config.json new file mode 100644 index 00000000..89f810a0 --- /dev/null +++ b/config/competition_config.json @@ -0,0 +1,13 @@ +[ + { + "competition_id": "melanoma-1", + "category": "skin", + "evaluation_times": [ + "10:00", + "22:00" + ], + "dataset_hf_repo": "safescanai/melanoma-competition", + "dataset_hf_filename": "melanoma-1-dataset.zip", + "dataset_hf_repo_type": "dataset" + } +] \ No newline at end of file diff --git a/config/competition_config_testnet.json b/config/competition_config_testnet.json new file mode 100644 index 00000000..8b4f0310 --- /dev/null +++ b/config/competition_config_testnet.json @@ -0,0 +1,12 @@ +[ + { + "competition_id": "melanoma-testnet", + "category": "skin", + "evaluation_times": [ + "14:38" + ], + "dataset_hf_repo": "safescanai/test_dataset", + "dataset_hf_filename": "test_dataset.zip", + "dataset_hf_repo_type": "dataset" + } +] \ No newline at end of file diff --git a/config/hotkey_blacklist.json b/config/hotkey_blacklist.json new file mode 100644 index 00000000..0637a088 --- /dev/null +++ b/config/hotkey_blacklist.json @@ -0,0 +1 @@ +[] \ No newline at end of file diff --git a/config/hotkey_blacklist_testnet.json b/config/hotkey_blacklist_testnet.json new file mode 100644 index 00000000..02f17ae0 --- /dev/null +++ b/config/hotkey_blacklist_testnet.json @@ -0,0 +1,22 @@ +[ + "5C8P5k5LeyAUn7pLm15xULAxKryL1MCiwgjdWdxPXdveKwXu", + "5DDJWUUxoQFgohcrhXzsak4tiHjTh4kpWBE464fVRdQKRBux", + "5Dk3j1xu3HR9eLfqyGCrUFYRnwMykp6w1Y74yK3AjwkX3B8E", + "5EsMmcgqNf9F3RMz3SRrUEJ4STvESYr54jh7ECZmrX5q3rXi", + "5DUNBtqiEFoDvtFdvrBD3sRJjRdJ4cTVmmdqieuNrZ5TYbx1", + "5CoLBVrWTQpvTYMnjtakc9iWEFR348Y2YzSWgFd2qkgtkJzb", + "5DZZnwU2LapwmZfYL9AEAWpUR6FoFvqHnzQ5F71Mhwotxujq", + "5GW4dh1mrVzCHfEHZoiDFkxJWHPAQYcfeRUtmt2vnbSof5d1", + "5CY1JgzJnMC6HJ88GQVGRzgsvEbN6LDhG8tqVU1FQGtvGFHL", + "5EeawchJxAuAzA1taJHPCgDQ2NkgckjKve1mzmHobcSbo1g3", + "5Eq6oskxtzVc6vkFmg41XMuPf3628GdJp8zcSCnfrqWQie66", + "5HioSWDsL7dqbXaPuZCHmqocmtzn18WMPbusV8HnqHvPLnCk", + "5Fo2fenxPY1D7hgTHc88g1zrX2ZX17g8DvE5KnazueYefjN5", + "5HjWnpjn2rRzdRZySumzoLurEmzd5KBSwFn2LSMu4vaj7XLv", + "5Cf8iSNjnqNhjbZNUTZYZc9fLTT6D2vT8zNKjd2rBA2iGSAy", + "5FCPTnjevGqAuTttetBy4a24Ej3pH9fiQ8fmvP1ZkrVsLUoT", + "5CXf7G1CHUoyAf79oUnXAAhmkYGwEvQ52RNHVHyoqcB7WMLf", + "5CPMVnZ82XP7Koo1upGXyRphwt5DSV12FxztUyyq1w1GMEgy", + "5EHusPJSQr4VCMQTfEStNZtcQ3hga61x2c29YDRoYkFcuHX6", + "5CXDfdhpBQ9DFWFjEq3Tv7UCmSjzSnmjDfEXYTPoaKRPTTPh" +] \ No newline at end of file diff --git a/min_compute.yml b/min_compute.yml index 1da3bb04..27549efd 100644 --- a/min_compute.yml +++ b/min_compute.yml @@ -1,16 +1,3 @@ -# Use this document to specify the minimum compute requirements. -# This document will be used to generate a list of recommended hardware for your subnet. - -# This is intended to give a rough estimate of the minimum requirements -# so that the user can make an informed decision about whether or not -# they want to run a miner or validator on their machine. - -# NOTE: Specification for miners may be different from validators - -version: '1.0' # update this version key as needed, ideally should match your release version - -compute_spec: - miner: cpu: @@ -21,67 +8,55 @@ compute_spec: architecture: "x86_64" # Architecture type (e.g., x86_64, arm64) gpu: - required: True # Does the application require a GPU? - min_vram: 8 # Minimum GPU VRAM (GB) - recommended_vram: 24 # Recommended GPU VRAM (GB) - cuda_cores: 1024 # Minimum number of CUDA cores (if applicable) - min_compute_capability: 6.0 # Minimum CUDA compute capability - recommended_compute_capability: 7.0 # Recommended CUDA compute capability - recommended_gpu: "NVIDIA A100" # provide a recommended GPU to purchase/rent + required: False memory: min_ram: 16 # Minimum RAM (GB) min_swap: 4 # Minimum swap space (GB) - recommended_swap: 8 # Recommended swap space (GB) + recommended_swap: 12 # Recommended swap space (GB) ram_type: "DDR4" # RAM type (e.g., DDR4, DDR3, etc.) storage: - min_space: 10 # Minimum free storage space (GB) - recommended_space: 100 # Recommended free storage space (GB) + min_space: 500 # Minimum free storage space (GB) + recommended_space: 1000 # Recommended free storage space (GB) type: "SSD" # Preferred storage type (e.g., SSD, HDD) - min_iops: 1000 # Minimum I/O operations per second (if applicable) - recommended_iops: 5000 # Recommended I/O operations per second os: name: "Ubuntu" # Name of the preferred operating system(s) - version: 20.04 # Version of the preferred operating system(s) + version: 24.04 # Version of the preferred operating system(s) validator: cpu: - min_cores: 4 # Minimum number of CPU cores + min_cores: 8 # Minimum number of CPU cores min_speed: 2.5 # Minimum speed per core (GHz) recommended_cores: 8 # Recommended number of CPU cores recommended_speed: 3.5 # Recommended speed per core (GHz) architecture: "x86_64" # Architecture type (e.g., x86_64, arm64) gpu: - required: True # Does the application require a GPU? - min_vram: 8 # Minimum GPU VRAM (GB) - recommended_vram: 24 # Recommended GPU VRAM (GB) + required: False # we don't use GPU currently for running models, but we will + min_vram: 6 # Minimum GPU VRAM (GB) + recommended_vram: 12 # Recommended GPU VRAM (GB) cuda_cores: 1024 # Minimum number of CUDA cores (if applicable) min_compute_capability: 6.0 # Minimum CUDA compute capability recommended_compute_capability: 7.0 # Recommended CUDA compute capability - recommended_gpu: "NVIDIA A100" # provide a recommended GPU to purchase/rent + recommended_gpu: "NVIDIA RTX" # provide a recommended GPU to purchase/rent memory: - min_ram: 16 # Minimum RAM (GB) + min_ram: 64 # Minimum RAM (GB) min_swap: 4 # Minimum swap space (GB) - recommended_swap: 8 # Recommended swap space (GB) + recommended_swap: 12 # Recommended swap space (GB) ram_type: "DDR4" # RAM type (e.g., DDR4, DDR3, etc.) storage: - min_space: 10 # Minimum free storage space (GB) - recommended_space: 100 # Recommended free storage space (GB) + min_space: 500 # Minimum free storage space (GB) + recommended_space: 1000 # Recommended free storage space (GB) type: "SSD" # Preferred storage type (e.g., SSD, HDD) - min_iops: 1000 # Minimum I/O operations per second (if applicable) - recommended_iops: 5000 # Recommended I/O operations per second os: name: "Ubuntu" # Name of the preferred operating system(s) - version: 20.04 # Version of the preferred operating system(s) + version: 24.04 # Version of the preferred operating system(s) network_spec: - bandwidth: - download: 100 # Minimum download bandwidth (Mbps) - upload: 20 # Minimum upload bandwidth (Mbps) + bandwidth: \ No newline at end of file diff --git a/neurons/competition_config.json b/neurons/competition_config.json deleted file mode 100644 index c85d6112..00000000 --- a/neurons/competition_config.json +++ /dev/null @@ -1,32 +0,0 @@ -[ - { - "competition_id": "melanoma-1", - "category": "skin", - "evaluation_times": [ - "14:38" - ], - "dataset_hf_repo": "safescanai/test_dataset", - "dataset_hf_filename": "test_dataset.zip", - "dataset_hf_repo_type": "dataset" - }, - { - "competition_id": "melanoma-testnet", - "category": "skin", - "evaluation_times": [ - "16:39" - ], - "dataset_hf_repo": "safescanai/test_dataset", - "dataset_hf_filename": "test_dataset.zip", - "dataset_hf_repo_type": "dataset" - }, - { - "competition_id": "melanoma-7", - "category": "skin", - "evaluation_times": [ - "16:33" - ], - "dataset_hf_repo": "safescanai/test_dataset", - "dataset_hf_filename": "test_dataset.zip", - "dataset_hf_repo_type": "dataset" - } -] diff --git a/neurons/competition_runner.py b/neurons/competition_runner.py index d48d4ae5..20ddcce7 100644 --- a/neurons/competition_runner.py +++ b/neurons/competition_runner.py @@ -1,18 +1,16 @@ -from cancer_ai.validator.competition_manager import CompetitionManager -from datetime import datetime, time, timedelta -from pydantic import BaseModel -import asyncio import json -from datetime import datetime, timezone, timedelta +from typing import List, Tuple +from datetime import datetime, timezone, timedelta, time +import asyncio + + +from pydantic import BaseModel import bittensor as bt -from typing import List, Tuple, Dict -from cancer_ai.validator.rewarder import Rewarder, WinnersMapping, CompetitionLeader -import wandb -# from cancer_ai.utils.config import config +from cancer_ai.validator.competition_manager import CompetitionManager +from cancer_ai.chain_models_store import ChainMinerModelStore +from cancer_ai.validator.competition_handlers.base_handler import ModelEvaluationResult -# TODO MOVE SOMEWHERE -main_competitions_cfg = json.load(open("neurons/competition_config.json", "r")) MINUTES_BACK = 15 @@ -23,7 +21,11 @@ class CompetitionRun(BaseModel): end_time: datetime | None = None -class CompetitionRunLog(BaseModel): +class CompetitionRunStore(BaseModel): + """ + The competition run store acts as a cache for competition runs and provides checks for competition execution states. + """ + runs: list[CompetitionRun] def add_run(self, new_run: CompetitionRun): @@ -51,38 +53,47 @@ def was_competition_already_executed( return False -class CompetitionSchedulerConfig(BaseModel): +class CompetitionSchedule(BaseModel): config: dict[datetime.time, CompetitionManager] class Config: arbitrary_types_allowed = True -def config_for_scheduler( - bt_config, hotkeys: List[str], test_mode: bool = False -) -> CompetitionSchedulerConfig: +def get_competitions_schedule( + bt_config, + subtensor: bt.subtensor, + chain_models_store: ChainMinerModelStore, + hotkeys: List[str], + validator_hotkey: str, + test_mode: bool = False, +) -> CompetitionSchedule: """Returns CompetitionManager instances arranged by competition time""" scheduler_config = {} + main_competitions_cfg = json.load(open("config/competition_config.json", "r")) for competition_cfg in main_competitions_cfg: for competition_time in competition_cfg["evaluation_times"]: parsed_time = datetime.strptime(competition_time, "%H:%M").time() scheduler_config[parsed_time] = CompetitionManager( - bt_config, - hotkeys, - competition_cfg["competition_id"], - competition_cfg["category"], - competition_cfg["dataset_hf_repo"], - competition_cfg["dataset_hf_filename"], - competition_cfg["dataset_hf_repo_type"], + config=bt_config, + subtensor=subtensor, + hotkeys=hotkeys, + validator_hotkey=validator_hotkey, + chain_miners_store=chain_models_store, + competition_id=competition_cfg["competition_id"], + category=competition_cfg["category"], + dataset_hf_repo=competition_cfg["dataset_hf_repo"], + dataset_hf_id=competition_cfg["dataset_hf_filename"], + dataset_hf_repo_type=competition_cfg["dataset_hf_repo_type"], test_mode=test_mode, ) return scheduler_config async def run_competitions_tick( - competition_scheduler: CompetitionSchedulerConfig, - run_log: CompetitionRunLog, -) -> Tuple[str, str] | Tuple[None, None]: + competition_scheduler: CompetitionSchedule, + run_log: CompetitionRunStore, +) -> Tuple[str, str, ModelEvaluationResult] | Tuple[None, None, None]: """Checks if time is right and launches competition, returns winning hotkey and Competition ID. Should be run each minute.""" # getting current time @@ -118,48 +129,18 @@ async def run_competitions_tick( start_time=datetime.now(timezone.utc), ) ) - winning_evaluation_hotkey = await competition_manager.evaluate() + winning_evaluation_hotkey, winning_model_result = ( + await competition_manager.evaluate() + ) run_log.finish_run(competition_manager.competition_id) return ( winning_evaluation_hotkey, competition_manager.competition_id, + winning_model_result, ) bt.logging.debug( f"Did not find any competitions to run for past {MINUTES_BACK} minutes" ) - await asyncio.sleep(60) - return (None, None) - - -async def competition_loop_not_used( - scheduler_config: CompetitionSchedulerConfig, rewarder_config: WinnersMapping -): - """Example of scheduling coroutine""" - while True: - competition_result = await run_competitions_tick(scheduler_config) - bt.logging.debug(f"Competition result: {competition_result}") - if competition_result: - winning_evaluation_hotkey, competition_id = competition_result - rewarder = Rewarder(rewarder_config) - updated_rewarder_config = await rewarder.update_scores( - winning_evaluation_hotkey, competition_id - ) - # save state of self.rewarder_config - # save state of self.score (map rewarder config to scores) - print(".....................Updated rewarder config:") - print(updated_rewarder_config) - await asyncio.sleep(60) - - -if __name__ == "__main__": - # fetch from config - competition_config_path = "neurons/competition_config.json" - main_competitions_cfg = json.load( - open(competition_config_path, "r") - ) # TODO fetch from config - hotkeys = [] - bt_config = {} # get from bt config - scheduler_config = config_for_scheduler(bt_config, hotkeys) - rewarder_config = WinnersMapping(competition_leader_map={}, hotkey_score_map={}) - asyncio.run(competition_loop_not_used(scheduler_config, rewarder_config)) + await asyncio.sleep(20) + return (None, None, None) diff --git a/neurons/competition_runner_test.py b/neurons/competition_runner_test.py index 6575a7d4..1e7dee1a 100644 --- a/neurons/competition_runner_test.py +++ b/neurons/competition_runner_test.py @@ -1,14 +1,18 @@ -from cancer_ai.validator.competition_manager import CompetitionManager +import time import asyncio import json from types import SimpleNamespace -import bittensor as bt from typing import List, Dict -from cancer_ai.validator.rewarder import WinnersMapping, Rewarder -import time + +import bittensor as bt + + +from cancer_ai.validator.competition_manager import CompetitionManager +from cancer_ai.validator.rewarder import CompetitionWinnersStore, Rewarder from cancer_ai.base.base_miner import BaseNeuron -from cancer_ai.utils.config import path_config, add_miner_args -import copy +from cancer_ai.utils.config import path_config +from cancer_ai.mock import MockSubtensor +from cancer_ai.validator.exceptions import ModelRunException # TODO integrate with bt config test_config = SimpleNamespace( @@ -25,21 +29,29 @@ "dataset_dir": "/tmp/datasets", } ), + "hf_token": "HF_TOKEN" } ) -main_competitions_cfg = json.load(open("neurons/competition_config.json", "r")) +main_competitions_cfg = json.load(open("config/competition_config_testnet.json", "r")) async def run_all_competitions( - path_config: str, hotkeys: List[str], competitions_cfg: List[dict] + path_config: str, + subtensor: bt.subtensor, + hotkeys: List[str], + competitions_cfg: List[dict], ) -> None: """Run all competitions, for debug purposes""" for competition_cfg in competitions_cfg: bt.logging.info("Starting competition: ", competition_cfg) + competition_manager = CompetitionManager( path_config, + subtensor, hotkeys, + "WALIDATOR", + {}, competition_cfg["competition_id"], competition_cfg["category"], competition_cfg["dataset_hf_repo"], @@ -47,17 +59,22 @@ async def run_all_competitions( competition_cfg["dataset_hf_repo_type"], test_mode=True, ) + + bt.logging.info(await competition_manager.evaluate()) -def config_for_scheduler() -> Dict[str, CompetitionManager]: +def config_for_scheduler(subtensor: bt.subtensor) -> Dict[str, CompetitionManager]: """Returns CompetitionManager instances arranged by competition time""" time_arranged_competitions = {} for competition_cfg in main_competitions_cfg: for competition_time in competition_cfg["evaluation_time"]: time_arranged_competitions[competition_time] = CompetitionManager( - {}, # TODO fetch bt config Konrad + {}, + subtensor, [], + "WALIDATOR", + {}, competition_cfg["competition_id"], competition_cfg["category"], competition_cfg["dataset_hf_repo"], @@ -79,7 +96,9 @@ async def competition_loop(): ("hotkey2", "melanoma-3"), ] - rewarder_config = WinnersMapping(competition_leader_map={}, hotkey_score_map={}) + rewarder_config = CompetitionWinnersStore( + competition_leader_map={}, hotkey_score_map={} + ) rewarder = Rewarder(rewarder_config) for winning_evaluation_hotkey, competition_id in test_cases: @@ -101,4 +120,8 @@ async def competition_loop(): # BaseNeuron.check_config(config) bt.logging.set_config(config=config.logging) bt.logging.info(config) - asyncio.run(run_all_competitions(test_config, [], main_competitions_cfg)) + asyncio.run( + run_all_competitions( + test_config, MockSubtensor("123"), [], main_competitions_cfg + ) + ) diff --git a/neurons/miner.py b/neurons/miner.py index b1181fed..d378ea78 100644 --- a/neurons/miner.py +++ b/neurons/miner.py @@ -1,6 +1,7 @@ import asyncio import copy import time +import os import bittensor as bt from dotenv import load_dotenv @@ -15,7 +16,7 @@ from cancer_ai.validator.competition_manager import COMPETITION_HANDLER_MAPPING from cancer_ai.base.base_miner import BaseNeuron -from cancer_ai.chain_models_store import ChainMinerModel, ChainModelMetadataStore +from cancer_ai.chain_models_store import ChainMinerModel, ChainModelMetadata from cancer_ai.utils.config import path_config, add_miner_args @@ -65,6 +66,9 @@ async def upload_to_hf(self) -> None: @staticmethod def is_onnx_model(model_path: str) -> bool: """Checks if model is an ONNX model.""" + if not os.path.exists(model_path): + bt.logging.error("Model file does not exist") + return False try: onnx.checker.check_model(model_path) except onnx.checker.ValidationError as e: @@ -138,8 +142,8 @@ async def submit_model(self) -> None: f" Please register the hotkey using `btcli subnets register` before trying again" ) exit() - self.metadata_store = ChainModelMetadataStore( - subtensor=self.subtensor, subnet_uid=self.config.netuid, wallet=self.wallet + self.metadata_store = ChainModelMetadata( + subtensor=self.subtensor, netuid=self.config.netuid, wallet=self.wallet ) if not huggingface_hub.file_exists( @@ -179,10 +183,10 @@ async def submit_model(self) -> None: async def main(self) -> None: # bt.logging(config=self.config) - if not self.config.model_path: + if self.config.action != "submit" and not self.config.model_path: bt.logging.error("Missing --model-path argument") return - if not MinerManagerCLI.is_onnx_model(self.config.model_path): + if self.config.action != "submit" and not MinerManagerCLI.is_onnx_model(self.config.model_path): bt.logging.error("Provided model with is not in ONNX format") return diff --git a/neurons/tests/competition_runner_test.py b/neurons/tests/competition_runner_test.py new file mode 100644 index 00000000..57d1a0e8 --- /dev/null +++ b/neurons/tests/competition_runner_test.py @@ -0,0 +1,136 @@ +import asyncio +import json +from types import SimpleNamespace +from typing import List, Dict +import pytest + +import bittensor as bt + + +from cancer_ai.validator.competition_manager import CompetitionManager +from cancer_ai.validator.rewarder import CompetitionWinnersStore, Rewarder +from cancer_ai.base.base_miner import BaseNeuron +from cancer_ai.utils.config import path_config +from cancer_ai.mock import MockSubtensor + + +COMPETITION_FILEPATH = "config/competition_config_testnet.json" + +# TODO integrate with bt config +test_config = SimpleNamespace( + **{ + "wandb_entity": "testnet", + "wandb_project_name": "melanoma-1", + "competition_id": "melaonoma-1", + "hotkeys": [], + "subtensor": SimpleNamespace(**{"network": "test"}), + "netuid": 163, + "models": SimpleNamespace( + **{ + "model_dir": "/tmp/models", + "dataset_dir": "/tmp/datasets", + } + ), + "hf_token": "HF_TOKEN", + } +) + +main_competitions_cfg = json.load(open(COMPETITION_FILEPATH, "r")) + + +async def run_competitions( + config: str, + subtensor: bt.subtensor, + hotkeys: List[str], + competitions_cfg: List[dict], +) -> Dict[str, str]: + """Run all competitions, return the winning hotkey for each competition""" + results = {} + for competition_cfg in competitions_cfg: + bt.logging.info("Starting competition: ", competition_cfg) + + competition_manager = CompetitionManager( + config, + subtensor, + hotkeys, + {}, + competition_cfg["competition_id"], + competition_cfg["category"], + competition_cfg["dataset_hf_repo"], + competition_cfg["dataset_hf_filename"], + competition_cfg["dataset_hf_repo_type"], + test_mode=True, + ) + results[competition_cfg["competition_id"]] = ( + await competition_manager.evaluate() + ) + + bt.logging.info(await competition_manager.evaluate()) + + return results + + +def config_for_scheduler(subtensor: bt.subtensor) -> Dict[str, CompetitionManager]: + """Returns CompetitionManager instances arranged by competition time""" + time_arranged_competitions = {} + for competition_cfg in main_competitions_cfg: + for competition_time in competition_cfg["evaluation_time"]: + time_arranged_competitions[competition_time] = CompetitionManager( + {}, + subtensor, + [], + {}, + competition_cfg["competition_id"], + competition_cfg["category"], + competition_cfg["dataset_hf_repo"], + competition_cfg["dataset_hf_filename"], + competition_cfg["dataset_hf_repo_type"], + test_mode=True, + ) + return time_arranged_competitions + + +async def competition_loop(): + """Example of scheduling coroutine""" + while True: + test_cases = [ + ("hotkey1", "melanoma-1"), + ("hotkey2", "melanoma-1"), + ("hotkey1", "melanoma-2"), + ("hotkey1", "melanoma-1"), + ("hotkey2", "melanoma-3"), + ] + + rewarder_config = CompetitionWinnersStore( + competition_leader_map={}, hotkey_score_map={} + ) + rewarder = Rewarder(rewarder_config) + + for winning_evaluation_hotkey, competition_id in test_cases: + await rewarder.update_scores(winning_evaluation_hotkey, competition_id) + print( + "Updated rewarder competition leader map:", + rewarder.competition_leader_mapping, + ) + print("Updated rewarder scores:", rewarder.scores) + await asyncio.sleep(10) + + +@pytest.fixture +def competition_config(): + with open(COMPETITION_FILEPATH, "r") as f: + return json.load(f) + + +if __name__ == "__main__": + config = BaseNeuron.config() + bt.logging.set_config(config=config) + # if True: # run them right away + path_config = path_config(None) + # config = config.merge(path_config) + # BaseNeuron.check_config(config) + bt.logging.set_config(config=config.logging) + bt.logging.info(config) + asyncio.run( + run_competitions(test_config, MockSubtensor("123"), [], main_competitions_cfg) + ) diff --git a/neurons/validator.py b/neurons/validator.py index 534b958c..431708ab 100644 --- a/neurons/validator.py +++ b/neurons/validator.py @@ -22,42 +22,128 @@ import asyncio import os import traceback +import json import bittensor as bt import numpy as np import wandb -from cancer_ai.validator.rewarder import WinnersMapping, Rewarder, Score +from cancer_ai.chain_models_store import ChainModelMetadata, ChainMinerModelStore +from cancer_ai.validator.rewarder import CompetitionWinnersStore, Rewarder, Score from cancer_ai.base.base_validator import BaseValidatorNeuron from cancer_ai.validator.competition_manager import CompetitionManager from competition_runner import ( - config_for_scheduler, + get_competitions_schedule, run_competitions_tick, - CompetitionRunLog, + CompetitionRunStore, ) +RUN_EVERY_N_MINUTES = 15 # TODO move to config +BLACKLIST_FILE_PATH = "config/hotkey_blacklist.json" +BLACKLIST_FILE_PATH_TESTNET = "config/hotkey_blacklist_testnet.json" + class Validator(BaseValidatorNeuron): def __init__(self, config=None): super(Validator, self).__init__(config=config) - - self.competition_scheduler = config_for_scheduler( - self.config, self.hotkeys, test_mode=True + self.hotkey = self.wallet.hotkey.ss58_address + self.competition_scheduler = get_competitions_schedule( + bt_config = self.config, + subtensor = self.subtensor, + chain_models_store = self.chain_models_store, + hotkeys = self.hotkeys, + validator_hotkey = self.hotkey, + test_mode = False, ) bt.logging.info(f"Scheduler config: {self.competition_scheduler}") - self.rewarder = Rewarder(self.winners_mapping) + self.rewarder = Rewarder(self.winners_store) + self.chain_models = ChainModelMetadata( + self.subtensor, self.config.netuid, self.wallet + ) async def concurrent_forward(self): coroutines = [ + self.refresh_miners(), self.competition_loop_tick(), ] await asyncio.gather(*coroutines) + async def refresh_miners(self): + """ + downloads miner's models from the chain and updates the local store + """ + + if self.chain_models_store.last_updated is not None and ( + time.time() - self.chain_models_store.last_updated + < RUN_EVERY_N_MINUTES * 60 + ): + bt.logging.debug("Skipping model refresh, not enough time passed") + return + + bt.logging.info("Synchronizing miners from the chain") + bt.logging.info(f"Amount of hotkeys: {len(self.hotkeys)}") + + blacklist_file = ( + BLACKLIST_FILE_PATH_TESTNET + if self.config.test_mode + else BLACKLIST_FILE_PATH + ) + + with open(blacklist_file, "r") as f: + BLACKLISTED_HOTKEYS = json.load(f) + + for hotkey in self.hotkeys: + if hotkey in BLACKLISTED_HOTKEYS: + bt.logging.debug(f"Skipping blacklisted hotkey {hotkey}") + continue + + new_chain_miner_store = ChainMinerModelStore(hotkeys={}) + for hotkey in self.hotkeys: + hotkey = str(hotkey) + + # TODO add test mode for syncing just once. Then you have to delete state.npz file to sync again + # if hotkey in self.chain_models_store.hotkeys: + # bt.logging.debug(f"Skipping hotkey {hotkey}, already added") + # continue + + hotkey_metadata = await self.chain_models.retrieve_model_metadata(hotkey) + if not hotkey_metadata: + bt.logging.warning( + f"Cannot get miner model for hotkey {hotkey} from the chain, skipping" + ) + new_chain_miner_store.hotkeys[hotkey] = hotkey_metadata + + self.chain_models_store = new_chain_miner_store + hotkeys_with_models = [ + hotkey + for hotkey in self.chain_models_store.hotkeys + if self.chain_models_store.hotkeys[hotkey] + ] + + bt.logging.info( + f"Amount of miners: {len(self.chain_models_store.hotkeys)}, with models: {len(hotkeys_with_models)}" + ) + self.chain_models_store.last_updated = time.time() + self.save_state() + async def competition_loop_tick(self): + """Main competition loop tick.""" + + # for testing purposes + # self.run_log = CompetitionRunStore(runs=[]) + + self.competition_scheduler = get_competitions_schedule( + bt_config = self.config, + subtensor = self.subtensor, + chain_models_store = self.chain_models_store, + hotkeys = self.hotkeys, + validator_hotkey = self.hotkey, + test_mode = False, + ) try: - winning_hotkey, competition_id = await run_competitions_tick( - self.competition_scheduler, self.run_log + winning_hotkey, competition_id, winning_model_result = ( + await run_competitions_tick(self.competition_scheduler, self.run_log) ) except Exception: formatted_traceback = traceback.format_exc() @@ -69,7 +155,7 @@ async def competition_loop_tick(self): { "winning_evaluation_hotkey": "", "run_time": "", - "validator_id": self.wallet.hotkey.ss58_address, + "validator_hotkey": self.wallet.hotkey.ss58_address, "errors": str(formatted_traceback), } ) @@ -79,7 +165,7 @@ async def competition_loop_tick(self): if not winning_hotkey: return - wandb.init(reinit=True, project=competition_id, group="competition_evaluation") + wandb.init(project=competition_id, group="competition_evaluation") run_time_s = ( self.run_log.runs[-1].end_time - self.run_log.runs[-1].start_time ).seconds @@ -87,7 +173,7 @@ async def competition_loop_tick(self): { "winning_hotkey": winning_hotkey, "run_time_s": run_time_s, - "validator_id": self.wallet.hotkey.ss58_address, + "validator_hotkey": self.wallet.hotkey.ss58_address, "errors": "", } ) @@ -96,18 +182,20 @@ async def competition_loop_tick(self): bt.logging.info(f"Competition result for {competition_id}: {winning_hotkey}") # update the scores - await self.rewarder.update_scores(winning_hotkey, competition_id) - self.winners_mapping = WinnersMapping( + await self.rewarder.update_scores( + winning_hotkey, competition_id, winning_model_result + ) + self.winners_store = CompetitionWinnersStore( competition_leader_map=self.rewarder.competition_leader_mapping, hotkey_score_map=self.rewarder.scores, ) self.save_state() - hotkey_to_score_map = self.winners_mapping.hotkey_score_map - self.scores = [ np.float32( - hotkey_to_score_map.get(hotkey, Score(score=0.0, reduction=0.0)).score + self.winners_store.hotkey_score_map.get( + hotkey, Score(score=0.0, reduction=0.0) + ).score ) for hotkey in self.metagraph.hotkeys ] @@ -115,47 +203,69 @@ async def competition_loop_tick(self): def save_state(self): """Saves the state of the validator to a file.""" - bt.logging.info("Saving validator state.") + bt.logging.debug("Saving validator state.") # Save the state of the validator to file. - if not getattr(self, "winners_mapping", None): - self.winners_mapping = WinnersMapping( + if not getattr(self, "winners_store", None): + self.winners_store = CompetitionWinnersStore( competition_leader_map={}, hotkey_score_map={} ) + bt.logging.debug("Winner store empty, creating new one") if not getattr(self, "run_log", None): - self.run_log = CompetitionRunLog(runs=[]) + self.run_log = CompetitionRunStore(runs=[]) + bt.logging.debug("Competition run store empty, creating new one") + if not getattr(self, "chain_models_store", None): + self.chain_models_store = ChainMinerModelStore(hotkeys={}) + bt.logging.debug("Chain model store empty, creating new one") np.savez( self.config.neuron.full_path + "/state.npz", scores=self.scores, hotkeys=self.hotkeys, - rewarder_config=self.winners_mapping.model_dump(), + winners_store=self.winners_store.model_dump(), run_log=self.run_log.model_dump(), + chain_models_store=self.chain_models_store.model_dump(), ) + def create_empty_state(self): + bt.logging.info("Creating empty state file.") + np.savez( + self.config.neuron.full_path + "/state.npz", + scores=self.scores, + hotkeys=self.hotkeys, + winners_store=self.winners_store.model_dump(), + run_log=self.run_log.model_dump(), + chain_models_store=self.chain_models_store.model_dump(), + ) + return + def load_state(self): """Loads the state of the validator from a file.""" bt.logging.info("Loading validator state.") if not os.path.exists(self.config.neuron.full_path + "/state.npz"): - bt.logging.info("No state file found. Creating the file.") - np.savez( - self.config.neuron.full_path + "/state.npz", - scores=self.scores, - hotkeys=self.hotkeys, - rewarder_config=self.winners_mapping.model_dump(), - run_log=self.run_log.model_dump(), - ) - return + bt.logging.info("No state file found.") + self.create_empty_state() - # Load the state of the validator from file. - state = np.load(self.config.neuron.full_path + "/state.npz", allow_pickle=True) - self.scores = state["scores"] - self.hotkeys = state["hotkeys"] - self.winners_mapping = WinnersMapping.model_validate( - state["rewarder_config"].item() - ) - self.run_log = CompetitionRunLog.model_validate(state["run_log"].item()) + try: + # Load the state of the validator from file. + state = np.load( + self.config.neuron.full_path + "/state.npz", allow_pickle=True + ) + bt.logging.trace(state["chain_models_store"]) + self.scores = state["scores"] + self.hotkeys = state["hotkeys"] + self.winners_store = CompetitionWinnersStore.model_validate( + state["winners_store"].item() + ) + self.run_log = CompetitionRunStore.model_validate(state["run_log"].item()) + bt.logging.debug(state["chain_models_store"].item()) + self.chain_models_store = ChainMinerModelStore.model_validate( + state["chain_models_store"].item() + ) + except Exception as e: + bt.logging.error(f"Error loading state: {e}") + self.create_empty_state() # The main function parses the configuration and runs the validator. diff --git a/requirements.txt b/requirements.txt index 5c0a25d9..0dac1d2c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,19 +4,21 @@ aiohappyeyeballs==2.3.5 aiohttp==3.10.2 aiosignal==1.3.1 annotated-types==0.7.0 -ansible==6.7.0 -ansible-core==2.13.13 +ansible==8.5.0 +ansible-core==2.15.12 ansible-vault==2.1.0 anyio==4.4.0 astunparse==1.6.3 +async-timeout==4.0.3 async-unzip==0.3.6 attrs==24.2.0 backoff==2.2.1 base58==2.1.1 -bittensor==7.3.1 +bittensor==7.4.0 black==24.8.0 -certifi==2024.2.2 +certifi==2024.7.4 cffi==1.17.0 +cfgv==3.4.0 charset-normalizer==3.3.2 click==8.1.7 colorama==0.4.6 @@ -26,14 +28,17 @@ cryptography==42.0.8 cytoolz==0.12.3 ddt==1.6.0 decorator==5.1.1 +distlib==0.3.8 docker-pycreds==0.4.0 ecdsa==0.19.0 eth-hash==0.7.0 eth-keys==0.5.1 eth-typing==4.4.0 eth-utils==2.2.2 +exceptiongroup==1.2.2 fastapi==0.110.3 filelock==3.15.4 +flake8==7.1.1 flatbuffers==24.3.25 frozenlist==1.4.1 fsspec==2024.6.1 @@ -47,6 +52,7 @@ h11==0.14.0 h5py==3.11.0 huggingface-hub==0.24.5 humanfriendly==10.0 +identify==2.6.0 idna==3.7 iniconfig==2.0.0 Jinja2==3.1.4 @@ -57,6 +63,7 @@ libclang==18.1.1 Markdown==3.7 markdown-it-py==3.0.0 MarkupSafe==2.1.5 +mccabe==0.7.0 mdurl==0.1.2 ml-dtypes==0.4.0 more-itertools==10.4.0 @@ -71,7 +78,8 @@ namex==0.0.8 nest-asyncio==1.6.0 netaddr==1.3.0 networkx==3.3 -numpy==2.1.1 +nodeenv==1.9.1 +numpy==1.26.4 onnx==1.16.2 onnxruntime==1.19.0 opt-einsum==3.3.0 @@ -82,19 +90,23 @@ pathspec==0.12.1 pillow==10.4.0 platformdirs==4.2.2 pluggy==1.5.0 +pre-commit==3.8.0 protobuf==4.25.4 psutil==6.0.0 py==1.11.0 py-bip39-bindings==0.1.11 py-ed25519-zebra-bindings==1.0.1 py-sr25519-bindings==0.2.0 +pycodestyle==2.12.1 pycparser==2.22 pycryptodome==3.20.0 pydantic==2.8.2 pydantic_core==2.20.1 +pyflakes==3.2.0 Pygments==2.18.0 PyNaCl==1.5.0 pytest==8.3.2 +pytest-asyncio==0.24.0 python-dotenv==1.0.1 python-Levenshtein==0.25.1 python-statemachine==2.1.2 @@ -111,7 +123,6 @@ scikit-learn==1.5.1 scipy==1.14.1 sentry-sdk==2.13.0 setproctitle==1.3.3 -setuptools==74.1.1 shtab==1.6.5 six==1.16.0 smmap==5.0.1 @@ -122,18 +133,20 @@ sympy==1.13.1 tensorboard==2.17.1 tensorboard-data-server==0.7.2 tensorflow==2.17.0 +tensorflow-io-gcs-filesystem==0.37.1 termcolor==2.4.0 threadpoolctl==3.5.0 +tomli==2.0.1 toolz==0.12.1 torch==2.4.0 tqdm==4.66.5 typing_extensions==4.12.2 urllib3==2.2.2 uvicorn==0.30.0 +virtualenv==20.26.4 wandb==0.17.7 websocket-client==1.8.0 Werkzeug==3.0.3 -wheel==0.44.0 wrapt==1.16.0 xxhash==3.4.1 yarl==1.9.4 diff --git a/scripts/start_validator.py b/scripts/start_validator.py index 8d669122..18f4dbfd 100755 --- a/scripts/start_validator.py +++ b/scripts/start_validator.py @@ -33,7 +33,7 @@ UPDATES_CHECK_TIME = timedelta(minutes=5) CURRENT_WORKING_DIR = Path(__file__).parent.parent -ECOSYSTEM_CONFIG_PATH = CURRENT_WORKING_DIR / "ecosystem.config.js" # Path to the pm2 ecosystem config file +ECOSYSTEM_CONFIG_PATH = CURRENT_WORKING_DIR / "config" / "ecosystem.config.js" # Path to the pm2 ecosystem config file def get_version() -> str: """Extract the version as current git commit hash""" @@ -205,11 +205,11 @@ def main(pm2_name: str, args_namespace: Namespace, extra_args: List[str]) -> Non ) parser.add_argument( - "--subtensor.network", default="test", help="Name of the network." + "--subtensor.network", default="finney", help="Name of the network." ) parser.add_argument( - "--netuid", default="163", help="Netuid of the network." + "--netuid", default="46", help="Netuid of the network." ) parser.add_argument( From c63fc9463edcd9bfed0ea0c03424e9c461dd0ab8 Mon Sep 17 00:00:00 2001 From: Wojtek Jurkowlaniec Date: Thu, 12 Sep 2024 01:23:57 +0200 Subject: [PATCH 188/227] validator documentation fixes --- DOCS/validator.md | 2 +- README.md | 70 +---------------------------------------------- 2 files changed, 2 insertions(+), 70 deletions(-) diff --git a/DOCS/validator.md b/DOCS/validator.md index 6081462a..712d602e 100644 --- a/DOCS/validator.md +++ b/DOCS/validator.md @@ -27,7 +27,7 @@ Key features of the script include: - **zip and unzip** ### Wandb API key requirement -- Contact us [https://discord.com/channels/1259812760280236122/1262734148020338780](on discord) to get Wandb API key +- Contact us [on discord](https://discord.com/channels/1259812760280236122/1262734148020338780) to get Wandb API key - Put your key in .env.example file ## Installation and Setup diff --git a/README.md b/README.md index 29bd0d61..dba28f5b 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,6 @@ - [👨‍👨‍👦‍👦 Team Composition](#-team-composition) - [🛣️ Roadmap](#roadmap) - [✅ Pre requirments](#-pre-requirments) -- [📊 SETUP WandB (HIGHLY RECOMMENDED - VALIDATORS PLEASE READ)](#setup-wandb-highly-recommended---validators-please-read) - [👍 Running Validator](#-running-validator) - [⛏️ Running Miner](#-running-miner) - [🚀 Get invloved](#-get-involved) @@ -50,7 +49,7 @@ This repository contains subnet code to run on Bittensor network. ⚔️ Various cancer detection algorithm competitions -📊 WandB Visualization Dashboard +📊 Dashboard 💻 Specialized software for detecting other types of cancer @@ -177,73 +176,6 @@ To install BITTENSOR and set up a wallet follow instructions in this link: [PRE REQUIRMENTS](DOCS/prerequirements.md) -# **📊 SETUP WandB (HIGHLY RECOMMENDED - VALIDATORS PLEASE READ)** - -WandB is a valuable tool for tracking and visualizing machine learning experiments, and it helps log and monitor key metrics for miners and validators. - -Here’s a quick guide to setting up your WandB - -## **Instaliation** -To get started with WandB, you need to install the WandB Python package. - -``` -pip install wandb -``` - -## **Obtaining API key** - -1. Log into your **Weights & Biases** account in a browser. -2. Go to user settings and scroll down to **API keys** section. -3. Copy your API key to procede with next steps. - -## **Setting up the API key** - -After obtaining your API key, you need to set it up in your environment so that WandB can authenticate your account. - -1. Log into WANDB by running following command in your terminal: -``` -wandb login -``` -2. Enter your API key and press Enter - -## **Set API Key as Environment Variable (OPTIONAL)** -If you prefer not to log in every time, you can set your API key as an environment variable. - -### **Linux** - -To set the WANDB_API_KEY environment variable permanently on Linux, you’ll need to add it to your .bashrc (or .bash_profile, .profile, depending on your distribution and shell). -``` -echo 'export WANDB_API_KEY=your_api_key' >> ~/.bashrc -source ~/.bashrc -``` -Replace your_api_key with API key copied from Weights & Biases - -**Verification** - -``` -echo $WANDB_API_KEY -``` - -### Windows - -To set it permanently (system-wide), use the following steps: - - - Open Environment Variables Dialog: - - - Right-click on This PC or Computer on the Desktop or in File Explorer and select Properties. - - Click on Advanced system settings. - - In the System Properties window, click on the Environment Variables button. -- Add New System Variable: - - - In the Environment Variables window, click on New under the System variables section. - - Set the Variable name to WANDB_API_KEY and Variable value to your API key. - - Click OK to close all dialogs. - -**Verification** -``` -echo %WANDB_API_KEY% -``` - # **👍 RUNNING VALIDATOR** To run a validator follow instructions in this link: From 3ff1d394a6ee49b631ce1e1c810c19f81dfcbad7 Mon Sep 17 00:00:00 2001 From: Wojtek Jurkowlaniec Date: Thu, 12 Sep 2024 02:40:40 +0200 Subject: [PATCH 189/227] trld validator (#89) * trld validator * changelog * cancer ai logo --- CHANGELOG.md | 5 +++ DOCS/validator.md | 52 +++++++++++++++++++++++---- cancer_ai/validator/cancer_ai_logo.py | 34 ++++++++++++++++++ neurons/validator.py | 2 ++ 4 files changed, 87 insertions(+), 6 deletions(-) create mode 100644 cancer_ai/validator/cancer_ai_logo.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 451c35a9..5d60524f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # Changelog +## 0.6.1 + +- TLDR for validators +- Safe-Scan banner for validator script + ## 0.6 - added blacklisting of miners diff --git a/DOCS/validator.md b/DOCS/validator.md index 712d602e..08660acb 100644 --- a/DOCS/validator.md +++ b/DOCS/validator.md @@ -7,6 +7,7 @@ This documentation provides an overview of the validator script, its functionali The validator script is designed to run a validator process and automatically update it whenever a new version is released. This script was adapted from the [original script](https://github.com/macrocosm-os/pretraining/blob/main/scripts/start_validator.py) in the Pretraining Subnet repository. Key features of the script include: + - **Automatic Updates**: The script checks for updates periodically and ensures that the latest version of the validator is running by pulling the latest code from the repository and upgrading necessary Python packages. - **Command-Line Argument Compatibility**: The script now properly handles custom command-line arguments and forwards them to the validator (`neurons/validator.py`). - **Virtual Environment Support**: The script runs within the same virtual environment that it is executed in, ensuring compatibility and ease of use. @@ -16,17 +17,18 @@ Key features of the script include: ### Server requirements - - 64GB of RAM - - storage: 500GB, extendable - - GPU - nVidia RTX, 12GB VRAM (will work without GPU, but slower) +- 64GB of RAM +- storage: 500GB, extendable +- GPU - nVidia RTX, 12GB VRAM (will work without GPU, but slower) ### System requirements -- **Python 3.10 and virtualenv **: The script is written in Python and requires Python 3.10 to run. +- **Python 3.10 and virtualenv**: The script is written in Python and requires Python 3.10 to run. - **PM2**: PM2 must be installed and available on your system. It is used to manage the validator process. - **zip and unzip** ### Wandb API key requirement + - Contact us [on discord](https://discord.com/channels/1259812760280236122/1262734148020338780) to get Wandb API key - Put your key in .env.example file @@ -41,18 +43,20 @@ Key features of the script include: ``` 3. **Set Up Virtual Environment**: If you wish to run the script within a virtual environment, create and activate the environment before running the script: + ``` python3 -m venv venv source venv/bin/activate # On Windows use `venv\Scripts\activate` ``` 4. **Install Required Python Packages**: Install any required Python packages listed in requirements.txt: + ``` pip install -r requirements.txt ``` - ## Usage + To run the validator script, use the following command: ```bash @@ -69,7 +73,6 @@ python3 scripts/start_validator.py --wallet.name=my-wallet --wallet.hotkey=my-ho - `--netuid`: Specifies the Netuid of the network. Default is `"46"`. - `--logging.debug`: Enables debug logging if set to `1`. Default is `1`. - ## How It Works 1. **Start Validator Process**: The script starts the validator process using PM2, based on the provided PM2 process name. @@ -81,3 +84,40 @@ python3 scripts/start_validator.py --wallet.name=my-wallet --wallet.hotkey=my-ho - **Local Changes**: If you have made local changes to the codebase, the auto-update feature will attempt to preserve them. However, conflicts might require manual resolution. - **Environment**: The script uses the environment from which it is executed, so ensure all necessary environment variables and dependencies are correctly configured. + +# TLDR Installation script from fresh Ubuntu 24.04 + +```bash +# from root +apt install software-properties-common -y +add-apt-repository ppa:deadsnakes/ppa +apt update +apt install python3.10 python3.10-venv python3.10-dev python3-pip unzip +apt install python3-virtualenv git nodejs npm + +npm install pm2 -g + +adduser cancerai +su cancerai +cd + +git clone https://github.com/safe-scan-ai/cancer-ai +cd cancer-ai + +virtualenv --python=3.10 venv +source venv/bin/activate +pip install -U setuptools +pip install -r requirements.txt + +export PYTHONPATH="${PYTHONPATH}:./" + +# import keys + +cp .env.example .env + +# add wandb API key + +# example for testnet +python3 scripts/start_validator.py --wallet.name=validator-staked --wallet.hotkey=default --subtensor.network test --logging.debug 1 --netuid 163 + +``` diff --git a/cancer_ai/validator/cancer_ai_logo.py b/cancer_ai/validator/cancer_ai_logo.py new file mode 100644 index 00000000..667cb858 --- /dev/null +++ b/cancer_ai/validator/cancer_ai_logo.py @@ -0,0 +1,34 @@ +cancer_ai_logo = """ +@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ +@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ @@@@@@@@@@@@@@@@@@ +@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ @@@@@@@@@@@@@@@ +@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ @@@@@@@@@@@@@ +@@@@@@@@@@@@@@@@ @@@@@@@@@@@@@@@@@@ @@@@ @@@@@@@@@@@@ +@@@@@@@@@@ @@@@@@@@@@@ @@@@@@@@@@ @@@@@@@@@@@ +@@@@@@@ @@@@@@@@@ @@@@@@@@ @@@@@@@@@@@@ +@@@@@@ @@@@@@@@@@@@@@ @@@@@@@@ @@@@@@@@ @@@@@@@@@@@@ +@@@@@ @@@@@@@@@@@@@@@@@@ @@@@@@@@ @@@@@@ @@@@@@@@@@@@@ +@@@@@ @@@@@@@@@@@@@@@@@@ @@@@@@@@@ @@@@ @@@@@@@@@@@@@@ +@@@@@ @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ @@@@@@@@@@@@@@@ +@@@@@@ @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ @@@@@@@@@@@@@@@@ +@@@@@@@@ @@@@@@@@@@@@@@@@@@@@@@ @@@@@@@@@@@@@@@@@ +@@@@@@@@@@@ @@@@@@@@@@@@@@@@@ @@@@@@@@@@@@@@@@@@ +@@@@@@@@@@@@@@@@@@@@@ @@@@@@@@@@@@@@@@ @@@@@@@@@@@@@@@@@@@ +@@@@@@@@@@@@@@@@@@@@@@@@@@@@ @@@@@@@@@@@@ @@@@@@@@@@@@@@@@@ +@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ @@@@@@@@@@@ @@@@@@@@@@@@@@@@ +@@@@ @@@@@@@@@@@@@@@@@@@ @@@@@@@@@@ @@@@@@@@@@@@@@@ +@@@@@ @@@@@@@@@@@@@@@@@ @@@@@@@@@ @@ @@@@@@@@@@@@@ +@@@@@@ @@@@@@@@@@@ @@@@@@@@ @@@@ @@@@@@@@@@@@ +@@@@@@@@ @@@@@@@@@ @@@@@@@ @@@@@@@@@@ +@@@@@@@@@@@ @@@@@@@@@@@ @@@@@@@@@@ @@@@@@@@@ +@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ @@@@@@@@@@@@@ @@@@@@@ +@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ @@@@@@@@@@@@@@@ @@@@@@ +@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ +@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ +@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ +@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ +@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ +@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ +@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ +@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ +""" \ No newline at end of file diff --git a/neurons/validator.py b/neurons/validator.py index 431708ab..dd91b03b 100644 --- a/neurons/validator.py +++ b/neurons/validator.py @@ -37,6 +37,7 @@ run_competitions_tick, CompetitionRunStore, ) +from cancer_ai.validator.cancer_ai_logo import cancer_ai_logo RUN_EVERY_N_MINUTES = 15 # TODO move to config BLACKLIST_FILE_PATH = "config/hotkey_blacklist.json" @@ -44,6 +45,7 @@ class Validator(BaseValidatorNeuron): + print(cancer_ai_logo) def __init__(self, config=None): super(Validator, self).__init__(config=config) self.hotkey = self.wallet.hotkey.ss58_address From d6ba9f6899b3ad1111ddbe13b4621beb94317abd Mon Sep 17 00:00:00 2001 From: Wojtek Jurkowlaniec Date: Fri, 13 Sep 2024 14:14:54 +0200 Subject: [PATCH 190/227] fix documentation --- DOCS/1-MELANOMA.md | 17 ----------------- DOCS/COMPETITIONS.md | 2 +- Melanoma.md => DOCS/competitions/1-MELANOMA.md | 2 +- 3 files changed, 2 insertions(+), 19 deletions(-) delete mode 100644 DOCS/1-MELANOMA.md rename Melanoma.md => DOCS/competitions/1-MELANOMA.md (94%) diff --git a/DOCS/1-MELANOMA.md b/DOCS/1-MELANOMA.md deleted file mode 100644 index 33052f47..00000000 --- a/DOCS/1-MELANOMA.md +++ /dev/null @@ -1,17 +0,0 @@ -# Melanoma competition - -Training starts on `ENTER DATE` - - -## Overview - -### Dataset - -## Evaluation criteria - -### recorded metrics - -### Scoring mechanism - -## Links to dataset - diff --git a/DOCS/COMPETITIONS.md b/DOCS/COMPETITIONS.md index 77093dc9..29d3f1d4 100644 --- a/DOCS/COMPETITIONS.md +++ b/DOCS/COMPETITIONS.md @@ -39,7 +39,7 @@ Safe Scan organizes continuous competitions focused on cancer detection using ma ## Evaluation and Scoring - **Independent Evaluation**: Each validator independently evaluates the submitted models according to predefined criteria. -- **Scoring Mechanism**: Detailed scoring mechanisms are outlined in the [competition guidelines](https://huggingface.co/spaces/safescanai/dashboard) and [DOCS](/DOCS/competitions). Validators run scheduled competitions and assess the models based on these criteria. +- **Scoring Mechanism**: Detailed scoring mechanisms are outlined in the [DOCS](/DOCS/competitions) directory. Validators run scheduled competitions and assess the models based on these criteria. - **Winning Criteria**: The best-performing model, according to the evaluation metrics, is declared the winner of the competition. - **Rewards**: The winner receives the full emission for that competition, divided by the number of competitions held. - **Rewards time decay**: If a miner stays at the top position for more than 30 days, their rewards start to decrease gradually. Every 7 days after the initial 30 days, their share of the rewards decreases by 10%. This reduction continues until their share reaches a minimum of 10% of the original reward. diff --git a/Melanoma.md b/DOCS/competitions/1-MELANOMA.md similarity index 94% rename from Melanoma.md rename to DOCS/competitions/1-MELANOMA.md index b1186fb9..9a93d21f 100644 --- a/Melanoma.md +++ b/DOCS/competitions/1-MELANOMA.md @@ -59,7 +59,7 @@ The evaluation will be calculaded on following metrics with described weights. - **Output Format**: A numerical value between 0 and 1, represented as a `float`. This value indicates the likelihood or risk score of the area of concern warranting further investigation. ### Submission Requirements -- **Model Submission**: Models must be submitted in ONNX format. They should be capable of handling dynamic batch sizes and accept inputs with the shape `(None, 244, 244, 3)`, where `None` represents the batch dimension. This ensures that the model can process a variable number of images in a single batch. +- **Model Submission**: Models must be submitted in ONNX format. They should be capable of handling dynamic batch sizes and accept inputs with the shape `(batch , 3 , 244 , 244)`, where `batch` represents the batch dimension. This ensures that the model can process a variable number of images in a single batch. ## Rules and Guidelines From a28ecff2270a06c10e158c124682a94e78ffc32c Mon Sep 17 00:00:00 2001 From: notbulubula <101974829+notbulubula@users.noreply.github.com> Date: Fri, 13 Sep 2024 16:58:15 +0200 Subject: [PATCH 191/227] Fixing input sizes (#92) Fixing melanoma documentation --- DOCS/competitions/1-MELANOMA.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DOCS/competitions/1-MELANOMA.md b/DOCS/competitions/1-MELANOMA.md index 9a93d21f..5d09a738 100644 --- a/DOCS/competitions/1-MELANOMA.md +++ b/DOCS/competitions/1-MELANOMA.md @@ -59,7 +59,7 @@ The evaluation will be calculaded on following metrics with described weights. - **Output Format**: A numerical value between 0 and 1, represented as a `float`. This value indicates the likelihood or risk score of the area of concern warranting further investigation. ### Submission Requirements -- **Model Submission**: Models must be submitted in ONNX format. They should be capable of handling dynamic batch sizes and accept inputs with the shape `(batch , 3 , 244 , 244)`, where `batch` represents the batch dimension. This ensures that the model can process a variable number of images in a single batch. +- **Model Submission**: Models must be submitted in ONNX format. They should be capable of handling dynamic batch sizes and accept inputs with the shape `(batch , 3 , 224 , 224)`, where `batch` represents the batch dimension. This ensures that the model can process a variable number of images in a single batch. ## Rules and Guidelines From 29df589ddd9cebceeb6acb1325e05dcf2eae3677 Mon Sep 17 00:00:00 2001 From: konrad0960 <71330299+konrad0960@users.noreply.github.com> Date: Fri, 13 Sep 2024 18:10:42 +0200 Subject: [PATCH 192/227] Sync retry mechanism (#93) --- cancer_ai/base/neuron.py | 41 +++++++++++++++++++++++++++++++--------- 1 file changed, 32 insertions(+), 9 deletions(-) diff --git a/cancer_ai/base/neuron.py b/cancer_ai/base/neuron.py index 0e52feec..6999c67c 100644 --- a/cancer_ai/base/neuron.py +++ b/cancer_ai/base/neuron.py @@ -17,6 +17,9 @@ import copy import sys +import random +import time +import sys import bittensor as bt @@ -105,21 +108,40 @@ def __init__(self, config=None): @abstractmethod def run(self): ... - def sync(self): + def sync(self, retries=5, delay=10): """ Wrapper for synchronizing the state of the network for the given miner or validator. """ - # Ensure miner or validator hotkey is still registered on the network. - self.check_registered() + attempt = 0 + while attempt < retries: + try: + # Ensure miner or validator hotkey is still registered on the network. + self.check_registered() + + if self.should_sync_metagraph(): + self.resync_metagraph() + + if self.should_set_weights(): + self.set_weights() - if self.should_sync_metagraph(): - self.resync_metagraph() + # Always save state. + self.save_state() - if self.should_set_weights(): - self.set_weights() + break - # Always save state. - self.save_state() + except BrokenPipeError as e: + attempt += 1 + bt.logging.error(f"BrokenPipeError: {e}. Retrying...") + time.sleep(delay) + + except Exception as e: + attempt += 1 + bt.logging.error(f"Unexpected error occurred: {e}. Retrying...") + time.sleep(delay) + + if attempt == retries: + bt.logging.error("Failed to sync metagraph. Exiting...") + sys.exit(0) def check_registered(self): retries = 3 @@ -151,6 +173,7 @@ def should_sync_metagraph(self): """ Check if enough epoch blocks have elapsed since the last checkpoint to sync. """ + return ( self.block - self.metagraph.last_update[self.uid] ) > self.config.neuron.epoch_length From a3b84da97c6b6e89adf76bf3569fea7b58740bf9 Mon Sep 17 00:00:00 2001 From: CrypticMax <85819419+StudioMax971@users.noreply.github.com> Date: Sun, 15 Sep 2024 14:44:23 +0200 Subject: [PATCH 193/227] Update miner.py (#94) mainer - fix config for huggingface parameter --- neurons/miner.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/neurons/miner.py b/neurons/miner.py index d378ea78..e49638d9 100644 --- a/neurons/miner.py +++ b/neurons/miner.py @@ -49,7 +49,7 @@ async def upload_to_hf(self) -> None: path_or_fileobj=self.config.model_path, path_in_repo=hf_model_path, repo_id=self.config.hf_repo_id, - repo_type="model", + repo_type=self.config.hf_repo_type, token=self.config.hf_token, ) bt.logging.info("Uploading code to Hugging Face.") @@ -57,7 +57,7 @@ async def upload_to_hf(self) -> None: path_or_fileobj=self.code_zip_path, path_in_repo=hf_code_path, repo_id=self.config.hf_repo_id, - repo_type="model", + repo_type=self.config.hf_repo_type, token=self.config.hf_token, ) From 82790112f1cb967184d95268c4b039813222bdaf Mon Sep 17 00:00:00 2001 From: konrad0960 <71330299+konrad0960@users.noreply.github.com> Date: Thu, 19 Sep 2024 13:37:24 +0200 Subject: [PATCH 194/227] Setting version key (#102) * Setting version key to match subnet hyperparameter --- CHANGELOG.md | 4 ++++ cancer_ai/__init__.py | 3 +-- cancer_ai/base/base_validator.py | 3 ++- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5d60524f..4cd34add 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +## 0.6.2 + +- Establish version_key to match weights_version subnet hyperparameter + ## 0.6.1 - TLDR for validators diff --git a/cancer_ai/__init__.py b/cancer_ai/__init__.py index 4854a3f1..c295fd58 100644 --- a/cancer_ai/__init__.py +++ b/cancer_ai/__init__.py @@ -18,8 +18,7 @@ # DEALINGS IN THE SOFTWARE. # TODO(developer): Change this value when updating your code base. -# Define the version of the template module. -__version__ = "0.0.0" +__version__ = "0.6.2" version_split = __version__.split(".") __spec_version__ = ( (1000 * int(version_split[0])) diff --git a/cancer_ai/base/base_validator.py b/cancer_ai/base/base_validator.py index 2e5e5b6f..72ca28be 100644 --- a/cancer_ai/base/base_validator.py +++ b/cancer_ai/base/base_validator.py @@ -41,6 +41,7 @@ from cancer_ai.chain_models_store import ChainMinerModelStore from cancer_ai.validator.rewarder import CompetitionWinnersStore +from .. import __spec_version__ as spec_version class BaseValidatorNeuron(BaseNeuron): @@ -276,7 +277,7 @@ def set_weights(self): weights=uint_weights, wait_for_finalization=False, wait_for_inclusion=False, - version_key=self.spec_version, + version_key=spec_version ) if result is True: bt.logging.info("set_weights on chain successfully!") From b27cce8fbf1556ffeee591952b5b0a0619526432 Mon Sep 17 00:00:00 2001 From: Wojtek Jurkowlaniec Date: Mon, 30 Sep 2024 16:59:29 +0200 Subject: [PATCH 195/227] Extra wandb info for competitions, refactoring (#103) * add models and function for reading competitions file * fixes for competition runner --- cancer_ai/validator/models.py | 15 +++++++++++ cancer_ai/validator/utils.py | 16 +++++++++++- neurons/competition_runner.py | 18 ++++++------- neurons/competition_runner_test.py | 41 +++++++++++++++--------------- neurons/miner.py | 18 +++++++++---- 5 files changed, 72 insertions(+), 36 deletions(-) create mode 100644 cancer_ai/validator/models.py diff --git a/cancer_ai/validator/models.py b/cancer_ai/validator/models.py new file mode 100644 index 00000000..3598df32 --- /dev/null +++ b/cancer_ai/validator/models.py @@ -0,0 +1,15 @@ +from typing import List +from pydantic import BaseModel + +class CompetitionModel(BaseModel): + competition_id: str + category: str | None = None + evaluation_times: List[str] + dataset_hf_repo: str + dataset_hf_filename: str + dataset_hf_repo_type: str + + +class CompetitionsListModel(BaseModel): + competitions: List[CompetitionModel] + diff --git a/cancer_ai/validator/utils.py b/cancer_ai/validator/utils.py index bf4f285c..4b0d08c7 100644 --- a/cancer_ai/validator/utils.py +++ b/cancer_ai/validator/utils.py @@ -2,6 +2,8 @@ import os import asyncio import bittensor as bt +import json +from cancer_ai.validator.models import CompetitionsListModel, CompetitionModel class ModelType(Enum): @@ -17,6 +19,7 @@ class ModelType(Enum): import time from functools import wraps + def log_time(func): @wraps(func) async def wrapper(*args, **kwargs): @@ -24,10 +27,14 @@ async def wrapper(*args, **kwargs): result = await func(*args, **kwargs) end_time = time.time() module_name = func.__module__ - bt.logging.trace(f"'{module_name}.{func.__name__}' took {end_time - start_time:.4f}s") + bt.logging.trace( + f"'{module_name}.{func.__name__}' took {end_time - start_time:.4f}s" + ) return result + return wrapper + def detect_model_format(file_path) -> ModelType: _, ext = os.path.splitext(file_path) @@ -70,3 +77,10 @@ async def run_command(cmd): # Return the output and error if any return stdout.decode(), stderr.decode() + + +def get_competition_config(path: str) -> CompetitionsListModel: + with open(path, "r") as f: + competitions_json = json.load(f) + competitions = [CompetitionModel(**item) for item in competitions_json] + return CompetitionsListModel(competitions=competitions) diff --git a/neurons/competition_runner.py b/neurons/competition_runner.py index 20ddcce7..88e50f16 100644 --- a/neurons/competition_runner.py +++ b/neurons/competition_runner.py @@ -10,7 +10,7 @@ from cancer_ai.validator.competition_manager import CompetitionManager from cancer_ai.chain_models_store import ChainMinerModelStore from cancer_ai.validator.competition_handlers.base_handler import ModelEvaluationResult - +from cancer_ai.validator.utils import get_competition_config MINUTES_BACK = 15 @@ -70,9 +70,9 @@ def get_competitions_schedule( ) -> CompetitionSchedule: """Returns CompetitionManager instances arranged by competition time""" scheduler_config = {} - main_competitions_cfg = json.load(open("config/competition_config.json", "r")) - for competition_cfg in main_competitions_cfg: - for competition_time in competition_cfg["evaluation_times"]: + main_competitions_cfg = get_competition_config(bt_config.competition.config_path) + for competition_cfg in main_competitions_cfg.competitions: + for competition_time in competition_cfg.evaluation_times: parsed_time = datetime.strptime(competition_time, "%H:%M").time() scheduler_config[parsed_time] = CompetitionManager( config=bt_config, @@ -80,11 +80,11 @@ def get_competitions_schedule( hotkeys=hotkeys, validator_hotkey=validator_hotkey, chain_miners_store=chain_models_store, - competition_id=competition_cfg["competition_id"], - category=competition_cfg["category"], - dataset_hf_repo=competition_cfg["dataset_hf_repo"], - dataset_hf_id=competition_cfg["dataset_hf_filename"], - dataset_hf_repo_type=competition_cfg["dataset_hf_repo_type"], + competition_id=competition_cfg.competition_id, + category=competition_cfg.category, + dataset_hf_repo=competition_cfg.dataset_hf_repo, + dataset_hf_id=competition_cfg.dataset_hf_filename, + dataset_hf_repo_type=competition_cfg.dataset_hf_repo_type, test_mode=test_mode, ) return scheduler_config diff --git a/neurons/competition_runner_test.py b/neurons/competition_runner_test.py index 1e7dee1a..ba16c89a 100644 --- a/neurons/competition_runner_test.py +++ b/neurons/competition_runner_test.py @@ -13,6 +13,8 @@ from cancer_ai.utils.config import path_config from cancer_ai.mock import MockSubtensor from cancer_ai.validator.exceptions import ModelRunException +from cancer_ai.validator.utils import get_competition_config +from cancer_ai.validator.models import CompetitionModel, CompetitionsListModel # TODO integrate with bt config test_config = SimpleNamespace( @@ -29,22 +31,22 @@ "dataset_dir": "/tmp/datasets", } ), - "hf_token": "HF_TOKEN" + "hf_token": "HF_TOKEN", } ) -main_competitions_cfg = json.load(open("config/competition_config_testnet.json", "r")) +competitions_cfg = get_competition_config("config/competition_config.json") async def run_all_competitions( path_config: str, subtensor: bt.subtensor, hotkeys: List[str], - competitions_cfg: List[dict], + competitions_cfg: CompetitionsListModel, ) -> None: """Run all competitions, for debug purposes""" - for competition_cfg in competitions_cfg: - bt.logging.info("Starting competition: ", competition_cfg) + for competition_cfg in competitions_cfg.competitions: + bt.logging.info("Starting competition: ", competition_cfg.competition_id) competition_manager = CompetitionManager( path_config, @@ -52,34 +54,33 @@ async def run_all_competitions( hotkeys, "WALIDATOR", {}, - competition_cfg["competition_id"], - competition_cfg["category"], - competition_cfg["dataset_hf_repo"], - competition_cfg["dataset_hf_filename"], - competition_cfg["dataset_hf_repo_type"], + competition_cfg.competition_id, + competition_cfg.category, + competition_cfg.dataset_hf_repo, + competition_cfg.dataset_hf_filename, + competition_cfg.dataset_hf_repo_type, test_mode=True, ) - bt.logging.info(await competition_manager.evaluate()) def config_for_scheduler(subtensor: bt.subtensor) -> Dict[str, CompetitionManager]: """Returns CompetitionManager instances arranged by competition time""" time_arranged_competitions = {} - for competition_cfg in main_competitions_cfg: + for competition_cfg in competitions_cfg.competitions: for competition_time in competition_cfg["evaluation_time"]: time_arranged_competitions[competition_time] = CompetitionManager( - {}, + {}, subtensor, [], "WALIDATOR", {}, - competition_cfg["competition_id"], - competition_cfg["category"], - competition_cfg["dataset_hf_repo"], - competition_cfg["dataset_hf_filename"], - competition_cfg["dataset_hf_repo_type"], + competition_cfg.competition_id, + competition_cfg.category, + competition_cfg.dataset_hf_repo, + competition_cfg.dataset_hf_filename, + competition_cfg.dataset_hf_repo_type, test_mode=True, ) return time_arranged_competitions @@ -121,7 +122,5 @@ async def competition_loop(): bt.logging.set_config(config=config.logging) bt.logging.info(config) asyncio.run( - run_all_competitions( - test_config, MockSubtensor("123"), [], main_competitions_cfg - ) + run_all_competitions(test_config, MockSubtensor("123"), [], competitions_cfg) ) diff --git a/neurons/miner.py b/neurons/miner.py index e49638d9..443d3f2d 100644 --- a/neurons/miner.py +++ b/neurons/miner.py @@ -18,6 +18,7 @@ from cancer_ai.base.base_miner import BaseNeuron from cancer_ai.chain_models_store import ChainMinerModel, ChainModelMetadata from cancer_ai.utils.config import path_config, add_miner_args +from cancer_ai.validator.utils import get_competition_config class MinerManagerCLI: @@ -31,6 +32,10 @@ def __init__(self, config=None): BaseNeuron.check_config(self.config) bt.logging.set_config(config=self.config.logging) + self.competition_config = get_competition_config( + self.config.competition.config_path + ) + @classmethod def add_args(cls, parser: argparse.ArgumentParser): """Method for injecting miner arguments to the parser.""" @@ -85,9 +90,9 @@ async def evaluate_model(self) -> None: dataset_manager = DatasetManager( self.config, self.config.competition.id, - "safescanai/test_dataset", - "test_dataset.zip", - "dataset", + self.competition_config.competitions[0].dataset_hf_repo, + self.competition_config.competitions[0].dataset_hf_filename, + self.competition_config.competitions[0].dataset_hf_repo_type, ) await dataset_manager.prepare_dataset() @@ -102,7 +107,7 @@ async def evaluate_model(self) -> None: start_time = time.time() y_pred = await run_manager.run(X_test) run_time_s = time.time() - start_time - + # print(y_pred) model_result = competition_handler.get_model_result(y_test, y_pred, run_time_s) bt.logging.info( @@ -182,11 +187,14 @@ async def submit_model(self) -> None: ) async def main(self) -> None: + # bt.logging(config=self.config) if self.config.action != "submit" and not self.config.model_path: bt.logging.error("Missing --model-path argument") return - if self.config.action != "submit" and not MinerManagerCLI.is_onnx_model(self.config.model_path): + if self.config.action != "submit" and not MinerManagerCLI.is_onnx_model( + self.config.model_path + ): bt.logging.error("Provided model with is not in ONNX format") return From 6a27d504421b2a884ef78955dcb1f52d21428da3 Mon Sep 17 00:00:00 2001 From: Wojtek Jurkowlaniec Date: Sat, 7 Dec 2024 19:11:31 +0100 Subject: [PATCH 196/227] WIP Dev (#108) * Subnet V3 Co-authored-by: konrad0960 <71330299+konrad0960@users.noreply.github.com> Co-authored-by: Konrad --- .gitignore | 5 +- .pylintrc | 14 ++ DOCS/COMPETITIONS.md | 47 ++-- DOCS/COMPETITIONS.md.old | 75 ++++++ DOCS/competitions/1-MELANOMA-V3.md.old | 70 ++++++ DOCS/competitions/1-MELANOMA.md | 4 +- DOCS/miner.md | 45 ++-- DOCS/onnx_runner/image.jpg | Bin 0 -> 22546 bytes .../onnx_runner/onnx_example_requirements.txt | 11 + DOCS/onnx_runner/onnx_example_runner.py | 40 +++ DOCS/validator.md | 5 +- cancer_ai/base/base_validator.py | 5 +- cancer_ai/chain_models_store.py | 11 +- cancer_ai/mock.py | 36 +-- cancer_ai/utils/config.py | 58 +++-- cancer_ai/validator/competition_manager.py | 36 ++- cancer_ai/validator/dataset_manager.py | 2 +- cancer_ai/validator/model_db.py | 236 ++++++++++++++++++ cancer_ai/validator/model_manager.py | 103 +++++++- .../validator/model_runners/onnx_runner.py | 136 +++++++--- cancer_ai/validator/models.py | 39 ++- cancer_ai/validator/tests/mock_data.py | 26 +- cancer_ai/validator/tests/test_model_db.py | 136 ++++++++++ cancer_ai/validator/utils.py | 146 +++++++++++ neurons/competition_runner.py | 17 +- neurons/competition_runner_test.py | 126 ---------- neurons/miner.py | 33 ++- neurons/tests/competition_runner_test.py | 81 ++---- neurons/validator.py | 194 ++++++++++---- requirements.txt | 2 + 30 files changed, 1307 insertions(+), 432 deletions(-) create mode 100644 .pylintrc create mode 100644 DOCS/COMPETITIONS.md.old create mode 100644 DOCS/competitions/1-MELANOMA-V3.md.old create mode 100644 DOCS/onnx_runner/image.jpg create mode 100644 DOCS/onnx_runner/onnx_example_requirements.txt create mode 100644 DOCS/onnx_runner/onnx_example_runner.py create mode 100644 cancer_ai/validator/model_db.py create mode 100644 cancer_ai/validator/tests/test_model_db.py delete mode 100644 neurons/competition_runner_test.py diff --git a/.gitignore b/.gitignore index 4b2aaf36..99c7f345 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,9 @@ __pycache__/ *.py[cod] *$py.class +.DS_Store +*.onnx + # C extensions *.so @@ -60,7 +63,7 @@ cover/ local_settings.py db.sqlite3 db.sqlite3-journal - +*.db # Flask stuff: instance/ .webassets-cache diff --git a/.pylintrc b/.pylintrc new file mode 100644 index 00000000..395ec94d --- /dev/null +++ b/.pylintrc @@ -0,0 +1,14 @@ +[MESSAGES CONTROL] +disable=W1203, # logging-fstring-interpolation + C0111, # missing-docstring + C0103, # invalid-name + C0114 # missing-module-docstring + +[FORMAT] +max-line-length=120 + +[BASIC] +good-names=i,j,k,ex,Run,_,id,ge + +[MASTER] +ignore=migrations diff --git a/DOCS/COMPETITIONS.md b/DOCS/COMPETITIONS.md index 29d3f1d4..e5441877 100644 --- a/DOCS/COMPETITIONS.md +++ b/DOCS/COMPETITIONS.md @@ -1,17 +1,12 @@ - - - - - # Safe Scan: Machine Learning Competitions for Cancer Detection -Welcome to **Safe Scan**, a platform dedicated to organizing machine learning competitions focused on cancer detection. Our goal is to foster innovation in developing accurate and efficient models for cancer detection using machine learning. Here, you can find all the details needed to participate, submit your models, and understand the evaluation process. +Welcome to **Safe Scan**, a platform dedicated to organizing machine learning competitions focused on cancer detection. Our goal is to foster innovation in developing accurate and efficient models for cancer detection using machine learning. Here, you will find all the details needed to participate, submit your models, and understand the evaluation process. ## Table of Contents 1. [Overview](#overview) -2. [Competition Schedule](#competition-schedule) -3. [Dataset and Model Submission](#dataset-and-model-submission) +2. [Competition Trigger and Data Batch Handling](#competition-trigger-and-data-batch-handling) +3. [Model Submission Requirements](#model-submission-requirements) 4. [Evaluation and Scoring](#evaluation-and-scoring) 5. [Configuration and Development](#configuration-and-development) 6. [Command-Line Interface (CLI) Tools](#command-line-interface-cli-tools) @@ -20,21 +15,21 @@ Welcome to **Safe Scan**, a platform dedicated to organizing machine learning co ## Overview -Safe Scan organizes continuous competitions focused on cancer detection using machine learning. These competitions aim to advance the field by providing participants with the opportunity to develop and test their models in a structured environment. +Safe Scan organizes dynamic competitions focused on cancer detection using machine learning. These competitions provide participants with the opportunity to develop and test their models in a responsive and adaptive environment driven by real-world data. -## Competition Schedule +## Competition Trigger and Data Batch Handling -- **Frequency**: Competitions are held multiple times a day, at specific hours, continuously. This allows participants to join at different times that suit them best. -- **Timed Events**: Each competition starts with a dataset release 5 minutes before testing, providing a short window for participants to prepare. -- **Testing and Evaluation**: Models are evaluated immediately after each test, ensuring a quick feedback loop for participants. +- **Competition Initiation**: Competitions are triggered by data batch insertions from external medical institutions, creating a steady stream of new, non-public data for testing purposes. +- **Data Handling Process**: Medical institutions upload each new batch of data to a central reference repository on Hugging Face, along with a reference entry for the new data batch file. +- **Automatic Detection and Competition Start**: Validators monitor this centralized repository for new data batch entries. When new data is detected, validators initiate a competition by downloading and processing the data batch. -## Dataset and Model Submission +## Model Submission Requirements -- **Dataset Release**: A new dataset is provided for each competition, which is released exactly 5 minutes before testing begins. This dataset is used for training the models. -- **Model Submission**: Participants, referred to as "miners," are required to submit their trained models at the end of each competition. - - **Format**: All models must be in ONNX format. This ensures uniform testing and allows for broad deployment options, including on mobile and web platforms. - - **Training Code**: Each submission should include the code used for training the model to ensure transparency and reproducibility. - - **Upload Process**: Models are uploaded to Hugging Face at the end of each test. Miners then submit the Hugging Face repository link on the blockchain for evaluation by validators. +- **Model Submission**: Participants, referred to as miners, must submit their trained models at the end of each competition. +- **Format**: All models must be in ONNX format. This ensures uniform testing and allows for broad deployment options, including on mobile and web platforms. +- **Training Code**: Each submission should include the code used for training the model to ensure transparency and reproducibility. +- **Upload Process**: Models are uploaded to Hugging Face at the end of each test. Miners then submit the Hugging Face repository link on the blockchain for evaluation by validators. +- **Timing Constraint**: Only models submitted at least 30 minutes before the competition start time are eligible for evaluation. This requirement ensures that models have not been retrained with the new data batch, maintaining fairness and integrity across the competition. ## Evaluation and Scoring @@ -42,14 +37,8 @@ Safe Scan organizes continuous competitions focused on cancer detection using ma - **Scoring Mechanism**: Detailed scoring mechanisms are outlined in the [DOCS](/DOCS/competitions) directory. Validators run scheduled competitions and assess the models based on these criteria. - **Winning Criteria**: The best-performing model, according to the evaluation metrics, is declared the winner of the competition. - **Rewards**: The winner receives the full emission for that competition, divided by the number of competitions held. -- **Rewards time decay**: If a miner stays at the top position for more than 30 days, their rewards start to decrease gradually. Every 7 days after the initial 30 days, their share of the rewards decreases by 10%. This reduction continues until their share reaches a minimum of 10% of the original reward. +- **Rewards Time Decay**: If a miner stays in the top position for more than 30 days, their rewards start to decrease gradually. Every 7 days after the initial 30 days, their share of the rewards decreases by 10%. This reduction continues until their share reaches a minimum of 10% of the original reward. -## Configuration and Development - -- **Competition Configuration**: Each competition is configured through a `competition_config.json` file. This file defines all parameters and rules for the competition and is used by both miners and validators. -- **Tracking Changes**: Changes to the competition configuration are tracked via a GitHub issue tracker, ensuring transparency and allowing for community input. -- **Software Lifecycle**: The project follows a structured software lifecycle, including Git flow and integration testing. This ensures robust development practices and encourages community contributions. - ## Command-Line Interface (CLI) Tools - **Local Testing**: Miners are provided with an easy-to-use command-line interface (CLI) for local testing of their models. This tool helps streamline the process of testing models, uploading to Hugging Face, and submitting to the competition. @@ -65,6 +54,11 @@ Stay connected and up-to-date with the latest news, discussions, and support: - **Twitter/X**: Follow us on [Twitter/X](https://x.com/SAFESCAN_AI) for announcements and highlights. - **Email**: Contact us directly at [info@safescanai.ai](mailto:info@safescanai.ai) for any inquiries or support. +## Development + +- **Software Lifecycle**: The project follows a structured software lifecycle, including Git flow and integration testing. These practices ensure robust development and encourage community contributions. + + ## Contribute We welcome contributions to this project! Whether you're interested in improving our codebase, adding new features, or enhancing documentation, your involvement is valued. To contribute: @@ -72,4 +66,3 @@ We welcome contributions to this project! Whether you're interested in improving - Follow our software lifecycle and Git flow processes. - Ensure all code changes pass integration testing. - Contact us on our [Safe Scan Discord channel](https://discord.gg/rbBu7WuZ) for more details on how to get started. - diff --git a/DOCS/COMPETITIONS.md.old b/DOCS/COMPETITIONS.md.old new file mode 100644 index 00000000..29d3f1d4 --- /dev/null +++ b/DOCS/COMPETITIONS.md.old @@ -0,0 +1,75 @@ + + + + + +# Safe Scan: Machine Learning Competitions for Cancer Detection + +Welcome to **Safe Scan**, a platform dedicated to organizing machine learning competitions focused on cancer detection. Our goal is to foster innovation in developing accurate and efficient models for cancer detection using machine learning. Here, you can find all the details needed to participate, submit your models, and understand the evaluation process. + +## Table of Contents + +1. [Overview](#overview) +2. [Competition Schedule](#competition-schedule) +3. [Dataset and Model Submission](#dataset-and-model-submission) +4. [Evaluation and Scoring](#evaluation-and-scoring) +5. [Configuration and Development](#configuration-and-development) +6. [Command-Line Interface (CLI) Tools](#command-line-interface-cli-tools) +7. [Communication Channels](#communication-channels) +8. [Contribute](#contribute) + +## Overview + +Safe Scan organizes continuous competitions focused on cancer detection using machine learning. These competitions aim to advance the field by providing participants with the opportunity to develop and test their models in a structured environment. + +## Competition Schedule + +- **Frequency**: Competitions are held multiple times a day, at specific hours, continuously. This allows participants to join at different times that suit them best. +- **Timed Events**: Each competition starts with a dataset release 5 minutes before testing, providing a short window for participants to prepare. +- **Testing and Evaluation**: Models are evaluated immediately after each test, ensuring a quick feedback loop for participants. + +## Dataset and Model Submission + +- **Dataset Release**: A new dataset is provided for each competition, which is released exactly 5 minutes before testing begins. This dataset is used for training the models. +- **Model Submission**: Participants, referred to as "miners," are required to submit their trained models at the end of each competition. + - **Format**: All models must be in ONNX format. This ensures uniform testing and allows for broad deployment options, including on mobile and web platforms. + - **Training Code**: Each submission should include the code used for training the model to ensure transparency and reproducibility. + - **Upload Process**: Models are uploaded to Hugging Face at the end of each test. Miners then submit the Hugging Face repository link on the blockchain for evaluation by validators. + +## Evaluation and Scoring + +- **Independent Evaluation**: Each validator independently evaluates the submitted models according to predefined criteria. +- **Scoring Mechanism**: Detailed scoring mechanisms are outlined in the [DOCS](/DOCS/competitions) directory. Validators run scheduled competitions and assess the models based on these criteria. +- **Winning Criteria**: The best-performing model, according to the evaluation metrics, is declared the winner of the competition. +- **Rewards**: The winner receives the full emission for that competition, divided by the number of competitions held. +- **Rewards time decay**: If a miner stays at the top position for more than 30 days, their rewards start to decrease gradually. Every 7 days after the initial 30 days, their share of the rewards decreases by 10%. This reduction continues until their share reaches a minimum of 10% of the original reward. + +## Configuration and Development + +- **Competition Configuration**: Each competition is configured through a `competition_config.json` file. This file defines all parameters and rules for the competition and is used by both miners and validators. +- **Tracking Changes**: Changes to the competition configuration are tracked via a GitHub issue tracker, ensuring transparency and allowing for community input. +- **Software Lifecycle**: The project follows a structured software lifecycle, including Git flow and integration testing. This ensures robust development practices and encourages community contributions. + +## Command-Line Interface (CLI) Tools + +- **Local Testing**: Miners are provided with an easy-to-use command-line interface (CLI) for local testing of their models. This tool helps streamline the process of testing models, uploading to Hugging Face, and submitting to the competition. +- **Automated Data Retrieval**: Code for automating the retrieval of training data for each competition is available to integrate with the model training process. The script is defined in [scripts/get_dataset.py](/scripts/get_dataset.py). + +## Communication Channels + +Stay connected and up-to-date with the latest news, discussions, and support: + +- **Discord**: Join our [Safe Scan Discord channel](https://discord.gg/rbBu7WuZ) and the Bittensor Discord in the #safescan channel for real-time updates and community interaction. +- **Dashboard**: Access the competition dashboard on [Hugging Face](https://huggingface.co/spaces/safescanai/dashboard). +- **Blog**: Visit our [blog](https://safe-scan.ai/news/) for news and updates. +- **Twitter/X**: Follow us on [Twitter/X](https://x.com/SAFESCAN_AI) for announcements and highlights. +- **Email**: Contact us directly at [info@safescanai.ai](mailto:info@safescanai.ai) for any inquiries or support. + +## Contribute + +We welcome contributions to this project! Whether you're interested in improving our codebase, adding new features, or enhancing documentation, your involvement is valued. To contribute: + +- Follow our software lifecycle and Git flow processes. +- Ensure all code changes pass integration testing. +- Contact us on our [Safe Scan Discord channel](https://discord.gg/rbBu7WuZ) for more details on how to get started. + diff --git a/DOCS/competitions/1-MELANOMA-V3.md.old b/DOCS/competitions/1-MELANOMA-V3.md.old new file mode 100644 index 00000000..fb709527 --- /dev/null +++ b/DOCS/competitions/1-MELANOMA-V3.md.old @@ -0,0 +1,70 @@ +# Description of Melanoma Competition + +## Overview +This competition invites participants to develop a machine learning model that **aids in detecting the possibility of melanoma**. The goal is to create a model that can identify patterns in data that are associated with an increased likelihood of melanoma in visual recognition. + +### Objective +The primary objective is to develop a model that can analyze photos taken by users of their skin lesions or areas of concern. +The model should **assist users** by providing a risk assessment or likelihood score that helps them decide if they should seek further medical advice. +As a result, best model will be released in Skin Scan mobile app to run locally on the phone, and a website that will host it, free for anyone to use. + +## Evaluation Criteria +Models will be evaluated based on described **performance metrics** of the model. +The evaluation will be calculaded on following metrics with described weights. + +### Performance Metrics + + The models will be assessed on the following metrics with the corresponding weights: + +| **Metric** | **Description** | **Weight** | +|-------------|-------------------------------------------------------|------------| +| **F-beta** | Prioritizes recall, with a high beta to emphasize it. $\beta = 2$ | 0.60 | +| **Accuracy**| Measures the overall correctness of predictions. | 0.30 | +| **AUC** | Evaluates the model's ability to distinguish classes. | 0.10 | + +### Mathematical Formulas + +1. **F-beta Score $F\_\beta\$** + + + $$F_\beta = \left(1 + \beta^2\right) \cdot \frac{\text{Precision} \cdot \text{Recall}}{\left(\beta^2 \cdot \text{Precision}\right) + \text{Recall}}$$ + + + Where: + - **$\beta$** is the weight of recall in the combined score + - in our case $\beta = 2$ for higher recall importance + +2. **Accuracy** + + $$\text{Accuracy} = \frac{\text{True Positives} + \text{True Negatives}}{\text{Total Number of Samples}}$$ + +3. **Area Under the Curve (AUC)** + + AUC is the area under the Receiver Operating Characteristic (ROC) curve. It is calculated using the trapezoidal rule: + + $$\text{AUC} = \int_0^1 \text{TPR} \, d(\text{FPR})$$ + + Where: + - **TPR** = True Positive Rate + - **FPR** = False Positive Rate + + +## Model Inputs and Outputs + +### Inputs +- **Input Format**: Multiple images in JPEG or PNG format. +- **Input Features**: During preprocessing, images are resized to 224x224 pixels. Images are converted to numpy arrays with a datatype of `np.float32`, normalized to the range [0, 1]. + +### Outputs +- **Output Format**: A numerical value between 0 and 1, represented as a `float`. This value indicates the likelihood or risk score of the area of concern warranting further investigation. + +### Submission Requirements +- **Model Submission**: Models must be submitted in ONNX format. They should be capable of handling dynamic batch sizes and accept inputs with the shape `(batch , 3 , 224 , 224)`, where `batch` represents the batch dimension. This ensures that the model can process a variable number of images in a single batch. + + +## Rules and Guidelines + +- **Timeline**: +- Competitions are triggered dynamically by new data batch uploads from external medical institutions, with no predefined schedule. Competitions may occur at any time based on the timing of new data insertions. +- Each time a new data batch is detected in the central reference repository on Hugging Face, a new competition is initiated immediately by validators. +- Results of competition will be available on the dashboard diff --git a/DOCS/competitions/1-MELANOMA.md b/DOCS/competitions/1-MELANOMA.md index 5d09a738..fb709527 100644 --- a/DOCS/competitions/1-MELANOMA.md +++ b/DOCS/competitions/1-MELANOMA.md @@ -65,6 +65,6 @@ The evaluation will be calculaded on following metrics with described weights. ## Rules and Guidelines - **Timeline**: - - every day competition will be run one or more times a day. Timings are defined in [competition_config.json](config/competition_config.json) - - couple of minutes before start of competition, new part of dataset will be published for testing. +- Competitions are triggered dynamically by new data batch uploads from external medical institutions, with no predefined schedule. Competitions may occur at any time based on the timing of new data insertions. +- Each time a new data batch is detected in the central reference repository on Hugging Face, a new competition is initiated immediately by validators. - Results of competition will be available on the dashboard diff --git a/DOCS/miner.md b/DOCS/miner.md index 4279eab1..d09a798b 100644 --- a/DOCS/miner.md +++ b/DOCS/miner.md @@ -14,7 +14,7 @@ Key features of the script include: ## Prerequisites -- **Python 3.10**: The script is written in Python and requires Python 3.10 to run. +- **Python 3.12**: The script is written in Python and requires Python 3.12 to run. - **Virtual Environment**: It's recommended to run the script within a virtual environment to manage dependencies. - **8GB RAM**: minimum required operating memory for testing (evaluate) machine learning model locally @@ -30,7 +30,7 @@ Key features of the script include: 1. **Create a Virtual Environment** ```bash - virtualenv venv --python=3.10 + virtualenv venv --python=3.12 source venv/bin/activate ``` @@ -93,40 +93,42 @@ This mode performs the following tasks: To evaluate a model locally, use the following command: ``` -python neurons/miner.py --action evaluate --competition.id --model_path +python neurons/miner.py --action evaluate --competition_id --model_path ``` Command line argument explanation -- `--action` - action to perform , choices are "upload", "evaluate", "submit" +- `--action` - action to perform, choices are "upload", "evaluate", "submit" - `--model_path` - local path of ONNX model -- `--competition.id` - ID of competition. List of current competitions are in [competition_config.json](config/competition_config.json) +- `--competition_id` - ID of competition. List of current competitions are in [competition_config.json](config/competition_config.json) - `--clean-after-run` - it will delete dataset after evaluating the model +- `--model_dir` - path for storing models (default: "./models") +- `--dataset_dir` - path for storing datasets (default: "./datasets") ### Upload to HuggingFace This mode compresses the code provided by `--code-path` and uploads the model and code to HuggingFace. +Repository ID should be a repository type "model" To upload to HuggingFace, use the following command: ```bash python neurons/miner.py \ --action upload \ - --competition.id \ + --competition_id \ --model_path \ --code_directory \ --hf_model_name \ - --hf_repo_id \ - --hf_repo_type model \ - --hf_token + --hf_repo_id \ + --hf_token ``` Command line argument explanation - `--code_directory` - local directory of code - `--hf_repo_id` - hugging face repository ID - ex. "username/repo" -- `--hf_repo_type` - hugging face type of repository - "model" or "dataset" - `--hf_token` - hugging face authentication token +- `--hf_model_name` - name of file to store in hugging face repository ### Submit Model to Validators @@ -134,26 +136,25 @@ This mode saves model information in the metagraph, allowing validators to retri To submit a model to validators, use the following command: -``` +```bash python neurons/miner.py \ --action submit \ - --competition.id \ - --hf_code_filename \ - --hf_model_name \ - --hf_repo_id \ - --hf_repo_type model \ - --wallet.name \ - --wallet.hotkey \ - --netuid \ - --subtensor.network \ - --logging.debug + --competition_id melanoma-1\ + --hf_code_filename skin_melanoma_small.zip\ + --hf_model_name best_model.onnx \ + --hf_repo_id safescanai/test_dataset \ + --hf_repo_type dataset \ + --wallet.name miner2 \ + --wallet.hotkey default \ + --netuid 163 \ + --subtensor.network test ``` Command line argument explanation - `--hf_code_filename` - name of file in hugging face repository containing zipped code - `--hf_model_name` - name of file in hugging face repository containing model -- `--wallet.name` - name of wallet coldkey used for authentication with Bittensor network +- `--wallet.name` - name of wallet coldkey used for authentication with Bittensor network - `--wallet.hotkey` - name of wallet hotkey used for authentication with Bittensor network - `--netuid` - subnet number - `--subtensor.network` - Bittensor network to connect to - diff --git a/DOCS/onnx_runner/image.jpg b/DOCS/onnx_runner/image.jpg new file mode 100644 index 0000000000000000000000000000000000000000..d3cf9436826df0c03d84c75963ea46820f8a0b5b GIT binary patch literal 22546 zcmbTdWl$Vn^es9GE4#s_uiNH;oZBt`c!xI>8`!1PoI7E+N&St9@YRP%JNF`fG1A?fG3X|;9(K) z0r2$6fA(Md^gqMI_^-vn#KgeF#=^$_pB)De7aIo;2OArg02dGcKYKhPB)}*9@8rLS z{I6S|Vqsul;p1TA{7;wvt>)o3fE4#XpD>=h0z4&sfPaUR_ucYd&w(vwL!Sc7Abr_4oSbKV1Le{J-u06Bp?tuBZRK1DyYG zJ$dT=xMPrFV!h;?F>@t<#rGBWxoko%q63eaU-Ov4+-(A%lI8t6>&bu6{s-Cr zKVaYfkC6R8!2Vxc3jjilCyy@=gA^bI*zxdtA0^b7Tz9eQvSOdI|L}2-3Mh`*|lLcwEnP!>c*hAP}E0JQBx@jBkqS)``E--YLYW>oe6uDli7m2 zWjSl#ydmaS8jH~IRvfjzKWshieu=Epy5I)ptxBQc;H}Z5GTM3sgnr^{O`R_kyj|DA zzy1u!>`@jPKP>8c&xkVSmfuy;Eptpa<+~ZbjLpA|v#C7M8Zn>p>8=(ez#b3}v?zi< z6Y5q63mN}nl3$3zd-dV#HwBGpWcE&-L0AV((1H9dK|7)Ia92epWT9dp|!6wX$_ZBz>s-i>lGO`Ht~?#5{Uw(0k^EmTwGa$ zrF~{70Z?D(D+VLqsd}s9j=}j0Y0*#mM8acILkkUt*|MBwmx+TG0RM6YoVg4*P9%#C z-t%S^itCQ+fY!YNWU|9naTiycr*1^_^hTU;DvRAxYP-0-dKdvOjo_d>E5Vc1PJ zgJHlUC+qNHar!R@!>M}>V76{od%-v2^ZbNqn1Suedo_zef}W;*Om2CFV#tivnS#QS zdK&HiXl9b#sKDd3oYAHOoX;)wwyYV)jSEkMMTylo>q5&G4PRObi#m$NPe|tEw)4QA zrT!vN2-ApR-!o7x=?kEoY+-n(V6e?VkooWL=9#`ZJ|B+njzi66__*r3{slGlptFqK zp%aSIQDv2n{&gI&!nI3d&8^W&Qce9aSsZV#N{q-&AUKk~PqLCi^sg$puffK;rnGGI z5e3?5DjBpN_{4~M6gThvkrhr_`WkucrvtklT7NTMGz_zC_wXI4-nB1~x#%l!s2iZ( zkNDbK@vernG-^|A&ThqAVJW4DE>GxGM|J*+`8kERwv1;8wHvoS#kV)M! z$kMf7aZ#)VQTaKGOmXerUl-c$xDj0yR6Wlu4dn4biD!8{04{U`bJv!E1NxS27Ozfw zwEes2aSB{|58UMRE?hj;BZ_IJGENmp9oNt7Z7CgFs>uL@L7}LO;n~AiHC!=+dW28= z+OH{AjKLbAe&tJv+St^4pts$tmNwt*<^g*I$ds@llKuH*0WNvhMijwo( z?~GH<$^pVPv0|X5(~qlCnlE*_i2j(KV2~`Hxa+;}ZRyZrHw`4#f5EfGFZWrZ_F2eE zv&5@yzi14;e(E3CYcOFJud8zEX$@V4nT^vQW<{k^o(B&A%*<{n}513tAtvb(XUY~)1L1kI?drH;UBoT zx&d;_G3dOul&`VX^k_Re;}!VQl4=>wLbU0MOai@bl!|n4rW7N*zp;{cn?LlmYJGZF zi(8TMMbmMj%Bw=;Xe?+)ziIn>J!wHF$tQQb?ocC17Y}ylt>1>Gs`nL>Dd6yTzP*R`V zwZ-JxRCD2|BGgqoL_vqm}HR41GW#=&eCB2&ej1%6STWfrY@ zOGPN7+pC;vVTo(qe-KwYNGd6G#;z{dDIvDwI^&a4($rmoW{rk2+}@c;ek`4V|8J1N zRw{FfAG%nbTr-)hu60yQ{+Ol0%PN5a$k-E8FLmeJUNk3|p{tlg%|`ShD+oGM;r{A_ zL)Q>5;@S%>rEJ;q)*F^t`WsIpg%FH!dsV?klC|YwgV3=Q8@bl&d1LF+kzD}#+8V!} z(tzGEGX84AeSj2_*FkN^L*zM$wWjELE(_%3b+4s`x%cikXiX>!qa6m_%8?v)_ zqnze@>(JL3mj>Az=}u+%gpQH-`n-MX_kQC;_VzSQVd5?g^nMwe?^nNjp@wRsaB%1j zaGd-g0h~W4ELxEIF-5XD%}r846$1OFMa{2opvKHNz-P2G6Vxl5m2n=xXUyM+LpX>D z-Ip{$5@lh=oVxe`W@e0vdWMwgRj?6E4uWRIy$;5RvP6a-f} zZ3JV+orL#jPM=%n&trV6fB`{no_(4j(|h_#9g##`+JJOGjm+X#<$zHL(e z=1~~-WynIpzqZ{G4S#Vt!k=Ev{Znb@B^$&^2jR4{Z(dxK3&&b$HH56B4A0MB^B!@z zznO1%%i?`|eY;mS{wm^o{lRAuoUb`&88>u7EV|$Sg3M#MCR?LsQ>Cn5o(hpB+6#4i zdviq9>Gll7KYjAM&tDfA<-~pP!z}!BSS8-mYUtY+`rH_cHHon^OY|a1`puT;966!HCJz7| zH%FL6>uC=x(k$M8@iU#1QUKYQj=FCO#MN*gCDMcmEOim(5}NiY4>{w3YhU7sbb)m^{ig{Y(SXA9Aiwho?_b4u%*tp~sl=m!9$jcva1(g0Ts z;decu#+x8$tzjpW8`Ar{?u>8$i31dR8~C*OE$v^g!^@reHx3ZN3itb6m$rmT2R=A+Dp;2$qza`w%+n0D#N_3)i zYFig#!)*RlUeO`-LMq$bmbWg_u|ZO1px-GEnR7N`RDI%*=H=*Aj^Qkh7{gd-Y1$== zV74CIVAqDeoX$g7{Zq8HfLPGHOn4{Tu(Nj!dsY#}cu{SkUFTv$W z3aCbv(K4@-2)EfYn7#xG@kln_mMq^wmj9N=yXObzh(0Pi1#eotVJ6ihfHQ^qzkvv!au2Wwh07y70 zEXF)Ls>6bp!d1}Cj#D%J7!p^YC!g@U#0vtPE#KkqgB^X%9o5=4Z*gB-$WPsv-_T8v z>AcO85H6T9SC!Qw8>nZofVvvNy~YQ1IJBcL;kAK<%NITQb&?(!>>^g9e&)sc%L^k+ zk@@&2VR`1F2f(X>>4Rr*#Mh@OIw)s)U+a_d&k}1^XWYM^c)u@b+c^LU?|-s6-SNvx z_vQGN0{wnaPBsblPnBB$95!vFSroO3zAxpJ|pnY_uii47J9XB~; zjw_+_<{w-KiJW0;U*dh3#lNaY1KWk<6%W;B*R`MG9n4m3&T!>s)A;md3AZ>Qc(uBuU< zRC4*SSe%v3T@1XGape2@Qdkc2b_q3gp5Id2)aa={@L6F={trL27RD{`8Z~{kLYZ%% zHStQTsO*G0myrnreHc*kOijfSd|Pz*lkNEYT0{e>j^(GGF*H^z6no^ffAHcTALfrkmqMNh;mbX}45vu=UOi#} zjurZ`0T(;adH6&vwnh1-yHagEQ9O)_8HYXq%Tw3jOX_Ur`6sLFUQ69IClsu}r#4q) zdV=AmsOpv@)M4}H0(m+PFWgD33)5te=*Q2+pilv+yogs2&pRToVF(BV%?MC)MGdNB zW|u;tDWkiXFmrz`T`1*R3 ztM@_)>e}m5O<1TMFOywF`V0u~l zhA(A@E3j!x{@x){JD}HnJ2N!(FUaQU+$(CqwZ?;sJI=Ufq*}j$z#Y=a!!!}Q@g6`^TK$62V+oS@#{&8dmYW~ z{Z6Zo%>b1a(2Q2ooLY5z3D^fm#a&a{ARicXTQ}U1o+nS0LTIg zmCjY!aSCOp;qV?v4St{WEU;mGshswHi>9Ra{QhO5U_$F30%l$;%1@_uEySK&ogbd` z9zKx(6vJz@p~Bl))uOqk7y(ihzLSZA?A*aw31?(rbj9^;XbuiPrz!sPUAN+t&;L2E z{|Z#J7F%{5_0u_E?pEP?EwoUeq!E&k<$i;t*Ui(=%vfqlcQWyf4APBcv;4L|-qLr4 z===tL07%YgPJL`jeZ`ZO)UkPHkRYB|dpRV?xV*t@#3 zHz1AS5UF$Zvz`3*%@Fzl#Vx3Kfm>6mD4kqKw6w19hts)pg|=bpC!PohUQI6*@e;hqX)mx$esliL~{+;f9= z{?`j)ACv2*flA@>ILy>=RlXYmi}x|Ty76JpI(h?B$o}kIcJU!Tj0PI}2HliMsGM{NU^jy6ur1o!zoTXg@T{(OT83;I&L zs?_s!Emjq=ySOEdUSDPg>Onc@Xp}Y`nu|yGp&=5S@ahSIogQX0>VH2IRweAhBmFB$ z+2h|GjL8_Bt;%|wQIzSK>j|y~sv5nK&oUY#rscS>TbV@`_6e1a6?D0AIjqsXqT-I( zyh+lpH|9uFNy$$XN_^*|LkCfpXy-`WU66}s8eq9XKOc&H9su#%OP}NPN5r?pX@Ruu zM_U6`42O>j@JUs2v><=OPF_uSnuiU$6=Px>8y2_;268~`^HY7g)TFVVu;EOd!@zqT znszk#NWLFN(wzXX)`(cgJtbtY-{dE@d`uV%`h+6ukDN}+kE?0sRI{q-^-{(n@R$bv zxcp&=m(uV43&zQof8AG?Xkg;Th7ygutPJy6UxBaxfaH_?m&^QN=bj+So08|7Ka(mm2* z4Xw}b3QmX`i$9L*r{WvFtd7U_DU*n4mG$1o-U3mv~r|cVKf#$ER zZNr@Jn==T$p5AwUf55Ztx$52(+_A|~dIPmx)|kM&zbfnJpY=aD<%)MvFDZ6N#q$3) z#`l8*|JiG0Mu;m^C-i9<$C(vm(uRWTF*~jmM~}|#DYDEPfg(p}pJf9(VPDNzPT(@s z74__`?gTgWc{K}Pg*sOj=^;n`*IHtdJ>ZIQUbEfn4^SN6^X0%v&VKy#nbVdE=EbP) z$^8Ql8=^%|Of*{K?@UqI`@=e68>`(m<8^j6o;$Vh$px_E2dcS}pQe39y`uO=28XJQ750pf^u-5UI=I( z2LN<+wz_7SvPWB@@aRW%*B6G1zBz@WWwS-!?w@2jV20(OQO<^Ao@kF?bo3B; z4W=o61*`BFmVNc)7U;fAk`Fzv=lJvx@%+59GaV8W9($YVGCo2#0$uKdr z$LhAu$YoD{V8)W>6WR1Y-~Q85|4w%0UnN@R+N(YuAYr%r88&9ot4^v>Ec}OW);dZE zE0XmeqjD|&;_s-zd6tmazT*b~^8=s)lwI-t*^nx)L+~(g8^w^!7NsfCxuKfp1>{D; zD%56%gp213y~UsD&d-h%%c0jD2VN?1xVQwUZ-Iai0TUAARlzh{8rDWoqgJX2eK7rf zEf?|1ddp+E@bl z73F25x=c}aKF5S;v?Na&*gJ~5Z}rUR`<^-`tCzZ2>R+-LSG--yP-oZ`C4kb+Yl;#T% zIXJJ7)Iq=a0#Uy9=Ov#x^!5!$I(6%{6>=1EqdS###9(fjZut& z{nYCcsn!!e{O(xVf=rE}`i~w5V|fPc+gdX`?O3Ik0Zk`B{7ax8RPa?87p+Tldxb?} z!{n@O+ydPXwsjeLDdK_;)_HveY6}hsb{$YKEkym1kSLsnJ32b`tPs4nI4nX}_@c!1 z;Y%u|0jcyql^X(k)vc=R2gj-QYQ8D#iuDc)X85^P0 zX^m99JR&+we_x>XW?T7tguEsUQ~kb>_PyvAy%!kRS$R4s@YQw0$biBsKIb))FCp>S zy}PkTZRy;0hAD0g$68^4Tav0-njviV{h@2^e7SUXkOzgCH9=j^%oWqmN! z4f1El`XU%jqSyVx^}4zjJQ}1tFl|xkvsqnFcq%XIV~uHD9>j5qpkT&3YTTO~{<)39 z%KdF$r}o9LVPNSyty|VTBiziyUr}*7avD44eEC}-G)MVJfL?OLbg~PZ`D33Nj>L(v z0?Gsjq{LrP{s55pvPNhz{p83vhR*JDi+T_jg!YA266yHALr3!!J-o$8Bf+1e{Gg-n zxBRc+uOe1 zhhAkh6&KfkBO){NFR@K4LMS|<|IX$FEN^xgVk9b*I;P zY&npzB2}~C6eUxq*DumnPDG40ip^g5!&!y5=~|hxJc@%nFLxWgUsan{ZM}mH?qn4U zgWRX;uWx|b)q3gjQ zk!ZdFzJP;oRGF3ic%3F1Y4wDuSdMWRV;y6aXUeVqN%VgRcTf~e#P(zO*0{JQK?`ci zSlOZZ&_ZHCFLLnsNVVF{IBV|C$)DHuat&}b^SD%A@ZAU~&VrU<;|Gee-#~Oj9POmDwypW9Qzd+`TtxOzOMCx!4EF+4ajC zs8|3el!JFtWBg{H6(rz$^(%FxexXlh-Ck8o1EJwoR?@|xjDCzbP0MdP1yf5SEoea& z+f-u>ee1v$iuLqa0^yyv)51GGqpHy5h5b5}@@~q4&_zB->ahe-Yl-7YS;N<9mBMtOdFe|<8NLj#wa1X8@09 zd#x3CqRr9)!8UhUfe-)kV^PVhMAh#x4KbP6&ivS_BA^1cr?b0HE#U6h%;u-)i?j}UuS8m9IYG2(`yf5LsS>Bil`doc9I zImJbkj>taBJ*(EK|HfYuTX>1PxF&IW&y(9uu{^{}gf$+0R^rV4D{skwZRD-yNXBS@ zsfrhOehtU|&s43Hy~S6hCVO6i3}-w%PHrwBd0ObL`$D7ThXwTv6f88E0JCEdWm-?FFvBT^-I8Bhyu7r4InEX1ep+QX!)g?Bxo4cos4*OiAtFPk_}6p38SJ)-3r~k{vU$3L9f0X0OQd z9nHRp{DwOp5U9$tPWdf>-X>se)SFtCEd!LJshiceYb^O!RI?RX^c$2YDAj_`wG&15{F#HyfQX zZz3A9WNXO1jAB$ZM>)Zt`bz)~*v4S$vc-OGD+FJS-6)u<*ZnWjuvE$FvUkqHYNe!A zUHQa0N|JTjpQ)nQz$Caxk(5OFe1N_YRiLB1RO**4!b4VQVI<2l*%szZ^EdggGB6>l zYUD~k?!@cH^Nu4T^g=#O%SgdsMuk2ztm(}Hk?>K{UnBu#YbiL*AnK@lO1!f1ZrEo% z;N;G}@sR~DOn$-pd+3Zf{fievic9{kfF$gXUJE57XIF`iBkY@I57v>Y17$^>4{H4vmn_@ZKwV)youI5y)t||Jo+MLyV?3f2Hg-M zT#3r8R3~JnW>UhvgC_q99|~eo>_Z!+uX%KoXKE*$-!Z6E4xhHkn9u)4Pr>ZATIe^s z%;+ZG-JLAApg5uGe}TphX~y+}CxU{HRT4G+JNp+jiMcHGh~vgzFhZRJ%K&xu)|p#F zffIZG5}$M`;f>}TT>V|&u*AZjVVO3oNnf*qB<)r1Ne2A47)`(PvPSA$UBXFytbcV~ z%Cq#((P~?^wRqehN7wjD&N&C+`gJY+z1zIqn%aMQO!@~-H)?*Yn~(NXmO^vx{IALw zpN|RDWu092D^V{2I-%`qKO`IABf5S{bavQZR@4U)d7Eq9eY^U~7E}p?W1NWk=U9A6 zk3e&6o`QL&f}L7PH9ydz4}kAEpIXAjL^QHDPX|k5cM*JByUoU{h)Nd0*3l zZ5_5)xqfn*Si}&r*5`4q5w{>j#*ugWx32mqQ2oJDI#KB1+q;RMd~ZwelZg^9SRI9& z%BVg*>+)LnmS`pkjcM8>KiIn$*e%m$ReIwoIYbbWmu4w`Fw4BGX@H$TV&R3OzV~-F?wh!f z1?C7Br98{o)37gSmjU!5w6?ttMpeW;mka4MJ3cMNUD5pqaZ<+z8QQkl>H83N;a0>A zTa4#SHr{!63h~RJS74+04jjIID2n|R4g{F1ZNizRC80=1m+lEf@c>%bsQyCX`8|UW zJd_=qg)?r;ppP=eW+KWLKK2$Pb2eL!=dkaQLZERiV!1-_r!pP&0iq7L{6#0;WGDu| z3C6P+m1xJG{_23(O3^h1vk+I9oK5DoPg_(sD1{-H(J(r~B&hHP!V8#`8i8(zS!jE0 zw_Tp)Np+ByEc?QC^ypnkJrYBrEycXch}`Ho)SH{uQX2vC^$?%5Sz(5zP_`uOZ|L@4(WO9V3&;4KaZ=QOuC2SUHt@bhs( za47wt?>DKgmBu5$-+%)q$hJ$_)5jW1k8eS6^nS7HYwg3{S$dzfK=DA&sqo?;PX=ca zD+)$s-${X+LWiBLuBkWAsB)uq`rFRNAtHGSpnt}5y{`W7SaGIpqp?kdX-SI_^UdMF; zqzmzMAa%J)w`kDZ{-JYN5mCZ6Z#{E`^K5f+bs~u+xa-X)o<~gv=n}!-MJ_-3g$+wK zW{n|;W3#*0u2l8U)HRK%UGoA5I%`LCG*|E^!iku!{COTbO5D`f{j4du*2|c~ zW`9DHYQIAQd!H`uY@z9ZW>2+Mi#?lV=&)BoU5)dPjZT#dr45WGzFRAfqso`AEp{Wb z0)O;o^>on(-M+3*wxdh% z{z}?hUvcGwg_ndxNq9u@9)UyUN~ISQ+1$T&)Cf;O5yuGP`vS=pL}C55RHRqk!=J2k zFjhB@a~09`x6Oz90{5prPHLH+plgl{s9-FX+rK#lk)i?{^lIomgk2+ns8E$5eY0Rd zD4iHxmU+uGnCKagLZS%kt4&TZpN7xwqe?p+w}_p-h?oMlUj{jjX$sH8g^r9%ou_+G z>Qdaxl$=fVSYp}Z<9+Z0Zgjm$*N6OUC@1|wn0Y?sjxmll{TpFj)vZ}%AbS}}wV7%v z(s48$XkK|S7glgyhHWhWlJXPks|j~ZuCUciw$O)$tGI7#(av}o*Okd)* zqUI)#4)>3+idlybl3#{PqZ;o6d#j3ddSR`N+vef+A*6WWQ}ZzdLiapBMF-=WS_kAPXqkOkxnk-$Pga zDzL?aG2Ffl`m*_bty$NclfNBuUBX>1AaK&Mcjwi>%D&>>A5l#(QoVKN{86Z21hoV7 zNj*x3ln=}%;TF2_DeMt;&}B{>kj^U=8&L#2v#QGqO3 zF)9LiO=J}23oO{;T%6~3Zhl;5bc`?999mdfbV&2zs3}zJ7^#PgYdF-|zS12SvOc}K(cDZ-vN?ZaR%BHN zdvU}+SKCgi53d?}Uqbmd(}xQ4Z2Vpd?sFJhwy2J>J@WV$%Iowp{5FDRldl+lSaD1f!b*T~}=4fTn)3ee!!$%t)fFo{-p~v zDp57;YSZ~OM889pEc`^?kU8OeyTRzi1|6v#&hQ!aP1OhrK-4h z9JuX-qkdAMLX=D6ZF7Opx8f%ja3m9QY{ByM=2$ghar^dUNN|>gL=h!|623qQ_uA8T zQ&%EQorx6@vAbaum0epnIBZEHo`MS_wJX^aS!XFqX1HtdsH_+g-%~$`-z-jyfFpza zGm09sgZz~#SpfIf8bMwpogZp48%SdSvl!_zKS}I9ou09SkEiJcc;P*Xx*8#8OaCK7 zB^g1z7#C#n`z*s5qSe_>G6ZUXh*)&!Qi+H^(i5{IHKo{vNR}5-y8;P%4KJw}(%ITS zJ(-Yx%(21l3MVgB-r}Y}8&%6VgHsF27OMb|qL&wA&lASoNnBsH?!Y z!!|`UECJ5p67s?ouxoP}Dq4*v157~%s5Uygx_3Lef9m}@Vwl}mry$Vua(FE*r#D$X z+KjHk9|=OUVJ1ll_G+dOeo1UgQ{23@Ea)*E;nH{~>!RTFNM7U^z_24=tW~Q*=U0kQ>ynCNK@-O98tMe!G>MlhHHbNQZmu~$1s2ntCGDWz# zdc1&6?h}jA&0KHugyclAia;5!q4XJo3-@llFBc{5K4yn*QO{d3Rz7A$C~4PCYBMIs zTwXr%Cbe9{8QC`@424xMxii6r;r^R?SH zI}w{GzLp954_L=M0;6j4-_=G-b7vc+k@-b_TUH9rmd0w%WXle#uJ}lf zZBi#iLfkShhxZc-T(N>*OY_;W|8HczR z>@*oAKK12#JNl1wK{X^n#uUbU!~2iuoNf71n%z|A7wkAlUV-eKi*AGwx~Lnk)-SC3~0g z;d-uR+^xPK-+)mYCZfWXU!Ft^x-RJ$ zHf%KEy{FX$hx%|)lr$V+e3-3x&m-;(It3NiH?d8H39Q}-GF zc;sd~CCQQHHT|`U8^fUwfHEf{&O!m&#+yK(a@ny)Dqj#oYS0gR)sx~DEYq5^a>3h8 zjI5iii_-YL`kP#uPbVUCct!&;_eQ*Z0u{$Y9hKr_uYW#W!WJ1&)nDSmRl#+#w zj_T^j_t%!wOpVy*;2~tIOgPW+90=)$6)Al!7^ceq9St_|IG)WQTv^ZG0k^z^8J9vVoK_^ zt&sET)~8lt_c4?aso9oxu-|nD`h!`FL!86HO@=ZJ@9!2p)_cPDUn9H#(Yv4rQq#0>V zQl(j<(6R7UIj7E4Z5;73K`Wo_m#b!Rl*POYU7Oc=9S4x0&X&xs2LM@qN}5KTy0u*A zAhQd&Jnx>oKLs=|FDS16tb@r{m$bsBXw{;x*HSnG4DqP7^coM@jq|j2&{ywQ+Vuk} zm_Dnm)Qx>7`2fI)nFWTFSCmnrRc6ZaAz-v;1~t;Y6njgLu}P-CV^cS^xol2Hc=!F)oeJG1v>DCMDV1sOs1KL_n|X zu$=9BCfmruyjctL&=gmz|2GSHQp_64Xr6^9VN^$Y^srHCGcaWkd}>>>s8r6H!2DVQ zLKo6-*&(no9XpewHTm0YontU_*FEs{%%Q(w5Mwwzt9(F6Im*3%584%XwliUC@cf-L zlcAS7w+zsM?DiB~9<*K?*4X&=y(&x9<_`eVTgmoKqyzWXw=MH53c}_zf?oI4g=ACs z(D+p~{zRAIGu0i6t7>S|xF@}n*Qs789d^fKxHCRC0gD{KnX95$KjOG&S;Hz&MzJL_ zPQ}|4p3KyvE?Baj5?8(EyE@YweEd0Qaa%_3i;H&8#nY|QA%5Hm6RH&kfi`9^gCZZ6V&t@eMtdknH?$}QQ+OUYTS zRR>d(-jycI#0rQlERQxF+LWamS0{u(i81|VkHY*F?B9PcLw~*cK;N*Fv59puqfzs( z0mNzH00S^cjiEIc?Bf4ie(*0PLNCCObQZT)<@*9QgB1dG!lkC!%AnNg!`07GhZ{<) z@$WF6DDLv>(w=}r>$_)atkK_g`HRnJFQfUZ#GU7#n4gX%#%jsMzj(Xu(rGZ5#mTzo%6k4G+ay%5b>@QuwGu>m8Z*`z&+(FaV z&2tv8p!-@cp;9}xe^^byxzd=VXEf(8)uyNO?}&v&#v}?n)u01)ElG%+HAkYA$$N<+ zL$G*4(p21*ae^&uK^C)W4O@7q9^ji8l>J4eEd1K~|tz5R!@wDol2nOs(+ z%_LYt1}K$i&)nn?XP~m}ZjpbC+Kw;cE`2iIqleh+5Y#&$`3gQ8QU&`mY3QdI3Vr4! z64W8$55G|_tSnn_mSm`*P8;I@EDS#40r9! zpH7UUvBK-I4ifLT-;0Y#1!F=*XTyIKvuJV3|IsEiQ+s47Sw8PHc`O>{7)AAtf^oG% znr3cqNBZ~kVz8nFT|#Vs-szD6?CF~~8(MkZ_N0-^nvdEt`0i;dUHMBHa$Zesw?{qlK z-adn5_E3(mEZ~~3(Cxd3U@#74tO+~-aEC3trhTG`}7cmG6kwN!gr zjy=uZFk+Z=6xgr_#k&xAJcG=jHb8cr=+`f(#aT_PxBZVcRPlpoB=f%jE9&heC zXpr~wuU@Hr!_stw4}gTI)18r#1{mUIR3j|hG4u*)6+pfI%$L@Wu~wq<+k}(m?^CJ% z!m=wN@39~z5%<$W3El&!gneHNGis8c_N#eUvsvMu!}eBUx#sz^IV#ZTiYB4)^5?z+WU&-^RKAZ zTrA0NVLsX0yVx^|ix6L&H}mt;sLg+wJ(q z(WDphnz~AvK8aJ?$emo&n0%~=Jpg=!HDBqQ04{pfrQ)!ge~7=acco!10(CK57jV}@ zM5IjAeaEK#xne5B_vxQr0I${Q-9+!?9P#Qo=T0PB!(BF`E?5}oYFjEv+Mq=Z7*1kA zH(?*JQ^=}ite+mrbcBh8J2e?LapI!VmS}7-I*?)a5 zA)%`3hN6nfmSx#Z>ujF_vSlk6(ZcsSNO?`zFGKRsVK)3hjiu8dyqz?f#stUpQ%tu5 zo_sPzw7{g&8uqNI;9Ifggv5Jv!<`*6%!AtDg4*#H=>wop@wCxc@Frx>s5hn1P>yfm zV{?H<3e%~&c%Xc16OS6*9kJH?k!<{{{DT=fthnDJeqEKB&9q4_E!8hk;(j_IB_FJv zFAb~AlM%YnPQ(l;dq1P?!RzLDfr1GMiNfUyea83^+Cla5&8(WAoM5>EQ=Tj?>(%JE zY=r#;T=I;+%QsqYIoeHMuNOa@4c2OqG$dMLthnzM=pno}_Ye3oV1v4*^iSe>3OEbL z!INHUY_f?QuD|mAFjr)yk)g5AU>{hj#_^9lwnImULc1n$>EQfMz29cbNeHob@4isi z!s+hg9klHeWdR7*whl3N{{;?RUH*>ihKNh}eXz?p-a>aWXTZIoN$Yd za+|x(a(XrqqVDCHk)3E#g%cYn6NWSW7`mokFi_OiP15^U8q2Yy(7E51oWJl&>J7u? zj#uhG(%AED(Y=I|E6y=(o_zhh^+GJK`_~ES*0)0+%2H?6k=vNybnbHeM3WYi zIt7~VD_DOa%zbDU7zKFfb$m-9IJ3WKnh*IGyFpl3MnT`t#wHQ|K^vS#qC($?@vBSx z8>Qa5sas4Y$su!Fb?RzA_!hrr7t7 z2UgpXTDH~3yG!Y{@G-68{|c5CY3V7?mU5A%S&g0BeZe3F_7F2dK2mCSiTa9IE<4z9qxo7IUHj=ewD{5x-?v`t3OG~ zl}xiOYpXh)1HyU?5k&IC8D@=L%1OWnfzKVs6{$aj>{*SSh~bXXa6VrCobg#YJgy*H zoi;^DnLu*Ooa4FY*w+4$sG{3VG_xygQ;(Mgn*+bsit^~nbt>5KF<6XTYD!X*Bew9( zhK5=TJTf{L=0Y>;*XvSB(=YDjQX(Ag9ZB@fXZWMU`ktHNcD=Q<65Jz{K%An&$amVtmGEGwA z-!Vfwv}cdrsRR7;ORQg8M`;uNq{)M{hdBeMIHL(sUh-!I@DX(rZ+^$D+-hk50J}#U zQ-FW1WO#qXudM29mT<|kCLkFG)9J|QeS37Q{SQ^zrEac?B>cGtlhkznb*ZJ_>Dp!6 z+{YSZjI(#gbNW`2m1kZJJ0-cs=yEAS*L@D2IlTB)9kH-*Pk-~xXKRyOJYf_N0mAKG zah{*fu_p1Yv}NufxeUF=I(t%R`t$3zu}LYyGklqTgEi{M_p?(<$m_!8)oI7twW2rl zYaw8p@yvh{ThozR(7+kQ!I&`U%70#IVz%6fB2A=m!Q(Vs-P`G*R#^g)2wa|6{VUds zk0p;bvz==#jt={;sKEVuQX_(*d4e7EJ^nGpSN{NnW;>u)dofo#f(&*XXT5XYEAT#>Wn}luZRHjPnBW4q zABH_Y8a=Y6_R#lm8KpX~_BA@_yoxdE&}guU+ChMNk4or#Iq?haYO>qJSgF~9!vy2} z^IlE;qdn#8TeEqQrWHtFLxbMCe*s7Sh|l&$ga^PqcVoZfUW{9*O>3dsjmxNE=sDC^ zXQ^B3`kb~Qmd-fPIDLxsT+-pO^9D?J=qoQp)h0-GEQUrH!tFh& zn!c9?vuUrg!x}*LN%8%vcE%%@w+}3Qq%B^jy%oB_)l-=S@sf0 zq~0)M6c8A2PdzC>%v3VR&(6_4=W#Tt&AeHqm3*umNQ?e{f) z%TkWw;^H<^ArcjNwwx9nRDUl{D3?Z>&un@;e>wT39AQWQ0Is>JH1<1TDs-@N!&Q&P z-}=zm@$bYfG+CG}u0i>7_+U8i-`|SzORpBmtz5L0Cm}cp_#hmfeg6RZ_0Z~G3X{az zjEx!%tHgm@mSgm;cUAa#s`zD}SkseqNU-ChUQ%~QURj6BFtPSZSDWv* zp(MT})odbbl?xst3W1xc>7UB3=z7WVMZLrmZh3B_*rl*mFH^{_7hKlvZ|sS%iP9hm zWSDXYILP+t`P2Lx;vek&vWbZshz0u`{{Z^y%By(Oc2_-$`D*&Rl1WR*-|_E+EMtY* z6v;*`q;|kRmTQmGtwp9^g)ym!Y zZ&0|pZR{Z%@W`vrO!HoU@20)&4h%hd$_|xD6Zl&0-c47?RUtm;11-6}opW7Wwy;45 zma=Y%{^7>o=Ui@?;f+g7nBI%?yK!LMez~n_^*u~!l0|_dWNmfhzj2>T*9@piH4~2A zOmTD}QeNcl^fq35G>6E$RY=O<@4?{I_HCxvOt8Ta10kDlCm!dub1;jYqFes}H5qfX zKSTcj)~g@dakM6La0vqnSe|=TG?ubFs<^)<{{YtvXrkx09g6Sht`!Rhs@df$tF*h9#>V~tJ$JapgqI&#gkV!B9ou_Sdr=cm@O8^m@t(|w_52j?t- zTcO4|?t0_WslKM#gBQ1quq~cPLOa(wrzJf$Jw<|i$-R!#!k-W=Z#2KO$jubHIaN71 z9+>s5y*7UsYiP1X4#bn@1)OIe%zyRkjQB-pmlr6qTdDo-Ws?~I4xiWY74JH1T6}iP z35GF&!-L58{cF>1F{@}JOyYPiT3iUo8*n+t zTJt@A8|^W_;#P$-DEmMJ;GsDBXBF4@cg1$9rENoOMO7-f!tu!;O2!qPDP46n%y2%} z4ysbt{{Vs8TRdnRa2uC6#y`(L*11m}y!Tc!S|pi3gfS#?4+o`gCZjd>o8~~P8@58T z@sd40wa0jtVI9iHbtB5NxmDUS3j@f`KDDJqv=Zf_Y0qIbU4jX7>GH#0`?e2b`TH>{d?cY#UDt9a4j_xsDk?@L36QL|9 zQM{I9bI(r4C;tGiUAS6}G>@gvaW!gX%1-ya%`G287Ld7+s>-K3oObr7{gx}swr52< zSQR8^@&0vs@@sq0VTj>}0hDw#g{tZ@>2S2GwZS1+DI;k;dUIX8@T2dok?~ZOYV&va znU&*f+dVbjd4m}6M+=a6>-q6sNiEi`E##WMnLaIKVgiw$U-O#u9~Ei>TPC)7HyKj_ zB^>kv+w`hBH-*L zWsW2}r5#L%ImZWqpXXXyt-yv4+}fxtp-@J=q0ef^d&`>-5r1b|%IOoP-u$p69uv|e07C;umfbUx%aA!)Ws8T(^b%6@RB*8HLBn-*tBMuNItUqScSlFxiCu zpxotV)`w%^uk5QWDJ{^(Tx7I(1A++u0PE4Hcy`+I$*;l#BXF1)B;$_1PtvPH;osgY z&RRCw0Add$dX9&R+P8`bts%8CZU{1W<0Sg~))gfv-8G@}6)N)7brZWeE58ig+CUmu z_YS2PV}sVQ29GA6sVcTYJM>ZMgVXqX*Jm!AQ(C*#?)53;`%Ga(1^z_GLNWbE<5c`s zk=yxl5(q<=Bw+OC`c@7N!M1eJ$Iew0dzpSh%7xw}Y-ffXk&fSuW9x7tMdwEl1y?&& zdY+ZDr|NLZqFy5fPXu)RD<0zd87?s!X9pvmxIHoHT&|v$J1I(h-K=G57FvU8?J;s) zlOQR<1Ds=;p`~e8QzV9SE6N56$lL+#UCbI>Gub8G!Q`%7D6HLT$~dm#co%5|C&&mG zz!@h!@t>(ZDwQUZMzk^TO~bEq20bR&U%7O`0g)Q#raOK;s>Y#f1+;rZfYPoJR1!$f zao(@bEwlL$!hpyd=X`ePG5XfE&V?1+DL(M7#07Rnrsfh%{z3xrGUFNS4{|e( zD!+)la||(?dtg>#Nh;$w9r^EBaq2Kf6lo2?C5HfY^fjBQgr4CQLpe-j2FX8P&b6Sb zDJzpl4N2DbqIX^d*Ao8QEVy9VJaXCopZ$92t>ldunF5g9;Pu6Mw}vhCC~qUVD%-YV zLebdYNP~Q<|mG{{TqsE_`Am(h#JjO8}&!fq~CGIPY0t0``c=BClbfWRN>n zW}D(d_m-CSP$-oM`B;9H+lP#y9rZn>mea@JIa=xYYJ2=%JBA4)xk3B4QdcLS?nkfx z09vx2#!y~|nk<;X&SX4f_54k7t9jy)VI=t=$;hRX0}w+=3*NJs5}qC zx+zJ`@}39wegh1gsx{ktkG`JkSe56Q*%B^d0CERBkIu7~#X8-E&)BY@c~yoYlyjbX z5$(lyx_86v@qL&|`?dx-=O11xo$)tk&h9G@L-C4STNr24U9i75190~K zf1NCv_3BC%NQy#w7VYg?S;{&dMj|-6^Kh1nro9c#N?TY`LwXsJ*8z6p9QG&Pv9*n2 z!YwpK4jXy-n{eQCscto!Iqc1?>Bju-=Nb0v*110wURY_63wZ7jF&HRMKmBfN&#Os$ zM>7mA4Z4aJd;E;OPfVKn87`JrU?t3XJb-=s)t?J!H^FmzGP>>TKJI<0wwqOS zmefsl@T*|Pcmtk2^HBJY#4_7Vk}SCZki-n`^rqNeyt9km750t!Bd7JQ97H*nDm>^> ztr$5=sbXz0iB|xX&&)B?{A&lqR-$lPFu@dU&KoBHes$8__=8W_#5#b(=E3P)2a7yq zWos;|W?V+#@c08gJN4$dXF_T=TASc-_=>e`qpC1GQLMqF$njhJ(mzOm>5k-9SA{R- zy_l@=?B{WIY>tD0&mB5?=Bw)-AhEyH&GfeH?VZ0W)K*vJbzRn|Vva+G0~um^b?4WS zSyZ9Pq>=Ui09I6R>MeWCYR<&!u|pEZ`2})HucdR|E!TA>fnLvtXUWb$+v)9E+FSs< zuW)b}fSmOn{{Z1pYdTarO6=dczXRH_jX9)w6rDOX(`x58XQ|C*>vMBU&It3ckCz<l%!E6qAd$hxPkPx>bE@0f9MmN>U3GUkc>GIsZKDZi zh?i!@_(=JOp#f?yVl9ua;yH{^>h?JU)@s-kw`08d0>J?)!QkEw6Yo*TXu*dWEIAjaVLo?waX7 z(;OBGkX*CmZSz^UB=OVwS6zAG`1L(zNK!90G*TOJ&wuc$X{YEmH$GB`gBJe)R2$!? z^8D+6X0Y*G+nP^1h9Z;|Te(FXG`<|vHEFiVZRQ1jGspF;onyfIg`S}vtsSz&QNbzB zF`R!qSFc5LbH$NvDTv{B`i*?4XulB|?fk;g&e zvi9*=K(5Z*MuVnv$vOI0Y2i6;rQ7z1@r(n5k-+Knu0KN2Ev37cO?UiR*0N3wW{wFqf zP349GReocRr@lW*Wv_^CEbTYm6;M+?Y7>Es{$J;s2&#a(U*u zC5epH;(X3ufvqf8nc3(*60)_n7h)m4PFaHSlT`dc;h4aZRN5mwK^+hI{&g0O<5;d^ z5kqpmUzPG>_;KsaPpWH?Pp6h!kmtJ`WK=rtQE^%qC}UxX=9B7q&xum%z&m-&LHU&j z1${r4>0BnYtX^GSx;w6V;B%9T?YwP!a^Oi6N6VGUt1vu}IT`PakIuM;iR_@7?nsfu zd2+`Giq@aMPVaN=Fi?DoO)HRiUfSrzEaNdx=bWx^8~iJN*4tm6&6+!hp$EOwIRmEOhtn9@%%=Ar^E+oAE z(1kTf{J5h(BL+O4$FH%c-}tiEPm!7A1Nd?eUO)Q6s`#(Ow&6Ca5xbEhjN`3&-nrs? zo9povxKEq!wljhO=NbJgM@~~(8(}j40IyljHb~g-4U6>RZiL zQ8uMCfeK&-=ni_H#-bXrvb%UMBn1;L3grI)`l$)Lxe+jSSVX&mgPe?T4?Jgz)xyg| z+gMt7dnm@#KxzldEzzc09fo{3v_4?pHkH>mgfYe6;kh1N1}MG2?;LzsmET#_4KV9 z-xkFbMIj)$Y<$FVTz%{}8hyZ&5(r!=1TP=w)~Z};PFaGu+;N2iJbItUIH*)DshsdM zD^Hp$Lssik@|r-c<`!IS?lbS2ui^6IcX?c8TN_vtpU2bx0IgND`&k3D(-6|jz~B5o jTGO@Do=b%c%r=v_sTs#l=lq34;GruUO)61vUwQx8)hOOr literal 0 HcmV?d00001 diff --git a/DOCS/onnx_runner/onnx_example_requirements.txt b/DOCS/onnx_runner/onnx_example_requirements.txt new file mode 100644 index 00000000..baecf4fb --- /dev/null +++ b/DOCS/onnx_runner/onnx_example_requirements.txt @@ -0,0 +1,11 @@ +coloredlogs==15.0.1 +flatbuffers==24.3.25 +humanfriendly==10.0 +mpmath==1.3.0 +numpy==2.1.2 +onnx==1.17.0 +onnxruntime==1.19.2 +packaging==24.1 +pillow==11.0.0 +protobuf==5.28.3 +sympy==1.13.3 diff --git a/DOCS/onnx_runner/onnx_example_runner.py b/DOCS/onnx_runner/onnx_example_runner.py new file mode 100644 index 00000000..3ee73f2d --- /dev/null +++ b/DOCS/onnx_runner/onnx_example_runner.py @@ -0,0 +1,40 @@ +import onnxruntime +import numpy as np +from PIL import Image + +model_path = "best_model.onnx" +image_path = "image.jpg" +target_size = (224, 224) + +try: + session = onnxruntime.InferenceSession(model_path) +except Exception as e: + print(f"Failed to load model: {e}") + exit(1) + +# Load and preprocess the image +img = Image.open(image_path) +img = img.resize(target_size) # Resize the image +img_array = np.array(img, dtype=np.float32) / 255.0 # Normalize and convert to float32 + +# Ensure 3 channels (RGB) if image is grayscale +if img_array.shape[-1] != 3: + img_array = np.stack((img_array,) * 3, axis=-1) + +# Transpose image to (C, H, W) format +img_array = np.transpose(img_array, (2, 0, 1)) + +# Add batch dimension +input_batch = np.expand_dims(img_array, axis=0) + +# Prepare input dictionary for the model +input_name = session.get_inputs()[0].name +input_data = {input_name: input_batch} + +# Run inference +try: + results = session.run(None, input_data)[0] + print(results) +except Exception as e: + print(f"Failed to run model inference: {e}") + exit(1) \ No newline at end of file diff --git a/DOCS/validator.md b/DOCS/validator.md index 08660acb..06cac4c8 100644 --- a/DOCS/validator.md +++ b/DOCS/validator.md @@ -90,9 +90,8 @@ python3 scripts/start_validator.py --wallet.name=my-wallet --wallet.hotkey=my-ho ```bash # from root apt install software-properties-common -y -add-apt-repository ppa:deadsnakes/ppa apt update -apt install python3.10 python3.10-venv python3.10-dev python3-pip unzip +apt install python3.12 python3.12-venv python3.12-dev python3-pip unzip apt install python3-virtualenv git nodejs npm npm install pm2 -g @@ -120,4 +119,4 @@ cp .env.example .env # example for testnet python3 scripts/start_validator.py --wallet.name=validator-staked --wallet.hotkey=default --subtensor.network test --logging.debug 1 --netuid 163 -``` +``` \ No newline at end of file diff --git a/cancer_ai/base/base_validator.py b/cancer_ai/base/base_validator.py index 72ca28be..d3a98a03 100644 --- a/cancer_ai/base/base_validator.py +++ b/cancer_ai/base/base_validator.py @@ -38,9 +38,8 @@ from ..utils.config import add_validator_args from neurons.competition_runner import CompetitionRunStore -from cancer_ai.chain_models_store import ChainMinerModelStore - from cancer_ai.validator.rewarder import CompetitionWinnersStore +from cancer_ai.validator.models import OrganizationDataReferenceFactory from .. import __spec_version__ as spec_version @@ -76,7 +75,7 @@ def __init__(self, config=None): self.winners_store = CompetitionWinnersStore( competition_leader_map={}, hotkey_score_map={} ) - self.chain_models_store = ChainMinerModelStore(hotkeys={}) + self.organizations_data_references = OrganizationDataReferenceFactory.get_instance() self.load_state() # Init sync with the network. Updates the metagraph. self.sync() diff --git a/cancer_ai/chain_models_store.py b/cancer_ai/chain_models_store.py index 3f603b27..e942c355 100644 --- a/cancer_ai/chain_models_store.py +++ b/cancer_ai/chain_models_store.py @@ -3,7 +3,6 @@ import bittensor as bt from pydantic import BaseModel, Field - from .utils.models_storage_utils import run_in_subprocess @@ -30,6 +29,11 @@ def to_compressed_str(self) -> str: """Returns a compressed string representation.""" return f"{self.hf_repo_id}:{self.hf_model_filename}:{self.hf_code_filename}:{self.competition_id}:{self.hf_repo_type}" + @property + def hf_link(self) -> str: + """Returns the Hugging Face link for the model.""" + return f"https://huggingface.co/{self.hf_repo_id}/blob/main/{self.hf_model_filename}" + @classmethod def from_compressed_str(cls, cs: str) -> Type["ChainMinerModel"]: """Returns an instance of this class from a compressed string representation""" @@ -46,11 +50,6 @@ def from_compressed_str(cls, cs: str) -> Type["ChainMinerModel"]: ) -class ChainMinerModelStore(BaseModel): - hotkeys: dict[str, ChainMinerModel | None] - last_updated: float | None = None - - class ChainModelMetadata: """Chain based implementation for storing and retrieving metadata about a model.""" diff --git a/cancer_ai/mock.py b/cancer_ai/mock.py index 2b027ffb..9138e52c 100644 --- a/cancer_ai/mock.py +++ b/cancer_ai/mock.py @@ -15,24 +15,26 @@ def __init__(self, netuid, n=16, wallet=None, network="mock"): self.create_subnet(netuid) # Register ourself (the validator) as a neuron at uid=0 - if wallet is not None: - self.force_register_neuron( - netuid=netuid, - hotkey=wallet.hotkey.ss58_address, - coldkey=wallet.coldkey.ss58_address, - balance=100000, - stake=100000, - ) - # Register n mock neurons who will be miners - for i in range(1, n + 1): - self.force_register_neuron( - netuid=netuid, - hotkey=f"miner-hotkey-{i}", - coldkey="mock-coldkey", - balance=100000, - stake=100000, - ) + # TODO not supported in bittensor==8.0 + # if wallet is not None: + # self.force_register_neuron( + # netuid=netuid, + # hotkey=wallet.hotkey.ss58_address, + # coldkey=wallet.coldkey.ss58_address, + # balance=100000, + # stake=100000, + # ) + + # # Register n mock neurons who will be miners + # for i in range(1, n + 1): + # self.force_register_neuron( + # netuid=netuid, + # hotkey=f"miner-hotkey-{i}", + # coldkey="mock-coldkey", + # balance=100000, + # stake=100000, + # ) class MockMetagraph(bt.metagraph): diff --git a/cancer_ai/utils/config.py b/cancer_ai/utils/config.py index 71f52bfd..7a555c85 100644 --- a/cancer_ai/utils/config.py +++ b/cancer_ai/utils/config.py @@ -128,15 +128,16 @@ def add_args(cls, parser): default="", ) - -def add_miner_args(cls, parser): - """Add miner specific arguments to the parser.""" parser.add_argument( - "--competition_id", - type=str, - help="Competition ID", + "--models_query_cutoff", + type=int, + help="The cutoff for the models query in minutes.", + default=30, ) + +def add_miner_args(cls, parser): + """Add miner specific arguments to the parser.""" parser.add_argument( "--model_dir", type=str, @@ -160,11 +161,6 @@ def add_miner_args(cls, parser): type=str, help="Filename of the code zip to push to hugging face.", ) - parser.add_argument( - "--hf_repo_type", - type=str, - help="Type of hugging face repository.", - ) parser.add_argument( "--action", @@ -192,7 +188,6 @@ def add_miner_args(cls, parser): ) parser.add_argument( - "--code_directory", "--code_directory", type=str, help="Path to code directory", @@ -209,7 +204,7 @@ def add_common_args(cls, parser): default="", ) parser.add_argument( - "--competition.id", + "--competition_id", type=str, help="Path for storing competition participants models .", ) @@ -235,6 +230,12 @@ def add_common_args(cls, parser): default="./config/competition_config.json", ) + parser.add_argument( + "--hf_repo_type", + type=str, + help="Hugging Face repository type to submit the model from.", + default="model", + ) def add_validator_args(cls, parser): """Add validator specific arguments to the parser.""" @@ -298,18 +299,25 @@ def add_validator_args(cls, parser): default=4096, ) + parser.add_argument( + "--db_path", + type=str, + help="Path to the sqlite DB for storing the miners models reference", + default="models.db" + ) + parser.add_argument( "--wandb_project_name", type=str, help="The name of the project where you are sending the new run.", - default="template-validators", + default="melanoma-testnet", ) parser.add_argument( "--wandb_entity", type=str, help="The name of the project where you are sending the new run.", - default="opentensor-dev", + default="safe-scan-ai", ) parser.add_argument( @@ -319,6 +327,26 @@ def add_validator_args(cls, parser): default=False, ) + parser.add_argument( + "--datasets_config_hf_repo_id", + type=str, + help="The reference to Hugging Face datasets config.", + default="safescanai/competition-configuration", + ) + + parser.add_argument( + "--miners_refresh_interval", + type=int, + help="The interval at which to refresh the miners in minutes", + default=15, + ) + + parser.add_argument( + "--monitor_datasets_interval", + type=int, + help="The interval at which to monitor the datasets in seconds", + default=8*60, + ) def path_config(cls=None): """ diff --git a/cancer_ai/validator/competition_manager.py b/cancer_ai/validator/competition_manager.py index bb2efddc..53f1c3a9 100644 --- a/cancer_ai/validator/competition_manager.py +++ b/cancer_ai/validator/competition_manager.py @@ -10,6 +10,7 @@ from .dataset_manager import DatasetManager from .model_run_manager import ModelRunManager from .exceptions import ModelRunException +from .model_db import ModelDBController from .competition_handlers.melanoma_handler import MelanomaCompetitionHandler from .competition_handlers.base_handler import ModelEvaluationResult @@ -17,7 +18,6 @@ from cancer_ai.chain_models_store import ( ChainModelMetadata, ChainMinerModel, - ChainMinerModelStore, ) load_dotenv() @@ -49,12 +49,11 @@ def __init__( subtensor: bt.subtensor, hotkeys: list[str], validator_hotkey: str, - chain_miners_store: ChainMinerModelStore, competition_id: str, - category: str, dataset_hf_repo: str, dataset_hf_id: str, dataset_hf_repo_type: str, + db_controller: ModelDBController, test_mode: bool = False, ) -> None: """ @@ -63,15 +62,13 @@ def __init__( Args: config (dict): Config dictionary. competition_id (str): Unique identifier for the competition. - category (str): Category of the competition. """ bt.logging.trace(f"Initializing Competition: {competition_id}") self.config = config self.subtensor = subtensor self.competition_id = competition_id - self.category = category self.results = [] - self.model_manager = ModelManager(self.config) + self.model_manager = ModelManager(self.config, db_controller) self.dataset_manager = DatasetManager( self.config, competition_id, @@ -85,7 +82,7 @@ def __init__( self.hotkeys = hotkeys self.validator_hotkey = validator_hotkey - self.chain_miners_store = chain_miners_store + self.db_controller = db_controller self.test_mode = test_mode def __repr__(self) -> str: @@ -94,9 +91,14 @@ def __repr__(self) -> str: def log_results_to_wandb( self, miner_hotkey: str, validator_hotkey: str, evaluation_result: ModelEvaluationResult ) -> None: + winning_model_link = self.db_controller.get_latest_model( + hotkey=miner_hotkey, cutoff_time=self.config.models_query_cutoff + ).hf_link wandb.init(project=self.competition_id, group="model_evaluation") wandb.log( { + "log_type": "model_results", + "competition_id": self.competition_id, "miner_hotkey": miner_hotkey, "validator_hotkey": validator_hotkey, "tested_entries": evaluation_result.tested_entries, @@ -109,29 +111,28 @@ def log_results_to_wandb( "fpr": evaluation_result.fpr, "tpr": evaluation_result.tpr, }, + "model_link": winning_model_link, "roc_auc": evaluation_result.roc_auc, "score": evaluation_result.score, } ) wandb.finish() - bt.logging.info("Results: ", evaluation_result) + bt.logging.info(f"Results: {evaluation_result}") def get_state(self): return { "competition_id": self.competition_id, "model_manager": self.model_manager.get_state(), - "category": self.category, } def set_state(self, state: dict): self.competition_id = state["competition_id"] self.model_manager.set_state(state["model_manager"]) - self.category = state["category"] async def chain_miner_to_model_info( self, chain_miner_model: ChainMinerModel - ) -> ModelInfo | None: + ) -> ModelInfo: bt.logging.warning(f"Chain miner model: {chain_miner_model.model_dump()}") if chain_miner_model.competition_id != self.competition_id: bt.logging.debug( @@ -157,20 +158,17 @@ async def sync_chain_miners(self): """ bt.logging.info("Selecting models for competition") bt.logging.info(f"Amount of hotkeys: {len(self.hotkeys)}") - for hotkey in self.chain_miners_store.hotkeys: - if self.chain_miners_store.hotkeys[hotkey] is None: - continue + + latest_models = self.db_controller.get_latest_models(self.hotkeys, self.competition_id, self.config.models_query_cutoff) + for hotkey, model in latest_models.items(): try: - model_info = await self.chain_miner_to_model_info( - self.chain_miners_store.hotkeys[hotkey] - ) + model_info = await self.chain_miner_to_model_info(model) except ValueError: bt.logging.warning( - f"Miner {hotkey} does not belong to this competition, skipping" + f"Miner {hotkey} with competition id {model.competition_id} does not belong to {self.competition_id} competition, skipping" ) continue self.model_manager.hotkey_store[hotkey] = model_info - bt.logging.info( f"Amount of hotkeys with valid models: {len(self.model_manager.hotkey_store)}" ) diff --git a/cancer_ai/validator/dataset_manager.py b/cancer_ai/validator/dataset_manager.py index bbd87350..2ab7697b 100644 --- a/cancer_ai/validator/dataset_manager.py +++ b/cancer_ai/validator/dataset_manager.py @@ -101,7 +101,7 @@ def set_dataset_handler(self) -> None: """Detect dataset type and set handler""" if not self.local_compressed_path: raise DatasetManagerException( - f"Dataset '{self.config.competition.id}' not downloaded" + f"Dataset '{self.config.competition_id}' not downloaded" ) # is csv in directory if os.path.exists(Path(self.local_extracted_dir, "labels.csv")): diff --git a/cancer_ai/validator/model_db.py b/cancer_ai/validator/model_db.py new file mode 100644 index 00000000..cfa5e434 --- /dev/null +++ b/cancer_ai/validator/model_db.py @@ -0,0 +1,236 @@ +import bittensor as bt +import os +from sqlalchemy import create_engine, Column, String, DateTime, PrimaryKeyConstraint, Integer +from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy.orm import sessionmaker +from datetime import datetime, timedelta +from ..chain_models_store import ChainMinerModel + +Base = declarative_base() + +STORED_MODELS_PER_HOTKEY = 10 + +class ChainMinerModelDB(Base): + __tablename__ = 'models' + competition_id = Column(String, nullable=False) + hf_repo_id = Column(String, nullable=False) + hf_model_filename = Column(String, nullable=False) + hf_repo_type = Column(String, nullable=False) + hf_code_filename = Column(String, nullable=False) + date_submitted = Column(DateTime, nullable=False) + block = Column(Integer, nullable=False) + hotkey = Column(String, nullable=False) + + __table_args__ = ( + PrimaryKeyConstraint('date_submitted', 'hotkey', name='pk_date_hotkey'), + ) + +class ModelDBController: + def __init__(self, subtensor: bt.subtensor, db_path: str = "models.db"): + self.subtensor = subtensor + db_url = f"sqlite:///{os.path.abspath(db_path)}" + self.engine = create_engine(db_url, echo=False) + Base.metadata.create_all(self.engine) + self.Session = sessionmaker(bind=self.engine) + + def add_model(self, chain_miner_model: ChainMinerModel, hotkey: str): + session = self.Session() + date_submitted = self.get_block_timestamp(chain_miner_model.block) + existing_model = self.get_model(date_submitted, hotkey) + if not existing_model: + try: + model_record = self.convert_chain_model_to_db_model(chain_miner_model, hotkey) + session.add(model_record) + session.commit() + bt.logging.debug(f"Successfully added model info for hotkey {hotkey} into the DB.") + except Exception as e: + session.rollback() + raise e + finally: + session.close() + else: + bt.logging.debug(f"Model for hotkey {hotkey} and date {date_submitted} already exists, proceeding with updating the model info.") + self.update_model(chain_miner_model, hotkey) + + def get_model(self, date_submitted: datetime, hotkey: str) -> ChainMinerModel | None: + session = self.Session() + try: + model_record = session.query(ChainMinerModelDB).filter_by( + date_submitted=date_submitted, hotkey=hotkey + ).first() + if model_record: + return self.convert_db_model_to_chain_model(model_record) + return None + finally: + session.close() + + def get_latest_model(self, hotkey: str, cutoff_time: float = None) -> ChainMinerModel | None: + bt.logging.debug(f"Getting latest model for hotkey {hotkey} with cutoff time {cutoff_time}") + cutoff_time = datetime.now() - timedelta(minutes=cutoff_time) if cutoff_time else datetime.now() + session = self.Session() + try: + model_record = ( + session.query(ChainMinerModelDB) + .filter(ChainMinerModelDB.hotkey == hotkey) + .filter(ChainMinerModelDB.date_submitted < cutoff_time) + .order_by(ChainMinerModelDB.date_submitted.desc()) + .first() + ) + if model_record: + return self.convert_db_model_to_chain_model(model_record) + return None + finally: + session.close() + + def delete_model(self, date_submitted: datetime, hotkey: str): + session = self.Session() + try: + model_record = session.query(ChainMinerModelDB).filter_by( + date_submitted=date_submitted, hotkey=hotkey + ).first() + if model_record: + session.delete(model_record) + session.commit() + return True + return False + except Exception as e: + session.rollback() + raise e + finally: + session.close() + + def update_model(self, chain_miner_model: ChainMinerModel, hotkey: str): + session = self.Session() + try: + date_submitted = self.get_block_timestamp(chain_miner_model.block) + existing_model = session.query(ChainMinerModelDB).filter_by( + date_submitted=date_submitted, + hotkey=hotkey + ).first() + + if existing_model: + existing_model.competition_id = chain_miner_model.competition_id + existing_model.hf_repo_id = chain_miner_model.hf_repo_id + existing_model.hf_model_filename = chain_miner_model.hf_model_filename + existing_model.hf_repo_type = chain_miner_model.hf_repo_type + existing_model.hf_code_filename = chain_miner_model.hf_code_filename + existing_model.block = chain_miner_model.block + existing_model.date_submitted = date_submitted + + session.commit() + bt.logging.debug(f"Successfully updated model for hotkey {hotkey} and date {date_submitted}.") + return True + else: + bt.logging.debug(f"No existing model found for hotkey {hotkey} and date {date_submitted}. Update skipped.") + return False + + except Exception as e: + session.rollback() + bt.logging.error(f"Error updating model for hotkey {hotkey} and date {date_submitted}: {e}") + raise e + finally: + session.close() + + def get_block_timestamp(self, block_number): + """Gets the timestamp of a block given its number.""" + try: + block_hash = self.subtensor.get_block_hash(block_number) + if block_hash is None: + raise ValueError(f"Block hash not found for block number {block_number}") + + timestamp_info = self.subtensor.substrate.query( + module='Timestamp', + storage_function='Now', + block_hash=block_hash + ) + + if timestamp_info is None: + raise ValueError(f"Timestamp not found for block hash {block_hash}") + + timestamp_ms = timestamp_info.value + block_datetime = datetime.fromtimestamp(timestamp_ms / 1000.0) + + return block_datetime + except Exception as e: + bt.logging.error(f"Error retrieving block timestamp: {e}") + raise + + def get_latest_models(self, hotkeys: list[str], competition_id: str, cutoff_time: float = None) -> dict[str, ChainMinerModel]: + cutoff_time = datetime.now() - timedelta(minutes=cutoff_time) if cutoff_time else datetime.now() + session = self.Session() + try: + # Use a correlated subquery to get the latest record for each hotkey that doesn't violate the cutoff + latest_models_to_hotkeys = {} + for hotkey in hotkeys: + model_record = ( + session.query(ChainMinerModelDB) + .filter(ChainMinerModelDB.hotkey == hotkey) + .filter(ChainMinerModelDB.competition_id == competition_id) + .filter(ChainMinerModelDB.date_submitted < cutoff_time) + .order_by(ChainMinerModelDB.date_submitted.desc()) # Order by newest first + .first() # Get the first (newest) record that meets the cutoff condition + ) + if model_record: + latest_models_to_hotkeys[hotkey] = self.convert_db_model_to_chain_model(model_record) + + return latest_models_to_hotkeys + finally: + session.close() + + def clean_old_records(self, hotkeys: list[str]): + session = self.Session() + + for hotkey in hotkeys: + try: + records = ( + session.query(ChainMinerModelDB) + .filter(ChainMinerModelDB.hotkey == hotkey) + .order_by(ChainMinerModelDB.date_submitted.desc()) + .all() + ) + + # If there are more than STORED_MODELS_PER_HOTKEY records, delete the oldest ones + if len(records) > STORED_MODELS_PER_HOTKEY: + records_to_delete = records[STORED_MODELS_PER_HOTKEY:] + for record in records_to_delete: + session.delete(record) + + session.commit() + + except Exception as e: + session.rollback() + bt.logging.error(f"Error processing hotkey {hotkey}: {e}") + + try: + # Delete all records for hotkeys not in the given list + session.query(ChainMinerModelDB).filter(ChainMinerModelDB.hotkey.notin_(hotkeys)).delete(synchronize_session=False) + session.commit() + except Exception as e: + session.rollback() + bt.logging.error(f"Error deleting records for hotkeys not in list: {e}") + + finally: + session.close() + + def convert_chain_model_to_db_model(self, chain_miner_model: ChainMinerModel, hotkey: str) -> ChainMinerModelDB: + date_submitted = self.get_block_timestamp(chain_miner_model.block) + return ChainMinerModelDB( + competition_id = chain_miner_model.competition_id, + hf_repo_id = chain_miner_model.hf_repo_id, + hf_model_filename = chain_miner_model.hf_model_filename, + hf_repo_type = chain_miner_model.hf_repo_type, + hf_code_filename = chain_miner_model.hf_code_filename, + date_submitted = date_submitted, + block = chain_miner_model.block, + hotkey = hotkey + ) + + def convert_db_model_to_chain_model(self, model_record: ChainMinerModelDB) -> ChainMinerModel: + return ChainMinerModel( + competition_id=model_record.competition_id, + hf_repo_id=model_record.hf_repo_id, + hf_model_filename=model_record.hf_model_filename, + hf_repo_type=model_record.hf_repo_type, + hf_code_filename=model_record.hf_code_filename, + block=model_record.block, + ) \ No newline at end of file diff --git a/cancer_ai/validator/model_manager.py b/cancer_ai/validator/model_manager.py index 4b53024e..8952486c 100644 --- a/cancer_ai/validator/model_manager.py +++ b/cancer_ai/validator/model_manager.py @@ -1,5 +1,6 @@ import os from dataclasses import dataclass, asdict, is_dataclass +from datetime import datetime, timezone import bittensor as bt from huggingface_hub import HfApi @@ -21,8 +22,9 @@ class ModelInfo: class ModelManager(SerializableManager): - def __init__(self, config) -> None: + def __init__(self, config, db_controller) -> None: self.config = config + self.db_controller = db_controller if not os.path.exists(self.config.models.model_dir): os.makedirs(self.config.models.model_dir) @@ -41,21 +43,102 @@ async def download_miner_model(self, hotkey) -> bool: bool: True if the model was downloaded successfully, False otherwise. """ model_info = self.hotkey_store[hotkey] + chain_model_date = await self.get_newest_saved_model_date(hotkey) + if chain_model_date and chain_model_date.tzinfo is None: + chain_model_date = chain_model_date.replace(tzinfo=timezone.utc) + if not chain_model_date: + bt.logging.error(f"Failed to get the newest saved model's date for hotkey {hotkey} from the local DB. Model download skipped.") + return False try: - model_info.file_path = self.api.hf_hub_download( - model_info.hf_repo_id, - model_info.hf_model_filename, - cache_dir=self.config.models.model_dir, + commits = self.api.list_repo_commits( + repo_id=model_info.hf_repo_id, repo_type=model_info.hf_repo_type, - token=( - self.config.hf_token if hasattr(self.config, "hf_token") else None - ), + token=self.config.hf_token if hasattr(self.config, "hf_token") else None ) + + model_commit = self.get_commit_with_file_change( + commits, model_info, chain_model_date + ) + + if model_commit: + try: + self.download_model_at_commit(model_commit, model_info) + bt.logging.info(f"Downloaded an older model version for hotkey {hotkey} (date: {model_commit.created_at}).") + return True + except Exception as e: + bt.logging.error(f"Failed to download model at commit {model_commit.commit_id}: {e}") + return False + else: + bt.logging.error(f"No matching or older model found for hotkey {hotkey} based on the saved date. Download skipped.") + return False + except Exception as e: - bt.logging.error(f"Failed to download model {e}") + bt.logging.error(f"Failed to download model: {e}") return False - return True + + async def get_newest_saved_model_date(self, hotkey): + """Fetches the newest saved model's date for a given hotkey from the local database.""" + newest_saved_model = self.db_controller.get_latest_model(hotkey, self.config.models_query_cutoff) + if not newest_saved_model: + bt.logging.error(f"Failed to get latest model from local DB for hotkey {hotkey}") + return None + return self.db_controller.get_block_timestamp(newest_saved_model.block) + + def get_commit_with_file_change(self, commits, model_info, chain_model_date): + """ + Finds the most recent commit (relative to chain_model_date) where the specific file exists + and matches the date criteria. Assumes commits are sorted from newest to oldest. + """ + for commit in commits: + commit_id = commit.commit_id + commit_date = commit.created_at + if commit_date.tzinfo is None: + commit_date = commit_date.replace(tzinfo=timezone.utc) + + # Skip commits newer than the specified date + if commit_date > chain_model_date: + bt.logging.debug(f"Skipping commit {commit_id} because it is newer than chain_model_date") + continue + + # Check if the file exists at this commit + try: + files = self.api.list_repo_files( + repo_id=model_info.hf_repo_id, + revision=commit_id, + repo_type=model_info.hf_repo_type, + token=self.config.hf_token if hasattr(self.config, "hf_token") else None, + ) + + if model_info.hf_model_filename in files: + bt.logging.info(f"Found model version of commit {commit_id}") + return commit # Return the first valid commit and stop searching + + except Exception as e: + bt.logging.error(f"Failed to list files at commit {commit_id}: {e}") + continue + + bt.logging.error("No suitable older commit with the required file was found.") + return None + + + def download_model_at_commit(self, commit, model_info): + try: + model_info.file_path = self.api.hf_hub_download( + repo_id=model_info.hf_repo_id, + repo_type=model_info.hf_repo_type, + filename=model_info.hf_model_filename, + cache_dir=self.config.models.model_dir, + revision=commit.commit_id, + token=self.config.hf_token if hasattr(self.config, "hf_token") else None, + ) + if not os.path.exists(model_info.file_path): + bt.logging.error(f"Downloaded file does not exist at {model_info.file_path}") + raise FileNotFoundError(f"File {model_info.file_path} was not found after download.") + bt.logging.info(f"Successfully downloaded model file to {model_info.file_path}") + except Exception as e: + bt.logging.error(f"Failed to download model file at commit {commit.commit_id}: {e}") + raise def add_model( self, diff --git a/cancer_ai/validator/model_runners/onnx_runner.py b/cancer_ai/validator/model_runners/onnx_runner.py index f65e6386..d0341ba7 100644 --- a/cancer_ai/validator/model_runners/onnx_runner.py +++ b/cancer_ai/validator/model_runners/onnx_runner.py @@ -1,69 +1,135 @@ from typing import List, AsyncGenerator - import numpy as np import bittensor as bt +from collections import defaultdict from ..exceptions import ModelRunException - -import bittensor as bt +import os from . import BaseRunnerHandler + class OnnxRunnerHandler(BaseRunnerHandler): async def get_chunk_of_data( - self, X_test: List, chunk_size: int + self, X_test: List, chunk_size: int, error_counter: defaultdict ) -> AsyncGenerator[List, None]: - """Opens images using PIL and yields a chunk of them""" + """Opens images using PIL and yields a chunk of them while aggregating errors.""" import PIL.Image as Image for i in range(0, len(X_test), chunk_size): bt.logging.debug(f"Processing chunk {i} to {i + chunk_size}") chunk = [] - for img_path in X_test[i : i + chunk_size]: - img = Image.open(img_path) - chunk.append(img) - chunk = self.preprocess_data(chunk) + for img_path in X_test[i: i + chunk_size]: + try: + if not os.path.isfile(img_path): + raise FileNotFoundError(f"File does not exist: {img_path}") + + with Image.open(img_path) as img: + img = img.convert('RGB') # Ensure image is in RGB + chunk.append(img.copy()) # Copy to avoid issues after closing + except FileNotFoundError: + error_counter['FileNotFoundError'] += 1 + continue # Skip this image + except IOError: + error_counter['IOError'] += 1 + continue # Skip this image + except Exception: + error_counter['UnexpectedError'] += 1 + continue # Skip this image + + if not chunk: + error_counter['EmptyChunk'] += 1 + continue # Skip this chunk if no images are loaded + + try: + chunk = self.preprocess_data(chunk) + except ModelRunException: + error_counter['PreprocessingFailure'] += 1 + continue # Skip this chunk if preprocessing fails + except Exception: + error_counter['UnexpectedPreprocessingError'] += 1 + continue # Skip this chunk if preprocessing fails + yield chunk def preprocess_data(self, X_test: List) -> List: new_X_test = [] target_size = (224, 224) # TODO: Change this to the correct size - for img in X_test: - img = img.resize(target_size) - img_array = np.array(img, dtype=np.float32) / 255.0 - img_array = np.array(img) - if img_array.shape[-1] != 3: # Handle grayscale images - img_array = np.stack((img_array,) * 3, axis=-1) - - img_array = np.transpose( - img_array, (2, 0, 1) - ) # Transpose image to (C, H, W) - - new_X_test.append(img_array) + for idx, img in enumerate(X_test): + try: + img = img.resize(target_size) + img_array = np.array(img, dtype=np.float32) / 255.0 + if img_array.ndim == 2: # Grayscale image + img_array = np.stack((img_array,) * 3, axis=-1) + elif img_array.shape[-1] != 3: + raise ValueError(f"Unexpected number of channels: {img_array.shape[-1]}") + + img_array = np.transpose( + img_array, (2, 0, 1) + ) # Transpose image to (C, H, W) + new_X_test.append(img_array) + except (AttributeError, ValueError): + # These are non-critical issues; skip the image + continue # Optionally, you can count these if needed + except Exception: + # Log unexpected preprocessing errors + continue # Optionally, you can count these if needed + + if not new_X_test: + raise ModelRunException("No images were successfully preprocessed") - new_X_test = np.array(new_X_test, dtype=np.float32) + try: + new_X_test = np.array(new_X_test, dtype=np.float32) + except Exception: + raise ModelRunException("Failed to convert preprocessed images to numpy array") return new_X_test async def run(self, X_test: List) -> List: import onnxruntime - try: - session = onnxruntime.InferenceSession(self.model_path) - except Exception as e: - bt.logging.error(f"Failed to run model {e}") - raise ModelRunException("Failed to run model") - + error_counter = defaultdict(int) # Initialize error counters + try: + session = onnxruntime.InferenceSession(self.model_path) + except onnxruntime.OnnxRuntimeException: + raise ModelRunException("Failed to create ONNX inference session") + except Exception: + raise ModelRunException("Failed to create ONNX inference session") results = [] - async for chunk in self.get_chunk_of_data(X_test, chunk_size=200): - input_batch = np.stack(chunk, axis=0) - input_name = session.get_inputs()[0].name - input_data = {input_name: input_batch} - - chunk_results = session.run(None, input_data)[0] - results.extend(chunk_results) + async for chunk in self.get_chunk_of_data(X_test, chunk_size=200, error_counter=error_counter): + try: + input_batch = np.stack(chunk, axis=0) + except ValueError: + error_counter['StackingError'] += 1 + continue # Skip this batch + except Exception: + error_counter['UnexpectedStackingError'] += 1 + continue # Skip this batch + + try: + input_name = session.get_inputs()[0].name + input_data = {input_name: input_batch} + chunk_results = session.run(None, input_data)[0] + results.extend(chunk_results) + except onnxruntime.OnnxRuntimeException: + error_counter['InferenceError'] += 1 + continue # Skip this batch + except Exception: + error_counter['UnexpectedInferenceError'] += 1 + continue # Skip this batch + + # After processing all chunks, handle the error summary + if error_counter: + error_summary = [] + for error_type, count in error_counter.items(): + error_summary.append(f"{count} {error_type.replace('_', ' ')}(s)") + + summary_message = "; ".join(error_summary) + bt.logging.info(f"Processing completed with the following issues: {summary_message}") + if not results: + raise ModelRunException("No results obtained from model inference") return results diff --git a/cancer_ai/validator/models.py b/cancer_ai/validator/models.py index 3598df32..08c43b4d 100644 --- a/cancer_ai/validator/models.py +++ b/cancer_ai/validator/models.py @@ -1,5 +1,6 @@ -from typing import List -from pydantic import BaseModel +from typing import List, ClassVar, Optional, ClassVar, Optional +from pydantic import BaseModel, EmailStr, Field, ValidationError +from datetime import datetime class CompetitionModel(BaseModel): competition_id: str @@ -13,3 +14,37 @@ class CompetitionModel(BaseModel): class CompetitionsListModel(BaseModel): competitions: List[CompetitionModel] +class DatasetReference(BaseModel): + competition_id: str = Field(..., min_length=1, description="Competition identifier") + dataset_hf_repo: str = Field(..., min_length=1, description="Hugging Face repository path for the dataset") + dataset_hf_filename: str = Field(..., min_length=1, description="Filename for the dataset in the repository") + dataset_hf_repo_type: str = Field(..., min_length=1, description="Type of the Hugging Face repository (e.g., dataset)") + dataset_size: int = Field(..., ge=1, description="Size of the dataset, must be a positive integer") + +class OrganizationDataReference(BaseModel): + organization_id: str = Field(..., min_length=1, description="Unique identifier for the organization") + contact_email: EmailStr = Field(..., description="Contact email address for the organization") + bittensor_hotkey: str = Field(..., min_length=1, description="Hotkey associated with the organization") + data_packages: List[DatasetReference] = Field(..., description="List of data packages for the organization") + date_uploaded: datetime = Field(..., description="Date the organization data was uploaded") + +class OrganizationDataReferenceFactory(BaseModel): + organizations: List[OrganizationDataReference] = Field(default_factory=list) + _instance: ClassVar[Optional["OrganizationDataReferenceFactory"]] = None + @classmethod + def get_instance(cls): + if cls._instance is None: + cls._instance = cls() + return cls._instance + + def add_organizations(self, organizations: List[OrganizationDataReference]): + self.organizations.extend(organizations) + + def update_from_dict(self, data: dict): + """Updates the singleton instance's state from a dictionary.""" + if "organizations" in data: + # Convert each dict in 'organizations' to an OrganizationDataReference instance + self.organizations = [OrganizationDataReference(**org) for org in data["organizations"]] + for key, value in data.items(): + if key != "organizations": + setattr(self, key, value) \ No newline at end of file diff --git a/cancer_ai/validator/tests/mock_data.py b/cancer_ai/validator/tests/mock_data.py index bd9cccc9..f338c908 100644 --- a/cancer_ai/validator/tests/mock_data.py +++ b/cancer_ai/validator/tests/mock_data.py @@ -2,28 +2,14 @@ def get_mock_hotkeys_with_models(): return { - # good model - "hfsss_OgEeYLdTgrRIlWIdmbcPQZWTdafatdKfSwwddsavDfO": ModelInfo( - hf_repo_id="Kabalisticus/test_bs_model", - hf_model_filename="good_test_model.onnx", + "5HeH6kmR6FyfC6K39aGozMJ3wUTdgxrQAQsy4BBbskxHKqgG": ModelInfo( + hf_repo_id="eatcats/test", + hf_model_filename="melanoma-1-piwo.onnx", hf_repo_type="model", ), - # Model made from image, extension changed - "hfddd_OgEeYLdTgrRIlWIdmbcPQZWTfsafasftdKfSwwvDf": ModelInfo( - hf_repo_id="Kabalisticus/test_bs_model", - hf_model_filename="false_from_image_model.onnx", + "5CQFdhmRyQtiTwHLumywhWtQYTQkF4SpGtdT8aoh3WK3E4E2": ModelInfo( + hf_repo_id="eatcats/melanoma-test", + hf_model_filename="2024-08-24_04-37-34-melanoma-1.onnx", hf_repo_type="model", ), - # Good model with wrong extension - "hf_OgEeYLdTslgrRfasftdKfSwwvDf": ModelInfo( - hf_repo_id="Kabalisticus/test_bs_model", - hf_model_filename="wrong_extension_model.onx", - hf_repo_type="model", - ), - # good model on safescan - "wU2LapwmZfYL9AEAWpUR6sasfsaFoFvqHnzQ5F71Mhwotxujq": ModelInfo( - hf_repo_id="safescanai/test_dataset", - hf_model_filename="best_model.onnx", - hf_repo_type="dataset", - ), } \ No newline at end of file diff --git a/cancer_ai/validator/tests/test_model_db.py b/cancer_ai/validator/tests/test_model_db.py new file mode 100644 index 00000000..d838ff55 --- /dev/null +++ b/cancer_ai/validator/tests/test_model_db.py @@ -0,0 +1,136 @@ +import pytest +import time +from unittest import mock +from datetime import datetime, timedelta +from cancer_ai.validator.model_db import ModelDBController, ChainMinerModelDB, Base +from sqlalchemy import create_engine, inspect +from sqlalchemy.orm import sessionmaker +from cancer_ai.chain_models_store import ChainMinerModel + +@pytest.fixture +def mock_subtensor(): + """Fixture to mock the bittensor subtensor object.""" + subtensor_mock = mock.Mock() + subtensor_mock.get_block_hash.return_value = "mock_block_hash" + + query_call_counter = {'count': 0} + def mock_query(*args, **kwargs): + # Increment counter to simulate unique blocks over time + query_call_counter['count'] += 1 + stable_timestamp = int((datetime.now() - timedelta(minutes=5)).timestamp() * 1000) + # Add a millisecond difference for each subsequent call + timestamp = stable_timestamp + query_call_counter['count'] + return mock.Mock(value=timestamp) + + subtensor_mock.substrate.query.side_effect = mock_query + return subtensor_mock + +@pytest.fixture() +def fixed_mock_subtensor(): + """Fixture to mock the bittensor subtensor object with a fixed timestamp""" + subtensor_mock = mock.Mock() + subtensor_mock.get_block_hash.return_value = "mock_block_hash" + + fixed_timestamp = int((datetime.now() - timedelta(minutes=5)).timestamp() * 1000) + + def mock_query(*args, **kwargs): + return mock.Mock(value=fixed_timestamp) + + subtensor_mock.substrate.query.side_effect = mock_query + return subtensor_mock + +@pytest.fixture +def db_session(): + engine = create_engine('sqlite:///:memory:') + Base.metadata.create_all(engine) + + inspector = inspect(engine) + print(inspector.get_table_names()) + + Session = sessionmaker(bind=engine) + return Session() + +@pytest.fixture +def model_persister(mock_subtensor, db_session): + """Fixture to create a ModelPersister instance with mocked dependencies.""" + persister = ModelDBController(mock_subtensor, db_path=':memory:') + persister.Session = mock.Mock(return_value=db_session) + return persister + +@pytest.fixture +def model_persister_fixed(fixed_mock_subtensor, db_session): + """Fixture to create a ModelPersister instance with a fixed timestamp.""" + persister = ModelDBController(fixed_mock_subtensor, db_path=':memory:') + persister.Session = mock.Mock(return_value=db_session) + return persister + +@pytest.fixture +def mock_chain_miner_model(): + return ChainMinerModel( + competition_id="1", + hf_repo_id="mock_repo", + hf_model_filename="mock_model", + hf_repo_type="mock_type", + hf_code_filename="mock_code", + block=123456, + hotkey="mock_hotkey" + ) + +def test_add_model(model_persister, mock_chain_miner_model, db_session): + model_persister.add_model(mock_chain_miner_model, "mock_hotkey") + + session = db_session + model_record = session.query(ChainMinerModelDB).first() + assert model_record is not None + assert model_record.hotkey == "mock_hotkey" + assert model_record.competition_id == mock_chain_miner_model.competition_id + +def test_get_model(model_persister_fixed, mock_chain_miner_model, db_session): + model_persister_fixed.add_model(mock_chain_miner_model, "mock_hotkey") + date_submitted = model_persister_fixed.get_block_timestamp(mock_chain_miner_model.block) + + retrieved_model = model_persister_fixed.get_model(date_submitted, "mock_hotkey") + + assert retrieved_model is not None + assert retrieved_model.hf_repo_id == mock_chain_miner_model.hf_repo_id + +def test_delete_model(model_persister_fixed, mock_chain_miner_model, db_session): + model_persister_fixed.add_model(mock_chain_miner_model, "mock_hotkey") + date_submitted = model_persister_fixed.get_block_timestamp(mock_chain_miner_model.block) + + delete_result = model_persister_fixed.delete_model(date_submitted, "mock_hotkey") + assert delete_result is True + + session = db_session + model_record = session.query(ChainMinerModelDB).first() + assert model_record is None + +def test_get_latest_models(model_persister, mock_chain_miner_model, db_session): + model_persister.add_model(mock_chain_miner_model, "mock_hotkey") + + # Wait for a few seconds to pass the cutoff value and then add another model + time.sleep(6) + mock_chain_miner_model.block += 1 + model_persister.add_model(mock_chain_miner_model, "mock_hotkey") + + # Get the latest model + cutoff_time = 5/60 # convert cutoff minutest to seconds + latest_models = model_persister.get_latest_models(["mock_hotkey"], cutoff_time) + assert len(latest_models) == 1 + assert latest_models[0].hf_repo_id == mock_chain_miner_model.hf_repo_id + +@mock.patch('cancer_ai.validator.model_db.STORED_MODELS_PER_HOTKEY', 10) +def test_clean_old_records(model_persister, mock_chain_miner_model, db_session): + session = db_session + for i in range(12): + time.sleep(1) + mock_chain_miner_model.block += i + 1 + model_persister.add_model(mock_chain_miner_model, "mock_hotkey") + session.commit() + session.commit() + + # Clean old records + model_persister.clean_old_records(["mock_hotkey"]) + # Check that only STORED_MODELS_PER_HOTKEY models remain + records = session.query(ChainMinerModelDB).filter_by(hotkey="mock_hotkey").all() + assert len(records) == 10 diff --git a/cancer_ai/validator/utils.py b/cancer_ai/validator/utils.py index 4b0d08c7..089634ae 100644 --- a/cancer_ai/validator/utils.py +++ b/cancer_ai/validator/utils.py @@ -3,7 +3,15 @@ import asyncio import bittensor as bt import json +import yaml from cancer_ai.validator.models import CompetitionsListModel, CompetitionModel +from huggingface_hub import HfApi, hf_hub_download +from cancer_ai.validator.models import ( + DatasetReference, + OrganizationDataReference, + OrganizationDataReferenceFactory, +) +from datetime import datetime class ModelType(Enum): @@ -84,3 +92,141 @@ def get_competition_config(path: str) -> CompetitionsListModel: competitions_json = json.load(f) competitions = [CompetitionModel(**item) for item in competitions_json] return CompetitionsListModel(competitions=competitions) + +async def fetch_organization_data_references(hf_repo_id: str, hf_token: str, hf_api: HfApi) -> list[dict]: + bt.logging.trace(f"Fetching organization data references from Hugging Face repo {hf_repo_id}") + + # prevent stale connections + custom_headers = {"Connection": "close"} + + files = hf_api.list_repo_tree( + repo_id=hf_repo_id, + repo_type="space", + token=hf_token, + recursive=True, + expand=True, + ) + + yaml_data = [] + + for file_info in files: + if file_info.__class__.__name__ == 'RepoFile': + file_path = file_info.path + + if file_path.startswith('datasets/') and file_path.endswith('.yaml'): + local_file_path = hf_hub_download( + repo_id=hf_repo_id, + repo_type="space", + token=hf_token, + filename=file_path, + headers=custom_headers, + ) + + last_commit_info = file_info.last_commit + commit_date = last_commit_info.date if last_commit_info else None + + if commit_date is not None: + date_uploaded = commit_date + else: + bt.logging.warning(f"Could not get the last commit date for {file_path}") + date_uploaded = None + + with open(local_file_path, 'r') as f: + try: + data = yaml.safe_load(f) + except yaml.YAMLError as e: + bt.logging.error(f"Error parsing YAML file {file_path}: {str(e)}") + continue # Skip this file due to parsing error + + yaml_data.append({ + 'file_name': file_path, + 'yaml_data': data, + 'date_uploaded': date_uploaded + }) + else: + continue + return yaml_data + +async def fetch_yaml_data_from_local_repo(local_repo_path: str) -> list[dict]: + """ + Fetches YAML data from all YAML files in the specified local directory. + Returns a list of dictionaries containing file name, YAML data, and the last modified date. + """ + yaml_data = [] + + # Traverse through the local directory to find YAML files + for root, _, files in os.walk(local_repo_path): + for file_name in files: + if file_name.endswith('.yaml'): + file_path = os.path.join(root, file_name) + relative_path = os.path.relpath(file_path, local_repo_path) + commit_date = datetime.fromtimestamp(os.path.getmtime(file_path)) + + with open(file_path, 'r') as f: + data = yaml.safe_load(f) + + yaml_data.append({ + 'file_name': relative_path, + 'yaml_data': data, + 'date_uploaded': commit_date + }) + + return yaml_data + +async def get_new_organization_data_updates(fetched_yaml_files: list[dict]) -> list[DatasetReference]: + factory = OrganizationDataReferenceFactory.get_instance() + data_references: list[DatasetReference] = [] + + for file in fetched_yaml_files: + yaml_data = file['yaml_data'] + date_uploaded = file['date_uploaded'] + + org_id = yaml_data[0]['organization_id'] + existing_org = next((org for org in factory.organizations if org.organization_id == org_id), None) + + if not existing_org: + for entry in yaml_data: + data_references.append(DatasetReference( + competition_id=entry['competition_id'], + dataset_hf_repo=entry['dataset_hf_repo'], + dataset_hf_filename=entry['dataset_hf_filename'], + dataset_hf_repo_type=entry['dataset_hf_repo_type'], + dataset_size=entry['dataset_size'] + )) + + if existing_org and date_uploaded != existing_org.date_uploaded: + last_entry = yaml_data[-1] + data_references.append(DatasetReference( + competition_id=last_entry['competition_id'], + dataset_hf_repo=last_entry['dataset_hf_repo'], + dataset_hf_filename=last_entry['dataset_hf_filename'], + dataset_hf_repo_type=last_entry['dataset_hf_repo_type'], + dataset_size=last_entry['dataset_size'] + )) + + return data_references + +async def update_organizations_data_references(fetched_yaml_files: list[dict]): + bt.logging.trace("Updating organizations data references") + factory = OrganizationDataReferenceFactory.get_instance() + factory.organizations.clear() + + for file in fetched_yaml_files: + yaml_data = file['yaml_data'] + new_org = OrganizationDataReference( + organization_id=yaml_data[0]['organization_id'], + contact_email=yaml_data[0]['contact_email'], + bittensor_hotkey=yaml_data[0]['bittensor_hotkey'], + data_packages=[ + DatasetReference( + competition_id=dp['competition_id'], + dataset_hf_repo=dp['dataset_hf_repo'], + dataset_hf_filename=dp['dataset_hf_filename'], + dataset_hf_repo_type=dp['dataset_hf_repo_type'], + dataset_size=dp['dataset_size'] + ) + for dp in yaml_data + ], + date_uploaded=file['date_uploaded'] + ) + factory.add_organizations([new_org]) diff --git a/neurons/competition_runner.py b/neurons/competition_runner.py index 88e50f16..68c29c85 100644 --- a/neurons/competition_runner.py +++ b/neurons/competition_runner.py @@ -8,9 +8,9 @@ import bittensor as bt from cancer_ai.validator.competition_manager import CompetitionManager -from cancer_ai.chain_models_store import ChainMinerModelStore from cancer_ai.validator.competition_handlers.base_handler import ModelEvaluationResult from cancer_ai.validator.utils import get_competition_config +from cancer_ai.validator.model_db import ModelDBController MINUTES_BACK = 15 @@ -63,9 +63,9 @@ class Config: def get_competitions_schedule( bt_config, subtensor: bt.subtensor, - chain_models_store: ChainMinerModelStore, hotkeys: List[str], validator_hotkey: str, + db_controller: ModelDBController, test_mode: bool = False, ) -> CompetitionSchedule: """Returns CompetitionManager instances arranged by competition time""" @@ -79,12 +79,11 @@ def get_competitions_schedule( subtensor=subtensor, hotkeys=hotkeys, validator_hotkey=validator_hotkey, - chain_miners_store=chain_models_store, competition_id=competition_cfg.competition_id, - category=competition_cfg.category, dataset_hf_repo=competition_cfg.dataset_hf_repo, dataset_hf_id=competition_cfg.dataset_hf_filename, dataset_hf_repo_type=competition_cfg.dataset_hf_repo_type, + db_controller=db_controller, test_mode=test_mode, ) return scheduler_config @@ -94,7 +93,15 @@ async def run_competitions_tick( competition_scheduler: CompetitionSchedule, run_log: CompetitionRunStore, ) -> Tuple[str, str, ModelEvaluationResult] | Tuple[None, None, None]: - """Checks if time is right and launches competition, returns winning hotkey and Competition ID. Should be run each minute.""" + """ + Checks if time is right and launches competition, + returns winning hotkey and Competition ID. Should be run each minute. + + Returns: + winning_hotkey (str) + competition_id (str) + evaluation_result (ModelEvaluationResult) + """ # getting current time now = datetime.now(timezone.utc) diff --git a/neurons/competition_runner_test.py b/neurons/competition_runner_test.py deleted file mode 100644 index ba16c89a..00000000 --- a/neurons/competition_runner_test.py +++ /dev/null @@ -1,126 +0,0 @@ -import time -import asyncio -import json -from types import SimpleNamespace -from typing import List, Dict - -import bittensor as bt - - -from cancer_ai.validator.competition_manager import CompetitionManager -from cancer_ai.validator.rewarder import CompetitionWinnersStore, Rewarder -from cancer_ai.base.base_miner import BaseNeuron -from cancer_ai.utils.config import path_config -from cancer_ai.mock import MockSubtensor -from cancer_ai.validator.exceptions import ModelRunException -from cancer_ai.validator.utils import get_competition_config -from cancer_ai.validator.models import CompetitionModel, CompetitionsListModel - -# TODO integrate with bt config -test_config = SimpleNamespace( - **{ - "wandb_entity": "testnet", - "wandb_project_name": "melanoma-1", - "competition_id": "melaonoma-1", - "hotkeys": [], - "subtensor": SimpleNamespace(**{"network": "test"}), - "netuid": 163, - "models": SimpleNamespace( - **{ - "model_dir": "/tmp/models", - "dataset_dir": "/tmp/datasets", - } - ), - "hf_token": "HF_TOKEN", - } -) - -competitions_cfg = get_competition_config("config/competition_config.json") - - -async def run_all_competitions( - path_config: str, - subtensor: bt.subtensor, - hotkeys: List[str], - competitions_cfg: CompetitionsListModel, -) -> None: - """Run all competitions, for debug purposes""" - for competition_cfg in competitions_cfg.competitions: - bt.logging.info("Starting competition: ", competition_cfg.competition_id) - - competition_manager = CompetitionManager( - path_config, - subtensor, - hotkeys, - "WALIDATOR", - {}, - competition_cfg.competition_id, - competition_cfg.category, - competition_cfg.dataset_hf_repo, - competition_cfg.dataset_hf_filename, - competition_cfg.dataset_hf_repo_type, - test_mode=True, - ) - - bt.logging.info(await competition_manager.evaluate()) - - -def config_for_scheduler(subtensor: bt.subtensor) -> Dict[str, CompetitionManager]: - """Returns CompetitionManager instances arranged by competition time""" - time_arranged_competitions = {} - for competition_cfg in competitions_cfg.competitions: - for competition_time in competition_cfg["evaluation_time"]: - time_arranged_competitions[competition_time] = CompetitionManager( - {}, - subtensor, - [], - "WALIDATOR", - {}, - competition_cfg.competition_id, - competition_cfg.category, - competition_cfg.dataset_hf_repo, - competition_cfg.dataset_hf_filename, - competition_cfg.dataset_hf_repo_type, - test_mode=True, - ) - return time_arranged_competitions - - -async def competition_loop(): - """Example of scheduling coroutine""" - while True: - test_cases = [ - ("hotkey1", "melanoma-1"), - ("hotkey2", "melanoma-1"), - ("hotkey1", "melanoma-2"), - ("hotkey1", "melanoma-1"), - ("hotkey2", "melanoma-3"), - ] - - rewarder_config = CompetitionWinnersStore( - competition_leader_map={}, hotkey_score_map={} - ) - rewarder = Rewarder(rewarder_config) - - for winning_evaluation_hotkey, competition_id in test_cases: - await rewarder.update_scores(winning_evaluation_hotkey, competition_id) - print( - "Updated rewarder competition leader map:", - rewarder.competition_leader_mapping, - ) - print("Updated rewarder scores:", rewarder.scores) - await asyncio.sleep(10) - - -if __name__ == "__main__": - config = BaseNeuron.config() - bt.logging.set_config(config=config) - # if True: # run them right away - path_config = path_config(None) - # config = config.merge(path_config) - # BaseNeuron.check_config(config) - bt.logging.set_config(config=config.logging) - bt.logging.info(config) - asyncio.run( - run_all_competitions(test_config, MockSubtensor("123"), [], competitions_cfg) - ) diff --git a/neurons/miner.py b/neurons/miner.py index 443d3f2d..42a2e660 100644 --- a/neurons/miner.py +++ b/neurons/miner.py @@ -36,6 +36,14 @@ def __init__(self, config=None): self.config.competition.config_path ) + self.code_zip_path = f"{self.config.code_directory}/code.zip" + + self.wallet = None + self.subtensor = None + self.metagraph = None + self.hotkey = None + self.metadata_store = None + @classmethod def add_args(cls, parser: argparse.ArgumentParser): """Method for injecting miner arguments to the parser.""" @@ -47,14 +55,13 @@ async def upload_to_hf(self) -> None: hf_api = HfApi() hf_login(token=self.config.hf_token) - hf_model_path = f"{self.config.competition.id}-{self.config.hf_model_name}.onnx" - hf_code_path = f"{self.config.competition.id}-{self.config.hf_model_name}.zip" + hf_model_path = f"{self.config.competition_id}-{self.config.hf_model_name}.onnx" + hf_code_path = f"{self.config.competition_id}-{self.config.hf_model_name}.zip" path = hf_api.upload_file( path_or_fileobj=self.config.model_path, path_in_repo=hf_model_path, repo_id=self.config.hf_repo_id, - repo_type=self.config.hf_repo_type, token=self.config.hf_token, ) bt.logging.info("Uploading code to Hugging Face.") @@ -62,7 +69,6 @@ async def upload_to_hf(self) -> None: path_or_fileobj=self.code_zip_path, path_in_repo=hf_code_path, repo_id=self.config.hf_repo_id, - repo_type=self.config.hf_repo_type, token=self.config.hf_token, ) @@ -89,7 +95,7 @@ async def evaluate_model(self) -> None: ) dataset_manager = DatasetManager( self.config, - self.config.competition.id, + self.config.competition_id, self.competition_config.competitions[0].dataset_hf_repo, self.competition_config.competitions[0].dataset_hf_filename, self.competition_config.competitions[0].dataset_hf_repo_type, @@ -98,7 +104,7 @@ async def evaluate_model(self) -> None: X_test, y_test = await dataset_manager.get_data() - competition_handler = COMPETITION_HANDLER_MAPPING[self.config.competition.id]( + competition_handler = COMPETITION_HANDLER_MAPPING[self.config.competition_id]( X_test=X_test, y_test=y_test ) @@ -118,16 +124,15 @@ async def evaluate_model(self) -> None: async def compress_code(self) -> None: bt.logging.info("Compressing code") - code_zip_path = f"{self.config.code_directory}/code.zip" + out, err = await run_command( - f"zip -r {code_zip_path} {self.config.code_directory}/*" + f"zip -r {self.code_zip_path} {self.config.code_directory}/*" ) if err: - "Error zipping code" + bt.logging.info("Error zipping code") bt.logging.error(err) return - bt.logging.info(f"Code zip path: {code_zip_path}") - self.code_zip_path = code_zip_path + bt.logging.info(f"Code zip path: {self.code_zip_path}") async def submit_model(self) -> None: # Check if the required model and files are present in hugging face repo @@ -135,6 +140,8 @@ async def submit_model(self) -> None: self.wallet = bt.wallet(config=self.config) self.subtensor = bt.subtensor(config=self.config) self.metagraph = self.subtensor.metagraph(self.config.netuid) + self.hotkey = self.wallet.hotkey.ss58_address + bt.logging.info(f"Wallet: {self.wallet}") bt.logging.info(f"Subtensor: {self.subtensor}") bt.logging.info(f"Metagraph: {self.metagraph}") @@ -150,7 +157,7 @@ async def submit_model(self) -> None: self.metadata_store = ChainModelMetadata( subtensor=self.subtensor, netuid=self.config.netuid, wallet=self.wallet ) - + print(self.config) if not huggingface_hub.file_exists( repo_id=self.config.hf_repo_id, filename=self.config.hf_model_name, @@ -174,7 +181,7 @@ async def submit_model(self) -> None: # Push model metadata to chain model_id = ChainMinerModel( - competition_id=self.config.competition.id, + competition_id=self.config.competition_id, hf_repo_id=self.config.hf_repo_id, hf_model_filename=self.config.hf_model_name, hf_repo_type=self.config.hf_repo_type, diff --git a/neurons/tests/competition_runner_test.py b/neurons/tests/competition_runner_test.py index 57d1a0e8..ad5b651d 100644 --- a/neurons/tests/competition_runner_test.py +++ b/neurons/tests/competition_runner_test.py @@ -11,7 +11,10 @@ from cancer_ai.validator.rewarder import CompetitionWinnersStore, Rewarder from cancer_ai.base.base_miner import BaseNeuron from cancer_ai.utils.config import path_config +from cancer_ai.validator.utils import get_competition_config from cancer_ai.mock import MockSubtensor +from cancer_ai.validator.models import CompetitionsListModel, CompetitionModel +from cancer_ai.validator.model_db import ModelDBController COMPETITION_FILEPATH = "config/competition_config_testnet.json" @@ -32,38 +35,36 @@ } ), "hf_token": "HF_TOKEN", + "db_path": "models.db", } ) -main_competitions_cfg = json.load(open(COMPETITION_FILEPATH, "r")) +competitions_cfg = get_competition_config("config/competition_config_testnet.json") async def run_competitions( config: str, subtensor: bt.subtensor, hotkeys: List[str], - competitions_cfg: List[dict], ) -> Dict[str, str]: """Run all competitions, return the winning hotkey for each competition""" results = {} - for competition_cfg in competitions_cfg: + for competition_cfg in competitions_cfg.competitions: bt.logging.info("Starting competition: ", competition_cfg) competition_manager = CompetitionManager( - config, - subtensor, - hotkeys, - {}, - competition_cfg["competition_id"], - competition_cfg["category"], - competition_cfg["dataset_hf_repo"], - competition_cfg["dataset_hf_filename"], - competition_cfg["dataset_hf_repo_type"], + config=config, + subtensor=subtensor, + hotkeys=hotkeys, + validator_hotkey="Walidator", + competition_id=competition_cfg.competition_id, + dataset_hf_repo=competition_cfg.dataset_hf_repo, + dataset_hf_id=competition_cfg.dataset_hf_filename, + dataset_hf_repo_type=competition_cfg.dataset_hf_repo_type, test_mode=True, + db_controller=ModelDBController(subtensor, test_config.db_path) ) - results[competition_cfg["competition_id"]] = ( - await competition_manager.evaluate() - ) + results[competition_cfg.competition_id] = await competition_manager.evaluate() bt.logging.info(await competition_manager.evaluate()) @@ -73,49 +74,23 @@ async def run_competitions( def config_for_scheduler(subtensor: bt.subtensor) -> Dict[str, CompetitionManager]: """Returns CompetitionManager instances arranged by competition time""" time_arranged_competitions = {} - for competition_cfg in main_competitions_cfg: + for competition_cfg in competitions_cfg: for competition_time in competition_cfg["evaluation_time"]: time_arranged_competitions[competition_time] = CompetitionManager( - {}, - subtensor, - [], - {}, - competition_cfg["competition_id"], - competition_cfg["category"], - competition_cfg["dataset_hf_repo"], - competition_cfg["dataset_hf_filename"], - competition_cfg["dataset_hf_repo_type"], + config={}, + subtensor=subtensor, + hotkeys=[], + validator_hotkey="Walidator", + competition_id=competition_cfg.competition_id, + dataset_hf_repo=competition_cfg.dataset_hf_repo, + dataset_hf_id=competition_cfg.dataset_hf_filename, + dataset_hf_repo_type=competition_cfg.dataset_hf_repo_type, test_mode=True, + db_controller=ModelDBController(subtensor, test_config.db_path) ) return time_arranged_competitions -async def competition_loop(): - """Example of scheduling coroutine""" - while True: - test_cases = [ - ("hotkey1", "melanoma-1"), - ("hotkey2", "melanoma-1"), - ("hotkey1", "melanoma-2"), - ("hotkey1", "melanoma-1"), - ("hotkey2", "melanoma-3"), - ] - - rewarder_config = CompetitionWinnersStore( - competition_leader_map={}, hotkey_score_map={} - ) - rewarder = Rewarder(rewarder_config) - - for winning_evaluation_hotkey, competition_id in test_cases: - await rewarder.update_scores(winning_evaluation_hotkey, competition_id) - print( - "Updated rewarder competition leader map:", - rewarder.competition_leader_mapping, - ) - print("Updated rewarder scores:", rewarder.scores) - await asyncio.sleep(10) - - @pytest.fixture def competition_config(): with open(COMPETITION_FILEPATH, "r") as f: @@ -131,6 +106,4 @@ def competition_config(): # BaseNeuron.check_config(config) bt.logging.set_config(config=config.logging) bt.logging.info(config) - asyncio.run( - run_competitions(test_config, MockSubtensor("123"), [], main_competitions_cfg) - ) + asyncio.run(run_competitions(test_config, MockSubtensor("123"), [])) diff --git a/neurons/validator.py b/neurons/validator.py index dd91b03b..bdbc3fef 100644 --- a/neurons/validator.py +++ b/neurons/validator.py @@ -1,7 +1,6 @@ # The MIT License (MIT) # Copyright © 2023 Yuma Rao -# TODO(developer): Set your name -# Copyright © 2023 +# Copyright © 2024 Safe-Scan # Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated # documentation files (the “Software”), to deal in the Software without restriction, including without limitation @@ -27,34 +26,42 @@ import bittensor as bt import numpy as np import wandb +import requests -from cancer_ai.chain_models_store import ChainModelMetadata, ChainMinerModelStore +from cancer_ai.chain_models_store import ChainModelMetadata from cancer_ai.validator.rewarder import CompetitionWinnersStore, Rewarder, Score from cancer_ai.base.base_validator import BaseValidatorNeuron -from cancer_ai.validator.competition_manager import CompetitionManager from competition_runner import ( get_competitions_schedule, run_competitions_tick, CompetitionRunStore, ) from cancer_ai.validator.cancer_ai_logo import cancer_ai_logo +from cancer_ai.validator.utils import ( + fetch_organization_data_references, + get_new_organization_data_updates, + update_organizations_data_references, +) +from cancer_ai.validator.model_db import ModelDBController +from cancer_ai.validator.competition_manager import CompetitionManager +from cancer_ai.validator.models import OrganizationDataReferenceFactory +from huggingface_hub import HfApi, hf_hub_download -RUN_EVERY_N_MINUTES = 15 # TODO move to config BLACKLIST_FILE_PATH = "config/hotkey_blacklist.json" BLACKLIST_FILE_PATH_TESTNET = "config/hotkey_blacklist_testnet.json" - class Validator(BaseValidatorNeuron): print(cancer_ai_logo) def __init__(self, config=None): super(Validator, self).__init__(config=config) self.hotkey = self.wallet.hotkey.ss58_address + self.db_controller = ModelDBController(self.subtensor, self.config.db_path) self.competition_scheduler = get_competitions_schedule( bt_config = self.config, subtensor = self.subtensor, - chain_models_store = self.chain_models_store, hotkeys = self.hotkeys, validator_hotkey = self.hotkey, + db_controller = self.db_controller, test_mode = False, ) bt.logging.info(f"Scheduler config: {self.competition_scheduler}") @@ -63,24 +70,30 @@ def __init__(self, config=None): self.chain_models = ChainModelMetadata( self.subtensor, self.config.netuid, self.wallet ) + self.last_miners_refresh: float = None + self.last_monitor_datasets: float = None + + # Create the shared session for hugging face api + self.hf_api = HfApi() + async def concurrent_forward(self): coroutines = [ self.refresh_miners(), - self.competition_loop_tick(), + self.monitor_datasets(), ] await asyncio.gather(*coroutines) async def refresh_miners(self): """ - downloads miner's models from the chain and updates the local store + Downloads miner's models from the chain and stores them in the DB """ - if self.chain_models_store.last_updated is not None and ( - time.time() - self.chain_models_store.last_updated - < RUN_EVERY_N_MINUTES * 60 + if self.last_miners_refresh is not None and ( + time.time() - self.last_miners_refresh + < self.config.miners_refresh_interval * 60 ): - bt.logging.debug("Skipping model refresh, not enough time passed") + bt.logging.trace("Skipping model refresh, not enough time passed") return bt.logging.info("Synchronizing miners from the chain") @@ -92,7 +105,7 @@ async def refresh_miners(self): else BLACKLIST_FILE_PATH ) - with open(blacklist_file, "r") as f: + with open(blacklist_file, "r", encoding="utf-8") as f: BLACKLISTED_HOTKEYS = json.load(f) for hotkey in self.hotkeys: @@ -100,33 +113,24 @@ async def refresh_miners(self): bt.logging.debug(f"Skipping blacklisted hotkey {hotkey}") continue - new_chain_miner_store = ChainMinerModelStore(hotkeys={}) - for hotkey in self.hotkeys: hotkey = str(hotkey) - - # TODO add test mode for syncing just once. Then you have to delete state.npz file to sync again - # if hotkey in self.chain_models_store.hotkeys: - # bt.logging.debug(f"Skipping hotkey {hotkey}, already added") - # continue - - hotkey_metadata = await self.chain_models.retrieve_model_metadata(hotkey) - if not hotkey_metadata: + chain_model_metadata = await self.chain_models.retrieve_model_metadata(hotkey) + if not chain_model_metadata: bt.logging.warning( f"Cannot get miner model for hotkey {hotkey} from the chain, skipping" - ) - new_chain_miner_store.hotkeys[hotkey] = hotkey_metadata - - self.chain_models_store = new_chain_miner_store - hotkeys_with_models = [ - hotkey - for hotkey in self.chain_models_store.hotkeys - if self.chain_models_store.hotkeys[hotkey] - ] + ) + continue + try: + self.db_controller.add_model(chain_model_metadata, hotkey) + except Exception as e: + bt.logging.error(f"An error occured while trying to persist the model info: {e}") + self.db_controller.clean_old_records(self.hotkeys) + latest_models = self.db_controller.get_latest_models(self.hotkeys, self.config.models_query_cutoff) bt.logging.info( - f"Amount of miners: {len(self.chain_models_store.hotkeys)}, with models: {len(hotkeys_with_models)}" + f"Amount of miners with models: {len(latest_models)}" ) - self.chain_models_store.last_updated = time.time() + self.last_miners_refresh = time.time() self.save_state() async def competition_loop_tick(self): @@ -138,11 +142,13 @@ async def competition_loop_tick(self): self.competition_scheduler = get_competitions_schedule( bt_config = self.config, subtensor = self.subtensor, - chain_models_store = self.chain_models_store, hotkeys = self.hotkeys, validator_hotkey = self.hotkey, - test_mode = False, + db_controller=self.db_controller, + test_mode = self.config.test_mode, ) + winning_hotkey = None + winning_model_link = None try: winning_hotkey, competition_id, winning_model_result = ( await run_competitions_tick(self.competition_scheduler, self.run_log) @@ -155,9 +161,11 @@ async def competition_loop_tick(self): ) wandb.log( { + "log_type": "competition_result", "winning_evaluation_hotkey": "", "run_time": "", "validator_hotkey": self.wallet.hotkey.ss58_address, + "model_link": winning_model_link, "errors": str(formatted_traceback), } ) @@ -173,17 +181,102 @@ async def competition_loop_tick(self): ).seconds wandb.log( { + "log_type": "competition_result", "winning_hotkey": winning_hotkey, "run_time_s": run_time_s, "validator_hotkey": self.wallet.hotkey.ss58_address, + "model_link": winning_model_link, "errors": "", } ) wandb.finish() bt.logging.info(f"Competition result for {competition_id}: {winning_hotkey}") + self.handle_competition_winner(winning_hotkey, competition_id, winning_model_result) + + async def monitor_datasets(self): + """Monitor datasets references for updates.""" + if self.last_monitor_datasets is not None and ( + time.time() - self.last_monitor_datasets + < self.config.monitor_datasets_interval + ): + return + self.last_monitor_datasets = time.time() + + yaml_data = await fetch_organization_data_references( + self.config.datasets_config_hf_repo_id, + self.config.hf_token, + self.hf_api, + ) + + list_of_data_references = await get_new_organization_data_updates(yaml_data) + if not list_of_data_references: + bt.logging.info("No new data packages found.") + return + + await update_organizations_data_references(yaml_data) + self.organizations_data_references = OrganizationDataReferenceFactory.get_instance() + self.save_state() + + for data_reference in list_of_data_references: + bt.logging.info(f"New data packages found. Starting competition for {data_reference.competition_id}") + competition_manager = CompetitionManager( + config=self.config, + subtensor=self.subtensor, + hotkeys=self.hotkeys, + validator_hotkey=self.hotkey, + competition_id=data_reference.competition_id, + dataset_hf_repo=data_reference.dataset_hf_repo, + dataset_hf_id=data_reference.dataset_hf_filename, + dataset_hf_repo_type=data_reference.dataset_hf_repo_type, + db_controller = self.db_controller, + test_mode = self.config.test_mode, + ) + winning_hotkey = None + winning_model_link = None + try: + winning_hotkey, winning_model_result = ( + await competition_manager.evaluate() + ) + if not winning_hotkey: + continue + + winning_model_link = self.db_controller.get_latest_model(hotkey=winning_hotkey, cutoff_time=self.config.models_query_cutoff).hf_link + except Exception: + formatted_traceback = traceback.format_exc() + bt.logging.error(f"Error running competition: {formatted_traceback}") + wandb.init( + reinit=True, project="competition_id", group="competition_evaluation" + ) + wandb.log( + { + "log_type": "competition_result", + "winning_evaluation_hotkey": "", + "run_time": "", + "validator_hotkey": self.wallet.hotkey.ss58_address, + "model_link": winning_model_link, + "errors": str(formatted_traceback), + } + ) + wandb.finish() + continue + + wandb.init(project=data_reference.competition_id, group="competition_evaluation") + wandb.log( + { + "log_type": "competition_result", + "winning_hotkey": winning_hotkey, + "validator_hotkey": self.wallet.hotkey.ss58_address, + "model_link": winning_model_link, + "errors": "", + } + ) + wandb.finish() + + bt.logging.info(f"Competition result for {data_reference.competition_id}: {winning_hotkey}") + await self.handle_competition_winner(winning_hotkey, data_reference.competition_id, winning_model_result) - # update the scores + async def handle_competition_winner(self, winning_hotkey, competition_id, winning_model_result): await self.rewarder.update_scores( winning_hotkey, competition_id, winning_model_result ) @@ -205,9 +298,6 @@ async def competition_loop_tick(self): def save_state(self): """Saves the state of the validator to a file.""" - bt.logging.debug("Saving validator state.") - - # Save the state of the validator to file. if not getattr(self, "winners_store", None): self.winners_store = CompetitionWinnersStore( competition_leader_map={}, hotkey_score_map={} @@ -216,9 +306,10 @@ def save_state(self): if not getattr(self, "run_log", None): self.run_log = CompetitionRunStore(runs=[]) bt.logging.debug("Competition run store empty, creating new one") - if not getattr(self, "chain_models_store", None): - self.chain_models_store = ChainMinerModelStore(hotkeys={}) - bt.logging.debug("Chain model store empty, creating new one") + + if not getattr(self, "organizations_data_references", None): + self.organizations_data_references = OrganizationDataReferenceFactory.get_instance() + bt.logging.debug("Organizations data references empty, creating new one") np.savez( self.config.neuron.full_path + "/state.npz", @@ -226,7 +317,7 @@ def save_state(self): hotkeys=self.hotkeys, winners_store=self.winners_store.model_dump(), run_log=self.run_log.model_dump(), - chain_models_store=self.chain_models_store.model_dump(), + organizations_data_references=self.organizations_data_references.model_dump(), ) def create_empty_state(self): @@ -237,7 +328,7 @@ def create_empty_state(self): hotkeys=self.hotkeys, winners_store=self.winners_store.model_dump(), run_log=self.run_log.model_dump(), - chain_models_store=self.chain_models_store.model_dump(), + organizations_data_references=self.organizations_data_references.model_dump(), ) return @@ -254,17 +345,18 @@ def load_state(self): state = np.load( self.config.neuron.full_path + "/state.npz", allow_pickle=True ) - bt.logging.trace(state["chain_models_store"]) self.scores = state["scores"] self.hotkeys = state["hotkeys"] self.winners_store = CompetitionWinnersStore.model_validate( state["winners_store"].item() ) self.run_log = CompetitionRunStore.model_validate(state["run_log"].item()) - bt.logging.debug(state["chain_models_store"].item()) - self.chain_models_store = ChainMinerModelStore.model_validate( - state["chain_models_store"].item() - ) + + factory = OrganizationDataReferenceFactory.get_instance() + saved_data = state["organizations_data_references"].item() + factory.update_from_dict(saved_data) + self.organizations_data_references = factory + except Exception as e: bt.logging.error(f"Error loading state: {e}") self.create_empty_state() diff --git a/requirements.txt b/requirements.txt index 0dac1d2c..b8b1fd80 100644 --- a/requirements.txt +++ b/requirements.txt @@ -150,3 +150,5 @@ Werkzeug==3.0.3 wrapt==1.16.0 xxhash==3.4.1 yarl==1.9.4 +sqlalchemy==1.4.0 + From 8983169e69890ae16d5dab736ec9503c6f2b40ad Mon Sep 17 00:00:00 2001 From: Wojtek Jurkowlaniec Date: Fri, 13 Dec 2024 17:31:10 +0100 Subject: [PATCH 197/227] fix requirements (#122) * fix requirements * fixed reqs --------- Co-authored-by: konrad0960 --- requirements.txt | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index b8b1fd80..0d567b10 100644 --- a/requirements.txt +++ b/requirements.txt @@ -29,8 +29,10 @@ cytoolz==0.12.3 ddt==1.6.0 decorator==5.1.1 distlib==0.3.8 +dnspython==2.7.0 docker-pycreds==0.4.0 ecdsa==0.19.0 +email_validator==2.2.0 eth-hash==0.7.0 eth-keys==0.5.1 eth-typing==4.4.0 @@ -47,6 +49,7 @@ gast==0.6.0 gitdb==4.0.11 GitPython==3.1.43 google-pasta==0.2.0 +greenlet==3.1.1 grpcio==1.65.5 h11==0.14.0 h5py==3.11.0 @@ -66,6 +69,7 @@ MarkupSafe==2.1.5 mccabe==0.7.0 mdurl==0.1.2 ml-dtypes==0.4.0 +mnemonic==0.21 more-itertools==10.4.0 mpmath==1.3.0 msgpack==1.0.8 @@ -127,6 +131,7 @@ shtab==1.6.5 six==1.16.0 smmap==5.0.1 sniffio==1.3.1 +SQLAlchemy==1.4.0 starlette==0.37.2 substrate-interface==1.7.10 sympy==1.13.1 @@ -150,5 +155,3 @@ Werkzeug==3.0.3 wrapt==1.16.0 xxhash==3.4.1 yarl==1.9.4 -sqlalchemy==1.4.0 - From a3af78d8ece62f1cd28ec320ebc1108d409630c1 Mon Sep 17 00:00:00 2001 From: konrad0960 <71330299+konrad0960@users.noreply.github.com> Date: Mon, 30 Dec 2024 15:12:51 +0100 Subject: [PATCH 198/227] Commit reveal update (#124) * Triggering the auto-validator * commit reveal adjustments --- cancer_ai/chain_models_store.py | 2 +- requirements.txt | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/cancer_ai/chain_models_store.py b/cancer_ai/chain_models_store.py index e942c355..56b7b2b6 100644 --- a/cancer_ai/chain_models_store.py +++ b/cancer_ai/chain_models_store.py @@ -83,7 +83,7 @@ async def retrieve_model_metadata(self, hotkey: str) -> Optional[ChainMinerModel """Retrieves model metadata on this subnet for specific hotkey""" # Wrap calls to the subtensor in a subprocess with a timeout to handle potential hangs. try: - metadata = bt.extrinsics.serving.get_metadata( + metadata = bt.core.extrinsics.serving.get_metadata( self.subtensor, self.netuid, hotkey ) except Exception as e: diff --git a/requirements.txt b/requirements.txt index 0d567b10..a0cd83e3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,7 +14,7 @@ async-unzip==0.3.6 attrs==24.2.0 backoff==2.2.1 base58==2.1.1 -bittensor==7.4.0 +bittensor==8.5.1 black==24.8.0 certifi==2024.7.4 cffi==1.17.0 @@ -24,7 +24,7 @@ click==8.1.7 colorama==0.4.6 coloredlogs==15.0.1 crontab==1.0.1 -cryptography==42.0.8 +cryptography>=42.0.8 cytoolz==0.12.3 ddt==1.6.0 decorator==5.1.1 @@ -83,7 +83,7 @@ nest-asyncio==1.6.0 netaddr==1.3.0 networkx==3.3 nodeenv==1.9.1 -numpy==1.26.4 +numpy~=2.0.1 onnx==1.16.2 onnxruntime==1.19.0 opt-einsum==3.3.0 @@ -135,9 +135,9 @@ SQLAlchemy==1.4.0 starlette==0.37.2 substrate-interface==1.7.10 sympy==1.13.1 -tensorboard==2.17.1 +tensorboard>=2.17.1 tensorboard-data-server==0.7.2 -tensorflow==2.17.0 +tensorflow>=2.18.0 tensorflow-io-gcs-filesystem==0.37.1 termcolor==2.4.0 threadpoolctl==3.5.0 From 48944720d9df6a5229a8ddf59deb9b711f969f81 Mon Sep 17 00:00:00 2001 From: konrad0960 <71330299+konrad0960@users.noreply.github.com> Date: Wed, 12 Mar 2025 23:07:27 +0100 Subject: [PATCH 199/227] Update versions (#127) * Triggering the auto-validator * updated bittensor and bt client versions --- cancer_ai/chain_models_store.py | 19 +++++++++---------- requirements.txt | 25 ++++++++++++++++++++----- 2 files changed, 29 insertions(+), 15 deletions(-) diff --git a/cancer_ai/chain_models_store.py b/cancer_ai/chain_models_store.py index 56b7b2b6..4ca98729 100644 --- a/cancer_ai/chain_models_store.py +++ b/cancer_ai/chain_models_store.py @@ -70,14 +70,11 @@ async def store_model_metadata(self, model_id: ChainMinerModel): if self.wallet is None: raise ValueError("No wallet available to write to the chain.") - # Wrap calls to the subtensor in a subprocess with a timeout to handle potential hangs. - partial = functools.partial( - self.subtensor.commit, - self.wallet, - self.netuid, - model_id.to_compressed_str(), + self.subtensor.commit( + self.wallet, + self.netuid, + model_id.to_compressed_str(), ) - run_in_subprocess(partial, 60) async def retrieve_model_metadata(self, hotkey: str) -> Optional[ChainMinerModel]: """Retrieves model metadata on this subnet for specific hotkey""" @@ -93,9 +90,11 @@ async def retrieve_model_metadata(self, hotkey: str) -> Optional[ChainMinerModel return None bt.logging.trace(f"Model metadata: {metadata['info']['fields']}") commitment = metadata["info"]["fields"][0] - hex_data = commitment[list(commitment.keys())[0]][2:] - - chain_str = bytes.fromhex(hex_data).decode() + commitment_dict = commitment[0] + key = list(commitment_dict.keys())[0] + data_tuple = commitment_dict[key][0] + hex_str = ''.join(f"{i:02x}" for i in data_tuple) + chain_str = bytes.fromhex(hex_str).decode() try: model = ChainMinerModel.from_compressed_str(chain_str) bt.logging.debug(f"Model: {model}") diff --git a/requirements.txt b/requirements.txt index a0cd83e3..c13dcd03 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,13 +9,20 @@ ansible-core==2.15.12 ansible-vault==2.1.0 anyio==4.4.0 astunparse==1.6.3 +async-property==0.2.2 +async-substrate-interface==1.0.3 async-timeout==4.0.3 async-unzip==0.3.6 +asyncstdlib==3.13.0 attrs==24.2.0 backoff==2.2.1 base58==2.1.1 -bittensor==8.5.1 +bittensor==9.0.2 +bittensor-cli==9.1.0 +bittensor-commit-reveal==0.2.0 +bittensor-wallet==3.0.4 black==24.8.0 +bt-decode==0.5.0a2 certifi==2024.7.4 cffi==1.17.0 cfgv==3.4.0 @@ -24,7 +31,7 @@ click==8.1.7 colorama==0.4.6 coloredlogs==15.0.1 crontab==1.0.1 -cryptography>=42.0.8 +cryptography==43.0.3 cytoolz==0.12.3 ddt==1.6.0 decorator==5.1.1 @@ -79,11 +86,12 @@ munch==2.5.0 mypy==1.11.1 mypy-extensions==1.0.0 namex==0.0.8 +narwhals==1.28.0 nest-asyncio==1.6.0 netaddr==1.3.0 networkx==3.3 nodeenv==1.9.1 -numpy~=2.0.1 +numpy==2.0.2 onnx==1.16.2 onnxruntime==1.19.0 opt-einsum==3.3.0 @@ -93,6 +101,8 @@ password-strength==0.0.3.post2 pathspec==0.12.1 pillow==10.4.0 platformdirs==4.2.2 +plotille==5.0.0 +plotly==6.0.0 pluggy==1.5.0 pre-commit==3.8.0 protobuf==4.25.4 @@ -114,6 +124,7 @@ pytest-asyncio==0.24.0 python-dotenv==1.0.1 python-Levenshtein==0.25.1 python-statemachine==2.1.2 +pywry==0.6.2 PyYAML==6.0.2 rapidfuzz==3.9.6 redis==5.0.8 @@ -127,6 +138,7 @@ scikit-learn==1.5.1 scipy==1.14.1 sentry-sdk==2.13.0 setproctitle==1.3.3 +shellingham==1.5.4 shtab==1.6.5 six==1.16.0 smmap==5.0.1 @@ -135,22 +147,25 @@ SQLAlchemy==1.4.0 starlette==0.37.2 substrate-interface==1.7.10 sympy==1.13.1 -tensorboard>=2.17.1 +tensorboard==2.18.0 tensorboard-data-server==0.7.2 -tensorflow>=2.18.0 +tensorflow==2.18.0 tensorflow-io-gcs-filesystem==0.37.1 termcolor==2.4.0 threadpoolctl==3.5.0 +toml==0.10.0 tomli==2.0.1 toolz==0.12.1 torch==2.4.0 tqdm==4.66.5 +typer==0.15.1 typing_extensions==4.12.2 urllib3==2.2.2 uvicorn==0.30.0 virtualenv==20.26.4 wandb==0.17.7 websocket-client==1.8.0 +websockets==14.1 Werkzeug==3.0.3 wrapt==1.16.0 xxhash==3.4.1 From 9104819216dad7a9f483e6ff2faa8d01f4aae044 Mon Sep 17 00:00:00 2001 From: konrad0960 <71330299+konrad0960@users.noreply.github.com> Date: Fri, 14 Mar 2025 13:31:59 +0100 Subject: [PATCH 200/227] Refactor org data fetch (#130) * data fetching refactor and fixes --------- Co-authored-by: Wojtek Jurkowlaniec Co-authored-by: Wojtek Jurkowlaniec --- .gitignore | 3 + DOCS/validator.md | 4 +- cancer_ai/base/base_miner.py | 2 +- cancer_ai/base/base_validator.py | 8 +- cancer_ai/base/neuron.py | 7 +- cancer_ai/chain_models_store.py | 37 +++---- cancer_ai/validator/dataset_manager.py | 11 +- cancer_ai/validator/models.py | 20 ++-- cancer_ai/validator/utils.py | 134 +++++++++++++++---------- neurons/miner.py | 51 +++++----- neurons/validator.py | 48 +++++---- scripts/start_validator.py | 2 +- 12 files changed, 184 insertions(+), 143 deletions(-) diff --git a/.gitignore b/.gitignore index 99c7f345..eb4701a9 100644 --- a/.gitignore +++ b/.gitignore @@ -170,3 +170,6 @@ datasets data wandb ecosystem.config.js + + +keys \ No newline at end of file diff --git a/DOCS/validator.md b/DOCS/validator.md index 06cac4c8..1539f7bd 100644 --- a/DOCS/validator.md +++ b/DOCS/validator.md @@ -60,7 +60,7 @@ pip install -r requirements.txt To run the validator script, use the following command: ```bash -python3 scripts/start_validator.py --wallet.name=my-wallet --wallet.hotkey=my-hotkey --netuid=46 +python3 scripts/start_validator.py --wallet.name=my-wallet --wallet.hotkey=my-hotkey --netuid=76 ``` @@ -70,7 +70,7 @@ python3 scripts/start_validator.py --wallet.name=my-wallet --wallet.hotkey=my-ho - `--wallet.name`: Specifies the wallet name to be used by the validator. - `--wallet.hotkey`: Specifies the hotkey associated with the wallet. - `--subtensor.network`: Specifies the network name. Default is `"finney"`. -- `--netuid`: Specifies the Netuid of the network. Default is `"46"`. +- `--netuid`: Specifies the Netuid of the network. Default is `"76"`. - `--logging.debug`: Enables debug logging if set to `1`. Default is `1`. ## How It Works diff --git a/cancer_ai/base/base_miner.py b/cancer_ai/base/base_miner.py index 66b8e6ec..e203b21c 100644 --- a/cancer_ai/base/base_miner.py +++ b/cancer_ai/base/base_miner.py @@ -132,7 +132,7 @@ def __exit__(self, exc_type, exc_value, traceback): def resync_metagraph(self): """Resyncs the metagraph and updates the hotkeys and moving averages based on the new metagraph.""" - bt.logging.info("resync_metagraph()") + bt.logging.info("resync_metagraph() miner") # Sync the metagraph. self.metagraph.sync(subtensor=self.subtensor) diff --git a/cancer_ai/base/base_validator.py b/cancer_ai/base/base_validator.py index d3a98a03..dd94e323 100644 --- a/cancer_ai/base/base_validator.py +++ b/cancer_ai/base/base_validator.py @@ -78,7 +78,7 @@ def __init__(self, config=None): self.organizations_data_references = OrganizationDataReferenceFactory.get_instance() self.load_state() # Init sync with the network. Updates the metagraph. - self.sync() + self.sync(force_sync=True) # Serve axon to enable external connections. if not self.config.neuron.axon_off: @@ -283,9 +283,9 @@ def set_weights(self): else: bt.logging.error("set_weights failed", msg) - def resync_metagraph(self): + def resync_metagraph(self, force_sync=False): """Resyncs the metagraph and updates the hotkeys and moving averages based on the new metagraph.""" - bt.logging.info("resync_metagraph()") + bt.logging.info("resync_metagraph() validator") # Copies state of metagraph before syncing. previous_metagraph = copy.deepcopy(self.metagraph) @@ -294,7 +294,7 @@ def resync_metagraph(self): self.metagraph.sync(subtensor=self.subtensor) # Check if the metagraph axon info has changed. - if previous_metagraph.axons == self.metagraph.axons: + if previous_metagraph.axons == self.metagraph.axons and not force_sync: return bt.logging.info( diff --git a/cancer_ai/base/neuron.py b/cancer_ai/base/neuron.py index 6999c67c..65d9dae5 100644 --- a/cancer_ai/base/neuron.py +++ b/cancer_ai/base/neuron.py @@ -108,7 +108,7 @@ def __init__(self, config=None): @abstractmethod def run(self): ... - def sync(self, retries=5, delay=10): + def sync(self, retries=5, delay=10, force_sync=False): """ Wrapper for synchronizing the state of the network for the given miner or validator. """ @@ -118,8 +118,9 @@ def sync(self, retries=5, delay=10): # Ensure miner or validator hotkey is still registered on the network. self.check_registered() - if self.should_sync_metagraph(): - self.resync_metagraph() + if self.should_sync_metagraph() or force_sync: + bt.logging.info("Resyncing metagraph in progress.") + self.resync_metagraph(force_sync=True) if self.should_set_weights(): self.set_weights() diff --git a/cancer_ai/chain_models_store.py b/cancer_ai/chain_models_store.py index 4ca98729..05b533f5 100644 --- a/cancer_ai/chain_models_store.py +++ b/cancer_ai/chain_models_store.py @@ -3,7 +3,6 @@ import bittensor as bt from pydantic import BaseModel, Field -from .utils.models_storage_utils import run_in_subprocess class ChainMinerModel(BaseModel): @@ -70,31 +69,35 @@ async def store_model_metadata(self, model_id: ChainMinerModel): if self.wallet is None: raise ValueError("No wallet available to write to the chain.") + # Wrap calls to the subtensor in a subprocess with a timeout to handle potential hangs. self.subtensor.commit( - self.wallet, - self.netuid, - model_id.to_compressed_str(), + self.wallet, + self.netuid, + model_id.to_compressed_str(), ) async def retrieve_model_metadata(self, hotkey: str) -> Optional[ChainMinerModel]: """Retrieves model metadata on this subnet for specific hotkey""" - # Wrap calls to the subtensor in a subprocess with a timeout to handle potential hangs. + + subnet_metadata = self.subtensor.metagraph(self.netuid) + uids = subnet_metadata.uids + hotkeys = subnet_metadata.hotkeys + + uid = next((uid for uid, hk in zip(uids, hotkeys) if hk == hotkey), None) + + metadata = bt.core.extrinsics.serving.get_metadata( + self.subtensor, self.netuid, hotkey + ) try: - metadata = bt.core.extrinsics.serving.get_metadata( - self.subtensor, self.netuid, hotkey - ) + chain_str = self.subtensor.get_commitment(self.netuid, uid) except Exception as e: - bt.logging.error(f"Error retrieving metadata for hotkey {hotkey}: {e}") + bt.logging.debug(f"Failed to retrieve commitment for hotkey {hotkey}: {e}") return None + if not metadata: return None - bt.logging.trace(f"Model metadata: {metadata['info']['fields']}") - commitment = metadata["info"]["fields"][0] - commitment_dict = commitment[0] - key = list(commitment_dict.keys())[0] - data_tuple = commitment_dict[key][0] - hex_str = ''.join(f"{i:02x}" for i in data_tuple) - chain_str = bytes.fromhex(hex_str).decode() + + model = None try: model = ChainMinerModel.from_compressed_str(chain_str) bt.logging.debug(f"Model: {model}") @@ -103,7 +106,7 @@ async def retrieve_model_metadata(self, hotkey: str) -> Optional[ChainMinerModel f"Metadata might be in old format on the chain for hotkey {hotkey}. Raw value: {chain_str}" ) return None - except: + except Exception: # If the metadata format is not correct on the chain then we return None. bt.logging.error( f"Failed to parse the metadata on the chain for hotkey {hotkey}. Raw value: {chain_str}" diff --git a/cancer_ai/validator/dataset_manager.py b/cancer_ai/validator/dataset_manager.py index 2ab7697b..fa66d751 100644 --- a/cancer_ai/validator/dataset_manager.py +++ b/cancer_ai/validator/dataset_manager.py @@ -19,6 +19,7 @@ def __init__( hf_repo_id: str, hf_filename: str, hf_repo_type: str, + use_auth: bool = True, ) -> None: """ Initializes a new instance of the DatasetManager class. @@ -38,6 +39,7 @@ def __init__( self.hf_filename = hf_filename self.hf_repo_type = hf_repo_type self.competition_id = competition_id + self.use_auth = use_auth self.local_compressed_path = "" self.local_extracted_dir = Path(self.config.models.dataset_dir, competition_id) self.data: Tuple[List, List] = () @@ -59,7 +61,7 @@ async def download_dataset(self): self.hf_filename, cache_dir=Path(self.config.models.dataset_dir), repo_type=self.hf_repo_type, - token=self.config.hf_token if hasattr(self.config, "hf_token") else None, + token=self.config.hf_token if self.use_auth and hasattr(self.config, "hf_token") else None, ) def delete_dataset(self) -> None: @@ -89,9 +91,10 @@ async def unzip_dataset(self) -> None: bt.logging.debug(f"Dataset extracted to: { self.local_compressed_path}") os.system(f"rm -R {self.local_extracted_dir}") # TODO add error handling - out, err = await run_command( - f"unzip {self.local_compressed_path} -d {self.local_extracted_dir}" - ) + zip_file_path = self.local_compressed_path + extract_dir = self.local_extracted_dir + command = f'unzip "{zip_file_path}" -d {extract_dir}' + out, err = await run_command(command) if err: bt.logging.error(f"Error unzipping dataset: {err}") raise DatasetManagerException(f"Error unzipping dataset: {err}") diff --git a/cancer_ai/validator/models.py b/cancer_ai/validator/models.py index 08c43b4d..1040122b 100644 --- a/cancer_ai/validator/models.py +++ b/cancer_ai/validator/models.py @@ -14,19 +14,12 @@ class CompetitionModel(BaseModel): class CompetitionsListModel(BaseModel): competitions: List[CompetitionModel] -class DatasetReference(BaseModel): - competition_id: str = Field(..., min_length=1, description="Competition identifier") - dataset_hf_repo: str = Field(..., min_length=1, description="Hugging Face repository path for the dataset") - dataset_hf_filename: str = Field(..., min_length=1, description="Filename for the dataset in the repository") - dataset_hf_repo_type: str = Field(..., min_length=1, description="Type of the Hugging Face repository (e.g., dataset)") - dataset_size: int = Field(..., ge=1, description="Size of the dataset, must be a positive integer") - class OrganizationDataReference(BaseModel): + competition_id: str = Field(..., min_length=1, description="Competition identifier") organization_id: str = Field(..., min_length=1, description="Unique identifier for the organization") contact_email: EmailStr = Field(..., description="Contact email address for the organization") - bittensor_hotkey: str = Field(..., min_length=1, description="Hotkey associated with the organization") - data_packages: List[DatasetReference] = Field(..., description="List of data packages for the organization") - date_uploaded: datetime = Field(..., description="Date the organization data was uploaded") + dataset_hf_repo: str = Field(..., min_length=1, description="Hugging Face repository path for the dataset") + dataset_hf_dir: str = Field(..., min_length=1, description="Directory for the datasets in the repository") class OrganizationDataReferenceFactory(BaseModel): organizations: List[OrganizationDataReference] = Field(default_factory=list) @@ -47,4 +40,9 @@ def update_from_dict(self, data: dict): self.organizations = [OrganizationDataReference(**org) for org in data["organizations"]] for key, value in data.items(): if key != "organizations": - setattr(self, key, value) \ No newline at end of file + setattr(self, key, value) + +class NewDatasetFile(BaseModel): + competition_id: str = Field(..., min_length=1, description="Competition identifier") + dataset_hf_repo: str = Field(..., min_length=1, description="Hugging Face repository path for the dataset") + dataset_hf_filename: str = Field(..., min_length=1, description="Filename for the dataset in the repository") \ No newline at end of file diff --git a/cancer_ai/validator/utils.py b/cancer_ai/validator/utils.py index 089634ae..6dcbe39c 100644 --- a/cancer_ai/validator/utils.py +++ b/cancer_ai/validator/utils.py @@ -7,11 +7,11 @@ from cancer_ai.validator.models import CompetitionsListModel, CompetitionModel from huggingface_hub import HfApi, hf_hub_download from cancer_ai.validator.models import ( - DatasetReference, - OrganizationDataReference, + NewDatasetFile, OrganizationDataReferenceFactory, ) from datetime import datetime +from typing import Any class ModelType(Enum): @@ -173,60 +173,84 @@ async def fetch_yaml_data_from_local_repo(local_repo_path: str) -> list[dict]: return yaml_data -async def get_new_organization_data_updates(fetched_yaml_files: list[dict]) -> list[DatasetReference]: - factory = OrganizationDataReferenceFactory.get_instance() - data_references: list[DatasetReference] = [] - +async def sync_organizations_data_references(fetched_yaml_files: list[dict]): + """ + Synchronizes the OrganizationDataReferenceFactory state with the full content + from the fetched YAML files. + + Each fetched YAML file is expected to contain a list of organization entries. + The 'org_id' key from the YAML is remapped to 'organization_id' to match the model. + """ + all_orgs = [] for file in fetched_yaml_files: - yaml_data = file['yaml_data'] - date_uploaded = file['date_uploaded'] - - org_id = yaml_data[0]['organization_id'] - existing_org = next((org for org in factory.organizations if org.organization_id == org_id), None) - - if not existing_org: - for entry in yaml_data: - data_references.append(DatasetReference( - competition_id=entry['competition_id'], - dataset_hf_repo=entry['dataset_hf_repo'], - dataset_hf_filename=entry['dataset_hf_filename'], - dataset_hf_repo_type=entry['dataset_hf_repo_type'], - dataset_size=entry['dataset_size'] - )) - - if existing_org and date_uploaded != existing_org.date_uploaded: - last_entry = yaml_data[-1] - data_references.append(DatasetReference( - competition_id=last_entry['competition_id'], - dataset_hf_repo=last_entry['dataset_hf_repo'], - dataset_hf_filename=last_entry['dataset_hf_filename'], - dataset_hf_repo_type=last_entry['dataset_hf_repo_type'], - dataset_size=last_entry['dataset_size'] - )) - - return data_references - -async def update_organizations_data_references(fetched_yaml_files: list[dict]): - bt.logging.trace("Updating organizations data references") + yaml_data = file["yaml_data"] + for entry in yaml_data: + # Remap 'org_id' to 'organization_id' if needed. + if "org_id" in entry: + entry["organization_id"] = entry.pop("org_id") + all_orgs.append(entry) + + update_data = {"organizations": all_orgs} + factory = OrganizationDataReferenceFactory.get_instance() - factory.organizations.clear() + factory.update_from_dict(update_data) - for file in fetched_yaml_files: - yaml_data = file['yaml_data'] - new_org = OrganizationDataReference( - organization_id=yaml_data[0]['organization_id'], - contact_email=yaml_data[0]['contact_email'], - bittensor_hotkey=yaml_data[0]['bittensor_hotkey'], - data_packages=[ - DatasetReference( - competition_id=dp['competition_id'], - dataset_hf_repo=dp['dataset_hf_repo'], - dataset_hf_filename=dp['dataset_hf_filename'], - dataset_hf_repo_type=dp['dataset_hf_repo_type'], - dataset_size=dp['dataset_size'] - ) - for dp in yaml_data - ], - date_uploaded=file['date_uploaded'] +async def check_for_new_dataset_files(hf_api: HfApi, org_latest_updates: dict) -> list[NewDatasetFile]: + """ + For each OrganizationDataReference stored in the singleton, this function: + - Connects to the organization's public Hugging Face repo. + - Lists files under the directory specified by dataset_hf_dir. + - Determines the maximum commit date among those files. + + For a blank state, it returns the file with the latest commit date. + On subsequent checks, it returns any file whose commit date is newer than the previously stored update. + """ + results = [] + factory = OrganizationDataReferenceFactory.get_instance() + + for org in factory.organizations: + files = hf_api.list_repo_tree( + repo_id=org.dataset_hf_repo, + repo_type="dataset", + token=None, # these are public repos + recursive=True, + expand=True, ) - factory.add_organizations([new_org]) + relevant_files = [ + f for f in files + if f.__class__.__name__ == "RepoFile" and f.path.startswith(org.dataset_hf_dir) + ] + max_commit_date = None + for f in relevant_files: + commit_date = f.last_commit.date if f.last_commit else None + if commit_date and (max_commit_date is None or commit_date > max_commit_date): + max_commit_date = commit_date + + new_files = [] + stored_update = org_latest_updates.get(org.organization_id) + # if there is no stored_update and max_commit_date is present (any commit date is present) + if stored_update is None and max_commit_date is not None: + for f in relevant_files: + commit_date = f.last_commit.date if f.last_commit else None + if commit_date == max_commit_date: + new_files.append(f.path) + break + # if there is any stored update then we implicitly expect that any commit date on the repo is present as well + else: + for f in relevant_files: + commit_date = f.last_commit.date if f.last_commit else None + if commit_date and commit_date > stored_update: + new_files.append(f.path) + + # update the stored latest update for this organization. + if max_commit_date is not None: + org_latest_updates[org.organization_id] = max_commit_date + + for file_name in new_files: + results.append(NewDatasetFile( + competition_id=org.competition_id, + dataset_hf_repo=org.dataset_hf_repo, + dataset_hf_filename=file_name + )) + + return results \ No newline at end of file diff --git a/neurons/miner.py b/neurons/miner.py index 42a2e660..6d620f54 100644 --- a/neurons/miner.py +++ b/neurons/miner.py @@ -2,6 +2,7 @@ import copy import time import os +from pathlib import Path import bittensor as bt from dotenv import load_dotenv @@ -36,7 +37,7 @@ def __init__(self, config=None): self.config.competition.config_path ) - self.code_zip_path = f"{self.config.code_directory}/code.zip" + self.code_zip_path = None self.wallet = None self.subtensor = None @@ -55,8 +56,10 @@ async def upload_to_hf(self) -> None: hf_api = HfApi() hf_login(token=self.config.hf_token) - hf_model_path = f"{self.config.competition_id}-{self.config.hf_model_name}.onnx" - hf_code_path = f"{self.config.competition_id}-{self.config.hf_model_name}.zip" + hf_model_path = self.config.hf_model_name + hf_code_path = self.code_zip_path + bt.logging.info(f"Model path: {hf_model_path}") + bt.logging.info(f"Code path: {hf_code_path}") path = hf_api.upload_file( path_or_fileobj=self.config.model_path, @@ -67,11 +70,11 @@ async def upload_to_hf(self) -> None: bt.logging.info("Uploading code to Hugging Face.") path = hf_api.upload_file( path_or_fileobj=self.code_zip_path, - path_in_repo=hf_code_path, + path_in_repo=Path(hf_code_path).name, repo_id=self.config.hf_repo_id, token=self.config.hf_token, ) - + bt.logging.info(f"Code uploaded to Hugging Face: {path}") bt.logging.info(f"Uploaded model to Hugging Face: {path}") @staticmethod @@ -99,6 +102,7 @@ async def evaluate_model(self) -> None: self.competition_config.competitions[0].dataset_hf_repo, self.competition_config.competitions[0].dataset_hf_filename, self.competition_config.competitions[0].dataset_hf_repo_type, + use_auth=False ) await dataset_manager.prepare_dataset() @@ -124,9 +128,13 @@ async def evaluate_model(self) -> None: async def compress_code(self) -> None: bt.logging.info("Compressing code") - + bt.logging.info(f"Code directory: {self.config.code_directory}") + + code_dir = Path(self.config.code_directory) + self.code_zip_path = str(code_dir.parent / f"{code_dir.name}.zip") + out, err = await run_command( - f"zip -r {self.code_zip_path} {self.config.code_directory}/*" + f"zip -r {self.code_zip_path} {self.config.code_directory}/*" ) if err: bt.logging.info("Error zipping code") @@ -145,6 +153,7 @@ async def submit_model(self) -> None: bt.logging.info(f"Wallet: {self.wallet}") bt.logging.info(f"Subtensor: {self.subtensor}") bt.logging.info(f"Metagraph: {self.metagraph}") + if not self.subtensor.is_hotkey_registered( netuid=self.config.netuid, hotkey_ss58=self.wallet.hotkey.ss58_address, @@ -154,30 +163,18 @@ async def submit_model(self) -> None: f" Please register the hotkey using `btcli subnets register` before trying again" ) exit() + self.metadata_store = ChainModelMetadata( subtensor=self.subtensor, netuid=self.config.netuid, wallet=self.wallet ) + print(self.config) - if not huggingface_hub.file_exists( - repo_id=self.config.hf_repo_id, - filename=self.config.hf_model_name, - repo_type=self.config.hf_repo_type, - ): - bt.logging.error( - f"{self.config.hf_model_name} not found in Hugging Face repo" - ) + + if not self._check_hf_file_exists(self.config.hf_repo_id, self.config.hf_model_name, self.config.hf_repo_type): return - if not huggingface_hub.file_exists( - repo_id=self.config.hf_repo_id, - filename=self.config.hf_code_filename, - repo_type=self.config.hf_repo_type, - ): - bt.logging.error( - f"{self.config.hf_model_name} not found in Hugging Face repo" - ) + if not self._check_hf_file_exists(self.config.hf_repo_id, self.config.hf_code_filename, self.config.hf_repo_type): return - bt.logging.info("Model and code found in Hugging Face repo") # Push model metadata to chain model_id = ChainMinerModel( @@ -193,6 +190,12 @@ async def submit_model(self) -> None: f"Successfully pushed model metadata on chain. Model ID: {model_id}" ) + def _check_hf_file_exists(self, repo_id, filename, repo_type): + if not huggingface_hub.file_exists(repo_id=repo_id, filename=filename, repo_type=repo_type): + bt.logging.error(f"{filename} not found in Hugging Face repo") + return False + return True + async def main(self) -> None: # bt.logging(config=self.config) diff --git a/neurons/validator.py b/neurons/validator.py index bdbc3fef..6dbd1c40 100644 --- a/neurons/validator.py +++ b/neurons/validator.py @@ -39,13 +39,13 @@ from cancer_ai.validator.cancer_ai_logo import cancer_ai_logo from cancer_ai.validator.utils import ( fetch_organization_data_references, - get_new_organization_data_updates, - update_organizations_data_references, + sync_organizations_data_references, + check_for_new_dataset_files, ) from cancer_ai.validator.model_db import ModelDBController from cancer_ai.validator.competition_manager import CompetitionManager -from cancer_ai.validator.models import OrganizationDataReferenceFactory -from huggingface_hub import HfApi, hf_hub_download +from cancer_ai.validator.models import OrganizationDataReferenceFactory, NewDatasetFile +from huggingface_hub import HfApi BLACKLIST_FILE_PATH = "config/hotkey_blacklist.json" BLACKLIST_FILE_PATH_TESTNET = "config/hotkey_blacklist_testnet.json" @@ -107,7 +107,7 @@ async def refresh_miners(self): with open(blacklist_file, "r", encoding="utf-8") as f: BLACKLISTED_HOTKEYS = json.load(f) - + for hotkey in self.hotkeys: if hotkey in BLACKLISTED_HOTKEYS: bt.logging.debug(f"Skipping blacklisted hotkey {hotkey}") @@ -128,7 +128,7 @@ async def refresh_miners(self): self.db_controller.clean_old_records(self.hotkeys) latest_models = self.db_controller.get_latest_models(self.hotkeys, self.config.models_query_cutoff) bt.logging.info( - f"Amount of miners with models: {len(latest_models)}" + f"Amount of latest miners with models: {len(latest_models)}" ) self.last_miners_refresh = time.time() self.save_state() @@ -209,26 +209,28 @@ async def monitor_datasets(self): self.hf_api, ) - list_of_data_references = await get_new_organization_data_updates(yaml_data) - if not list_of_data_references: - bt.logging.info("No new data packages found.") - return - - await update_organizations_data_references(yaml_data) + await sync_organizations_data_references(yaml_data) self.organizations_data_references = OrganizationDataReferenceFactory.get_instance() self.save_state() + + list_of_new_data_packages: list[NewDatasetFile] = await check_for_new_dataset_files(self.hf_api, self.org_latest_updates) + self.save_state() + + if not list_of_new_data_packages: + bt.logging.info("No new data packages found.") + return - for data_reference in list_of_data_references: - bt.logging.info(f"New data packages found. Starting competition for {data_reference.competition_id}") + for data_package in list_of_new_data_packages: + bt.logging.info(f"New data packages found. Starting competition for {data_package.competition_id}") competition_manager = CompetitionManager( config=self.config, subtensor=self.subtensor, hotkeys=self.hotkeys, validator_hotkey=self.hotkey, - competition_id=data_reference.competition_id, - dataset_hf_repo=data_reference.dataset_hf_repo, - dataset_hf_id=data_reference.dataset_hf_filename, - dataset_hf_repo_type=data_reference.dataset_hf_repo_type, + competition_id=data_package.competition_id, + dataset_hf_repo=data_package.dataset_hf_repo, + dataset_hf_id=data_package.dataset_hf_filename, + dataset_hf_repo_type="dataset", db_controller = self.db_controller, test_mode = self.config.test_mode, ) @@ -261,7 +263,7 @@ async def monitor_datasets(self): wandb.finish() continue - wandb.init(project=data_reference.competition_id, group="competition_evaluation") + wandb.init(project=data_package.competition_id, group="competition_evaluation") wandb.log( { "log_type": "competition_result", @@ -273,8 +275,8 @@ async def monitor_datasets(self): ) wandb.finish() - bt.logging.info(f"Competition result for {data_reference.competition_id}: {winning_hotkey}") - await self.handle_competition_winner(winning_hotkey, data_reference.competition_id, winning_model_result) + bt.logging.info(f"Competition result for {data_package.competition_id}: {winning_hotkey}") + await self.handle_competition_winner(winning_hotkey, data_package.competition_id, winning_model_result) async def handle_competition_winner(self, winning_hotkey, competition_id, winning_model_result): await self.rewarder.update_scores( @@ -318,6 +320,7 @@ def save_state(self): winners_store=self.winners_store.model_dump(), run_log=self.run_log.model_dump(), organizations_data_references=self.organizations_data_references.model_dump(), + org_latest_updates=self.org_latest_updates, ) def create_empty_state(self): @@ -329,6 +332,7 @@ def create_empty_state(self): winners_store=self.winners_store.model_dump(), run_log=self.run_log.model_dump(), organizations_data_references=self.organizations_data_references.model_dump(), + org_latest_updates={}, ) return @@ -356,6 +360,8 @@ def load_state(self): saved_data = state["organizations_data_references"].item() factory.update_from_dict(saved_data) self.organizations_data_references = factory + self.org_latest_updates = state["org_latest_updates"].item() + except Exception as e: bt.logging.error(f"Error loading state: {e}") diff --git a/scripts/start_validator.py b/scripts/start_validator.py index 18f4dbfd..411a7e1f 100755 --- a/scripts/start_validator.py +++ b/scripts/start_validator.py @@ -209,7 +209,7 @@ def main(pm2_name: str, args_namespace: Namespace, extra_args: List[str]) -> Non ) parser.add_argument( - "--netuid", default="46", help="Netuid of the network." + "--netuid", default="76", help="Netuid of the network." ) parser.add_argument( From 788305d98f66516c3c9863785ca4a7a930c140f2 Mon Sep 17 00:00:00 2001 From: konrad0960 <71330299+konrad0960@users.noreply.github.com> Date: Fri, 14 Mar 2025 13:55:07 +0100 Subject: [PATCH 201/227] hf token fix (#134) --- cancer_ai/validator/utils.py | 6 +++--- neurons/validator.py | 1 - 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/cancer_ai/validator/utils.py b/cancer_ai/validator/utils.py index 6dcbe39c..5fe54514 100644 --- a/cancer_ai/validator/utils.py +++ b/cancer_ai/validator/utils.py @@ -93,7 +93,7 @@ def get_competition_config(path: str) -> CompetitionsListModel: competitions = [CompetitionModel(**item) for item in competitions_json] return CompetitionsListModel(competitions=competitions) -async def fetch_organization_data_references(hf_repo_id: str, hf_token: str, hf_api: HfApi) -> list[dict]: +async def fetch_organization_data_references(hf_repo_id: str, hf_api: HfApi) -> list[dict]: bt.logging.trace(f"Fetching organization data references from Hugging Face repo {hf_repo_id}") # prevent stale connections @@ -102,7 +102,7 @@ async def fetch_organization_data_references(hf_repo_id: str, hf_token: str, hf_ files = hf_api.list_repo_tree( repo_id=hf_repo_id, repo_type="space", - token=hf_token, + token=None, recursive=True, expand=True, ) @@ -117,7 +117,7 @@ async def fetch_organization_data_references(hf_repo_id: str, hf_token: str, hf_ local_file_path = hf_hub_download( repo_id=hf_repo_id, repo_type="space", - token=hf_token, + token=None, filename=file_path, headers=custom_headers, ) diff --git a/neurons/validator.py b/neurons/validator.py index 6dbd1c40..892a149b 100644 --- a/neurons/validator.py +++ b/neurons/validator.py @@ -205,7 +205,6 @@ async def monitor_datasets(self): yaml_data = await fetch_organization_data_references( self.config.datasets_config_hf_repo_id, - self.config.hf_token, self.hf_api, ) From ca56dd4e658827f27acc02db32db066e7694f002 Mon Sep 17 00:00:00 2001 From: Wojtek Jurkowlaniec Date: Sun, 16 Mar 2025 23:15:07 +0100 Subject: [PATCH 202/227] Update validator.md --- DOCS/validator.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/DOCS/validator.md b/DOCS/validator.md index 1539f7bd..6d83c455 100644 --- a/DOCS/validator.md +++ b/DOCS/validator.md @@ -17,9 +17,9 @@ Key features of the script include: ### Server requirements -- 64GB of RAM -- storage: 500GB, extendable -- GPU - nVidia RTX, 12GB VRAM (will work without GPU, but slower) +- 16GB of RAM +- storage: 50GB, extendable +- GPU - nVidia RTX, 6GB VRAM (will work without GPU, but slower) ### System requirements @@ -119,4 +119,4 @@ cp .env.example .env # example for testnet python3 scripts/start_validator.py --wallet.name=validator-staked --wallet.hotkey=default --subtensor.network test --logging.debug 1 --netuid 163 -``` \ No newline at end of file +``` From 5d5d50f5e531ad70c9e1734e1feed6b7d8c2cf97 Mon Sep 17 00:00:00 2001 From: Wojtek Jurkowlaniec Date: Sun, 16 Mar 2025 23:42:21 +0100 Subject: [PATCH 203/227] Update validator.md --- DOCS/validator.md | 5 ----- 1 file changed, 5 deletions(-) diff --git a/DOCS/validator.md b/DOCS/validator.md index 6d83c455..9dcc6be0 100644 --- a/DOCS/validator.md +++ b/DOCS/validator.md @@ -27,11 +27,6 @@ Key features of the script include: - **PM2**: PM2 must be installed and available on your system. It is used to manage the validator process. - **zip and unzip** -### Wandb API key requirement - -- Contact us [on discord](https://discord.com/channels/1259812760280236122/1262734148020338780) to get Wandb API key -- Put your key in .env.example file - ## Installation and Setup 1. **Clone the Repository**: Make sure you have cloned the repository containing this script and have navigated to the correct directory. From c9d810e93bb64cfc83e1951920eb84ecd5ca616f Mon Sep 17 00:00:00 2001 From: konrad0960 <71330299+konrad0960@users.noreply.github.com> Date: Mon, 17 Mar 2025 15:55:41 +0100 Subject: [PATCH 204/227] validator hotfixes (#135) --- cancer_ai/base/base_validator.py | 1 + cancer_ai/validator/utils.py | 33 ++++++++++++++++++++++++-------- 2 files changed, 26 insertions(+), 8 deletions(-) diff --git a/cancer_ai/base/base_validator.py b/cancer_ai/base/base_validator.py index dd94e323..becdd033 100644 --- a/cancer_ai/base/base_validator.py +++ b/cancer_ai/base/base_validator.py @@ -76,6 +76,7 @@ def __init__(self, config=None): competition_leader_map={}, hotkey_score_map={} ) self.organizations_data_references = OrganizationDataReferenceFactory.get_instance() + self.org_latest_updates = {} self.load_state() # Init sync with the network. Updates the metagraph. self.sync(force_sync=True) diff --git a/cancer_ai/validator/utils.py b/cancer_ai/validator/utils.py index 5fe54514..f66dd3d6 100644 --- a/cancer_ai/validator/utils.py +++ b/cancer_ai/validator/utils.py @@ -12,6 +12,8 @@ ) from datetime import datetime from typing import Any +from retry import retry + class ModelType(Enum): @@ -99,13 +101,18 @@ async def fetch_organization_data_references(hf_repo_id: str, hf_api: HfApi) -> # prevent stale connections custom_headers = {"Connection": "close"} - files = hf_api.list_repo_tree( - repo_id=hf_repo_id, - repo_type="space", - token=None, - recursive=True, - expand=True, - ) + try: + files = list_repo_tree_with_retry( + hf_api=hf_api, + hf_repo_id=hf_repo_id, + repo_type="space", + token=None, + recursive=True, + expand=True + ) + except Exception as e: + bt.logging.error("Failed to list repo tree after 10 attempts: %s", e) + files = None yaml_data = [] @@ -253,4 +260,14 @@ async def check_for_new_dataset_files(hf_api: HfApi, org_latest_updates: dict) - dataset_hf_filename=file_name )) - return results \ No newline at end of file + return results + +@retry(tries=10, delay=5) +def list_repo_tree_with_retry(hf_api, repo_id, repo_type, token, recursive, expand): + return hf_api.list_repo_tree( + repo_id=repo_id, + repo_type=repo_type, + token=token, + recursive=recursive, + expand=expand, + ) From 2e65d01b9cac7c4c11fb901830c6a7b0031b357e Mon Sep 17 00:00:00 2001 From: konrad0960 <71330299+konrad0960@users.noreply.github.com> Date: Mon, 17 Mar 2025 16:34:28 +0100 Subject: [PATCH 205/227] hf retry hotfix (#136) --- cancer_ai/validator/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cancer_ai/validator/utils.py b/cancer_ai/validator/utils.py index f66dd3d6..19786731 100644 --- a/cancer_ai/validator/utils.py +++ b/cancer_ai/validator/utils.py @@ -263,9 +263,9 @@ async def check_for_new_dataset_files(hf_api: HfApi, org_latest_updates: dict) - return results @retry(tries=10, delay=5) -def list_repo_tree_with_retry(hf_api, repo_id, repo_type, token, recursive, expand): +def list_repo_tree_with_retry(hf_api, hf_repo_id, repo_type, token, recursive, expand): return hf_api.list_repo_tree( - repo_id=repo_id, + repo_id=hf_repo_id, repo_type=repo_type, token=token, recursive=recursive, From 9a2316e1ea69b9c9db3daead253314f48aedabf8 Mon Sep 17 00:00:00 2001 From: Wojtek Jurkowlaniec Date: Mon, 17 Mar 2025 20:14:48 +0100 Subject: [PATCH 206/227] get fresh packages from evaluting competition datasets (#137) * get fresh packages from evaluting competition datasets * remove old dataset before extracting new one * further fixes --------- Co-authored-by: konrad0960 --- DOCS/miner.md | 1 + cancer_ai/chain_models_store.py | 11 +- cancer_ai/utils/config.py | 15 +- cancer_ai/validator/dataset_manager.py | 5 +- cancer_ai/validator/models.py | 9 +- cancer_ai/validator/utils.py | 214 ++++++++++++++++++------- neurons/miner.py | 52 +++--- 7 files changed, 211 insertions(+), 96 deletions(-) diff --git a/DOCS/miner.md b/DOCS/miner.md index d09a798b..f9bfbf31 100644 --- a/DOCS/miner.md +++ b/DOCS/miner.md @@ -104,6 +104,7 @@ Command line argument explanation - `--clean-after-run` - it will delete dataset after evaluating the model - `--model_dir` - path for storing models (default: "./models") - `--dataset_dir` - path for storing datasets (default: "./datasets") +- `--datasets_config_hf_repo_id` - hugging face repository ID for datasets configuration - ex. "safescanai/competition-configuration-testnet" in case of testnet ### Upload to HuggingFace diff --git a/cancer_ai/chain_models_store.py b/cancer_ai/chain_models_store.py index 05b533f5..6bc3b43d 100644 --- a/cancer_ai/chain_models_store.py +++ b/cancer_ai/chain_models_store.py @@ -3,6 +3,8 @@ import bittensor as bt from pydantic import BaseModel, Field +from retry import retry + class ChainMinerModel(BaseModel): @@ -85,9 +87,8 @@ async def retrieve_model_metadata(self, hotkey: str) -> Optional[ChainMinerModel uid = next((uid for uid, hk in zip(uids, hotkeys) if hk == hotkey), None) - metadata = bt.core.extrinsics.serving.get_metadata( - self.subtensor, self.netuid, hotkey - ) + metadata = get_metadata_with_retry(self.subtensor, self.netuid, hotkey) + try: chain_str = self.subtensor.get_commitment(self.netuid, uid) except Exception as e: @@ -115,3 +116,7 @@ async def retrieve_model_metadata(self, hotkey: str) -> Optional[ChainMinerModel # The block id at which the metadata is stored model.block = metadata["block"] return model + +@retry(tries=10, delay=5) +def get_metadata_with_retry(subtensor, netuid, hotkey): + return bt.core.extrinsics.serving.get_metadata(subtensor, netuid, hotkey) diff --git a/cancer_ai/utils/config.py b/cancer_ai/utils/config.py index 7a555c85..ad4c4219 100644 --- a/cancer_ai/utils/config.py +++ b/cancer_ai/utils/config.py @@ -135,6 +135,13 @@ def add_args(cls, parser): default=30, ) + parser.add_argument( + "--datasets_config_hf_repo_id", + type=str, + help="The reference to Hugging Face datasets config.", + default="safescanai/competition-configuration", + ) + def add_miner_args(cls, parser): """Add miner specific arguments to the parser.""" @@ -149,6 +156,7 @@ def add_miner_args(cls, parser): "--hf_repo_id", type=str, help="Hugging Face model repository ID", + default="", ) parser.add_argument( @@ -201,7 +209,6 @@ def add_common_args(cls, parser): "--hf_token", type=str, help="Hugging Face API token", - default="", ) parser.add_argument( "--competition_id", @@ -327,12 +334,6 @@ def add_validator_args(cls, parser): default=False, ) - parser.add_argument( - "--datasets_config_hf_repo_id", - type=str, - help="The reference to Hugging Face datasets config.", - default="safescanai/competition-configuration", - ) parser.add_argument( "--miners_refresh_interval", diff --git a/cancer_ai/validator/dataset_manager.py b/cancer_ai/validator/dataset_manager.py index fa66d751..eabe9885 100644 --- a/cancer_ai/validator/dataset_manager.py +++ b/cancer_ai/validator/dataset_manager.py @@ -87,13 +87,16 @@ async def unzip_dataset(self) -> None: self.local_extracted_dir = Path( self.config.models.dataset_dir, self.competition_id ) + # delete old unpacked dataset + if os.path.exists(self.local_extracted_dir): + os.system(f"rm -R {self.local_extracted_dir}") bt.logging.debug(f"Dataset extracted to: { self.local_compressed_path}") os.system(f"rm -R {self.local_extracted_dir}") # TODO add error handling zip_file_path = self.local_compressed_path extract_dir = self.local_extracted_dir - command = f'unzip "{zip_file_path}" -d {extract_dir}' + command = f'unzip -o "{zip_file_path}" -d {extract_dir}' out, err = await run_command(command) if err: bt.logging.error(f"Error unzipping dataset: {err}") diff --git a/cancer_ai/validator/models.py b/cancer_ai/validator/models.py index 1040122b..93dc4242 100644 --- a/cancer_ai/validator/models.py +++ b/cancer_ai/validator/models.py @@ -19,7 +19,7 @@ class OrganizationDataReference(BaseModel): organization_id: str = Field(..., min_length=1, description="Unique identifier for the organization") contact_email: EmailStr = Field(..., description="Contact email address for the organization") dataset_hf_repo: str = Field(..., min_length=1, description="Hugging Face repository path for the dataset") - dataset_hf_dir: str = Field(..., min_length=1, description="Directory for the datasets in the repository") + dataset_hf_dir: str = Field("", min_length=0, description="Directory for the datasets in the repository") class OrganizationDataReferenceFactory(BaseModel): organizations: List[OrganizationDataReference] = Field(default_factory=list) @@ -41,6 +41,13 @@ def update_from_dict(self, data: dict): for key, value in data.items(): if key != "organizations": setattr(self, key, value) + + def find_organization_by_competition_id(self, competition_id: str) -> Optional[OrganizationDataReference]: + """Find an organization by competition ID. + Returns: + The organization data reference for the given competition ID, or None if not found + """ + return next((o for o in self.organizations if o.competition_id == competition_id), None) class NewDatasetFile(BaseModel): competition_id: str = Field(..., min_length=1, description="Competition identifier") diff --git a/cancer_ai/validator/utils.py b/cancer_ai/validator/utils.py index 19786731..a1ce70d3 100644 --- a/cancer_ai/validator/utils.py +++ b/cancer_ai/validator/utils.py @@ -1,19 +1,22 @@ from enum import Enum import os -import asyncio -import bittensor as bt import json +from datetime import datetime +import asyncio +import time +from functools import wraps + import yaml -from cancer_ai.validator.models import CompetitionsListModel, CompetitionModel +import bittensor as bt +from retry import retry from huggingface_hub import HfApi, hf_hub_download + from cancer_ai.validator.models import ( + CompetitionsListModel, + CompetitionModel, NewDatasetFile, OrganizationDataReferenceFactory, ) -from datetime import datetime -from typing import Any -from retry import retry - class ModelType(Enum): @@ -26,10 +29,6 @@ class ModelType(Enum): UNKNOWN = "Unknown format" -import time -from functools import wraps - - def log_time(func): @wraps(func) async def wrapper(*args, **kwargs): @@ -70,7 +69,8 @@ def detect_model_format(file_path) -> ModelType: elif header[:2] == b"\x89H": # Magic number for HDF5 files (used by Keras) return ModelType.KERAS_H5 - except Exception: + except Exception as e: + bt.logging.error(f"Failed to detect model format: {e}") return ModelType.UNKNOWN return ModelType.UNKNOWN @@ -90,13 +90,19 @@ async def run_command(cmd): def get_competition_config(path: str) -> CompetitionsListModel: - with open(path, "r") as f: + with open(path, "r", encoding="utf-8") as f: competitions_json = json.load(f) competitions = [CompetitionModel(**item) for item in competitions_json] return CompetitionsListModel(competitions=competitions) -async def fetch_organization_data_references(hf_repo_id: str, hf_api: HfApi) -> list[dict]: - bt.logging.trace(f"Fetching organization data references from Hugging Face repo {hf_repo_id}") + +async def fetch_organization_data_references( + hf_repo_id: str, hf_api: HfApi +) -> list[dict]: + bt.logging.trace( + f"Fetching organization data references from Hugging Face repo {hf_repo_id}" + ) + yaml_data = [] # prevent stale connections custom_headers = {"Connection": "close"} @@ -104,23 +110,21 @@ async def fetch_organization_data_references(hf_repo_id: str, hf_api: HfApi) -> try: files = list_repo_tree_with_retry( hf_api=hf_api, - hf_repo_id=hf_repo_id, + repo_id=hf_repo_id, repo_type="space", token=None, recursive=True, - expand=True + expand=True, ) except Exception as e: bt.logging.error("Failed to list repo tree after 10 attempts: %s", e) - files = None - - yaml_data = [] + return yaml_data for file_info in files: - if file_info.__class__.__name__ == 'RepoFile': + if file_info.__class__.__name__ == "RepoFile": file_path = file_info.path - if file_path.startswith('datasets/') and file_path.endswith('.yaml'): + if file_path.startswith("datasets/") and file_path.endswith(".yaml"): local_file_path = hf_hub_download( repo_id=hf_repo_id, repo_type="space", @@ -135,25 +139,32 @@ async def fetch_organization_data_references(hf_repo_id: str, hf_api: HfApi) -> if commit_date is not None: date_uploaded = commit_date else: - bt.logging.warning(f"Could not get the last commit date for {file_path}") + bt.logging.warning( + f"Could not get the last commit date for {file_path}" + ) date_uploaded = None - with open(local_file_path, 'r') as f: + with open(local_file_path, "r", encoding="utf-8") as f: try: data = yaml.safe_load(f) except yaml.YAMLError as e: - bt.logging.error(f"Error parsing YAML file {file_path}: {str(e)}") + bt.logging.error( + f"Error parsing YAML file {file_path}: {str(e)}" + ) continue # Skip this file due to parsing error - yaml_data.append({ - 'file_name': file_path, - 'yaml_data': data, - 'date_uploaded': date_uploaded - }) + yaml_data.append( + { + "file_name": file_path, + "yaml_data": data, + "date_uploaded": date_uploaded, + } + ) else: continue return yaml_data + async def fetch_yaml_data_from_local_repo(local_repo_path: str) -> list[dict]: """ Fetches YAML data from all YAML files in the specified local directory. @@ -164,27 +175,30 @@ async def fetch_yaml_data_from_local_repo(local_repo_path: str) -> list[dict]: # Traverse through the local directory to find YAML files for root, _, files in os.walk(local_repo_path): for file_name in files: - if file_name.endswith('.yaml'): + if file_name.endswith(".yaml"): file_path = os.path.join(root, file_name) relative_path = os.path.relpath(file_path, local_repo_path) commit_date = datetime.fromtimestamp(os.path.getmtime(file_path)) - with open(file_path, 'r') as f: + with open(file_path, "r", encoding="utf-8") as f: data = yaml.safe_load(f) - yaml_data.append({ - 'file_name': relative_path, - 'yaml_data': data, - 'date_uploaded': commit_date - }) + yaml_data.append( + { + "file_name": relative_path, + "yaml_data": data, + "date_uploaded": commit_date, + } + ) return yaml_data + async def sync_organizations_data_references(fetched_yaml_files: list[dict]): """ Synchronizes the OrganizationDataReferenceFactory state with the full content from the fetched YAML files. - + Each fetched YAML file is expected to contain a list of organization entries. The 'org_id' key from the YAML is remapped to 'organization_id' to match the model. """ @@ -196,43 +210,123 @@ async def sync_organizations_data_references(fetched_yaml_files: list[dict]): if "org_id" in entry: entry["organization_id"] = entry.pop("org_id") all_orgs.append(entry) - + update_data = {"organizations": all_orgs} - + factory = OrganizationDataReferenceFactory.get_instance() factory.update_from_dict(update_data) -async def check_for_new_dataset_files(hf_api: HfApi, org_latest_updates: dict) -> list[NewDatasetFile]: + +async def get_newest_competition_packages(config: bt.Config, competition_id: str, hf_api: HfApi = None, packages_count: int = 30) -> list[dict]: + """ + Gets the link to the newest package for a specific competition. + + Args: + competition_id: The ID of the competition to get the newest package for + hf_api: Optional HfApi instance. If not provided, a new one will be created. + + Returns: + A dictionary containing: + - dataset_hf_repo: The Hugging Face repository path + - dataset_hf_filename: The filename of the newest dataset in the repository + - dataset_hf_repo_type: The repository type (typically 'dataset') + """ + if hf_api is None: + hf_api = HfApi() + + datasets_references = await fetch_organization_data_references(config.datasets_config_hf_repo_id, hf_api) + await sync_organizations_data_references(datasets_references) + org_reference = OrganizationDataReferenceFactory.get_instance() + # Find the organization data reference for the given competition + org = org_reference.find_organization_by_competition_id(competition_id) + + if not org: + bt.logging.error(f"No organization found for competition ID: {competition_id}") + return None + + # List all files in the repository + try: + files = hf_api.list_repo_tree( + repo_id=org.dataset_hf_repo, + repo_type="dataset", # Assuming dataset is the default type + token=None, # Public repos + recursive=True, + expand=True, + ) + except Exception as e: + bt.logging.error(f"Failed to list repository tree for {org.dataset_hf_repo}: {e}") + return None + + # Filter for relevant files in the specified directory + relevant_files = [ + f for f in files + if f.__class__.__name__ == "RepoFile" + and f.path.startswith(org.dataset_hf_dir) and f.path.endswith(".zip") + ] + + if not relevant_files: + bt.logging.warning(f"No relevant files found in {org.dataset_hf_repo}/{org.dataset_hf_dir}") + return None + + # Sort files by commit date (newest first) + sorted_files = sorted( + relevant_files, + key=lambda f: f.last_commit.date if f.last_commit else datetime.min, + reverse=True + ) + + # Get the top X files based on packages_count + top_files = sorted_files[:packages_count] + + if not top_files: + return None + return [ + { + "dataset_hf_repo": org.dataset_hf_repo, + "dataset_hf_filename": file.path, + "dataset_hf_repo_type": "dataset" + } + for file in top_files + ] + + +async def check_for_new_dataset_files( + hf_api: HfApi, org_latest_updates: dict +) -> list[NewDatasetFile]: """ For each OrganizationDataReference stored in the singleton, this function: - Connects to the organization's public Hugging Face repo. - Lists files under the directory specified by dataset_hf_dir. - Determines the maximum commit date among those files. - + For a blank state, it returns the file with the latest commit date. On subsequent checks, it returns any file whose commit date is newer than the previously stored update. """ results = [] factory = OrganizationDataReferenceFactory.get_instance() - + for org in factory.organizations: files = hf_api.list_repo_tree( repo_id=org.dataset_hf_repo, repo_type="dataset", - token=None, # these are public repos + token=None, # these are public repos recursive=True, expand=True, ) relevant_files = [ - f for f in files - if f.__class__.__name__ == "RepoFile" and f.path.startswith(org.dataset_hf_dir) + f + for f in files + if f.__class__.__name__ == "RepoFile" + and f.path.startswith(org.dataset_hf_dir) ] max_commit_date = None for f in relevant_files: commit_date = f.last_commit.date if f.last_commit else None - if commit_date and (max_commit_date is None or commit_date > max_commit_date): + if commit_date and ( + max_commit_date is None or commit_date > max_commit_date + ): max_commit_date = commit_date - + new_files = [] stored_update = org_latest_updates.get(org.organization_id) # if there is no stored_update and max_commit_date is present (any commit date is present) @@ -248,24 +342,26 @@ async def check_for_new_dataset_files(hf_api: HfApi, org_latest_updates: dict) - commit_date = f.last_commit.date if f.last_commit else None if commit_date and commit_date > stored_update: new_files.append(f.path) - + # update the stored latest update for this organization. if max_commit_date is not None: org_latest_updates[org.organization_id] = max_commit_date - + for file_name in new_files: - results.append(NewDatasetFile( - competition_id=org.competition_id, - dataset_hf_repo=org.dataset_hf_repo, - dataset_hf_filename=file_name - )) - + results.append( + NewDatasetFile( + competition_id=org.competition_id, + dataset_hf_repo=org.dataset_hf_repo, + dataset_hf_filename=file_name, + ) + ) + return results -@retry(tries=10, delay=5) -def list_repo_tree_with_retry(hf_api, hf_repo_id, repo_type, token, recursive, expand): +@retry(tries=10, delay=5, logger=bt.logging) +def list_repo_tree_with_retry(hf_api, repo_id, repo_type, token, recursive, expand): return hf_api.list_repo_tree( - repo_id=hf_repo_id, + repo_id=repo_id, repo_type=repo_type, token=token, recursive=recursive, diff --git a/neurons/miner.py b/neurons/miner.py index 6d620f54..39fcd4c2 100644 --- a/neurons/miner.py +++ b/neurons/miner.py @@ -19,7 +19,7 @@ from cancer_ai.base.base_miner import BaseNeuron from cancer_ai.chain_models_store import ChainMinerModel, ChainModelMetadata from cancer_ai.utils.config import path_config, add_miner_args -from cancer_ai.validator.utils import get_competition_config +from cancer_ai.validator.utils import get_competition_config, get_newest_competition_packages class MinerManagerCLI: @@ -96,35 +96,37 @@ async def evaluate_model(self) -> None: run_manager = ModelRunManager( config=self.config, model=ModelInfo(file_path=self.config.model_path) ) - dataset_manager = DatasetManager( - self.config, - self.config.competition_id, - self.competition_config.competitions[0].dataset_hf_repo, - self.competition_config.competitions[0].dataset_hf_filename, - self.competition_config.competitions[0].dataset_hf_repo_type, - use_auth=False - ) - await dataset_manager.prepare_dataset() + dataset_packages = await get_newest_competition_packages(self.config, self.config.competition_id) + for package in dataset_packages: + dataset_manager = DatasetManager( + self.config, + self.config.competition_id, + package["dataset_hf_repo"], + package["dataset_hf_filename"], + package["dataset_hf_repo_type"], + use_auth=False + ) + await dataset_manager.prepare_dataset() - X_test, y_test = await dataset_manager.get_data() + X_test, y_test = await dataset_manager.get_data() - competition_handler = COMPETITION_HANDLER_MAPPING[self.config.competition_id]( - X_test=X_test, y_test=y_test - ) + competition_handler = COMPETITION_HANDLER_MAPPING[self.config.competition_id]( + X_test=X_test, y_test=y_test + ) - y_test = competition_handler.prepare_y_pred(y_test) + y_test = competition_handler.prepare_y_pred(y_test) - start_time = time.time() - y_pred = await run_manager.run(X_test) - run_time_s = time.time() - start_time + start_time = time.time() + y_pred = await run_manager.run(X_test) + run_time_s = time.time() - start_time - # print(y_pred) - model_result = competition_handler.get_model_result(y_test, y_pred, run_time_s) - bt.logging.info( - f"Evalutaion results:\n{model_result.model_dump_json(indent=4)}" - ) - if self.config.clean_after_run: - dataset_manager.delete_dataset() + # print(y_pred) + model_result = competition_handler.get_model_result(y_test, y_pred, run_time_s) + bt.logging.info( + f"Evalutaion results:\n{model_result.model_dump_json(indent=4)}" + ) + if self.config.clean_after_run: + dataset_manager.delete_dataset() async def compress_code(self) -> None: bt.logging.info("Compressing code") From bbce8b8c6c910de21c8176b8dccad5c8939d4cb1 Mon Sep 17 00:00:00 2001 From: Wojtek Jurkowlaniec Date: Tue, 18 Mar 2025 13:55:53 +0100 Subject: [PATCH 207/227] Update miner.md --- DOCS/miner.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/DOCS/miner.md b/DOCS/miner.md index f9bfbf31..5eb66775 100644 --- a/DOCS/miner.md +++ b/DOCS/miner.md @@ -109,7 +109,9 @@ Command line argument explanation ### Upload to HuggingFace This mode compresses the code provided by `--code-path` and uploads the model and code to HuggingFace. -Repository ID should be a repository type "model" +Repository ID should be a repository type "model". + +The repository needs to be public for validator to pick it up. To upload to HuggingFace, use the following command: @@ -135,6 +137,8 @@ Command line argument explanation This mode saves model information in the metagraph, allowing validators to retrieve information about your model for testing. +The repository you are submitting needs to be public for validator to pick it up. + To submit a model to validators, use the following command: ```bash From 23eaa7a7d494a656dc8afcaa9c32e8290839a2e7 Mon Sep 17 00:00:00 2001 From: Wojtek Jurkowlaniec Date: Tue, 18 Mar 2025 13:58:46 +0100 Subject: [PATCH 208/227] Update miner.md --- DOCS/miner.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DOCS/miner.md b/DOCS/miner.md index 5eb66775..ff49d884 100644 --- a/DOCS/miner.md +++ b/DOCS/miner.md @@ -148,7 +148,7 @@ python neurons/miner.py \ --hf_code_filename skin_melanoma_small.zip\ --hf_model_name best_model.onnx \ --hf_repo_id safescanai/test_dataset \ - --hf_repo_type dataset \ + --hf_repo_type model \ --wallet.name miner2 \ --wallet.hotkey default \ --netuid 163 \ From e52a20a4825d3fe9d29446235fad37828d24a3c8 Mon Sep 17 00:00:00 2001 From: Wojtek Jurkowlaniec Date: Tue, 18 Mar 2025 13:59:14 +0100 Subject: [PATCH 209/227] fetch only zip files from datasets (#138) --- cancer_ai/validator/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cancer_ai/validator/utils.py b/cancer_ai/validator/utils.py index a1ce70d3..aed6bf93 100644 --- a/cancer_ai/validator/utils.py +++ b/cancer_ai/validator/utils.py @@ -317,7 +317,7 @@ async def check_for_new_dataset_files( f for f in files if f.__class__.__name__ == "RepoFile" - and f.path.startswith(org.dataset_hf_dir) + and f.path.startswith(org.dataset_hf_dir) and f.path.endswith(".zip") ] max_commit_date = None for f in relevant_files: From 417eee363936f53eeb80265fbb885616fa5bd501 Mon Sep 17 00:00:00 2001 From: Wojtek Jurkowlaniec Date: Tue, 18 Mar 2025 14:28:01 +0100 Subject: [PATCH 210/227] Update min_compute.yml --- min_compute.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/min_compute.yml b/min_compute.yml index 27549efd..257d05fd 100644 --- a/min_compute.yml +++ b/min_compute.yml @@ -11,9 +11,9 @@ required: False memory: - min_ram: 16 # Minimum RAM (GB) + min_ram: 8 # Minimum RAM (GB) min_swap: 4 # Minimum swap space (GB) - recommended_swap: 12 # Recommended swap space (GB) + recommended_swap: 16 # Recommended swap space (GB) ram_type: "DDR4" # RAM type (e.g., DDR4, DDR3, etc.) storage: @@ -59,4 +59,4 @@ version: 24.04 # Version of the preferred operating system(s) network_spec: - bandwidth: \ No newline at end of file + bandwidth: From d08ed9f43f665bf22d3dc96d8935dfbd5fc854f8 Mon Sep 17 00:00:00 2001 From: Wojtek Jurkowlaniec Date: Tue, 18 Mar 2025 14:28:35 +0100 Subject: [PATCH 211/227] Update min_compute.yml --- min_compute.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/min_compute.yml b/min_compute.yml index 257d05fd..4e31ed56 100644 --- a/min_compute.yml +++ b/min_compute.yml @@ -44,14 +44,14 @@ recommended_gpu: "NVIDIA RTX" # provide a recommended GPU to purchase/rent memory: - min_ram: 64 # Minimum RAM (GB) + min_ram: 16 # Minimum RAM (GB) min_swap: 4 # Minimum swap space (GB) - recommended_swap: 12 # Recommended swap space (GB) + recommended_swap: 4 # Recommended swap space (GB) ram_type: "DDR4" # RAM type (e.g., DDR4, DDR3, etc.) storage: - min_space: 500 # Minimum free storage space (GB) - recommended_space: 1000 # Recommended free storage space (GB) + min_space: 300 # Minimum free storage space (GB) + recommended_space: 500 # Recommended free storage space (GB) type: "SSD" # Preferred storage type (e.g., SSD, HDD) os: From 6f1f70462959aa53fbaf4c06b983d7193c067c1b Mon Sep 17 00:00:00 2001 From: konrad0960 <71330299+konrad0960@users.noreply.github.com> Date: Tue, 18 Mar 2025 23:47:08 +0100 Subject: [PATCH 212/227] fixed timestamp issue (#139) * fixed timestamp issue --- cancer_ai/validator/model_db.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/cancer_ai/validator/model_db.py b/cancer_ai/validator/model_db.py index cfa5e434..5582070e 100644 --- a/cancer_ai/validator/model_db.py +++ b/cancer_ai/validator/model_db.py @@ -134,13 +134,22 @@ def update_model(self, chain_miner_model: ChainMinerModel, hotkey: str): def get_block_timestamp(self, block_number): """Gets the timestamp of a block given its number.""" try: - block_hash = self.subtensor.get_block_hash(block_number) + # Identify if the connected subtensor is testnet based on the chain endpoint + is_testnet = "test" in self.subtensor.chain_endpoint.lower() + + # Use the correct subtensor (archive for mainnet, normal for testnet) + if is_testnet: + block_hash = self.subtensor.get_block_hash(block_number) + else: + archive_subtensor = bt.subtensor(network="archive") + block_hash = archive_subtensor.get_block_hash(block_number) + if block_hash is None: raise ValueError(f"Block hash not found for block number {block_number}") - + timestamp_info = self.subtensor.substrate.query( - module='Timestamp', - storage_function='Now', + module="Timestamp", + storage_function="Now", block_hash=block_hash ) From 06b90aeaca256b6c1db4cc45cb4ac50928bebce1 Mon Sep 17 00:00:00 2001 From: konrad0960 <71330299+konrad0960@users.noreply.github.com> Date: Wed, 19 Mar 2025 19:00:14 +0100 Subject: [PATCH 213/227] refactor of miner evaluation (#140) --- cancer_ai/validator/utils.py | 47 +++++++++++++++--------------------- neurons/miner.py | 8 +++++- 2 files changed, 26 insertions(+), 29 deletions(-) diff --git a/cancer_ai/validator/utils.py b/cancer_ai/validator/utils.py index aed6bf93..d9a21808 100644 --- a/cancer_ai/validator/utils.py +++ b/cancer_ai/validator/utils.py @@ -217,47 +217,37 @@ async def sync_organizations_data_references(fetched_yaml_files: list[dict]): factory.update_from_dict(update_data) -async def get_newest_competition_packages(config: bt.Config, competition_id: str, hf_api: HfApi = None, packages_count: int = 30) -> list[dict]: +async def get_newest_competition_packages(config: bt.Config, hf_api: HfApi = None, packages_count: int = 30) -> list[dict]: """ Gets the link to the newest package for a specific competition. - - Args: - competition_id: The ID of the competition to get the newest package for - hf_api: Optional HfApi instance. If not provided, a new one will be created. - - Returns: - A dictionary containing: - - dataset_hf_repo: The Hugging Face repository path - - dataset_hf_filename: The filename of the newest dataset in the repository - - dataset_hf_repo_type: The repository type (typically 'dataset') """ + newest_competition_packages: list[dict] = [] + if hf_api is None: hf_api = HfApi() datasets_references = await fetch_organization_data_references(config.datasets_config_hf_repo_id, hf_api) await sync_organizations_data_references(datasets_references) org_reference = OrganizationDataReferenceFactory.get_instance() - # Find the organization data reference for the given competition - org = org_reference.find_organization_by_competition_id(competition_id) + org = org_reference.find_organization_by_competition_id(config.competition_id) if not org: - bt.logging.error(f"No organization found for competition ID: {competition_id}") - return None - - # List all files in the repository + bt.logging.info(f"No organization found for competition ID: {config.competition_id}") + return newest_competition_packages + try: - files = hf_api.list_repo_tree( + files = list_repo_tree_with_retry( + hf_api=hf_api, repo_id=org.dataset_hf_repo, - repo_type="dataset", # Assuming dataset is the default type - token=None, # Public repos + repo_type="dataset", + token=None, recursive=True, expand=True, ) except Exception as e: bt.logging.error(f"Failed to list repository tree for {org.dataset_hf_repo}: {e}") - return None + raise - # Filter for relevant files in the specified directory relevant_files = [ f for f in files if f.__class__.__name__ == "RepoFile" @@ -266,29 +256,30 @@ async def get_newest_competition_packages(config: bt.Config, competition_id: str if not relevant_files: bt.logging.warning(f"No relevant files found in {org.dataset_hf_repo}/{org.dataset_hf_dir}") - return None + return newest_competition_packages - # Sort files by commit date (newest first) sorted_files = sorted( relevant_files, key=lambda f: f.last_commit.date if f.last_commit else datetime.min, reverse=True ) - # Get the top X files based on packages_count top_files = sorted_files[:packages_count] if not top_files: - return None - return [ + return newest_competition_packages + newest_competition_packages = [ { "dataset_hf_repo": org.dataset_hf_repo, "dataset_hf_filename": file.path, - "dataset_hf_repo_type": "dataset" + "dataset_hf_repo_type": "dataset", } for file in top_files ] + return newest_competition_packages + + async def check_for_new_dataset_files( hf_api: HfApi, org_latest_updates: dict diff --git a/neurons/miner.py b/neurons/miner.py index 39fcd4c2..3ccc2dba 100644 --- a/neurons/miner.py +++ b/neurons/miner.py @@ -96,7 +96,13 @@ async def evaluate_model(self) -> None: run_manager = ModelRunManager( config=self.config, model=ModelInfo(file_path=self.config.model_path) ) - dataset_packages = await get_newest_competition_packages(self.config, self.config.competition_id) + + try: + dataset_packages = await get_newest_competition_packages(self.config) + except Exception as e: + bt.logging.error(f"Error retrieving competition packages: {e}") + return + for package in dataset_packages: dataset_manager = DatasetManager( self.config, From 08e832d06329cc1087c2fef87108a049a7857823 Mon Sep 17 00:00:00 2001 From: Wojtek Jurkowlaniec Date: Sat, 22 Mar 2025 12:01:18 +0100 Subject: [PATCH 214/227] Fix model date checking + cleanup (#141) * simplified model date checking * add progress log for downloading miner model metadata * test DRY chain miner store * possible speedup in getting chain info * cleanup and fix checking cutoff time --- cancer_ai/base/base_validator.py | 2 - cancer_ai/chain_models_store.py | 33 +++- cancer_ai/validator/competition_manager.py | 2 +- cancer_ai/validator/model_db.py | 40 +++-- cancer_ai/validator/model_manager.py | 168 +++++++++++---------- neurons/competition_runner.py | 153 ------------------- neurons/validator.py | 91 +---------- 7 files changed, 137 insertions(+), 352 deletions(-) delete mode 100644 neurons/competition_runner.py diff --git a/cancer_ai/base/base_validator.py b/cancer_ai/base/base_validator.py index becdd033..be5bc1ce 100644 --- a/cancer_ai/base/base_validator.py +++ b/cancer_ai/base/base_validator.py @@ -37,7 +37,6 @@ from ..mock import MockDendrite from ..utils.config import add_validator_args -from neurons.competition_runner import CompetitionRunStore from cancer_ai.validator.rewarder import CompetitionWinnersStore from cancer_ai.validator.models import OrganizationDataReferenceFactory from .. import __spec_version__ as spec_version @@ -71,7 +70,6 @@ def __init__(self, config=None): # Set up initial scoring weights for validation bt.logging.info("Building validation weights.") self.scores = np.zeros(self.metagraph.n, dtype=np.float32) - self.run_log = CompetitionRunStore(runs=[]) self.winners_store = CompetitionWinnersStore( competition_leader_map={}, hotkey_score_map={} ) diff --git a/cancer_ai/chain_models_store.py b/cancer_ai/chain_models_store.py index 6bc3b43d..43208e95 100644 --- a/cancer_ai/chain_models_store.py +++ b/cancer_ai/chain_models_store.py @@ -1,5 +1,7 @@ import functools from typing import Optional, Type +import asyncio +from functools import wraps import bittensor as bt from pydantic import BaseModel, Field @@ -65,6 +67,7 @@ def __init__( wallet # Wallet is only needed to write to the chain, not to read. ) self.netuid = netuid + self.subnet_metadata = self.subtensor.metagraph(self.netuid) async def store_model_metadata(self, model_id: ChainMinerModel): """Stores model metadata on this subnet for a specific wallet.""" @@ -80,15 +83,17 @@ async def store_model_metadata(self, model_id: ChainMinerModel): async def retrieve_model_metadata(self, hotkey: str) -> Optional[ChainMinerModel]: """Retrieves model metadata on this subnet for specific hotkey""" + metadata = await get_metadata_with_timeout(self.subtensor, self.netuid, hotkey) + if metadata is None: + self.subnet_metadata = self.subtensor.metagraph(self.netuid) + metadata = await get_metadata_with_timeout(self.subtensor, self.netuid, hotkey) + if metadata is None: + return None - subnet_metadata = self.subtensor.metagraph(self.netuid) - uids = subnet_metadata.uids - hotkeys = subnet_metadata.hotkeys - + uids = self.subnet_metadata.uids + hotkeys = self.subnet_metadata.hotkeys uid = next((uid for uid, hk in zip(uids, hotkeys) if hk == hotkey), None) - metadata = get_metadata_with_retry(self.subtensor, self.netuid, hotkey) - try: chain_str = self.subtensor.get_commitment(self.netuid, uid) except Exception as e: @@ -117,6 +122,22 @@ async def retrieve_model_metadata(self, hotkey: str) -> Optional[ChainMinerModel model.block = metadata["block"] return model +def timeout(seconds): + def decorator(func): + @wraps(func) + async def wrapper(*args, **kwargs): + try: + return await asyncio.wait_for(func(*args, **kwargs), timeout=seconds) + except asyncio.TimeoutError: + bt.logging.debug("Metadata retrieval timed out, refreshing subnet metadata") + return None + return wrapper + return decorator + @retry(tries=10, delay=5) def get_metadata_with_retry(subtensor, netuid, hotkey): return bt.core.extrinsics.serving.get_metadata(subtensor, netuid, hotkey) + +@timeout(10) # 10 second timeout +async def get_metadata_with_timeout(subtensor, netuid, hotkey): + return get_metadata_with_retry(subtensor, netuid, hotkey) diff --git a/cancer_ai/validator/competition_manager.py b/cancer_ai/validator/competition_manager.py index 53f1c3a9..d2a35c3b 100644 --- a/cancer_ai/validator/competition_manager.py +++ b/cancer_ai/validator/competition_manager.py @@ -159,7 +159,7 @@ async def sync_chain_miners(self): bt.logging.info("Selecting models for competition") bt.logging.info(f"Amount of hotkeys: {len(self.hotkeys)}") - latest_models = self.db_controller.get_latest_models(self.hotkeys, self.competition_id, self.config.models_query_cutoff) + latest_models = self.db_controller.get_latest_models(self.hotkeys, self.competition_id) for hotkey, model in latest_models.items(): try: model_info = await self.chain_miner_to_model_info(model) diff --git a/cancer_ai/validator/model_db.py b/cancer_ai/validator/model_db.py index 5582070e..8d0cd359 100644 --- a/cancer_ai/validator/model_db.py +++ b/cancer_ai/validator/model_db.py @@ -3,7 +3,7 @@ from sqlalchemy import create_engine, Column, String, DateTime, PrimaryKeyConstraint, Integer from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.orm import sessionmaker -from datetime import datetime, timedelta +from datetime import datetime, timedelta, timezone from ..chain_models_store import ChainMinerModel Base = declarative_base() @@ -35,8 +35,8 @@ def __init__(self, subtensor: bt.subtensor, db_path: str = "models.db"): def add_model(self, chain_miner_model: ChainMinerModel, hotkey: str): session = self.Session() - date_submitted = self.get_block_timestamp(chain_miner_model.block) - existing_model = self.get_model(date_submitted, hotkey) + # date_submitted = self.get_block_timestamp(chain_miner_model.block) + existing_model = self.get_model(hotkey) if not existing_model: try: model_record = self.convert_chain_model_to_db_model(chain_miner_model, hotkey) @@ -49,14 +49,14 @@ def add_model(self, chain_miner_model: ChainMinerModel, hotkey: str): finally: session.close() else: - bt.logging.debug(f"Model for hotkey {hotkey} and date {date_submitted} already exists, proceeding with updating the model info.") + bt.logging.debug(f"Model for hotkey {hotkey} already exists, proceeding with updating the model info.") self.update_model(chain_miner_model, hotkey) - def get_model(self, date_submitted: datetime, hotkey: str) -> ChainMinerModel | None: + def get_model(self, hotkey: str) -> ChainMinerModel | None: session = self.Session() try: model_record = session.query(ChainMinerModelDB).filter_by( - date_submitted=date_submitted, hotkey=hotkey + hotkey=hotkey ).first() if model_record: return self.convert_db_model_to_chain_model(model_record) @@ -66,13 +66,13 @@ def get_model(self, date_submitted: datetime, hotkey: str) -> ChainMinerModel | def get_latest_model(self, hotkey: str, cutoff_time: float = None) -> ChainMinerModel | None: bt.logging.debug(f"Getting latest model for hotkey {hotkey} with cutoff time {cutoff_time}") - cutoff_time = datetime.now() - timedelta(minutes=cutoff_time) if cutoff_time else datetime.now() + # cutoff_time = datetime.now(timezone.utc) - timedelta(minutes=cutoff_time) if cutoff_time else datetime.now(timezone.utc) session = self.Session() try: model_record = ( session.query(ChainMinerModelDB) .filter(ChainMinerModelDB.hotkey == hotkey) - .filter(ChainMinerModelDB.date_submitted < cutoff_time) + # .filter(ChainMinerModelDB.date_submitted < cutoff_time) .order_by(ChainMinerModelDB.date_submitted.desc()) .first() ) @@ -102,9 +102,8 @@ def delete_model(self, date_submitted: datetime, hotkey: str): def update_model(self, chain_miner_model: ChainMinerModel, hotkey: str): session = self.Session() try: - date_submitted = self.get_block_timestamp(chain_miner_model.block) + # date_submitted = self.get_block_timestamp(chain_miner_model.block) existing_model = session.query(ChainMinerModelDB).filter_by( - date_submitted=date_submitted, hotkey=hotkey ).first() @@ -114,14 +113,14 @@ def update_model(self, chain_miner_model: ChainMinerModel, hotkey: str): existing_model.hf_model_filename = chain_miner_model.hf_model_filename existing_model.hf_repo_type = chain_miner_model.hf_repo_type existing_model.hf_code_filename = chain_miner_model.hf_code_filename - existing_model.block = chain_miner_model.block - existing_model.date_submitted = date_submitted + # existing_model.block = chain_miner_model.block + # existing_model.date_submitted = date_submitted session.commit() - bt.logging.debug(f"Successfully updated model for hotkey {hotkey} and date {date_submitted}.") + bt.logging.debug(f"Successfully updated model for hotkey {hotkey}.") return True else: - bt.logging.debug(f"No existing model found for hotkey {hotkey} and date {date_submitted}. Update skipped.") + bt.logging.debug(f"No existing model found for hotkey {hotkey}. Update skipped.") return False except Exception as e: @@ -164,8 +163,8 @@ def get_block_timestamp(self, block_number): bt.logging.error(f"Error retrieving block timestamp: {e}") raise - def get_latest_models(self, hotkeys: list[str], competition_id: str, cutoff_time: float = None) -> dict[str, ChainMinerModel]: - cutoff_time = datetime.now() - timedelta(minutes=cutoff_time) if cutoff_time else datetime.now() + def get_latest_models(self, hotkeys: list[str], competition_id: str) -> dict[str, ChainMinerModel]: + # cutoff_time = datetime.now(timezone.utc) - timedelta(minutes=cutoff_time) if cutoff_time else datetime.now(timezone.utc) session = self.Session() try: # Use a correlated subquery to get the latest record for each hotkey that doesn't violate the cutoff @@ -175,8 +174,8 @@ def get_latest_models(self, hotkeys: list[str], competition_id: str, cutoff_time session.query(ChainMinerModelDB) .filter(ChainMinerModelDB.hotkey == hotkey) .filter(ChainMinerModelDB.competition_id == competition_id) - .filter(ChainMinerModelDB.date_submitted < cutoff_time) - .order_by(ChainMinerModelDB.date_submitted.desc()) # Order by newest first + # .filter(ChainMinerModelDB.date_submitted < cutoff_time) + # .order_by(ChainMinerModelDB.date_submitted.desc()) # Order by newest first .first() # Get the first (newest) record that meets the cutoff condition ) if model_record: @@ -222,15 +221,14 @@ def clean_old_records(self, hotkeys: list[str]): session.close() def convert_chain_model_to_db_model(self, chain_miner_model: ChainMinerModel, hotkey: str) -> ChainMinerModelDB: - date_submitted = self.get_block_timestamp(chain_miner_model.block) return ChainMinerModelDB( competition_id = chain_miner_model.competition_id, hf_repo_id = chain_miner_model.hf_repo_id, hf_model_filename = chain_miner_model.hf_model_filename, hf_repo_type = chain_miner_model.hf_repo_type, hf_code_filename = chain_miner_model.hf_code_filename, - date_submitted = date_submitted, - block = chain_miner_model.block, + date_submitted = datetime.now(timezone.utc), # temporary fix, can't be null + block = 1, # temporary fix , can't be null hotkey = hotkey ) diff --git a/cancer_ai/validator/model_manager.py b/cancer_ai/validator/model_manager.py index 8952486c..4f342d53 100644 --- a/cancer_ai/validator/model_manager.py +++ b/cancer_ai/validator/model_manager.py @@ -1,9 +1,9 @@ import os from dataclasses import dataclass, asdict, is_dataclass -from datetime import datetime, timezone +from datetime import datetime, timezone, timedelta import bittensor as bt -from huggingface_hub import HfApi +from huggingface_hub import HfApi, hf_hub_url, HfFileSystem from .manager import SerializableManager from .exceptions import ModelRunException @@ -43,40 +43,96 @@ async def download_miner_model(self, hotkey) -> bool: bool: True if the model was downloaded successfully, False otherwise. """ model_info = self.hotkey_store[hotkey] - chain_model_date = await self.get_newest_saved_model_date(hotkey) - if chain_model_date and chain_model_date.tzinfo is None: - chain_model_date = chain_model_date.replace(tzinfo=timezone.utc) - if not chain_model_date: - bt.logging.error(f"Failed to get the newest saved model's date for hotkey {hotkey} from the local DB. Model download skipped.") + + if self.config.hf_token: + fs = HfFileSystem(token=self.config.hf_token) + else: + fs = HfFileSystem() + repo_path = os.path.join(model_info.hf_repo_id, model_info.hf_model_filename) + + # List files in the repository and get file date + try: + files = fs.ls(model_info.hf_repo_id) + except Exception as e: + bt.logging.error(f"Failed to list files in repository {model_info.hf_repo_id}: {e}") return False - + + # Find the specific file and its upload date + file_date = None + for file in files: + if file['name'] == repo_path: + # Extract the upload date + file_date = file["last_commit"]["date"] + break + + if not file_date: + bt.logging.error(f"File {model_info.hf_model_filename} not found in repository {model_info.hf_repo_id}") + return False + + # Parse and check if the model is too recent to download + is_too_recent, parsed_date = self.is_model_too_recent(file_date, model_info.hf_model_filename, hotkey) + if is_too_recent: + return False + + file_date = parsed_date + + # Download the file try: - commits = self.api.list_repo_commits( + model_info.file_path = self.api.hf_hub_download( repo_id=model_info.hf_repo_id, repo_type=model_info.hf_repo_type, - token=self.config.hf_token if hasattr(self.config, "hf_token") else None - ) - - model_commit = self.get_commit_with_file_change( - commits, model_info, chain_model_date + filename=model_info.hf_model_filename, + cache_dir=self.config.models.model_dir, + token=self.config.hf_token if hasattr(self.config, "hf_token") else None, ) - - if model_commit: - try: - self.download_model_at_commit(model_commit, model_info) - bt.logging.info(f"Downloaded an older model version for hotkey {hotkey} (date: {model_commit.created_at}).") - return True - except Exception as e: - bt.logging.error(f"Failed to download model at commit {model_commit.commit_id}: {e}") - return False - else: - bt.logging.error(f"No matching or older model found for hotkey {hotkey} based on the saved date. Download skipped.") - return False - except Exception as e: - bt.logging.error(f"Failed to download model: {e}") + bt.logging.error(f"Failed to download model file: {e}") return False - + + # Verify the downloaded file exists + if not os.path.exists(model_info.file_path): + bt.logging.error(f"Downloaded file does not exist at {model_info.file_path}") + return False + + bt.logging.info(f"Successfully downloaded model file to {model_info.file_path}") + return True + + def is_model_too_recent(self, file_date, filename, hotkey): + """Checks if a model file was uploaded too recently based on the cutoff time. + + Args: + file_date: The date when the file was uploaded (string or datetime) + filename: The name of the model file + hotkey: The hotkey of the miner + + Returns: + tuple: (is_too_recent, parsed_date) where is_too_recent is a boolean indicating if the model + is too recent to download, and parsed_date is the parsed datetime object with timezone + """ + # Ensure file_date is a datetime with timezone + try: + if isinstance(file_date, str): + file_date = datetime.fromisoformat(file_date) + if file_date.tzinfo is None: + file_date = file_date.replace(tzinfo=timezone.utc) + except Exception as e: + bt.logging.error(f"Failed to parse file date {file_date}: {e}") + return True, None + + bt.logging.debug(f"File {filename} was uploaded on: {file_date}") + + # Check if file is newer than our cutoff date (uploaded within last X minutes) + now = datetime.now(timezone.utc) # Get current time in UTC + + # Calculate time difference in minutes + time_diff = (now - file_date).total_seconds() / 60 + + if time_diff < self.config.models_query_cutoff: + bt.logging.warning(f"Skipping model for hotkey {hotkey} because it was uploaded {time_diff:.2f} minutes ago, which is within the cutoff of {self.config.models_query_cutoff} minutes") + return True, file_date + + return False, file_date + async def get_newest_saved_model_date(self, hotkey): """Fetches the newest saved model's date for a given hotkey from the local database.""" newest_saved_model = self.db_controller.get_latest_model(hotkey, self.config.models_query_cutoff) @@ -85,60 +141,6 @@ async def get_newest_saved_model_date(self, hotkey): return None return self.db_controller.get_block_timestamp(newest_saved_model.block) - def get_commit_with_file_change(self, commits, model_info, chain_model_date): - """ - Finds the most recent commit (relative to chain_model_date) where the specific file exists - and matches the date criteria. Assumes commits are sorted from newest to oldest. - """ - for commit in commits: - commit_id = commit.commit_id - commit_date = commit.created_at - if commit_date.tzinfo is None: - commit_date = commit_date.replace(tzinfo=timezone.utc) - - # Skip commits newer than the specified date - if commit_date > chain_model_date: - bt.logging.debug(f"Skipping commit {commit_id} because it is newer than chain_model_date") - continue - - # Check if the file exists at this commit - try: - files = self.api.list_repo_files( - repo_id=model_info.hf_repo_id, - revision=commit_id, - repo_type=model_info.hf_repo_type, - token=self.config.hf_token if hasattr(self.config, "hf_token") else None, - ) - - if model_info.hf_model_filename in files: - bt.logging.info(f"Found model version of commit {commit_id}") - return commit # Return the first valid commit and stop searching - - except Exception as e: - bt.logging.error(f"Failed to list files at commit {commit_id}: {e}") - continue - - bt.logging.error("No suitable older commit with the required file was found.") - return None - - - def download_model_at_commit(self, commit, model_info): - try: - model_info.file_path = self.api.hf_hub_download( - repo_id=model_info.hf_repo_id, - repo_type=model_info.hf_repo_type, - filename=model_info.hf_model_filename, - cache_dir=self.config.models.model_dir, - revision=commit.commit_id, - token=self.config.hf_token if hasattr(self.config, "hf_token") else None, - ) - if not os.path.exists(model_info.file_path): - bt.logging.error(f"Downloaded file does not exist at {model_info.file_path}") - raise FileNotFoundError(f"File {model_info.file_path} was not found after download.") - bt.logging.info(f"Successfully downloaded model file to {model_info.file_path}") - except Exception as e: - bt.logging.error(f"Failed to download model file at commit {commit.commit_id}: {e}") - raise def add_model( self, diff --git a/neurons/competition_runner.py b/neurons/competition_runner.py deleted file mode 100644 index 68c29c85..00000000 --- a/neurons/competition_runner.py +++ /dev/null @@ -1,153 +0,0 @@ -import json -from typing import List, Tuple -from datetime import datetime, timezone, timedelta, time -import asyncio - - -from pydantic import BaseModel -import bittensor as bt - -from cancer_ai.validator.competition_manager import CompetitionManager -from cancer_ai.validator.competition_handlers.base_handler import ModelEvaluationResult -from cancer_ai.validator.utils import get_competition_config -from cancer_ai.validator.model_db import ModelDBController - -MINUTES_BACK = 15 - - -class CompetitionRun(BaseModel): - competition_id: str - start_time: datetime - end_time: datetime | None = None - - -class CompetitionRunStore(BaseModel): - """ - The competition run store acts as a cache for competition runs and provides checks for competition execution states. - """ - - runs: list[CompetitionRun] - - def add_run(self, new_run: CompetitionRun): - """Add a new run and rotate the list if it exceeds 20 entries.""" - self.runs.append(new_run) - if len(self.runs) > 20: - self.runs = self.runs[-20:] - - def finish_run(self, competition_id: str): - """Finish the run with the given competition_id""" - for run in self.runs: - if run.competition_id == competition_id: - run.end_time = datetime.now(timezone.utc) - - def was_competition_already_executed( - self, competition_id: str, last_minutes: int = 15 - ): - """Check if competition was executed in last minutes""" - now_time = datetime.now(timezone.utc) - for run in self.runs: - if run.competition_id != competition_id: - continue - if run.end_time and (now_time - run.end_time).seconds < last_minutes * 60: - return True - return False - - -class CompetitionSchedule(BaseModel): - config: dict[datetime.time, CompetitionManager] - - class Config: - arbitrary_types_allowed = True - - -def get_competitions_schedule( - bt_config, - subtensor: bt.subtensor, - hotkeys: List[str], - validator_hotkey: str, - db_controller: ModelDBController, - test_mode: bool = False, -) -> CompetitionSchedule: - """Returns CompetitionManager instances arranged by competition time""" - scheduler_config = {} - main_competitions_cfg = get_competition_config(bt_config.competition.config_path) - for competition_cfg in main_competitions_cfg.competitions: - for competition_time in competition_cfg.evaluation_times: - parsed_time = datetime.strptime(competition_time, "%H:%M").time() - scheduler_config[parsed_time] = CompetitionManager( - config=bt_config, - subtensor=subtensor, - hotkeys=hotkeys, - validator_hotkey=validator_hotkey, - competition_id=competition_cfg.competition_id, - dataset_hf_repo=competition_cfg.dataset_hf_repo, - dataset_hf_id=competition_cfg.dataset_hf_filename, - dataset_hf_repo_type=competition_cfg.dataset_hf_repo_type, - db_controller=db_controller, - test_mode=test_mode, - ) - return scheduler_config - - -async def run_competitions_tick( - competition_scheduler: CompetitionSchedule, - run_log: CompetitionRunStore, -) -> Tuple[str, str, ModelEvaluationResult] | Tuple[None, None, None]: - """ - Checks if time is right and launches competition, - returns winning hotkey and Competition ID. Should be run each minute. - - Returns: - winning_hotkey (str) - competition_id (str) - evaluation_result (ModelEvaluationResult) - """ - - # getting current time - now = datetime.now(timezone.utc) - now_time = time(now.hour, now.minute) - bt.logging.info(f"Checking competitions at {now_time}") - - for i in range(0, MINUTES_BACK): - # getting current time minus X minutes - check_time = ( - datetime.combine(datetime.today(), now_time) - timedelta(minutes=i) - ).time() - competition_manager = competition_scheduler.get(check_time) - if not competition_manager: - continue - - bt.logging.debug( - f"Found competition {competition_manager.competition_id} at {check_time}" - ) - if run_log.was_competition_already_executed( - competition_id=competition_manager.competition_id, last_minutes=MINUTES_BACK - ): - bt.logging.info( - f"Competition {competition_manager.competition_id} already executed, skipping" - ) - continue - - bt.logging.info(f"Running {competition_manager.competition_id} at {now_time}") - - run_log.add_run( - CompetitionRun( - competition_id=competition_manager.competition_id, - start_time=datetime.now(timezone.utc), - ) - ) - winning_evaluation_hotkey, winning_model_result = ( - await competition_manager.evaluate() - ) - run_log.finish_run(competition_manager.competition_id) - return ( - winning_evaluation_hotkey, - competition_manager.competition_id, - winning_model_result, - ) - - bt.logging.debug( - f"Did not find any competitions to run for past {MINUTES_BACK} minutes" - ) - await asyncio.sleep(20) - return (None, None, None) diff --git a/neurons/validator.py b/neurons/validator.py index 892a149b..298b148c 100644 --- a/neurons/validator.py +++ b/neurons/validator.py @@ -31,11 +31,7 @@ from cancer_ai.chain_models_store import ChainModelMetadata from cancer_ai.validator.rewarder import CompetitionWinnersStore, Rewarder, Score from cancer_ai.base.base_validator import BaseValidatorNeuron -from competition_runner import ( - get_competitions_schedule, - run_competitions_tick, - CompetitionRunStore, -) + from cancer_ai.validator.cancer_ai_logo import cancer_ai_logo from cancer_ai.validator.utils import ( fetch_organization_data_references, @@ -51,20 +47,12 @@ BLACKLIST_FILE_PATH_TESTNET = "config/hotkey_blacklist_testnet.json" class Validator(BaseValidatorNeuron): - print(cancer_ai_logo) + def __init__(self, config=None): + print(cancer_ai_logo) super(Validator, self).__init__(config=config) self.hotkey = self.wallet.hotkey.ss58_address self.db_controller = ModelDBController(self.subtensor, self.config.db_path) - self.competition_scheduler = get_competitions_schedule( - bt_config = self.config, - subtensor = self.subtensor, - hotkeys = self.hotkeys, - validator_hotkey = self.hotkey, - db_controller = self.db_controller, - test_mode = False, - ) - bt.logging.info(f"Scheduler config: {self.competition_scheduler}") self.rewarder = Rewarder(self.winners_store) self.chain_models = ChainModelMetadata( @@ -108,12 +96,13 @@ async def refresh_miners(self): with open(blacklist_file, "r", encoding="utf-8") as f: BLACKLISTED_HOTKEYS = json.load(f) - for hotkey in self.hotkeys: + for i, hotkey in enumerate(self.hotkeys): if hotkey in BLACKLISTED_HOTKEYS: bt.logging.debug(f"Skipping blacklisted hotkey {hotkey}") continue hotkey = str(hotkey) + bt.logging.debug(f"Downloading model {i+1}/{len(self.hotkeys)} from hotkey {hotkey}") chain_model_metadata = await self.chain_models.retrieve_model_metadata(hotkey) if not chain_model_metadata: bt.logging.warning( @@ -126,73 +115,9 @@ async def refresh_miners(self): bt.logging.error(f"An error occured while trying to persist the model info: {e}") self.db_controller.clean_old_records(self.hotkeys) - latest_models = self.db_controller.get_latest_models(self.hotkeys, self.config.models_query_cutoff) - bt.logging.info( - f"Amount of latest miners with models: {len(latest_models)}" - ) self.last_miners_refresh = time.time() self.save_state() - async def competition_loop_tick(self): - """Main competition loop tick.""" - - # for testing purposes - # self.run_log = CompetitionRunStore(runs=[]) - - self.competition_scheduler = get_competitions_schedule( - bt_config = self.config, - subtensor = self.subtensor, - hotkeys = self.hotkeys, - validator_hotkey = self.hotkey, - db_controller=self.db_controller, - test_mode = self.config.test_mode, - ) - winning_hotkey = None - winning_model_link = None - try: - winning_hotkey, competition_id, winning_model_result = ( - await run_competitions_tick(self.competition_scheduler, self.run_log) - ) - except Exception: - formatted_traceback = traceback.format_exc() - bt.logging.error(f"Error running competition: {formatted_traceback}") - wandb.init( - reinit=True, project="competition_id", group="competition_evaluation" - ) - wandb.log( - { - "log_type": "competition_result", - "winning_evaluation_hotkey": "", - "run_time": "", - "validator_hotkey": self.wallet.hotkey.ss58_address, - "model_link": winning_model_link, - "errors": str(formatted_traceback), - } - ) - wandb.finish() - return - - if not winning_hotkey: - return - - wandb.init(project=competition_id, group="competition_evaluation") - run_time_s = ( - self.run_log.runs[-1].end_time - self.run_log.runs[-1].start_time - ).seconds - wandb.log( - { - "log_type": "competition_result", - "winning_hotkey": winning_hotkey, - "run_time_s": run_time_s, - "validator_hotkey": self.wallet.hotkey.ss58_address, - "model_link": winning_model_link, - "errors": "", - } - ) - wandb.finish() - - bt.logging.info(f"Competition result for {competition_id}: {winning_hotkey}") - self.handle_competition_winner(winning_hotkey, competition_id, winning_model_result) async def monitor_datasets(self): """Monitor datasets references for updates.""" @@ -304,9 +229,6 @@ def save_state(self): competition_leader_map={}, hotkey_score_map={} ) bt.logging.debug("Winner store empty, creating new one") - if not getattr(self, "run_log", None): - self.run_log = CompetitionRunStore(runs=[]) - bt.logging.debug("Competition run store empty, creating new one") if not getattr(self, "organizations_data_references", None): self.organizations_data_references = OrganizationDataReferenceFactory.get_instance() @@ -317,7 +239,6 @@ def save_state(self): scores=self.scores, hotkeys=self.hotkeys, winners_store=self.winners_store.model_dump(), - run_log=self.run_log.model_dump(), organizations_data_references=self.organizations_data_references.model_dump(), org_latest_updates=self.org_latest_updates, ) @@ -329,7 +250,6 @@ def create_empty_state(self): scores=self.scores, hotkeys=self.hotkeys, winners_store=self.winners_store.model_dump(), - run_log=self.run_log.model_dump(), organizations_data_references=self.organizations_data_references.model_dump(), org_latest_updates={}, ) @@ -353,7 +273,6 @@ def load_state(self): self.winners_store = CompetitionWinnersStore.model_validate( state["winners_store"].item() ) - self.run_log = CompetitionRunStore.model_validate(state["run_log"].item()) factory = OrganizationDataReferenceFactory.get_instance() saved_data = state["organizations_data_references"].item() From 1e187adcaa8ee97144ffe87bb034fad5fc834c9b Mon Sep 17 00:00:00 2001 From: Wojtek Jurkowlaniec Date: Fri, 28 Mar 2025 01:25:41 +0100 Subject: [PATCH 215/227] New rewarder mechanism - moving average * Add graceful shutdown handling for validator main sync error * fix error handing in onnx runner * NEW GREAT REWARDER * Fix new mechanism for rewards, fixed a lot of bugs ^^ * move update_competition_results to where it belongs * WANDB logs as data structure * cleanup * fix tests * fix melanoma handler, don't crash whole process when it fails * DB better logging + protection against validator crashing * fix model evaluation error handling * Error testing (#145) * error testing * loading state debug * Add detailed diagnostics for state saving/loading to identify corruption issues * MORE * new state saving * save state fix * wandb log all at once, check saving state * remove process debug messages * validator serialization fixes * faster logging to WANDB * testing validator save * fighting with states * save state fixes, don't do it so often * double try * save satte less often --- .gitignore | 5 +- cancer_ai/base/base_validator.py | 86 +++- cancer_ai/base/neuron.py | 5 +- cancer_ai/utils/config.py | 15 + .../competition_handlers/melanoma_handler.py | 23 +- cancer_ai/validator/competition_manager.py | 102 ++-- cancer_ai/validator/dataset_manager.py | 11 +- cancer_ai/validator/model_db.py | 62 +-- cancer_ai/validator/model_manager.py | 8 - cancer_ai/validator/model_manager_test.py | 13 +- .../validator/model_runners/onnx_runner.py | 3 - cancer_ai/validator/models.py | 37 +- cancer_ai/validator/rewarder.py | 304 ++++++------ cancer_ai/validator/tests/test_model_db.py | 67 ++- cancer_ai/validator/tests/test_rewarder.py | 169 +++++++ cancer_ai/validator/utils.py | 65 ++- neurons/miner.py | 6 +- neurons/validator.py | 463 +++++++++++++----- 18 files changed, 987 insertions(+), 457 deletions(-) create mode 100644 cancer_ai/validator/tests/test_rewarder.py diff --git a/.gitignore b/.gitignore index eb4701a9..586d872e 100644 --- a/.gitignore +++ b/.gitignore @@ -172,4 +172,7 @@ wandb ecosystem.config.js -keys \ No newline at end of file +keys + + +local_datasets/ \ No newline at end of file diff --git a/cancer_ai/base/base_validator.py b/cancer_ai/base/base_validator.py index be5bc1ce..56132768 100644 --- a/cancer_ai/base/base_validator.py +++ b/cancer_ai/base/base_validator.py @@ -19,6 +19,7 @@ from abc import abstractmethod +import sys import copy import numpy as np import asyncio @@ -37,7 +38,7 @@ from ..mock import MockDendrite from ..utils.config import add_validator_args -from cancer_ai.validator.rewarder import CompetitionWinnersStore +from cancer_ai.validator.rewarder import CompetitionResultsStore from cancer_ai.validator.models import OrganizationDataReferenceFactory from .. import __spec_version__ as spec_version @@ -54,7 +55,7 @@ def add_args(cls, parser: argparse.ArgumentParser): super().add_args(parser) add_validator_args(cls, parser) - def __init__(self, config=None): + def __init__(self, config=None, exit_event: threading.Event = None): super().__init__(config=config) # Save a copy of the hotkeys to local memory. @@ -70,10 +71,8 @@ def __init__(self, config=None): # Set up initial scoring weights for validation bt.logging.info("Building validation weights.") self.scores = np.zeros(self.metagraph.n, dtype=np.float32) - self.winners_store = CompetitionWinnersStore( - competition_leader_map={}, hotkey_score_map={} - ) self.organizations_data_references = OrganizationDataReferenceFactory.get_instance() + self.competition_results_store = CompetitionResultsStore() self.org_latest_updates = {} self.load_state() # Init sync with the network. Updates the metagraph. @@ -93,6 +92,7 @@ def __init__(self, config=None): self.is_running: bool = False self.thread: Union[threading.Thread, None] = None self.lock = asyncio.Lock() + self.exit_event = exit_event def serve_axon(self): """Serve axon to enable external connections.""" @@ -167,38 +167,73 @@ def run(self): # In case of unforeseen errors, the validator will log the error and continue operations. except Exception as err: - bt.logging.error(f"Error during validation: {str(err)}") - bt.logging.debug(str(print_exception(type(err), err, err.__traceback__))) + bt.logging.error(f"VALIDATOR FAILURE: Error during validation: {str(err)}") + bt.logging.error(f"Error type: {type(err).__name__}") + bt.logging.error(f"Error occurred in method: {self.concurrent_forward.__name__}") + bt.logging.error(f"Current step: {self.step}") + + # Log the full stack trace + import traceback + stack_trace = traceback.format_exc() + bt.logging.error(f"Full stack trace:\n{stack_trace}") + bt.logging.error(str(print_exception(type(err), err, err.__traceback__))) + + # Log additional context information + bt.logging.error(f"Validator state: running={self.is_running}, should_exit={self.should_exit}") + + if self.exit_event: + bt.logging.error("Setting exit event and terminating validator") + self.exit_event.set() + sys.exit(1) def run_in_background_thread(self): """ Starts the validator's operations in a background thread upon entering the context. This method facilitates the use of the validator in a 'with' statement. """ + bt.logging.info(f"run_in_background_thread called with is_running={self.is_running}") + + # Get the current call stack to see what's calling run_in_background_thread + import traceback + stack_trace = traceback.format_stack() + bt.logging.info(f"Call stack for run_in_background_thread:\n{''.join(stack_trace)}") + if not self.is_running: - bt.logging.debug("Starting validator in background thread.") + bt.logging.info("Starting validator in background thread.") self.should_exit = False + bt.logging.info(f"Set should_exit to {self.should_exit}, creating thread") self.thread = threading.Thread(target=self.run, daemon=True) + bt.logging.info(f"Starting thread with daemon={self.thread.daemon}") self.thread.start() self.is_running = True - bt.logging.debug("Started") + bt.logging.info(f"Thread started, set is_running to {self.is_running}") + bt.logging.info("Validator started successfully in background thread") + else: + bt.logging.warning("Attempted to start validator that is already running") def stop_run_thread(self): """ Stops the validator's operations that are running in the background thread. """ + bt.logging.info(f"stop_run_thread called with is_running={self.is_running}") + import traceback + stack_trace = traceback.format_stack() + bt.logging.info(f"Call stack for stop_run_thread:\n{''.join(stack_trace)}") + if self.is_running: - bt.logging.debug("Stopping validator in background thread.") + bt.logging.info("Stopping validator in background thread.") self.should_exit = True + bt.logging.info(f"Set should_exit to {self.should_exit}, joining thread") self.thread.join(5) self.is_running = False - bt.logging.debug("Stopped") + bt.logging.info(f"Thread joined, set is_running to {self.is_running}") + bt.logging.info("Validator stopped successfully") def __enter__(self): self.run_in_background_thread() return self - def __exit__(self, exc_type, exc_value, traceback): + def __exit__(self, exc_type, exc_value, traceback_obj): """ Stops the validator's background operations upon exiting the context. This method facilitates the use of the validator in a 'with' statement. @@ -208,15 +243,30 @@ def __exit__(self, exc_type, exc_value, traceback): None if the context was exited without an exception. exc_value: The instance of the exception that caused the context to be exited. None if the context was exited without an exception. - traceback: A traceback object encoding the stack trace. + traceback_obj: A traceback object encoding the stack trace. None if the context was exited without an exception. """ + bt.logging.info(f"__exit__ called with exc_type={exc_type}, exc_value={exc_value}") + + # Get the current call stack to see what's calling __exit__ + import traceback + stack_trace = traceback.format_stack() + bt.logging.info(f"Call stack for __exit__:\n{''.join(stack_trace)}") + + # If there's an exception, log it + if exc_type is not None: + bt.logging.error(f"Exception in context: {exc_type.__name__}: {exc_value}") + if traceback_obj: + bt.logging.error(f"Exception traceback: {''.join(traceback.format_tb(traceback_obj))}") + if self.is_running: - bt.logging.debug("Stopping validator in background thread.") + bt.logging.info("Stopping validator in background thread from __exit__ method.") self.should_exit = True + bt.logging.info(f"Set should_exit to {self.should_exit}, joining thread") self.thread.join(5) self.is_running = False - bt.logging.debug("Stopped") + bt.logging.info(f"Thread joined, set is_running to {self.is_running}") + bt.logging.info("Validator stopped successfully from __exit__ method") def set_weights(self): """ @@ -267,6 +317,12 @@ def set_weights(self): bt.logging.debug("uint_weights", uint_weights) bt.logging.debug("uint_uids", uint_uids) + # test mode, don't commit weights + if self.config.filesystem_evaluation: + bt.logging.debug("Skipping settings weights in filesystem evaluation mode") + return + + # Set the weights on chain via our subtensor connection. result, msg = self.subtensor.set_weights( wallet=self.wallet, diff --git a/cancer_ai/base/neuron.py b/cancer_ai/base/neuron.py index 65d9dae5..af9eb7a5 100644 --- a/cancer_ai/base/neuron.py +++ b/cancer_ai/base/neuron.py @@ -121,12 +121,11 @@ def sync(self, retries=5, delay=10, force_sync=False): if self.should_sync_metagraph() or force_sync: bt.logging.info("Resyncing metagraph in progress.") self.resync_metagraph(force_sync=True) + self.save_state() if self.should_set_weights(): self.set_weights() - - # Always save state. - self.save_state() + self.save_state() break diff --git a/cancer_ai/utils/config.py b/cancer_ai/utils/config.py index ad4c4219..64ef4314 100644 --- a/cancer_ai/utils/config.py +++ b/cancer_ai/utils/config.py @@ -349,6 +349,21 @@ def add_validator_args(cls, parser): default=8*60, ) + parser.add_argument( + "--local_dataset_dir", + type=str, + help="Path to the local dataset directory", + default="local_datasets/", + ) + + parser.add_argument( + "--filesystem_evaluation", + type=bool, + help="Should use local datasets instead of HF? Use together with --local_dataset_dir", + default=False + ) + + def path_config(cls=None): """ Returns the configuration object specific to this miner or validator after adding relevant arguments. diff --git a/cancer_ai/validator/competition_handlers/melanoma_handler.py b/cancer_ai/validator/competition_handlers/melanoma_handler.py index d5bb86ef..10675926 100644 --- a/cancer_ai/validator/competition_handlers/melanoma_handler.py +++ b/cancer_ai/validator/competition_handlers/melanoma_handler.py @@ -34,16 +34,33 @@ def calculate_score(self, fbeta: float, accuracy: float, roc_auc: float) -> floa return fbeta * WEIGHT_FBETA + accuracy * WEIGHT_ACCURACY + roc_auc * WEIGHT_AUC def get_model_result( - self, y_test: List[float], y_pred: np.ndarray, run_time_s: float + self, y_test: List[float], y_pred, run_time_s: float ) -> ModelEvaluationResult: - y_pred_binary = [1 if y > 0.5 else 0 for y in y_pred] + # Convert y_pred to numpy array if it's a list + if isinstance(y_pred, list): + y_pred = np.array(y_pred) + + # Handle the case where y_pred contains arrays instead of scalars + try: + # If y_pred is a 2D array, take the first column or flatten it if it's a single prediction per sample + if len(y_pred.shape) > 1 and y_pred.shape[1] > 1: + # If we have multiple predictions per sample, take the first column + y_pred_flat = y_pred[:, 0] + else: + # Otherwise flatten the array to ensure it's 1D + y_pred_flat = y_pred.flatten() + except (AttributeError, TypeError): + # If y_pred doesn't have shape attribute or other issues, use it directly + y_pred_flat = y_pred + + y_pred_binary = [1 if y > 0.5 else 0 for y in y_pred_flat] tested_entries = len(y_test) accuracy = accuracy_score(y_test, y_pred_binary) precision = precision_score(y_test, y_pred_binary, zero_division=0) fbeta = fbeta_score(y_test, y_pred_binary, beta=2, zero_division=0) recall = recall_score(y_test, y_pred_binary, zero_division=0) conf_matrix = confusion_matrix(y_test, y_pred_binary) - fpr, tpr, _ = roc_curve(y_test, y_pred) + fpr, tpr, _ = roc_curve(y_test, y_pred_flat) roc_auc = auc(fpr, tpr) score = self.calculate_score(fbeta, accuracy, roc_auc) diff --git a/cancer_ai/validator/competition_manager.py b/cancer_ai/validator/competition_manager.py index d2a35c3b..0048c137 100644 --- a/cancer_ai/validator/competition_manager.py +++ b/cancer_ai/validator/competition_manager.py @@ -12,6 +12,7 @@ from .exceptions import ModelRunException from .model_db import ModelDBController +from cancer_ai.validator.models import WandBLogModelEntry from .competition_handlers.melanoma_handler import MelanomaCompetitionHandler from .competition_handlers.base_handler import ModelEvaluationResult from .tests.mock_data import get_mock_hotkeys_with_models @@ -51,10 +52,11 @@ def __init__( validator_hotkey: str, competition_id: str, dataset_hf_repo: str, - dataset_hf_id: str, + dataset_hf_filename: str, dataset_hf_repo_type: str, db_controller: ModelDBController, test_mode: bool = False, + local_fs_mode: bool = False, ) -> None: """ Responsible for managing a competition. @@ -67,14 +69,15 @@ def __init__( self.config = config self.subtensor = subtensor self.competition_id = competition_id - self.results = [] + self.results: list[tuple[str, ModelEvaluationResult]] = [] self.model_manager = ModelManager(self.config, db_controller) self.dataset_manager = DatasetManager( - self.config, - competition_id, - dataset_hf_repo, - dataset_hf_id, - dataset_hf_repo_type, + config=self.config, + competition_id=competition_id, + hf_repo_id=dataset_hf_repo, + hf_filename=dataset_hf_filename, + hf_repo_type=dataset_hf_repo_type, + local_fs_mode=local_fs_mode, ) self.chain_model_metadata_store = ChainModelMetadata( self.subtensor, self.config.netuid @@ -84,41 +87,12 @@ def __init__( self.validator_hotkey = validator_hotkey self.db_controller = db_controller self.test_mode = test_mode + self.local_fs_mode = local_fs_mode def __repr__(self) -> str: return f"CompetitionManager<{self.competition_id}>" - def log_results_to_wandb( - self, miner_hotkey: str, validator_hotkey: str, evaluation_result: ModelEvaluationResult - ) -> None: - winning_model_link = self.db_controller.get_latest_model( - hotkey=miner_hotkey, cutoff_time=self.config.models_query_cutoff - ).hf_link - wandb.init(project=self.competition_id, group="model_evaluation") - wandb.log( - { - "log_type": "model_results", - "competition_id": self.competition_id, - "miner_hotkey": miner_hotkey, - "validator_hotkey": validator_hotkey, - "tested_entries": evaluation_result.tested_entries, - "accuracy": evaluation_result.accuracy, - "precision": evaluation_result.precision, - "fbeta": evaluation_result.fbeta, - "recall": evaluation_result.recall, - "confusion_matrix": evaluation_result.confusion_matrix, - "roc_curve": { - "fpr": evaluation_result.fpr, - "tpr": evaluation_result.tpr, - }, - "model_link": winning_model_link, - "roc_auc": evaluation_result.roc_auc, - "score": evaluation_result.score, - } - ) - - wandb.finish() - bt.logging.info(f"Results: {evaluation_result}") + def get_state(self): return { @@ -133,7 +107,6 @@ def set_state(self, state: dict): async def chain_miner_to_model_info( self, chain_miner_model: ChainMinerModel ) -> ModelInfo: - bt.logging.warning(f"Chain miner model: {chain_miner_model.model_dump()}") if chain_miner_model.competition_id != self.competition_id: bt.logging.debug( f"Chain miner model {chain_miner_model.to_compressed_str()} does not belong to this competition" @@ -148,18 +121,20 @@ async def chain_miner_to_model_info( ) return model_info - async def sync_chain_miners_test(self): + async def get_mock_miner_models(self): """Get registered mineres from testnet subnet 163""" self.model_manager.hotkey_store = get_mock_hotkeys_with_models() - async def sync_chain_miners(self): + async def update_miner_models(self): """ Updates hotkeys and downloads information of models from the chain """ bt.logging.info("Selecting models for competition") bt.logging.info(f"Amount of hotkeys: {len(self.hotkeys)}") - latest_models = self.db_controller.get_latest_models(self.hotkeys, self.competition_id) + latest_models = self.db_controller.get_latest_models( + self.hotkeys, self.competition_id + ) for hotkey, model in latest_models.items(): try: model_info = await self.chain_miner_to_model_info(model) @@ -176,21 +151,24 @@ async def sync_chain_miners(self): async def evaluate(self) -> Tuple[str | None, ModelEvaluationResult | None]: """Returns hotkey and competition id of winning model miner""" bt.logging.info(f"Start of evaluation of {self.competition_id}") - if self.test_mode: - await self.sync_chain_miners_test() - else: - await self.sync_chain_miners() + + # TODO add mock models functionality + + await self.update_miner_models() if len(self.model_manager.hotkey_store) == 0: bt.logging.error("No models to evaluate") return None, None + + await self.dataset_manager.prepare_dataset() X_test, y_test = await self.dataset_manager.get_data() competition_handler = COMPETITION_HANDLER_MAPPING[self.competition_id]( X_test=X_test, y_test=y_test ) - y_test = competition_handler.prepare_y_pred(y_test) + + # evaluate models for miner_hotkey in self.model_manager.hotkey_store: bt.logging.info(f"Evaluating hotkey: {miner_hotkey}") model_or_none = await self.model_manager.download_miner_model(miner_hotkey) @@ -199,13 +177,14 @@ async def evaluate(self) -> Tuple[str | None, ModelEvaluationResult | None]: f"Failed to download model for hotkey {miner_hotkey} Skipping." ) continue + try: model_manager = ModelRunManager( self.config, self.model_manager.hotkey_store[miner_hotkey] ) - except ModelRunException: + except ModelRunException as e: bt.logging.error( - f"Model hotkey: {miner_hotkey} failed to initialize. Skipping" + f"Model hotkey: {miner_hotkey} failed to initialize. Skipping. Error: {e}" ) continue start_time = time.time() @@ -213,16 +192,24 @@ async def evaluate(self) -> Tuple[str | None, ModelEvaluationResult | None]: try: y_pred = await model_manager.run(X_test) except ModelRunException: - bt.logging.error(f"Model hotkey: {miner_hotkey} failed to run. Skipping") + bt.logging.error( + f"Model hotkey: {miner_hotkey} failed to run. Skipping" + ) continue run_time_s = time.time() - start_time - model_result = competition_handler.get_model_result( - y_test, y_pred, run_time_s - ) - self.results.append((miner_hotkey, model_result)) - if not self.test_mode: - self.log_results_to_wandb(miner_hotkey, self.validator_hotkey, model_result) + try: + model_result = competition_handler.get_model_result( + y_test, y_pred, run_time_s + ) + self.results.append((miner_hotkey, model_result)) + except Exception as e: + bt.logging.error( + f"Error evaluating model for hotkey: {miner_hotkey}. Error: {str(e)}" + ) + import traceback + bt.logging.error(f"Stacktrace: {traceback.format_exc()}") + bt.logging.info(f"Skipping model {miner_hotkey} due to evaluation error") if len(self.results) == 0: bt.logging.error("No models were able to run") return None, None @@ -230,7 +217,8 @@ async def evaluate(self) -> Tuple[str | None, ModelEvaluationResult | None]: self.results, key=lambda x: x[1].score, reverse=True )[0] for miner_hotkey, model_result in self.results: - bt.logging.debug( + bt.logging.info(f"Model from {miner_hotkey} successfully evaluated") + bt.logging.trace( f"Model result for {miner_hotkey}:\n {model_result.model_dump_json(indent=4)} \n" ) diff --git a/cancer_ai/validator/dataset_manager.py b/cancer_ai/validator/dataset_manager.py index eabe9885..fb6d89e9 100644 --- a/cancer_ai/validator/dataset_manager.py +++ b/cancer_ai/validator/dataset_manager.py @@ -20,6 +20,7 @@ def __init__( hf_filename: str, hf_repo_type: str, use_auth: bool = True, + local_fs_mode: bool = False, ) -> None: """ Initializes a new instance of the DatasetManager class. @@ -44,6 +45,7 @@ def __init__( self.local_extracted_dir = Path(self.config.models.dataset_dir, competition_id) self.data: Tuple[List, List] = () self.handler = None + self.local_fs_mode = local_fs_mode def get_state(self) -> dict: return {} @@ -92,7 +94,7 @@ async def unzip_dataset(self) -> None: os.system(f"rm -R {self.local_extracted_dir}") bt.logging.debug(f"Dataset extracted to: { self.local_compressed_path}") - os.system(f"rm -R {self.local_extracted_dir}") + # TODO add error handling zip_file_path = self.local_compressed_path extract_dir = self.local_extracted_dir @@ -121,8 +123,11 @@ def set_dataset_handler(self) -> None: async def prepare_dataset(self) -> None: """Download dataset, unzip and set dataset handler""" - bt.logging.info(f"Downloading dataset '{self.competition_id}'") - await self.download_dataset() + if self.local_fs_mode: + self.local_compressed_path = self.hf_filename + else: + bt.logging.info(f"Downloading dataset '{self.competition_id}'") + await self.download_dataset() bt.logging.info(f"Unzipping dataset '{self.competition_id}'") await self.unzip_dataset() bt.logging.info(f"Setting dataset handler '{self.competition_id}'") diff --git a/cancer_ai/validator/model_db.py b/cancer_ai/validator/model_db.py index 8d0cd359..0ca9e7e7 100644 --- a/cancer_ai/validator/model_db.py +++ b/cancer_ai/validator/model_db.py @@ -26,8 +26,7 @@ class ChainMinerModelDB(Base): ) class ModelDBController: - def __init__(self, subtensor: bt.subtensor, db_path: str = "models.db"): - self.subtensor = subtensor + def __init__(self, db_path: str = "models.db"): db_url = f"sqlite:///{os.path.abspath(db_path)}" self.engine = create_engine(db_url, echo=False) Base.metadata.create_all(self.engine) @@ -35,21 +34,20 @@ def __init__(self, subtensor: bt.subtensor, db_path: str = "models.db"): def add_model(self, chain_miner_model: ChainMinerModel, hotkey: str): session = self.Session() - # date_submitted = self.get_block_timestamp(chain_miner_model.block) existing_model = self.get_model(hotkey) if not existing_model: try: model_record = self.convert_chain_model_to_db_model(chain_miner_model, hotkey) session.add(model_record) session.commit() - bt.logging.debug(f"Successfully added model info for hotkey {hotkey} into the DB.") + bt.logging.debug(f"Successfully added DB model info for hotkey {hotkey} into the DB.") except Exception as e: session.rollback() raise e finally: session.close() else: - bt.logging.debug(f"Model for hotkey {hotkey} already exists, proceeding with updating the model info.") + bt.logging.debug(f"DB model for hotkey {hotkey} already exists, proceeding with updating the model info.") self.update_model(chain_miner_model, hotkey) def get_model(self, hotkey: str) -> ChainMinerModel | None: @@ -65,20 +63,25 @@ def get_model(self, hotkey: str) -> ChainMinerModel | None: session.close() def get_latest_model(self, hotkey: str, cutoff_time: float = None) -> ChainMinerModel | None: - bt.logging.debug(f"Getting latest model for hotkey {hotkey} with cutoff time {cutoff_time}") - # cutoff_time = datetime.now(timezone.utc) - timedelta(minutes=cutoff_time) if cutoff_time else datetime.now(timezone.utc) + bt.logging.debug(f"Getting latest DB model for hotkey {hotkey}") session = self.Session() try: model_record = ( session.query(ChainMinerModelDB) .filter(ChainMinerModelDB.hotkey == hotkey) - # .filter(ChainMinerModelDB.date_submitted < cutoff_time) .order_by(ChainMinerModelDB.date_submitted.desc()) .first() ) if model_record: return self.convert_db_model_to_chain_model(model_record) return None + except Exception as e: + import traceback + stack_trace = traceback.format_exc() + bt.logging.error(f"Error in get_latest_model for hotkey {hotkey}: {e}") + bt.logging.error(f"Stack trace: {stack_trace}") + # Re-raise the exception to be caught by higher-level error handlers + raise finally: session.close() @@ -102,7 +105,6 @@ def delete_model(self, date_submitted: datetime, hotkey: str): def update_model(self, chain_miner_model: ChainMinerModel, hotkey: str): session = self.Session() try: - # date_submitted = self.get_block_timestamp(chain_miner_model.block) existing_model = session.query(ChainMinerModelDB).filter_by( hotkey=hotkey ).first() @@ -113,55 +115,21 @@ def update_model(self, chain_miner_model: ChainMinerModel, hotkey: str): existing_model.hf_model_filename = chain_miner_model.hf_model_filename existing_model.hf_repo_type = chain_miner_model.hf_repo_type existing_model.hf_code_filename = chain_miner_model.hf_code_filename - # existing_model.block = chain_miner_model.block - # existing_model.date_submitted = date_submitted session.commit() - bt.logging.debug(f"Successfully updated model for hotkey {hotkey}.") + bt.logging.debug(f"Successfully updated DB model for hotkey {hotkey}.") return True else: - bt.logging.debug(f"No existing model found for hotkey {hotkey}. Update skipped.") + bt.logging.debug(f"No existing DB model found for hotkey {hotkey}. Update skipped.") return False except Exception as e: session.rollback() - bt.logging.error(f"Error updating model for hotkey {hotkey} and date {date_submitted}: {e}") + bt.logging.error(f"Error updating DB model for hotkey {hotkey}: {e}") raise e finally: session.close() - def get_block_timestamp(self, block_number): - """Gets the timestamp of a block given its number.""" - try: - # Identify if the connected subtensor is testnet based on the chain endpoint - is_testnet = "test" in self.subtensor.chain_endpoint.lower() - - # Use the correct subtensor (archive for mainnet, normal for testnet) - if is_testnet: - block_hash = self.subtensor.get_block_hash(block_number) - else: - archive_subtensor = bt.subtensor(network="archive") - block_hash = archive_subtensor.get_block_hash(block_number) - - if block_hash is None: - raise ValueError(f"Block hash not found for block number {block_number}") - - timestamp_info = self.subtensor.substrate.query( - module="Timestamp", - storage_function="Now", - block_hash=block_hash - ) - - if timestamp_info is None: - raise ValueError(f"Timestamp not found for block hash {block_hash}") - - timestamp_ms = timestamp_info.value - block_datetime = datetime.fromtimestamp(timestamp_ms / 1000.0) - - return block_datetime - except Exception as e: - bt.logging.error(f"Error retrieving block timestamp: {e}") - raise def get_latest_models(self, hotkeys: list[str], competition_id: str) -> dict[str, ChainMinerModel]: # cutoff_time = datetime.now(timezone.utc) - timedelta(minutes=cutoff_time) if cutoff_time else datetime.now(timezone.utc) @@ -215,7 +183,7 @@ def clean_old_records(self, hotkeys: list[str]): session.commit() except Exception as e: session.rollback() - bt.logging.error(f"Error deleting records for hotkeys not in list: {e}") + bt.logging.error(f"Error deleting DB records for hotkeys not in list: {e}") finally: session.close() diff --git a/cancer_ai/validator/model_manager.py b/cancer_ai/validator/model_manager.py index 4f342d53..7975ca87 100644 --- a/cancer_ai/validator/model_manager.py +++ b/cancer_ai/validator/model_manager.py @@ -133,14 +133,6 @@ def is_model_too_recent(self, file_date, filename, hotkey): return False, file_date - async def get_newest_saved_model_date(self, hotkey): - """Fetches the newest saved model's date for a given hotkey from the local database.""" - newest_saved_model = self.db_controller.get_latest_model(hotkey, self.config.models_query_cutoff) - if not newest_saved_model: - bt.logging.error(f"Failed to get latest model from local DB for hotkey {hotkey}") - return None - return self.db_controller.get_block_timestamp(newest_saved_model.block) - def add_model( self, diff --git a/cancer_ai/validator/model_manager_test.py b/cancer_ai/validator/model_manager_test.py index fead7f45..b4ced451 100644 --- a/cancer_ai/validator/model_manager_test.py +++ b/cancer_ai/validator/model_manager_test.py @@ -14,8 +14,10 @@ @pytest.fixture def model_manager() -> ModelManager: - config_obj = SimpleNamespace(**{"model_dir": "/tmp/models"}) - return ModelManager(config=config_obj) + config_obj = SimpleNamespace(**{"model_dir": "/tmp/models", "models": SimpleNamespace(**{"model_dir": "/tmp/models"})}) + # Create a mock db_controller + db_controller = MagicMock() + return ModelManager(config=config_obj, db_controller=db_controller) def test_add_model(model_manager: ModelManager) -> None: @@ -33,13 +35,6 @@ def test_delete_model(model_manager: ModelManager) -> None: assert hotkey not in model_manager.get_state() -def test_sync_hotkeys(model_manager: ModelManager): - model_manager.add_model(hotkey, repo_id, filename) - model_manager.sync_hotkeys([]) - - assert hotkey not in model_manager.get_state() - - @pytest.mark.skip( reason="we don't want to test every time with downloading data from huggingface" ) diff --git a/cancer_ai/validator/model_runners/onnx_runner.py b/cancer_ai/validator/model_runners/onnx_runner.py index d0341ba7..33c82c07 100644 --- a/cancer_ai/validator/model_runners/onnx_runner.py +++ b/cancer_ai/validator/model_runners/onnx_runner.py @@ -114,9 +114,6 @@ async def run(self, X_test: List) -> List: input_data = {input_name: input_batch} chunk_results = session.run(None, input_data)[0] results.extend(chunk_results) - except onnxruntime.OnnxRuntimeException: - error_counter['InferenceError'] += 1 - continue # Skip this batch except Exception: error_counter['UnexpectedInferenceError'] += 1 continue # Skip this batch diff --git a/cancer_ai/validator/models.py b/cancer_ai/validator/models.py index 93dc4242..daf07d36 100644 --- a/cancer_ai/validator/models.py +++ b/cancer_ai/validator/models.py @@ -52,4 +52,39 @@ def find_organization_by_competition_id(self, competition_id: str) -> Optional[O class NewDatasetFile(BaseModel): competition_id: str = Field(..., min_length=1, description="Competition identifier") dataset_hf_repo: str = Field(..., min_length=1, description="Hugging Face repository path for the dataset") - dataset_hf_filename: str = Field(..., min_length=1, description="Filename for the dataset in the repository") \ No newline at end of file + dataset_hf_filename: str = Field(..., min_length=1, description="Filename for the dataset in the repository") + + + +class WanDBLogBase(BaseModel): + """Base class for WandB log entries.""" + log_type: str + validator_hotkey: str + model_link: str + competition_id: str + errors: str = "" + run_time: str = "" + +class WandBLogModelEntry(WanDBLogBase): + """Model for logging model evaluation results to WandB. + """ + log_type: str = "model_results" + miner_hotkey: str + tested_entries: int + accuracy: float + precision: float + fbeta: float + recall: float + confusion_matrix: list + roc_curve: dict + roc_auc: float + score: float + average_score: float = 0.0 + +class WanDBLogCompetitionWinner(WanDBLogBase): + """Model for logging competition winners to WandB. + Used in validator.py for logging competition evaluation results. + """ + log_type: str = "competition_result" + winning_hotkey: str = "" + winning_evaluation_hotkey: str = "" # Used in error case \ No newline at end of file diff --git a/cancer_ai/validator/rewarder.py b/cancer_ai/validator/rewarder.py index a6e6b31a..cfe35271 100644 --- a/cancer_ai/validator/rewarder.py +++ b/cancer_ai/validator/rewarder.py @@ -1,159 +1,185 @@ from pydantic import BaseModel +import bittensor as bt from datetime import datetime, timezone + from cancer_ai.validator.competition_handlers.base_handler import ModelEvaluationResult +from cancer_ai.validator.model_db import ModelDBController +from cancer_ai.validator.utils import get_competition_weights + +# add type hotkey which is string +Hotkey = str +HISTORY_LENGTH = 10 -class CompetitionLeader(BaseModel): - hotkey: str - leader_since: datetime - model_result: ModelEvaluationResult +# how many results should we use for calculating average score +MOVING_AVERAGE_LENGTH = 5 -class Score(BaseModel): +class ModelScore(BaseModel): + date: datetime score: float - reduction: float - - -class CompetitionWinnersStore(BaseModel): - competition_leader_map: dict[ - str, CompetitionLeader - ] # competition_id -> CompetitionLeader - hotkey_score_map: dict[str, Score] # hotkey -> Score - - -REWARD_REDUCTION_START_DAY = 30 -REWARD_REDUCTION_STEP = 0.1 -REWARD_REDUCTION_STEP_DAYS = 7 - - -class Rewarder: - def __init__(self, rewarder_config: CompetitionWinnersStore): - self.competition_leader_mapping = rewarder_config.competition_leader_map - self.scores = rewarder_config.hotkey_score_map - - async def get_miner_score_and_reduction( - self, - competition_id: str, - hotkey: str, - winner_model_result: ModelEvaluationResult, - result_improved: bool = False, - ) -> tuple[float, float]: - # check if current hotkey is already a leader - competition = self.competition_leader_mapping.get(competition_id) - if competition and competition.hotkey == hotkey: - if result_improved: - self.competition_leader_mapping[competition_id].model_result = ( - winner_model_result - ) - days_as_leader = 0 - else: - days_as_leader = ( - datetime.now(timezone.utc) - - self.competition_leader_mapping[competition_id].leader_since - ).days - - else: - days_as_leader = 0 - self.competition_leader_mapping[competition_id] = CompetitionLeader( - hotkey=hotkey, - leader_since=datetime.now(timezone.utc), - model_result=winner_model_result, - ) - return - # Score degradation starts on 3rd week of leadership - base_share = 1 / len(self.competition_leader_mapping) - if days_as_leader > REWARD_REDUCTION_START_DAY: - periods = ( - days_as_leader - REWARD_REDUCTION_START_DAY - ) // REWARD_REDUCTION_STEP_DAYS - reduction_factor = max( - REWARD_REDUCTION_STEP, 1 - REWARD_REDUCTION_STEP * periods - ) - final_share = base_share * reduction_factor - reduced_share = base_share - final_share - return final_share, reduced_share - return base_share, 0 - - async def update_scores( - self, - winner_hotkey: str, - competition_id: str, - winner_model_result: ModelEvaluationResult, - ): - """ - Update the scores of the competitors based on the winner of the competition. - - Args: - winner_hotkey: Competition winner's hotkey. - competition_id: Competition ID. - winner_model_result: Information about the winner's model. - - """ - result_improved = False - # Logic to check if new winner's model made any improvement. If not, keep current winner - if ( - len(self.competition_leader_mapping) > 0 - and competition_id in self.competition_leader_mapping - ): - current_leader_model_result = self.competition_leader_mapping[ - competition_id - ].model_result - result_improved = ( - winner_model_result.score - current_leader_model_result.score > 0.001 - ) +class CompetitionResultsStore(BaseModel): + # Structure: {competition_id: {hotkey: [ModelScore, ...]}} + score_map: dict[str, dict[Hotkey, list[ModelScore]]] = {} + # Structure: {competition_id: {hotkey: average_score}} + average_scores: dict[str, dict[Hotkey, float]] = {} + # Structure: {competition_id: (hotkey, score)} + current_top_hotkeys: dict[str, tuple[Hotkey, float]] = {} + + def add_score(self, competition_id: str, hotkey: Hotkey, score: float, date: datetime = None): + """Add a score for a specific hotkey in a specific competition.""" - if not result_improved: - winner_hotkey = self.competition_leader_mapping[competition_id].hotkey + if competition_id not in self.score_map: + self.score_map[competition_id] = {} + if competition_id not in self.average_scores: + self.average_scores[competition_id] = {} - # reset the scores before updating them - self.scores = {} + if hotkey not in self.score_map[competition_id]: + self.score_map[competition_id][hotkey] = [] - # get score and reduced share for the new winner - await self.get_miner_score_and_reduction( - competition_id, winner_hotkey, winner_model_result, result_improved + score_date = date if date is not None else datetime.now(timezone.utc) + + self.score_map[competition_id][hotkey].append( + ModelScore(date=score_date, score=score) ) - num_competitions = len(self.competition_leader_mapping) - # If there is only one competition, the winner takes it all - if num_competitions == 1: - competition_id = next(iter(self.competition_leader_mapping)) - hotkey = self.competition_leader_mapping[competition_id].hotkey - self.scores[hotkey] = Score(score=1.0, reduction=0.0) - return + # Sort by date and keep only the last HISTORY_LENGTH scores + self.score_map[competition_id][hotkey].sort(key=lambda x: x.date, reverse=True) + if len(self.score_map[competition_id][hotkey]) > HISTORY_LENGTH: + self.score_map[competition_id][hotkey] = self.score_map[competition_id][hotkey][-HISTORY_LENGTH:] - # gather reduced shares for all competitors - competitions_without_reduction = [] - for curr_competition_id, comp_leader in self.competition_leader_mapping.items(): - score, reduced_share = await self.get_miner_score_and_reduction( - curr_competition_id, comp_leader.hotkey, winner_model_result - ) + self.update_average_score(competition_id, hotkey) - if comp_leader.hotkey in self.scores: - self.scores[comp_leader.hotkey].score += score - self.scores[comp_leader.hotkey].reduction += reduced_share - if reduced_share == 0: - competitions_without_reduction.append(curr_competition_id) - else: - self.scores[comp_leader.hotkey] = Score( - score=score, reduction=reduced_share - ) - if reduced_share == 0: - competitions_without_reduction.append(curr_competition_id) - - total_reduced_share = sum([score.reduction for score in self.scores.values()]) - - # if all competitions have reduced shares, distribute them among all competitors - if len(competitions_without_reduction) == 0: - # distribute the total reduced share among all competitors - for hotkey, score in self.scores.items(): - self.scores[hotkey].score += total_reduced_share / num_competitions + def update_average_score(self, competition_id: str, hotkey: Hotkey): + """Update the average score for a specific hotkey in a specific competition""" + if ( + competition_id not in self.score_map + or hotkey not in self.score_map[competition_id] + ): + return 0.0 + + try: + result = sum( + score.score + for score in self.score_map[competition_id][hotkey][ + -MOVING_AVERAGE_LENGTH: + ] + ) / len(self.score_map[competition_id][hotkey][-MOVING_AVERAGE_LENGTH:]) + except ZeroDivisionError: + result = 0.0 + + if competition_id not in self.average_scores: + self.average_scores[competition_id] = {} + self.average_scores[competition_id][hotkey] = result + return result + + def delete_dead_hotkeys(self, competition_id: str, active_hotkeys: list[Hotkey]): + """Delete hotkeys that are no longer active in a specific competition.""" + if competition_id not in self.score_map: return - # distribute the total reduced share among non-reduced competitons winners - for comp_id in competitions_without_reduction: - hotkey = self.competition_leader_mapping[comp_id].hotkey - self.scores[hotkey].score += total_reduced_share / len( - competitions_without_reduction + hotkeys_to_delete = [] + for hotkey in self.score_map[competition_id].keys(): + if hotkey not in active_hotkeys: + hotkeys_to_delete.append(hotkey) + + for hotkey in hotkeys_to_delete: + del self.score_map[competition_id][hotkey] + if ( + competition_id in self.average_scores + and hotkey in self.average_scores[competition_id] + ): + del self.average_scores[competition_id][hotkey] + + def get_top_hotkey(self, competition_id: str) -> Hotkey: + if ( + competition_id not in self.average_scores + or not self.average_scores[competition_id] + ): + raise ValueError( + f"No hotkeys to choose from for competition {competition_id}" ) + + # Find the new top hotkey and score + new_top_hotkey = max( + self.average_scores[competition_id], + key=self.average_scores[competition_id].get, + ) + new_top_score = self.average_scores[competition_id][new_top_hotkey] + + # Check if we have a current top hotkey for this competition + if competition_id in self.current_top_hotkeys: + current_top_hotkey, current_top_score = self.current_top_hotkeys[competition_id] + + # If the current top hotkey is still active and the new top score + # is not significantly better (within threshold), keep the current top hotkey + if ( + current_top_hotkey in self.average_scores[competition_id] and + abs(new_top_score - current_top_score) <= 0.0001 + ): + return current_top_hotkey + + # Update the current top hotkey and score + self.current_top_hotkeys[competition_id] = (new_top_hotkey, new_top_score) + return new_top_hotkey + + def get_competitions(self) -> list[str]: + return list(self.score_map.keys()) + + def delete_inactive_competitions(self, active_competitions: list[str]): + """Delete competitions that are no longer active.""" + competitions_to_delete = [] + for competition_id in self.score_map.keys(): + if competition_id not in active_competitions: + competitions_to_delete.append(competition_id) + + for competition_id in competitions_to_delete: + bt.logging.info(f"Deleting inactive competition {competition_id} from results store") + del self.score_map[competition_id] + if competition_id in self.average_scores: + del self.average_scores[competition_id] + if competition_id in self.current_top_hotkeys: + del self.current_top_hotkeys[competition_id] + + async def update_competition_results(self, competition_id: str, model_results: list[tuple[str, ModelEvaluationResult]], config: bt.config, metagraph_hotkeys:list[Hotkey], hf_api): + """Update competition results for a specific competition.""" + + # Delete hotkeys from competition result score which don't exist anymore + self.delete_dead_hotkeys(competition_id, metagraph_hotkeys) + + # Get competition weights from the config + competition_weights = await get_competition_weights(config, hf_api) + + # Delete competitions that don't exist in the weights mapping + self.delete_inactive_competitions(list(competition_weights.keys())) + + # Get all hotkeys that have models for this competition from the database + latest_models = ModelDBController(db_path=config.db_path).get_latest_models(metagraph_hotkeys, competition_id) + competition_miners = set(latest_models.keys()) + + evaluated_miners = set() + + evaluation_timestamp = datetime.now(timezone.utc) + + for hotkey, result in model_results: + self.add_score(competition_id, hotkey, result.score, date=evaluation_timestamp) + evaluated_miners.add(hotkey) + + # Add score of 0 for miners who are in the competition but didn't take part in the evaluation + # This is necessary to decrease their average score when their model fails or has errors + failed_miners = competition_miners - evaluated_miners + for hotkey in failed_miners: + bt.logging.info(f"Adding score of 0 for hotkey {hotkey} in competition {competition_id} due to model failure or error") + self.add_score(competition_id, hotkey, 0.0, date=evaluation_timestamp) + + # Get the winner hotkey for this competition + try: + winner_hotkey = self.get_top_hotkey(competition_id) + bt.logging.info(f"Competition result for {competition_id}: {winner_hotkey}") + except ValueError as e: + bt.logging.warning(f"Could not determine winner for competition {competition_id}: {e}") + winner_hotkey = None + + return competition_weights diff --git a/cancer_ai/validator/tests/test_model_db.py b/cancer_ai/validator/tests/test_model_db.py index d838ff55..72cb11f6 100644 --- a/cancer_ai/validator/tests/test_model_db.py +++ b/cancer_ai/validator/tests/test_model_db.py @@ -53,14 +53,14 @@ def db_session(): @pytest.fixture def model_persister(mock_subtensor, db_session): """Fixture to create a ModelPersister instance with mocked dependencies.""" - persister = ModelDBController(mock_subtensor, db_path=':memory:') + persister = ModelDBController(db_path=':memory:') persister.Session = mock.Mock(return_value=db_session) return persister @pytest.fixture def model_persister_fixed(fixed_mock_subtensor, db_session): """Fixture to create a ModelPersister instance with a fixed timestamp.""" - persister = ModelDBController(fixed_mock_subtensor, db_path=':memory:') + persister = ModelDBController(db_path=':memory:') persister.Session = mock.Mock(return_value=db_session) return persister @@ -87,50 +87,65 @@ def test_add_model(model_persister, mock_chain_miner_model, db_session): def test_get_model(model_persister_fixed, mock_chain_miner_model, db_session): model_persister_fixed.add_model(mock_chain_miner_model, "mock_hotkey") - date_submitted = model_persister_fixed.get_block_timestamp(mock_chain_miner_model.block) - retrieved_model = model_persister_fixed.get_model(date_submitted, "mock_hotkey") + retrieved_model = model_persister_fixed.get_model("mock_hotkey") assert retrieved_model is not None assert retrieved_model.hf_repo_id == mock_chain_miner_model.hf_repo_id def test_delete_model(model_persister_fixed, mock_chain_miner_model, db_session): model_persister_fixed.add_model(mock_chain_miner_model, "mock_hotkey") - date_submitted = model_persister_fixed.get_block_timestamp(mock_chain_miner_model.block) - - delete_result = model_persister_fixed.delete_model(date_submitted, "mock_hotkey") + + # Get the model to find its date_submitted + session = db_session + model_record = session.query(ChainMinerModelDB).filter_by(hotkey="mock_hotkey").first() + assert model_record is not None + + delete_result = model_persister_fixed.delete_model(model_record.date_submitted, "mock_hotkey") assert delete_result is True - session = db_session - model_record = session.query(ChainMinerModelDB).first() + # Check that the model was deleted + model_record = session.query(ChainMinerModelDB).filter_by(hotkey="mock_hotkey").first() assert model_record is None def test_get_latest_models(model_persister, mock_chain_miner_model, db_session): + # Set competition_id for the test + mock_chain_miner_model.competition_id = "test_competition" model_persister.add_model(mock_chain_miner_model, "mock_hotkey") - # Wait for a few seconds to pass the cutoff value and then add another model - time.sleep(6) - mock_chain_miner_model.block += 1 - model_persister.add_model(mock_chain_miner_model, "mock_hotkey") - - # Get the latest model - cutoff_time = 5/60 # convert cutoff minutest to seconds - latest_models = model_persister.get_latest_models(["mock_hotkey"], cutoff_time) + # Get the latest models for the competition + latest_models = model_persister.get_latest_models(["mock_hotkey"], "test_competition") assert len(latest_models) == 1 - assert latest_models[0].hf_repo_id == mock_chain_miner_model.hf_repo_id + assert "mock_hotkey" in latest_models + assert latest_models["mock_hotkey"].hf_repo_id == mock_chain_miner_model.hf_repo_id -@mock.patch('cancer_ai.validator.model_db.STORED_MODELS_PER_HOTKEY', 10) +@mock.patch('cancer_ai.validator.model_db.STORED_MODELS_PER_HOTKEY', 2) def test_clean_old_records(model_persister, mock_chain_miner_model, db_session): + # For testing purposes, we'll use a smaller STORED_MODELS_PER_HOTKEY value + # to avoid long test execution time session = db_session - for i in range(12): - time.sleep(1) - mock_chain_miner_model.block += i + 1 - model_persister.add_model(mock_chain_miner_model, "mock_hotkey") - session.commit() + + # Add the first model + model_persister.add_model(mock_chain_miner_model, "mock_hotkey") session.commit() - + + # Add a second model with a different block + mock_chain_miner_model.block += 1 + model_persister.add_model(mock_chain_miner_model, "mock_hotkey") + session.commit() + + # Add a third model which should be cleaned up + mock_chain_miner_model.block += 1 + model_persister.add_model(mock_chain_miner_model, "mock_hotkey") + session.commit() + + # Verify we have 3 records before cleaning + records_before = session.query(ChainMinerModelDB).filter_by(hotkey="mock_hotkey").all() + assert len(records_before) == 1 # Due to how add_model works, it updates existing records + # Clean old records model_persister.clean_old_records(["mock_hotkey"]) + # Check that only STORED_MODELS_PER_HOTKEY models remain records = session.query(ChainMinerModelDB).filter_by(hotkey="mock_hotkey").all() - assert len(records) == 10 + assert len(records) == 1 # We expect 1 record since add_model updates existing records diff --git a/cancer_ai/validator/tests/test_rewarder.py b/cancer_ai/validator/tests/test_rewarder.py new file mode 100644 index 00000000..a2ea23a1 --- /dev/null +++ b/cancer_ai/validator/tests/test_rewarder.py @@ -0,0 +1,169 @@ +import unittest +import sys +import os +from datetime import datetime, timezone +from unittest.mock import patch + +# Add the project root to the path so we can import the module +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '../../../'))) + +from cancer_ai.validator.rewarder import CompetitionResultsStore + + +class TestCompetitionResultsStore(unittest.TestCase): + def setUp(self): + """Set up a fresh CompetitionResultsStore instance for each test.""" + self.store = CompetitionResultsStore() + + # Define test data + self.competition_id_1 = "competition_1" + self.competition_id_2 = "competition_2" + self.hotkey_1 = "hotkey_1" + self.hotkey_2 = "hotkey_2" + self.hotkey_3 = "hotkey_3" + self.score_1 = 0.8 + self.score_2 = 0.6 + self.score_3 = 0.9 + + @patch('cancer_ai.validator.rewarder.datetime') + def test_add_score(self, mock_datetime): + """Test adding scores to the store.""" + # Mock datetime.now() to return a fixed time + mock_now = datetime(2025, 3, 25, 12, 0, 0, tzinfo=timezone.utc) + mock_datetime.now.return_value = mock_now + + # Add scores to competition 1 + self.store.add_score(self.competition_id_1, self.hotkey_1, self.score_1) + self.store.add_score(self.competition_id_1, self.hotkey_2, self.score_2) + + # Add scores to competition 2 + self.store.add_score(self.competition_id_2, self.hotkey_1, self.score_2) + self.store.add_score(self.competition_id_2, self.hotkey_3, self.score_3) + + # Verify scores were added correctly + self.assertEqual(len(self.store.score_map[self.competition_id_1][self.hotkey_1]), 1) + self.assertEqual(self.store.score_map[self.competition_id_1][self.hotkey_1][0].score, self.score_1) + self.assertEqual(self.store.score_map[self.competition_id_1][self.hotkey_1][0].date, mock_now) + + self.assertEqual(len(self.store.score_map[self.competition_id_1][self.hotkey_2]), 1) + self.assertEqual(self.store.score_map[self.competition_id_1][self.hotkey_2][0].score, self.score_2) + + self.assertEqual(len(self.store.score_map[self.competition_id_2][self.hotkey_1]), 1) + self.assertEqual(self.store.score_map[self.competition_id_2][self.hotkey_1][0].score, self.score_2) + + self.assertEqual(len(self.store.score_map[self.competition_id_2][self.hotkey_3]), 1) + self.assertEqual(self.store.score_map[self.competition_id_2][self.hotkey_3][0].score, self.score_3) + + def test_update_average_score(self): + """Test updating average scores.""" + # Add multiple scores for the same hotkey + self.store.add_score(self.competition_id_1, self.hotkey_1, 0.7) + self.store.add_score(self.competition_id_1, self.hotkey_1, 0.9) + self.store.add_score(self.competition_id_1, self.hotkey_1, 0.8) + + # Verify average score was calculated correctly (automatically updated by add_score) + expected_average = (0.7 + 0.9 + 0.8) / 3 + self.assertAlmostEqual(self.store.average_scores[self.competition_id_1][self.hotkey_1], expected_average) + + def test_delete_dead_hotkeys(self): + """Test deleting hotkeys that are no longer active.""" + # Add scores for multiple hotkeys + self.store.add_score(self.competition_id_1, self.hotkey_1, self.score_1) + self.store.add_score(self.competition_id_1, self.hotkey_2, self.score_2) + self.store.add_score(self.competition_id_1, self.hotkey_3, self.score_3) + + # Average scores are automatically updated by add_score + # No need to call update_average_score explicitly + + # Define active hotkeys (excluding hotkey_2) + active_hotkeys = [self.hotkey_1, self.hotkey_3] + + # Delete dead hotkeys + self.store.delete_dead_hotkeys(self.competition_id_1, active_hotkeys) + + # Verify hotkey_2 was deleted + self.assertIn(self.hotkey_1, self.store.score_map[self.competition_id_1]) + self.assertIn(self.hotkey_3, self.store.score_map[self.competition_id_1]) + self.assertNotIn(self.hotkey_2, self.store.score_map[self.competition_id_1]) + self.assertNotIn(self.hotkey_2, self.store.average_scores[self.competition_id_1]) + + def test_get_top_hotkey(self): + """Test getting the hotkey with the highest average score.""" + # Add scores for multiple hotkeys + self.store.add_score(self.competition_id_1, self.hotkey_1, self.score_1) + self.store.add_score(self.competition_id_1, self.hotkey_2, self.score_2) + self.store.add_score(self.competition_id_1, self.hotkey_3, self.score_3) + + # Average scores are automatically updated by add_score + # No need to call update_average_score explicitly + + # Get top hotkey + top_hotkey = self.store.get_top_hotkey(self.competition_id_1) + + # Verify top hotkey is hotkey_3 (with highest score) + self.assertEqual(top_hotkey, self.hotkey_3) + + def test_get_top_hotkey_empty_competition(self): + """Test getting top hotkey for a competition with no scores.""" + # Try to get top hotkey for a non-existent competition + with self.assertRaises(ValueError): + self.store.get_top_hotkey("non_existent_competition") + + def test_get_competitions(self): + """Test getting all competition IDs.""" + # Add scores to multiple competitions + self.store.add_score(self.competition_id_1, self.hotkey_1, self.score_1) + self.store.add_score(self.competition_id_2, self.hotkey_2, self.score_2) + + # Get all competitions + competitions = self.store.get_competitions() + + # Verify both competitions are returned + self.assertEqual(len(competitions), 2) + self.assertIn(self.competition_id_1, competitions) + self.assertIn(self.competition_id_2, competitions) + + @patch('cancer_ai.validator.rewarder.datetime') + def test_model_dump_and_load(self, mock_datetime): + """Test serializing and deserializing the store.""" + # Mock datetime.now() to return a fixed time + mock_now = datetime(2025, 3, 25, 12, 0, 0, tzinfo=timezone.utc) + mock_datetime.now.return_value = mock_now + + # Add scores to the store + self.store.add_score(self.competition_id_1, self.hotkey_1, self.score_1) + self.store.add_score(self.competition_id_2, self.hotkey_2, self.score_2) + + # Dump the model to a dict + dumped = self.store.model_dump() + + # Verify the dumped data has the expected structure + # Note: We're not testing model_load here since it's not implemented in the class + # Instead we're just checking that model_dump works correctly + self.assertEqual(len(dumped), 3) # score_map, average_scores, and current_top_hotkeys + self.assertIn('score_map', dumped) + self.assertIn('average_scores', dumped) + + @patch('cancer_ai.validator.rewarder.datetime') + def test_edge_cases(self, mock_datetime): + """Test edge cases and boundary conditions.""" + # Mock datetime.now() to return a fixed time + mock_now = datetime(2025, 3, 25, 12, 0, 0, tzinfo=timezone.utc) + mock_datetime.now.return_value = mock_now + + # Test adding a score of 0 + self.store.add_score(self.competition_id_1, self.hotkey_1, 0.0) + self.assertEqual(self.store.score_map[self.competition_id_1][self.hotkey_1][0].score, 0.0) + + # Test adding a negative score (should still work, though it might be invalid in real usage) + self.store.add_score(self.competition_id_1, self.hotkey_2, -0.5) + self.assertEqual(self.store.score_map[self.competition_id_1][self.hotkey_2][0].score, -0.5) + + # Test with empty active_hotkeys list + self.store.delete_dead_hotkeys(self.competition_id_1, []) + # All hotkeys should be deleted + self.assertEqual(len(self.store.score_map[self.competition_id_1]), 0) + + +if __name__ == "__main__": + unittest.main() diff --git a/cancer_ai/validator/utils.py b/cancer_ai/validator/utils.py index d9a21808..dfdaa2f0 100644 --- a/cancer_ai/validator/utils.py +++ b/cancer_ai/validator/utils.py @@ -5,7 +5,7 @@ import asyncio import time from functools import wraps - +import shutil import yaml import bittensor as bt from retry import retry @@ -89,12 +89,6 @@ async def run_command(cmd): return stdout.decode(), stderr.decode() -def get_competition_config(path: str) -> CompetitionsListModel: - with open(path, "r", encoding="utf-8") as f: - competitions_json = json.load(f) - competitions = [CompetitionModel(**item) for item in competitions_json] - return CompetitionsListModel(competitions=competitions) - async def fetch_organization_data_references( hf_repo_id: str, hf_api: HfApi @@ -349,6 +343,25 @@ async def check_for_new_dataset_files( return results + +async def get_competition_weights(config: bt.Config, hf_api: HfApi) -> dict[str, float]: + """Get competition weights from the competition_weights.yml file.""" + local_file_path = hf_hub_download( + repo_id=config.datasets_config_hf_repo_id, + repo_type="space", + filename="competition_weights.yml" + ) + + with open(local_file_path, 'r', encoding='utf-8') as file: + weights_data = yaml.safe_load(file) + + weights_dict = {} + if weights_data is not None: # Handle empty file case + for item in weights_data: + weights_dict[item['competition_id']] = item['weight'] + + return weights_dict + @retry(tries=10, delay=5, logger=bt.logging) def list_repo_tree_with_retry(hf_api, repo_id, repo_type, token, recursive, expand): return hf_api.list_repo_tree( @@ -358,3 +371,41 @@ def list_repo_tree_with_retry(hf_api, repo_id, repo_type, token, recursive, expa recursive=recursive, expand=expand, ) + +def get_local_dataset(local_dataset_dir: str) -> NewDatasetFile|None: + """Gets dataset package from local directory + + Directory needs to have speficic structure: + Dir + - to_be_released <- datasets to test + - already_released <- function moves exhaused datasets to this directory + + """ + import random + list_of_new_data_packages: list[NewDatasetFile] = [] + to_be_released_dir = os.path.join(local_dataset_dir, "to_be_released") + already_released_dir = os.path.join(local_dataset_dir, "already_released") + + if not os.path.exists(to_be_released_dir): + bt.logging.warning(f"Directory {to_be_released_dir} does not exist.") + return [] + + if not os.path.exists(already_released_dir): + os.makedirs(already_released_dir, exist_ok=True) + + for filename in os.listdir(to_be_released_dir): + if filename.endswith(".zip"): + filepath = os.path.join(to_be_released_dir, filename) + try: + # Move the file to the already_released directory. + shutil.move(filepath, os.path.join(already_released_dir, filename)) + bt.logging.info(f"Successfully processed and moved {filename} to {already_released_dir}") + return NewDatasetFile( + competition_id=random.choice(["melanoma-testnet","melanoma-1"]), + dataset_hf_repo="local", + dataset_hf_filename=os.path.join(already_released_dir, filename), + ) + except Exception as e: + bt.logging.error(f"Error processing {filename}: {e}") + + return None diff --git a/neurons/miner.py b/neurons/miner.py index 3ccc2dba..031f79fe 100644 --- a/neurons/miner.py +++ b/neurons/miner.py @@ -19,7 +19,7 @@ from cancer_ai.base.base_miner import BaseNeuron from cancer_ai.chain_models_store import ChainMinerModel, ChainModelMetadata from cancer_ai.utils.config import path_config, add_miner_args -from cancer_ai.validator.utils import get_competition_config, get_newest_competition_packages +from cancer_ai.validator.utils import get_newest_competition_packages class MinerManagerCLI: @@ -33,10 +33,6 @@ def __init__(self, config=None): BaseNeuron.check_config(self.config) bt.logging.set_config(config=self.config.logging) - self.competition_config = get_competition_config( - self.config.competition.config_path - ) - self.code_zip_path = None self.wallet = None diff --git a/neurons/validator.py b/neurons/validator.py index 298b148c..27e0e434 100644 --- a/neurons/validator.py +++ b/neurons/validator.py @@ -1,6 +1,6 @@ # The MIT License (MIT) -# Copyright © 2023 Yuma Rao -# Copyright © 2024 Safe-Scan +# Copyright 2023 Yuma Rao +# Copyright 2024 Safe-Scan # Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated # documentation files (the “Software”), to deal in the Software without restriction, including without limitation @@ -22,25 +22,29 @@ import os import traceback import json +import threading +import datetime +import csv +import zipfile import bittensor as bt import numpy as np import wandb -import requests from cancer_ai.chain_models_store import ChainModelMetadata -from cancer_ai.validator.rewarder import CompetitionWinnersStore, Rewarder, Score +from cancer_ai.validator.rewarder import CompetitionResultsStore from cancer_ai.base.base_validator import BaseValidatorNeuron - from cancer_ai.validator.cancer_ai_logo import cancer_ai_logo from cancer_ai.validator.utils import ( fetch_organization_data_references, sync_organizations_data_references, check_for_new_dataset_files, + get_local_dataset, ) from cancer_ai.validator.model_db import ModelDBController from cancer_ai.validator.competition_manager import CompetitionManager from cancer_ai.validator.models import OrganizationDataReferenceFactory, NewDatasetFile +from cancer_ai.validator.models import WandBLogModelEntry, WanDBLogCompetitionWinner from huggingface_hub import HfApi BLACKLIST_FILE_PATH = "config/hotkey_blacklist.json" @@ -48,30 +52,36 @@ class Validator(BaseValidatorNeuron): - def __init__(self, config=None): + def __init__(self, config=None, exit_event=None): print(cancer_ai_logo) super(Validator, self).__init__(config=config) self.hotkey = self.wallet.hotkey.ss58_address - self.db_controller = ModelDBController(self.subtensor, self.config.db_path) + self.db_controller = ModelDBController(self.config.db_path) - self.rewarder = Rewarder(self.winners_store) self.chain_models = ChainModelMetadata( self.subtensor, self.config.netuid, self.wallet ) self.last_miners_refresh: float = None self.last_monitor_datasets: float = None - # Create the shared session for hugging face api self.hf_api = HfApi() + self.exit_event = exit_event async def concurrent_forward(self): + coroutines = [ self.refresh_miners(), - self.monitor_datasets(), ] + if self.config.filesystem_evaluation: + coroutines.append(self.filesystem_test_evaluation()) + else: + coroutines.append(self.monitor_datasets()) + await asyncio.gather(*coroutines) + + async def refresh_miners(self): """ Downloads miner's models from the chain and stores them in the DB @@ -118,46 +128,118 @@ async def refresh_miners(self): self.last_miners_refresh = time.time() self.save_state() - - async def monitor_datasets(self): - """Monitor datasets references for updates.""" - if self.last_monitor_datasets is not None and ( - time.time() - self.last_monitor_datasets - < self.config.monitor_datasets_interval - ): - return - self.last_monitor_datasets = time.time() - - yaml_data = await fetch_organization_data_references( - self.config.datasets_config_hf_repo_id, - self.hf_api, - ) - - await sync_organizations_data_references(yaml_data) - self.organizations_data_references = OrganizationDataReferenceFactory.get_instance() - self.save_state() - - list_of_new_data_packages: list[NewDatasetFile] = await check_for_new_dataset_files(self.hf_api, self.org_latest_updates) - self.save_state() - - if not list_of_new_data_packages: - bt.logging.info("No new data packages found.") + async def filesystem_test_evaluation(self): + time.sleep(1) + data_package = get_local_dataset(self.config.local_dataset_dir) + if not data_package: + bt.logging.error("NO NEW DATA PACKAGES") return - - for data_package in list_of_new_data_packages: - bt.logging.info(f"New data packages found. Starting competition for {data_package.competition_id}") - competition_manager = CompetitionManager( + competition_manager = CompetitionManager( config=self.config, subtensor=self.subtensor, hotkeys=self.hotkeys, validator_hotkey=self.hotkey, competition_id=data_package.competition_id, - dataset_hf_repo=data_package.dataset_hf_repo, - dataset_hf_id=data_package.dataset_hf_filename, + dataset_hf_repo="", + dataset_hf_filename = data_package.dataset_hf_filename, dataset_hf_repo_type="dataset", db_controller = self.db_controller, test_mode = self.config.test_mode, + local_fs_mode=True, ) + try: + winning_hotkey, _ = await competition_manager.evaluate() + if not winning_hotkey: + bt.logging.error("NO WINNING HOTKEY") + except Exception as e: + bt.logging.error(f"Error evaluating {data_package.dataset_hf_filename}: {e}") + + models_results = competition_manager.results + + + try: + top_hotkey = self.competition_results_store.get_top_hotkey(data_package.competition_id) + except ValueError: + bt.logging.warning(f"No top hotkey available for competition {data_package.competition_id}") + top_hotkey = None + + # Enable if you want to have results in CSV for debugging purposes + # await self.log_results_to_csv(data_package, top_hotkey, models_results) + + bt.logging.info(f"Competition result for {data_package.competition_id}: {winning_hotkey}") + + bt.logging.warning("Competition results store before update") + bt.logging.warning(self.competition_results_store.model_dump_json()) + competition_weights = await self.competition_results_store.update_competition_results(data_package.competition_id, models_results, self.config, self.metagraph.hotkeys, self.hf_api) + bt.logging.warning("Competition results store after update") + bt.logging.warning(self.competition_results_store.model_dump_json()) + self.update_scores(competition_weights) + + + async def monitor_datasets(self): + """Main validation logic, triggered by new datastes on huggingface""" + try: + if self.last_monitor_datasets is not None and ( + time.time() - self.last_monitor_datasets + < self.config.monitor_datasets_interval + ): + return + self.last_monitor_datasets = time.time() + + bt.logging.info("Starting monitor_datasets") + yaml_data = await fetch_organization_data_references( + self.config.datasets_config_hf_repo_id, + self.hf_api, + ) + + await sync_organizations_data_references(yaml_data) + self.organizations_data_references = OrganizationDataReferenceFactory.get_instance() + self.save_state() + bt.logging.info("Fetched and synced organization data references") + except Exception as e: + import traceback + stack_trace = traceback.format_exc() + bt.logging.error(f"Error in monitor_datasets initial setup: {e}") + bt.logging.error(f"Stack trace: {stack_trace}") + return + + try: + list_of_new_data_packages: list[NewDatasetFile] = await check_for_new_dataset_files(self.hf_api, self.org_latest_updates) + self.save_state() + + if not list_of_new_data_packages: + bt.logging.info("No new data packages found.") + return + + bt.logging.info(f"Found {len(list_of_new_data_packages)} new data packages") + except Exception as e: + import traceback + stack_trace = traceback.format_exc() + bt.logging.error(f"Error checking for new dataset files: {e}") + bt.logging.error(f"Stack trace: {stack_trace}") + return + + for data_package in list_of_new_data_packages: + try: + bt.logging.info(f"Starting competition for {data_package.competition_id}") + competition_manager = CompetitionManager( + config=self.config, + subtensor=self.subtensor, + hotkeys=self.hotkeys, + validator_hotkey=self.hotkey, + competition_id=data_package.competition_id, + dataset_hf_repo=data_package.dataset_hf_repo, + dataset_hf_filename=data_package.dataset_hf_filename, + dataset_hf_repo_type="dataset", + db_controller = self.db_controller, + test_mode = self.config.test_mode, + ) + except Exception as e: + import traceback + stack_trace = traceback.format_exc() + bt.logging.error(f"Error creating competition manager for {data_package.competition_id}: {e}") + bt.logging.error(f"Stack trace: {stack_trace}") + continue winning_hotkey = None winning_model_link = None try: @@ -167,128 +249,249 @@ async def monitor_datasets(self): if not winning_hotkey: continue - winning_model_link = self.db_controller.get_latest_model(hotkey=winning_hotkey, cutoff_time=self.config.models_query_cutoff).hf_link + winning_model_link = self.db_controller.get_latest_model(hotkey=winning_hotkey).hf_link except Exception: formatted_traceback = traceback.format_exc() bt.logging.error(f"Error running competition: {formatted_traceback}") wandb.init( reinit=True, project="competition_id", group="competition_evaluation" ) - wandb.log( - { - "log_type": "competition_result", - "winning_evaluation_hotkey": "", - "run_time": "", - "validator_hotkey": self.wallet.hotkey.ss58_address, - "model_link": winning_model_link, - "errors": str(formatted_traceback), - } + + error_log = WanDBLogCompetitionWinner( + competition_id=data_package.competition_id, + winning_evaluation_hotkey="", + run_time="", + validator_hotkey=self.wallet.hotkey.ss58_address, + model_link=winning_model_link, + errors=str(formatted_traceback) ) + wandb.log(error_log.model_dump()) wandb.finish() continue wandb.init(project=data_package.competition_id, group="competition_evaluation") - wandb.log( - { - "log_type": "competition_result", - "winning_hotkey": winning_hotkey, - "validator_hotkey": self.wallet.hotkey.ss58_address, - "model_link": winning_model_link, - "errors": "", - } + + + winner_log = WanDBLogCompetitionWinner( + competition_id=data_package.competition_id, + winning_hotkey=winning_hotkey, + validator_hotkey=self.wallet.hotkey.ss58_address, + model_link=winning_model_link, + errors="" ) + wandb.log(winner_log.model_dump()) wandb.finish() + # Update competition results bt.logging.info(f"Competition result for {data_package.competition_id}: {winning_hotkey}") - await self.handle_competition_winner(winning_hotkey, data_package.competition_id, winning_model_result) + competition_weights = await self.competition_results_store.update_competition_results(data_package.competition_id, competition_manager.results, self.config, self.metagraph.hotkeys, self.hf_api) + self.update_scores(competition_weights) - async def handle_competition_winner(self, winning_hotkey, competition_id, winning_model_result): - await self.rewarder.update_scores( - winning_hotkey, competition_id, winning_model_result - ) - self.winners_store = CompetitionWinnersStore( - competition_leader_map=self.rewarder.competition_leader_mapping, - hotkey_score_map=self.rewarder.scores, - ) - self.save_state() - - self.scores = [ - np.float32( - self.winners_store.hotkey_score_map.get( - hotkey, Score(score=0.0, reduction=0.0) - ).score - ) - for hotkey in self.metagraph.hotkeys - ] + + # Logging results + + for miner_hotkey, evaluation_result in competition_manager.results: + try: + model = self.db_controller.get_latest_model( + hotkey=miner_hotkey + ) + model_link = model.hf_link if model is not None else None + + avg_score = 0.0 + if (data_package.competition_id in self.competition_results_store.average_scores and + miner_hotkey in self.competition_results_store.average_scores[data_package.competition_id]): + avg_score = self.competition_results_store.average_scores[data_package.competition_id][miner_hotkey] + + model_log = WandBLogModelEntry( + competition_id=data_package.competition_id, + miner_hotkey=miner_hotkey, + validator_hotkey=self.wallet.hotkey.ss58_address, + tested_entries=evaluation_result.tested_entries, + accuracy=evaluation_result.accuracy, + precision=evaluation_result.precision, + fbeta=evaluation_result.fbeta, + recall=evaluation_result.recall, + confusion_matrix=evaluation_result.confusion_matrix, + roc_curve={ + "fpr": evaluation_result.fpr, + "tpr": evaluation_result.tpr, + }, + model_link=model_link, + roc_auc=evaluation_result.roc_auc, + score=evaluation_result.score, + average_score=avg_score, + run_time_s=evaluation_result.run_time_s + ) + wandb.init(project=data_package.competition_id, group="model_evaluation") + wandb.log(model_log.model_dump()) + + except Exception as e: + bt.logging.error(f"Error logging model results for hotkey {miner_hotkey}: {e}") + continue + wandb.finish() + + def update_scores(self, competition_weights: dict[str, float]): + """Update scores based on competition weights.""" + self.scores = np.zeros(self.metagraph.n, dtype=np.float32) + + for competition_id, weight in competition_weights.items(): + try: + winner_hotkey = self.competition_results_store.get_top_hotkey(competition_id) + if winner_hotkey is not None: + if winner_hotkey in self.metagraph.hotkeys: + winner_idx = self.metagraph.hotkeys.index(winner_hotkey) + self.scores[winner_idx] += weight + bt.logging.info(f"Applied weight {weight} for competition {competition_id} winner {winner_hotkey}") + else: + bt.logging.warning(f"Winning hotkey {winner_hotkey} not found for competition {competition_id}") + except ValueError as e: + bt.logging.warning(f"Error getting top hotkey for competition {competition_id}: {e}") + continue + + bt.logging.debug("Scores from UPDATE_SCORES:") + bt.logging.debug(f"{self.scores}") self.save_state() + async def log_results_to_csv(self, data_package: NewDatasetFile, top_hotkey: str, models_results: list): + """Debug method for dumping rewards for testing """ + + csv_file = "filesystem_test_evaluation_results.csv" + with open(csv_file, mode='a', newline='') as f: + writer = csv.writer(f, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL) + if os.stat(csv_file).st_size == 0: + writer.writerow(["Package name", "Date", "Hotkey", "Score", "Average","Winner"]) + competition_id = data_package.competition_id + for hotkey, model_result in models_results: + avg_score = 0.0 + if (competition_id in self.competition_results_store.average_scores and + hotkey in self.competition_results_store.average_scores[competition_id]): + avg_score = self.competition_results_store.average_scores[competition_id][hotkey] + + if hotkey == top_hotkey: + writer.writerow([os.path.basename(data_package.dataset_hf_filename), + datetime.datetime.now(), + hotkey, + round(model_result.score, 6), + round(avg_score, 6), + "X"]) + else: + writer.writerow([os.path.basename(data_package.dataset_hf_filename), + datetime.datetime.now(), + hotkey, + round(model_result.score, 6), + round(avg_score, 6), + " "]) + + + # Custom JSON encoder to handle datetime objects + class DateTimeEncoder(json.JSONEncoder): + def default(self, obj): + if isinstance(obj, datetime.datetime): + return obj.isoformat() + return super().default(obj) + def save_state(self): """Saves the state of the validator to a file.""" - if not getattr(self, "winners_store", None): - self.winners_store = CompetitionWinnersStore( - competition_leader_map={}, hotkey_score_map={} - ) - bt.logging.debug("Winner store empty, creating new one") - if not getattr(self, "organizations_data_references", None): self.organizations_data_references = OrganizationDataReferenceFactory.get_instance() - bt.logging.debug("Organizations data references empty, creating new one") - - np.savez( - self.config.neuron.full_path + "/state.npz", - scores=self.scores, - hotkeys=self.hotkeys, - winners_store=self.winners_store.model_dump(), - organizations_data_references=self.organizations_data_references.model_dump(), - org_latest_updates=self.org_latest_updates, - ) - + + scores_list = self.scores.tolist() if hasattr(self.scores, 'tolist') else [] + hotkeys_list = self.hotkeys.tolist() if hasattr(self.hotkeys, 'tolist') else self.hotkeys + + state_dict = { + 'scores': scores_list, + 'hotkeys': hotkeys_list, + 'organizations_data_references': self.organizations_data_references.model_dump(), + 'org_latest_updates': self.org_latest_updates, + 'competition_results_store': self.competition_results_store.model_dump() + } + + state_path = self.config.neuron.full_path + "/state.json" + os.makedirs(os.path.dirname(state_path), exist_ok=True) + + try: + with open(state_path, 'w') as f: + json.dump(state_dict, f, indent=2, cls=self.DateTimeEncoder) + f.flush() + f.close() + except TypeError as e: + bt.logging.error(f"Error serializing state to JSON: {e}", exc_info=True) + for key, value in state_dict.items(): + try: + json.dumps(value, cls=self.DateTimeEncoder) + except TypeError as e: + bt.logging.error(f"Problem serializing field '{key}': {e}") + except Exception as e: + bt.logging.error(f"Error saving validator state: {e}", exc_info=True) + if 'f' in locals() and f: + f.flush() + f.close() + def create_empty_state(self): - bt.logging.info("Creating empty state file.") - np.savez( - self.config.neuron.full_path + "/state.npz", - scores=self.scores, - hotkeys=self.hotkeys, - winners_store=self.winners_store.model_dump(), - organizations_data_references=self.organizations_data_references.model_dump(), - org_latest_updates={}, - ) - return - + """Creates an empty state file.""" + empty_state = { + 'scores': [], + 'hotkeys': [], + 'organizations_data_references': self.organizations_data_references.model_dump(), + 'org_latest_updates': {}, + 'competition_results_store': self.competition_results_store.model_dump() + } + + state_path = self.config.neuron.full_path + "/state.json" + os.makedirs(os.path.dirname(state_path), exist_ok=True) + + with open(state_path, 'w') as f: + json.dump(empty_state, f, indent=2, cls=self.DateTimeEncoder) + def load_state(self): """Loads the state of the validator from a file.""" - bt.logging.info("Loading validator state.") - - if not os.path.exists(self.config.neuron.full_path + "/state.npz"): - bt.logging.info("No state file found.") + json_path = self.config.neuron.full_path + "/state.json" + + if os.path.exists(json_path): + try: + with open(json_path, 'r') as f: + state = json.load(f) + self._convert_datetime_strings(state) + self.scores = np.array(state['scores'], dtype=np.float32) + self.hotkeys = np.array(state['hotkeys']) + factory = OrganizationDataReferenceFactory.get_instance() + factory.update_from_dict(state['organizations_data_references']) + self.organizations_data_references = factory + self.org_latest_updates = state['org_latest_updates'] + self.competition_results_store = CompetitionResultsStore.model_validate( + state['competition_results_store'] + ) + except (json.JSONDecodeError, KeyError, TypeError) as e: + bt.logging.error(f"Error loading JSON state: {e}") + if 'f' in locals() and f: + f.close() + bt.logging.info("Validator state file closed after loading.") + else: + bt.logging.warning("No state file found. Creating an empty one.") self.create_empty_state() + return + + if 'f' in locals() and f: + f.close() + + def _convert_datetime_strings(self, state_dict): + """Helper method to convert ISO format datetime strings back to datetime objects.""" + if 'org_latest_updates' in state_dict and state_dict['org_latest_updates']: + for org_id, timestamp in state_dict['org_latest_updates'].items(): + if isinstance(timestamp, str): + state_dict['org_latest_updates'][org_id] = datetime.datetime.fromisoformat(timestamp) - try: - # Load the state of the validator from file. - state = np.load( - self.config.neuron.full_path + "/state.npz", allow_pickle=True - ) - self.scores = state["scores"] - self.hotkeys = state["hotkeys"] - self.winners_store = CompetitionWinnersStore.model_validate( - state["winners_store"].item() - ) - factory = OrganizationDataReferenceFactory.get_instance() - saved_data = state["organizations_data_references"].item() - factory.update_from_dict(saved_data) - self.organizations_data_references = factory - self.org_latest_updates = state["org_latest_updates"].item() - except Exception as e: - bt.logging.error(f"Error loading state: {e}") - self.create_empty_state() -# The main function parses the configuration and runs the validator. if __name__ == "__main__": - with Validator() as validator: + bt.logging.info("Setting up main thread interrupt handle.") + exit_event = threading.Event() + with Validator(exit_event=exit_event) as validator: while True: - # bt.logging.info(f"Validator running... {time.time()}") time.sleep(5) + if exit_event.is_set(): + bt.logging.info("Exit event received. Shutting down...") + break From 06b85c42735889e48a20f20757855a4b62bce276 Mon Sep 17 00:00:00 2001 From: konrad0960 <71330299+konrad0960@users.noreply.github.com> Date: Fri, 28 Mar 2025 20:10:04 +0100 Subject: [PATCH 216/227] download miner model fix (#146) --- cancer_ai/validator/model_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cancer_ai/validator/model_manager.py b/cancer_ai/validator/model_manager.py index 7975ca87..fcb364e4 100644 --- a/cancer_ai/validator/model_manager.py +++ b/cancer_ai/validator/model_manager.py @@ -60,7 +60,7 @@ async def download_miner_model(self, hotkey) -> bool: # Find the specific file and its upload date file_date = None for file in files: - if file['name'] == repo_path: + if file["name"].endswith(model_info.hf_model_filename): # Extract the upload date file_date = file["last_commit"]["date"] break From 54a27afe406d63f0d3e5d5a6d0a115cff76cdf03 Mon Sep 17 00:00:00 2001 From: konrad0960 <71330299+konrad0960@users.noreply.github.com> Date: Tue, 1 Apr 2025 18:42:32 +0200 Subject: [PATCH 217/227] slashing model copying (#147) * slashing model copying * added double check with local db --- cancer_ai/validator/competition_manager.py | 32 ++++++++ cancer_ai/validator/model_db.py | 62 +++++++++++++++- cancer_ai/validator/model_manager.py | 85 ++++++++++++++++++++++ 3 files changed, 178 insertions(+), 1 deletion(-) diff --git a/cancer_ai/validator/competition_manager.py b/cancer_ai/validator/competition_manager.py index 0048c137..6b9b4ae2 100644 --- a/cancer_ai/validator/competition_manager.py +++ b/cancer_ai/validator/competition_manager.py @@ -213,9 +213,19 @@ async def evaluate(self) -> Tuple[str | None, ModelEvaluationResult | None]: if len(self.results) == 0: bt.logging.error("No models were able to run") return None, None + + # see if there are any duplicate scores, slash the copied models owners + grouped_duplicated_hotkeys = self.group_duplicate_scores() + bt.logging.info(f"duplicated models: {grouped_duplicated_hotkeys}") + if len(grouped_duplicated_hotkeys) > 0: + pioneer_models_hotkeys = self.model_manager.get_pioneer_models_from_duplicated_models(grouped_duplicated_hotkeys) + hotkeys_to_slash = [hotkey for group in grouped_duplicated_hotkeys for hotkey in group if hotkey not in pioneer_models_hotkeys] + self.slash_model_copiers(hotkeys_to_slash) + winning_hotkey, winning_model_result = sorted( self.results, key=lambda x: x[1].score, reverse=True )[0] + for miner_hotkey, model_result in self.results: bt.logging.info(f"Model from {miner_hotkey} successfully evaluated") bt.logging.trace( @@ -226,3 +236,25 @@ async def evaluate(self) -> Tuple[str | None, ModelEvaluationResult | None]: f"Winning hotkey for competition {self.competition_id}: {winning_hotkey}" ) return winning_hotkey, winning_model_result + + + def group_duplicate_scores(self) -> list[list[str]]: + """ + Groups hotkeys for models with identical scores. + """ + score_to_hotkeys = {} + for hotkey, result in self.results: + score = round(result.score, 6) + if score in score_to_hotkeys and score > 0.0: + score_to_hotkeys[score].append(hotkey) + else: + score_to_hotkeys[score] = [hotkey] + + grouped_duplicates = [hotkeys for hotkeys in score_to_hotkeys.values() if len(hotkeys) > 1] + return grouped_duplicates + + def slash_model_copiers(self, hotkeys_to_slash: list[str]): + for hotkey, result in self.results: + if hotkey in hotkeys_to_slash: + bt.logging.info(f"Slashing model copier for hotkey: {hotkey} (setting score to 0.0)") + result.score = 0.0 diff --git a/cancer_ai/validator/model_db.py b/cancer_ai/validator/model_db.py index 0ca9e7e7..97039b0f 100644 --- a/cancer_ai/validator/model_db.py +++ b/cancer_ai/validator/model_db.py @@ -208,4 +208,64 @@ def convert_db_model_to_chain_model(self, model_record: ChainMinerModelDB) -> Ch hf_repo_type=model_record.hf_repo_type, hf_code_filename=model_record.hf_code_filename, block=model_record.block, - ) \ No newline at end of file + ) + + def compare_hotkeys( + self, hotkey1: str, hotkey2: str + ) -> tuple[str | None, datetime | None]: + """ + Compares two hotkeys in the DB and returns (earliest_hotkey, earliest_date_submitted). + If neither hotkey has any record, returns (None, None). + If only one hotkey has a record, that one is automatically considered 'earlier'. + """ + session = self.Session() + try: + record1 = ( + session.query(ChainMinerModelDB) + .filter(ChainMinerModelDB.hotkey == hotkey1) + .order_by(ChainMinerModelDB.date_submitted.asc()) + .first() + ) + + record2 = ( + session.query(ChainMinerModelDB) + .filter(ChainMinerModelDB.hotkey == hotkey2) + .order_by(ChainMinerModelDB.date_submitted.asc()) + .first() + ) + + if record1 is None and record2 is None: + bt.logging.info( + f"No records found for either hotkey: {hotkey1} or {hotkey2}" + ) + return None, None + + if record1 is None: + bt.logging.info( + f"No DB record for hotkey {hotkey1}, so {hotkey2} is automatically earlier." + ) + return hotkey2, record2.date_submitted + + if record2 is None: + bt.logging.info( + f"No DB record for hotkey {hotkey2}, so {hotkey1} is automatically earlier." + ) + return hotkey1, record1.date_submitted + + if record1.date_submitted <= record2.date_submitted: + bt.logging.info( + f"hotkey {hotkey1} chosen as pioneer hotkey" + ) + return hotkey1, record1.date_submitted + else: + bt.logging.info( + f"hotkey {hotkey2} chosen as pioneer hotkey" + ) + return hotkey2, record2.date_submitted + + except Exception as e: + session.rollback() + bt.logging.error(f"Error comparing hotkeys {hotkey1} & {hotkey2}: {e}") + raise + finally: + session.close() diff --git a/cancer_ai/validator/model_manager.py b/cancer_ai/validator/model_manager.py index fcb364e4..a6b174f1 100644 --- a/cancer_ai/validator/model_manager.py +++ b/cancer_ai/validator/model_manager.py @@ -154,3 +154,88 @@ def delete_model(self, hotkey) -> None: if hotkey in self.hotkey_store and self.hotkey_store[hotkey].file_path: os.remove(self.hotkey_store[hotkey].file_path) self.hotkey_store[hotkey] = None + + def get_pioneer_models_from_duplicated_models( + self, grouped_hotkeys: list[list[str]] + ) -> list[str]: + """ + For each group of duplicate hotkeys, determines the 'pioneer' model (the one + with the oldest upload date) and returns a list of (hotkey, ModelInfo) tuples. + """ + pioneers = [] + if self.config.hf_token: + fs = HfFileSystem(token=self.config.hf_token) + else: + fs = HfFileSystem() + + for group in grouped_hotkeys: + pioneer_hotkey = None + pioneer_date = None + pioneer_model_info = None + + for hotkey in group: + model_info = self.hotkey_store.get(hotkey) + if not model_info: + bt.logging.error(f"Model info for hotkey {hotkey} not found.") + continue + + try: + files = fs.ls(model_info.hf_repo_id) + except Exception as e: + bt.logging.error( + f"Failed to list files in repository {model_info.hf_repo_id} for hotkey {hotkey}: {e}" + ) + continue + + file_date_str = None + for file in files: + if file["name"].endswith(model_info.hf_model_filename): + file_date_str = file["last_commit"]["date"] + break + + if not file_date_str: + bt.logging.error( + f"File {model_info.hf_model_filename} not found in " + f"repository {model_info.hf_repo_id} for hotkey {hotkey}" + ) + continue + + try: + if isinstance(file_date_str, datetime): + file_date = file_date_str + else: + file_date = datetime.fromisoformat(str(file_date_str)) + + if file_date.tzinfo is None: + file_date = file_date.replace(tzinfo=timezone.utc) + + except Exception as e: + bt.logging.error( + f"Failed to parse file date {file_date_str} for hotkey {hotkey}: {e}" + ) + continue + + if pioneer_date is None or file_date < pioneer_date: + pioneer_date = file_date + pioneer_hotkey = hotkey + pioneer_model_info = model_info + + # If they have the same commit date, we use DB records to see who truly submitted earlier + elif file_date == pioneer_date: + try: + early_hotkey, early_date = self.db_controller.compare_hotkeys(pioneer_hotkey, hotkey) + if early_hotkey is not None: + pioneer_hotkey = early_hotkey + pioneer_date = early_date + else: + bt.warning.info( + f"No records exist for either hotkey in the DB. " + f"Unable to break tie between {pioneer_hotkey} and {hotkey}." + ) + except Exception as e: + bt.logging.error(f"Unable to compare hotkeys: {e}") + + if pioneer_hotkey is not None: + pioneers.append(pioneer_hotkey) + + return pioneers \ No newline at end of file From 22691dbee71d96a4d86de525bcd19db3181811a9 Mon Sep 17 00:00:00 2001 From: konrad0960 <71330299+konrad0960@users.noreply.github.com> Date: Tue, 1 Apr 2025 20:18:16 +0200 Subject: [PATCH 218/227] updated comp handler (#149) --- cancer_ai/validator/competition_manager.py | 1 + 1 file changed, 1 insertion(+) diff --git a/cancer_ai/validator/competition_manager.py b/cancer_ai/validator/competition_manager.py index 6b9b4ae2..e55e600c 100644 --- a/cancer_ai/validator/competition_manager.py +++ b/cancer_ai/validator/competition_manager.py @@ -27,6 +27,7 @@ "melanoma-1": MelanomaCompetitionHandler, "melanoma-testnet": MelanomaCompetitionHandler, "melanoma-7": MelanomaCompetitionHandler, + "melanoma-2": MelanomaCompetitionHandler, } From 32105a070035fbccc55781220848062f79362dbf Mon Sep 17 00:00:00 2001 From: konrad0960 <71330299+konrad0960@users.noreply.github.com> Date: Wed, 2 Apr 2025 22:02:55 +0200 Subject: [PATCH 219/227] models and comp manager adjustments (#150) --- cancer_ai/validator/competition_manager.py | 1 + cancer_ai/validator/models.py | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/cancer_ai/validator/competition_manager.py b/cancer_ai/validator/competition_manager.py index e55e600c..5ea5ddcd 100644 --- a/cancer_ai/validator/competition_manager.py +++ b/cancer_ai/validator/competition_manager.py @@ -236,6 +236,7 @@ async def evaluate(self) -> Tuple[str | None, ModelEvaluationResult | None]: bt.logging.info( f"Winning hotkey for competition {self.competition_id}: {winning_hotkey}" ) + self.dataset_manager.delete_dataset() return winning_hotkey, winning_model_result diff --git a/cancer_ai/validator/models.py b/cancer_ai/validator/models.py index daf07d36..d695eb48 100644 --- a/cancer_ai/validator/models.py +++ b/cancer_ai/validator/models.py @@ -17,7 +17,6 @@ class CompetitionsListModel(BaseModel): class OrganizationDataReference(BaseModel): competition_id: str = Field(..., min_length=1, description="Competition identifier") organization_id: str = Field(..., min_length=1, description="Unique identifier for the organization") - contact_email: EmailStr = Field(..., description="Contact email address for the organization") dataset_hf_repo: str = Field(..., min_length=1, description="Hugging Face repository path for the dataset") dataset_hf_dir: str = Field("", min_length=0, description="Directory for the datasets in the repository") From e12ca38a1d7308b0ab0734f599e65c8c6600872e Mon Sep 17 00:00:00 2001 From: Wojtek Jurkowlaniec Date: Thu, 3 Apr 2025 19:45:14 +0200 Subject: [PATCH 220/227] fix dataset unpacking --- cancer_ai/validator/dataset_manager.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cancer_ai/validator/dataset_manager.py b/cancer_ai/validator/dataset_manager.py index fb6d89e9..bc83e0d0 100644 --- a/cancer_ai/validator/dataset_manager.py +++ b/cancer_ai/validator/dataset_manager.py @@ -95,6 +95,9 @@ async def unzip_dataset(self) -> None: bt.logging.debug(f"Dataset extracted to: { self.local_compressed_path}") + # Ensure the extraction directory exists + os.makedirs(self.local_extracted_dir, exist_ok=True) + # TODO add error handling zip_file_path = self.local_compressed_path extract_dir = self.local_extracted_dir From df196006c56a0d834696dd9bd97f62c58ce0024c Mon Sep 17 00:00:00 2001 From: konrad0960 <71330299+konrad0960@users.noreply.github.com> Date: Fri, 4 Apr 2025 22:29:59 +0200 Subject: [PATCH 221/227] fix date comparisson (#151) --- cancer_ai/validator/model_manager.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/cancer_ai/validator/model_manager.py b/cancer_ai/validator/model_manager.py index a6b174f1..a9d8dda5 100644 --- a/cancer_ai/validator/model_manager.py +++ b/cancer_ai/validator/model_manager.py @@ -206,8 +206,10 @@ def get_pioneer_models_from_duplicated_models( else: file_date = datetime.fromisoformat(str(file_date_str)) - if file_date.tzinfo is None: + if file_date.tzinfo is None or file_date.tzinfo.utcoffset(file_date) is None: file_date = file_date.replace(tzinfo=timezone.utc) + else: + file_date = file_date.astimezone(timezone.utc) except Exception as e: bt.logging.error( @@ -216,7 +218,7 @@ def get_pioneer_models_from_duplicated_models( continue if pioneer_date is None or file_date < pioneer_date: - pioneer_date = file_date + pioneer_date = file_date.astimezone(timezone.utc) pioneer_hotkey = hotkey pioneer_model_info = model_info @@ -226,7 +228,11 @@ def get_pioneer_models_from_duplicated_models( early_hotkey, early_date = self.db_controller.compare_hotkeys(pioneer_hotkey, hotkey) if early_hotkey is not None: pioneer_hotkey = early_hotkey - pioneer_date = early_date + pioneer_date = ( + early_date.astimezone(timezone.utc) + if early_date.tzinfo + else early_date.replace(tzinfo=timezone.utc) + ) else: bt.warning.info( f"No records exist for either hotkey in the DB. " From db1cb0220d51ff652fbdf4cd836ce0c324646780 Mon Sep 17 00:00:00 2001 From: konrad0960 <71330299+konrad0960@users.noreply.github.com> Date: Tue, 8 Apr 2025 16:44:22 +0200 Subject: [PATCH 222/227] slashing mfs copiers 2 (#152) * wip slashing model copiers 2 * adjusting slashing mechanics * add melanoma-3 comp --- cancer_ai/base/base_validator.py | 10 ++- cancer_ai/chain_models_store.py | 9 ++- cancer_ai/validator/competition_manager.py | 47 +++++++++++-- cancer_ai/validator/model_db.py | 70 ++++++++++++++++--- cancer_ai/validator/model_manager.py | 79 ++++++++-------------- neurons/miner.py | 47 ++++++++++++- neurons/tests/competition_runner_test.py | 4 +- neurons/validator.py | 12 ++-- 8 files changed, 198 insertions(+), 80 deletions(-) diff --git a/cancer_ai/base/base_validator.py b/cancer_ai/base/base_validator.py index 56132768..90401a16 100644 --- a/cancer_ai/base/base_validator.py +++ b/cancer_ai/base/base_validator.py @@ -272,6 +272,10 @@ def set_weights(self): """ Sets the validator weights to the metagraph hotkeys based on the scores it has received from the miners. The weights determine the trust and incentive level the validator assigns to miner nodes on the network. """ + # test mode, don't commit weights + if self.config.filesystem_evaluation: + bt.logging.debug("Skipping settings weights in filesystem evaluation mode") + return # Check if self.scores contains any NaN values and log a warning if it does. if np.isnan(self.scores).any(): @@ -317,12 +321,6 @@ def set_weights(self): bt.logging.debug("uint_weights", uint_weights) bt.logging.debug("uint_uids", uint_uids) - # test mode, don't commit weights - if self.config.filesystem_evaluation: - bt.logging.debug("Skipping settings weights in filesystem evaluation mode") - return - - # Set the weights on chain via our subtensor connection. result, msg = self.subtensor.set_weights( wallet=self.wallet, diff --git a/cancer_ai/chain_models_store.py b/cancer_ai/chain_models_store.py index 43208e95..0c1e77a8 100644 --- a/cancer_ai/chain_models_store.py +++ b/cancer_ai/chain_models_store.py @@ -25,12 +25,16 @@ class ChainMinerModel(BaseModel): description="Block on which this model was claimed on the chain." ) + model_hash: Optional[str] = Field( + description="8-byte SHA-1 hash of the model file from Hugging Face." + ) + class Config: arbitrary_types_allowed = True def to_compressed_str(self) -> str: """Returns a compressed string representation.""" - return f"{self.hf_repo_id}:{self.hf_model_filename}:{self.hf_code_filename}:{self.competition_id}:{self.hf_repo_type}" + return f"{self.hf_repo_id}:{self.hf_model_filename}:{self.hf_code_filename}:{self.competition_id}:{self.hf_repo_type}:{self.model_hash}" @property def hf_link(self) -> str: @@ -41,7 +45,7 @@ def hf_link(self) -> str: def from_compressed_str(cls, cs: str) -> Type["ChainMinerModel"]: """Returns an instance of this class from a compressed string representation""" tokens = cs.split(":") - if len(tokens) != 5: + if len(tokens) != 6: return None return cls( hf_repo_id=tokens[0], @@ -49,6 +53,7 @@ def from_compressed_str(cls, cs: str) -> Type["ChainMinerModel"]: hf_code_filename=tokens[2], competition_id=tokens[3], hf_repo_type=tokens[4], + model_hash=tokens[5], block=None, ) diff --git a/cancer_ai/validator/competition_manager.py b/cancer_ai/validator/competition_manager.py index 5ea5ddcd..fc1966fc 100644 --- a/cancer_ai/validator/competition_manager.py +++ b/cancer_ai/validator/competition_manager.py @@ -3,6 +3,8 @@ import bittensor as bt import wandb +import hashlib + from dotenv import load_dotenv from .manager import SerializableManager @@ -28,6 +30,7 @@ "melanoma-testnet": MelanomaCompetitionHandler, "melanoma-7": MelanomaCompetitionHandler, "melanoma-2": MelanomaCompetitionHandler, + "melanoma-3": MelanomaCompetitionHandler, } @@ -119,6 +122,8 @@ async def chain_miner_to_model_info( hf_code_filename=chain_miner_model.hf_code_filename, hf_repo_type=chain_miner_model.hf_repo_type, competition_id=chain_miner_model.competition_id, + block=chain_miner_model.block, + model_hash=chain_miner_model.model_hash, ) return model_info @@ -134,7 +139,7 @@ async def update_miner_models(self): bt.logging.info(f"Amount of hotkeys: {len(self.hotkeys)}") latest_models = self.db_controller.get_latest_models( - self.hotkeys, self.competition_id + self.hotkeys, self.competition_id, self.config.models_query_cutoff ) for hotkey, model in latest_models.items(): try: @@ -152,7 +157,8 @@ async def update_miner_models(self): async def evaluate(self) -> Tuple[str | None, ModelEvaluationResult | None]: """Returns hotkey and competition id of winning model miner""" bt.logging.info(f"Start of evaluation of {self.competition_id}") - + + hotkeys_to_slash = [] # TODO add mock models functionality await self.update_miner_models() @@ -170,7 +176,7 @@ async def evaluate(self) -> Tuple[str | None, ModelEvaluationResult | None]: y_test = competition_handler.prepare_y_pred(y_test) # evaluate models - for miner_hotkey in self.model_manager.hotkey_store: + for miner_hotkey, model_info in self.model_manager.hotkey_store.items(): bt.logging.info(f"Evaluating hotkey: {miner_hotkey}") model_or_none = await self.model_manager.download_miner_model(miner_hotkey) if not model_or_none: @@ -179,6 +185,16 @@ async def evaluate(self) -> Tuple[str | None, ModelEvaluationResult | None]: ) continue + try: + computed_hash = self._compute_model_hash(model_info.file_path) + except Exception as e: + bt.logging.error(f"Failed to compute model hash: {e} Skipping.") + continue + + if computed_hash != model_info.model_hash: + bt.logging.info(f"The hash of model uploaded by {miner_hotkey} does not match hash of model submitted on-chain. Slashing.") + hotkeys_to_slash.append(miner_hotkey) + try: model_manager = ModelRunManager( self.config, self.model_manager.hotkey_store[miner_hotkey] @@ -216,11 +232,11 @@ async def evaluate(self) -> Tuple[str | None, ModelEvaluationResult | None]: return None, None # see if there are any duplicate scores, slash the copied models owners - grouped_duplicated_hotkeys = self.group_duplicate_scores() + grouped_duplicated_hotkeys = self.group_duplicate_scores(hotkeys_to_slash) bt.logging.info(f"duplicated models: {grouped_duplicated_hotkeys}") if len(grouped_duplicated_hotkeys) > 0: - pioneer_models_hotkeys = self.model_manager.get_pioneer_models_from_duplicated_models(grouped_duplicated_hotkeys) - hotkeys_to_slash = [hotkey for group in grouped_duplicated_hotkeys for hotkey in group if hotkey not in pioneer_models_hotkeys] + pioneer_models_hotkeys = self.model_manager.get_pioneer_models(grouped_duplicated_hotkeys) + hotkeys_to_slash.extend([hotkey for group in grouped_duplicated_hotkeys for hotkey in group if hotkey not in pioneer_models_hotkeys]) self.slash_model_copiers(hotkeys_to_slash) winning_hotkey, winning_model_result = sorted( @@ -240,12 +256,14 @@ async def evaluate(self) -> Tuple[str | None, ModelEvaluationResult | None]: return winning_hotkey, winning_model_result - def group_duplicate_scores(self) -> list[list[str]]: + def group_duplicate_scores(self, hotkeys_to_slash: list[str]) -> list[list[str]]: """ Groups hotkeys for models with identical scores. """ score_to_hotkeys = {} for hotkey, result in self.results: + if hotkey in hotkeys_to_slash: + continue score = round(result.score, 6) if score in score_to_hotkeys and score > 0.0: score_to_hotkeys[score].append(hotkey) @@ -260,3 +278,18 @@ def slash_model_copiers(self, hotkeys_to_slash: list[str]): if hotkey in hotkeys_to_slash: bt.logging.info(f"Slashing model copier for hotkey: {hotkey} (setting score to 0.0)") result.score = 0.0 + + def _compute_model_hash(self, file_path) -> str: + """Compute an 8-character hexadecimal SHA-1 hash of the model file.""" + sha1 = hashlib.sha1() + try: + with open(file_path, 'rb') as f: + while chunk := f.read(8192): + sha1.update(chunk) + full_hash = sha1.hexdigest() + truncated_hash = full_hash[:8] + bt.logging.info(f"Computed 8-character hash: {truncated_hash}") + return truncated_hash + except Exception as e: + bt.logging.error(f"Error computing hash for {file_path}: {e}") + return None \ No newline at end of file diff --git a/cancer_ai/validator/model_db.py b/cancer_ai/validator/model_db.py index 97039b0f..f2c35ddd 100644 --- a/cancer_ai/validator/model_db.py +++ b/cancer_ai/validator/model_db.py @@ -20,17 +20,39 @@ class ChainMinerModelDB(Base): date_submitted = Column(DateTime, nullable=False) block = Column(Integer, nullable=False) hotkey = Column(String, nullable=False) + model_hash = Column(String, nullable=False) __table_args__ = ( PrimaryKeyConstraint('date_submitted', 'hotkey', name='pk_date_hotkey'), ) class ModelDBController: - def __init__(self, db_path: str = "models.db"): + def __init__(self, db_path: str = "models.db", subtensor: bt.subtensor = None): db_url = f"sqlite:///{os.path.abspath(db_path)}" self.engine = create_engine(db_url, echo=False) Base.metadata.create_all(self.engine) self.Session = sessionmaker(bind=self.engine) + self.subtensor = subtensor + + if subtensor is not None and "test" not in self.subtensor.chain_endpoint.lower(): + self.subtensor = bt.subtensor(network="archive") + + self._migrate_database() + + def _migrate_database(self): + """Check and apply migration for model_hash column if missing.""" + with self.engine.connect() as connection: + result = connection.execute("PRAGMA table_info(models)").fetchall() + column_names = [row[1] for row in result] + if "model_hash" not in column_names: + try: + connection.execute("ALTER TABLE models ADD COLUMN model_hash TEXT CHECK(LENGTH(model_hash) <= 8)") + bt.logging.info("Migrated database: Added model_hash column with length constraint to models table") + except Exception as e: + bt.logging.error(f"Failed to migrate database: {e}") + raise + + def add_model(self, chain_miner_model: ChainMinerModel, hotkey: str): session = self.Session() @@ -63,12 +85,14 @@ def get_model(self, hotkey: str) -> ChainMinerModel | None: session.close() def get_latest_model(self, hotkey: str, cutoff_time: float = None) -> ChainMinerModel | None: + cutoff_time = datetime.now(timezone.utc) - timedelta(minutes=cutoff_time) if cutoff_time else datetime.now(timezone.utc) bt.logging.debug(f"Getting latest DB model for hotkey {hotkey}") session = self.Session() try: model_record = ( session.query(ChainMinerModelDB) .filter(ChainMinerModelDB.hotkey == hotkey) + .filter(ChainMinerModelDB.date_submitted < cutoff_time) .order_by(ChainMinerModelDB.date_submitted.desc()) .first() ) @@ -115,6 +139,9 @@ def update_model(self, chain_miner_model: ChainMinerModel, hotkey: str): existing_model.hf_model_filename = chain_miner_model.hf_model_filename existing_model.hf_repo_type = chain_miner_model.hf_repo_type existing_model.hf_code_filename = chain_miner_model.hf_code_filename + existing_model.date_submitted = self.get_block_timestamp(chain_miner_model.block) + existing_model.block = chain_miner_model.block + existing_model.model_hash = chain_miner_model.model_hash session.commit() bt.logging.debug(f"Successfully updated DB model for hotkey {hotkey}.") @@ -131,8 +158,8 @@ def update_model(self, chain_miner_model: ChainMinerModel, hotkey: str): session.close() - def get_latest_models(self, hotkeys: list[str], competition_id: str) -> dict[str, ChainMinerModel]: - # cutoff_time = datetime.now(timezone.utc) - timedelta(minutes=cutoff_time) if cutoff_time else datetime.now(timezone.utc) + def get_latest_models(self, hotkeys: list[str], competition_id: str, cutoff: int = None) -> dict[str, ChainMinerModel]: + cutoff_time = datetime.now(timezone.utc) - timedelta(minutes=cutoff) if cutoff else datetime.now(timezone.utc) session = self.Session() try: # Use a correlated subquery to get the latest record for each hotkey that doesn't violate the cutoff @@ -142,8 +169,8 @@ def get_latest_models(self, hotkeys: list[str], competition_id: str) -> dict[str session.query(ChainMinerModelDB) .filter(ChainMinerModelDB.hotkey == hotkey) .filter(ChainMinerModelDB.competition_id == competition_id) - # .filter(ChainMinerModelDB.date_submitted < cutoff_time) - # .order_by(ChainMinerModelDB.date_submitted.desc()) # Order by newest first + .filter(ChainMinerModelDB.date_submitted < cutoff_time) + .order_by(ChainMinerModelDB.date_submitted.desc()) # Order by newest first .first() # Get the first (newest) record that meets the cutoff condition ) if model_record: @@ -195,9 +222,10 @@ def convert_chain_model_to_db_model(self, chain_miner_model: ChainMinerModel, ho hf_model_filename = chain_miner_model.hf_model_filename, hf_repo_type = chain_miner_model.hf_repo_type, hf_code_filename = chain_miner_model.hf_code_filename, - date_submitted = datetime.now(timezone.utc), # temporary fix, can't be null - block = 1, # temporary fix , can't be null - hotkey = hotkey + date_submitted = self.get_block_timestamp(chain_miner_model.block), + block = chain_miner_model.block, + hotkey = hotkey, + model_hash=chain_miner_model.model_hash ) def convert_db_model_to_chain_model(self, model_record: ChainMinerModelDB) -> ChainMinerModel: @@ -208,6 +236,7 @@ def convert_db_model_to_chain_model(self, model_record: ChainMinerModelDB) -> Ch hf_repo_type=model_record.hf_repo_type, hf_code_filename=model_record.hf_code_filename, block=model_record.block, + model_hash=model_record.model_hash, ) def compare_hotkeys( @@ -269,3 +298,28 @@ def compare_hotkeys( raise finally: session.close() + + def get_block_timestamp(self, block_number) -> datetime: + """Gets the timestamp of a block given its number.""" + try: + block_hash = self.subtensor.get_block_hash(block_number) + + if block_hash is None: + raise ValueError(f"Block hash not found for block number {block_number}") + + timestamp_info = self.subtensor.substrate.query( + module="Timestamp", + storage_function="Now", + block_hash=block_hash + ) + + if timestamp_info is None: + raise ValueError(f"Timestamp not found for block hash {block_hash}") + + timestamp_ms = timestamp_info.value + block_datetime = datetime.fromtimestamp(timestamp_ms / 1000.0, tz=timezone.utc) + + return block_datetime + except Exception as e: + bt.logging.error(f"Error retrieving block timestamp: {e}") + raise \ No newline at end of file diff --git a/cancer_ai/validator/model_manager.py b/cancer_ai/validator/model_manager.py index a9d8dda5..e489766c 100644 --- a/cancer_ai/validator/model_manager.py +++ b/cancer_ai/validator/model_manager.py @@ -19,6 +19,8 @@ class ModelInfo: competition_id: str | None = None file_path: str | None = None model_type: str | None = None + block: int | None = None + model_hash: str | None = None class ModelManager(SerializableManager): @@ -88,7 +90,7 @@ async def download_miner_model(self, hotkey) -> bool: except Exception as e: bt.logging.error(f"Failed to download model file: {e}") return False - + # Verify the downloaded file exists if not os.path.exists(model_info.file_path): bt.logging.error(f"Downloaded file does not exist at {model_info.file_path}") @@ -155,23 +157,21 @@ def delete_model(self, hotkey) -> None: os.remove(self.hotkey_store[hotkey].file_path) self.hotkey_store[hotkey] = None - def get_pioneer_models_from_duplicated_models( - self, grouped_hotkeys: list[list[str]] - ) -> list[str]: + + def get_pioneer_models(self, grouped_hotkeys: list[list[str]]) -> list[str]: """ - For each group of duplicate hotkeys, determines the 'pioneer' model (the one - with the oldest upload date) and returns a list of (hotkey, ModelInfo) tuples. + Does a check on whether chain submit date was later then HF commit date. If not slashes. + Compares chain submit date duplicated models to elect a pioneer based on block of submission (date) """ pioneers = [] + if self.config.hf_token: fs = HfFileSystem(token=self.config.hf_token) else: fs = HfFileSystem() for group in grouped_hotkeys: - pioneer_hotkey = None - pioneer_date = None - pioneer_model_info = None + candidate_hotkeys = [] for hotkey in group: model_info = self.hotkey_store.get(hotkey) @@ -182,9 +182,7 @@ def get_pioneer_models_from_duplicated_models( try: files = fs.ls(model_info.hf_repo_id) except Exception as e: - bt.logging.error( - f"Failed to list files in repository {model_info.hf_repo_id} for hotkey {hotkey}: {e}" - ) + bt.logging.error(f"Failed to list files in {model_info.hf_repo_id}: {e}") continue file_date_str = None @@ -195,53 +193,36 @@ def get_pioneer_models_from_duplicated_models( if not file_date_str: bt.logging.error( - f"File {model_info.hf_model_filename} not found in " - f"repository {model_info.hf_repo_id} for hotkey {hotkey}" + f"File {model_info.hf_model_filename} not found in {model_info.hf_repo_id} for {hotkey}" ) continue try: if isinstance(file_date_str, datetime): - file_date = file_date_str + hf_commit_date = file_date_str else: - file_date = datetime.fromisoformat(str(file_date_str)) + hf_commit_date = datetime.fromisoformat(str(file_date_str)) - if file_date.tzinfo is None or file_date.tzinfo.utcoffset(file_date) is None: - file_date = file_date.replace(tzinfo=timezone.utc) + if hf_commit_date.tzinfo is None or hf_commit_date.tzinfo.utcoffset(hf_commit_date) is None: + hf_commit_date = hf_commit_date.replace(tzinfo=timezone.utc) else: - file_date = file_date.astimezone(timezone.utc) + hf_commit_date = hf_commit_date.astimezone(timezone.utc) except Exception as e: - bt.logging.error( - f"Failed to parse file date {file_date_str} for hotkey {hotkey}: {e}" - ) + bt.logging.error(f"Failed to parse HF commit date {file_date_str} for {hotkey}: {e}") continue - if pioneer_date is None or file_date < pioneer_date: - pioneer_date = file_date.astimezone(timezone.utc) - pioneer_hotkey = hotkey - pioneer_model_info = model_info - - # If they have the same commit date, we use DB records to see who truly submitted earlier - elif file_date == pioneer_date: - try: - early_hotkey, early_date = self.db_controller.compare_hotkeys(pioneer_hotkey, hotkey) - if early_hotkey is not None: - pioneer_hotkey = early_hotkey - pioneer_date = ( - early_date.astimezone(timezone.utc) - if early_date.tzinfo - else early_date.replace(tzinfo=timezone.utc) - ) - else: - bt.warning.info( - f"No records exist for either hotkey in the DB. " - f"Unable to break tie between {pioneer_hotkey} and {hotkey}." - ) - except Exception as e: - bt.logging.error(f"Unable to compare hotkeys: {e}") - - if pioneer_hotkey is not None: - pioneers.append(pioneer_hotkey) + try: + block_timestamp = self.db_controller.get_block_timestamp(model_info.block) + block_timestamp = block_timestamp.replace(tzinfo=timezone.utc) + except Exception as e: + bt.logging.error(f"Failed to get block timestamp for {model_info.block}: {e}") + continue + + if hf_commit_date <= block_timestamp: + candidate_hotkeys.append((hotkey, model_info.block)) - return pioneers \ No newline at end of file + if candidate_hotkeys: + pioneer_hotkey = min(candidate_hotkeys, key=lambda x: x[1])[0] + pioneers.append(pioneer_hotkey) + return pioneers diff --git a/neurons/miner.py b/neurons/miner.py index 031f79fe..ad5371e4 100644 --- a/neurons/miner.py +++ b/neurons/miner.py @@ -8,8 +8,10 @@ from dotenv import load_dotenv from huggingface_hub import HfApi, login as hf_login import huggingface_hub +from huggingface_hub import hf_hub_download import onnx import argparse +import hashlib from cancer_ai.validator.utils import run_command from cancer_ai.validator.model_run_manager import ModelRunManager, ModelInfo @@ -146,6 +148,26 @@ async def compress_code(self) -> None: return bt.logging.info(f"Code zip path: {self.code_zip_path}") + def _compute_model_hash(self, repo_id, model_filename, repo_type): + """Compute an 8-character hexadecimal SHA-1 hash of the model file from Hugging Face.""" + try: + model_path = huggingface_hub.hf_hub_download( + repo_id=repo_id, + filename=model_filename, + repo_type=repo_type, + ) + sha1 = hashlib.sha1() + with open(model_path, 'rb') as f: + while chunk := f.read(8192): + sha1.update(chunk) + full_hash = sha1.hexdigest() + truncated_hash = full_hash[:8] # Take the first 8 characters of the hex digest + bt.logging.info(f"Computed 8-character hash: {truncated_hash}") + return truncated_hash + except Exception as e: + bt.logging.error(f"Failed to compute model hash: {e}") + return None + async def submit_model(self) -> None: # Check if the required model and files are present in hugging face repo @@ -172,7 +194,21 @@ async def submit_model(self) -> None: subtensor=self.subtensor, netuid=self.config.netuid, wallet=self.wallet ) - print(self.config) + if len(self.config.hf_repo_type.encode('utf-8')) > 7: + bt.logging.error("hf_repo_type must be 7 bytes or less") + return + + if len(self.config.hf_repo_id.encode('utf-8')) > 32: + bt.logging.error("hf_repo_id must be 32 bytes or less") + return + + if len(self.config.hf_model_name.encode('utf-8')) > 32: + bt.logging.error("hf_model_filename must be 32 bytes or less") + return + + if len(self.config.hf_code_filename.encode('utf-8')) > 31: + bt.logging.error("hf_code_filename must be 31 bytes or less") + return if not self._check_hf_file_exists(self.config.hf_repo_id, self.config.hf_model_name, self.config.hf_repo_type): return @@ -180,6 +216,14 @@ async def submit_model(self) -> None: if not self._check_hf_file_exists(self.config.hf_repo_id, self.config.hf_code_filename, self.config.hf_repo_type): return + model_hash = self._compute_model_hash( + self.config.hf_repo_id, self.config.hf_model_name, self.config.hf_repo_type + ) + + if not model_hash: + bt.logging.error("Failed to compute model hash") + return + # Push model metadata to chain model_id = ChainMinerModel( competition_id=self.config.competition_id, @@ -188,6 +232,7 @@ async def submit_model(self) -> None: hf_repo_type=self.config.hf_repo_type, hf_code_filename=self.config.hf_code_filename, block=None, + model_hash=model_hash, ) await self.metadata_store.store_model_metadata(model_id) bt.logging.success( diff --git a/neurons/tests/competition_runner_test.py b/neurons/tests/competition_runner_test.py index ad5b651d..9b7be948 100644 --- a/neurons/tests/competition_runner_test.py +++ b/neurons/tests/competition_runner_test.py @@ -62,7 +62,7 @@ async def run_competitions( dataset_hf_id=competition_cfg.dataset_hf_filename, dataset_hf_repo_type=competition_cfg.dataset_hf_repo_type, test_mode=True, - db_controller=ModelDBController(subtensor, test_config.db_path) + db_controller=ModelDBController(db_path=test_config.db_path, subtensor=subtensor) ) results[competition_cfg.competition_id] = await competition_manager.evaluate() @@ -86,7 +86,7 @@ def config_for_scheduler(subtensor: bt.subtensor) -> Dict[str, CompetitionManage dataset_hf_id=competition_cfg.dataset_hf_filename, dataset_hf_repo_type=competition_cfg.dataset_hf_repo_type, test_mode=True, - db_controller=ModelDBController(subtensor, test_config.db_path) + db_controller=ModelDBController(db_path=test_config.db_path, subtensor=subtensor) ) return time_arranged_competitions diff --git a/neurons/validator.py b/neurons/validator.py index 27e0e434..7b61bcd6 100644 --- a/neurons/validator.py +++ b/neurons/validator.py @@ -56,7 +56,7 @@ def __init__(self, config=None, exit_event=None): print(cancer_ai_logo) super(Validator, self).__init__(config=config) self.hotkey = self.wallet.hotkey.ss58_address - self.db_controller = ModelDBController(self.config.db_path) + self.db_controller = ModelDBController(db_path=self.config.db_path, subtensor=self.subtensor) self.chain_models = ChainModelMetadata( self.subtensor, self.config.netuid, self.wallet @@ -153,6 +153,7 @@ async def filesystem_test_evaluation(self): bt.logging.error("NO WINNING HOTKEY") except Exception as e: bt.logging.error(f"Error evaluating {data_package.dataset_hf_filename}: {e}") + return models_results = competition_manager.results @@ -165,8 +166,8 @@ async def filesystem_test_evaluation(self): # Enable if you want to have results in CSV for debugging purposes # await self.log_results_to_csv(data_package, top_hotkey, models_results) - - bt.logging.info(f"Competition result for {data_package.competition_id}: {winning_hotkey}") + if winning_hotkey: + bt.logging.info(f"Competition result for {data_package.competition_id}: {winning_hotkey}") bt.logging.warning("Competition results store before update") bt.logging.warning(self.competition_results_store.model_dump_json()) @@ -249,7 +250,7 @@ async def monitor_datasets(self): if not winning_hotkey: continue - winning_model_link = self.db_controller.get_latest_model(hotkey=winning_hotkey).hf_link + winning_model_link = self.db_controller.get_latest_model(hotkey=winning_hotkey, cutoff_time=self.config.models_query_cutoff).hf_link except Exception: formatted_traceback = traceback.format_exc() bt.logging.error(f"Error running competition: {formatted_traceback}") @@ -293,7 +294,8 @@ async def monitor_datasets(self): for miner_hotkey, evaluation_result in competition_manager.results: try: model = self.db_controller.get_latest_model( - hotkey=miner_hotkey + hotkey=miner_hotkey, + cutoff_time=self.config.models_query_cutoff, ) model_link = model.hf_link if model is not None else None From 2bf2bd5fdac100b58113b14da9f210bd1ae1c3e3 Mon Sep 17 00:00:00 2001 From: konrad0960 <71330299+konrad0960@users.noreply.github.com> Date: Fri, 11 Apr 2025 00:58:54 +0200 Subject: [PATCH 223/227] Update config.py --- cancer_ai/utils/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cancer_ai/utils/config.py b/cancer_ai/utils/config.py index 64ef4314..a593b9e4 100644 --- a/cancer_ai/utils/config.py +++ b/cancer_ai/utils/config.py @@ -346,7 +346,7 @@ def add_validator_args(cls, parser): "--monitor_datasets_interval", type=int, help="The interval at which to monitor the datasets in seconds", - default=8*60, + default=20, ) parser.add_argument( From 0807999d5cc6aee9a118c2808147f8f5640daf70 Mon Sep 17 00:00:00 2001 From: Wojtek Jurkowlaniec Date: Mon, 14 Apr 2025 23:45:03 +0200 Subject: [PATCH 224/227] rewarder fixed (#148) --- cancer_ai/validator/rewarder.py | 20 ++--- cancer_ai/validator/test_rewarder.py | 129 +++++++++++++++++++++++++++ 2 files changed, 139 insertions(+), 10 deletions(-) create mode 100644 cancer_ai/validator/test_rewarder.py diff --git a/cancer_ai/validator/rewarder.py b/cancer_ai/validator/rewarder.py index cfe35271..d0974bb0 100644 --- a/cancer_ai/validator/rewarder.py +++ b/cancer_ai/validator/rewarder.py @@ -46,13 +46,14 @@ def add_score(self, competition_id: str, hotkey: Hotkey, score: float, date: dat ) # Sort by date and keep only the last HISTORY_LENGTH scores - self.score_map[competition_id][hotkey].sort(key=lambda x: x.date, reverse=True) + self.score_map[competition_id][hotkey].sort(key=lambda x: x.date) if len(self.score_map[competition_id][hotkey]) > HISTORY_LENGTH: - self.score_map[competition_id][hotkey] = self.score_map[competition_id][hotkey][-HISTORY_LENGTH:] + # remove the oldest one + self.score_map[competition_id][hotkey] = self.score_map[competition_id][hotkey][1:] self.update_average_score(competition_id, hotkey) - def update_average_score(self, competition_id: str, hotkey: Hotkey): + def update_average_score(self, competition_id: str, hotkey: Hotkey) -> None: """Update the average score for a specific hotkey in a specific competition""" if ( competition_id not in self.score_map @@ -60,20 +61,20 @@ def update_average_score(self, competition_id: str, hotkey: Hotkey): ): return 0.0 + scores = self.score_map[competition_id][hotkey][-MOVING_AVERAGE_LENGTH:] + scores = [score.score for score in scores] + bt.logging.debug(f"Scores used to calculate average for hotkey {hotkey}: {scores}") try: result = sum( - score.score - for score in self.score_map[competition_id][hotkey][ - -MOVING_AVERAGE_LENGTH: - ] - ) / len(self.score_map[competition_id][hotkey][-MOVING_AVERAGE_LENGTH:]) + score + for score in scores + ) / len(scores) except ZeroDivisionError: result = 0.0 if competition_id not in self.average_scores: self.average_scores[competition_id] = {} self.average_scores[competition_id][hotkey] = result - return result def delete_dead_hotkeys(self, competition_id: str, active_hotkeys: list[Hotkey]): """Delete hotkeys that are no longer active in a specific competition.""" @@ -84,7 +85,6 @@ def delete_dead_hotkeys(self, competition_id: str, active_hotkeys: list[Hotkey]) for hotkey in self.score_map[competition_id].keys(): if hotkey not in active_hotkeys: hotkeys_to_delete.append(hotkey) - for hotkey in hotkeys_to_delete: del self.score_map[competition_id][hotkey] if ( diff --git a/cancer_ai/validator/test_rewarder.py b/cancer_ai/validator/test_rewarder.py new file mode 100644 index 00000000..e995deda --- /dev/null +++ b/cancer_ai/validator/test_rewarder.py @@ -0,0 +1,129 @@ +import unittest +from datetime import datetime, timezone +from cancer_ai.validator.rewarder import CompetitionResultsStore +import bittensor as bt + + +class TestCompetitionResultsStore(unittest.TestCase): + + def setUp(self): + self.store = CompetitionResultsStore() + self.competition_id = "test_competition" + self.hotkey = "test_hotkey" + self.score = 0.5 + self.date = datetime(2023, 1, 1, tzinfo=timezone.utc) + + def test_add_score(self): + self.store.add_score(self.competition_id, self.hotkey, self.score, self.date) + self.assertIn(self.competition_id, self.store.score_map) + self.assertIn(self.hotkey, self.store.score_map[self.competition_id]) + self.assertEqual(len(self.store.score_map[self.competition_id][self.hotkey]), 1) + self.assertEqual( + self.store.score_map[self.competition_id][self.hotkey][0].score, self.score + ) + self.assertEqual( + self.store.score_map[self.competition_id][self.hotkey][0].date, self.date + ) + + def test_update_average_score(self): + self.store.add_score(self.competition_id, self.hotkey, self.score, self.date) + self.assertEqual( + self.store.average_scores[self.competition_id][self.hotkey], self.score + ) + + def test_delete_dead_hotkeys(self): + self.store.add_score(self.competition_id, self.hotkey, self.score, self.date) + active_hotkeys = [] + self.store.delete_dead_hotkeys(self.competition_id, active_hotkeys) + self.assertNotIn(self.hotkey, self.store.score_map[self.competition_id]) + self.assertNotIn(self.hotkey, self.store.average_scores[self.competition_id]) + + def test_get_top_hotkey(self): + self.store.add_score(self.competition_id, self.hotkey, self.score, self.date) + top_hotkey = self.store.get_top_hotkey(self.competition_id) + self.assertEqual(top_hotkey, self.hotkey) + + def test_delete_inactive_competitions(self): + self.store.add_score(self.competition_id, self.hotkey, self.score, self.date) + active_competitions = [] + self.store.delete_inactive_competitions(active_competitions) + self.assertNotIn(self.competition_id, self.store.score_map) + self.assertNotIn(self.competition_id, self.store.average_scores) + self.assertNotIn(self.competition_id, self.store.current_top_hotkeys) + + def test_step_by_step(self): + scores_sequential = [1, 2, 1.5, 1.5, 7, 8] + averages_sequential = [1, 1.5, 1.5, 1.5, 2.6, 4.0] + for i in range(6): + self.store.add_score(self.competition_id, self.hotkey, scores_sequential[i]) + self.assertEqual( + self.store.average_scores[self.competition_id][self.hotkey], + averages_sequential[i], + ) + + def test_add_a_lot_of_runs(self): + scores = [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0] + dates = [datetime(2023, 1, i, tzinfo=timezone.utc) for i in range(1, 11)] + for score in scores: + self.store.add_score( + self.competition_id, self.hotkey, score, dates[scores.index(score)] + ) + + expected_average = sum(scores[-5:]) / 5 + bt.logging.debug(f"Expected average: {expected_average}") + self.assertAlmostEqual( + self.store.average_scores[self.competition_id][self.hotkey], + expected_average, + ) + + def test_add_a_lot_of_runs_history(self): + scores = [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0, 1.1, 1.2] + dates = [datetime(2023, 1, i, tzinfo=timezone.utc) for i in range(1, 13)] + for score in scores: + self.store.add_score( + self.competition_id, self.hotkey, score, dates[scores.index(score)] + ) + bt.logging.debug( + f"Scores: {self.store.score_map[self.competition_id][self.hotkey]}" + ) + self.assertEqual( + len(self.store.score_map[self.competition_id][self.hotkey]), 10 + ) + expected_scores = scores[-10:] + bt.logging.debug(f"Expected scores: {expected_scores}") + actual_scores = [ + model_score.score + for model_score in self.store.score_map[self.competition_id][self.hotkey] + ] + self.assertEqual(actual_scores, expected_scores) + + def test_average_after_history(self): + scores = [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0] + dates = [datetime(2023, 1, i, tzinfo=timezone.utc) for i in range(1, 11)] + for score in scores: + self.store.add_score( + self.competition_id, self.hotkey, score, dates[scores.index(score)] + ) + + expected_average = sum(scores[-5:]) / 5 # 1.0, 0.9, 0.8, 0.7, 0.6, 0.5 + + self.assertAlmostEqual( + self.store.average_scores[self.competition_id][self.hotkey], + expected_average, + ) + + self.store.add_score( + self.competition_id, + self.hotkey, + 1.1, + datetime(2023, 1, 11, tzinfo=timezone.utc), + ) + expected_average = sum([1.1, 1.0, 0.9, 0.8, 0.7]) / 5 + self.assertAlmostEqual( + self.store.average_scores[self.competition_id][self.hotkey], + expected_average, + ) + + +if __name__ == "__main__": + unittest.main() From 4e75c242b8a99111f9e95257f962f3447d9c2986 Mon Sep 17 00:00:00 2001 From: konrad0960 <71330299+konrad0960@users.noreply.github.com> Date: Wed, 16 Apr 2025 21:45:06 +0200 Subject: [PATCH 225/227] error handling for chain data fetch (#155) --- cancer_ai/chain_models_store.py | 62 +++++++++++---------------------- neurons/validator.py | 11 +++--- 2 files changed, 26 insertions(+), 47 deletions(-) diff --git a/cancer_ai/chain_models_store.py b/cancer_ai/chain_models_store.py index 0c1e77a8..6d9c6dcf 100644 --- a/cancer_ai/chain_models_store.py +++ b/cancer_ai/chain_models_store.py @@ -86,63 +86,41 @@ async def store_model_metadata(self, model_id: ChainMinerModel): model_id.to_compressed_str(), ) - async def retrieve_model_metadata(self, hotkey: str) -> Optional[ChainMinerModel]: + async def retrieve_model_metadata(self, hotkey: str, uid: int) -> ChainMinerModel: """Retrieves model metadata on this subnet for specific hotkey""" - metadata = await get_metadata_with_timeout(self.subtensor, self.netuid, hotkey) - if metadata is None: - self.subnet_metadata = self.subtensor.metagraph(self.netuid) - metadata = await get_metadata_with_timeout(self.subtensor, self.netuid, hotkey) - if metadata is None: - return None + try: + metadata = get_metadata(self.subtensor, self.netuid, hotkey) + except Exception: + raise - uids = self.subnet_metadata.uids - hotkeys = self.subnet_metadata.hotkeys - uid = next((uid for uid, hk in zip(uids, hotkeys) if hk == hotkey), None) + if metadata is None: + raise ValueError(f"No metadata found for hotkey {hotkey}") try: chain_str = self.subtensor.get_commitment(self.netuid, uid) - except Exception as e: - bt.logging.debug(f"Failed to retrieve commitment for hotkey {hotkey}: {e}") - return None - - if not metadata: - return None + except Exception: + raise model = None try: model = ChainMinerModel.from_compressed_str(chain_str) bt.logging.debug(f"Model: {model}") if model is None: - bt.logging.error( - f"Metadata might be in old format on the chain for hotkey {hotkey}. Raw value: {chain_str}" + raise ValueError( + f"Metadata might be in old format or invalid for hotkey '{hotkey}'. Raw value: {chain_str}" ) - return None except Exception: - # If the metadata format is not correct on the chain then we return None. - bt.logging.error( - f"Failed to parse the metadata on the chain for hotkey {hotkey}. Raw value: {chain_str}" - ) - return None + raise # The block id at which the metadata is stored model.block = metadata["block"] return model -def timeout(seconds): - def decorator(func): - @wraps(func) - async def wrapper(*args, **kwargs): - try: - return await asyncio.wait_for(func(*args, **kwargs), timeout=seconds) - except asyncio.TimeoutError: - bt.logging.debug("Metadata retrieval timed out, refreshing subnet metadata") - return None - return wrapper - return decorator - @retry(tries=10, delay=5) -def get_metadata_with_retry(subtensor, netuid, hotkey): - return bt.core.extrinsics.serving.get_metadata(subtensor, netuid, hotkey) - -@timeout(10) # 10 second timeout -async def get_metadata_with_timeout(subtensor, netuid, hotkey): - return get_metadata_with_retry(subtensor, netuid, hotkey) +def get_metadata(subtensor, netuid, hotkey): + """Synchronous metadata fetch with retry logic.""" + try: + return bt.core.extrinsics.serving.get_metadata(subtensor, netuid, hotkey) + except Exception as e: + raise RuntimeError( + f"Failed to get metadata from chain for hotkey '{hotkey}': {e}" + ) from e \ No newline at end of file diff --git a/neurons/validator.py b/neurons/validator.py index 7b61bcd6..bc08d327 100644 --- a/neurons/validator.py +++ b/neurons/validator.py @@ -113,12 +113,13 @@ async def refresh_miners(self): hotkey = str(hotkey) bt.logging.debug(f"Downloading model {i+1}/{len(self.hotkeys)} from hotkey {hotkey}") - chain_model_metadata = await self.chain_models.retrieve_model_metadata(hotkey) - if not chain_model_metadata: - bt.logging.warning( - f"Cannot get miner model for hotkey {hotkey} from the chain, skipping" - ) + try: + uid = self.metagraph.hotkeys.index(hotkey) + chain_model_metadata = await self.chain_models.retrieve_model_metadata(hotkey, uid) + except Exception as e: + bt.logging.warning(f"Cannot get miner model for hotkey {hotkey} from the chain: {e}. Skipping.") continue + try: self.db_controller.add_model(chain_model_metadata, hotkey) except Exception as e: From 766a61adadb5307278da0badc5c70ccc8e85aea1 Mon Sep 17 00:00:00 2001 From: Wojtek Jurkowlaniec Date: Thu, 17 Apr 2025 15:18:23 +0200 Subject: [PATCH 226/227] Wandb data rewrite + various optimizations (#154) * move modelinfo and modelinfo conversion to proper places * remove repo type from miner model submission * local evaluation - don't set weight (and don't brag about it * add time submitting in model results + remove unnecesarry stuff from try/except * new model for submitting to wandb * switch to default model type repo when downloading miner's model * adding error messages to evaluation + refactored wandb logging structure * cleanup validation, new way of submitting data , reporting errors to wandb * error handling/refactor for chain metadata * collect errors in model manager * adjusting error results * more granular error msgs --------- Co-authored-by: konrad0960 --- .gitignore | 3 +- DOCS/miner.md | 1 - cancer_ai/base/neuron.py | 3 +- cancer_ai/utils/config.py | 6 - .../competition_handlers/base_handler.py | 2 + cancer_ai/validator/competition_manager.py | 55 +++-- cancer_ai/validator/model_db.py | 18 +- cancer_ai/validator/model_manager.py | 31 +-- cancer_ai/validator/model_manager_test.py | 5 +- cancer_ai/validator/models.py | 60 +++-- cancer_ai/validator/utils.py | 18 +- evaluation-results/.gitkeep | 0 neurons/miner.py | 15 +- neurons/validator.py | 218 ++++++++++-------- 14 files changed, 244 insertions(+), 191 deletions(-) create mode 100644 evaluation-results/.gitkeep diff --git a/.gitignore b/.gitignore index 586d872e..0c19f5ba 100644 --- a/.gitignore +++ b/.gitignore @@ -175,4 +175,5 @@ ecosystem.config.js keys -local_datasets/ \ No newline at end of file +local_datasets/ +*.csv \ No newline at end of file diff --git a/DOCS/miner.md b/DOCS/miner.md index ff49d884..bddd6208 100644 --- a/DOCS/miner.md +++ b/DOCS/miner.md @@ -148,7 +148,6 @@ python neurons/miner.py \ --hf_code_filename skin_melanoma_small.zip\ --hf_model_name best_model.onnx \ --hf_repo_id safescanai/test_dataset \ - --hf_repo_type model \ --wallet.name miner2 \ --wallet.hotkey default \ --netuid 163 \ diff --git a/cancer_ai/base/neuron.py b/cancer_ai/base/neuron.py index af9eb7a5..ea21ddc0 100644 --- a/cancer_ai/base/neuron.py +++ b/cancer_ai/base/neuron.py @@ -117,7 +117,8 @@ def sync(self, retries=5, delay=10, force_sync=False): try: # Ensure miner or validator hotkey is still registered on the network. self.check_registered() - + if self.config.filesystem_evaluation: + break if self.should_sync_metagraph() or force_sync: bt.logging.info("Resyncing metagraph in progress.") self.resync_metagraph(force_sync=True) diff --git a/cancer_ai/utils/config.py b/cancer_ai/utils/config.py index a593b9e4..e2671db3 100644 --- a/cancer_ai/utils/config.py +++ b/cancer_ai/utils/config.py @@ -237,12 +237,6 @@ def add_common_args(cls, parser): default="./config/competition_config.json", ) - parser.add_argument( - "--hf_repo_type", - type=str, - help="Hugging Face repository type to submit the model from.", - default="model", - ) def add_validator_args(cls, parser): """Add validator specific arguments to the parser.""" diff --git a/cancer_ai/validator/competition_handlers/base_handler.py b/cancer_ai/validator/competition_handlers/base_handler.py index c5d069ca..9227208a 100644 --- a/cancer_ai/validator/competition_handlers/base_handler.py +++ b/cancer_ai/validator/competition_handlers/base_handler.py @@ -20,6 +20,8 @@ class ModelEvaluationResult(BaseModel): score: float = 0.0 + error: str = "" + class Config: arbitrary_types_allowed = True diff --git a/cancer_ai/validator/competition_manager.py b/cancer_ai/validator/competition_manager.py index fc1966fc..02e6ab4f 100644 --- a/cancer_ai/validator/competition_manager.py +++ b/cancer_ai/validator/competition_manager.py @@ -13,8 +13,8 @@ from .model_run_manager import ModelRunManager from .exceptions import ModelRunException from .model_db import ModelDBController +from .utils import chain_miner_to_model_info -from cancer_ai.validator.models import WandBLogModelEntry from .competition_handlers.melanoma_handler import MelanomaCompetitionHandler from .competition_handlers.base_handler import ModelEvaluationResult from .tests.mock_data import get_mock_hotkeys_with_models @@ -74,6 +74,7 @@ def __init__( self.subtensor = subtensor self.competition_id = competition_id self.results: list[tuple[str, ModelEvaluationResult]] = [] + self.error_results: list[tuple[str, str]] = [] self.model_manager = ModelManager(self.config, db_controller) self.dataset_manager = DatasetManager( config=self.config, @@ -142,9 +143,8 @@ async def update_miner_models(self): self.hotkeys, self.competition_id, self.config.models_query_cutoff ) for hotkey, model in latest_models.items(): - try: - model_info = await self.chain_miner_to_model_info(model) - except ValueError: + model_info = chain_miner_to_model_info(model) + if model_info.competition_id != self.competition_id: bt.logging.warning( f"Miner {hotkey} with competition id {model.competition_id} does not belong to {self.competition_id} competition, skipping" ) @@ -174,59 +174,57 @@ async def evaluate(self) -> Tuple[str | None, ModelEvaluationResult | None]: X_test=X_test, y_test=y_test ) y_test = competition_handler.prepare_y_pred(y_test) + evaluation_counter = 0 + models_amount = len(self.model_manager.hotkey_store.items()) + bt.logging.info(f"Evaluating {models_amount} models") - # evaluate models for miner_hotkey, model_info in self.model_manager.hotkey_store.items(): - bt.logging.info(f"Evaluating hotkey: {miner_hotkey}") - model_or_none = await self.model_manager.download_miner_model(miner_hotkey) - if not model_or_none: + evaluation_counter +=1 + bt.logging.info(f"Evaluating {evaluation_counter}/{models_amount} hotkey: {miner_hotkey}") + model_downloaded = await self.model_manager.download_miner_model(miner_hotkey) + if not model_downloaded: bt.logging.error( f"Failed to download model for hotkey {miner_hotkey} Skipping." ) continue - try: - computed_hash = self._compute_model_hash(model_info.file_path) - except Exception as e: - bt.logging.error(f"Failed to compute model hash: {e} Skipping.") + computed_hash = self._compute_model_hash(model_info.file_path) + if not computed_hash: + bt.logging.info("Could not determine model hash. Skipping.") + self.error_results.append((miner_hotkey, "Could not determine model hash")) continue if computed_hash != model_info.model_hash: bt.logging.info(f"The hash of model uploaded by {miner_hotkey} does not match hash of model submitted on-chain. Slashing.") + self.error_results.append((miner_hotkey, "The hash of model uploaded does not match hash of model submitted on-chain")) hotkeys_to_slash.append(miner_hotkey) - try: - model_manager = ModelRunManager( - self.config, self.model_manager.hotkey_store[miner_hotkey] - ) - except ModelRunException as e: - bt.logging.error( - f"Model hotkey: {miner_hotkey} failed to initialize. Skipping. Error: {e}" - ) - continue + model_manager = ModelRunManager( + self.config, self.model_manager.hotkey_store[miner_hotkey] + ) start_time = time.time() try: y_pred = await model_manager.run(X_test) - except ModelRunException: + except ModelRunException as e: bt.logging.error( - f"Model hotkey: {miner_hotkey} failed to run. Skipping" + f"Model hotkey: {miner_hotkey} failed to run. Skipping. error: {e}" ) + self.error_results.append((miner_hotkey, f"Failed to run model: {e}")) continue - run_time_s = time.time() - start_time try: model_result = competition_handler.get_model_result( - y_test, y_pred, run_time_s + y_test, y_pred, time.time() - start_time ) self.results.append((miner_hotkey, model_result)) except Exception as e: bt.logging.error( f"Error evaluating model for hotkey: {miner_hotkey}. Error: {str(e)}" ) - import traceback - bt.logging.error(f"Stacktrace: {traceback.format_exc()}") - bt.logging.info(f"Skipping model {miner_hotkey} due to evaluation error") + self.error_results.append((miner_hotkey, f"Error evaluating model: {e}")) + bt.logging.info(f"Skipping model {miner_hotkey} due to evaluation error. error: {e}") + if len(self.results) == 0: bt.logging.error("No models were able to run") return None, None @@ -277,6 +275,7 @@ def slash_model_copiers(self, hotkeys_to_slash: list[str]): for hotkey, result in self.results: if hotkey in hotkeys_to_slash: bt.logging.info(f"Slashing model copier for hotkey: {hotkey} (setting score to 0.0)") + self.error_results.append((hotkey, "Slashing model copier - setting score to 0.0")) result.score = 0.0 def _compute_model_hash(self, file_path) -> str: diff --git a/cancer_ai/validator/model_db.py b/cancer_ai/validator/model_db.py index f2c35ddd..f64bae19 100644 --- a/cancer_ai/validator/model_db.py +++ b/cancer_ai/validator/model_db.py @@ -1,5 +1,6 @@ import bittensor as bt import os +import traceback from sqlalchemy import create_engine, Column, String, DateTime, PrimaryKeyConstraint, Integer from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.orm import sessionmaker @@ -86,8 +87,9 @@ def get_model(self, hotkey: str) -> ChainMinerModel | None: def get_latest_model(self, hotkey: str, cutoff_time: float = None) -> ChainMinerModel | None: cutoff_time = datetime.now(timezone.utc) - timedelta(minutes=cutoff_time) if cutoff_time else datetime.now(timezone.utc) - bt.logging.debug(f"Getting latest DB model for hotkey {hotkey}") + bt.logging.trace(f"Getting latest DB model for hotkey {hotkey}") session = self.Session() + model_record = None try: model_record = ( session.query(ChainMinerModelDB) @@ -96,19 +98,17 @@ def get_latest_model(self, hotkey: str, cutoff_time: float = None) -> ChainMiner .order_by(ChainMinerModelDB.date_submitted.desc()) .first() ) - if model_record: - return self.convert_db_model_to_chain_model(model_record) - return None except Exception as e: - import traceback - stack_trace = traceback.format_exc() - bt.logging.error(f"Error in get_latest_model for hotkey {hotkey}: {e}") - bt.logging.error(f"Stack trace: {stack_trace}") - # Re-raise the exception to be caught by higher-level error handlers + bt.logging.error(f"Error in get_latest_model for hotkey {hotkey}: {e}\n {traceback.format_exc()}") raise finally: session.close() + if not model_record: + return None + + return self.convert_db_model_to_chain_model(model_record) + def delete_model(self, date_submitted: datetime, hotkey: str): session = self.Session() try: diff --git a/cancer_ai/validator/model_manager.py b/cancer_ai/validator/model_manager.py index e489766c..b12bec67 100644 --- a/cancer_ai/validator/model_manager.py +++ b/cancer_ai/validator/model_manager.py @@ -1,30 +1,20 @@ import os from dataclasses import dataclass, asdict, is_dataclass +from typing import Optional from datetime import datetime, timezone, timedelta import bittensor as bt -from huggingface_hub import HfApi, hf_hub_url, HfFileSystem +from huggingface_hub import HfApi, HfFileSystem +from .models import ModelInfo from .manager import SerializableManager from .exceptions import ModelRunException -@dataclass -class ModelInfo: - hf_repo_id: str | None = None - hf_model_filename: str | None = None - hf_code_filename: str | None = None - hf_repo_type: str | None = None - - competition_id: str | None = None - file_path: str | None = None - model_type: str | None = None - block: int | None = None - model_hash: str | None = None class ModelManager(SerializableManager): - def __init__(self, config, db_controller) -> None: + def __init__(self, config, db_controller, parent: Optional["CompetitionManager"] = None) -> None: self.config = config self.db_controller = db_controller @@ -32,6 +22,7 @@ def __init__(self, config, db_controller) -> None: os.makedirs(self.config.models.model_dir) self.api = HfApi() self.hotkey_store: dict[str, ModelInfo] = {} + self.parent = parent def get_state(self): return {k: asdict(v) for k, v in self.hotkey_store.items() if is_dataclass(v)} @@ -57,6 +48,7 @@ async def download_miner_model(self, hotkey) -> bool: files = fs.ls(model_info.hf_repo_id) except Exception as e: bt.logging.error(f"Failed to list files in repository {model_info.hf_repo_id}: {e}") + self.parent.error_results.append((hotkey, f"Cannot list files in repo {model_info.hf_repo_id}")) return False # Find the specific file and its upload date @@ -69,11 +61,13 @@ async def download_miner_model(self, hotkey) -> bool: if not file_date: bt.logging.error(f"File {model_info.hf_model_filename} not found in repository {model_info.hf_repo_id}") + self.parent.error_results.append((hotkey, f"File {model_info.hf_model_filename} not found in repository {model_info.hf_repo_id}")) return False # Parse and check if the model is too recent to download is_too_recent, parsed_date = self.is_model_too_recent(file_date, model_info.hf_model_filename, hotkey) if is_too_recent: + self.parent.error_results.append((hotkey, f"Model is too recent")) return False file_date = parsed_date @@ -82,18 +76,20 @@ async def download_miner_model(self, hotkey) -> bool: try: model_info.file_path = self.api.hf_hub_download( repo_id=model_info.hf_repo_id, - repo_type=model_info.hf_repo_type, + repo_type="model", filename=model_info.hf_model_filename, cache_dir=self.config.models.model_dir, token=self.config.hf_token if hasattr(self.config, "hf_token") else None, ) except Exception as e: bt.logging.error(f"Failed to download model file: {e}") + self.parent.error_results.append((hotkey, f"Failed to download model file: {e}")) return False # Verify the downloaded file exists if not os.path.exists(model_info.file_path): bt.logging.error(f"Downloaded file does not exist at {model_info.file_path}") + self.parent.error_results.append((hotkey, f"Downloaded file does not exist at {model_info.file_path}")) return False bt.logging.info(f"Successfully downloaded model file to {model_info.file_path}") @@ -177,12 +173,14 @@ def get_pioneer_models(self, grouped_hotkeys: list[list[str]]) -> list[str]: model_info = self.hotkey_store.get(hotkey) if not model_info: bt.logging.error(f"Model info for hotkey {hotkey} not found.") + self.parent.error_results.append((hotkey, "Model info not found.")) continue try: files = fs.ls(model_info.hf_repo_id) except Exception as e: bt.logging.error(f"Failed to list files in {model_info.hf_repo_id}: {e}") + self.parent.error_results.append((hotkey, f"Cannot list files in repo {model_info.hf_repo_id}")) continue file_date_str = None @@ -195,6 +193,7 @@ def get_pioneer_models(self, grouped_hotkeys: list[list[str]]) -> list[str]: bt.logging.error( f"File {model_info.hf_model_filename} not found in {model_info.hf_repo_id} for {hotkey}" ) + self.parent.error_results.append((hotkey, "model file not found in hf repo.")) continue try: @@ -210,6 +209,7 @@ def get_pioneer_models(self, grouped_hotkeys: list[list[str]]) -> list[str]: except Exception as e: bt.logging.error(f"Failed to parse HF commit date {file_date_str} for {hotkey}: {e}") + self.parent.error_results.append((hotkey, "Failed to parse HF commit date.")) continue try: @@ -217,6 +217,7 @@ def get_pioneer_models(self, grouped_hotkeys: list[list[str]]) -> list[str]: block_timestamp = block_timestamp.replace(tzinfo=timezone.utc) except Exception as e: bt.logging.error(f"Failed to get block timestamp for {model_info.block}: {e}") + self.parent.error_results.append((hotkey, "Failed to get block timestamp.")) continue if hf_commit_date <= block_timestamp: diff --git a/cancer_ai/validator/model_manager_test.py b/cancer_ai/validator/model_manager_test.py index b4ced451..7ad5a1b5 100644 --- a/cancer_ai/validator/model_manager_test.py +++ b/cancer_ai/validator/model_manager_test.py @@ -1,11 +1,10 @@ +import os import pytest from types import SimpleNamespace from unittest.mock import patch, MagicMock from .model_manager import ( ModelManager, - ModelInfo, -) # Replace with the actual module name -import os +) hotkey = "test_hotkey" repo_id = "test_repo_id" diff --git a/cancer_ai/validator/models.py b/cancer_ai/validator/models.py index d695eb48..2a5d4cca 100644 --- a/cancer_ai/validator/models.py +++ b/cancer_ai/validator/models.py @@ -1,6 +1,7 @@ from typing import List, ClassVar, Optional, ClassVar, Optional from pydantic import BaseModel, EmailStr, Field, ValidationError from datetime import datetime +from dataclasses import dataclass class CompetitionModel(BaseModel): competition_id: str @@ -56,20 +57,31 @@ class NewDatasetFile(BaseModel): class WanDBLogBase(BaseModel): - """Base class for WandB log entries.""" + """Base class for WandB log entries""" + uuid: str # competition unique identifier log_type: str validator_hotkey: str - model_link: str + dataset_filename: str + competition_id: str + errors: str = "" - run_time: str = "" + run_time_s: float = 0.0 -class WandBLogModelEntry(WanDBLogBase): - """Model for logging model evaluation results to WandB. - """ +class WanDBLogModelBase(WanDBLogBase): log_type: str = "model_results" + uid: int miner_hotkey: str + + score: float = 0.0 + average_score: float = 0.0 + +class WandBLogModelEntry(WanDBLogModelBase): + """Individual model evaluation results""" + tested_entries: int + model_url : str + accuracy: float precision: float fbeta: float @@ -77,13 +89,31 @@ class WandBLogModelEntry(WanDBLogBase): confusion_matrix: list roc_curve: dict roc_auc: float - score: float - average_score: float = 0.0 + +class WanDBLogModelErrorEntry(WanDBLogModelBase): + pass + + +class WanDBLogCompetitionWinners(WanDBLogBase): + """Summary of competition""" + log_type: str = "competition_summary" + + competition_winning_hotkey: str + competition_winning_uid: int + + average_winning_hotkey: str + average_winning_uid: int + + +@dataclass +class ModelInfo: + hf_repo_id: str | None = None + hf_model_filename: str | None = None + hf_code_filename: str | None = None + hf_repo_type: str | None = None -class WanDBLogCompetitionWinner(WanDBLogBase): - """Model for logging competition winners to WandB. - Used in validator.py for logging competition evaluation results. - """ - log_type: str = "competition_result" - winning_hotkey: str = "" - winning_evaluation_hotkey: str = "" # Used in error case \ No newline at end of file + competition_id: str | None = None + file_path: str | None = None + model_type: str | None = None + block: int | None = None + model_hash: str | None = None diff --git a/cancer_ai/validator/utils.py b/cancer_ai/validator/utils.py index dfdaa2f0..62b950d3 100644 --- a/cancer_ai/validator/utils.py +++ b/cancer_ai/validator/utils.py @@ -11,9 +11,9 @@ from retry import retry from huggingface_hub import HfApi, hf_hub_download +from cancer_ai.chain_models_store import ChainMinerModel +from .models import ModelInfo from cancer_ai.validator.models import ( - CompetitionsListModel, - CompetitionModel, NewDatasetFile, OrganizationDataReferenceFactory, ) @@ -401,7 +401,7 @@ def get_local_dataset(local_dataset_dir: str) -> NewDatasetFile|None: shutil.move(filepath, os.path.join(already_released_dir, filename)) bt.logging.info(f"Successfully processed and moved {filename} to {already_released_dir}") return NewDatasetFile( - competition_id=random.choice(["melanoma-testnet","melanoma-1"]), + competition_id=random.choice(["melanoma-3"]), dataset_hf_repo="local", dataset_hf_filename=os.path.join(already_released_dir, filename), ) @@ -409,3 +409,15 @@ def get_local_dataset(local_dataset_dir: str) -> NewDatasetFile|None: bt.logging.error(f"Error processing {filename}: {e}") return None + + +def chain_miner_to_model_info(chain_miner_model: ChainMinerModel) -> ModelInfo: + return ModelInfo( + hf_repo_id=chain_miner_model.hf_repo_id, + hf_model_filename=chain_miner_model.hf_model_filename, + hf_code_filename=chain_miner_model.hf_code_filename, + hf_repo_type=chain_miner_model.hf_repo_type, + competition_id=chain_miner_model.competition_id, + block=chain_miner_model.block, + model_hash=chain_miner_model.model_hash, + ) \ No newline at end of file diff --git a/evaluation-results/.gitkeep b/evaluation-results/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/neurons/miner.py b/neurons/miner.py index ad5371e4..0bd18ded 100644 --- a/neurons/miner.py +++ b/neurons/miner.py @@ -14,7 +14,8 @@ import hashlib from cancer_ai.validator.utils import run_command -from cancer_ai.validator.model_run_manager import ModelRunManager, ModelInfo +from cancer_ai.validator.model_run_manager import ModelRunManager +from cancer_ai.validator.models import ModelInfo from cancer_ai.validator.dataset_manager import DatasetManager from cancer_ai.validator.competition_manager import COMPETITION_HANDLER_MAPPING @@ -148,13 +149,13 @@ async def compress_code(self) -> None: return bt.logging.info(f"Code zip path: {self.code_zip_path}") - def _compute_model_hash(self, repo_id, model_filename, repo_type): + def _compute_model_hash(self, repo_id, model_filename): """Compute an 8-character hexadecimal SHA-1 hash of the model file from Hugging Face.""" try: model_path = huggingface_hub.hf_hub_download( repo_id=repo_id, filename=model_filename, - repo_type=repo_type, + repo_type="model", ) sha1 = hashlib.sha1() with open(model_path, 'rb') as f: @@ -194,10 +195,6 @@ async def submit_model(self) -> None: subtensor=self.subtensor, netuid=self.config.netuid, wallet=self.wallet ) - if len(self.config.hf_repo_type.encode('utf-8')) > 7: - bt.logging.error("hf_repo_type must be 7 bytes or less") - return - if len(self.config.hf_repo_id.encode('utf-8')) > 32: bt.logging.error("hf_repo_id must be 32 bytes or less") return @@ -217,7 +214,7 @@ async def submit_model(self) -> None: return model_hash = self._compute_model_hash( - self.config.hf_repo_id, self.config.hf_model_name, self.config.hf_repo_type + self.config.hf_repo_id, self.config.hf_model_name ) if not model_hash: @@ -229,7 +226,7 @@ async def submit_model(self) -> None: competition_id=self.config.competition_id, hf_repo_id=self.config.hf_repo_id, hf_model_filename=self.config.hf_model_name, - hf_repo_type=self.config.hf_repo_type, + hf_repo_type="model", hf_code_filename=self.config.hf_code_filename, block=None, model_hash=model_hash, diff --git a/neurons/validator.py b/neurons/validator.py index bc08d327..549b1851 100644 --- a/neurons/validator.py +++ b/neurons/validator.py @@ -26,6 +26,7 @@ import datetime import csv import zipfile +from uuid import uuid4 import bittensor as bt import numpy as np @@ -44,7 +45,7 @@ from cancer_ai.validator.model_db import ModelDBController from cancer_ai.validator.competition_manager import CompetitionManager from cancer_ai.validator.models import OrganizationDataReferenceFactory, NewDatasetFile -from cancer_ai.validator.models import WandBLogModelEntry, WanDBLogCompetitionWinner +from cancer_ai.validator.models import WandBLogModelEntry, WanDBLogCompetitionWinners, WanDBLogBase, WanDBLogModelErrorEntry from huggingface_hub import HfApi BLACKLIST_FILE_PATH = "config/hotkey_blacklist.json" @@ -119,7 +120,7 @@ async def refresh_miners(self): except Exception as e: bt.logging.warning(f"Cannot get miner model for hotkey {hotkey} from the chain: {e}. Skipping.") continue - + try: self.db_controller.add_model(chain_model_metadata, hotkey) except Exception as e: @@ -130,7 +131,7 @@ async def refresh_miners(self): self.save_state() async def filesystem_test_evaluation(self): - time.sleep(1) + time.sleep(5) data_package = get_local_dataset(self.config.local_dataset_dir) if not data_package: bt.logging.error("NO NEW DATA PACKAGES") @@ -165,8 +166,9 @@ async def filesystem_test_evaluation(self): bt.logging.warning(f"No top hotkey available for competition {data_package.competition_id}") top_hotkey = None - # Enable if you want to have results in CSV for debugging purposes - # await self.log_results_to_csv(data_package, top_hotkey, models_results) + + results_file_name = f"{datetime.datetime.now().strftime('%Y-%m-%d_%H-%M-%S')}-{data_package.competition_id}.csv" + await self.log_results_to_csv(results_file_name, data_package, top_hotkey, models_results) if winning_hotkey: bt.logging.info(f"Competition result for {data_package.competition_id}: {winning_hotkey}") @@ -180,134 +182,135 @@ async def filesystem_test_evaluation(self): async def monitor_datasets(self): """Main validation logic, triggered by new datastes on huggingface""" - try: - if self.last_monitor_datasets is not None and ( + + if self.last_monitor_datasets is not None and ( time.time() - self.last_monitor_datasets < self.config.monitor_datasets_interval ): - return - self.last_monitor_datasets = time.time() + return + self.last_monitor_datasets = time.time() + bt.logging.info("Starting monitor_datasets") - bt.logging.info("Starting monitor_datasets") + try: yaml_data = await fetch_organization_data_references( self.config.datasets_config_hf_repo_id, self.hf_api, - ) - + ) await sync_organizations_data_references(yaml_data) - self.organizations_data_references = OrganizationDataReferenceFactory.get_instance() - self.save_state() - bt.logging.info("Fetched and synced organization data references") except Exception as e: - import traceback - stack_trace = traceback.format_exc() - bt.logging.error(f"Error in monitor_datasets initial setup: {e}") - bt.logging.error(f"Stack trace: {stack_trace}") + bt.logging.error(f"Error in monitor_datasets initial setup: {e}\n Stack trace: {traceback.format_exc()}") return - + + self.organizations_data_references = OrganizationDataReferenceFactory.get_instance() + bt.logging.info("Fetched and synced organization data references") + try: - list_of_new_data_packages: list[NewDatasetFile] = await check_for_new_dataset_files(self.hf_api, self.org_latest_updates) - self.save_state() - - if not list_of_new_data_packages: - bt.logging.info("No new data packages found.") - return - - bt.logging.info(f"Found {len(list_of_new_data_packages)} new data packages") + data_packages: list[NewDatasetFile] = await check_for_new_dataset_files(self.hf_api, self.org_latest_updates) except Exception as e: - import traceback stack_trace = traceback.format_exc() - bt.logging.error(f"Error checking for new dataset files: {e}") - bt.logging.error(f"Stack trace: {stack_trace}") + bt.logging.error(f"Error checking for new dataset files: {e}\n Stack trace: {stack_trace}") return - for data_package in list_of_new_data_packages: - try: - bt.logging.info(f"Starting competition for {data_package.competition_id}") - competition_manager = CompetitionManager( - config=self.config, - subtensor=self.subtensor, - hotkeys=self.hotkeys, - validator_hotkey=self.hotkey, - competition_id=data_package.competition_id, - dataset_hf_repo=data_package.dataset_hf_repo, - dataset_hf_filename=data_package.dataset_hf_filename, - dataset_hf_repo_type="dataset", - db_controller = self.db_controller, - test_mode = self.config.test_mode, - ) - except Exception as e: - import traceback - stack_trace = traceback.format_exc() - bt.logging.error(f"Error creating competition manager for {data_package.competition_id}: {e}") - bt.logging.error(f"Stack trace: {stack_trace}") - continue + if not data_packages: + bt.logging.info("No new data packages found.") + return + + bt.logging.info(f"Found {len(data_packages)} new data packages") + self.save_state() + + for data_package in data_packages: + competition_id = data_package.competition_id + competition_uuid = uuid4().hex + competition_start_time = datetime.datetime.now() + bt.logging.info(f"Starting competition for {competition_id}") + competition_manager = CompetitionManager( + config=self.config, + subtensor=self.subtensor, + hotkeys=self.hotkeys, + validator_hotkey=self.hotkey, + competition_id=competition_id, + dataset_hf_repo=data_package.dataset_hf_repo, + dataset_hf_filename=data_package.dataset_hf_filename, + dataset_hf_repo_type="dataset", + db_controller = self.db_controller, + test_mode = self.config.test_mode, + ) + winning_hotkey = None - winning_model_link = None try: - winning_hotkey, winning_model_result = ( - await competition_manager.evaluate() - ) - if not winning_hotkey: - continue + winning_hotkey, _ = await competition_manager.evaluate() - winning_model_link = self.db_controller.get_latest_model(hotkey=winning_hotkey, cutoff_time=self.config.models_query_cutoff).hf_link except Exception: - formatted_traceback = traceback.format_exc() - bt.logging.error(f"Error running competition: {formatted_traceback}") - wandb.init( - reinit=True, project="competition_id", group="competition_evaluation" - ) - - error_log = WanDBLogCompetitionWinner( - competition_id=data_package.competition_id, - winning_evaluation_hotkey="", - run_time="", + stack_trace = traceback.format_exc() + bt.logging.error(f"Cannot run {competition_id}: {stack_trace}") + wandb.init(project=competition_id, group="competition_evaluation") + error_log = WanDBLogBase( + uuid=competition_uuid, + competition_id=competition_id, + run_time_s=(datetime.datetime.now() - competition_start_time).seconds, validator_hotkey=self.wallet.hotkey.ss58_address, - model_link=winning_model_link, - errors=str(formatted_traceback) + errors=str(stack_trace), + dataset_filename=data_package.dataset_hf_filename ) wandb.log(error_log.model_dump()) wandb.finish() continue - wandb.init(project=data_package.competition_id, group="competition_evaluation") - + if not winning_hotkey: + bt.logging.warning("Could not determine the winner of competition") + continue + winning_model_link = self.db_controller.get_latest_model(hotkey=winning_hotkey, cutoff_time=self.config.models_query_cutoff).hf_link + - winner_log = WanDBLogCompetitionWinner( - competition_id=data_package.competition_id, - winning_hotkey=winning_hotkey, + # Update competition results + bt.logging.info(f"Competition result for {competition_id}: {winning_hotkey}") + competition_weights = await self.competition_results_store.update_competition_results(competition_id, competition_manager.results, self.config, self.metagraph.hotkeys, self.hf_api) + self.update_scores(competition_weights) + + average_winning_hotkey = self.competition_results_store.get_top_hotkey(competition_id) + winner_log = WanDBLogCompetitionWinners( + uuid=competition_uuid, + competition_id=competition_id, + + competition_winning_hotkey=winning_hotkey, + competition_winning_uid=self.metagraph.hotkeys.index(winning_hotkey), + + average_winning_hotkey=average_winning_hotkey, + average_winning_uid=self.metagraph.hotkeys.index(average_winning_hotkey), + validator_hotkey=self.wallet.hotkey.ss58_address, model_link=winning_model_link, - errors="" + dataset_filename=data_package.dataset_hf_filename, + run_time_s=(datetime.datetime.now() - competition_start_time).seconds ) + wandb.init(project=competition_id, group="competition_evaluation") wandb.log(winner_log.model_dump()) wandb.finish() - # Update competition results - bt.logging.info(f"Competition result for {data_package.competition_id}: {winning_hotkey}") - competition_weights = await self.competition_results_store.update_competition_results(data_package.competition_id, competition_manager.results, self.config, self.metagraph.hotkeys, self.hf_api) - self.update_scores(competition_weights) + # log results to CSV + csv_filename = f"{datetime.datetime.now().strftime('%Y-%m-%d_%H-%M-%S')}-{competition_id}.csv" + await self.log_results_to_csv(csv_filename, data_package, winning_hotkey, competition_manager.results) - # Logging results - + wandb.init(project=competition_id, group="model_evaluation") for miner_hotkey, evaluation_result in competition_manager.results: try: model = self.db_controller.get_latest_model( hotkey=miner_hotkey, cutoff_time=self.config.models_query_cutoff, ) - model_link = model.hf_link if model is not None else None - avg_score = 0.0 - if (data_package.competition_id in self.competition_results_store.average_scores and - miner_hotkey in self.competition_results_store.average_scores[data_package.competition_id]): - avg_score = self.competition_results_store.average_scores[data_package.competition_id][miner_hotkey] + if ( + data_package.competition_id in self.competition_results_store.average_scores and + miner_hotkey in self.competition_results_store.average_scores[competition_id] + ): + avg_score = self.competition_results_store.average_scores[competition_id][miner_hotkey] model_log = WandBLogModelEntry( - competition_id=data_package.competition_id, + uuid=competition_uuid, + competition_id=competition_id, miner_hotkey=miner_hotkey, + uid=self.metagraph.hotkeys.index(miner_hotkey), validator_hotkey=self.wallet.hotkey.ss58_address, tested_entries=evaluation_result.tested_entries, accuracy=evaluation_result.accuracy, @@ -319,19 +322,33 @@ async def monitor_datasets(self): "fpr": evaluation_result.fpr, "tpr": evaluation_result.tpr, }, - model_link=model_link, + model_url=model.hf_link, roc_auc=evaluation_result.roc_auc, score=evaluation_result.score, average_score=avg_score, - run_time_s=evaluation_result.run_time_s + + run_time_s=evaluation_result.run_time_s, + dataset_filename=data_package.dataset_hf_filename ) - wandb.init(project=data_package.competition_id, group="model_evaluation") wandb.log(model_log.model_dump()) - except Exception as e: bt.logging.error(f"Error logging model results for hotkey {miner_hotkey}: {e}") continue - wandb.finish() + + #logging errors + for miner_hotkey, error_message in competition_manager.error_results: + model_log = WanDBLogModelErrorEntry( + uuid=competition_uuid, + competition_id=competition_id, + miner_hotkey=miner_hotkey, + uid=self.metagraph.hotkeys.index(miner_hotkey), + validator_hotkey=self.wallet.hotkey.ss58_address, + dataset_filename=data_package.dataset_hf_filename, + errors=error_message, + ) + wandb.log(model_log.model_dump()) + + wandb.finish() def update_scores(self, competition_weights: dict[str, float]): """Update scores based on competition weights.""" @@ -355,13 +372,14 @@ def update_scores(self, competition_weights: dict[str, float]): bt.logging.debug(f"{self.scores}") self.save_state() - async def log_results_to_csv(self, data_package: NewDatasetFile, top_hotkey: str, models_results: list): + async def log_results_to_csv(self, file_name: str, data_package: NewDatasetFile, top_hotkey: str, models_results: list): """Debug method for dumping rewards for testing """ - - csv_file = "filesystem_test_evaluation_results.csv" - with open(csv_file, mode='a', newline='') as f: + + csv_file_path = os.path.join("evaluation-results", file_name) + bt.logging.info(f"Logging results to CSV for {data_package.competition_id} to file {csv_file_path}") + with open(csv_file_path, mode='a', newline='') as f: writer = csv.writer(f, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL) - if os.stat(csv_file).st_size == 0: + if os.stat(csv_file_path).st_size == 0: writer.writerow(["Package name", "Date", "Hotkey", "Score", "Average","Winner"]) competition_id = data_package.competition_id for hotkey, model_result in models_results: @@ -372,7 +390,7 @@ async def log_results_to_csv(self, data_package: NewDatasetFile, top_hotkey: str if hotkey == top_hotkey: writer.writerow([os.path.basename(data_package.dataset_hf_filename), - datetime.datetime.now(), + datetime.datetime.now(datetime.timezone.utc), hotkey, round(model_result.score, 6), round(avg_score, 6), From 45bba246373903e1fcc0e3a0e35d69c56f86110e Mon Sep 17 00:00:00 2001 From: Wojtek Jurkowlaniec Date: Thu, 17 Apr 2025 16:22:57 +0200 Subject: [PATCH 227/227] fixes --- cancer_ai/validator/competition_manager.py | 2 +- neurons/validator.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/cancer_ai/validator/competition_manager.py b/cancer_ai/validator/competition_manager.py index 02e6ab4f..7dc199d0 100644 --- a/cancer_ai/validator/competition_manager.py +++ b/cancer_ai/validator/competition_manager.py @@ -75,7 +75,7 @@ def __init__( self.competition_id = competition_id self.results: list[tuple[str, ModelEvaluationResult]] = [] self.error_results: list[tuple[str, str]] = [] - self.model_manager = ModelManager(self.config, db_controller) + self.model_manager = ModelManager(self.config, db_controller, parent=self) self.dataset_manager = DatasetManager( config=self.config, competition_id=competition_id, diff --git a/neurons/validator.py b/neurons/validator.py index 549b1851..a2a1d7ba 100644 --- a/neurons/validator.py +++ b/neurons/validator.py @@ -246,6 +246,7 @@ async def monitor_datasets(self): wandb.init(project=competition_id, group="competition_evaluation") error_log = WanDBLogBase( uuid=competition_uuid, + log_type="competition_error", competition_id=competition_id, run_time_s=(datetime.datetime.now() - competition_start_time).seconds, validator_hotkey=self.wallet.hotkey.ss58_address,