Merge pull request #69 from foundryservices/dev

Merge Dev into Staging
foundryservices · Jan 8, 2025 · 7b5da0f · 7b5da0f
2 parents 83661ab + f2e4bd6
commit 7b5da0f
Show file tree

Hide file tree

Showing 12 changed files with 933 additions and 689 deletions.
diff --git a/.gitignore b/.gitignore
@@ -128,6 +128,8 @@ venv/
 ENV/
 env.bak/
 venv.bak/
+# branch specific environments that start with issue number
+.[0-9]*-*
 
 # Spyder project settings
 .spyderproject
@@ -166,3 +168,5 @@ timestamp.txt
 # localnet config
 miner.config.local.js
 validator.config.local.js
+
+local_data/
diff --git a/Makefile b/Makefile
@@ -1,23 +1,24 @@
 ################################################################################
 #                               User Parameters                                #
 ################################################################################
-coldkey = default
-validator_hotkey = validator
-miner_hotkey = miner
-netuid = $(testnet_netuid)
-network = $(testnet)
-logging_level = info # options= ['info', 'debug', 'trace']
+# coldkey = validator
+coldkey = miner
+validator_hotkey = default
+miner_hotkey = default
+netuid = $(localnet_netuid)
+network = $(localnet)
+logging_level = debug # options= ['info', 'debug', 'trace']
 
 
 ################################################################################
 #                             Network Parameters                               #
 ################################################################################
 finney = wss://entrypoint-finney.opentensor.ai:443
 testnet = wss://test.finney.opentensor.ai:443
-locanet = ws://127.0.0.1:9944
+localnet = ws://127.0.0.1:9945
 
 finney_netuid = 28
-testnet_netuid = 93
+testnet_netuid = 272
 localnet_netuid = 1
 
 
@@ -41,7 +42,7 @@ validator:
 		--subtensor.chain_endpoint $(network) \
 		--axon.port 8091 \
 		--netuid $(netuid) \
-		--logging.level $(logging_level)
+		--logging.$(logging_level)
 
 miner:
 	pm2 start python --name miner  -- ./snp_oracle/neurons/miner.py \
@@ -50,7 +51,8 @@ miner:
 		--subtensor.chain_endpoint $(network) \
 		--axon.port 8092 \
 		--netuid $(netuid) \
-		--logging.level $(logging_level) \
+		--logging.$(logging_level) \
 		--vpermit_tao_limit 2 \
-		--hf_repo_id foundryservices/bittensor-sn28-base-lstm \
+		--blacklist.force_validator_permit true \
+		--hf_repo_id foundryservices/mining_models \
 		--model mining_models/base_lstm_new.h5
diff --git a/README.md b/README.md
@@ -7,7 +7,7 @@
 
 <div align="center">
 
-| This repository is the official codebase<br>for Bittensor Subnet 28 (SN28) v1.0.0+,<br>which was released on February 20th 2024. | **Testnet UID:**  93 <br> **Mainnet UID:**  28 |
+| This repository is the official codebase<br>for Bittensor Subnet 28 (SN28) v1.0.0+,<br>which was released on February 20th 2024. | **Testnet UID:**  272 <br> **Mainnet UID:**  28 |
 | - | - |
 
 </div>

diff --git a/docs/miners.md b/docs/miners.md
@@ -10,7 +10,7 @@
 
 <div align="center">
 
-| This repository is the official codebase<br>for Bittensor Subnet 28 (SN28) v1.0.0+,<br>which was released on February 20th 2024. | **Testnet UID:**  93 <br> **Mainnet UID:**  28 |
+| This repository is the official codebase<br>for Bittensor Subnet 28 (SN28) v1.0.0+,<br>which was released on February 20th 2024. | **Testnet UID:**  272 <br> **Mainnet UID:**  28 |
 | - | - |
 
 </div>

diff --git a/docs/validators.md b/docs/validators.md
@@ -2,7 +2,7 @@
 
 <div align="center">
 
-| This repository is the official codebase<br>for Bittensor Subnet 28 (SN28) v1.0.0+,<br>which was released on February 20th 2024. | **Testnet UID:**  93 <br> **Mainnet UID:**  28 |
+| This repository is the official codebase<br>for Bittensor Subnet 28 (SN28) v1.0.0+,<br>which was released on February 20th 2024. | **Testnet UID:**  272 <br> **Mainnet UID:**  28 |
 | - | - |
 
 </div>

diff --git a/poetry.lock b/poetry.lock
diff --git a/pyproject.toml b/pyproject.toml
@@ -13,12 +13,12 @@ readme = "README.md"
 python = ">3.9.1,<3.12"
 
 # Bittensor Version Strict
-bittensor = "7.4.0"
+bittensor = "8.5.1"
 
 # Bittensor Dependencies We Also Need
 setuptools = "~70.0.0"
 pydantic = "^2.3.0"
-numpy = "^1.26"
+numpy = ">=2.0.1,<2.1.0"
 
 # Subnet Specific Dependencies
 torch = "^2.5.1"
@@ -35,7 +35,7 @@ pandas-market-calendars = "^4.4.2"
 python-dotenv = "^1.0.1"
 scikit-learn = "^1.6.0"
 wandb = "^0.19.1"
-cryptography = ">=42.0.5,<42.1.0"
+cryptography = ">=43.0.1,<43.1.0"
 
 [tool.poetry.group.dev.dependencies]
 pre-commit-hooks = "5.0.0"

diff --git a/snp_oracle/base_miner/predict.py b/snp_oracle/base_miner/predict.py
@@ -10,9 +10,9 @@
 from snp_oracle.base_miner.model import create_and_save_base_model_lstm
 
 
-def predict(timestamp: datetime, scaler: MinMaxScaler, model, type) -> float:
+def predict(timestamp: datetime, scaler: MinMaxScaler, model, type: str) -> tuple[np.ndarray, pd.DataFrame]:
     """
-    Predicts the close price of the next 5m interval
+    Predicts the close price of the next 5m interval and returns the input data used for prediction.
 
     The predict function also ensures that the data is procured - using yahoo finance's python module,
     prepares the data and gets basic technical analysis metrics, and finally predicts the model
@@ -26,11 +26,14 @@ def predict(timestamp: datetime, scaler: MinMaxScaler, model, type) -> float:
         :type scaler: sklearn.preprocessing.MinMaxScaler
 
         :param model: The model used to make the predictions - in this case a .h5 file
-        :type model: A keras model instance
+        :type model: tensorflow.keras.Model
+
+        :param type: The type of model being used (e.g. "regression" or "lstm")
+        :type type: str
 
     Output:
-        :returns: The close price of the 5m period that ends at the timestamp passed by the validator
-        :rtype: float
+        :returns: A tuple containing (prediction array, input data used for prediction)
+        :rtype: tuple[numpy.ndarray, pandas.DataFrame]
     """
     # calling this to get the data - the information passed by the validator contains
     # only a timestamp, it is on the miners to get the data and prepare is according to their requirements
@@ -50,7 +53,7 @@ def predict(timestamp: datetime, scaler: MinMaxScaler, model, type) -> float:
     # Check if matching_row is empty
     if matching_row.empty:
         print("No matching row found for the given timestamp.")
-        return 0.0
+        return 0.0, pd.DataFrame()
 
     # data.to_csv('mining_models/base_miner_data.csv')
     input = matching_row[
@@ -67,6 +70,8 @@ def predict(timestamp: datetime, scaler: MinMaxScaler, model, type) -> float:
         ]
     ]
 
+    input_df = input.copy()
+
     if type != "regression":
         input = np.array(input, dtype=np.float32).reshape(1, -1)
         input = np.reshape(input, (1, 1, input.shape[1]))
@@ -77,7 +82,7 @@ def predict(timestamp: datetime, scaler: MinMaxScaler, model, type) -> float:
         prediction = scaler.inverse_transform(prediction.reshape(1, -1))
 
     print(prediction)
-    return prediction
+    return prediction, input_df
 
 
 # Uncomment this section if you wanna do a local test without having to run the miner
@@ -95,5 +100,5 @@ def predict(timestamp: datetime, scaler: MinMaxScaler, model, type) -> float:
 
     mse = create_and_save_base_model_lstm(scaler, X, y)
     model = load_model("mining_models/base_lstm_new.h5")
-    prediction = predict(timestamp, scaler, model, type="lstm")
+    prediction, _ = predict(timestamp, scaler, model, type="lstm")
     print(prediction[0])
diff --git a/snp_oracle/neurons/miner.py b/snp_oracle/neurons/miner.py
@@ -3,12 +3,12 @@
 import typing
 
 import bittensor as bt
+import tensorflow as tf
 from cryptography.fernet import Fernet
 
 # ML imports
 from dotenv import load_dotenv
 from huggingface_hub import hf_hub_download
-from tensorflow.keras.models import load_model
 
 import snp_oracle.predictionnet as predictionnet
 from snp_oracle.base_miner.get_data import prep_data, scale_data
@@ -29,8 +29,13 @@ class Miner(BaseMinerNeuron):
     """
 
     def __init__(self, config=None):
+        bt.logging.info("Initializing Miner...")
+        bt.logging.info(f"Initial config: {config}")
+
         super(Miner, self).__init__(config=config)
-        print(config)
+
+        bt.logging.info(f"Config after super init: {self.config}")
+        bt.logging.info(f"Config model path: {self.config.model if self.config else 'No config'}")
         # TODO(developer): Anything specific to your use case you can do here
         self.model_loc = self.config.model
         if self.config.neuron.device == "cpu":
@@ -162,7 +167,7 @@ async def forward(self, synapse: predictionnet.protocol.Challenge) -> prediction
             f"👈 Received prediction request from: {synapse.dendrite.hotkey} for timestamp: {synapse.timestamp}"
         )
 
-        model_filename = f"{self.wallet.hotkey.ss58_address}{os.path.splitext(self.config.model)[1]}"
+        model_filename = f"{self.wallet.hotkey.ss58_address}/models/model{os.path.splitext(self.config.model)[1]}"
 
         timestamp = synapse.timestamp
         synapse.repo_id = self.config.hf_repo_id
@@ -184,17 +189,24 @@ async def forward(self, synapse: predictionnet.protocol.Challenge) -> prediction
             )
             bt.logging.info(f"Model downloaded from huggingface at {model_path}")
 
-        model = load_model(model_path)
+        model = tf.keras.models.load_model(model_path)
         data = prep_data()
 
+        scaler, _, _ = scale_data(data)
+        # mse = create_and_save_base_model_lstm(scaler, X, y)
+
+        # type needs to be changed based on the algo you're running
+        # any algo specific change logic can be added to predict function in predict.py
+        prediction, input_df = predict(timestamp, scaler, model, type="lstm")
+
         # Generate encryption key for this request
         encryption_key = Fernet.generate_key()
 
         # Upload encrypted data to HuggingFace
         hf_interface = MinerHfInterface(self.config)
         success, metadata = hf_interface.upload_data(
             hotkey=self.wallet.hotkey.ss58_address,
-            data=data,
+            data=input_df,
             repo_id=self.config.hf_repo_id,
             encryption_key=encryption_key,
         )
@@ -208,12 +220,6 @@ async def forward(self, synapse: predictionnet.protocol.Challenge) -> prediction
         else:
             bt.logging.error(f"Data upload failed: {metadata['error']}")
 
-        scaler, _, _ = scale_data(data)
-        # mse = create_and_save_base_model_lstm(scaler, X, y)
-
-        # type needs to be changed based on the algo you're running
-        # any algo specific change logic can be added to predict function in predict.py
-        prediction = predict(timestamp, scaler, model, type="lstm")
         bt.logging.info(f"Prediction: {prediction}")
         # pred_np_array = np.array(prediction).reshape(-1, 1)
 
@@ -237,17 +243,27 @@ def print_info(self):
         metagraph = self.metagraph
         self.uid = self.metagraph.hotkeys.index(self.wallet.hotkey.ss58_address)
 
-        log = (
-            "Miner | "
-            f"Step:{self.step} | "
-            f"UID:{self.uid} | "
-            f"Block:{self.block} | "
-            f"Stake:{metagraph.S[self.uid]:.4f} | "
-            f"Trust:{metagraph.T[self.uid]:.4f} | "
-            f"Incentive:{metagraph.I[self.uid]:.4f} | "
-            f"Emission:{metagraph.E[self.uid]:.4f}"
-        )
-        bt.logging.info(log)
+        # Get all values in one go to avoid multiple concurrent requests
+        try:
+            current_block = self.block  # Single websocket call
+            stake = float(metagraph.S[self.uid])
+            trust = float(metagraph.T[self.uid])
+            incentive = float(metagraph.I[self.uid])
+            emission = float(metagraph.E[self.uid])
+
+            log = (
+                "Miner | "
+                f"Step:{self.step} | "
+                f"UID:{self.uid} | "
+                f"Block:{current_block} | "
+                f"Stake:{stake:.4f} | "
+                f"Trust:{trust:.4f} | "
+                f"Incentive:{incentive:.4f} | "
+                f"Emission:{emission:.4f}"
+            )
+            bt.logging.info(log)
+        except Exception as e:
+            bt.logging.error(f"Error getting miner info: {e}")
 
 
 # This is the main function, which runs the miner.

diff --git a/snp_oracle/neurons/validator.py b/snp_oracle/neurons/validator.py
@@ -129,17 +129,27 @@ def print_info(self):
         metagraph = self.metagraph
         self.uid = self.metagraph.hotkeys.index(self.wallet.hotkey.ss58_address)
 
-        log = (
-            "Validator | "
-            f"Step:{self.step} | "
-            f"UID:{self.uid} | "
-            f"Block:{self.block} | "
-            f"Stake:{metagraph.S[self.uid]} | "
-            f"VTrust:{metagraph.Tv[self.uid]} | "
-            f"Dividend:{metagraph.D[self.uid]} | "
-            f"Emission:{metagraph.E[self.uid]}"
-        )
-        bt.logging.info(log)
+        # Get all values in one go to avoid multiple concurrent requests
+        try:
+            current_block = self.block  # Single websocket call
+            stake = float(metagraph.S[self.uid])
+            vtrust = float(metagraph.Tv[self.uid])
+            dividend = float(metagraph.D[self.uid])
+            emission = float(metagraph.E[self.uid])
+
+            log = (
+                "Validator | "
+                f"Step:{self.step} | "
+                f"UID:{self.uid} | "
+                f"Block:{current_block} | "
+                f"Stake:{stake:.4f} | "
+                f"VTrust:{vtrust:.4f} | "
+                f"Dividend:{dividend:.4f} | "
+                f"Emission:{emission:.4f}"
+            )
+            bt.logging.info(log)
+        except Exception as e:
+            bt.logging.error(f"Error getting validator info: {e}")
 
 
 # The main function parses the configuration and runs the validator.