260 require data input

.env.template

@@ -1,4 +1,10 @@
 # Copy the contents on this file to a new file called .env
+WANDB_API_KEY='REPLACE_WITH_WANDB_API_KEY'
+MINER_HF_ACCESS_TOKEN='REPLACE_WITH_HUGGINGFACE_ACCESS_KEY'
+HF_ACCESS_TOKEN='REPLACE_WITH_HUGGINGFACE_ACCESS_KEY'
 
-HF_ACCESS_TOKEN = 'REPLACE_WITH_HUGGINGFACE_ACCESS_KEY'
-WANDB_API_KEY = 'REPLACE_WITH_WANDB_API_KEY'
+# Git credentials
+GIT_TOKEN='REPLACE_WITH_GIT_TOKEN'
+GIT_USERNAME='REPLACE_WITH_GIT_USERNAME'
+GIT_NAME="REPLACE_WITH_GIT_NAME"
+GIT_EMAIL="REPLACE_WITH_GIT_EMAIL"

miner.config.js

@@ -3,7 +3,7 @@ module.exports = {
     {
       name: 'miner',
       script: 'python3',
-      args: './neurons/miner.py --netuid 28 --logging.debug --logging.trace --subtensor.network local --wallet.name walletName --wallet.hotkey hotkeyName --axon.port 8091 --hf_repo_id foundryservices/bittensor-sn28-base-lstm --model mining_models/base_lstm_new.h5'
+      args: './snp_oracle/neurons/miner.py --netuid 28 --logging.debug --logging.trace --subtensor.network local --wallet.name walletName --wallet.hotkey hotkeyName --axon.port 8091 --hf_repo_id foundryservices/bittensor-sn28-base-lstm --model mining_models/base_lstm_new.h5'
     },
   ],
 };

pyproject.toml

@@ -30,6 +30,7 @@ tensorflow = "^2.18.0"
 yfinance = "^0.2.50"
 huggingface-hub = "^0.27.0"
 loguru = "^0.7.3"
+pyarrow = "^18.1.0"
 pandas-market-calendars = "^4.4.2"
 python-dotenv = "^1.0.1"
 scikit-learn = "^1.6.0"

snp_oracle/base_miner/get_data.py

@@ -29,7 +29,7 @@ def prep_data(drop_na: bool = True) -> DataFrame:
     """
     # Fetch S&P 500 data - when capturing data any interval, the max we can go back is 60 days
     # using Yahoo Finance's Python SDK
-    data = yf.download("^GSPC", period="60d", interval="5m")
+    data = yf.download("^GSPC", period="1m", interval="5m")
 
     # Calculate technical indicators - all technical indicators computed here are based on the 5m data
     # For example - SMA_50, is not a 50-day moving average, but is instead a 50 5m moving average

snp_oracle/neurons/miner.py

@@ -3,6 +3,9 @@
 import typing
 
 import bittensor as bt
+from cryptography.fernet import Fernet
+
+# ML imports
 from dotenv import load_dotenv
 from huggingface_hub import hf_hub_download
 from tensorflow.keras.models import load_model
@@ -183,6 +186,28 @@ async def forward(self, synapse: predictionnet.protocol.Challenge) -> prediction
 
         model = load_model(model_path)
         data = prep_data()
+
+        # Generate encryption key for this request
+        encryption_key = Fernet.generate_key()
+
+        # Upload encrypted data to HuggingFace
+        hf_interface = MinerHfInterface(self.config)
+        success, metadata = hf_interface.upload_data(
+            hotkey=self.wallet.hotkey.ss58_address,
+            data=data,
+            repo_id=self.config.hf_repo_id,
+            encryption_key=encryption_key,
+        )
+
+        if success:
+            bt.logging.success(f"Encrypted data uploaded successfully to {metadata['data_path']}")
+            synapse.data = metadata["data_path"]  # Store the data path in synapse
+            synapse.decryption_key = encryption_key  # Provide key to validator
+            bt.logging.info(f"synapse.decryption_key: {synapse.decryption_key}")
+            bt.logging.info(f"synapse.data: {synapse.data}")
+        else:
+            bt.logging.error(f"Data upload failed: {metadata['error']}")
+
         scaler, _, _ = scale_data(data)
         # mse = create_and_save_base_model_lstm(scaler, X, y)
 

snp_oracle/predictionnet/protocol.py

@@ -6,13 +6,16 @@
 
 class Challenge(bt.Synapse):
     """
-    A simple dummy protocol representation which uses bt.Synapse as its base.
-    This protocol helps in handling dummy request and response communication between
-    the miner and the validator.
+    Protocol for handling encrypted prediction challenges between miners and validators.
+    Includes secure handling of decryption keys and manages model/data references.
 
     Attributes:
-    - dummy_input: An integer value representing the input request sent by the validator.
-    - dummy_output: An optional integer value which, when filled, represents the response from the miner.
+        repo_id: Repository identifier where the model is stored
+        model: Identifier for the specific model to use
+        decryption_key: Securely stored key for decrypting data/models
+        data: Identifier for the data to be used
+        timestamp: Time at which the validation is taking place
+        prediction: List of predicted values for next 6 5m candles
     """
 
     repo_id: Optional[str] = pydantic.Field(
@@ -27,34 +30,45 @@ class Challenge(bt.Synapse):
         description="Which model to use",
     )
 
-    # Required request input, filled by sending dendrite caller.
+    decryption_key: Optional[bytes] = pydantic.Field(
+        default=None,
+        title="Decryption Key",
+        description="Secure key for decrypting sensitive data/models",
+    )
+
+    data: Optional[str] = pydantic.Field(
+        default=None,
+        title="Data ID",
+        description="Which data to use",
+    )
+
     timestamp: str = pydantic.Field(
         ...,
         title="Timestamp",
         description="The time stamp at which the validation is taking place for",
         allow_mutation=False,
     )
-    # Optional request output, filled by recieving axon.
+
     prediction: Optional[List[float]] = pydantic.Field(
         default=None,
         title="Predictions",
         description="Next 6 5m candles' predictions for closing price of S&P 500",
     )
 
-    def deserialize(self) -> int:
+    def deserialize(self) -> List[float]:
         """
-        Deserialize the dummy output. This method retrieves the response from
-        the miner in the form of dummy_output, deserializes it and returns it
-        as the output of the dendrite.query() call.
+        Deserialize the prediction output from the miner.
 
         Returns:
-        - int: The deserialized response, which in this case is the value of dummy_output.
-
-        Example:
-        Assuming a Dummy instance has a dummy_output value of 5:
-        >>> dummy_instance = Dummy(dummy_input=4)
-        >>> dummy_instance.dummy_output = 5
-        >>> dummy_instance.deserialize()
-        5
+            List[float]: The deserialized predictions for the next 6 5m candles.
         """
         return self.prediction
+
+    def get_decryption_key(self) -> Optional[bytes]:
+        """
+        Safely retrieve the decryption key when needed.
+
+        Returns:
+            Optional[bytes]: The raw Fernet key bytes if set, None otherwise.
+        """
+        return self.decryption_key

snp_oracle/predictionnet/utils/config.py

@@ -138,6 +138,13 @@ def add_args(cls, parser):
             default=4096,
         )
 
+        parser.add_argument(
+            "--neuron.organization",
+            type=str,
+            help="HuggingFace organization name for dataset storage",
+            default="foundryservices",
+        )
+
     # MINER ONLY CONFIG
     else:
         bt.logging.debug("Adding miner-specific arguments")