diff --git a/examples/command/portals/snowflake/example-7/README.md b/examples/command/portals/snowflake/example-7/README.md new file mode 100644 index 00000000000..1eb91d92e6e --- /dev/null +++ b/examples/command/portals/snowflake/example-7/README.md @@ -0,0 +1,363 @@ +# Access Microsoft SQL Server from Snowpark Container Services + +![Architecture](./diagram.png) + +## Get started with Ockam + +[Signup for Ockam](https://www.ockam.io/signup) and then run the following commands on your workstation: + +```sh +# Install Ockam Command +curl --proto '=https' --tlsv1.2 -sSfL https://install.command.ockam.io | bash && source "$HOME/.ockam/env" + +# Enroll with Ockam Orchestrator. +ockam enroll + +# Create an enrollment ticket for the node that will run in snowpark container services +ockam project ticket --usage-count 1 --expires-in 4h --attribute snowflake > snowflake_inlet.ticket + +# Create an enrollemnt ticket for the node that will run from a linux machine where thesql server is reachable from +ockam project ticket --usage-count 1 --expires-in 4h --attribute mssql --relay mssql > mssql_outlet.ticket + +# Note egress allow list for ockam. This will be used to create a network rule in snowflake. +ockam project show --jq .egress_allow_list + +``` + +## Setup Ockam node next to MS SQL Server + +- Copy `setup_ockam_outlet.sh` to the linux machine where the MS SQL Server is reachable from. +- Copy `mssql_outlet.ticket` to the same location as `setup_ockam_outlet.sh` script + +```sh +# Run the setup script +chmod +x setup_ockam_outlet.sh +DB_ENDPOINT="HOST:1433" ./setup_ockam_outlet.sh +``` + +## Setup Snowflake + + +```sql +USE ROLE ACCOUNTADMIN; + +--Create Role +CREATE ROLE MSSQL_API_ROLE; +GRANT ROLE MSSQL_API_ROLE TO ROLE ACCOUNTADMIN; + +--Create Database +CREATE DATABASE IF NOT EXISTS MSSQL_API_DB; +GRANT OWNERSHIP ON DATABASE MSSQL_API_DB TO ROLE MSSQL_API_ROLE COPY CURRENT GRANTS; + +USE DATABASE MSSQL_API_DB; + +--Create Warehouse +CREATE OR REPLACE WAREHOUSE MSSQL_API_WH WITH WAREHOUSE_SIZE='X-SMALL'; +GRANT USAGE ON WAREHOUSE MSSQL_API_WH TO ROLE MSSQL_API_ROLE; + +--Create compute pool +CREATE COMPUTE POOL MSSQL_API_CP + MIN_NODES = 1 + MAX_NODES = 5 + INSTANCE_FAMILY = CPU_X64_XS; + +GRANT USAGE ON COMPUTE POOL MSSQL_API_CP TO ROLE MSSQL_API_ROLE; +GRANT MONITOR ON COMPUTE POOL MSSQL_API_CP TO ROLE MSSQL_API_ROLE; + + +--Wait till compute pool is in idle or ready state +DESCRIBE COMPUTE POOL MSSQL_API_CP; + +--Create schema +CREATE SCHEMA IF NOT EXISTS MSSQL_API_SCHEMA; +GRANT ALL PRIVILEGES ON SCHEMA MSSQL_API_SCHEMA TO ROLE MSSQL_API_ROLE; + + +--Create Image Repository +CREATE IMAGE REPOSITORY IF NOT EXISTS MSSQL_API_REPOSITORY; +GRANT READ ON IMAGE REPOSITORY MSSQL_API_REPOSITORY TO ROLE MSSQL_API_ROLE; + +--Note repository_url value to be used to build and publish consumer image to snowflake +SHOW IMAGE REPOSITORIES; + +``` + +## Push Ockam docker image and MS SQL Server client docker image + +```sh +cd mssql_client + +#TODO +hycwvdm-test-account.registry.snowflakecomputing.com/mssql_api_db/mssql_api_schema/mssql_api_repository + +# Use the repository_url +docker login + +docker buildx build --platform linux/amd64 --load -t /mssql_client:latest . + +docker push /mssql_client:latest + +# Push Ockam +docker pull ghcr.io/build-trust/ockam:0.146.0@sha256:b13ed188dbde6f5cae9d2c9c9e9305f9c36a009b1e5c126ac0d066537510f895 + +docker tag ghcr.io/build-trust/ockam:0.146.0@sha256:b13ed188dbde6f5cae9d2c9c9e9305f9c36a009b1e5c126ac0d066537510f895 /ockam:latest + +docker push /ockam:latest + +cd - +``` + +## Create an Ockam node and API Client to connect to MS SQL Server in Snowpark Container Services + +> [!IMPORTANT] +> Replace `TODO` values in `VALUE_LIST` with the output of `ockam project show --jq .egress_allow_list` command in previous step. + +```sh +#Example +VALUE_LIST = ("XXXXX-*.projects.orchestrator.ockam.io:443"); +``` + +> [!IMPORTANT] +> Replace `` with contents of `snowflake_inlet.ticket` generated in previous step +> Replace `TODO` values in `MSSQL_DATABASE`, `MSSQL_USER`, `MSSQL_PASSWORD` with values from MS SQL Server. Make sure the user has access to the database and perform the necessary operations. + +- Create a network rule to allow the Ockam node to connect to your ockam project. + +```sql +USE ROLE ACCOUNTADMIN; + +-- Update VALUE_LIST with ockam egress details +CREATE NETWORK RULE OCKAM_OUT TYPE = 'HOST_PORT' MODE = 'EGRESS' +VALUE_LIST = ("TODO"); + +CREATE OR REPLACE EXTERNAL ACCESS INTEGRATION OCKAM +ALLOWED_NETWORK_RULES = (OCKAM_OUT) ENABLED = true; + +GRANT USAGE ON INTEGRATION OCKAM TO ROLE MSSQL_API_ROLE; + +``` + +- Setup Service + +```sql +USE ROLE MSSQL_API_ROLE; +USE DATABASE MSSQL_API_DB; +USE WAREHOUSE MSSQL_API_WH; +USE SCHEMA MSSQL_API_SCHEMA; + + +CREATE OR REPLACE NETWORK RULE OCSP_OUT +TYPE = 'HOST_PORT' MODE= 'EGRESS' +VALUE_LIST = ('ocsp.snowflakecomputing.com:80'); + +-- Create access integration + +USE ROLE ACCOUNTADMIN; +GRANT CREATE INTEGRATION ON ACCOUNT TO ROLE MSSQL_API_ROLE; + +CREATE OR REPLACE EXTERNAL ACCESS INTEGRATION OCSP +ALLOWED_NETWORK_RULES = (OCSP_OUT) +ENABLED = true; + +GRANT USAGE ON INTEGRATION OCSP TO ROLE MSSQL_API_ROLE; + +-- Create Service. Make sure to replace variables. + +CREATE SERVICE MSSQL_API_CLIENT + IN COMPUTE POOL MSSQL_API_CP + FROM SPECIFICATION +$$ + spec: + endpoints: + - name: http-endpoint + port: 8080 + public: false + protocol: HTTP + - name: ockam-inlet + port: 1443 + public: false + protocol: TCP + containers: + - name: ockam-inlet + image: /mssql_api_db/MSSQL_API_SCHEMA/mssql_api_repository/ockam + env: + OCKAM_DISABLE_UPGRADE_CHECK: true + OCKAM_TELEMETRY_EXPORT: false + args: + - node + - create + - --foreground + - --enrollment-ticket + - "" + - --configuration + - | + tcp-inlet: + from: 0.0.0.0:1433 + via: mssql + allow: mssql + - name: http-endpoint + image: /mssql_api_db/mssql_api_schema/mssql_api_repository/mssql_client + env: + SNOWFLAKE_WAREHOUSE: MSSQL_API_WH + MSSQL_DATABASE: 'TODO' + MSSQL_USER: 'TODO' + MSSQL_PASSWORD: 'TODO' + resources: + requests: + cpu: 0.5 + memory: 128M + limits: + cpu: 1 + memory: 256M +$$ +MIN_INSTANCES=1 +MAX_INSTANCES=1 +EXTERNAL_ACCESS_INTEGRATIONS = (OCSP, OCKAM); + +-- Check service status +SHOW SERVICES; +SELECT SYSTEM$GET_SERVICE_STATUS('MSSQL_API_CLIENT'); + +-- Check service logs +CALL SYSTEM$GET_SERVICE_LOGS('MSSQL_API_CLIENT', '0', 'http-endpoint', 100); +CALL SYSTEM$GET_SERVICE_LOGS('MSSQL_API_CLIENT', '0', 'ockam-inlet', 100); + +``` +> [!IMPORTANT] +> - `http-endpoint` is the endpoint that will be used to connect to the MS SQL Server. You will see `Successfully connected to SQL Server` in the logs upon successful connection. +> - `ockam-inlet` is the endpoint that will be used to connect to the Ockam node. Logs will indicate if there are any errors starting the node. + +## Stored procedures to use the API + +```sql +-- These functions are dependent the service, and needs to be created after the service is created + +USE ROLE ACCOUNTADMIN; + +-- Query +CREATE OR REPLACE FUNCTION _ockam_query_mssql(query STRING) + RETURNS VARCHAR + CALLED ON NULL INPUT + VOLATILE + SERVICE = MSSQL_API_CLIENT + ENDPOINT = 'http-endpoint' + AS '/query'; + + +CREATE OR REPLACE PROCEDURE ockam_mssql_query(QUERY STRING) + RETURNS TABLE() + LANGUAGE PYTHON + RUNTIME_VERSION = '3.11' + PACKAGES = ('snowflake-snowpark-python') + HANDLER = 'wrap_query' + EXECUTE AS OWNER +AS ' +import json + +def wrap_query(session, query): + data = json.loads(session.sql(f"SELECT _ockam_query_mssql($${query}$$);").collect()[0][0]) + keys = data[0] + values = data[1:] + return session.create_dataframe(values).to_df(keys) +'; + +-- Execute Statement +CREATE OR REPLACE FUNCTION _ockam_mssql_execute(QUERY STRING) + RETURNS VARCHAR + CALLED ON NULL INPUT + VOLATILE + SERVICE = MSSQL_API_CLIENT + ENDPOINT = 'http-endpoint' + AS '/execute'; + + +CREATE OR REPLACE PROCEDURE ockam_mssql_execute(QUERY STRING) + RETURNS STRING + LANGUAGE PYTHON + RUNTIME_VERSION = '3.11' + PACKAGES = ('snowflake-snowpark-python') + HANDLER = 'wrap_execute' + EXECUTE AS OWNER +AS ' +def wrap_execute(session, query): + return session.sql(f"SELECT _ockam_mssql_execute($${query}$$);").collect()[0][0] +'; + +-- Insert Statement +CREATE OR REPLACE FUNCTION ockam_mssql_insert(QUERY STRING, ENTRIES ARRAY) + RETURNS VARCHAR + CALLED ON NULL INPUT + VOLATILE + SERVICE = MSSQL_API_CLIENT + ENDPOINT = 'http-endpoint' + AS '/insert'; + +``` + +## Test the API + +__Execute a statement__ +```sql + +USE ROLE ACCOUNTADMIN; + +CALL OCKAM_MSSQL_EXECUTE($$ +IF NOT EXISTS (SELECT * FROM sys.tables WHERE name = 'PETS') +BEGIN + CREATE TABLE PETS ( + NAME NVARCHAR(100), + BREED NVARCHAR(100) + ); +END +$$); +``` + +__Insert data__ +```sql +CALL OCKAM_MSSQL_EXECUTE($$ +INSERT INTO PETS VALUES ('Toby', 'Beagle'); +$$); +``` + +__Query the PostgreSQL database__ +```sql +CALL OCKAM_MSSQL_QUERY('SELECT * FROM PETS'); +``` + +__Join PostgreSQL and Snowflake tables__ +```sql +-- Create two similar tables `PETS`, in MS SQL Server and Snowflake, but with different fields + +-- Create the table in MS SQL Server +CALL OCKAM_MSSQL_EXECUTE($$ +IF OBJECT_ID('PETS', 'U') IS NOT NULL + DROP TABLE PETS; + +IF NOT EXISTS (SELECT * FROM sys.tables WHERE name = 'PETS') +BEGIN + CREATE TABLE PETS ( + NAME NVARCHAR(100), + BREED NVARCHAR(100) + ); +END; + +INSERT INTO PETS VALUES ('Max', 'Golden Retriever'); +INSERT INTO PETS VALUES ('Bella', 'Poodle'); +$$); + +-- Create the table in Snowflake + +CREATE TABLE IF NOT EXISTS PETS (NAME VARCHAR(100), YEAR_OF_BIRTH INT); +INSERT INTO PETS VALUES ('Max', 2018); +INSERT INTO PETS VALUES ('Bella', 2019); + + +-- First we query the remote MS SQL Server +CALL OCKAM_MSSQL_QUERY('SELECT * FROM PETS'); +SET pets_query=LAST_QUERY_ID(); + +-- Join the results of the MS SQL Server query with the Snowflake table +SELECT * FROM TABLE(RESULT_SCAN($pets_query)) as MSSQL_PETS + INNER JOIN PETS ON MSSQL_PETS.NAME = PETS.NAME; +``` + diff --git a/examples/command/portals/snowflake/example-7/diagram.png b/examples/command/portals/snowflake/example-7/diagram.png new file mode 100644 index 00000000000..d5aaa542c70 Binary files /dev/null and b/examples/command/portals/snowflake/example-7/diagram.png differ diff --git a/examples/command/portals/snowflake/example-7/mssql_client/Dockerfile b/examples/command/portals/snowflake/example-7/mssql_client/Dockerfile new file mode 100644 index 00000000000..be3bded0811 --- /dev/null +++ b/examples/command/portals/snowflake/example-7/mssql_client/Dockerfile @@ -0,0 +1,12 @@ +ARG BASE_IMAGE=python:3.10-slim-buster +FROM $BASE_IMAGE + +RUN pip install --upgrade pip && \ + pip install flask snowflake snowflake-connector-python + +RUN pip install pymssql==2.2.11 + +COPY service.py ./ +COPY connection.py ./ + +CMD ["python3", "service.py"] diff --git a/examples/command/portals/snowflake/example-7/mssql_client/connection.py b/examples/command/portals/snowflake/example-7/mssql_client/connection.py new file mode 100644 index 00000000000..fcf4c6e58c5 --- /dev/null +++ b/examples/command/portals/snowflake/example-7/mssql_client/connection.py @@ -0,0 +1,52 @@ +import os +import logging +import snowflake.connector +from snowflake.snowpark import Session + +def session() -> Session: + """ + Create a session for the connection + :return: Session + """ + logging.info(f"Create a session") + return Session.builder.configs({"connection": connection()}).create() + + +def connection() -> snowflake.connector.SnowflakeConnection: + """ + Create a connection, either from inside the native application when deployed + or with user/password credentials when testing locally + + :return: A SnowflakeConnection + """ + logging.info(f"Create a connection") + if os.path.isfile("/snowflake/session/token"): + logging.info(f"Use an OAUTH token") + creds = { + "account": os.getenv("SNOWFLAKE_ACCOUNT"), + "host": os.getenv("SNOWFLAKE_HOST"), + "port": os.getenv("SNOWFLAKE_PORT"), + "protocol": "https", + "warehouse": os.getenv("SNOWFLAKE_WAREHOUSE"), + "database": os.getenv("SNOWFLAKE_DATABASE"), + "schema": os.getenv("SNOWFLAKE_SCHEMA"), + "role": os.getenv("SNOWFLAKE_ROLE"), + "authenticator": "oauth", + "token": open("/snowflake/session/token", "r").read(), + "client_session_keep_alive": True, + "ocsp_fail_open": False, + "validate_default_parameters": True, + } + logging.info(f"the creds are {creds}") + else: + creds = { + "account": os.getenv("SNOWFLAKE_ACCOUNT"), + "user": os.getenv("SNOWFLAKE_USER"), + "password": os.getenv("SNOWFLAKE_PASSWORD"), + "warehouse": os.getenv("SNOWFLAKE_WAREHOUSE"), + "database": os.getenv("SNOWFLAKE_DATABASE"), + "schema": os.getenv("SNOWFLAKE_SCHEMA"), + "client_session_keep_alive": True + } + + return snowflake.connector.connect(**creds) diff --git a/examples/command/portals/snowflake/example-7/mssql_client/service.py b/examples/command/portals/snowflake/example-7/mssql_client/service.py new file mode 100644 index 00000000000..101e30360b6 --- /dev/null +++ b/examples/command/portals/snowflake/example-7/mssql_client/service.py @@ -0,0 +1,127 @@ +import logging +import os +import sys +import pymssql +from flask import request +from flask import Flask + +# Environment variables +LOG_LEVEL = os.getenv('LOG_LEVEL', 'DEBUG').upper() +logging.basicConfig(level=LOG_LEVEL, format='%(asctime)s - %(levelname)s - %(message)s') + +app = Flask(__name__) + +MSSQL_USER = os.environ.get('MSSQL_USER') +MSSQL_PASSWORD = os.environ.get('MSSQL_PASSWORD') +MSSQL_DATABASE = os.environ.get('MSSQL_DATABASE') +MSSQL_SERVER = os.environ.get('ENDPOINT_HOST', 'localhost') + +# Create connection function +def get_connection(): + return pymssql.connect( + server=MSSQL_SERVER, + user=MSSQL_USER, + password=MSSQL_PASSWORD, + database=MSSQL_DATABASE + ) + +@app.route("/query", methods=["POST"]) +def query(): + message = request.json + logging.info(f"Received message: {message}") + user_query = message['data'][0][1] + logging.info(f"Received query: {user_query}") + + with get_connection() as conn: + with conn.cursor() as cursor: + cursor.execute(user_query) + columns = [column[0] for column in cursor.description] + rows = [list(row) for row in cursor.fetchall()] + rows = [columns] + rows + data = { + 'data': [[0, rows]], + } + logging.info(f"Returning data: {data}") + return data + +@app.route("/execute", methods=["POST"]) +def execute(): + message = request.json + logging.info(f"Received message: {message}") + user_query = message['data'][0][1] + logging.info(f"Received query: {user_query}") + + with get_connection() as conn: + with conn.cursor() as cursor: + cursor.execute(user_query) + conn.commit() + return { + 'data': [[0, "SUCCESS"]], + } + +@app.route("/insert", methods=["POST"]) +def insert(): + message = request.json + logging.info(f"Received message: {message}") + user_query = message['data'][0][1] + logging.info(f"Received query: {user_query}") + values = message['data'][0][2] + logging.info(f"Received values: {values}") + + if len(values) == 0: + return { + 'data': [[0, "SUCCESS"]], + } + + with get_connection() as conn: + with conn.cursor() as cursor: + for value in values: + cursor.execute(user_query, value) + conn.commit() + return { + 'data': [[0, "SUCCESS"]], + } + +@app.route("/ready", methods=["GET"]) +def ready(): + return {} + +def print_environment_variables(): + """ + Print the relevant environment variables for diagnostic + """ + relevant_vars = [ + 'SNOWFLAKE_ACCOUNT', + 'SNOWFLAKE_WAREHOUSE', + 'SNOWFLAKE_HOST', + 'SNOWFLAKE_DATABASE', + 'SNOWFLAKE_SCHEMA', + 'SNOWFLAKE_ROLE', + 'SNOWFLAKE_USER', + 'LOG_LEVEL', + 'MSSQL_DATABASE', + ] + + logging.info("Application environment variables:") + for var in relevant_vars: + value = os.getenv(var, 'Not set') + if var in globals(): + value = globals()[var] + logging.info(f"{var}: {value}") + +def test_connection(): + """Test the database connection with a simple system query""" + try: + with get_connection() as conn: + with conn.cursor() as cursor: + cursor.execute("SELECT @@VERSION AS SQLServerVersion;") + version = cursor.fetchone()[0] + logging.info(f"Successfully connected to SQL Server") + logging.info(f"Server Version: {version}") + except Exception as e: + logging.error(f"Connection test failed: {str(e)}") + +if __name__ == "__main__": + print_environment_variables() + test_connection() + app.run(host='0.0.0.0', port=8080) diff --git a/examples/command/portals/snowflake/example-7/setup_ockam_outlet.sh b/examples/command/portals/snowflake/example-7/setup_ockam_outlet.sh new file mode 100644 index 00000000000..b3cfc0cf215 --- /dev/null +++ b/examples/command/portals/snowflake/example-7/setup_ockam_outlet.sh @@ -0,0 +1,48 @@ +#!/bin/bash +set -e +############# +# Usage +# DB_ENDPOINT="SQLSERVER_HOST:1433" ./setup_ockam_outlet.sh +############# + +############## +# Check if mssql_outlet.ticket exists +if [ ! -f "mssql_outlet.ticket" ]; then + echo "ERROR: mssql_outlet.ticket file not found" + exit 1 +fi + +# Validate DB_ENDPOINT existence and format +if [ -z "$DB_ENDPOINT" ] || ! echo "$DB_ENDPOINT" | grep -qE '^[a-zA-Z0-9.-]+:[0-9]+$'; then + echo "ERROR: DB_ENDPOINT must be set and in format hostname:port" + exit 1 +fi + +echo "Starting Ockam installation..." +if ! curl --proto '=https' --tlsv1.2 -sSfL https://install.command.ockam.io | bash; then + echo "ERROR: Failed to install Ockam Command" + exit 1 +fi + +source "$HOME/.ockam/env" + +echo "Setup ockam node" +export OCKAM_LOG_MAX_FILES=5 +export OCKAM_HOME=/opt/ockam_home + +# Read the enrollment ticket +ENROLLMENT_TICKET=$(cat mssql_outlet.ticket) + +echo "Enrolling project with ticket" +ockam node create --enrollment-ticket "$ENROLLMENT_TICKET" \ +--configuration " +{ + "relay": "mssql", + "tcp-outlet": { + "to": "${DB_ENDPOINT}", + "allow": "snowflake" + } +} +" + +echo "Ockam setup complete"