docs: snowflake mssql example

examples/command/portals/snowflake/example-7/README.md

@@ -0,0 +1,363 @@
+# Access Microsoft SQL Server from Snowpark Container Services
+
+![Architecture](./diagram.png)
+
+## Get started with Ockam
+
+[Signup for Ockam](https://www.ockam.io/signup) and then run the following commands on your workstation:
+
+```sh
+# Install Ockam Command
+curl --proto '=https' --tlsv1.2 -sSfL https://install.command.ockam.io | bash && source "$HOME/.ockam/env"
+
+# Enroll with Ockam Orchestrator.
+ockam enroll
+
+# Create an enrollment ticket for the node that will run in snowpark container services
+ockam project ticket --usage-count 1 --expires-in 4h --attribute snowflake > snowflake_inlet.ticket
+
+# Create an enrollemnt ticket for the node that will run from a linux machine where thesql server is reachable from
+ockam project ticket --usage-count 1 --expires-in 4h --attribute mssql --relay mssql > mssql_outlet.ticket
+
+# Note egress allow list for ockam. This will be used to create a network rule in snowflake.
+ockam project show --jq .egress_allow_list
+
+```
+
+## Setup Ockam node next to MS SQL Server
+
+- Copy `setup_ockam_outlet.sh` to the linux machine where the MS SQL Server is reachable from.
+- Copy `mssql_outlet.ticket` to the same location as `setup_ockam_outlet.sh` script
+
+```sh
+# Run the setup script
+chmod +x setup_ockam_outlet.sh
+DB_ENDPOINT="HOST:1433" ./setup_ockam_outlet.sh
+```
+
+## Setup Snowflake
+
+
+```sql
+USE ROLE ACCOUNTADMIN;
+
+--Create Role
+CREATE ROLE MSSQL_API_ROLE;
+GRANT ROLE MSSQL_API_ROLE TO ROLE ACCOUNTADMIN;
+
+--Create Database
+CREATE DATABASE IF NOT EXISTS MSSQL_API_DB;
+GRANT OWNERSHIP ON DATABASE MSSQL_API_DB TO ROLE MSSQL_API_ROLE COPY CURRENT GRANTS;
+
+USE DATABASE MSSQL_API_DB;
+
+--Create Warehouse
+CREATE OR REPLACE WAREHOUSE MSSQL_API_WH WITH WAREHOUSE_SIZE='X-SMALL';
+GRANT USAGE ON WAREHOUSE MSSQL_API_WH TO ROLE MSSQL_API_ROLE;
+
+--Create compute pool
+CREATE COMPUTE POOL MSSQL_API_CP
+  MIN_NODES = 1
+  MAX_NODES = 5
+  INSTANCE_FAMILY = CPU_X64_XS;
+
+GRANT USAGE ON COMPUTE POOL MSSQL_API_CP TO ROLE MSSQL_API_ROLE;
+GRANT MONITOR ON COMPUTE POOL MSSQL_API_CP TO ROLE MSSQL_API_ROLE;
+
+
+--Wait till compute pool is in idle or ready state
+DESCRIBE COMPUTE POOL MSSQL_API_CP;
+
+--Create schema
+CREATE SCHEMA IF NOT EXISTS MSSQL_API_SCHEMA;
+GRANT ALL PRIVILEGES ON SCHEMA MSSQL_API_SCHEMA TO ROLE MSSQL_API_ROLE;
+
+
+--Create Image Repository
+CREATE IMAGE REPOSITORY IF NOT EXISTS MSSQL_API_REPOSITORY;
+GRANT READ ON IMAGE REPOSITORY MSSQL_API_REPOSITORY TO ROLE MSSQL_API_ROLE;
+
+--Note repository_url value to be used to build and publish consumer image to snowflake
+SHOW IMAGE REPOSITORIES;
+
+```
+
+## Push Ockam docker image and MS SQL Server client docker image
+
+```sh
+cd mssql_client
+
+#TODO
+hycwvdm-test-account.registry.snowflakecomputing.com/mssql_api_db/mssql_api_schema/mssql_api_repository
+
+# Use the repository_url
+docker login <repository_url>
+
+docker buildx build --platform linux/amd64 --load -t <repository_url>/mssql_client:latest .
+
+docker push <repository_url>/mssql_client:latest
+
+# Push Ockam
+docker pull ghcr.io/build-trust/ockam:0.146.0@sha256:b13ed188dbde6f5cae9d2c9c9e9305f9c36a009b1e5c126ac0d066537510f895
+
+docker tag ghcr.io/build-trust/ockam:0.146.0@sha256:b13ed188dbde6f5cae9d2c9c9e9305f9c36a009b1e5c126ac0d066537510f895 <repository_url>/ockam:latest
+
+docker push <repository_url>/ockam:latest
+
+cd -
+```
+
+## Create an Ockam node and API Client to connect to MS SQL Server in Snowpark Container Services
+
+> [!IMPORTANT]
+> Replace `TODO` values in `VALUE_LIST` with the output of `ockam project show --jq .egress_allow_list` command in previous step.
+
+```sh
+#Example
+VALUE_LIST = ("XXXXX-*.projects.orchestrator.ockam.io:443");
+```
+
+> [!IMPORTANT]
+> Replace `<OCKAM_ENROLLMENT_TICKET>` with contents of `snowflake_inlet.ticket` generated in previous step
+> Replace `TODO` values in `MSSQL_DATABASE`, `MSSQL_USER`, `MSSQL_PASSWORD` with values from MS SQL Server. Make sure the user has access to the database and perform the necessary operations.
+
+- Create a network rule to allow the Ockam node to connect to your ockam project.
+
+```sql
+USE ROLE ACCOUNTADMIN;
+
+-- Update VALUE_LIST with ockam egress details
+CREATE NETWORK RULE OCKAM_OUT TYPE = 'HOST_PORT' MODE = 'EGRESS'
+VALUE_LIST = ("TODO");
+
+CREATE OR REPLACE EXTERNAL ACCESS INTEGRATION OCKAM
+ALLOWED_NETWORK_RULES = (OCKAM_OUT) ENABLED = true;
+
+GRANT USAGE ON INTEGRATION OCKAM TO ROLE MSSQL_API_ROLE;
+
+```
+
+- Setup Service
+
+```sql
+USE ROLE MSSQL_API_ROLE;
+USE DATABASE MSSQL_API_DB;
+USE WAREHOUSE MSSQL_API_WH;
+USE SCHEMA MSSQL_API_SCHEMA;
+
+
+CREATE OR REPLACE NETWORK RULE OCSP_OUT
+TYPE = 'HOST_PORT' MODE= 'EGRESS'
+VALUE_LIST = ('ocsp.snowflakecomputing.com:80');
+
+-- Create access integration
+
+USE ROLE ACCOUNTADMIN;
+GRANT CREATE INTEGRATION ON ACCOUNT TO ROLE MSSQL_API_ROLE;
+
+CREATE OR REPLACE EXTERNAL ACCESS INTEGRATION OCSP
+ALLOWED_NETWORK_RULES = (OCSP_OUT)
+ENABLED = true;
+
+GRANT USAGE ON INTEGRATION OCSP TO ROLE MSSQL_API_ROLE;
+
+-- Create Service. Make sure to replace variables.
+
+CREATE SERVICE MSSQL_API_CLIENT
+  IN COMPUTE POOL MSSQL_API_CP
+  FROM SPECIFICATION
+$$
+    spec:
+      endpoints:
+      - name: http-endpoint
+        port: 8080
+        public: false
+        protocol: HTTP
+      - name: ockam-inlet
+        port: 1443
+        public: false
+        protocol: TCP
+      containers:
+      - name: ockam-inlet
+        image: /mssql_api_db/MSSQL_API_SCHEMA/mssql_api_repository/ockam
+        env:
+            OCKAM_DISABLE_UPGRADE_CHECK: true
+            OCKAM_TELEMETRY_EXPORT: false
+        args:
+          - node
+          - create
+          - --foreground
+          - --enrollment-ticket
+          - "<OCKAM_ENROLLMENT_TICKET>"
+          - --configuration
+          - |
+            tcp-inlet:
+              from: 0.0.0.0:1433
+              via: mssql
+              allow: mssql
+      - name: http-endpoint
+        image: /mssql_api_db/mssql_api_schema/mssql_api_repository/mssql_client
+        env:
+          SNOWFLAKE_WAREHOUSE: MSSQL_API_WH
+          MSSQL_DATABASE: 'TODO'
+          MSSQL_USER: 'TODO'
+          MSSQL_PASSWORD: 'TODO'
+        resources:
+          requests:
+            cpu: 0.5
+            memory: 128M
+          limits:
+            cpu: 1
+            memory: 256M
+$$
+MIN_INSTANCES=1
+MAX_INSTANCES=1
+EXTERNAL_ACCESS_INTEGRATIONS = (OCSP, OCKAM);
+
+-- Check service status
+SHOW SERVICES;
+SELECT SYSTEM$GET_SERVICE_STATUS('MSSQL_API_CLIENT');
+
+-- Check service logs
+CALL SYSTEM$GET_SERVICE_LOGS('MSSQL_API_CLIENT', '0', 'http-endpoint', 100);
+CALL SYSTEM$GET_SERVICE_LOGS('MSSQL_API_CLIENT', '0', 'ockam-inlet', 100);
+
+```
+> [!IMPORTANT]
+> - `http-endpoint` is the endpoint that will be used to connect to the MS SQL Server. You will see `Successfully connected to SQL Server` in the logs upon successful connection.
+> - `ockam-inlet` is the endpoint that will be used to connect to the Ockam node. Logs will indicate if there are any errors starting the node.
+
+## Stored procedures to use the API
+
+```sql
+-- These functions are dependent the service, and needs to be created after the service is created
+
+USE ROLE ACCOUNTADMIN;
+
+-- Query
+CREATE OR REPLACE FUNCTION _ockam_query_mssql(query STRING)
+    RETURNS VARCHAR
+    CALLED ON NULL INPUT
+    VOLATILE
+    SERVICE = MSSQL_API_CLIENT
+    ENDPOINT = 'http-endpoint'
+    AS '/query';
+
+
+CREATE OR REPLACE PROCEDURE ockam_mssql_query(QUERY STRING)
+    RETURNS TABLE()
+    LANGUAGE PYTHON
+    RUNTIME_VERSION = '3.11'
+    PACKAGES = ('snowflake-snowpark-python')
+    HANDLER = 'wrap_query'
+    EXECUTE AS OWNER
+AS '
+import json
+
+def wrap_query(session, query):
+    data = json.loads(session.sql(f"SELECT _ockam_query_mssql($${query}$$);").collect()[0][0])
+    keys = data[0]
+    values = data[1:]
+    return session.create_dataframe(values).to_df(keys)
+';
+
+-- Execute Statement
+CREATE OR REPLACE FUNCTION _ockam_mssql_execute(QUERY STRING)
+    RETURNS VARCHAR
+    CALLED ON NULL INPUT
+    VOLATILE
+    SERVICE = MSSQL_API_CLIENT
+    ENDPOINT = 'http-endpoint'
+    AS '/execute';
+
+
+CREATE OR REPLACE PROCEDURE ockam_mssql_execute(QUERY STRING)
+    RETURNS STRING
+    LANGUAGE PYTHON
+    RUNTIME_VERSION = '3.11'
+    PACKAGES = ('snowflake-snowpark-python')
+    HANDLER = 'wrap_execute'
+    EXECUTE AS OWNER
+AS '
+def wrap_execute(session, query):
+    return session.sql(f"SELECT _ockam_mssql_execute($${query}$$);").collect()[0][0]
+';
+
+-- Insert Statement
+CREATE OR REPLACE FUNCTION ockam_mssql_insert(QUERY STRING, ENTRIES ARRAY)
+    RETURNS VARCHAR
+    CALLED ON NULL INPUT
+    VOLATILE
+    SERVICE = MSSQL_API_CLIENT
+    ENDPOINT = 'http-endpoint'
+    AS '/insert';
+
+```
+
+## Test the API
+
+__Execute a statement__
+```sql
+
+USE ROLE ACCOUNTADMIN;
+
+CALL OCKAM_MSSQL_EXECUTE($$
+IF NOT EXISTS (SELECT * FROM sys.tables WHERE name = 'PETS')
+BEGIN
+    CREATE TABLE PETS (
+        NAME NVARCHAR(100),
+        BREED NVARCHAR(100)
+    );
+END
+$$);
+```
+
+__Insert data__
+```sql
+CALL OCKAM_MSSQL_EXECUTE($$
+INSERT INTO PETS VALUES ('Toby', 'Beagle');
+$$);
+```
+
+__Query the PostgreSQL database__
+```sql
+CALL OCKAM_MSSQL_QUERY('SELECT * FROM PETS');
+```
+
+__Join PostgreSQL and Snowflake tables__
+```sql
+-- Create two similar tables `PETS`, in MS SQL Server and Snowflake, but with different fields
+
+-- Create the table in MS SQL Server
+CALL OCKAM_MSSQL_EXECUTE($$
+IF OBJECT_ID('PETS', 'U') IS NOT NULL
+    DROP TABLE PETS;
+
+IF NOT EXISTS (SELECT * FROM sys.tables WHERE name = 'PETS')
+BEGIN
+    CREATE TABLE PETS (
+        NAME NVARCHAR(100),
+        BREED NVARCHAR(100)
+    );
+END;
+
+INSERT INTO PETS VALUES ('Max', 'Golden Retriever');
+INSERT INTO PETS VALUES ('Bella', 'Poodle');
+$$);
+
+-- Create the table in Snowflake
+
+CREATE TABLE IF NOT EXISTS PETS (NAME VARCHAR(100), YEAR_OF_BIRTH INT);
+INSERT INTO PETS VALUES ('Max',  2018);
+INSERT INTO PETS VALUES ('Bella', 2019);
+
+
+-- First we query the remote MS SQL Server
+CALL OCKAM_MSSQL_QUERY('SELECT * FROM PETS');
+SET pets_query=LAST_QUERY_ID();
+
+-- Join the results of the MS SQL Server query with the Snowflake table
+SELECT * FROM TABLE(RESULT_SCAN($pets_query)) as MSSQL_PETS
+    INNER JOIN PETS ON MSSQL_PETS.NAME = PETS.NAME;
+```
+

examples/command/portals/snowflake/example-7/mssql_client/Dockerfile

@@ -0,0 +1,12 @@
+ARG BASE_IMAGE=python:3.10-slim-buster
+FROM $BASE_IMAGE
+
+RUN pip install --upgrade pip && \
+    pip install flask snowflake snowflake-connector-python
+
+RUN pip install pymssql==2.2.11
+
+COPY service.py ./
+COPY connection.py ./
+
+CMD ["python3", "service.py"]