diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml deleted file mode 100644 index c26c7a1b..00000000 --- a/.github/workflows/deploy.yml +++ /dev/null @@ -1,46 +0,0 @@ -name: Deploy -on: - push: - branches: [main] - workflow_dispatch: - -concurrency: - group: "pages" - cancel-in-progress: true - -jobs: - build_and_deploy: - name: Build and Deploy - permissions: - pages: write - id-token: write - environment: - name: github-pages - url: ${{ steps.deployment.outputs.page_url }} - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v3 - - name: Setup Node.js - uses: actions/setup-node@v3 - with: - node-version: 18 - - name: Install dependencies - run: npm ci - - name: Create workshop database - run: npm run create:db - env: - GITHUB_REPO_URL: ${{ github.event.repository.html_url }} - - name: Build website - run: npm run build:website - env: - SEARCH_URL: ${{ secrets.SEARCH_URL }} - - name: Setup GitHub Pages - uses: actions/configure-pages@v2 - - name: Upload artifact - uses: actions/upload-pages-artifact@v1 - with: - path: packages/website/dist/website - - name: Deploy to GitHub Pages - id: deployment - uses: actions/deploy-pages@v1 diff --git a/.github/workflows/release-cli.yml b/.github/workflows/release-cli.yml deleted file mode 100644 index f34983d5..00000000 --- a/.github/workflows/release-cli.yml +++ /dev/null @@ -1,35 +0,0 @@ -name: Release CLI -on: - workflow_dispatch: - # push: - # branches: [main] - -concurrency: - group: "release-cli" - -permissions: - contents: write - issues: write - pull-requests: write - -jobs: - release: - runs-on: ubuntu-latest - if: github.ref == 'refs/heads/main' - steps: - - uses: actions/checkout@v3 - with: - persist-credentials: false - - uses: actions/setup-node@v3 - with: - node-version: 18 - - run: npm ci - - run: npm run build:cli - - run: cd packages/cli && npx semantic-release-monorepo - if: success() - env: - GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} - NPM_TOKEN: ${{ secrets.NPM_TOKEN }} - # Need owner/admin account to bypass branch protection - GIT_COMMITTER_NAME: sinedied - GIT_COMMITTER_EMAIL: noda@free.fr diff --git a/workshops/aks-openai-embeddings/assets/architecture_aks_oai.png b/workshops/aks-openai-embeddings/assets/architecture_aks_oai.png deleted file mode 100644 index 8dd3c105..00000000 Binary files a/workshops/aks-openai-embeddings/assets/architecture_aks_oai.png and /dev/null differ diff --git a/workshops/aks-openai-embeddings/assets/openaiadddocs.png b/workshops/aks-openai-embeddings/assets/openaiadddocs.png deleted file mode 100644 index 1cc6a5fa..00000000 Binary files a/workshops/aks-openai-embeddings/assets/openaiadddocs.png and /dev/null differ diff --git a/workshops/aks-openai-embeddings/assets/openaichat.png b/workshops/aks-openai-embeddings/assets/openaichat.png deleted file mode 100644 index 8f818067..00000000 Binary files a/workshops/aks-openai-embeddings/assets/openaichat.png and /dev/null differ diff --git a/workshops/aks-openai-embeddings/workshop.md b/workshops/aks-openai-embeddings/workshop.md deleted file mode 100644 index f75a6d1f..00000000 --- a/workshops/aks-openai-embeddings/workshop.md +++ /dev/null @@ -1,371 +0,0 @@ ---- -published: true # Optional. Set to true to publish the workshop (default: false) -type: workshop # Required. -title: AKS and OpenAI Embeddings Workshop # Required. Full title of the workshop -short_title: AKS and OpenAI Embeddings Workshop # Optional. Short title displayed in the header -description: This is a workshop for cloud engineers who would like to learn how build secure AKS environments for their AI applications using OpenAI, Redis, Langchain and various Azure AI services like the form recognizer # Required. -level: intermediate # Required. Can be 'beginner', 'intermediate' or 'advanced' -authors: # Required. You can add as many authors as needed - - Ayobami Ayodeji -contacts: # Required. Must match the number of authors - - "@mosabami" -duration_minutes: 90 # Required. Estimated duration in minutes -tags: aks, openai, python, langchain, redis # Required. Tags for filtering and searching -#banner_url: assets/banner.jpg # Optional. Should be a 1280x640px image -#video_url: https://youtube.com/link # Optional. Link to a video of the workshop -#audience: students # Optional. Audience of the workshop (students, pro devs, etc.) -#wt_id: # Optional. Set advocacy tracking code for supported links -#oc_id: # Optional. Set marketing tracking code for supported links -#navigation_levels: 2 # Optional. Number of levels displayed in the side menu (default: 2) -#sections_title: # Optional. Override titles for each section to be displayed in the side bar -# - Section 1 title -# - Section 2 title ---- - -# AKS and OpenAI Embeddings Workshop - -AKS is a great platform for hosting modern AI based applications for various reasons. It provides a single control plane to host all the assets required to build applications from end to end and even allows the development of applications using a Microservice architecture. What this means is that the AI based components can be separated from the rest of the applications. AKS also allows hosting of some of Azure's AI services as containers withing your cluster, so that you can keep all the endpoints of your applications private as well as manage scaling however you need to. This is a significant advantage when it comes to securing your application. By hosting all components in a single control plane, you can streamline your DevOps process. - -In this workshop, you will perform a series of tasks with the end result being a Python based chatbot application deployed in a moderately well architected AKS environment, but allow more advanced environment architectures be deployed due to the flexibility of the AKS construciton tool being used. The appliction will have a page that allows the user upload text, pdf and other forms of media. This triggers a batch process that gets sequenced in Azure storage queue. It uses an Azure OpenAI embedding model to create embeddings of the uploaded document which then allows the chatbot answer questions based on the content of the data uploaded. For more information about the architecture of this workload, checkout the [AKS Landing Zone Accelerator](https://github.com/Azure/AKS-Landing-Zone-Accelerator/tree/main/Scenarios/AKS-OpenAI-CogServe-Redis-Embeddings#core-architecture-components) repo. - -![Architecture](assets/architecture_aks_oai.png) - -## Prerequisites - -
- -> This workshop was originally developed as part of the AKS landing zone accelerator program. Check out [the repo](https://aka.ms/akslza/aiscenario) for more information - -
- -You will need to have the following installed on your system if you are not using Azure cloud shell. The .devcontainer of the repo you will be cloning comes preinstalled with them: - -- Kubectl, preferably 1.25 and above ( `az aks install-cli` ) -- [Azure Subscription](https://azure.microsoft.com/free) -- [Azure CLI](https://learn.microsoft.com/cli/azure/what-is-azure-cli?WT.mc_id=containers-105184-pauyu) -- [Visual Studio Code](https://code.visualstudio.com/) -- [Docker Desktop](https://www.docker.com/products/docker-desktop/) -- [Git](https://git-scm.com/) -- Bash shell (e.g. [Windows Terminal](https://www.microsoft.com/p/windows-terminal/9n0dx20hk701) with [WSL](https://docs.microsoft.com/windows/wsl/install-win10) or [Azure Cloud Shell](https://shell.azure.com)) - -
- -> To Deploy this scenario, you must have Azure OpenAI Service enabled in your subscription. If you haven't registered it yet, follow the instructions [here](https://learn.microsoft.com/legal/cognitive-services/openai/limited-access) to do so. Registration may take a day. - -
- ---- - -## Fork and Clone the repo -
- -> Ensure you fork the repo before cloning your fork. If you don't fork it, you might not be able to push changes or run GitHub Actions required at the end of this workshop. - -
- -1. Skip this step if you have a local terminal ready with kubectl and Azure CLI installed. Using a web browser, navigate to the [Azure Cloud Shell](https://shell.azure.com). Ensure your Cloud Shell is set to Bash. If it is on PowerShell, click the drop down in the top left corner and select Bash. -1. Clone this repository locally, and change the directory to the `./infrastructure` folder. - ```bash - git clone --recurse-submodules https://github.com/Azure/AKS-Landing-Zone-Accelerator - - cd Scenarios/AKS-OpenAI-CogServe-Redis-Embeddings/infrastructure/ - ``` -1. Ensure you are signed into the `az` CLI (use `az login` if not) - -
- -> If running in **Github Code Spaces**, update submodules explicitly run in `AKS-Landing-Zone-Accelerator/Scenarios/AKS-OpenAI-CogServe-Redis-Embeddings/` - -
- -```bash -git submodule update --init --recursive -``` - ---- - -## Setup environment specific variables -This will set environment variables, including your preferred `Resource Group` name and `Azure Region` for the subsequent steps, and create the `resource group` where we will deploy the solution. - - > **Important** - > Set UNIQUESTRING to a value that will prevent your resources from clashing names, recommended combination of your initials, and 2-digit number (eg. js07) - -```bash -UNIQUESTRING= -RGNAME=embedding-openai-rg -LOCATION=eastus -SIGNEDINUSER=$(az ad signed-in-user show --query id --out tsv) && echo "Current user is $SIGNEDINUSER" - -``` - ---- - -## Deploy the environment using IaC (Bicep) and AKS Construction - -Create all the solution resources using the provided `bicep` template and capture the output environment configuration in variables that are used later in the process. -
- -> Our bicep template is using the [AKS-Construction](https://github.com/Azure/AKS-Construction) project to provision the AKS Cluster and associated cluster services/addons, in addition to the other workload specific resources. - -
- -
- -> Ensure you have enough **quota** to deploy the gpt-35-turbo and text-embedding-ada-002 models before running the command below. Failure to do this will lead to an "InsufficientQuota" error in the model deployment. Most subscriptions have quota of 1 of these models, so if you already have either of those models deployed, you might not be able to deploy another one in the same subscription and you might have to use that deployment as your model instead to proceed. If that is the case, use the **Reusing existing OpenAI Service** option. Otherwise use the **Deploy new resources** option. - -
- -#### Reusing existing OpenAI Service option - -If you are re-using existing OpenAI resource set following variables and pass them to Bicep template - -```bash -OPENAI_RGNAME= -OPENAI_ACCOUNTNAME= -``` - -Add optional variable variables to the script below -```bash -az deployment sub create \ - --name main-$UNIQUESTRING \ - --template-file main.bicep \ - --location=$LOCATION \ - --parameters UniqueString=$UNIQUESTRING \ - --parameters signedinuser=$SIGNEDINUSER \ - --parameters resourceGroupName=$RGNAME \ - --parameters openAIName=$OPENAI_ACCOUNTNAME \ - --parameters openAIRGName=$OPENAI_RGNAME - -``` - -#### Deploy new resources option -```bash -az deployment sub create \ - --name main-$UNIQUESTRING \ - --template-file main.bicep \ - --location=$LOCATION \ - --parameters UniqueString=$UNIQUESTRING \ - --parameters signedinuser=$SIGNEDINUSER \ - --parameters resourceGroupName=$RGNAME -``` -### Set Output Variables - -
- -> If you already had an OpenAI resource before you began this deployment that had either gpt-35-turbo or text-embedding-ada-002 turbo, you will need to change the OPENAI_ACCOUNTNAME, OPENAI_RGNAME, OPENAI_API_BASE, OPENAI_ENGINE and OPENAI_EMBEDDINGS_ENGINE environment variables in the commands below to the actual names used in your previous deployment. If you only created one of the two required models previously, you will need to create the other one manually. - -
- -```bash -KV_NAME=$(az deployment sub show --name main-$UNIQUESTRING --query properties.outputs.kvAppName.value -o tsv) && echo "The Key Vault name is $KV_NAME" -OIDCISSUERURL=$(az deployment sub show --name main-$UNIQUESTRING --query properties.outputs.aksOidcIssuerUrl.value -o tsv) && echo "The OIDC Issue URL is $OIDCISSUERURL" -AKSCLUSTER=$(az deployment sub show --name main-$UNIQUESTRING --query properties.outputs.aksClusterName.value -o tsv) && echo "The AKS cluster name is $AKSCLUSTER" -BLOB_ACCOUNT_NAME=$(az deployment sub show --name main-$UNIQUESTRING --query properties.outputs.blobAccountName.value -o tsv) && echo "The Azure Storage Blob account name is $BLOB_ACCOUNT_NAME" -FORMREC_ACCOUNT=$(az deployment sub show --name main-$UNIQUESTRING --query properties.outputs.formRecognizerName.value -o tsv) && echo "The Document Intelligence account name is $FORMREC_ACCOUNT" -FORM_RECOGNIZER_ENDPOINT=$(az deployment sub show --name main-$UNIQUESTRING --query properties.outputs.formRecognizerEndpoint.value -o tsv) && echo "The Document Intelligence endpoint URL is $FORM_RECOGNIZER_ENDPOINT" -TRANSLATOR_ACCOUNT=$(az deployment sub show --name main-$UNIQUESTRING --query properties.outputs.translatorName.value -o tsv) && echo "The Translator account name is $TRANSLATOR_ACCOUNT" -ACR_NAME=$(az acr list -g $RGNAME --query '[0]'.name -o tsv) && echo "The Azure OpenAI GPT Model is $ACR_NAME" -# If you created the OpenAI service separate from the deployment steps in this workshop, dont run the commands below, instead provide the name of your exisitng OpenAI deployments as the value of these environment variables. -OPENAI_ACCOUNTNAME=$(az deployment sub show --name main-$UNIQUESTRING --query properties.outputs.openAIAccountName.value -o tsv) && echo "The Azure OpenAI account name is $OPENAI_ACCOUNTNAME" -OPENAI_API_BASE=$(az deployment sub show --name main-$UNIQUESTRING --query properties.outputs.openAIURL.value -o tsv) && echo "The Azure OpenAI instance API URL is $OPENAI_API_BASE" -OPENAI_RGNAME=$(az deployment sub show --name main-$UNIQUESTRING --query properties.outputs.openAIRGName.value -o tsv) && echo "The Azure OpenAI Resource Group is $OPENAI_RGNAME" -OPENAI_ENGINE=$(az deployment sub show --name main-$UNIQUESTRING --query properties.outputs.openAIEngineName.value -o tsv) && echo "The Azure OpenAI GPT Model is $OPENAI_ENGINE" -OPENAI_EMBEDDINGS_ENGINE=$(az deployment sub show --name main-$UNIQUESTRING --query properties.outputs.openAIEmbeddingEngine.value -o tsv) && echo "The Azure OpenAI Embedding Model is $OPENAI_EMBEDDINGS_ENGINE" - -``` - -If variables are empty (some shells like zsh may have this issue) - see Troubleshooting section below. -
- -> Ensure you those commands above captured the correct values for the environment variables by using the echo command, otherwise you might run into errors in the next few commands. - -
- ---- - -## Store the resource keys Key Vault Secrets -OpenAI API, Blob Storage, Form Recognizer and Translator keys will be secured in Key Vault, and passed to the workload using the CSI Secret driver - -
- -> If you get a bad request error in any of the commands below, then it means the previous commands did not serialize the environment variable correctly. Use the echo command to get the name of the AI services used in the commands below and run the commands by replacing the environment variables with actual service names. - -
- -Enter the commands below to store the required secrets in Key vault - ```bash - az keyvault secret set --name openaiapikey --vault-name $KV_NAME --value $(az cognitiveservices account keys list -g $OPENAI_RGNAME -n $OPENAI_ACCOUNTNAME --query key1 -o tsv) - - az keyvault secret set --name formrecognizerkey --vault-name $KV_NAME --value $(az cognitiveservices account keys list -g $RGNAME -n $FORMREC_ACCOUNT --query key1 -o tsv) - - az keyvault secret set --name translatekey --vault-name $KV_NAME --value $(az cognitiveservices account keys list -g $RGNAME -n $TRANSLATOR_ACCOUNT --query key1 -o tsv) - - az keyvault secret set --name blobaccountkey --vault-name $KV_NAME --value $(az storage account keys list -g $RGNAME -n $BLOB_ACCOUNT_NAME --query \[1\].value -o tsv) - ``` - ---- - -## Federate AKS MI with Service account -Create and record the required federation to allow the CSI Secret driver to use the AD Workload identity, and to update the manifest files. - -
- -> If running the commands below in **zsh** or in **Github Code Spaces**, order of the variables is different. Make sure the variables make sense by taking a look at the echo'ed strings in your terminal. Use Option 1 below. If you are not using either of those two terminals, use Option 2 below. - -
- -#### Option 1 for **zsh** or **Github Code Spaces** - -```bash -CSIIdentity=($(az aks show -g $RGNAME -n $AKSCLUSTER --query "[addonProfiles.azureKeyvaultSecretsProvider.identity.resourceId,addonProfiles.azureKeyvaultSecretsProvider.identity.clientId]" -o tsv | cut -d '/' -f 5,9 --output-delimiter ' ')) - -CLIENT_ID=${CSIIdentity[3]} && echo "CLIENT_ID is $CLIENT_ID" -IDNAME=${CSIIdentity[2]} && echo "IDNAME is $IDNAME" -IDRG=${CSIIdentity[1]} && echo "IDRG is $IDRG" - -az identity federated-credential create --name aksfederatedidentity --identity-name $IDNAME --resource-group $IDRG --issuer $OIDCISSUERURL --subject system:serviceaccount:default:serversa -``` - -#### Option 2 for other Bash - -```bash -CSIIdentity=($(az aks show -g $RGNAME -n $AKSCLUSTER --query "[addonProfiles.azureKeyvaultSecretsProvider.identity.resourceId,addonProfiles.azureKeyvaultSecretsProvider.identity.clientId]" -o tsv | cut -d '/' -f 5,9 --output-delimiter ' ')) - -CLIENT_ID=${CSIIdentity[2]} && echo "CLIENT_ID is $CLIENT_ID" -IDNAME=${CSIIdentity[1]} && echo "IDNAME is $IDNAME" -IDRG=${CSIIdentity[0]} && echo "IDRG is $IDRG" - -az identity federated-credential create --name aksfederatedidentity --identity-name $IDNAME --resource-group $IDRG --issuer $OIDCISSUERURL --subject system:serviceaccount:default:serversa -``` - - -### Build ACR Image for the web app -```bash -cd ../App/ -az acr build --image oai-embeddings:v1 --registry $ACR_NAME -g $RGNAME -f ./WebApp.Dockerfile ./ -``` - - ---- - -## Deploy the Kubernetes Resources -In this step, you will deploy the kubernetes resources required to make the application run. This includes the ingress resources, deployments / pods, services, etc. - -1. Change directory to the Kubernetes manifests folder, deployment will be done using Kustomize declarations. - ```bash - cd ../kubernetes/ - ``` -1. Log into your AKS cluster and ensure you are properly logged in. The command should return the nodes in your cluster. - ```bash - az aks get-credentials -g $RGNAME -n $AKSCLUSTER - kubectl get nodes - ``` -1. Get your ingress's IP address so you have the URL required to access your application on your browser - ```bash - INGRESS_IP=$(kubectl get svc nginx -n app-routing-system -o jsonpath='{.status.loadBalancer.ingress[0].ip}') - echo "Ingress IP: $INGRESS_IP" - ``` -1. Save variables in a new .env file. Dont forget to change the env variables for the OPENAI_API_BASE, OPENAI_ENGIN and OPENAI_EMBEDDINGS_ENGINE variables in the command below to your actual openai deployment IF you already had an existing deployment. - ```bash - cat << EOF > .env - CLIENT_ID=$CLIENT_ID - TENANT_ID=$(az account show --query tenantId -o tsv) - KV_NAME=$KV_NAME - OPENAI_API_BASE=$OPENAI_API_BASE - OPENAI_ENGINE=$OPENAI_ENGINE - OPENAI_EMBEDDINGS_ENGINE=$OPENAI_EMBEDDINGS_ENGINE - LOCATION=$LOCATION - BLOB_ACCOUNT_NAME=$BLOB_ACCOUNT_NAME - FORM_RECOGNIZER_ENDPOINT=$FORM_RECOGNIZER_ENDPOINT - DNS_NAME=openai-$UNIQUESTRING.$INGRESS_IP.nip.io - ACR_IMAGE=$ACR_NAME.azurecr.io/oai-embeddings:v1 - EOF - ``` -1. Deploy the Kubernetes resources. Use option 1 if you are using kubectl < 1.25. Use option 2 if you are using kubectl >= 1.25 - - **Option 1:** - ```bash - kustomize build . > deploy-all.yaml - - kubectl apply -f deploy-all.yaml - ``` - **Option 2:** - ```bash - kubectl apply -k . - ``` - ---- - -## Test the app -1. Get the URL where the app can be reached - ```bash - kubectl get ingress - ``` -1. Copy the url under **HOSTS** and paste it in your browser. -1. Try asking the chatbot a domain specific question by heading to the **Chat** tab and typing a question there. You will notice it fail to answer it correctly. -1. Click on the `Add Document` tab in the left pane and either upload a PDF with domain information you would like to ask the chatbot about or copy and paste text containing the knowledge base in `Add text to the knowledge base` section, then click on `Compute Embeddings` -![add](./assets/openaiadddocs.png) -1. Head back to the **Chat** tab, try asking the same question again and watch the chatbot answer it correctly -![add](./assets/openaichat.png) - ---- - -## Troubleshooting - -
- -> Depending on your subscription OpenAI quota you may get deployment error - -```json -Inner Errors: -{"code": "InsufficientQuota", "message": "The specified capacity '120' of account deployment is bigger than available capacity '108' for UsageName 'Tokens Per Minute (thousands) - GPT-35-Turbo'."} -``` -There are few options - point deployment to the existing OpenAI resource instead of provisioning new one, or adjust quota. -Note: check if you have soft-deleted OpenAI instances taking up quota and purge them. -
- - -
- -> Depending on type of terminal you are using, the command to create environment variables by querying the **INFRA_RESULT** variable that gets created with the deployment might not work properly. You will notice then when you get bad request errors when running subsequent commands. Try using the **echo** command to print the values of those environment variables into your terminal and replace the environment variables like `$OPENAI_ACCOUNTNAME` and `$OIDCISSUERURL` with the actual string values. - -
- -
- -> If you notice that the api pod is stuck in *ContainerCreating* status, chances are that the federated identity was not created properly. To fix this, ensure that the "CSIIdentity" environment variable was created properly. You should then run the "az identity federated-credential create" command again using string values as opposed to environment variables. You can find the string values by using the **echo** command to print the environment variables in your terminal. It is the API deployment that brings the secrets from Key vault into the AKS cluster, so the other two pods require the API pod to be in a running state before they can start as well since they require the secrets. - -
- ---- - -# Build Intelligent Apps on AKS Challenge 1 - -Create CICD pipeline to automate OpenAI web application deployment to AKS. - -Assumption is that infrastructure, setting variables and keyvault secrets were done following OpenAI Scenario steps in [README.md](../AKS-OpenAI-CogServe-Redis-Embeddings/README.md) - -## Create GitHub Identity Federated with AAD - -Simplest way to create template for Github workflow is to use AKS **Automated deployments** wizard. -It will create Github identity federated with AAD and grant to it required permissions to AKS and ACR - -Fork `AKS-Landing-Zone-Accelerator` repo and use wizard option to "Deploy an application" pointing it to your fork and selecting provisioned ACR and AKS - -## Update GitHub workflow for Kustomize steps -Once deployment wizard is finished it will create PR with sample github flow that could be updated to match the steps required to run Kustomize - -- Add variables section and specify all variables that were queried from deployment -- Add Repo secret `CLIENT_ID` to value retrieved during infrastructure setup -- Add step to prepare `.env` file with all replacement variables -- Add step to bake Kubernetes manifest from Kustomize files -- Modify deployment step to refer to Kustomize built manifest - -The completed workflow can be found in the following repo in the [workflows folder](https://github.com/Azure/AKS-Landing-Zone-Accelerator/blob/main/.github/workflows/deploy-openai-embeddings-app.yaml). - - -
- -> You will need to update the workflow env variables to match the correct values for your deployment. - -
diff --git a/workshops/deploying-to-azure-container-apps/assets/aca.svg b/workshops/deploying-to-azure-container-apps/assets/aca.svg deleted file mode 100644 index 3378e07e..00000000 --- a/workshops/deploying-to-azure-container-apps/assets/aca.svg +++ /dev/null @@ -1 +0,0 @@ - \ No newline at end of file diff --git a/workshops/deploying-to-azure-container-apps/assets/add_secret_to_container_app.png b/workshops/deploying-to-azure-container-apps/assets/add_secret_to_container_app.png deleted file mode 100644 index 1b82ef3f..00000000 Binary files a/workshops/deploying-to-azure-container-apps/assets/add_secret_to_container_app.png and /dev/null differ diff --git a/workshops/deploying-to-azure-container-apps/assets/banner.jpg b/workshops/deploying-to-azure-container-apps/assets/banner.jpg deleted file mode 100644 index 9c87e96e..00000000 Binary files a/workshops/deploying-to-azure-container-apps/assets/banner.jpg and /dev/null differ diff --git a/workshops/deploying-to-azure-container-apps/assets/create_secret.png b/workshops/deploying-to-azure-container-apps/assets/create_secret.png deleted file mode 100644 index 0aaee6ea..00000000 Binary files a/workshops/deploying-to-azure-container-apps/assets/create_secret.png and /dev/null differ diff --git a/workshops/deploying-to-azure-container-apps/assets/edit_container_app.png b/workshops/deploying-to-azure-container-apps/assets/edit_container_app.png deleted file mode 100644 index 337f432d..00000000 Binary files a/workshops/deploying-to-azure-container-apps/assets/edit_container_app.png and /dev/null differ diff --git a/workshops/deploying-to-azure-container-apps/assets/view_new_revision.png b/workshops/deploying-to-azure-container-apps/assets/view_new_revision.png deleted file mode 100644 index 44bf88c7..00000000 Binary files a/workshops/deploying-to-azure-container-apps/assets/view_new_revision.png and /dev/null differ diff --git a/workshops/deploying-to-azure-container-apps/assets/welcome_to_build.gif b/workshops/deploying-to-azure-container-apps/assets/welcome_to_build.gif deleted file mode 100644 index 80699742..00000000 Binary files a/workshops/deploying-to-azure-container-apps/assets/welcome_to_build.gif and /dev/null differ diff --git a/workshops/deploying-to-azure-container-apps/workshop.md b/workshops/deploying-to-azure-container-apps/workshop.md deleted file mode 100644 index 91863827..00000000 --- a/workshops/deploying-to-azure-container-apps/workshop.md +++ /dev/null @@ -1,378 +0,0 @@ ---- -published: true -type: workshop -title: Deploying to Azure Container Apps -short_title: Deploying to ACA -description: Learn how to deploy containers to Azure Container Apps. -level: beginner -authors: - - Josh Duffney -contacts: - - joshduffney -duration_minutes: 30 -tags: Azure, Azure Container Apps, Azure Container Registry, Go, Golang, Containers, Docker ---- - -# Deploying to Azure Container Apps - -In this workshop, you'll learn how deploy a containerized application to Azure Container Apps. Azure Container Apps allows you to deploy containerized applications without having to manage the underlying infrastructure, leaving you to focus on your application. - -## Objectives - -You'll learn how to: -- Create an Azure Container Apps environment, -- Deploy a web application to Azure Container Apps from a local development space -- Edit and update the web application - -## Prerequisites - -| | | -|----------------------|------------------------------------------------------| -| Azure account | [Get a free Azure account](https://azure.microsoft.com/free) | -| Azure CLI | [Install Azure CLI](https://docs.microsoft.com/en-us/cli/azure/install-azure-cli) | -| Docker | [Install Docker](https://docs.docker.com/get-docker/) | -| Git | [Install Git](https://git-scm.com/downloads) | - ---- -## Set up Local Development Space -In this section, you'll clone a github repository to get the workshops code on your local computer. - -Open up your prefered terminal (This tutorial has powershell and bash instructions). if you'd like, make a folder to contain your work: - -```powershell -mkdir Reactor-Summer-2023 -cd Reactor-Summer-2023 -``` - -Clone the [Deploying to Azure Container Apps](https://github.com/duffney/deploying-to-aca) repository to your local machine using the following command: - -```powershell -git clone 'https://github.com/duffney/deploying-to-aca.git' -``` - -Change into the root of the repository: - -```powershell -cd deploying-to-aca -``` - - -### Add the containerapp extension to the Azure CLI - -Run the following command to add the containerapp extension to the Azure CLI: - -```powershell -az extension add --name containerapp -``` - -You now have code on your local device. -
- -> Optional: explore the files you've downloaded - open them in visual Studio Code, list the files with the terminal command "ls", hypothesize what the different files do, and what the final web app might look like. - - -
- ---- - -## Set up Azure Resources - -Next we need to set up the azure resources that will support our work. - -### Log in to the Azure CLI - -Run the following command to log in to the Azure CLI: - -```powershell -az login -``` -### Create a resource group - -
-PowerShell - -```powershell -$resource_group='myResourceGroup' -$location='northcentralus' - -az group create ` ---name $resource_group ` ---location $location -``` - -
- -
-Bash - -```bash -resource_group='myResourceGroup' -location='northcentralus' - -az group create \ ---name $resource_group \ ---location $location -``` - -
- -### Create an Empty Container Registry - -Next, run the following command to deploy an Azure Container Registry instance: - -
-PowerShell - -```powershell -$random = Get-Random -Minimum 1000 -Maximum 9999 -$acr_name="myregistry$random" - -az acr create ` - --name $acr_name ` - --resource-group $resource_group ` - --sku Basic ` - --admin-enabled true ` - --location $location -``` - -
- -
-Bash - -```bash -random=$RANDOM -acr_name="myregistry$random" - -az acr create \ ---name $acr_name \ ---resource-group $resource_group \ ---sku Basic \ ---admin-enabled true \ ---location $location -``` - -
- -### Log in to Azure Container Registry - -Run the following commands to log in to your ACR instance: - - -```powershell -az acr login --name $acr_name -``` -### Create an Azure Container Apps environment - -An Azure Container Apps environment is a logical grouping of resources that are used to deploy containerized applications. Within an environment, you can deploy one or more container apps and share resources such as a container registry and secrets. - -Run the following command to create an Azure Container Apps environment: - -
- -PowerShell - -```powershell -$container_app_environment_name='myContainerAppEnvironment' - -az containerapp env create ` - --name $container_app_environment_name ` - --resource-group $resource_group ` - --location $location -``` - -
- -
- -Bash - -```bash -container_app_environment_name='myContainerAppEnvironment' - -az containerapp env create \ ---name $container_app_environment_name \ ---resource-group $resource_group \ ---location $location -``` - -
- ---- - -## Build and Push the Container Image - -In this section, you'll build and push a container image to Azure Container Registry (ACR). That image will be used in the next section to deploy a container app to Azure Container Apps. - -### Create a container image and push it to ACR - - -Build the container image using the following command: - -
-PowerShell - -```powershell -$image_name='webapp' -$tag='v1.0' -$server="$acr_name.azurecr.io" - -az acr build --registry $acr_name --image "$server/${image_name}:${tag}" . -``` - -
- -
-Bash - -```bash -image_name='webapp' -tag='v1.0' -server="$acr_name.azurecr.io" - -az acr build --registry $acr_name --image "$server/$image_name:$tag" . -``` - -
- ---- - -## Deploy a Container image to Azure Container Apps - -In this section, you'll deploy a containerized Go web application to Azure Container Apps. The application will be accessible via an external ingress and will use environment variables and Azure Container Registry secrets to modify the application's behavior. - - - - -### Create the container app - -Container apps define the container image to deploy, the environment variables to set, and the secrets and or volumes to mount. You can pull images from Azure Container Registry or Docker Hub and set environment variables and secrets from Azure Key Vault. Container apps can also be deployed with an external ingress, which allows you to access the application from outside the environment. Internal ingress is also available, which allows you to access the application from within the environment. - -Run the following commands to create a container app: - -
- -PowerShell - -```powershell -$container_app_name='my-container-app' -$password=az acr credential show --name $acr_name --output tsv --query "passwords[0].value" - -az containerapp create ` - --name $container_app_name ` - --resource-group $resource_group ` - --environment $container_app_environment_name ` - --image "$server/${image_name}:${tag}" ` - --target-port 8080 ` - --ingress 'external' ` - --registry-server $server ` - --registry-username $acr_name ` - --registry-password $password ` - --query properties.configuration.ingress.fqdn ` - --output tsv - -``` - -
- -
- -Bash - -```bash -container_app_name='my-container-app' -password=$(az acr credential show --name $acr_name --output tsv --query "passwords[0].value" | tr -d '\r') - -az containerapp create \ ---name $container_app_name \ ---resource-group $resource_group \ ---environment $container_app_environment_name \ ---image "$server/$image_name:$tag" \ ---registry-server $server \ ---registry-username $acr_name \ ---registry-password $password \ ---target-port 8080 \ ---ingress 'external' \ ---query properties.configuration.ingress.fqdn -``` - -
- -
- -> Browse to the URL returned by the command to view the application. - -
- ---- - -## Update Your App - -In this section, you'll update your container app. - -Revisions allow you to deploy new versions of the container app without having to create a new container app. Revisions can be deployed with a new container image, environment variables, secrets, and volumes. - -You'll trigger a new deployment by updating updating the container app's environment variables using a contanier app secret. - -### Create a secret - -In the [Azure Portal](https://portal.azure.com/), navigate to your Azure Container App that was deployed to the `myResourceGroup` resource group. - -Next, follow the steps below to create a secret: - -1. Select **Secrets** from the left-hand menu under **Settings**. -2. Select **+ Add**. -3. Enter `welcome-secret` as the secret's **Key**. -4. Leave **Container Apps Secret** selected. -5. Enter your name for the **Value**. -6. Click **Add**. - -![Create a secret](./assets/create_secret.png) - -### Edit the container app - -Next, you need to update the container app to use the new secret as an environment variable to change the configuration of the web app. Once the seed is updated, a new revision will be deployed. - -Follow the steps below to update the container app: - -1. Select **Containers** from the left-hand menu under **Application**. -2. Click **Edit and Deploy**. -3. Check the box next to your container app, and then click **Edit**. - -![Add the secret to the container app as an environment variable](./assets/add_secret_to_container_app.png) - -### Add the secret to the container app as an environment variable - -Once the container app is open for editing, follow the steps below to add the secret as an environment variable: - -1. Under **Environment Variables**, click **+ Add**. -2. Enter `VISITOR_NAME` for the **Name**. -3. Select `Reference a secret` as the source. -4. Then, select `welcome-secret` as the **Value**. -5. Click **Save**. -6. Click **Create** to deploy the new revision. - -![Edit the container app](./assets/edit_container_app.png) - -### View the new revision - -Once the container app is updated, a new revision will be deployed. Azure Container apps supports zero downtime deployment. As the new revision is provisioned, the oldder revision will recieve all the traffic. Only once the new revision is fully provisioned will the container app switch traffic to the new revision. This way, the app has no downtime. - -Follow the steps below to track the update: - -1. Select **Revision Management** from the left-hand menu under **Application**. -2. refresh the page to see the revisions change status and traffic. -3. Once the more recently-created revision has 100% traffic, the app will be updated. -4. After the new revision is set, the previous revision will deactivate. View older revisions in the "inactive Revisions" tab - -![View the new revision](./assets/view_new_revision.png) - - - - - -
- -> Return to the Web App and refresh the page to see the new message. - -
diff --git a/workshops/fabric-e2e-rag/assets/aisearch.png b/workshops/fabric-e2e-rag/assets/aisearch.png deleted file mode 100644 index ecc384d4..00000000 Binary files a/workshops/fabric-e2e-rag/assets/aisearch.png and /dev/null differ diff --git a/workshops/fabric-e2e-rag/assets/aiservice.png b/workshops/fabric-e2e-rag/assets/aiservice.png deleted file mode 100644 index 8c2c9e8c..00000000 Binary files a/workshops/fabric-e2e-rag/assets/aiservice.png and /dev/null differ diff --git a/workshops/fabric-e2e-rag/assets/chunking-vector.svg b/workshops/fabric-e2e-rag/assets/chunking-vector.svg deleted file mode 100644 index 33038bc2..00000000 --- a/workshops/fabric-e2e-rag/assets/chunking-vector.svg +++ /dev/null @@ -1,21 +0,0 @@ - - - - - - - - Original PDF Filefaq.pdfTextChunkingfaq.pdf Page 1faq.pdf Page 2faq.pdf Page 3Chunks (page-based)GenerateEmbeddings[0.12, 0.45, 0.67, 0.89, 0.34][0.98, 0.76, 0.54, 0.32, 0.10][0.56, 0.78, 0.90, 0.12, 0.34]EmbeddingsVector storeIndexIndex 2{ "file": "fax.pdf", "chunk":"page-1", "embedding" : [0.12, 0.45, 0.67, 0.89, 0.34], "text": "Welcome to Contoso Real Estate! We’re"}{ "file": "fax.pdf", "chunk":"page-2", "embedding" : [0.98, 0.76, 0.54, 0.32, 0.10], "text": "How to Cancel a Confirmed Booking" }{ "file": "fax.pdf", "chunk":"page-3", "embedding" : [0.56, 0.78, 0.90, 0.12, 0.34] "text": "How to Report a Problem with a Listing " }Storing embeddingsin vector store"text" field value have been shorten for illustration purposes. In the actual indexed document, you'll have the entire text used for generating the embedding (in this case, the whole page). \ No newline at end of file diff --git a/workshops/fabric-e2e-rag/assets/deployments.png b/workshops/fabric-e2e-rag/assets/deployments.png deleted file mode 100644 index 2fb68289..00000000 Binary files a/workshops/fabric-e2e-rag/assets/deployments.png and /dev/null differ diff --git a/workshops/fabric-e2e-rag/assets/lakehouse.png b/workshops/fabric-e2e-rag/assets/lakehouse.png deleted file mode 100644 index 70f6b921..00000000 Binary files a/workshops/fabric-e2e-rag/assets/lakehouse.png and /dev/null differ diff --git a/workshops/fabric-e2e-rag/assets/main.bicep b/workshops/fabric-e2e-rag/assets/main.bicep deleted file mode 100644 index 94075954..00000000 --- a/workshops/fabric-e2e-rag/assets/main.bicep +++ /dev/null @@ -1,104 +0,0 @@ -param openailocation string = 'canadaeast' -param location string = resourceGroup().location -param cognitiveSearchName string = substring('rag-cogsearch${uniqueString(resourceGroup().id)}', 0, 24) -param azureOpenAiServiceName string = substring('rag-azureoai${uniqueString(resourceGroup().id)}', 0, 24) -param azureAiServiceName string = substring('rag-aiservices${uniqueString(resourceGroup().id)}', 0, 24) - -resource cognitiveSearch_resource 'Microsoft.Search/searchServices@2023-11-01' ={ - name: cognitiveSearchName - location: location - sku: { - name: 'free' - } - properties: { - replicaCount: 1 - partitionCount: 1 - hostingMode: 'default' - publicNetworkAccess: 'enabled' - networkRuleSet: { - ipRules: [] - } - encryptionWithCmk: { - enforcement: 'Unspecified' - } - disableLocalAuth: false - authOptions: { - apiKeyOnly: {} - } - } -} - -// resource azureOpenAiService_resource 'Microsoft.CognitiveServices/accounts@2023-10-01-preview' = { -// name: azureOpenAiServiceName -// location: openailocation -// sku: { -// name: 'S0' -// } -// kind: 'OpenAI' -// properties: { -// customSubDomainName: azureOpenAiServiceName -// networkAcls: { -// defaultAction: 'Allow' -// virtualNetworkRules: [] -// ipRules: [] -// } -// publicNetworkAccess: 'Enabled' -// } -// } - -// resource gpt4_deployment 'Microsoft.CognitiveServices/accounts/deployments@2023-10-01-preview' = { -// parent: azureOpenAiService_resource -// name: 'gpt-4' -// sku: { -// name: 'Standard' -// capacity: 10 -// } -// properties: { -// model: { -// format: 'OpenAI' -// name: 'gpt-4' -// version: '0613' -// } -// versionUpgradeOption: 'OnceNewDefaultVersionAvailable' -// currentCapacity: 10 -// raiPolicyName: 'Microsoft.Default' -// } -// } - -// resource adaTextEmbeddingResource 'Microsoft.CognitiveServices/accounts/deployments@2023-10-01-preview' = { -// parent: azureOpenAiService_resource -// name: 'text-embedding-ada-002' -// sku: { -// name: 'Standard' -// capacity: 120 -// } -// properties: { -// model: { -// format: 'OpenAI' -// name: 'text-embedding-ada-002' -// version: '2' -// } -// versionUpgradeOption: 'OnceNewDefaultVersionAvailable' -// currentCapacity: 120 -// raiPolicyName: 'Microsoft.Default' -// } -// } - - -resource azureAiService_resource 'Microsoft.CognitiveServices/accounts@2023-10-01-preview' = { - name: azureAiServiceName - location: location - sku: { - name: 'S0' - } - kind: 'CognitiveServices' - properties: { - customSubDomainName: azureAiServiceName - networkAcls: { - defaultAction: 'Allow' - virtualNetworkRules: [] - ipRules: [] - } - publicNetworkAccess: 'Enabled' - } -} diff --git a/workshops/fabric-e2e-rag/assets/main.json b/workshops/fabric-e2e-rag/assets/main.json deleted file mode 100644 index d7ef69af..00000000 --- a/workshops/fabric-e2e-rag/assets/main.json +++ /dev/null @@ -1,79 +0,0 @@ -{ - "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", - "contentVersion": "1.0.0.0", - "metadata": { - "_generator": { - "name": "bicep", - "version": "0.25.3.34343", - "templateHash": "6968767025245883167" - } - }, - "parameters": { - "openailocation": { - "type": "string", - "defaultValue": "canadaeast" - }, - "location": { - "type": "string", - "defaultValue": "[resourceGroup().location]" - }, - "cognitiveSearchName": { - "type": "string", - "defaultValue": "[substring(format('rag-cogsearch{0}', uniqueString(resourceGroup().id)), 0, 24)]" - }, - "azureOpenAiServiceName": { - "type": "string", - "defaultValue": "[substring(format('rag-azureoai{0}', uniqueString(resourceGroup().id)), 0, 24)]" - }, - "azureAiServiceName": { - "type": "string", - "defaultValue": "[substring(format('rag-aiservices{0}', uniqueString(resourceGroup().id)), 0, 24)]" - } - }, - "resources": [ - { - "type": "Microsoft.Search/searchServices", - "apiVersion": "2023-11-01", - "name": "[parameters('cognitiveSearchName')]", - "location": "[parameters('location')]", - "sku": { - "name": "free" - }, - "properties": { - "replicaCount": 1, - "partitionCount": 1, - "hostingMode": "default", - "publicNetworkAccess": "enabled", - "networkRuleSet": { - "ipRules": [] - }, - "encryptionWithCmk": { - "enforcement": "Unspecified" - }, - "disableLocalAuth": false, - "authOptions": { - "apiKeyOnly": {} - } - } - }, - { - "type": "Microsoft.CognitiveServices/accounts", - "apiVersion": "2023-10-01-preview", - "name": "[parameters('azureAiServiceName')]", - "location": "[parameters('location')]", - "sku": { - "name": "S0" - }, - "kind": "CognitiveServices", - "properties": { - "customSubDomainName": "[parameters('azureAiServiceName')]", - "networkAcls": { - "defaultAction": "Allow", - "virtualNetworkRules": [], - "ipRules": [] - }, - "publicNetworkAccess": "Enabled" - } - } - ] -} \ No newline at end of file diff --git a/workshops/fabric-e2e-rag/assets/new_notebook.png b/workshops/fabric-e2e-rag/assets/new_notebook.png deleted file mode 100644 index cc0dbee8..00000000 Binary files a/workshops/fabric-e2e-rag/assets/new_notebook.png and /dev/null differ diff --git a/workshops/fabric-e2e-rag/assets/openai.png b/workshops/fabric-e2e-rag/assets/openai.png deleted file mode 100644 index 4d0592d2..00000000 Binary files a/workshops/fabric-e2e-rag/assets/openai.png and /dev/null differ diff --git a/workshops/fabric-e2e-rag/assets/response.png b/workshops/fabric-e2e-rag/assets/response.png deleted file mode 100644 index 2f6e9243..00000000 Binary files a/workshops/fabric-e2e-rag/assets/response.png and /dev/null differ diff --git a/workshops/fabric-e2e-rag/assets/schema.png b/workshops/fabric-e2e-rag/assets/schema.png deleted file mode 100644 index baa6d7e5..00000000 Binary files a/workshops/fabric-e2e-rag/assets/schema.png and /dev/null differ diff --git a/workshops/fabric-e2e-rag/workshop.md b/workshops/fabric-e2e-rag/workshop.md deleted file mode 100644 index b0ee664e..00000000 --- a/workshops/fabric-e2e-rag/workshop.md +++ /dev/null @@ -1,715 +0,0 @@ ---- -published: true -type: workshop -title: Building RAG (Retrieval augmented generation) Application on Microsoft Fabric & Azure Open AI -short_title: Building RAG Application on Microsoft Fabric & Azure Open AI -description: This workshop will guide you through the process of building a Retrieval augmented generation (RAG) application on Microsoft Fabric and Azure Open AI. -level: beginner -authors: - - Josh Ndemenge -contacts: - - '@Jcardif' -duration_minutes: 60 -tags: data, Microsoft Fabric, Azure Open AI -banner_url: assets/architecture.png -sections_title: - - Welcome - - Introduction - - Environment Setup - - Loading and Preprocessing PDF Documents - - Generating and Storing Embeddings - - Retrieving Relevant Documents and Answering Questions - - Conclusion - -wt_id: data-114676-jndemenge - ---- - -# Welcome - -In this workshop, we'll demonstrate how to develop a context-aware question answering framework for any form of a document using [OpenAI models](https://azure.microsoft.com/products/ai-services/openai-service), [SynapseML](https://microsoft.github.io/SynapseML/) and [Azure AI Services](https://azure.microsoft.com/products/cognitive-services/). The source of data for this workshop is a PDF document, however, the same framework can be easily extended to other document formats too. - -## Goals - -You'll learn how to: - -- Pre-process PDF Documents using [Azure AI Document Intelligence](https://azure.microsoft.com/products/ai-services/ai-document-intelligence) in Azure AI Services. -- Perform text chunking using SynapseML. -- Generate embeddings for the chunks using SynapseML and [Azure OpenAI Services](https://azure.microsoft.com/products/cognitive-services/openai-service). -- Store the embeddings in [Azure AI Search](https://azure.microsoft.com/products/search). -- Build a question answering pipeline. - -## Pre-requisites - -| | | -|----------------------|------------------------------------------------------| -| Azure account | [Get a free Azure account](https://azure.microsoft.com/free) | -| Microsoft Fabric License | [Microsoft Fabric Licenses](https://learn.microsoft.com/fabric/enterprise/licenses?WT.mc_id=data-114676-jndemenge) | -| A workspace in Microsoft Fabric | [Create a Microsoft Fabric workspace](https://learn.microsoft.com/fabric/data-warehouse/tutorial-create-workspace?WT.mc_id=data-114676-jndemenge) | -| Access to Azure OpenAI API *(optional)* | [Request access to Azure OpenAI](https://aka.ms/oaiapply) | -| A Web browser | [Get Microsoft Edge](https://www.microsoft.com/edge) | -| Python knowledge | [Python for beginners](https://learn.microsoft.com/training/paths/beginner-python/) | - -
- -> Since we are using the Pre-Built Open AI Models in Microsoft Fabric you do not need to request or have access to the Azure OpenAI API. However, if you are using the trial version of Microsoft Fabric or do not have an F64+ capacity, you will need to request access to the Azure OpenAI API. - -
- ---- - -# Introduction - -Analyzing structured data has been an easy process for some time but the same cannot be said for unstructured data. Unstructured data, such as text, images, and videos, is more difficult to analyze and interpret. However, with the advent of advanced AI models, such as OpenAI's GPT-3 and GPT-4, it is now becoming easier to analyze and gain insights from unstructured data. - -An example of such analysis is the ability to query a document for specific information using natural language which is achievable though a combination of information retrieval and language generation. - -By leveraging the RAG (Retrieval-Augmented Generation) framework, you can create a powerful question-and-answering pipeline that uses a large language model (LLM) and you own data to generate responses. - -The architecture of such an application is as shown below: - -![Architecture diagram connecting Azure OpenAI with Azure AI Search and Document Intelligence](assets/schema.png) - -To get an in-depth understanding of the RAG framework, refer to [this workshop](https://moaw.dev/workshop/gh:azure-samples/azure-openai-rag-workshop/base/docs/) - ---- - -# Environment Setup - -To continue with this workshop, you'll need to create a Lakehouse in your Microsoft Fabric workspace and deploy the necessary resources in your Azure account. We'll detail the steps to do this below. - - -## Azure Setup - -To complete this workshop you'll need an Azure account. If you don't have one, you can create a [free account](https://azure.microsoft.com/free/?WT.mc_id=data-0000-cxa) before you begin. - -
- -> Ensure that the subscription you are using has the permissions to create and manage resources. - -
- -Navigate to the [Azure Portal](https://portal.azure.com) and click on `Create a resource` and search for `Azure AI Search`. Click on the `Azure AI Search` resource and then click on `Create`. - -![Screenshot of Azure AI Search creation wizard in Azure Portal](assets/aisearch.png) - -In the creation wizard, create a new resource group or select an existing one. To minimize costs, change the pricing tier from Standard to Free. Click `Review + Create` and once the validation is successful, click `Create`. - -Similarly, create another resource for the `Azure AI Services` by clicking on `Create a resource` and searching for `Azure AI Services`. Click on the `Azure AI Services` resource and then click on `Create`. - -![Screenshot of Azure AI Services creation wizard in Azure Portal](assets/aiservice.png) - -In the creation wizard, select the same resource group that you used for the Azure AI Search resource. The only available pricing tier is Standard, but you can apply free credits if you have any. Accept the Responsible AI Notice. With all fields filled out, click `Review + Create`. Once the validation is successful, click `Create`. - -## Azure OpenAI Set Up - -If your Microsoft Fabric has a SKU of F64 or higher, you can skip this step. - -However, if you're using the trial version of Microsoft Fabric or it does not have an F64+ SKU, you will need to request access to the Azure OpenAI API [using this form](https://aka.ms/oaiapply). Once you have access to the Azure OpenAI API, you'll need to create the Azure OpenAI resource in the Azure Portal. - -To do this navigate to the [Azure Portal] and click on `Create a resource` and search for `Azure OpenAI`. Click on the `Azure OpenAI` resource and then click on `Create`. - -![Screenshot of Azure OpenAI creation wizard in Azure Portal](assets/openai.png) - -Fill out all the required fields and click `Review + Create` and once the validation is successful, click `Create`. Also ensure that you select the same ***Resource Group*** as the ***Azure AI Search resource*** and the ***Azure AI Services resource***. - -Next you'll need to create new model deployments. To do this navigate to the [Azure OpenAI Studio](https://oai.azure.com/portal). Under management click on `Deployments` and then click on `Create Deployment`. You'll need to create two deployments, one for the `text-embedding-ada-002` model and another for the `gpt-35-turbo-16k` model. - -![Screenshot of "Deploy model" dialog in Azure OpenAI Studio](assets/deployments.png) - -
- -> You will have to provide the keys and deployment names for the Azure OpenAI resource in sections that are using the Azure OpenAI models. - -
- -## Create a Lakehouse - -To create a new Lakehouse in your Microsoft Fabric workspace, open the Synapse Data Engineering experience and select the `Lakehouse` button. Provide a name of `rag_workshop` and select `Create`. - -![Screenshot of New Lakehouse dialog in Synapse Data Engineering tab](assets/lakehouse.png) - -To learn more about Lakehouses in Microsoft Fabric, refer to [this Lakehouse tutorial](https://learn.microsoft.com/fabric/data-engineering/tutorial-build-lakehouse#create-a-lakehouse?WT.mc_id=data-114676-jndemenge). - - ---- - -# Loading and Pre-processing PDF Documents - -Now that we have all the necessary resources deployed, we can begin building the RAG application. This section covers the process of loading and preprocessing PDF documents using Document Intelligence in Azure AI Services. - -To do this, we'll perform the following steps: - -- Load the PDF document into a Spark DataFrame. -- Read the documents using the Azure AI Document Intelligence in Azure AI Services. -- Extract the text from the PDF documents. -- Use SynapseML to split the documents into chunks for more granular representation and processing of the document content. - -## Configure Azure API keys - -To begin, navigate back to the `rag_workshop` Lakehouse in your workspace and create a new notebook by selecting `Open Notebook` and selecting `New Notebook` from the options. - -This will open a new notebook. Select the `Save as` icon and rename the notebook to `analyze_and_create_embeddings`. - -Next you'll need to provide the keys for Azure AI Services to access the services. Copy the values from the Azure Portal and paste them into the following code cell. - -```python -# Azure AI Search -AI_SEARCH_NAME = "" -AI_SEARCH_INDEX_NAME = "rag-demo-index" -AI_SEARCH_API_KEY = "" - -# Azure AI Services -AI_SERVICES_KEY = "" -AI_SERVICES_LOCATION = "" -``` - -
- -> In a production scenario, it is recommended to store the credentials securely in Azure Key Vault. To access secrets stored in Azure Key Vault, [use the `mssparkutils` library](https://learn.microsoft.com/fabric/data-engineering/microsoft-spark-utilities#credentials-utilities) as shown below: -> -> ```python -> from notebookutils.mssparkutils.credentials import getSecret -> -> KEYVAULT_ENDPOINT = "https://YOUR-KEY-VAULT-NAME.vault.azure.net/" -> # Azure AI Search -> AI_SEARCH_NAME = "" -> AI_SEARCH_INDEX_NAME = "rag-demo-index" -> AI_SEARCH_API_KEY = getSecret(KEYVAULT_ENDPOINT, "SEARCH-SECRET-NAME") -> # Azure AI Services -> AI_SERVICES_KEY = getSecret(KEYVAULT_ENDPOINT, "AI-SERVICES-SECRET-NAME") -> AI_SERVICES_LOCATION = "" -> ``` - -
- - -## Loading & Analyzing the Document - -In this workshop, we will be using a specific document named [support.pdf](https://github.com/Azure-Samples/azure-openai-rag-workshop/blob/main/data/support.pdf) which will be the source of our data. - -To download the document, paste the following code in a new cell and run it. - -```python -import requests -import os - -url = "https://github.com/Azure-Samples/azure-openai-rag-workshop/raw/main/data/support.pdf" -response = requests.get(url) - -# Specify your path here -path = "/lakehouse/default/Files/" - -# Ensure the directory exists -os.makedirs(path, exist_ok=True) - -# Write the content to a file in the specified path -filename = url.rsplit("/")[-1] -with open(os.path.join(path, filename), "wb") as f: - f.write(response.content) -``` - -This downloads and stores the document in the `Files` directory in the Lakehouse. - -Next, load the PDF document into a Spark DataFrame using the `spark.read.format("binaryFile")` method provided by Apache Spark: - -```python -from pyspark.sql.functions import udf -from pyspark.sql.types import StringType - -document_path = f"Files/{filename}" - -df = spark.read.format("binaryFile").load(document_path).select("_metadata.file_name", "content").limit(10).cache() - -display(df) -``` - -This code will read the PDF document and create a Spark DataFrame named `df` with the contents of the PDF. The DataFrame will have a schema that represents the structure of the PDF document, including its textual content. - -Next, we'll use the Azure AI Document Intelligence to read the PDF documents and extract the text from them. - -We utilize [SynapseML](https://microsoft.github.io/SynapseML/), an ecosystem of tools designed to enhance the distributed computing framework [Apache Spark](https://github.com/apache/spark). SynapseML introduces advanced networking capabilities to the Spark ecosystem and offers user-friendly SparkML transformers for various [Azure AI Services](https://azure.microsoft.com/products/ai-services). - -Additionally, we employ `AnalyzeDocument` from Azure AI Services to extract the complete document content and present it in the designated columns called `output_content` and `paragraph`. - -```python -from synapse.ml.services import AnalyzeDocument -from pyspark.sql.functions import col - -analyze_document = ( - AnalyzeDocument() - .setPrebuiltModelId("prebuilt-layout") - .setSubscriptionKey(AI_SERVICES_KEY) - .setLocation(AI_SERVICES_LOCATION) - .setImageBytesCol("content") - .setOutputCol("result") -) - -analyzed_df = ( - analyze_document.transform(df) - .withColumn("output_content", col("result.analyzeResult.content")) - .withColumn("paragraphs", col("result.analyzeResult.paragraphs")) -).cache() -``` - -We can observe the analyzed Spark DataFrame named ```analyzed_df``` using the following code. Note that we drop the `content` column as it is not needed anymore. - -```python -analyzed_df = analyzed_df.drop("content") -display(analyzed_df) -``` - ---- - -# Generating and Storing Embeddings - -Now that we have the text content of the PDF documents, we can generate embeddings for the text using Azure OpenAI. Embeddings are vector representations of the text that can be used to compare the similarity between different pieces of text. - -![Diagram of flow from entire PDF file to chunks to embeddings](assets/chunking-vector.svg) - -This process begins by splitting the text into chunks, then for each of the chunks we generate embeddings using Azure OpenAI. These embeddings are then stored in Azure AI Search. - -## Text Chunking - -Before we can generate the embeddings, we need to split the text into chunks. To do this we leverage SynapseML’s PageSplitter to divide the documents into smaller sections, which are subsequently stored in the `chunks` column. This allows for more granular representation and processing of the document content. - -```python -from synapse.ml.featurize.text import PageSplitter - -ps = ( - PageSplitter() - .setInputCol("output_content") - .setMaximumPageLength(4000) - .setMinimumPageLength(3000) - .setOutputCol("chunks") -) - -splitted_df = ps.transform(analyzed_df) -display(splitted_df) -``` - -Note that the chunks for each document are presented in a single row inside an array. In order to embed all the chunks in the following cells, we need to have each chunk in a separate row. - -```python -from pyspark.sql.functions import posexplode, col, concat - -# Each "chunks" column contains the chunks for a single document in an array -# The posexplode function will separate each chunk into its own row -exploded_df = splitted_df.select("file_name", posexplode(col("chunks")).alias("chunk_index", "chunk")) - -# Add a unique identifier for each chunk -exploded_df = exploded_df.withColumn("unique_id", concat(exploded_df.file_name, exploded_df.chunk_index)) - -display(exploded_df) -``` - -From this code snippet we first explode these arrays so there is only one chunk in each row, then filter the Spark DataFrame in order to only keep the path to the document and the chunk in a single row. - -## Generating Embeddings - -Next we'll generate the embeddings for each chunk. To do this we utilize both SynapseML and Azure OpenAI Service. By integrating the built in Azure OpenAI service with SynapseML, we can leverage the power of the Apache Spark distributed computing framework to process numerous prompts using the OpenAI service. - -```python -from synapse.ml.services import OpenAIEmbedding - -embedding = ( - OpenAIEmbedding() - .setDeploymentName("text-embedding-ada-002") - .setTextCol("chunk") - .setErrorCol("error") - .setOutputCol("embeddings") -) - -df_embeddings = embedding.transform(exploded_df) - -display(df_embeddings) -``` - -This integration enables the SynapseML embedding client to generate embeddings in a distributed manner, enabling efficient processing of large volumes of data. If you're interested in applying large language models at a distributed scale using Azure OpenAI and Azure Synapse Analytics, you can refer to [this approach](https://microsoft.github.io/SynapseML/docs/Explore%20Algorithms/OpenAI/). - -For more detailed information on generating embeddings with Azure OpenAI, see: [Learn how to generate embeddings with Azure OpenAI](https://learn.microsoft.com/azure/cognitive-services/openai/how-to/embeddings?tabs=console&WT.mc_id=data-114676-jndemenge). - -
- -> If you're using the Azure OpenAI resource deployed on Microsoft Azure, you will need to provide the key as well as the deployment name for the Azure OpenAI resource, using `setDeploymentName` and `setSubscriptionKey`: -> -> ```python -> .setDeploymentName('YOUR-DEPLOYMENT_NAME') -> .setSubscriptionKey('YOUR-AZURE-OPENAI-KEY') -> ``` - -
- -## Storing Embeddings - -[Azure AI Search](https://learn.microsoft.com/azure/search/search-what-is-azure-search?WT.mc_id=data-114676-jndemenge) is a powerful search engine that includes the ability to perform full text search, vector search, and hybrid search. For more examples of its vector search capabilities, see the [azure-search-vector-samples repository](https://github.com/Azure/azure-search-vector-samples/). - -Storing data in Azure AI Search involves two main steps: - -1. **Creating the index:** The first step is to define the schema of the search index, which includes the properties of each field as well as any vector search strategies that will be used. - -2. **Adding chunked documents and embeddings:** The second step is to upload the chunked documents, along with their corresponding embeddings, to the index. This allows for efficient storage and retrieval of the data using hybrid and vector search. - -The following code snippet demonstrates how to create an index in Azure AI Search using the [Azure AI Search REST API](https://learn.microsoft.com/rest/api/searchservice/indexes/create-or-update). This code creates an index with fields for the unique identifier of each document, the text content of the document, and the vector embedding of the text content. - -```python -import requests -import json - -# Length of the embedding vector (OpenAI ada-002 generates embeddings of length 1536) -EMBEDDING_LENGTH = 1536 - -# Create index for AI Search with fields id, content, and contentVector -# Note the datatypes for each field below -url = f"https://{AI_SEARCH_NAME}.search.windows.net/indexes/{AI_SEARCH_INDEX_NAME}?api-version=2023-11-01" -payload = json.dumps( - { - "name": AI_SEARCH_INDEX_NAME, - "fields": [ - # Unique identifier for each document - { - "name": "id", - "type": "Edm.String", - "key": True, - "filterable": True, - }, - # Text content of the document - { - "name": "content", - "type": "Edm.String", - "searchable": True, - "retrievable": True, - }, - # Vector embedding of the text content - { - "name": "contentVector", - "type": "Collection(Edm.Single)", - "searchable": True, - "retrievable": True, - "dimensions": EMBEDDING_LENGTH, - "vectorSearchProfile": "vectorConfig", - }, - ], - "vectorSearch": { - "algorithms": [{"name": "hnswConfig", "kind": "hnsw", "hnswParameters": {"metric": "cosine"}}], - "profiles": [{"name": "vectorConfig", "algorithm": "hnswConfig"}], - }, - } -) -headers = {"Content-Type": "application/json", "api-key": AI_SEARCH_API_KEY} - -response = requests.request("PUT", url, headers=headers, data=payload) -if response.status_code == 201: - print("Index created!") -elif response.status_code == 204: - print("Index updated!") -else: - print(f"HTTP request failed with status code {response.status_code}") - print(f"HTTP response body: {response.text}") -``` - -The next step is to upload the chunks to the newly created Azure AI Search index. The [Azure AI Search REST API](https://learn.microsoft.com/rest/api/searchservice/addupdate-or-delete-documents) supports up to 1000 "documents" per request. Note that in this case, each of our "documents" is in fact a chunk of the original file. - -In order to efficiently upload the chunks to the Azure AI Search index, we'll use the `mapPartitions` function to process each partition of the dataframe. For each partition, the `upload_rows` function will collect 1000 rows at a time and upload them to the index. The function will then return the start and end index of the rows that were uploaded, as well as the status of the insertion, so that we know if the upload was successful or not. - -```python -import re - -from pyspark.sql.functions import monotonically_increasing_id - - -def insert_into_index(documents): - """Uploads a list of 'documents' to Azure AI Search index.""" - - url = f"https://{AI_SEARCH_NAME}.search.windows.net/indexes/{AI_SEARCH_INDEX_NAME}/docs/index?api-version=2023-11-01" - - payload = json.dumps({"value": documents}) - headers = { - "Content-Type": "application/json", - "api-key": AI_SEARCH_API_KEY, - } - - response = requests.request("POST", url, headers=headers, data=payload) - - if response.status_code == 200 or response.status_code == 201: - return "Success" - else: - return f"Failure: {response.text}" - -def make_safe_id(row_id: str): - """Strips disallowed characters from row id for use as Azure AI search document ID.""" - return re.sub("[^0-9a-zA-Z_-]", "_", row_id) - - -def upload_rows(rows): - """Uploads the rows in a Spark dataframe to Azure AI Search. - Limits uploads to 1000 rows at a time due to Azure AI Search API limits. - """ - BATCH_SIZE = 1000 - rows = list(rows) - for i in range(0, len(rows), BATCH_SIZE): - row_batch = rows[i : i + BATCH_SIZE] - documents = [] - for row in rows: - documents.append( - { - "id": make_safe_id(row["unique_id"]), - "content": row["chunk"], - "contentVector": row["embeddings"].tolist(), - "@search.action": "upload", - }, - ) - status = insert_into_index(documents) - yield [row_batch[0]["row_index"], row_batch[-1]["row_index"], status] - -# Add ID to help track what rows were successfully uploaded -df_embeddings = df_embeddings.withColumn("row_index", monotonically_increasing_id()) - -# Run upload_batch on partitions of the dataframe -res = df_embeddings.rdd.mapPartitions(upload_rows) -display(res.toDF(["start_index", "end_index", "insertion_status"])) -``` - -
- -> You can also use the [azure-search-documents Python package](https://pypi.org/project/azure-search-documents/) for Azure AI Search operations. -> You would first need to install that package into the Spark environment. See [Library management in Fabric environments](https://learn.microsoft.com/fabric/data-engineering/environment-manage-library) - -
- ---- - -# Retrieving Relevant Documents and Answering Questions - -After processing the document, we can proceed to pose a question. We will use [SynapseML](https://microsoft.github.io/SynapseML/docs/Explore%20Algorithms/OpenAI/Quickstart%20-%20OpenAI%20Embedding/) to convert the user's question into an embedding and then utilize cosine similarity to retrieve the top K document chunks that closely match the user's question. - -## Configure Environment & Azure API Keys - -Create a new notebook in the Lakehouse and save it as `rag_application`. We'll use this notebook to build the RAG application. - -Next we'll need to provide the credentials for access to Azure AI Search. You can copy the values from the previous notebook or from Azure Portal. - -```python -# Azure AI Search -AI_SEARCH_NAME = '' -AI_SEARCH_INDEX_NAME = 'rag-demo-index' -AI_SEARCH_API_KEY = '' -``` - - -
- -> In a production scenario, it is recommended to store the credentials securely in Azure Key Vault. To access secrets stored in Azure Key Vault, [use the `mssparkutils` library](https://learn.microsoft.com/fabric/data-engineering/microsoft-spark-utilities#credentials-utilities) as shown below: -> -> ```python -> from notebookutils.mssparkutils.credentials import getSecret -> -> KEYVAULT_ENDPOINT = "https://YOUR-KEY-VAULT-NAME.vault.azure.net/" -> # Azure AI Search -> AI_SEARCH_NAME = "" -> AI_SEARCH_INDEX_NAME = "rag-demo-index" -> AI_SEARCH_API_KEY = getSecret(KEYVAULT_ENDPOINT, "SEARCH-SECRET-NAME") -> ``` - -
- -## Generate Embeddings for the User Question - -The first step of the retrieval process is to generate embeddings for the user's question. - -The following function takes a user's question as input and converts it into an embedding using the `text-embedding-ada-002` model. This code assumes you're using the [Pre-built AI Services in Microsoft Fabric](https://learn.microsoft.com/fabric/data-science/ai-services/ai-services-overview?WT.mc_id=data-114676-jndemenge). - -```python -def gen_question_embedding(user_question): - """Generates embedding for user_question using SynapseML.""" - from synapse.ml.services import OpenAIEmbedding - - df_ques = spark.createDataFrame([(user_question, 1)], ["questions", "dummy"]) - embedding = ( - OpenAIEmbedding() - .setDeploymentName('text-embedding-ada-002') - .setTextCol("questions") - .setErrorCol("errorQ") - .setOutputCol("embeddings") - ) - df_ques_embeddings = embedding.transform(df_ques) - row = df_ques_embeddings.collect()[0] - question_embedding = row.embeddings.tolist() - return question_embedding -``` - -
- -> If you're using the Azure OpenAI resource deployed on Microsoft Azure, you will need to provide the key as well as the deployment name for the Azure OpenAI resource, using `setDeploymentName` and `setSubscriptionKey`: -> -> ```python -> .setDeploymentName('YOUR-DEPLOYMENT_NAME') -> .setSubscriptionKey('YOUR-AZURE-OPENAI-KEY') -> ``` - -
- -## Retrieve Relevant Documents - -The next step is to use the user question and its embedding to retrieve the top K most relevant document chunks from the search index. -The following function retrieves the top K entries using hybrid search: - -```python -import json -import requests - -def retrieve_top_chunks(k, question, question_embedding): - """Retrieve the top K entries from Azure AI Search using hybrid search.""" - url = f"https://{AI_SEARCH_NAME}.search.windows.net/indexes/{AI_SEARCH_INDEX_NAME}/docs/search?api-version=2023-11-01" - - payload = json.dumps({ - "search": question, - "top": k, - "vectorQueries": [ - { - "vector": question_embedding, - "k": k, - "fields": "contentVector", - "kind": "vector" - } - ] - }) - - headers = { - "Content-Type": "application/json", - "api-key": AI_SEARCH_API_KEY, - } - - response = requests.request("POST", url, headers=headers, data=payload) - output = json.loads(response.text) - return output -``` - -With those functions defined, we can define a function that takes a user's question, generates an embedding for the question, retrieves the top K document chunks, and concatenates the content of the retrieved documents to form the context for the user's question. - -```python -def get_context(user_question, retrieved_k = 5): - # Generate embeddings for the question - question_embedding = gen_question_embedding(user_question) - - # Retrieve the top K entries - output = retrieve_top_chunks(retrieved_k, user_question, question_embedding) - - # concatenate the content of the retrieved documents - context = [chunk["content"] for chunk in output["value"]] - - return context -``` - -## Answering the User's Question - -Finally, we can define a function that takes a user's question, retrieves the context for the question, and sends both the context and the question to a large language model to generate a response. For this demo, we'll use the `gpt-35-turbo-16k`, a model that is optimized for conversation. The code below assumes you're using the [Pre-built AI Services in Microsoft Fabric](https://learn.microsoft.com/fabric/data-science/ai-services/ai-services-overview?WT.mc_id=data-114676-jndemenge). - -```python -from pyspark.sql import Row -from synapse.ml.services.openai import OpenAIChatCompletion - - -def make_message(role, content): - return Row(role=role, content=content, name=role) - -def get_response(user_question): - context = get_context(user_question) - - # Write a prompt with context and user_question as variables - prompt = f""" - context: {context} - Answer the question based on the context above. - If the information to answer the question is not present in the given context then reply "I don't know". - """ - - chat_df = spark.createDataFrame( - [ - ( - [ - make_message( - "system", prompt - ), - make_message("user", user_question), - ], - ), - ] - ).toDF("messages") - - chat_completion = ( - OpenAIChatCompletion() - .setDeploymentName("gpt-35-turbo-16k") # deploymentName could be one of {gpt-35-turbo, gpt-35-turbo-16k} - .setMessagesCol("messages") - .setErrorCol("error") - .setOutputCol("chat_completions") - ) - - result_df = chat_completion.transform(chat_df).select("chat_completions.choices.message.content") - - result = [] - for row in result_df.collect(): - content_string = ' '.join(row['content']) - result.append(content_string) - - # Join the list into a single string - result = ' '.join(result) - - return result -``` - -
- -> If you're using the Azure OpenAI resource deployed on Microsoft Azure, you will need to provide the key as well as the deployment name for the Azure OpenAI resource, using `setDeploymentName` and `setSubscriptionKey`: -> -> ```python -> .setDeploymentName('YOUR-DEPLOYMENT_NAME') -> .setSubscriptionKey('YOUR-AZURE-OPENAI-KEY') -> ``` - -
- -Finally, we can call that function with an example question to see the response: - -```python -user_question = "how do i make a booking?" -response = get_response(user_question) -print(response) -``` - -This gives a result similar to the following: - -![Screenshot of notebook showing LLM response about making a booking](assets/response.png) - ---- - -# Conclusion - -This concludes this workshop, we hope you enjoyed it and learned something new. - -If you had any issues while following this workshop, please let us know by [creating a new issue](https://github.com/microsoft/moaw/issues) on the github repository. - -## Clean up resources - -
- -> After completing the workshop, remember to delete the Azure Resources you created to avoid incurring unnecessary costs! - -
- -To delete the resources, navigate to the resource group you created earlier and click on the `Delete` button. - -## Resources - -To learn more about Retrieval Augmented Generation (RAG) using Azure Search an Azure OpenAI, refer to the following resources: - -- [Retrieval Augmented Generation (RAG) in Azure AI Search](https://learn.microsoft.com/azure/search/retrieval-augmented-generation-overview?WT.mc_id=data-114676-jndemenge) -- [Use Azure OpenAI in Fabric with Python SDK and Synapse ML (preview)](https://learn.microsoft.com/fabric/data-science/ai-services/how-to-use-openai-sdk-synapse?WT.mc_id=data-114676-jndemenge) -- [Azure OpenAI for big data](https://microsoft.github.io/SynapseML/docs/Explore%20Algorithms/OpenAI/) - -***Bonus:*** For more information on creating RAG applications with Microsoft Fabric, refer to this blog post: [Using Microsoft Fabric’s Lakehouse Data and prompt flow in Azure Machine Learning Service to create RAG applications](https://blog.fabric.microsoft.com/en-us/blog/using-microsoft-fabrics-lakehouse-data-and-prompt-flow-in-azure-machine-learning-service-to-create-rag-applications). - -## References - -- This workshop URL: [aka.ms/ws/fabric-rag](https://aka.ms/ws/fabric-rag) -- If something does not work: [Report an issue](https://github.com/microsoft/moaw/issues) -- Live Workshop on [YouTube](https://www.youtube.com/watch?v=BfNiaaBOcM8) diff --git a/workshops/fabric-e2e-serengeti/assets/Annotation.png b/workshops/fabric-e2e-serengeti/assets/Annotation.png deleted file mode 100644 index f6c30097..00000000 Binary files a/workshops/fabric-e2e-serengeti/assets/Annotation.png and /dev/null differ diff --git a/workshops/fabric-e2e-serengeti/assets/Format_visual.png b/workshops/fabric-e2e-serengeti/assets/Format_visual.png deleted file mode 100644 index 77211e60..00000000 Binary files a/workshops/fabric-e2e-serengeti/assets/Format_visual.png and /dev/null differ diff --git a/workshops/fabric-e2e-serengeti/assets/Lakehouse_interface.png b/workshops/fabric-e2e-serengeti/assets/Lakehouse_interface.png deleted file mode 100644 index 46c6067a..00000000 Binary files a/workshops/fabric-e2e-serengeti/assets/Lakehouse_interface.png and /dev/null differ diff --git a/workshops/fabric-e2e-serengeti/assets/Original_Number_of_Sequences_per_Season.png b/workshops/fabric-e2e-serengeti/assets/Original_Number_of_Sequences_per_Season.png deleted file mode 100644 index 6b932d35..00000000 Binary files a/workshops/fabric-e2e-serengeti/assets/Original_Number_of_Sequences_per_Season.png and /dev/null differ diff --git a/workshops/fabric-e2e-serengeti/assets/SQL_endpoint.png b/workshops/fabric-e2e-serengeti/assets/SQL_endpoint.png deleted file mode 100644 index b5674639..00000000 Binary files a/workshops/fabric-e2e-serengeti/assets/SQL_endpoint.png and /dev/null differ diff --git a/workshops/fabric-e2e-serengeti/assets/Views.png b/workshops/fabric-e2e-serengeti/assets/Views.png deleted file mode 100644 index 0ae8c509..00000000 Binary files a/workshops/fabric-e2e-serengeti/assets/Views.png and /dev/null differ diff --git a/workshops/fabric-e2e-serengeti/assets/Workspace_interface.png b/workshops/fabric-e2e-serengeti/assets/Workspace_interface.png deleted file mode 100644 index 833488f7..00000000 Binary files a/workshops/fabric-e2e-serengeti/assets/Workspace_interface.png and /dev/null differ diff --git a/workshops/fabric-e2e-serengeti/assets/add-for-each.png b/workshops/fabric-e2e-serengeti/assets/add-for-each.png deleted file mode 100644 index 120b33a8..00000000 Binary files a/workshops/fabric-e2e-serengeti/assets/add-for-each.png and /dev/null differ diff --git a/workshops/fabric-e2e-serengeti/assets/analyze-and-transform-data.png b/workshops/fabric-e2e-serengeti/assets/analyze-and-transform-data.png deleted file mode 100644 index 9cc6faca..00000000 Binary files a/workshops/fabric-e2e-serengeti/assets/analyze-and-transform-data.png and /dev/null differ diff --git a/workshops/fabric-e2e-serengeti/assets/architecture.png b/workshops/fabric-e2e-serengeti/assets/architecture.png deleted file mode 100644 index 3cdd8579..00000000 Binary files a/workshops/fabric-e2e-serengeti/assets/architecture.png and /dev/null differ diff --git a/workshops/fabric-e2e-serengeti/assets/card.png b/workshops/fabric-e2e-serengeti/assets/card.png deleted file mode 100644 index 7d5b7eb3..00000000 Binary files a/workshops/fabric-e2e-serengeti/assets/card.png and /dev/null differ diff --git a/workshops/fabric-e2e-serengeti/assets/clustered_barchart.png b/workshops/fabric-e2e-serengeti/assets/clustered_barchart.png deleted file mode 100644 index d6b3b610..00000000 Binary files a/workshops/fabric-e2e-serengeti/assets/clustered_barchart.png and /dev/null differ diff --git a/workshops/fabric-e2e-serengeti/assets/complete-copy.png b/workshops/fabric-e2e-serengeti/assets/complete-copy.png deleted file mode 100644 index f2f85157..00000000 Binary files a/workshops/fabric-e2e-serengeti/assets/complete-copy.png and /dev/null differ diff --git a/workshops/fabric-e2e-serengeti/assets/create-connection.png b/workshops/fabric-e2e-serengeti/assets/create-connection.png deleted file mode 100644 index 2dfb22e8..00000000 Binary files a/workshops/fabric-e2e-serengeti/assets/create-connection.png and /dev/null differ diff --git a/workshops/fabric-e2e-serengeti/assets/create-data-pipeline.png b/workshops/fabric-e2e-serengeti/assets/create-data-pipeline.png deleted file mode 100644 index 4dbab764..00000000 Binary files a/workshops/fabric-e2e-serengeti/assets/create-data-pipeline.png and /dev/null differ diff --git a/workshops/fabric-e2e-serengeti/assets/create-lakehouse.png b/workshops/fabric-e2e-serengeti/assets/create-lakehouse.png deleted file mode 100644 index b8d9b168..00000000 Binary files a/workshops/fabric-e2e-serengeti/assets/create-lakehouse.png and /dev/null differ diff --git a/workshops/fabric-e2e-serengeti/assets/dashboard.png b/workshops/fabric-e2e-serengeti/assets/dashboard.png deleted file mode 100644 index dc247534..00000000 Binary files a/workshops/fabric-e2e-serengeti/assets/dashboard.png and /dev/null differ diff --git a/workshops/fabric-e2e-serengeti/assets/data_to_delta_tables.png b/workshops/fabric-e2e-serengeti/assets/data_to_delta_tables.png deleted file mode 100644 index d9027015..00000000 Binary files a/workshops/fabric-e2e-serengeti/assets/data_to_delta_tables.png and /dev/null differ diff --git a/workshops/fabric-e2e-serengeti/assets/data_to_delta_tables_output.png b/workshops/fabric-e2e-serengeti/assets/data_to_delta_tables_output.png deleted file mode 100644 index 9e6e2fec..00000000 Binary files a/workshops/fabric-e2e-serengeti/assets/data_to_delta_tables_output.png and /dev/null differ diff --git a/workshops/fabric-e2e-serengeti/assets/filter.png b/workshops/fabric-e2e-serengeti/assets/filter.png deleted file mode 100644 index 947c1794..00000000 Binary files a/workshops/fabric-e2e-serengeti/assets/filter.png and /dev/null differ diff --git a/workshops/fabric-e2e-serengeti/assets/import_notebook.png b/workshops/fabric-e2e-serengeti/assets/import_notebook.png deleted file mode 100644 index 22f371fc..00000000 Binary files a/workshops/fabric-e2e-serengeti/assets/import_notebook.png and /dev/null differ diff --git a/workshops/fabric-e2e-serengeti/assets/lakehouse-explorer.png b/workshops/fabric-e2e-serengeti/assets/lakehouse-explorer.png deleted file mode 100644 index c5bd9427..00000000 Binary files a/workshops/fabric-e2e-serengeti/assets/lakehouse-explorer.png and /dev/null differ diff --git a/workshops/fabric-e2e-serengeti/assets/leopard.png b/workshops/fabric-e2e-serengeti/assets/leopard.png deleted file mode 100644 index a15ad1c5..00000000 Binary files a/workshops/fabric-e2e-serengeti/assets/leopard.png and /dev/null differ diff --git a/workshops/fabric-e2e-serengeti/assets/load-to-tables.png b/workshops/fabric-e2e-serengeti/assets/load-to-tables.png deleted file mode 100644 index b697c827..00000000 Binary files a/workshops/fabric-e2e-serengeti/assets/load-to-tables.png and /dev/null differ diff --git a/workshops/fabric-e2e-serengeti/assets/load_data.png b/workshops/fabric-e2e-serengeti/assets/load_data.png deleted file mode 100644 index 198b34a0..00000000 Binary files a/workshops/fabric-e2e-serengeti/assets/load_data.png and /dev/null differ diff --git a/workshops/fabric-e2e-serengeti/assets/missing_lakehouse.png b/workshops/fabric-e2e-serengeti/assets/missing_lakehouse.png deleted file mode 100644 index 040f6ae3..00000000 Binary files a/workshops/fabric-e2e-serengeti/assets/missing_lakehouse.png and /dev/null differ diff --git a/workshops/fabric-e2e-serengeti/assets/mlflow_exp.png b/workshops/fabric-e2e-serengeti/assets/mlflow_exp.png deleted file mode 100644 index 39ab4cfa..00000000 Binary files a/workshops/fabric-e2e-serengeti/assets/mlflow_exp.png and /dev/null differ diff --git a/workshops/fabric-e2e-serengeti/assets/mlflow_model.png b/workshops/fabric-e2e-serengeti/assets/mlflow_model.png deleted file mode 100644 index 8e4743a6..00000000 Binary files a/workshops/fabric-e2e-serengeti/assets/mlflow_model.png and /dev/null differ diff --git a/workshops/fabric-e2e-serengeti/assets/model.png b/workshops/fabric-e2e-serengeti/assets/model.png deleted file mode 100644 index a1690ba7..00000000 Binary files a/workshops/fabric-e2e-serengeti/assets/model.png and /dev/null differ diff --git a/workshops/fabric-e2e-serengeti/assets/model_evaluation.png b/workshops/fabric-e2e-serengeti/assets/model_evaluation.png deleted file mode 100644 index 9fa381fe..00000000 Binary files a/workshops/fabric-e2e-serengeti/assets/model_evaluation.png and /dev/null differ diff --git a/workshops/fabric-e2e-serengeti/assets/model_training.png b/workshops/fabric-e2e-serengeti/assets/model_training.png deleted file mode 100644 index 9672d2d2..00000000 Binary files a/workshops/fabric-e2e-serengeti/assets/model_training.png and /dev/null differ diff --git a/workshops/fabric-e2e-serengeti/assets/notebooks/create_delta_lake_files.ipynb b/workshops/fabric-e2e-serengeti/assets/notebooks/create_delta_lake_files.ipynb deleted file mode 100644 index a8c36df8..00000000 --- a/workshops/fabric-e2e-serengeti/assets/notebooks/create_delta_lake_files.ipynb +++ /dev/null @@ -1 +0,0 @@ -{"cells":[{"cell_type":"code","execution_count":3,"id":"7960e481-fe2a-463f-8118-7cad08f2c191","metadata":{"jupyter":{"outputs_hidden":false,"source_hidden":false},"nteract":{"transient":{"deleting":false}}},"outputs":[{"data":{"application/vnd.livy.statement-meta+json":{"execution_finish_time":"2023-04-20T06:44:09.8363963Z","execution_start_time":"2023-04-20T06:44:09.4082676Z","livy_statement_state":"available","parent_msg_id":"901d891f-e229-43f3-ba8e-a558959907db","queued_time":"2023-04-20T06:44:08.5492385Z","session_id":"5d1aa8d0-49b9-4260-8dac-2ba686428313","session_start_time":"2023-04-20T06:44:09.2362833Z","spark_jobs":{"jobs":[],"limit":20,"numbers":{"FAILED":0,"RUNNING":0,"SUCCEEDED":0,"UNKNOWN":0},"rule":"ALL_DESC"},"spark_pool":null,"state":"finished","statement_id":6},"text/plain":["StatementMeta(, 5d1aa8d0-49b9-4260-8dac-2ba686428313, 6, Finished, Available)"]},"metadata":{},"output_type":"display_data"}],"source":["import os"]},{"cell_type":"code","execution_count":4,"id":"c71bc4ba-ae14-4f00-882d-13aba7555452","metadata":{"jupyter":{"outputs_hidden":false,"source_hidden":false},"nteract":{"transient":{"deleting":false}}},"outputs":[{"data":{"application/vnd.livy.statement-meta+json":{"execution_finish_time":"2023-04-20T06:44:10.5564444Z","execution_start_time":"2023-04-20T06:44:10.2004307Z","livy_statement_state":"available","parent_msg_id":"a1c0a454-5d4f-4743-a0bb-e80f5cb8290b","queued_time":"2023-04-20T06:44:08.5546358Z","session_id":"5d1aa8d0-49b9-4260-8dac-2ba686428313","session_start_time":null,"spark_jobs":{"jobs":[],"limit":20,"numbers":{"FAILED":0,"RUNNING":0,"SUCCEEDED":0,"UNKNOWN":0},"rule":"ALL_DESC"},"spark_pool":null,"state":"finished","statement_id":7},"text/plain":["StatementMeta(, 5d1aa8d0-49b9-4260-8dac-2ba686428313, 7, Finished, Available)"]},"metadata":{},"output_type":"display_data"}],"source":["# Annotations files S01 to S10\n","annotations_file_names = ['Annotations-S01.parquet', 'Annotations-S02.parquet', 'Annotations-S03.parquet', 'Annotations-S04.parquet', 'Annotations-S05.parquet', 'Annotations-S06.parquet', 'Annotations-S07.parquet', 'Annotations-S08.parquet', 'Annotations-S09.parquet', 'Annotations-S10.parquet']\n","\n","# Images files S01 to S10\n","images_file_names = ['Images-S01.parquet', 'Images-S02.parquet', 'Images-S03.parquet', 'Images-S04.parquet', 'Images-S05.parquet', 'Images-S06.parquet', 'Images-S07.parquet', 'Images-S08.parquet', 'Images-S09.parquet', 'Images-S10.parquet']\n","\n","# S11 files\n","s11_annotations_file_name = 'Annotations-S11.parquet'\n","s11_images_file_name = 'Images-S11.parquet'\n","\n","# Categories file\n","categories_file = 'Categories.parquet'\n","\n","# file paths\n","input_dir = 'Files/metadata/parquet'\n","output_dir = 'Files/metadata/delta-lake'"]},{"cell_type":"code","execution_count":7,"id":"c5f6a001-4ed6-4fdb-abf0-cc71df93ff4b","metadata":{"jupyter":{"outputs_hidden":false,"source_hidden":false},"nteract":{"transient":{"deleting":false}}},"outputs":[{"data":{"application/vnd.livy.statement-meta+json":{"execution_finish_time":"2023-04-20T06:46:53.1462521Z","execution_start_time":"2023-04-20T06:46:20.2847115Z","livy_statement_state":"available","parent_msg_id":"e856f324-d414-4db2-8083-972fd288f1b9","queued_time":"2023-04-20T06:46:19.958477Z","session_id":"5d1aa8d0-49b9-4260-8dac-2ba686428313","session_start_time":null,"spark_jobs":{"jobs":[{"completionTime":"2023-04-20T06:46:51.231GMT","dataRead":59,"dataWritten":0,"description":"Job group for statement 10:\n# read the annotations parquet files using spark and append as one dataframe\nannotations_df = spark.read.parquet(os.path.join(input_dir, annotations_file_names[0]))\nfor file_name in annotations_file_names[1:]:\n annotations_df = annotations_df.union(spark.read.parquet(os.path.join(input_dir, file_name)))\nprint(f'annotations : {annotations_df.count()}')\n\n# read the images parquet files using spark and append as one dataframe\nimages_df = spark.read.parquet(os.path.join(input_dir, images_file_names[0]))\nfor file_name in images_file_names[1:]:\n images_df = images_df.union(spark.read.parquet(os.path.join(input_dir, file_name)))\nprint(f'images : {images_df.count()}')\n\n# read the categories parquet file using spark\ncategories_df = spark.read.parquet(os.path.join(input_dir, categories_file))\nprint(f'categories : {categories_df.count()}')","jobGroup":"10","jobId":65,"killedTasksSummary":{},"name":"count at NativeMethodAccessorImpl.java:0","numActiveStages":0,"numActiveTasks":0,"numCompletedIndices":1,"numCompletedStages":1,"numCompletedTasks":1,"numFailedStages":0,"numFailedTasks":0,"numKilledTasks":0,"numSkippedStages":1,"numSkippedTasks":1,"numTasks":2,"rowCount":1,"stageIds":[78,79],"status":"SUCCEEDED","submissionTime":"2023-04-20T06:46:51.204GMT"},{"completionTime":"2023-04-20T06:46:51.194GMT","dataRead":572,"dataWritten":59,"description":"Job group for statement 10:\n# read the annotations parquet files using spark and append as one dataframe\nannotations_df = spark.read.parquet(os.path.join(input_dir, annotations_file_names[0]))\nfor file_name in annotations_file_names[1:]:\n annotations_df = annotations_df.union(spark.read.parquet(os.path.join(input_dir, file_name)))\nprint(f'annotations : {annotations_df.count()}')\n\n# read the images parquet files using spark and append as one dataframe\nimages_df = spark.read.parquet(os.path.join(input_dir, images_file_names[0]))\nfor file_name in images_file_names[1:]:\n images_df = images_df.union(spark.read.parquet(os.path.join(input_dir, file_name)))\nprint(f'images : {images_df.count()}')\n\n# read the categories parquet file using spark\ncategories_df = spark.read.parquet(os.path.join(input_dir, categories_file))\nprint(f'categories : {categories_df.count()}')","jobGroup":"10","jobId":64,"killedTasksSummary":{},"name":"count at NativeMethodAccessorImpl.java:0","numActiveStages":0,"numActiveTasks":0,"numCompletedIndices":1,"numCompletedStages":1,"numCompletedTasks":1,"numFailedStages":0,"numFailedTasks":0,"numKilledTasks":0,"numSkippedStages":0,"numSkippedTasks":0,"numTasks":1,"rowCount":62,"stageIds":[77],"status":"SUCCEEDED","submissionTime":"2023-04-20T06:46:50.142GMT"},{"completionTime":"2023-04-20T06:46:50.048GMT","dataRead":0,"dataWritten":0,"description":"Job group for statement 10:\n# read the annotations parquet files using spark and append as one dataframe\nannotations_df = spark.read.parquet(os.path.join(input_dir, annotations_file_names[0]))\nfor file_name in annotations_file_names[1:]:\n annotations_df = annotations_df.union(spark.read.parquet(os.path.join(input_dir, file_name)))\nprint(f'annotations : {annotations_df.count()}')\n\n# read the images parquet files using spark and append as one dataframe\nimages_df = spark.read.parquet(os.path.join(input_dir, images_file_names[0]))\nfor file_name in images_file_names[1:]:\n images_df = images_df.union(spark.read.parquet(os.path.join(input_dir, file_name)))\nprint(f'images : {images_df.count()}')\n\n# read the categories parquet file using spark\ncategories_df = spark.read.parquet(os.path.join(input_dir, categories_file))\nprint(f'categories : {categories_df.count()}')","jobGroup":"10","jobId":63,"killedTasksSummary":{},"name":"parquet at :0","numActiveStages":0,"numActiveTasks":0,"numCompletedIndices":1,"numCompletedStages":1,"numCompletedTasks":1,"numFailedStages":0,"numFailedTasks":0,"numKilledTasks":0,"numSkippedStages":0,"numSkippedTasks":0,"numTasks":1,"rowCount":0,"stageIds":[76],"status":"SUCCEEDED","submissionTime":"2023-04-20T06:46:49.604GMT"},{"completionTime":"2023-04-20T06:46:49.067GMT","dataRead":2662,"dataWritten":0,"description":"Job group for statement 10:\n# read the annotations parquet files using spark and append as one dataframe\nannotations_df = spark.read.parquet(os.path.join(input_dir, annotations_file_names[0]))\nfor file_name in annotations_file_names[1:]:\n annotations_df = annotations_df.union(spark.read.parquet(os.path.join(input_dir, file_name)))\nprint(f'annotations : {annotations_df.count()}')\n\n# read the images parquet files using spark and append as one dataframe\nimages_df = spark.read.parquet(os.path.join(input_dir, images_file_names[0]))\nfor file_name in images_file_names[1:]:\n images_df = images_df.union(spark.read.parquet(os.path.join(input_dir, file_name)))\nprint(f'images : {images_df.count()}')\n\n# read the categories parquet file using spark\ncategories_df = spark.read.parquet(os.path.join(input_dir, categories_file))\nprint(f'categories : {categories_df.count()}')","jobGroup":"10","jobId":62,"killedTasksSummary":{},"name":"count at NativeMethodAccessorImpl.java:0","numActiveStages":0,"numActiveTasks":0,"numCompletedIndices":1,"numCompletedStages":1,"numCompletedTasks":1,"numFailedStages":0,"numFailedTasks":0,"numKilledTasks":0,"numSkippedStages":1,"numSkippedTasks":47,"numTasks":48,"rowCount":47,"stageIds":[74,75],"status":"SUCCEEDED","submissionTime":"2023-04-20T06:46:49.040GMT"},{"completionTime":"2023-04-20T06:46:49.026GMT","dataRead":68315,"dataWritten":2662,"description":"Job group for statement 10:\n# read the annotations parquet files using spark and append as one dataframe\nannotations_df = spark.read.parquet(os.path.join(input_dir, annotations_file_names[0]))\nfor file_name in annotations_file_names[1:]:\n annotations_df = annotations_df.union(spark.read.parquet(os.path.join(input_dir, file_name)))\nprint(f'annotations : {annotations_df.count()}')\n\n# read the images parquet files using spark and append as one dataframe\nimages_df = spark.read.parquet(os.path.join(input_dir, images_file_names[0]))\nfor file_name in images_file_names[1:]:\n images_df = images_df.union(spark.read.parquet(os.path.join(input_dir, file_name)))\nprint(f'images : {images_df.count()}')\n\n# read the categories parquet file using spark\ncategories_df = spark.read.parquet(os.path.join(input_dir, categories_file))\nprint(f'categories : {categories_df.count()}')","jobGroup":"10","jobId":61,"killedTasksSummary":{},"name":"count at NativeMethodAccessorImpl.java:0","numActiveStages":0,"numActiveTasks":0,"numCompletedIndices":47,"numCompletedStages":1,"numCompletedTasks":47,"numFailedStages":0,"numFailedTasks":0,"numKilledTasks":0,"numSkippedStages":0,"numSkippedTasks":0,"numTasks":47,"rowCount":6679086,"stageIds":[73],"status":"SUCCEEDED","submissionTime":"2023-04-20T06:46:45.582GMT"},{"completionTime":"2023-04-20T06:46:45.301GMT","dataRead":0,"dataWritten":0,"description":"Job group for statement 10:\n# read the annotations parquet files using spark and append as one dataframe\nannotations_df = spark.read.parquet(os.path.join(input_dir, annotations_file_names[0]))\nfor file_name in annotations_file_names[1:]:\n annotations_df = annotations_df.union(spark.read.parquet(os.path.join(input_dir, file_name)))\nprint(f'annotations : {annotations_df.count()}')\n\n# read the images parquet files using spark and append as one dataframe\nimages_df = spark.read.parquet(os.path.join(input_dir, images_file_names[0]))\nfor file_name in images_file_names[1:]:\n images_df = images_df.union(spark.read.parquet(os.path.join(input_dir, file_name)))\nprint(f'images : {images_df.count()}')\n\n# read the categories parquet file using spark\ncategories_df = spark.read.parquet(os.path.join(input_dir, categories_file))\nprint(f'categories : {categories_df.count()}')","jobGroup":"10","jobId":60,"killedTasksSummary":{},"name":"parquet at :0","numActiveStages":0,"numActiveTasks":0,"numCompletedIndices":1,"numCompletedStages":1,"numCompletedTasks":1,"numFailedStages":0,"numFailedTasks":0,"numKilledTasks":0,"numSkippedStages":0,"numSkippedTasks":0,"numTasks":1,"rowCount":0,"stageIds":[72],"status":"SUCCEEDED","submissionTime":"2023-04-20T06:46:44.825GMT"},{"completionTime":"2023-04-20T06:46:44.347GMT","dataRead":0,"dataWritten":0,"description":"Job group for statement 10:\n# read the annotations parquet files using spark and append as one dataframe\nannotations_df = spark.read.parquet(os.path.join(input_dir, annotations_file_names[0]))\nfor file_name in annotations_file_names[1:]:\n annotations_df = annotations_df.union(spark.read.parquet(os.path.join(input_dir, file_name)))\nprint(f'annotations : {annotations_df.count()}')\n\n# read the images parquet files using spark and append as one dataframe\nimages_df = spark.read.parquet(os.path.join(input_dir, images_file_names[0]))\nfor file_name in images_file_names[1:]:\n images_df = images_df.union(spark.read.parquet(os.path.join(input_dir, file_name)))\nprint(f'images : {images_df.count()}')\n\n# read the categories parquet file using spark\ncategories_df = spark.read.parquet(os.path.join(input_dir, categories_file))\nprint(f'categories : {categories_df.count()}')","jobGroup":"10","jobId":59,"killedTasksSummary":{},"name":"parquet at :0","numActiveStages":0,"numActiveTasks":0,"numCompletedIndices":1,"numCompletedStages":1,"numCompletedTasks":1,"numFailedStages":0,"numFailedTasks":0,"numKilledTasks":0,"numSkippedStages":0,"numSkippedTasks":0,"numTasks":1,"rowCount":0,"stageIds":[71],"status":"SUCCEEDED","submissionTime":"2023-04-20T06:46:43.894GMT"},{"completionTime":"2023-04-20T06:46:43.446GMT","dataRead":0,"dataWritten":0,"description":"Job group for statement 10:\n# read the annotations parquet files using spark and append as one dataframe\nannotations_df = spark.read.parquet(os.path.join(input_dir, annotations_file_names[0]))\nfor file_name in annotations_file_names[1:]:\n annotations_df = annotations_df.union(spark.read.parquet(os.path.join(input_dir, file_name)))\nprint(f'annotations : {annotations_df.count()}')\n\n# read the images parquet files using spark and append as one dataframe\nimages_df = spark.read.parquet(os.path.join(input_dir, images_file_names[0]))\nfor file_name in images_file_names[1:]:\n images_df = images_df.union(spark.read.parquet(os.path.join(input_dir, file_name)))\nprint(f'images : {images_df.count()}')\n\n# read the categories parquet file using spark\ncategories_df = spark.read.parquet(os.path.join(input_dir, categories_file))\nprint(f'categories : {categories_df.count()}')","jobGroup":"10","jobId":58,"killedTasksSummary":{},"name":"parquet at :0","numActiveStages":0,"numActiveTasks":0,"numCompletedIndices":1,"numCompletedStages":1,"numCompletedTasks":1,"numFailedStages":0,"numFailedTasks":0,"numKilledTasks":0,"numSkippedStages":0,"numSkippedTasks":0,"numTasks":1,"rowCount":0,"stageIds":[70],"status":"SUCCEEDED","submissionTime":"2023-04-20T06:46:42.984GMT"},{"completionTime":"2023-04-20T06:46:42.460GMT","dataRead":0,"dataWritten":0,"description":"Job group for statement 10:\n# read the annotations parquet files using spark and append as one dataframe\nannotations_df = spark.read.parquet(os.path.join(input_dir, annotations_file_names[0]))\nfor file_name in annotations_file_names[1:]:\n annotations_df = annotations_df.union(spark.read.parquet(os.path.join(input_dir, file_name)))\nprint(f'annotations : {annotations_df.count()}')\n\n# read the images parquet files using spark and append as one dataframe\nimages_df = spark.read.parquet(os.path.join(input_dir, images_file_names[0]))\nfor file_name in images_file_names[1:]:\n images_df = images_df.union(spark.read.parquet(os.path.join(input_dir, file_name)))\nprint(f'images : {images_df.count()}')\n\n# read the categories parquet file using spark\ncategories_df = spark.read.parquet(os.path.join(input_dir, categories_file))\nprint(f'categories : {categories_df.count()}')","jobGroup":"10","jobId":57,"killedTasksSummary":{},"name":"parquet at :0","numActiveStages":0,"numActiveTasks":0,"numCompletedIndices":1,"numCompletedStages":1,"numCompletedTasks":1,"numFailedStages":0,"numFailedTasks":0,"numKilledTasks":0,"numSkippedStages":0,"numSkippedTasks":0,"numTasks":1,"rowCount":0,"stageIds":[69],"status":"SUCCEEDED","submissionTime":"2023-04-20T06:46:42.030GMT"},{"completionTime":"2023-04-20T06:46:41.524GMT","dataRead":0,"dataWritten":0,"description":"Job group for statement 10:\n# read the annotations parquet files using spark and append as one dataframe\nannotations_df = spark.read.parquet(os.path.join(input_dir, annotations_file_names[0]))\nfor file_name in annotations_file_names[1:]:\n annotations_df = annotations_df.union(spark.read.parquet(os.path.join(input_dir, file_name)))\nprint(f'annotations : {annotations_df.count()}')\n\n# read the images parquet files using spark and append as one dataframe\nimages_df = spark.read.parquet(os.path.join(input_dir, images_file_names[0]))\nfor file_name in images_file_names[1:]:\n images_df = images_df.union(spark.read.parquet(os.path.join(input_dir, file_name)))\nprint(f'images : {images_df.count()}')\n\n# read the categories parquet file using spark\ncategories_df = spark.read.parquet(os.path.join(input_dir, categories_file))\nprint(f'categories : {categories_df.count()}')","jobGroup":"10","jobId":56,"killedTasksSummary":{},"name":"parquet at :0","numActiveStages":0,"numActiveTasks":0,"numCompletedIndices":1,"numCompletedStages":1,"numCompletedTasks":1,"numFailedStages":0,"numFailedTasks":0,"numKilledTasks":0,"numSkippedStages":0,"numSkippedTasks":0,"numTasks":1,"rowCount":0,"stageIds":[68],"status":"SUCCEEDED","submissionTime":"2023-04-20T06:46:41.088GMT"},{"completionTime":"2023-04-20T06:46:40.621GMT","dataRead":0,"dataWritten":0,"description":"Job group for statement 10:\n# read the annotations parquet files using spark and append as one dataframe\nannotations_df = spark.read.parquet(os.path.join(input_dir, annotations_file_names[0]))\nfor file_name in annotations_file_names[1:]:\n annotations_df = annotations_df.union(spark.read.parquet(os.path.join(input_dir, file_name)))\nprint(f'annotations : {annotations_df.count()}')\n\n# read the images parquet files using spark and append as one dataframe\nimages_df = spark.read.parquet(os.path.join(input_dir, images_file_names[0]))\nfor file_name in images_file_names[1:]:\n images_df = images_df.union(spark.read.parquet(os.path.join(input_dir, file_name)))\nprint(f'images : {images_df.count()}')\n\n# read the categories parquet file using spark\ncategories_df = spark.read.parquet(os.path.join(input_dir, categories_file))\nprint(f'categories : {categories_df.count()}')","jobGroup":"10","jobId":55,"killedTasksSummary":{},"name":"parquet at :0","numActiveStages":0,"numActiveTasks":0,"numCompletedIndices":1,"numCompletedStages":1,"numCompletedTasks":1,"numFailedStages":0,"numFailedTasks":0,"numKilledTasks":0,"numSkippedStages":0,"numSkippedTasks":0,"numTasks":1,"rowCount":0,"stageIds":[67],"status":"SUCCEEDED","submissionTime":"2023-04-20T06:46:40.200GMT"},{"completionTime":"2023-04-20T06:46:39.739GMT","dataRead":0,"dataWritten":0,"description":"Job group for statement 10:\n# read the annotations parquet files using spark and append as one dataframe\nannotations_df = spark.read.parquet(os.path.join(input_dir, annotations_file_names[0]))\nfor file_name in annotations_file_names[1:]:\n annotations_df = annotations_df.union(spark.read.parquet(os.path.join(input_dir, file_name)))\nprint(f'annotations : {annotations_df.count()}')\n\n# read the images parquet files using spark and append as one dataframe\nimages_df = spark.read.parquet(os.path.join(input_dir, images_file_names[0]))\nfor file_name in images_file_names[1:]:\n images_df = images_df.union(spark.read.parquet(os.path.join(input_dir, file_name)))\nprint(f'images : {images_df.count()}')\n\n# read the categories parquet file using spark\ncategories_df = spark.read.parquet(os.path.join(input_dir, categories_file))\nprint(f'categories : {categories_df.count()}')","jobGroup":"10","jobId":54,"killedTasksSummary":{},"name":"parquet at :0","numActiveStages":0,"numActiveTasks":0,"numCompletedIndices":1,"numCompletedStages":1,"numCompletedTasks":1,"numFailedStages":0,"numFailedTasks":0,"numKilledTasks":0,"numSkippedStages":0,"numSkippedTasks":0,"numTasks":1,"rowCount":0,"stageIds":[66],"status":"SUCCEEDED","submissionTime":"2023-04-20T06:46:39.177GMT"},{"completionTime":"2023-04-20T06:46:38.685GMT","dataRead":0,"dataWritten":0,"description":"Job group for statement 10:\n# read the annotations parquet files using spark and append as one dataframe\nannotations_df = spark.read.parquet(os.path.join(input_dir, annotations_file_names[0]))\nfor file_name in annotations_file_names[1:]:\n annotations_df = annotations_df.union(spark.read.parquet(os.path.join(input_dir, file_name)))\nprint(f'annotations : {annotations_df.count()}')\n\n# read the images parquet files using spark and append as one dataframe\nimages_df = spark.read.parquet(os.path.join(input_dir, images_file_names[0]))\nfor file_name in images_file_names[1:]:\n images_df = images_df.union(spark.read.parquet(os.path.join(input_dir, file_name)))\nprint(f'images : {images_df.count()}')\n\n# read the categories parquet file using spark\ncategories_df = spark.read.parquet(os.path.join(input_dir, categories_file))\nprint(f'categories : {categories_df.count()}')","jobGroup":"10","jobId":53,"killedTasksSummary":{},"name":"parquet at :0","numActiveStages":0,"numActiveTasks":0,"numCompletedIndices":1,"numCompletedStages":1,"numCompletedTasks":1,"numFailedStages":0,"numFailedTasks":0,"numKilledTasks":0,"numSkippedStages":0,"numSkippedTasks":0,"numTasks":1,"rowCount":0,"stageIds":[65],"status":"SUCCEEDED","submissionTime":"2023-04-20T06:46:38.236GMT"},{"completionTime":"2023-04-20T06:46:37.713GMT","dataRead":0,"dataWritten":0,"description":"Job group for statement 10:\n# read the annotations parquet files using spark and append as one dataframe\nannotations_df = spark.read.parquet(os.path.join(input_dir, annotations_file_names[0]))\nfor file_name in annotations_file_names[1:]:\n annotations_df = annotations_df.union(spark.read.parquet(os.path.join(input_dir, file_name)))\nprint(f'annotations : {annotations_df.count()}')\n\n# read the images parquet files using spark and append as one dataframe\nimages_df = spark.read.parquet(os.path.join(input_dir, images_file_names[0]))\nfor file_name in images_file_names[1:]:\n images_df = images_df.union(spark.read.parquet(os.path.join(input_dir, file_name)))\nprint(f'images : {images_df.count()}')\n\n# read the categories parquet file using spark\ncategories_df = spark.read.parquet(os.path.join(input_dir, categories_file))\nprint(f'categories : {categories_df.count()}')","jobGroup":"10","jobId":52,"killedTasksSummary":{},"name":"parquet at :0","numActiveStages":0,"numActiveTasks":0,"numCompletedIndices":1,"numCompletedStages":1,"numCompletedTasks":1,"numFailedStages":0,"numFailedTasks":0,"numKilledTasks":0,"numSkippedStages":0,"numSkippedTasks":0,"numTasks":1,"rowCount":0,"stageIds":[64],"status":"SUCCEEDED","submissionTime":"2023-04-20T06:46:37.260GMT"},{"completionTime":"2023-04-20T06:46:36.776GMT","dataRead":0,"dataWritten":0,"description":"Job group for statement 10:\n# read the annotations parquet files using spark and append as one dataframe\nannotations_df = spark.read.parquet(os.path.join(input_dir, annotations_file_names[0]))\nfor file_name in annotations_file_names[1:]:\n annotations_df = annotations_df.union(spark.read.parquet(os.path.join(input_dir, file_name)))\nprint(f'annotations : {annotations_df.count()}')\n\n# read the images parquet files using spark and append as one dataframe\nimages_df = spark.read.parquet(os.path.join(input_dir, images_file_names[0]))\nfor file_name in images_file_names[1:]:\n images_df = images_df.union(spark.read.parquet(os.path.join(input_dir, file_name)))\nprint(f'images : {images_df.count()}')\n\n# read the categories parquet file using spark\ncategories_df = spark.read.parquet(os.path.join(input_dir, categories_file))\nprint(f'categories : {categories_df.count()}')","jobGroup":"10","jobId":51,"killedTasksSummary":{},"name":"parquet at :0","numActiveStages":0,"numActiveTasks":0,"numCompletedIndices":1,"numCompletedStages":1,"numCompletedTasks":1,"numFailedStages":0,"numFailedTasks":0,"numKilledTasks":0,"numSkippedStages":0,"numSkippedTasks":0,"numTasks":1,"rowCount":0,"stageIds":[63],"status":"SUCCEEDED","submissionTime":"2023-04-20T06:46:36.306GMT"},{"completionTime":"2023-04-20T06:46:35.829GMT","dataRead":2774,"dataWritten":0,"description":"Job group for statement 10:\n# read the annotations parquet files using spark and append as one dataframe\nannotations_df = spark.read.parquet(os.path.join(input_dir, annotations_file_names[0]))\nfor file_name in annotations_file_names[1:]:\n annotations_df = annotations_df.union(spark.read.parquet(os.path.join(input_dir, file_name)))\nprint(f'annotations : {annotations_df.count()}')\n\n# read the images parquet files using spark and append as one dataframe\nimages_df = spark.read.parquet(os.path.join(input_dir, images_file_names[0]))\nfor file_name in images_file_names[1:]:\n images_df = images_df.union(spark.read.parquet(os.path.join(input_dir, file_name)))\nprint(f'images : {images_df.count()}')\n\n# read the categories parquet file using spark\ncategories_df = spark.read.parquet(os.path.join(input_dir, categories_file))\nprint(f'categories : {categories_df.count()}')","jobGroup":"10","jobId":50,"killedTasksSummary":{},"name":"count at NativeMethodAccessorImpl.java:0","numActiveStages":0,"numActiveTasks":0,"numCompletedIndices":1,"numCompletedStages":1,"numCompletedTasks":1,"numFailedStages":0,"numFailedTasks":0,"numKilledTasks":0,"numSkippedStages":1,"numSkippedTasks":49,"numTasks":50,"rowCount":49,"stageIds":[61,62],"status":"SUCCEEDED","submissionTime":"2023-04-20T06:46:35.793GMT"},{"completionTime":"2023-04-20T06:46:35.780GMT","dataRead":91281,"dataWritten":2774,"description":"Job group for statement 10:\n# read the annotations parquet files using spark and append as one dataframe\nannotations_df = spark.read.parquet(os.path.join(input_dir, annotations_file_names[0]))\nfor file_name in annotations_file_names[1:]:\n annotations_df = annotations_df.union(spark.read.parquet(os.path.join(input_dir, file_name)))\nprint(f'annotations : {annotations_df.count()}')\n\n# read the images parquet files using spark and append as one dataframe\nimages_df = spark.read.parquet(os.path.join(input_dir, images_file_names[0]))\nfor file_name in images_file_names[1:]:\n images_df = images_df.union(spark.read.parquet(os.path.join(input_dir, file_name)))\nprint(f'images : {images_df.count()}')\n\n# read the categories parquet file using spark\ncategories_df = spark.read.parquet(os.path.join(input_dir, categories_file))\nprint(f'categories : {categories_df.count()}')","jobGroup":"10","jobId":49,"killedTasksSummary":{},"name":"count at NativeMethodAccessorImpl.java:0","numActiveStages":0,"numActiveTasks":0,"numCompletedIndices":49,"numCompletedStages":1,"numCompletedTasks":49,"numFailedStages":0,"numFailedTasks":0,"numKilledTasks":0,"numSkippedStages":0,"numSkippedTasks":0,"numTasks":49,"rowCount":6755139,"stageIds":[60],"status":"SUCCEEDED","submissionTime":"2023-04-20T06:46:32.370GMT"},{"completionTime":"2023-04-20T06:46:32.087GMT","dataRead":0,"dataWritten":0,"description":"Job group for statement 10:\n# read the annotations parquet files using spark and append as one dataframe\nannotations_df = spark.read.parquet(os.path.join(input_dir, annotations_file_names[0]))\nfor file_name in annotations_file_names[1:]:\n annotations_df = annotations_df.union(spark.read.parquet(os.path.join(input_dir, file_name)))\nprint(f'annotations : {annotations_df.count()}')\n\n# read the images parquet files using spark and append as one dataframe\nimages_df = spark.read.parquet(os.path.join(input_dir, images_file_names[0]))\nfor file_name in images_file_names[1:]:\n images_df = images_df.union(spark.read.parquet(os.path.join(input_dir, file_name)))\nprint(f'images : {images_df.count()}')\n\n# read the categories parquet file using spark\ncategories_df = spark.read.parquet(os.path.join(input_dir, categories_file))\nprint(f'categories : {categories_df.count()}')","jobGroup":"10","jobId":48,"killedTasksSummary":{},"name":"parquet at :0","numActiveStages":0,"numActiveTasks":0,"numCompletedIndices":1,"numCompletedStages":1,"numCompletedTasks":1,"numFailedStages":0,"numFailedTasks":0,"numKilledTasks":0,"numSkippedStages":0,"numSkippedTasks":0,"numTasks":1,"rowCount":0,"stageIds":[59],"status":"SUCCEEDED","submissionTime":"2023-04-20T06:46:31.617GMT"},{"completionTime":"2023-04-20T06:46:31.156GMT","dataRead":0,"dataWritten":0,"description":"Job group for statement 10:\n# read the annotations parquet files using spark and append as one dataframe\nannotations_df = spark.read.parquet(os.path.join(input_dir, annotations_file_names[0]))\nfor file_name in annotations_file_names[1:]:\n annotations_df = annotations_df.union(spark.read.parquet(os.path.join(input_dir, file_name)))\nprint(f'annotations : {annotations_df.count()}')\n\n# read the images parquet files using spark and append as one dataframe\nimages_df = spark.read.parquet(os.path.join(input_dir, images_file_names[0]))\nfor file_name in images_file_names[1:]:\n images_df = images_df.union(spark.read.parquet(os.path.join(input_dir, file_name)))\nprint(f'images : {images_df.count()}')\n\n# read the categories parquet file using spark\ncategories_df = spark.read.parquet(os.path.join(input_dir, categories_file))\nprint(f'categories : {categories_df.count()}')","jobGroup":"10","jobId":47,"killedTasksSummary":{},"name":"parquet at :0","numActiveStages":0,"numActiveTasks":0,"numCompletedIndices":1,"numCompletedStages":1,"numCompletedTasks":1,"numFailedStages":0,"numFailedTasks":0,"numKilledTasks":0,"numSkippedStages":0,"numSkippedTasks":0,"numTasks":1,"rowCount":0,"stageIds":[58],"status":"SUCCEEDED","submissionTime":"2023-04-20T06:46:30.732GMT"},{"completionTime":"2023-04-20T06:46:30.264GMT","dataRead":0,"dataWritten":0,"description":"Job group for statement 10:\n# read the annotations parquet files using spark and append as one dataframe\nannotations_df = spark.read.parquet(os.path.join(input_dir, annotations_file_names[0]))\nfor file_name in annotations_file_names[1:]:\n annotations_df = annotations_df.union(spark.read.parquet(os.path.join(input_dir, file_name)))\nprint(f'annotations : {annotations_df.count()}')\n\n# read the images parquet files using spark and append as one dataframe\nimages_df = spark.read.parquet(os.path.join(input_dir, images_file_names[0]))\nfor file_name in images_file_names[1:]:\n images_df = images_df.union(spark.read.parquet(os.path.join(input_dir, file_name)))\nprint(f'images : {images_df.count()}')\n\n# read the categories parquet file using spark\ncategories_df = spark.read.parquet(os.path.join(input_dir, categories_file))\nprint(f'categories : {categories_df.count()}')","jobGroup":"10","jobId":46,"killedTasksSummary":{},"name":"parquet at :0","numActiveStages":0,"numActiveTasks":0,"numCompletedIndices":1,"numCompletedStages":1,"numCompletedTasks":1,"numFailedStages":0,"numFailedTasks":0,"numKilledTasks":0,"numSkippedStages":0,"numSkippedTasks":0,"numTasks":1,"rowCount":0,"stageIds":[57],"status":"SUCCEEDED","submissionTime":"2023-04-20T06:46:29.828GMT"}],"limit":20,"numbers":{"FAILED":0,"RUNNING":0,"SUCCEEDED":27,"UNKNOWN":0},"rule":"ALL_DESC"},"spark_pool":null,"state":"finished","statement_id":10},"text/plain":["StatementMeta(, 5d1aa8d0-49b9-4260-8dac-2ba686428313, 10, Finished, Available)"]},"metadata":{},"output_type":"display_data"},{"name":"stdout","output_type":"stream","text":["annotations : 6755090\n","images : 6679039\n","categories : 61\n"]}],"source":["# read the annotations parquet files using spark and append as one dataframe\n","annotations_df = spark.read.parquet(os.path.join(input_dir, annotations_file_names[0]))\n","for file_name in annotations_file_names[1:]:\n"," annotations_df = annotations_df.union(spark.read.parquet(os.path.join(input_dir, file_name)))\n","print(f'annotations : {annotations_df.count()}')\n","\n","# read the images parquet files using spark and append as one dataframe\n","images_df = spark.read.parquet(os.path.join(input_dir, images_file_names[0]))\n","for file_name in images_file_names[1:]:\n"," images_df = images_df.union(spark.read.parquet(os.path.join(input_dir, file_name)))\n","print(f'images : {images_df.count()}')\n","\n","# read the categories parquet file using spark\n","categories_df = spark.read.parquet(os.path.join(input_dir, categories_file))\n","print(f'categories : {categories_df.count()}')"]},{"cell_type":"code","execution_count":10,"id":"27262c23-bc84-4cc2-b060-ee7e31437160","metadata":{"advisor":{"adviceMetadata":"{\"artifactId\":\"76cad586-c48c-4c6d-b703-019950925c2e\",\"activityId\":\"5d1aa8d0-49b9-4260-8dac-2ba686428313\",\"applicationId\":\"application_1681971504383_0001\",\"jobGroupId\":\"13\",\"advices\":{\"info\":3,\"warn\":2}}"}},"outputs":[{"data":{"application/vnd.livy.statement-meta+json":{"execution_finish_time":"2023-04-20T06:49:33.5361004Z","execution_start_time":"2023-04-20T06:48:56.0265366Z","livy_statement_state":"available","parent_msg_id":"7072a564-abeb-40f9-a5e2-7e115cae5fea","queued_time":"2023-04-20T06:48:55.7347141Z","session_id":"5d1aa8d0-49b9-4260-8dac-2ba686428313","session_start_time":null,"spark_jobs":{"jobs":[{"completionTime":"2023-04-20T06:49:31.649GMT","dataRead":4322,"dataWritten":0,"description":"Delta: Job group for statement 13:\n# write the annotations dataframe to a delta lake table\nannotations_df.write.format('delta').mode('overwrite').save(os.path.join(output_dir, 'annotations'))\n\n# write the images dataframe to a delta lake table\nimages_df.write.format('delta').mode('overwrite').save(os.path.join(output_dir, 'images'))\n\n# write the categories dataframe to a delta lake table\ncategories_df.write.format('delta').mode('overwrite').save(os.path.join(output_dir, 'categories')): Compute snapshot for version: 2","jobGroup":"13","jobId":116,"killedTasksSummary":{},"name":"$anonfun$recordDeltaOperation$5 at SynapseLoggingShim.scala:95","numActiveStages":0,"numActiveTasks":0,"numCompletedIndices":1,"numCompletedStages":1,"numCompletedTasks":1,"numFailedStages":0,"numFailedTasks":0,"numKilledTasks":0,"numSkippedStages":2,"numSkippedTasks":53,"numTasks":54,"rowCount":50,"stageIds":[171,169,170],"status":"SUCCEEDED","submissionTime":"2023-04-20T06:49:31.619GMT"},{"completionTime":"2023-04-20T06:49:31.600GMT","dataRead":4639,"dataWritten":4322,"description":"Delta: Job group for statement 13:\n# write the annotations dataframe to a delta lake table\nannotations_df.write.format('delta').mode('overwrite').save(os.path.join(output_dir, 'annotations'))\n\n# write the images dataframe to a delta lake table\nimages_df.write.format('delta').mode('overwrite').save(os.path.join(output_dir, 'images'))\n\n# write the categories dataframe to a delta lake table\ncategories_df.write.format('delta').mode('overwrite').save(os.path.join(output_dir, 'categories')): Compute snapshot for version: 2","jobGroup":"13","jobId":115,"killedTasksSummary":{},"name":"$anonfun$recordDeltaOperation$5 at SynapseLoggingShim.scala:95","numActiveStages":0,"numActiveTasks":0,"numCompletedIndices":50,"numCompletedStages":1,"numCompletedTasks":50,"numFailedStages":0,"numFailedTasks":0,"numKilledTasks":0,"numSkippedStages":1,"numSkippedTasks":3,"numTasks":53,"rowCount":60,"stageIds":[168,167],"status":"SUCCEEDED","submissionTime":"2023-04-20T06:49:31.417GMT"},{"completionTime":"2023-04-20T06:49:31.293GMT","dataRead":3179,"dataWritten":4639,"description":"Delta: Job group for statement 13:\n# write the annotations dataframe to a delta lake table\nannotations_df.write.format('delta').mode('overwrite').save(os.path.join(output_dir, 'annotations'))\n\n# write the images dataframe to a delta lake table\nimages_df.write.format('delta').mode('overwrite').save(os.path.join(output_dir, 'images'))\n\n# write the categories dataframe to a delta lake table\ncategories_df.write.format('delta').mode('overwrite').save(os.path.join(output_dir, 'categories')): Compute snapshot for version: 2","jobGroup":"13","jobId":114,"killedTasksSummary":{},"name":"$anonfun$recordDeltaOperation$5 at SynapseLoggingShim.scala:95","numActiveStages":0,"numActiveTasks":0,"numCompletedIndices":3,"numCompletedStages":1,"numCompletedTasks":3,"numFailedStages":0,"numFailedTasks":0,"numKilledTasks":0,"numSkippedStages":0,"numSkippedTasks":0,"numTasks":3,"rowCount":20,"stageIds":[166],"status":"SUCCEEDED","submissionTime":"2023-04-20T06:49:30.935GMT"},{"completionTime":"2023-04-20T06:49:29.367GMT","dataRead":1879,"dataWritten":0,"description":"Job group for statement 13:\n# write the annotations dataframe to a delta lake table\nannotations_df.write.format('delta').mode('overwrite').save(os.path.join(output_dir, 'annotations'))\n\n# write the images dataframe to a delta lake table\nimages_df.write.format('delta').mode('overwrite').save(os.path.join(output_dir, 'images'))\n\n# write the categories dataframe to a delta lake table\ncategories_df.write.format('delta').mode('overwrite').save(os.path.join(output_dir, 'categories'))","jobGroup":"13","jobId":113,"killedTasksSummary":{},"name":"$anonfun$recordDeltaOperation$5 at SynapseLoggingShim.scala:95","numActiveStages":0,"numActiveTasks":0,"numCompletedIndices":50,"numCompletedStages":1,"numCompletedTasks":50,"numFailedStages":0,"numFailedTasks":0,"numKilledTasks":0,"numSkippedStages":1,"numSkippedTasks":2,"numTasks":52,"rowCount":4,"stageIds":[165,164],"status":"SUCCEEDED","submissionTime":"2023-04-20T06:49:29.319GMT"},{"completionTime":"2023-04-20T06:49:29.253GMT","dataRead":2333,"dataWritten":2163,"description":"Job group for statement 13:\n# write the annotations dataframe to a delta lake table\nannotations_df.write.format('delta').mode('overwrite').save(os.path.join(output_dir, 'annotations'))\n\n# write the images dataframe to a delta lake table\nimages_df.write.format('delta').mode('overwrite').save(os.path.join(output_dir, 'images'))\n\n# write the categories dataframe to a delta lake table\ncategories_df.write.format('delta').mode('overwrite').save(os.path.join(output_dir, 'categories'))","jobGroup":"13","jobId":112,"killedTasksSummary":{},"name":"save at NativeMethodAccessorImpl.java:0","numActiveStages":0,"numActiveTasks":0,"numCompletedIndices":1,"numCompletedStages":1,"numCompletedTasks":1,"numFailedStages":0,"numFailedTasks":0,"numKilledTasks":0,"numSkippedStages":0,"numSkippedTasks":0,"numTasks":1,"rowCount":122,"stageIds":[163],"status":"SUCCEEDED","submissionTime":"2023-04-20T06:49:26.420GMT"},{"completionTime":"2023-04-20T06:49:25.956GMT","dataRead":4625,"dataWritten":0,"description":"Delta: Job group for statement 13:\n# write the annotations dataframe to a delta lake table\nannotations_df.write.format('delta').mode('overwrite').save(os.path.join(output_dir, 'annotations'))\n\n# write the images dataframe to a delta lake table\nimages_df.write.format('delta').mode('overwrite').save(os.path.join(output_dir, 'images'))\n\n# write the categories dataframe to a delta lake table\ncategories_df.write.format('delta').mode('overwrite').save(os.path.join(output_dir, 'categories')): Compute snapshot for version: 4","jobGroup":"13","jobId":111,"killedTasksSummary":{},"name":"$anonfun$recordDeltaOperation$5 at SynapseLoggingShim.scala:95","numActiveStages":0,"numActiveTasks":0,"numCompletedIndices":1,"numCompletedStages":1,"numCompletedTasks":1,"numFailedStages":0,"numFailedTasks":0,"numKilledTasks":0,"numSkippedStages":2,"numSkippedTasks":55,"numTasks":56,"rowCount":50,"stageIds":[161,162,160],"status":"SUCCEEDED","submissionTime":"2023-04-20T06:49:25.930GMT"},{"completionTime":"2023-04-20T06:49:25.916GMT","dataRead":36167,"dataWritten":4625,"description":"Delta: Job group for statement 13:\n# write the annotations dataframe to a delta lake table\nannotations_df.write.format('delta').mode('overwrite').save(os.path.join(output_dir, 'annotations'))\n\n# write the images dataframe to a delta lake table\nimages_df.write.format('delta').mode('overwrite').save(os.path.join(output_dir, 'images'))\n\n# write the categories dataframe to a delta lake table\ncategories_df.write.format('delta').mode('overwrite').save(os.path.join(output_dir, 'categories')): Compute snapshot for version: 4","jobGroup":"13","jobId":110,"killedTasksSummary":{},"name":"$anonfun$recordDeltaOperation$5 at SynapseLoggingShim.scala:95","numActiveStages":0,"numActiveTasks":0,"numCompletedIndices":50,"numCompletedStages":1,"numCompletedTasks":50,"numFailedStages":0,"numFailedTasks":0,"numKilledTasks":0,"numSkippedStages":1,"numSkippedTasks":5,"numTasks":55,"rowCount":120,"stageIds":[158,159],"status":"SUCCEEDED","submissionTime":"2023-04-20T06:49:25.728GMT"},{"completionTime":"2023-04-20T06:49:25.595GMT","dataRead":33146,"dataWritten":36167,"description":"Delta: Job group for statement 13:\n# write the annotations dataframe to a delta lake table\nannotations_df.write.format('delta').mode('overwrite').save(os.path.join(output_dir, 'annotations'))\n\n# write the images dataframe to a delta lake table\nimages_df.write.format('delta').mode('overwrite').save(os.path.join(output_dir, 'images'))\n\n# write the categories dataframe to a delta lake table\ncategories_df.write.format('delta').mode('overwrite').save(os.path.join(output_dir, 'categories')): Compute snapshot for version: 4","jobGroup":"13","jobId":109,"killedTasksSummary":{},"name":"$anonfun$recordDeltaOperation$5 at SynapseLoggingShim.scala:95","numActiveStages":0,"numActiveTasks":0,"numCompletedIndices":5,"numCompletedStages":1,"numCompletedTasks":5,"numFailedStages":0,"numFailedTasks":0,"numKilledTasks":0,"numSkippedStages":0,"numSkippedTasks":0,"numTasks":5,"rowCount":140,"stageIds":[157],"status":"SUCCEEDED","submissionTime":"2023-04-20T06:49:25.228GMT"},{"completionTime":"2023-04-20T06:49:23.414GMT","dataRead":6493,"dataWritten":0,"description":"Job group for statement 13:\n# write the annotations dataframe to a delta lake table\nannotations_df.write.format('delta').mode('overwrite').save(os.path.join(output_dir, 'annotations'))\n\n# write the images dataframe to a delta lake table\nimages_df.write.format('delta').mode('overwrite').save(os.path.join(output_dir, 'images'))\n\n# write the categories dataframe to a delta lake table\ncategories_df.write.format('delta').mode('overwrite').save(os.path.join(output_dir, 'categories'))","jobGroup":"13","jobId":108,"killedTasksSummary":{},"name":"$anonfun$recordDeltaOperation$5 at SynapseLoggingShim.scala:95","numActiveStages":0,"numActiveTasks":0,"numCompletedIndices":50,"numCompletedStages":1,"numCompletedTasks":50,"numFailedStages":0,"numFailedTasks":0,"numKilledTasks":0,"numSkippedStages":1,"numSkippedTasks":4,"numTasks":54,"rowCount":28,"stageIds":[155,156],"status":"SUCCEEDED","submissionTime":"2023-04-20T06:49:23.293GMT"},{"completionTime":"2023-04-20T06:49:23.220GMT","dataRead":175221620,"dataWritten":183660092,"description":"Job group for statement 13:\n# write the annotations dataframe to a delta lake table\nannotations_df.write.format('delta').mode('overwrite').save(os.path.join(output_dir, 'annotations'))\n\n# write the images dataframe to a delta lake table\nimages_df.write.format('delta').mode('overwrite').save(os.path.join(output_dir, 'images'))\n\n# write the categories dataframe to a delta lake table\ncategories_df.write.format('delta').mode('overwrite').save(os.path.join(output_dir, 'categories'))","jobGroup":"13","jobId":107,"killedTasksSummary":{},"name":"save at NativeMethodAccessorImpl.java:0","numActiveStages":0,"numActiveTasks":0,"numCompletedIndices":47,"numCompletedStages":1,"numCompletedTasks":47,"numFailedStages":0,"numFailedTasks":0,"numKilledTasks":0,"numSkippedStages":0,"numSkippedTasks":0,"numTasks":47,"rowCount":13358078,"stageIds":[154],"status":"SUCCEEDED","submissionTime":"2023-04-20T06:49:13.876GMT"},{"completionTime":"2023-04-20T06:49:13.302GMT","dataRead":4731,"dataWritten":0,"description":"Delta: Job group for statement 13:\n# write the annotations dataframe to a delta lake table\nannotations_df.write.format('delta').mode('overwrite').save(os.path.join(output_dir, 'annotations'))\n\n# write the images dataframe to a delta lake table\nimages_df.write.format('delta').mode('overwrite').save(os.path.join(output_dir, 'images'))\n\n# write the categories dataframe to a delta lake table\ncategories_df.write.format('delta').mode('overwrite').save(os.path.join(output_dir, 'categories')): Compute snapshot for version: 4","jobGroup":"13","jobId":106,"killedTasksSummary":{},"name":"$anonfun$recordDeltaOperation$5 at SynapseLoggingShim.scala:95","numActiveStages":0,"numActiveTasks":0,"numCompletedIndices":1,"numCompletedStages":1,"numCompletedTasks":1,"numFailedStages":0,"numFailedTasks":0,"numKilledTasks":0,"numSkippedStages":2,"numSkippedTasks":55,"numTasks":56,"rowCount":50,"stageIds":[153,151,152],"status":"SUCCEEDED","submissionTime":"2023-04-20T06:49:13.270GMT"},{"completionTime":"2023-04-20T06:49:13.256GMT","dataRead":46620,"dataWritten":4731,"description":"Delta: Job group for statement 13:\n# write the annotations dataframe to a delta lake table\nannotations_df.write.format('delta').mode('overwrite').save(os.path.join(output_dir, 'annotations'))\n\n# write the images dataframe to a delta lake table\nimages_df.write.format('delta').mode('overwrite').save(os.path.join(output_dir, 'images'))\n\n# write the categories dataframe to a delta lake table\ncategories_df.write.format('delta').mode('overwrite').save(os.path.join(output_dir, 'categories')): Compute snapshot for version: 4","jobGroup":"13","jobId":105,"killedTasksSummary":{},"name":"$anonfun$recordDeltaOperation$5 at SynapseLoggingShim.scala:95","numActiveStages":0,"numActiveTasks":0,"numCompletedIndices":50,"numCompletedStages":1,"numCompletedTasks":50,"numFailedStages":0,"numFailedTasks":0,"numKilledTasks":0,"numSkippedStages":1,"numSkippedTasks":5,"numTasks":55,"rowCount":120,"stageIds":[150,149],"status":"SUCCEEDED","submissionTime":"2023-04-20T06:49:13.069GMT"},{"completionTime":"2023-04-20T06:49:12.929GMT","dataRead":52031,"dataWritten":46620,"description":"Delta: Job group for statement 13:\n# write the annotations dataframe to a delta lake table\nannotations_df.write.format('delta').mode('overwrite').save(os.path.join(output_dir, 'annotations'))\n\n# write the images dataframe to a delta lake table\nimages_df.write.format('delta').mode('overwrite').save(os.path.join(output_dir, 'images'))\n\n# write the categories dataframe to a delta lake table\ncategories_df.write.format('delta').mode('overwrite').save(os.path.join(output_dir, 'categories')): Compute snapshot for version: 4","jobGroup":"13","jobId":104,"killedTasksSummary":{},"name":"$anonfun$recordDeltaOperation$5 at SynapseLoggingShim.scala:95","numActiveStages":0,"numActiveTasks":0,"numCompletedIndices":5,"numCompletedStages":1,"numCompletedTasks":5,"numFailedStages":0,"numFailedTasks":0,"numKilledTasks":0,"numSkippedStages":0,"numSkippedTasks":0,"numTasks":5,"rowCount":140,"stageIds":[148],"status":"SUCCEEDED","submissionTime":"2023-04-20T06:49:12.596GMT"},{"completionTime":"2023-04-20T06:49:10.830GMT","dataRead":7297,"dataWritten":0,"description":"Job group for statement 13:\n# write the annotations dataframe to a delta lake table\nannotations_df.write.format('delta').mode('overwrite').save(os.path.join(output_dir, 'annotations'))\n\n# write the images dataframe to a delta lake table\nimages_df.write.format('delta').mode('overwrite').save(os.path.join(output_dir, 'images'))\n\n# write the categories dataframe to a delta lake table\ncategories_df.write.format('delta').mode('overwrite').save(os.path.join(output_dir, 'categories'))","jobGroup":"13","jobId":103,"killedTasksSummary":{},"name":"$anonfun$recordDeltaOperation$5 at SynapseLoggingShim.scala:95","numActiveStages":0,"numActiveTasks":0,"numCompletedIndices":50,"numCompletedStages":1,"numCompletedTasks":50,"numFailedStages":0,"numFailedTasks":0,"numKilledTasks":0,"numSkippedStages":1,"numSkippedTasks":4,"numTasks":54,"rowCount":28,"stageIds":[147,146],"status":"SUCCEEDED","submissionTime":"2023-04-20T06:49:10.780GMT"},{"completionTime":"2023-04-20T06:49:10.632GMT","dataRead":188741311,"dataWritten":234586217,"description":"Job group for statement 13:\n# write the annotations dataframe to a delta lake table\nannotations_df.write.format('delta').mode('overwrite').save(os.path.join(output_dir, 'annotations'))\n\n# write the images dataframe to a delta lake table\nimages_df.write.format('delta').mode('overwrite').save(os.path.join(output_dir, 'images'))\n\n# write the categories dataframe to a delta lake table\ncategories_df.write.format('delta').mode('overwrite').save(os.path.join(output_dir, 'categories'))","jobGroup":"13","jobId":102,"killedTasksSummary":{},"name":"save at NativeMethodAccessorImpl.java:0","numActiveStages":0,"numActiveTasks":0,"numCompletedIndices":49,"numCompletedStages":1,"numCompletedTasks":49,"numFailedStages":0,"numFailedTasks":0,"numKilledTasks":0,"numSkippedStages":0,"numSkippedTasks":0,"numTasks":49,"rowCount":13510180,"stageIds":[145],"status":"SUCCEEDED","submissionTime":"2023-04-20T06:48:56.583GMT"}],"limit":20,"numbers":{"FAILED":0,"RUNNING":0,"SUCCEEDED":15,"UNKNOWN":0},"rule":"ALL_DESC"},"spark_pool":null,"state":"finished","statement_id":13},"text/plain":["StatementMeta(, 5d1aa8d0-49b9-4260-8dac-2ba686428313, 13, Finished, Available)"]},"metadata":{},"output_type":"display_data"}],"source":["# write the annotations dataframe to a delta lake table\n","annotations_df.write.format('delta').mode('overwrite').save(os.path.join(output_dir, 'annotations'))\n","\n","# write the images dataframe to a delta lake table\n","images_df.write.format('delta').mode('overwrite').save(os.path.join(output_dir, 'images'))\n","\n","# write the categories dataframe to a delta lake table\n","categories_df.write.format('delta').mode('overwrite').save(os.path.join(output_dir, 'categories'))"]},{"cell_type":"code","execution_count":13,"id":"e028a064-cdac-4833-9f8a-66b89488caff","metadata":{"advisor":{"adviceMetadata":"{\"artifactId\":\"76cad586-c48c-4c6d-b703-019950925c2e\",\"activityId\":\"5d1aa8d0-49b9-4260-8dac-2ba686428313\",\"applicationId\":\"application_1681971504383_0001\",\"jobGroupId\":\"16\",\"advices\":{\"info\":2,\"warn\":2}}"},"jupyter":{"outputs_hidden":false,"source_hidden":false},"nteract":{"transient":{"deleting":false}}},"outputs":[{"data":{"application/vnd.livy.statement-meta+json":{"execution_finish_time":"2023-04-20T07:01:38.1236776Z","execution_start_time":"2023-04-20T07:00:42.7239086Z","livy_statement_state":"available","parent_msg_id":"15da15f6-f7fe-4778-bf4d-3eb56e1c0d8b","queued_time":"2023-04-20T07:00:42.4154781Z","session_id":"5d1aa8d0-49b9-4260-8dac-2ba686428313","session_start_time":null,"spark_jobs":{"jobs":[{"completionTime":"2023-04-20T07:01:37.839GMT","dataRead":4400,"dataWritten":0,"description":"Delta: Job group for statement 16:\n# read and write S11 annotations and images\ns11_annotations_df = spark.read.parquet(os.path.join(input_dir, s11_annotations_file_name))\ns11_annotations_df.write.format('delta').mode('overwrite').save(os.path.join(output_dir, 's11annotations'))\n\ns11_images_df = spark.read.parquet(os.path.join(input_dir, s11_images_file_name))\ns11_images_df.write.format('delta').mode('overwrite').save(os.path.join(output_dir, 's11images')): Compute snapshot for version: 0","jobGroup":"16","jobId":152,"killedTasksSummary":{},"name":"$anonfun$recordDeltaOperation$5 at SynapseLoggingShim.scala:95","numActiveStages":0,"numActiveTasks":0,"numCompletedIndices":1,"numCompletedStages":1,"numCompletedTasks":1,"numFailedStages":0,"numFailedTasks":0,"numKilledTasks":0,"numSkippedStages":2,"numSkippedTasks":51,"numTasks":52,"rowCount":50,"stageIds":[219,217,218],"status":"SUCCEEDED","submissionTime":"2023-04-20T07:01:37.811GMT"},{"completionTime":"2023-04-20T07:01:37.798GMT","dataRead":2136,"dataWritten":4400,"description":"Delta: Job group for statement 16:\n# read and write S11 annotations and images\ns11_annotations_df = spark.read.parquet(os.path.join(input_dir, s11_annotations_file_name))\ns11_annotations_df.write.format('delta').mode('overwrite').save(os.path.join(output_dir, 's11annotations'))\n\ns11_images_df = spark.read.parquet(os.path.join(input_dir, s11_images_file_name))\ns11_images_df.write.format('delta').mode('overwrite').save(os.path.join(output_dir, 's11images')): Compute snapshot for version: 0","jobGroup":"16","jobId":151,"killedTasksSummary":{},"name":"$anonfun$recordDeltaOperation$5 at SynapseLoggingShim.scala:95","numActiveStages":0,"numActiveTasks":0,"numCompletedIndices":50,"numCompletedStages":1,"numCompletedTasks":50,"numFailedStages":0,"numFailedTasks":0,"numKilledTasks":0,"numSkippedStages":1,"numSkippedTasks":1,"numTasks":51,"rowCount":55,"stageIds":[215,216],"status":"SUCCEEDED","submissionTime":"2023-04-20T07:01:37.635GMT"},{"completionTime":"2023-04-20T07:01:37.500GMT","dataRead":2220,"dataWritten":2136,"description":"Delta: Job group for statement 16:\n# read and write S11 annotations and images\ns11_annotations_df = spark.read.parquet(os.path.join(input_dir, s11_annotations_file_name))\ns11_annotations_df.write.format('delta').mode('overwrite').save(os.path.join(output_dir, 's11annotations'))\n\ns11_images_df = spark.read.parquet(os.path.join(input_dir, s11_images_file_name))\ns11_images_df.write.format('delta').mode('overwrite').save(os.path.join(output_dir, 's11images')): Compute snapshot for version: 0","jobGroup":"16","jobId":150,"killedTasksSummary":{},"name":"$anonfun$recordDeltaOperation$5 at SynapseLoggingShim.scala:95","numActiveStages":0,"numActiveTasks":0,"numCompletedIndices":1,"numCompletedStages":1,"numCompletedTasks":1,"numFailedStages":0,"numFailedTasks":0,"numKilledTasks":0,"numSkippedStages":0,"numSkippedTasks":0,"numTasks":1,"rowCount":10,"stageIds":[214],"status":"SUCCEEDED","submissionTime":"2023-04-20T07:01:37.175GMT"},{"completionTime":"2023-04-20T07:01:31.868GMT","dataRead":0,"dataWritten":0,"description":"Job group for statement 16:\n# read and write S11 annotations and images\ns11_annotations_df = spark.read.parquet(os.path.join(input_dir, s11_annotations_file_name))\ns11_annotations_df.write.format('delta').mode('overwrite').save(os.path.join(output_dir, 's11annotations'))\n\ns11_images_df = spark.read.parquet(os.path.join(input_dir, s11_images_file_name))\ns11_images_df.write.format('delta').mode('overwrite').save(os.path.join(output_dir, 's11images'))","jobGroup":"16","jobId":149,"killedTasksSummary":{},"name":"","numActiveStages":0,"numActiveTasks":0,"numCompletedIndices":0,"numCompletedStages":0,"numCompletedTasks":0,"numFailedStages":0,"numFailedTasks":0,"numKilledTasks":0,"numSkippedStages":0,"numSkippedTasks":0,"numTasks":0,"rowCount":0,"stageIds":[],"status":"SUCCEEDED","submissionTime":"2023-04-20T07:01:31.868GMT"},{"completionTime":"2023-04-20T07:01:31.751GMT","dataRead":13101793,"dataWritten":13414630,"description":"Job group for statement 16:\n# read and write S11 annotations and images\ns11_annotations_df = spark.read.parquet(os.path.join(input_dir, s11_annotations_file_name))\ns11_annotations_df.write.format('delta').mode('overwrite').save(os.path.join(output_dir, 's11annotations'))\n\ns11_images_df = spark.read.parquet(os.path.join(input_dir, s11_images_file_name))\ns11_images_df.write.format('delta').mode('overwrite').save(os.path.join(output_dir, 's11images'))","jobGroup":"16","jobId":148,"killedTasksSummary":{},"name":"save at NativeMethodAccessorImpl.java:0","numActiveStages":0,"numActiveTasks":0,"numCompletedIndices":4,"numCompletedStages":1,"numCompletedTasks":4,"numFailedStages":0,"numFailedTasks":0,"numKilledTasks":0,"numSkippedStages":0,"numSkippedTasks":0,"numTasks":4,"rowCount":998802,"stageIds":[213],"status":"SUCCEEDED","submissionTime":"2023-04-20T07:01:25.189GMT"},{"completionTime":"2023-04-20T07:01:24.213GMT","dataRead":0,"dataWritten":0,"description":"Job group for statement 16:\n# read and write S11 annotations and images\ns11_annotations_df = spark.read.parquet(os.path.join(input_dir, s11_annotations_file_name))\ns11_annotations_df.write.format('delta').mode('overwrite').save(os.path.join(output_dir, 's11annotations'))\n\ns11_images_df = spark.read.parquet(os.path.join(input_dir, s11_images_file_name))\ns11_images_df.write.format('delta').mode('overwrite').save(os.path.join(output_dir, 's11images'))","jobGroup":"16","jobId":147,"killedTasksSummary":{},"name":"parquet at :0","numActiveStages":0,"numActiveTasks":0,"numCompletedIndices":1,"numCompletedStages":1,"numCompletedTasks":1,"numFailedStages":0,"numFailedTasks":0,"numKilledTasks":0,"numSkippedStages":0,"numSkippedTasks":0,"numTasks":1,"rowCount":0,"stageIds":[212],"status":"SUCCEEDED","submissionTime":"2023-04-20T07:01:23.750GMT"},{"completionTime":"2023-04-20T07:01:23.232GMT","dataRead":4502,"dataWritten":0,"description":"Delta: Job group for statement 16:\n# read and write S11 annotations and images\ns11_annotations_df = spark.read.parquet(os.path.join(input_dir, s11_annotations_file_name))\ns11_annotations_df.write.format('delta').mode('overwrite').save(os.path.join(output_dir, 's11annotations'))\n\ns11_images_df = spark.read.parquet(os.path.join(input_dir, s11_images_file_name))\ns11_images_df.write.format('delta').mode('overwrite').save(os.path.join(output_dir, 's11images')): Compute snapshot for version: 0","jobGroup":"16","jobId":146,"killedTasksSummary":{},"name":"$anonfun$recordDeltaOperation$5 at SynapseLoggingShim.scala:95","numActiveStages":0,"numActiveTasks":0,"numCompletedIndices":1,"numCompletedStages":1,"numCompletedTasks":1,"numFailedStages":0,"numFailedTasks":0,"numKilledTasks":0,"numSkippedStages":2,"numSkippedTasks":51,"numTasks":52,"rowCount":50,"stageIds":[209,210,211],"status":"SUCCEEDED","submissionTime":"2023-04-20T07:01:23.203GMT"},{"completionTime":"2023-04-20T07:01:23.188GMT","dataRead":2546,"dataWritten":4502,"description":"Delta: Job group for statement 16:\n# read and write S11 annotations and images\ns11_annotations_df = spark.read.parquet(os.path.join(input_dir, s11_annotations_file_name))\ns11_annotations_df.write.format('delta').mode('overwrite').save(os.path.join(output_dir, 's11annotations'))\n\ns11_images_df = spark.read.parquet(os.path.join(input_dir, s11_images_file_name))\ns11_images_df.write.format('delta').mode('overwrite').save(os.path.join(output_dir, 's11images')): Compute snapshot for version: 0","jobGroup":"16","jobId":145,"killedTasksSummary":{},"name":"$anonfun$recordDeltaOperation$5 at SynapseLoggingShim.scala:95","numActiveStages":0,"numActiveTasks":0,"numCompletedIndices":50,"numCompletedStages":1,"numCompletedTasks":50,"numFailedStages":0,"numFailedTasks":0,"numKilledTasks":0,"numSkippedStages":1,"numSkippedTasks":1,"numTasks":51,"rowCount":55,"stageIds":[208,207],"status":"SUCCEEDED","submissionTime":"2023-04-20T07:01:23.022GMT"},{"completionTime":"2023-04-20T07:01:22.900GMT","dataRead":3440,"dataWritten":2546,"description":"Delta: Job group for statement 16:\n# read and write S11 annotations and images\ns11_annotations_df = spark.read.parquet(os.path.join(input_dir, s11_annotations_file_name))\ns11_annotations_df.write.format('delta').mode('overwrite').save(os.path.join(output_dir, 's11annotations'))\n\ns11_images_df = spark.read.parquet(os.path.join(input_dir, s11_images_file_name))\ns11_images_df.write.format('delta').mode('overwrite').save(os.path.join(output_dir, 's11images')): Compute snapshot for version: 0","jobGroup":"16","jobId":144,"killedTasksSummary":{},"name":"$anonfun$recordDeltaOperation$5 at SynapseLoggingShim.scala:95","numActiveStages":0,"numActiveTasks":0,"numCompletedIndices":1,"numCompletedStages":1,"numCompletedTasks":1,"numFailedStages":0,"numFailedTasks":0,"numKilledTasks":0,"numSkippedStages":0,"numSkippedTasks":0,"numTasks":1,"rowCount":10,"stageIds":[206],"status":"SUCCEEDED","submissionTime":"2023-04-20T07:01:22.556GMT"},{"completionTime":"2023-04-20T07:01:20.702GMT","dataRead":0,"dataWritten":0,"description":"Job group for statement 16:\n# read and write S11 annotations and images\ns11_annotations_df = spark.read.parquet(os.path.join(input_dir, s11_annotations_file_name))\ns11_annotations_df.write.format('delta').mode('overwrite').save(os.path.join(output_dir, 's11annotations'))\n\ns11_images_df = spark.read.parquet(os.path.join(input_dir, s11_images_file_name))\ns11_images_df.write.format('delta').mode('overwrite').save(os.path.join(output_dir, 's11images'))","jobGroup":"16","jobId":143,"killedTasksSummary":{},"name":"","numActiveStages":0,"numActiveTasks":0,"numCompletedIndices":0,"numCompletedStages":0,"numCompletedTasks":0,"numFailedStages":0,"numFailedTasks":0,"numKilledTasks":0,"numSkippedStages":0,"numSkippedTasks":0,"numTasks":0,"rowCount":0,"stageIds":[],"status":"SUCCEEDED","submissionTime":"2023-04-20T07:01:20.702GMT"},{"completionTime":"2023-04-20T07:01:20.624GMT","dataRead":14238314,"dataWritten":17391704,"description":"Job group for statement 16:\n# read and write S11 annotations and images\ns11_annotations_df = spark.read.parquet(os.path.join(input_dir, s11_annotations_file_name))\ns11_annotations_df.write.format('delta').mode('overwrite').save(os.path.join(output_dir, 's11annotations'))\n\ns11_images_df = spark.read.parquet(os.path.join(input_dir, s11_images_file_name))\ns11_images_df.write.format('delta').mode('overwrite').save(os.path.join(output_dir, 's11images'))","jobGroup":"16","jobId":142,"killedTasksSummary":{},"name":"save at NativeMethodAccessorImpl.java:0","numActiveStages":0,"numActiveTasks":0,"numCompletedIndices":4,"numCompletedStages":1,"numCompletedTasks":4,"numFailedStages":0,"numFailedTasks":0,"numKilledTasks":0,"numSkippedStages":0,"numSkippedTasks":0,"numTasks":4,"rowCount":1012910,"stageIds":[205],"status":"SUCCEEDED","submissionTime":"2023-04-20T07:01:11.402GMT"},{"completionTime":"2023-04-20T07:01:10.472GMT","dataRead":0,"dataWritten":0,"description":"Job group for statement 16:\n# read and write S11 annotations and images\ns11_annotations_df = spark.read.parquet(os.path.join(input_dir, s11_annotations_file_name))\ns11_annotations_df.write.format('delta').mode('overwrite').save(os.path.join(output_dir, 's11annotations'))\n\ns11_images_df = spark.read.parquet(os.path.join(input_dir, s11_images_file_name))\ns11_images_df.write.format('delta').mode('overwrite').save(os.path.join(output_dir, 's11images'))","jobGroup":"16","jobId":141,"killedTasksSummary":{},"name":"parquet at :0","numActiveStages":0,"numActiveTasks":0,"numCompletedIndices":1,"numCompletedStages":1,"numCompletedTasks":1,"numFailedStages":0,"numFailedTasks":0,"numKilledTasks":0,"numSkippedStages":0,"numSkippedTasks":0,"numTasks":1,"rowCount":0,"stageIds":[204],"status":"SUCCEEDED","submissionTime":"2023-04-20T07:01:09.222GMT"}],"limit":20,"numbers":{"FAILED":0,"RUNNING":0,"SUCCEEDED":12,"UNKNOWN":0},"rule":"ALL_DESC"},"spark_pool":null,"state":"finished","statement_id":16},"text/plain":["StatementMeta(, 5d1aa8d0-49b9-4260-8dac-2ba686428313, 16, Finished, Available)"]},"metadata":{},"output_type":"display_data"}],"source":["# read and write S11 annotations and images\n","s11_annotations_df = spark.read.parquet(os.path.join(input_dir, s11_annotations_file_name))\n","s11_annotations_df.write.format('delta').mode('overwrite').save(os.path.join(output_dir, 's11annotations'))\n","\n","s11_images_df = spark.read.parquet(os.path.join(input_dir, s11_images_file_name))\n","s11_images_df.write.format('delta').mode('overwrite').save(os.path.join(output_dir, 's11images'))"]},{"cell_type":"code","execution_count":null,"id":"4be1127a-0b7b-4dcf-855d-e2f4fcde98e7","metadata":{"jupyter":{"outputs_hidden":false,"source_hidden":false},"nteract":{"transient":{"deleting":false}}},"outputs":[],"source":["import json\n","import os\n","import pandas as pd\n","\n","# Set the path to the directory containing the raw JSON data\n","raw_data_path = '/lakehouse/default/Files/raw-data'\n","\n","# Get the list of JSON files in the raw data path, and select the first 10 for the training set\n","train_set = os.listdir(raw_data_path)[:10]\n","\n","# Select the 11th file for the test set\n","test_set = os.listdir(raw_data_path)[10]\n","\n","# Initialize empty DataFrames to store images, annotations, and categories data\n","images = pd.DataFrame()\n","annotations = pd.DataFrame()\n","categories = pd.DataFrame()\n","\n","# Process each JSON file in the training set\n","for file in train_set:\n"," # Read the JSON file and load its data\n"," path = os.path.join(raw_data_path, file)\n"," with open(path) as f:\n"," data = json.load(f)\n","\n"," # Extract and concatenate the 'images' and 'annotations' data into their respective DataFrames\n"," images = pd.concat([images, pd.DataFrame(data['images'])])\n"," annotations = pd.concat([annotations, pd.DataFrame(data['annotations'])])\n","\n"," # The 'categories' data is the same for all files, so we only need to do it once\n"," if len(categories) == 0:\n"," categories = pd.DataFrame(data['categories'])\n","\n","\n","# Process the test set, similar to the training set\n","path = os.path.join(raw_data_path, test_set)\n","with open(path) as f:\n"," data = json.load(f)\n","\n","# Define the paths for saving the data as Delta lake tables\n","train_annotations_table = 'Tables/train_annotations'\n","train_images_table = 'Tables/train_images'\n","test_annotations_table = 'Tables/test_annotations'\n","test_images_table = 'Tables/test_images'\n","categories_table = 'Tables/categories'\n","\n","# Extract and convert the 'images' and 'annotations' data of the test set into DataFrames\n","test_images = pd.DataFrame(data['images'])\n","test_annotations = pd.DataFrame(data['annotations'])\n","\n","# Save the DataFrames as Delta lake tables\n","spark.createDataFrame(images).write.format('delta').mode('overwrite').save(train_images_table)\n","spark.createDataFrame(annotations).write.format('delta').mode('overwrite').save(train_annotations_table)\n","spark.createDataFrame(test_images).write.format('delta').mode('overwrite').save(test_images_table)\n","spark.createDataFrame(test_annotations).write.format('delta').mode('overwrite').save(test_annotations_table)\n","spark.createDataFrame(categories).write.format('delta').mode('overwrite').save(categories_table)\n","\n"]}],"metadata":{"kernel_info":{"name":"synapse_pyspark"},"kernelspec":{"display_name":"Synapse PySpark","name":"synapse_pyspark"},"language_info":{"name":"python"},"notebook_environment":{},"save_output":true,"spark_compute":{"compute_id":"/trident/default","session_options":{"conf":{},"enableDebugMode":false}},"synapse_widget":{"state":{},"version":"0.1"},"trident":{"lakehouse":{"default_lakehouse":"5f58e739-d87b-404a-9a7b-5e64738a82c5","default_lakehouse_name":"Serengeti_Lakehouse","default_lakehouse_workspace_id":"1eaa972c-c94a-47fa-90f5-bcad0a19f945","known_lakehouses":[{"id":"5f58e739-d87b-404a-9a7b-5e64738a82c5"}]}},"widgets":{}},"nbformat":4,"nbformat_minor":5} diff --git a/workshops/fabric-e2e-serengeti/assets/notebooks/prep_and_transform.ipynb b/workshops/fabric-e2e-serengeti/assets/notebooks/prep_and_transform.ipynb deleted file mode 100644 index 3c283df5..00000000 --- a/workshops/fabric-e2e-serengeti/assets/notebooks/prep_and_transform.ipynb +++ /dev/null @@ -1 +0,0 @@ -{"cells":[{"cell_type":"code","execution_count":null,"id":"f5a097b3-0e7a-424e-8e5b-f5b99f577e0d","metadata":{"collapsed":false,"jupyter":{"outputs_hidden":false,"source_hidden":false},"nteract":{"transient":{"deleting":false}}},"outputs":[],"source":["%pip install opencv-python imutils"]},{"cell_type":"code","execution_count":null,"id":"f09be3b2-4703-49cb-a1f2-64e73f5c2b18","metadata":{"jupyter":{"outputs_hidden":false,"source_hidden":false},"nteract":{"transient":{"deleting":false}}},"outputs":[],"source":["import multiprocessing\n","\n","num_cores = multiprocessing.cpu_count()\n","print(\"Number of available cores:\", num_cores)"]},{"cell_type":"markdown","id":"a659df9b-acb4-462b-818c-46dde1b74a92","metadata":{"nteract":{"transient":{"deleting":false}}},"source":["Define a function tat takes a pandas dataframe and since there are diffrent seasons in the training set and diffrent number of images per season and different images per season. By looking at the first few chracters of the seq_id we can know which season an image belongs to. \n","\n","This function takes a single argument `df`, which is the pandas DataFrame containing the `seq_id` column. The function first extracts the season from the `seq_id` column using a lambda function, and then counts the number of sequences in each season using the `value_counts` method of the pandas Series object. The counts are sorted by season using the `sort_index` method.\n","\n","The function then replaces the `'SER_'` prefix in the season labels with an empty string for easy visibility, and creates a bar plot using `plt.bar`. The x-axis represents the season labels and the y-axis represents the number of sequences in each season. Finally, the function displays the plot using `plt.show`.\n","\n","You can call this function with your DataFrame `df_train` to plot the number of sequences in each season. For example, you can call `plot_season_counts(df_train)` to plot the season counts."]},{"cell_type":"code","execution_count":null,"id":"5e3245c2-1faf-4929-80ac-fdf147b9122e","metadata":{"jupyter":{"outputs_hidden":false,"source_hidden":false},"nteract":{"transient":{"deleting":false}}},"outputs":[],"source":["import pandas as pd\n","import matplotlib.pyplot as plt\n","import seaborn as sns\n","\n","def plot_season_counts(df, title=\"Number of Sequences per Season\"):\n"," # Extract the season from the seq_id column using a lambda function\n"," df['season'] = df.seq_id.map(lambda x: x.split('#')[0])\n","\n"," # Count the number of sequences in each season, and sort the counts by season\n"," season_counts = df.season.value_counts().sort_index()\n","\n"," # Replace 'SER_' prefix in season labels with an empty string for easy visibility\n"," season_labels = [s.replace('SER_', '') for s in season_counts.index]\n","\n"," # Create a bar plot where the x-axis represents the season and the y-axis represents the number of sequences in that season\n"," sns.barplot(x=season_labels, y=season_counts.values)\n"," plt.xlabel('Season')\n"," plt.ylabel('Number of sequences')\n"," plt.title(title)\n"," plt.show()"]},{"cell_type":"markdown","id":"21deeac5-4eb4-45f2-801d-102fccd87a8a","metadata":{"nteract":{"transient":{"deleting":false}}},"source":["**Load Data**\n","\n","Read the annotations data from the lakehouse which forms our traininng set. from this we get information about each season's sequences and labels uisng this data which provided linka between the image_ids and sequences as well as what category is ins a given sequence.\n","\n","We filter out the relevant columns we'll need , *i.e season, seq_id, category_id, image_id and date_time*. We also need to filter out all records whose category_id is greater than 1 to exclue all empty and huma images which are not relevant for tgis training. \n","\n","We also remove any null values in the image_id column and drop any duplicate lrows, finally convert the spark df to a pandas dataframe for easier manipulation.\n"]},{"cell_type":"code","execution_count":null,"id":"5e9febed-7d83-4f05-87e8-27b074449696","metadata":{"collapsed":false,"jupyter":{"outputs_hidden":false,"source_hidden":false},"nteract":{"transient":{"deleting":false}}},"outputs":[],"source":["# Read all the annotations in the train table from the lakehouse\n","df = spark.sql(\"SELECT * FROM DemoLakehouse.train_annotations WHERE train_annotations.category_id > 1\")\n","\n","# filter out the season, sequence ID, category_id snf image_id\n","df_train = df.select(\"season\", \"seq_id\", \"category_id\", \"location\", \"image_id\", \"datetime\")\n","\n","# remove image_id wiTH null and duplicates\n","df_train = df_train.filter(df_train.image_id.isNotNull()).dropDuplicates()\n","\n","# convert df_train to pandas dataframe\n","df_train = df_train.toPandas()"]},{"cell_type":"markdown","id":"77fa7118-f22c-4df1-93e3-9da0ea69d360","metadata":{"nteract":{"transient":{"deleting":false}}},"source":["We can visualize the number of images we have for each sequence and as below by far most sequesnces have between 1 and 3 images in them"]},{"cell_type":"code","execution_count":null,"id":"bdc3b9c8-b535-438a-a9b7-5563a71f884a","metadata":{"jupyter":{"outputs_hidden":false,"source_hidden":false},"nteract":{"transient":{"deleting":false}}},"outputs":[],"source":["# Create the count plot\n","ax = sns.countplot(x=df_train.groupby('seq_id').size(), log=True)\n","\n","# Set the title and axis labels\n","ax.set_title('Number of images in each sequence')\n","ax.set_xlabel('Number of images')\n","ax.set_ylabel('Count of sequences')\n","\n","# Show the plot\n","plt.tight_layout()\n","plt.show()\n"]},{"cell_type":"markdown","id":"93430c5f-794b-48a6-830c-9a58e1675929","metadata":{"nteract":{"transient":{"deleting":false}}},"source":["Load the category names from the Categories table in the lakehouse. We'll then convert the spark dataframe to a pandas dataframe. Next the add a new\n","column called *label* in the df_train dataframe which is the category name for each category_id. Then remove the category_id column from df_train.\n","\n","Also rename the image_id column to filename and append the .JPG extension to the the values"]},{"cell_type":"code","execution_count":null,"id":"7ed02c3b-c3ef-4867-b62a-4267dc9315db","metadata":{"jupyter":{"outputs_hidden":false,"source_hidden":false},"nteract":{"transient":{"deleting":false}}},"outputs":[],"source":["import numpy as np\n","\n","# Load the Categories DataFrame into a pandas DataFrame\n","category_df = spark.sql(\"SELECT * FROM DemoLakehouse.categories\").toPandas()\n","\n","# Map category IDs to category names using a vectorized approach\n","category_map = pd.Series(category_df.name.values, index=category_df.id)\n","df_train['label'] = category_map[df_train.category_id].values\n","\n","# Drop the category_id column\n","df_train = df_train.drop('category_id', axis=1)\n","\n","# Rename the image_id column to filename\n","df_train = df_train.rename(columns={'image_id': 'filename'})\n","\n","# Append the .JPG extension to the filename column\n","df_train['filename'] = df_train.filename + '.JPG'"]},{"cell_type":"markdown","id":"8bbd79cb-f4f4-4c5d-9ed1-56f43134794c","metadata":{"nteract":{"transient":{"deleting":false}}},"source":["We'll then pick the first image from each sequence, the assumption is that the time period after a camera trap is triggered is the most likely time for an animal to be in the frame."]},{"cell_type":"code","execution_count":null,"id":"f8ea33bc-7818-4f40-a906-17c2e8283576","metadata":{"collapsed":false,"jupyter":{"outputs_hidden":false,"source_hidden":false},"nteract":{"transient":{"deleting":false}}},"outputs":[],"source":["# reduce to first frame only for all sequences\n","df_train = df_train.sort_values('filename').groupby('seq_id').first().reset_index()\n","\n","df_train.count()"]},{"cell_type":"markdown","id":"000360d5-9048-41e2-8d0a-59967159daf9","metadata":{"nteract":{"transient":{"deleting":false}}},"source":["The plot shows the distribution of the labels, where each bar represents the number of images with that label. The y-axis label is set to \"Label\", and the x-axis label is set to \"Number of images\". The horizontal orientation of the bars, the increased figure size, the added spacing between the labels, and the logarithmic scale of the x-axis make it easier to read the labels and normalize the distribution."]},{"cell_type":"code","execution_count":null,"id":"44617406-59e2-4963-959b-2023da838bb2","metadata":{"jupyter":{"outputs_hidden":false,"source_hidden":false},"nteract":{"transient":{"deleting":false}}},"outputs":[],"source":["# Create a horizontal bar plot where the y-axis represents the label and the x-axis represents the number of images with that label\n","plt.figure(figsize=(8, 12))\n","sns.countplot(y='label', data=df, order=df_train['label'].value_counts().index)\n","plt.xlabel('Number of images')\n","plt.ylabel('Label')\n","\n","# Set the x-axis scale to logarithmic\n","plt.xscale('log')\n","\n","plt.show()"]},{"cell_type":"markdown","id":"73f16ba4-af51-4108-b45c-288408553331","metadata":{"nteract":{"transient":{"deleting":false}}},"source":["Using the file name from the datafram use the defined function that takes the filename as input and returm the url to the image. The function is then applied to the `filename` column of a pandas DataFrame `df_train` to create a new column `image_url` containing the URLs for each image."]},{"cell_type":"code","execution_count":null,"id":"b85b86b9-76e9-4f4e-8520-aa91421c98a1","metadata":{"jupyter":{"outputs_hidden":false,"source_hidden":false},"nteract":{"transient":{"deleting":false}}},"outputs":[],"source":["# Define a function to apply to the filename column\n","def get_ImageUrl(filename):\n"," return f\"https://lilablobssc.blob.core.windows.net/snapshotserengeti-unzipped/{filename}\"\n","\n","# Create a new column in the dataframe using the apply method\n","df_train['image_url'] = df_train['filename'].apply(get_ImageUrl)\n"]},{"cell_type":"markdown","id":"7616d5cd-643a-4039-9bf8-83031f72a177","metadata":{"nteract":{"transient":{"deleting":false}}},"source":[]},{"cell_type":"code","execution_count":null,"id":"82a178c6-07a7-4242-9f67-7d3fd535bb20","metadata":{"jupyter":{"outputs_hidden":false,"source_hidden":false},"nteract":{"transient":{"deleting":false}}},"outputs":[],"source":["import urllib.request\n","\n","def display_random_image(label, random_state, width=500):\n"," # Filter the DataFrame to only include rows with the specified label\n"," df_filtered = df_train[df_train['label'] == label]\n"," \n"," # Select a random row from the filtered DataFrame\n"," row = df_filtered.sample(random_state=random_state).iloc[0]\n"," \n"," # Load the image from the URL and display it\n"," url = row['image_url']\n"," download_and_display_image(url, label)\n","\n","def download_and_display_image(url, label):\n"," image = plt.imread(urllib.request.urlopen(url), format='jpg')\n"," plt.imshow(image)\n"," plt.title(f\"Label: {label}\")\n"," plt.show()"]},{"cell_type":"code","execution_count":null,"id":"3477f501-8739-440b-b254-12ca982f42d8","metadata":{"jupyter":{"outputs_hidden":false,"source_hidden":false},"nteract":{"transient":{"deleting":false}}},"outputs":[],"source":["display_random_image(label='leopard', random_state=12)"]},{"cell_type":"code","execution_count":null,"id":"65616bf0-587b-447d-afb6-1740b9726e78","metadata":{"jupyter":{"outputs_hidden":false,"source_hidden":false},"nteract":{"transient":{"deleting":false}}},"outputs":[],"source":["def proportional_allocation_percentage(data, percentage):\n"," # Calculate the count of the original sample\n"," original_count = len(data)\n","\n"," # Calculate the count of the sample based on the percentage\n"," sample_count = int((percentage / 100) * original_count)\n","\n"," # Perform proportional allocation on the calculated sample count\n"," return proportional_allocation(data, sample_count)"]},{"cell_type":"code","execution_count":null,"id":"b37fb461-2e6b-47de-a3fd-e801a243e53f","metadata":{"jupyter":{"outputs_hidden":false,"source_hidden":false},"nteract":{"transient":{"deleting":false}}},"outputs":[],"source":["def proportional_allocation(data, sample_size):\n"," # Group the data by \"label\", \"season\", and \"location\" columns\n"," grouped_data = data.groupby([\"label\", \"season\", \"location\"])\n","\n"," # Calculate the proportion of each group in the original sample\n"," proportions = grouped_data.size() / len(data)\n","\n"," # Calculate the count of each group in the sample based on proportions\n"," sample_sizes = np.round(proportions * sample_size).astype(int)\n","\n"," # Calculate the difference between the desired sample size and the sum of rounded sample sizes\n"," size_difference = sample_size - sample_sizes.sum()\n","\n"," # Adjust the sample sizes to account for the difference\n"," if size_difference > 0:\n"," # If there is a shortage of items, allocate the additional items to the groups with the largest proportions\n"," largest_proportions = proportions.nlargest(size_difference)\n"," for group in largest_proportions.index:\n"," sample_sizes[group] += 1\n"," elif size_difference < 0:\n"," # If there is an excess of items, reduce the sample sizes from the groups with the smallest proportions\n"," smallest_proportions = proportions.nsmallest(-size_difference)\n"," for group in smallest_proportions.index:\n"," sample_sizes[group] -= 1\n","\n"," # Initialize an empty list to store the sample\n"," sample_data = []\n","\n"," # Iterate over each group and randomly sample the required count\n"," for group, count in zip(grouped_data.groups, sample_sizes):\n"," indices = grouped_data.groups[group]\n"," sample_indices = np.random.choice(indices, size=count, replace=False)\n"," sample_data.append(data.loc[sample_indices])\n","\n"," # Concatenate the sampled dataframes into a single dataframe\n"," sample_data = pd.concat(sample_data)\n","\n"," # Reset the index of the sample DataFrame\n"," sample_data.reset_index(drop=True, inplace=True)\n","\n"," return sample_data"]},{"cell_type":"code","execution_count":null,"id":"8c7c9454-2bf7-422a-a575-eec88b144536","metadata":{"jupyter":{"outputs_hidden":false,"source_hidden":false},"nteract":{"transient":{"deleting":false}}},"outputs":[],"source":["percent = 0.05\n","sampled_train = proportional_allocation_percentage(df_train, percent)\n","plot_season_counts(sampled_train, f\"{percent}% Sample from Original Number of Sequences per Season\")"]},{"cell_type":"code","execution_count":null,"id":"d67aed14","metadata":{},"outputs":[],"source":["import urllib.request\n","import cv2\n","import imutils\n","\n","def download_and_resize_image(url, path, kind):\n"," filename = os.path.basename(path)\n"," directory = os.path.dirname(path)\n","\n"," directory_path = f'/lakehouse/default/Files/images/{kind}/{directory}/'\n","\n"," # Create the directory if it does not exist\n"," os.makedirs(directory_path, exist_ok=True)\n","\n"," # check if file already exists\n"," if os.path.isfile(os.path.join(directory_path, filename)):\n"," return\n","\n"," # Download the image\n"," urllib.request.urlretrieve(url, filename)\n","\n"," # Read the image using OpenCV\n"," img = cv2.imread(filename)\n","\n"," # Resize the image to a reasonable ML training size using imutils\n"," resized_img = imutils.resize(img, width=224, height=224, inter=cv2.INTER_AREA)\n","\n"," # Save the resized image to a defined filepath\n"," cv2.imwrite(os.path.join(directory_path, filename), resized_img)"]},{"cell_type":"code","execution_count":null,"id":"ae502e79","metadata":{},"outputs":[],"source":["import concurrent.futures\n","\n","def execute_parallel_download(df, kind):\n"," # Use a process pool instead of a thread pool to avoid thread safety issues\n"," with concurrent.futures.ProcessPoolExecutor() as executor:\n"," # Batch process images instead of processing them one at a time\n"," urls = df['image_url'].tolist()\n"," paths = df['filename'].tolist()\n"," futures = [executor.submit(download_and_resize_image, url, path, kind) for url, path in zip(urls, paths)]\n"," # Wait for all tasks to complete\n"," concurrent.futures.wait(futures)"]},{"cell_type":"code","execution_count":null,"id":"35ab2975","metadata":{},"outputs":[],"source":["df = spark.sql(\"SELECT * FROM DemoLakehouse.test_annotations WHERE test_annotations.category_id > 1\")\n","\n","df_test = df.select(\"season\", \"seq_id\", \"category_id\", \"location\", \"image_id\", \"datetime\")\n","\n","df_test= df_test.filter(df_test.image_id.isNotNull()).dropDuplicates()\n","\n","df_test = df_test.toPandas()\n","\n","df_test['label'] = category_map[df_test.category_id].values\n","\n","df_test = df_test.drop('category_id', axis=1)\n","\n","df_test = df_test.rename(columns={'image_id':'filename'})\n","\n","df_test['filename'] = df_test.filename+ '.JPG'\n","\n","df_test = df_test.sort_values('filename').groupby('seq_id').first().reset_index()\n","\n","df_test['image_url'] = df_test['filename'].apply(get_ImageUrl)\n","\n","sampled_test = proportional_allocation_percentage(df_test, 0.27)"]},{"cell_type":"code","execution_count":null,"id":"e3ed3da2","metadata":{},"outputs":[],"source":["import os\n","\n","execute_parallel_download(sampled_train, 'train')\n","execute_parallel_download(sampled_test, 'test')\n"]},{"cell_type":"code","execution_count":null,"id":"0d11447a","metadata":{},"outputs":[],"source":["data_dir = '/lakehouse/default/Files/data/'\n","\n","train_data_file = os.path.join(data_dir, 'sampled_train.parquet')\n","test_data_file = os.path.join(data_dir, 'sampled_test.parquet')\n","\n","sampled_train.loc[:, ['filename', 'label']].to_parquet(train_data_file, engine='pyarrow', compression='snappy')\n","sampled_test.loc[:, ['filename', 'label']].to_parquet(test_data_file, engine='pyarrow', compression='snappy')\n"]}],"metadata":{"kernel_info":{"name":"synapse_pyspark"},"kernelspec":{"display_name":"base","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.10.9"},"notebook_environment":{},"save_output":true,"spark_compute":{"compute_id":"/trident/default","session_options":{"conf":{},"enableDebugMode":false}},"synapse_widget":{"state":{},"version":"0.1"},"trident":{"lakehouse":{"default_lakehouse":"5f58e739-d87b-404a-9a7b-5e64738a82c5","default_lakehouse_name":"Serengeti_Lakehouse","default_lakehouse_workspace_id":"1eaa972c-c94a-47fa-90f5-bcad0a19f945","known_lakehouses":[{"id":"5f58e739-d87b-404a-9a7b-5e64738a82c5"}]}},"widgets":{}},"nbformat":4,"nbformat_minor":5} diff --git a/workshops/fabric-e2e-serengeti/assets/notebooks/train-model.ipynb b/workshops/fabric-e2e-serengeti/assets/notebooks/train-model.ipynb deleted file mode 100644 index 8483ede2..00000000 --- a/workshops/fabric-e2e-serengeti/assets/notebooks/train-model.ipynb +++ /dev/null @@ -1 +0,0 @@ -{"cells":[{"cell_type":"code","source":["# Welcome to your new notebook\n","# Type here in the cell editor to add code!\n","%pip install torch"],"outputs":[{"output_type":"display_data","data":{"application/vnd.livy.statement-meta+json":{"spark_pool":null,"session_id":"2511b538-6adf-43c2-8c1f-705b65a98530","statement_id":9,"state":"finished","livy_statement_state":"available","queued_time":"2023-07-19T15:29:45.7992548Z","session_start_time":null,"execution_start_time":"2023-07-19T15:29:59.9443311Z","execution_finish_time":"2023-07-19T15:30:01.6315469Z","spark_jobs":{"numbers":{"RUNNING":0,"SUCCEEDED":0,"FAILED":0,"UNKNOWN":0},"jobs":[],"limit":20,"rule":"ALL_DESC"},"parent_msg_id":"1dfadd5d-b12e-4616-96a4-c23cdd05a24e"},"text/plain":"StatementMeta(, 2511b538-6adf-43c2-8c1f-705b65a98530, 9, Finished, Available)"},"metadata":{}},{"output_type":"execute_result","execution_count":2,"data":{},"metadata":{}},{"output_type":"stream","name":"stdout","text":["Requirement already satisfied: torch in /home/trusted-service-user/cluster-env/trident_env/lib/python3.10/site-packages (1.13.1)\nRequirement already satisfied: typing-extensions in /home/trusted-service-user/cluster-env/trident_env/lib/python3.10/site-packages (from torch) (4.5.0)\n\n\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.0\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m23.2\u001b[0m\n\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49m/nfs4/pyenv-98b8aa98-b6fc-413b-a5db-820fb80e5ba3/bin/python -m pip install --upgrade pip\u001b[0m\nNote: you may need to restart the kernel to use updated packages.\n"]},{"output_type":"execute_result","execution_count":2,"data":{},"metadata":{}},{"output_type":"stream","name":"stdout","text":["Warning: PySpark kernel has been restarted to use updated packages.\n\n"]}],"execution_count":2,"metadata":{},"id":"83c8001f-c587-401b-b868-4d8aee284035"},{"cell_type":"code","source":["%pip install torchvision"],"outputs":[{"output_type":"display_data","data":{"application/vnd.livy.statement-meta+json":{"spark_pool":null,"session_id":"2511b538-6adf-43c2-8c1f-705b65a98530","statement_id":-1,"state":"finished","livy_statement_state":"available","queued_time":"2023-07-19T15:29:52.2710191Z","session_start_time":null,"execution_start_time":"2023-07-19T15:31:24.9816756Z","execution_finish_time":"2023-07-19T15:31:24.9818159Z","spark_jobs":{"numbers":{"RUNNING":0,"SUCCEEDED":0,"FAILED":0,"UNKNOWN":0},"jobs":[],"limit":20,"rule":"ALL_DESC"},"parent_msg_id":"ae6b5a2d-f2da-4615-b0b5-8831d07ce36d"},"text/plain":"StatementMeta(, 2511b538-6adf-43c2-8c1f-705b65a98530, -1, Finished, Available)"},"metadata":{}},{"output_type":"execute_result","execution_count":3,"data":{},"metadata":{}},{"output_type":"stream","name":"stdout","text":["Collecting torchvision\n Downloading torchvision-0.15.2-cp310-cp310-manylinux1_x86_64.whl (6.0 MB)\n\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m6.0/6.0 MB\u001b[0m \u001b[31m106.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n\u001b[?25hRequirement already satisfied: numpy in /home/trusted-service-user/cluster-env/trident_env/lib/python3.10/site-packages (from torchvision) (1.23.5)\nCollecting torch==2.0.1\n Downloading torch-2.0.1-cp310-cp310-manylinux1_x86_64.whl (619.9 MB)\n\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m619.9/619.9 MB\u001b[0m \u001b[31m7.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n\u001b[?25hRequirement already satisfied: requests in /home/trusted-service-user/cluster-env/trident_env/lib/python3.10/site-packages (from torchvision) (2.28.2)\nRequirement already satisfied: pillow!=8.3.*,>=5.3.0 in /home/trusted-service-user/cluster-env/trident_env/lib/python3.10/site-packages (from torchvision) (9.4.0)\nCollecting nvidia-cuda-cupti-cu11==11.7.101\n Downloading nvidia_cuda_cupti_cu11-11.7.101-py3-none-manylinux1_x86_64.whl (11.8 MB)\n\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m11.8/11.8 MB\u001b[0m \u001b[31m90.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n\u001b[?25hCollecting triton==2.0.0\n Downloading triton-2.0.0-1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl (63.3 MB)\n\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m63.3/63.3 MB\u001b[0m \u001b[31m53.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n\u001b[?25hRequirement already satisfied: typing-extensions in /home/trusted-service-user/cluster-env/trident_env/lib/python3.10/site-packages (from torch==2.0.1->torchvision) (4.5.0)\nCollecting networkx\n Downloading networkx-3.1-py3-none-any.whl (2.1 MB)\n\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.1/2.1 MB\u001b[0m \u001b[31m146.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n\u001b[?25hCollecting nvidia-curand-cu11==10.2.10.91\n Downloading nvidia_curand_cu11-10.2.10.91-py3-none-manylinux1_x86_64.whl (54.6 MB)\n\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m54.6/54.6 MB\u001b[0m \u001b[31m56.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n\u001b[?25hCollecting nvidia-cusparse-cu11==11.7.4.91\n Downloading nvidia_cusparse_cu11-11.7.4.91-py3-none-manylinux1_x86_64.whl (173.2 MB)\n\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m173.2/173.2 MB\u001b[0m \u001b[31m6.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n\u001b[?25hCollecting nvidia-cublas-cu11==11.10.3.66\n Downloading nvidia_cublas_cu11-11.10.3.66-py3-none-manylinux1_x86_64.whl (317.1 MB)\n\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m317.1/317.1 MB\u001b[0m \u001b[31m17.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n\u001b[?25hCollecting nvidia-cusolver-cu11==11.4.0.1\n Downloading nvidia_cusolver_cu11-11.4.0.1-2-py3-none-manylinux1_x86_64.whl (102.6 MB)\n\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m102.6/102.6 MB\u001b[0m \u001b[31m50.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n\u001b[?25hRequirement already satisfied: jinja2 in /home/trusted-service-user/cluster-env/trident_env/lib/python3.10/site-packages (from torch==2.0.1->torchvision) (3.1.2)\nCollecting nvidia-nvtx-cu11==11.7.91\n Downloading nvidia_nvtx_cu11-11.7.91-py3-none-manylinux1_x86_64.whl (98 kB)\n\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m98.6/98.6 kB\u001b[0m \u001b[31m53.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n\u001b[?25hCollecting nvidia-cuda-runtime-cu11==11.7.99\n Downloading nvidia_cuda_runtime_cu11-11.7.99-py3-none-manylinux1_x86_64.whl (849 kB)\n\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m849.3/849.3 kB\u001b[0m \u001b[31m168.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n\u001b[?25hCollecting nvidia-cufft-cu11==10.9.0.58\n Downloading nvidia_cufft_cu11-10.9.0.58-py3-none-manylinux1_x86_64.whl (168.4 MB)\n\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m168.4/168.4 MB\u001b[0m \u001b[31m37.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n\u001b[?25hRequirement already satisfied: filelock in /home/trusted-service-user/cluster-env/trident_env/lib/python3.10/site-packages (from torch==2.0.1->torchvision) (3.11.0)\nCollecting nvidia-cudnn-cu11==8.5.0.96\n Downloading nvidia_cudnn_cu11-8.5.0.96-2-py3-none-manylinux1_x86_64.whl (557.1 MB)\n\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m557.1/557.1 MB\u001b[0m \u001b[31m7.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n\u001b[?25hCollecting nvidia-cuda-nvrtc-cu11==11.7.99\n Downloading nvidia_cuda_nvrtc_cu11-11.7.99-2-py3-none-manylinux1_x86_64.whl (21.0 MB)\n\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m21.0/21.0 MB\u001b[0m \u001b[31m99.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n\u001b[?25hCollecting nvidia-nccl-cu11==2.14.3\n Downloading nvidia_nccl_cu11-2.14.3-py3-none-manylinux1_x86_64.whl (177.1 MB)\n\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m177.1/177.1 MB\u001b[0m \u001b[31m32.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n\u001b[?25hCollecting sympy\n Downloading sympy-1.12-py3-none-any.whl (5.7 MB)\n\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m5.7/5.7 MB\u001b[0m \u001b[31m160.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m\n\u001b[?25hRequirement already satisfied: setuptools in /home/trusted-service-user/cluster-env/trident_env/lib/python3.10/site-packages (from nvidia-cublas-cu11==11.10.3.66->torch==2.0.1->torchvision) (67.6.1)\nRequirement already satisfied: wheel in /home/trusted-service-user/cluster-env/trident_env/lib/python3.10/site-packages (from nvidia-cublas-cu11==11.10.3.66->torch==2.0.1->torchvision) (0.40.0)\nCollecting lit\n Downloading lit-16.0.6.tar.gz (153 kB)\n\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m153.7/153.7 kB\u001b[0m \u001b[31m70.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n\u001b[?25h Installing build dependencies ... \u001b[?25l-\b \b\\\b \b|\b \bdone\n\u001b[?25h Getting requirements to build wheel ... \u001b[?25l-\b \bdone\n\u001b[?25h Installing backend dependencies ... \u001b[?25l-\b \b\\\b \bdone\n\u001b[?25h Preparing metadata (pyproject.toml) ... \u001b[?25l-\b \bdone\n\u001b[?25hCollecting cmake\n Downloading cmake-3.26.4-py2.py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl (24.0 MB)\n\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m24.0/24.0 MB\u001b[0m \u001b[31m86.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n\u001b[?25hRequirement already satisfied: urllib3<1.27,>=1.21.1 in /home/trusted-service-user/cluster-env/trident_env/lib/python3.10/site-packages (from requests->torchvision) (1.26.14)\nRequirement already satisfied: charset-normalizer<4,>=2 in /home/trusted-service-user/cluster-env/trident_env/lib/python3.10/site-packages (from requests->torchvision) (2.1.1)\nRequirement already satisfied: idna<4,>=2.5 in /home/trusted-service-user/cluster-env/trident_env/lib/python3.10/site-packages (from requests->torchvision) (3.4)\nRequirement already satisfied: certifi>=2017.4.17 in /home/trusted-service-user/cluster-env/trident_env/lib/python3.10/site-packages (from requests->torchvision) (2022.12.7)\nRequirement already satisfied: MarkupSafe>=2.0 in /home/trusted-service-user/cluster-env/trident_env/lib/python3.10/site-packages (from jinja2->torch==2.0.1->torchvision) (2.1.2)\nCollecting mpmath>=0.19\n Downloading mpmath-1.3.0-py3-none-any.whl (536 kB)\n\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m536.2/536.2 kB\u001b[0m \u001b[31m139.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n\u001b[?25hBuilding wheels for collected packages: lit\n Building wheel for lit (pyproject.toml) ... \u001b[?25l-\b \b\\\b \bdone\n\u001b[?25h Created wheel for lit: filename=lit-16.0.6-py3-none-any.whl size=93582 sha256=f3e249693591605b862739f91f821a526e6ae60121b222da338848f3c33ad3b9\n Stored in directory: /home/trusted-service-user/.cache/pip/wheels/14/f9/07/bb2308587bc2f57158f905a2325f6a89a2befa7437b2d7e137\nSuccessfully built lit\nInstalling collected packages: mpmath, lit, cmake, sympy, nvidia-nvtx-cu11, nvidia-nccl-cu11, nvidia-cusparse-cu11, nvidia-curand-cu11, nvidia-cufft-cu11, nvidia-cuda-runtime-cu11, nvidia-cuda-nvrtc-cu11, nvidia-cuda-cupti-cu11, nvidia-cublas-cu11, networkx, nvidia-cusolver-cu11, nvidia-cudnn-cu11, triton, torch, torchvision\n Attempting uninstall: torch\n Found existing installation: torch 1.13.1\n Not uninstalling torch at /home/trusted-service-user/cluster-env/trident_env/lib/python3.10/site-packages, outside environment /nfs4/pyenv-98b8aa98-b6fc-413b-a5db-820fb80e5ba3\n Can't uninstall 'torch'. No files were found to uninstall.\nSuccessfully installed cmake-3.26.4 lit-16.0.6 mpmath-1.3.0 networkx-3.1 nvidia-cublas-cu11-11.10.3.66 nvidia-cuda-cupti-cu11-11.7.101 nvidia-cuda-nvrtc-cu11-11.7.99 nvidia-cuda-runtime-cu11-11.7.99 nvidia-cudnn-cu11-8.5.0.96 nvidia-cufft-cu11-10.9.0.58 nvidia-curand-cu11-10.2.10.91 nvidia-cusolver-cu11-11.4.0.1 nvidia-cusparse-cu11-11.7.4.91 nvidia-nccl-cu11-2.14.3 nvidia-nvtx-cu11-11.7.91 sympy-1.12 torch-2.0.1 torchvision-0.15.2 triton-2.0.0\n\n\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.0\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m23.2\u001b[0m\n\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49m/nfs4/pyenv-98b8aa98-b6fc-413b-a5db-820fb80e5ba3/bin/python -m pip install --upgrade pip\u001b[0m\nNote: you may need to restart the kernel to use updated packages.\n"]},{"output_type":"execute_result","execution_count":3,"data":{},"metadata":{}},{"output_type":"stream","name":"stdout","text":["Warning: PySpark kernel has been restarted to use updated packages.\n\n"]}],"execution_count":3,"metadata":{"jupyter":{"source_hidden":false,"outputs_hidden":false},"nteract":{"transient":{"deleting":false}}},"id":"2651bd81-2ca3-4dbd-a285-7c237eab9d78"},{"cell_type":"code","source":["train_df = spark.sql(\"SELECT * FROM DemoLakehouse.sampled_train LIMIT 1000\")\r\n","\r\n","import pandas as pd\r\n","\r\n","# convert train_df to pandas dataframe\r\n","train_df = train_df.toPandas()"],"outputs":[{"output_type":"display_data","data":{"application/vnd.livy.statement-meta+json":{"spark_pool":null,"session_id":"2511b538-6adf-43c2-8c1f-705b65a98530","statement_id":17,"state":"finished","livy_statement_state":"available","queued_time":"2023-07-19T15:34:36.1803998Z","session_start_time":null,"execution_start_time":"2023-07-19T15:34:41.5038351Z","execution_finish_time":"2023-07-19T15:34:52.2065876Z","spark_jobs":{"numbers":{"RUNNING":0,"SUCCEEDED":6,"FAILED":0,"UNKNOWN":0},"jobs":[{"dataWritten":0,"dataRead":6682,"rowCount":294,"jobId":12,"name":"toPandas at /tmp/ipykernel_20773/1259328519.py:5","description":"Job group for statement 17:\ntrain_df = spark.sql(\"SELECT * FROM DemoLakehouse.sampled_train LIMIT 1000\")\n\nimport pandas as pd\n# convert train_df to pandas dataframe\ntrain_df = train_df.toPandas()","submissionTime":"2023-07-19T15:34:51.401GMT","completionTime":"2023-07-19T15:34:51.445GMT","stageIds":[19,20],"jobGroup":"17","status":"SUCCEEDED","numTasks":2,"numActiveTasks":0,"numCompletedTasks":1,"numSkippedTasks":1,"numFailedTasks":0,"numKilledTasks":0,"numCompletedIndices":1,"numActiveStages":0,"numCompletedStages":1,"numSkippedStages":1,"numFailedStages":0,"killedTasksSummary":{}},{"dataWritten":6682,"dataRead":7827,"rowCount":588,"jobId":11,"name":"toPandas at /tmp/ipykernel_20773/1259328519.py:5","description":"Job group for statement 17:\ntrain_df = spark.sql(\"SELECT * FROM DemoLakehouse.sampled_train LIMIT 1000\")\n\nimport pandas as pd\n# convert train_df to pandas dataframe\ntrain_df = train_df.toPandas()","submissionTime":"2023-07-19T15:34:51.195GMT","completionTime":"2023-07-19T15:34:51.382GMT","stageIds":[18],"jobGroup":"17","status":"SUCCEEDED","numTasks":1,"numActiveTasks":0,"numCompletedTasks":1,"numSkippedTasks":0,"numFailedTasks":0,"numKilledTasks":0,"numCompletedIndices":1,"numActiveStages":0,"numCompletedStages":1,"numSkippedStages":0,"numFailedStages":0,"killedTasksSummary":{}},{"dataWritten":0,"dataRead":1732,"rowCount":3,"jobId":10,"name":"toPandas at /tmp/ipykernel_20773/1259328519.py:5","description":"Delta: Job group for statement 17:\ntrain_df = spark.sql(\"SELECT * FROM DemoLakehouse.sampled_train LIMIT 1000\")\n\nimport pandas as pd\n# convert train_df to pandas dataframe\ntrain_df = train_df.toPandas(): Filtering files for query","submissionTime":"2023-07-19T15:34:50.535GMT","completionTime":"2023-07-19T15:34:51.017GMT","stageIds":[16,17],"jobGroup":"17","status":"SUCCEEDED","numTasks":51,"numActiveTasks":0,"numCompletedTasks":50,"numSkippedTasks":1,"numFailedTasks":0,"numKilledTasks":0,"numCompletedIndices":50,"numActiveStages":0,"numCompletedStages":1,"numSkippedStages":1,"numFailedStages":0,"killedTasksSummary":{}},{"dataWritten":0,"dataRead":4312,"rowCount":50,"jobId":9,"name":"toString at String.java:2994","description":"Delta: Job group for statement 17:\ntrain_df = spark.sql(\"SELECT * FROM DemoLakehouse.sampled_train LIMIT 1000\")\n\nimport pandas as pd\n# convert train_df to pandas dataframe\ntrain_df = train_df.toPandas(): Compute snapshot for version: 0","submissionTime":"2023-07-19T15:34:50.029GMT","completionTime":"2023-07-19T15:34:50.094GMT","stageIds":[15,13,14],"jobGroup":"17","status":"SUCCEEDED","numTasks":52,"numActiveTasks":0,"numCompletedTasks":1,"numSkippedTasks":51,"numFailedTasks":0,"numKilledTasks":0,"numCompletedIndices":1,"numActiveStages":0,"numCompletedStages":1,"numSkippedStages":2,"numFailedStages":0,"killedTasksSummary":{}},{"dataWritten":4312,"dataRead":1597,"rowCount":54,"jobId":8,"name":"toString at String.java:2994","description":"Delta: Job group for statement 17:\ntrain_df = spark.sql(\"SELECT * FROM DemoLakehouse.sampled_train LIMIT 1000\")\n\nimport pandas as pd\n# convert train_df to pandas dataframe\ntrain_df = train_df.toPandas(): Compute snapshot for version: 0","submissionTime":"2023-07-19T15:34:49.192GMT","completionTime":"2023-07-19T15:34:50.003GMT","stageIds":[12,11],"jobGroup":"17","status":"SUCCEEDED","numTasks":51,"numActiveTasks":0,"numCompletedTasks":50,"numSkippedTasks":1,"numFailedTasks":0,"numKilledTasks":0,"numCompletedIndices":50,"numActiveStages":0,"numCompletedStages":1,"numSkippedStages":1,"numFailedStages":0,"killedTasksSummary":{}},{"dataWritten":1597,"dataRead":1347,"rowCount":8,"jobId":7,"name":"toString at String.java:2994","description":"Delta: Job group for statement 17:\ntrain_df = spark.sql(\"SELECT * FROM DemoLakehouse.sampled_train LIMIT 1000\")\n\nimport pandas as pd\n# convert train_df to pandas dataframe\ntrain_df = train_df.toPandas(): Compute snapshot for version: 0","submissionTime":"2023-07-19T15:34:48.328GMT","completionTime":"2023-07-19T15:34:48.928GMT","stageIds":[10],"jobGroup":"17","status":"SUCCEEDED","numTasks":1,"numActiveTasks":0,"numCompletedTasks":1,"numSkippedTasks":0,"numFailedTasks":0,"numKilledTasks":0,"numCompletedIndices":1,"numActiveStages":0,"numCompletedStages":1,"numSkippedStages":0,"numFailedStages":0,"killedTasksSummary":{}}],"limit":20,"rule":"ALL_DESC"},"parent_msg_id":"30c4dbc1-c6eb-4de2-a007-1ae7b77b3d34"},"text/plain":"StatementMeta(, 2511b538-6adf-43c2-8c1f-705b65a98530, 17, Finished, Available)"},"metadata":{}}],"execution_count":4,"metadata":{"jupyter":{"source_hidden":false,"outputs_hidden":false},"nteract":{"transient":{"deleting":false}}},"id":"3659f7e5-26dc-4efa-abdd-699c967afded"},{"cell_type":"code","source":["# Create a new column in the dataframe to apply to the filename column tor read the image URL\r\n","train_df['image_url'] = train_df['filename'].apply(lambda filename: f\"/lakehouse/default/Files/images/train/{filename}\")\r\n","\r\n","train_df.head()"],"outputs":[{"output_type":"display_data","data":{"application/vnd.livy.statement-meta+json":{"spark_pool":null,"session_id":"2511b538-6adf-43c2-8c1f-705b65a98530","statement_id":18,"state":"finished","livy_statement_state":"available","queued_time":"2023-07-19T15:36:15.5726373Z","session_start_time":null,"execution_start_time":"2023-07-19T15:36:16.1667919Z","execution_finish_time":"2023-07-19T15:36:16.6045447Z","spark_jobs":{"numbers":{"RUNNING":0,"SUCCEEDED":0,"FAILED":0,"UNKNOWN":0},"jobs":[],"limit":20,"rule":"ALL_DESC"},"parent_msg_id":"b8a0ac78-9aa8-4234-bfb7-19081747d731"},"text/plain":"StatementMeta(, 2511b538-6adf-43c2-8c1f-705b65a98530, 18, Finished, Available)"},"metadata":{}},{"output_type":"execute_result","execution_count":10,"data":{"text/plain":" filename label \\\n0 S7/P13/P13_R1/S7_P13_R1_IMAG1518.JPG elephant \n1 S7/O10/O10_R2/S7_O10_R2_IMAG1454.JPG gazellegrants \n2 S2/H07/H07_R2/S2_H07_R2_PICT2795.JPG guineafowl \n3 S5/H07/H07_R1/S5_H07_R1_IMAG4660.JPG guineafowl \n4 S2/H07/H07_R1/S2_H07_R1_PICT0539.JPG hartebeest \n\n image_url \n0 /lakehouse/default/Files/images/train/S7/P13/P... \n1 /lakehouse/default/Files/images/train/S7/O10/O... \n2 /lakehouse/default/Files/images/train/S2/H07/H... \n3 /lakehouse/default/Files/images/train/S5/H07/H... \n4 /lakehouse/default/Files/images/train/S2/H07/H... ","text/html":"
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
filenamelabelimage_url
0S7/P13/P13_R1/S7_P13_R1_IMAG1518.JPGelephant/lakehouse/default/Files/images/train/S7/P13/P...
1S7/O10/O10_R2/S7_O10_R2_IMAG1454.JPGgazellegrants/lakehouse/default/Files/images/train/S7/O10/O...
2S2/H07/H07_R2/S2_H07_R2_PICT2795.JPGguineafowl/lakehouse/default/Files/images/train/S2/H07/H...
3S5/H07/H07_R1/S5_H07_R1_IMAG4660.JPGguineafowl/lakehouse/default/Files/images/train/S5/H07/H...
4S2/H07/H07_R1/S2_H07_R1_PICT0539.JPGhartebeest/lakehouse/default/Files/images/train/S2/H07/H...
\n
"},"metadata":{}}],"execution_count":5,"metadata":{"jupyter":{"source_hidden":false,"outputs_hidden":false},"nteract":{"transient":{"deleting":false}}},"id":"206543d5-a7ab-4971-b359-66292bcdfd53"},{"cell_type":"code","source":["from sklearn.preprocessing import LabelEncoder\r\n","\r\n","# Create a LabelEncoder object\r\n","le = LabelEncoder()\r\n","\r\n","# Fit the LabelEncoder to the label column in the train_df DataFrame\r\n","le.fit(train_df['label'])\r\n","\r\n","# Transform the label column to numerical labels using the LabelEncoder\r\n","train_df['labels'] = le.transform(train_df['label'])"],"outputs":[{"output_type":"display_data","data":{"application/vnd.livy.statement-meta+json":{"spark_pool":null,"session_id":"2511b538-6adf-43c2-8c1f-705b65a98530","statement_id":19,"state":"finished","livy_statement_state":"available","queued_time":"2023-07-19T15:37:56.9092353Z","session_start_time":null,"execution_start_time":"2023-07-19T15:37:57.5118916Z","execution_finish_time":"2023-07-19T15:38:02.8411901Z","spark_jobs":{"numbers":{"RUNNING":0,"SUCCEEDED":0,"FAILED":0,"UNKNOWN":0},"jobs":[],"limit":20,"rule":"ALL_DESC"},"parent_msg_id":"f50eacb6-49bc-4ec6-a56c-191efe328c88"},"text/plain":"StatementMeta(, 2511b538-6adf-43c2-8c1f-705b65a98530, 19, Finished, Available)"},"metadata":{}}],"execution_count":6,"metadata":{"jupyter":{"source_hidden":false,"outputs_hidden":false},"nteract":{"transient":{"deleting":false}}},"id":"502cec02-4311-4a40-ad70-bda74fdf513e"},{"cell_type":"code","source":["# Repeat the process for the test dataset\r\n","test_df = spark.sql(\"SELECT * FROM DemoLakehouse.sampled_test LIMIT 1000\")\r\n","\r\n","# convert test_df to pandas dataframe\r\n","test_df = test_df.toPandas()\r\n","\r\n","# Create a new column in the dataframe using the apply method\r\n","test_df['image_url'] = test_df['filename'].apply(lambda filename: f\"/lakehouse/default/Files/images/train/{filename}\")\r\n","\r\n","# Fit the LabelEncoder to the label column in the test_df DataFrame\r\n","le.fit(test_df['label'])\r\n","\r\n","# Transform the label column to numerical labels using the LabelEncoder\r\n","test_df['labels'] = le.transform(test_df['label'])\r\n","\r\n","# combine both the train and test dataset\r\n","data = pd.concat([test_df, train_df])\r\n","\r\n","# drop filename column \r\n","data = data[['image_url', 'labels']]"],"outputs":[{"output_type":"display_data","data":{"application/vnd.livy.statement-meta+json":{"spark_pool":null,"session_id":"2511b538-6adf-43c2-8c1f-705b65a98530","statement_id":20,"state":"finished","livy_statement_state":"available","queued_time":"2023-07-19T15:39:48.4423871Z","session_start_time":null,"execution_start_time":"2023-07-19T15:39:49.0907204Z","execution_finish_time":"2023-07-19T15:39:53.0571535Z","spark_jobs":{"numbers":{"RUNNING":0,"SUCCEEDED":6,"FAILED":0,"UNKNOWN":0},"jobs":[{"dataWritten":0,"dataRead":2349,"rowCount":146,"jobId":18,"name":"toPandas at /tmp/ipykernel_20773/203456709.py:5","description":"Job group for statement 20:\n# Repeat the process for the test dataset\ntest_df = spark.sql(\"SELECT * FROM DemoLakehouse.sampled_test LIMIT 1000\")\n\n# convert test_df to pandas dataframe\ntest_df = test_df.toPandas()\n\n# Create a new column in the dataframe using the apply method\ntest_df['image_url'] = test_df['filename'].apply(lambda filename: f\"/lakehouse/default/Files/images/train/{filename}\")\n\n# Fit the LabelEncoder to the label column in the test_df DataFrame\nle.fit(test_df['label'])\n\n# Transform the label column to numerical labels using the LabelEncoder\ntest_df['labels'] = le.transform(test_df['label'])\n\n# combine both the train and test dataset\ndata = pd.concat([test_df, train_df])\n\n# drop filename column \ndata = data[['image_url', 'labels']]","submissionTime":"2023-07-19T15:39:52.288GMT","completionTime":"2023-07-19T15:39:52.308GMT","stageIds":[30,31],"jobGroup":"20","status":"SUCCEEDED","numTasks":2,"numActiveTasks":0,"numCompletedTasks":1,"numSkippedTasks":1,"numFailedTasks":0,"numKilledTasks":0,"numCompletedIndices":1,"numActiveStages":0,"numCompletedStages":1,"numSkippedStages":1,"numFailedStages":0,"killedTasksSummary":{}},{"dataWritten":2349,"dataRead":4109,"rowCount":292,"jobId":17,"name":"toPandas at /tmp/ipykernel_20773/203456709.py:5","description":"Job group for statement 20:\n# Repeat the process for the test dataset\ntest_df = spark.sql(\"SELECT * FROM DemoLakehouse.sampled_test LIMIT 1000\")\n\n# convert test_df to pandas dataframe\ntest_df = test_df.toPandas()\n\n# Create a new column in the dataframe using the apply method\ntest_df['image_url'] = test_df['filename'].apply(lambda filename: f\"/lakehouse/default/Files/images/train/{filename}\")\n\n# Fit the LabelEncoder to the label column in the test_df DataFrame\nle.fit(test_df['label'])\n\n# Transform the label column to numerical labels using the LabelEncoder\ntest_df['labels'] = le.transform(test_df['label'])\n\n# combine both the train and test dataset\ndata = pd.concat([test_df, train_df])\n\n# drop filename column \ndata = data[['image_url', 'labels']]","submissionTime":"2023-07-19T15:39:52.139GMT","completionTime":"2023-07-19T15:39:52.278GMT","stageIds":[29],"jobGroup":"20","status":"SUCCEEDED","numTasks":1,"numActiveTasks":0,"numCompletedTasks":1,"numSkippedTasks":0,"numFailedTasks":0,"numKilledTasks":0,"numCompletedIndices":1,"numActiveStages":0,"numCompletedStages":1,"numSkippedStages":0,"numFailedStages":0,"killedTasksSummary":{}},{"dataWritten":0,"dataRead":1723,"rowCount":3,"jobId":16,"name":"toPandas at /tmp/ipykernel_20773/203456709.py:5","description":"Delta: Job group for statement 20:\n# Repeat the process for the test dataset\ntest_df = spark.sql(\"SELECT * FROM DemoLakehouse.sampled_test LIMIT 1000\")\n\n# convert test_df to pandas dataframe\ntest_df = test_df.toPandas()\n\n# Create a new column in the dataframe using the apply method\ntest_df['image_url'] = test_df['filename'].apply(lambda filename: f\"/lakehouse/default/Files/images/train/{filename}\")\n\n# Fit the LabelEncoder to the label column in the test_df DataFrame\nle.fit(test_df['label'])\n\n# Transform the label column to numerical labels using the LabelEncoder\ntest_df['labels'] = le.transform(test_df['label'])\n\n# combine both the train and test dataset\ndata = pd.concat([test_df, train_df])\n\n# drop filename column \ndata = data[['image_url', 'labels']]: Filtering files for query","submissionTime":"2023-07-19T15:39:51.823GMT","completionTime":"2023-07-19T15:39:52.088GMT","stageIds":[27,28],"jobGroup":"20","status":"SUCCEEDED","numTasks":51,"numActiveTasks":0,"numCompletedTasks":50,"numSkippedTasks":1,"numFailedTasks":0,"numKilledTasks":0,"numCompletedIndices":50,"numActiveStages":0,"numCompletedStages":1,"numSkippedStages":1,"numFailedStages":0,"killedTasksSummary":{}},{"dataWritten":0,"dataRead":4312,"rowCount":50,"jobId":15,"name":"toString at String.java:2994","description":"Delta: Job group for statement 20:\n# Repeat the process for the test dataset\ntest_df = spark.sql(\"SELECT * FROM DemoLakehouse.sampled_test LIMIT 1000\")\n\n# convert test_df to pandas dataframe\ntest_df = test_df.toPandas()\n\n# Create a new column in the dataframe using the apply method\ntest_df['image_url'] = test_df['filename'].apply(lambda filename: f\"/lakehouse/default/Files/images/train/{filename}\")\n\n# Fit the LabelEncoder to the label column in the test_df DataFrame\nle.fit(test_df['label'])\n\n# Transform the label column to numerical labels using the LabelEncoder\ntest_df['labels'] = le.transform(test_df['label'])\n\n# combine both the train and test dataset\ndata = pd.concat([test_df, train_df])\n\n# drop filename column \ndata = data[['image_url', 'labels']]: Compute snapshot for version: 0","submissionTime":"2023-07-19T15:39:51.528GMT","completionTime":"2023-07-19T15:39:51.568GMT","stageIds":[24,25,26],"jobGroup":"20","status":"SUCCEEDED","numTasks":52,"numActiveTasks":0,"numCompletedTasks":1,"numSkippedTasks":51,"numFailedTasks":0,"numKilledTasks":0,"numCompletedIndices":1,"numActiveStages":0,"numCompletedStages":1,"numSkippedStages":2,"numFailedStages":0,"killedTasksSummary":{}},{"dataWritten":4312,"dataRead":1589,"rowCount":54,"jobId":14,"name":"toString at String.java:2994","description":"Delta: Job group for statement 20:\n# Repeat the process for the test dataset\ntest_df = spark.sql(\"SELECT * FROM DemoLakehouse.sampled_test LIMIT 1000\")\n\n# convert test_df to pandas dataframe\ntest_df = test_df.toPandas()\n\n# Create a new column in the dataframe using the apply method\ntest_df['image_url'] = test_df['filename'].apply(lambda filename: f\"/lakehouse/default/Files/images/train/{filename}\")\n\n# Fit the LabelEncoder to the label column in the test_df DataFrame\nle.fit(test_df['label'])\n\n# Transform the label column to numerical labels using the LabelEncoder\ntest_df['labels'] = le.transform(test_df['label'])\n\n# combine both the train and test dataset\ndata = pd.concat([test_df, train_df])\n\n# drop filename column \ndata = data[['image_url', 'labels']]: Compute snapshot for version: 0","submissionTime":"2023-07-19T15:39:50.855GMT","completionTime":"2023-07-19T15:39:51.508GMT","stageIds":[22,23],"jobGroup":"20","status":"SUCCEEDED","numTasks":51,"numActiveTasks":0,"numCompletedTasks":50,"numSkippedTasks":1,"numFailedTasks":0,"numKilledTasks":0,"numCompletedIndices":50,"numActiveStages":0,"numCompletedStages":1,"numSkippedStages":1,"numFailedStages":0,"killedTasksSummary":{}},{"dataWritten":1589,"dataRead":1354,"rowCount":8,"jobId":13,"name":"toString at String.java:2994","description":"Delta: Job group for statement 20:\n# Repeat the process for the test dataset\ntest_df = spark.sql(\"SELECT * FROM DemoLakehouse.sampled_test LIMIT 1000\")\n\n# convert test_df to pandas dataframe\ntest_df = test_df.toPandas()\n\n# Create a new column in the dataframe using the apply method\ntest_df['image_url'] = test_df['filename'].apply(lambda filename: f\"/lakehouse/default/Files/images/train/{filename}\")\n\n# Fit the LabelEncoder to the label column in the test_df DataFrame\nle.fit(test_df['label'])\n\n# Transform the label column to numerical labels using the LabelEncoder\ntest_df['labels'] = le.transform(test_df['label'])\n\n# combine both the train and test dataset\ndata = pd.concat([test_df, train_df])\n\n# drop filename column \ndata = data[['image_url', 'labels']]: Compute snapshot for version: 0","submissionTime":"2023-07-19T15:39:50.615GMT","completionTime":"2023-07-19T15:39:50.689GMT","stageIds":[21],"jobGroup":"20","status":"SUCCEEDED","numTasks":1,"numActiveTasks":0,"numCompletedTasks":1,"numSkippedTasks":0,"numFailedTasks":0,"numKilledTasks":0,"numCompletedIndices":1,"numActiveStages":0,"numCompletedStages":1,"numSkippedStages":0,"numFailedStages":0,"killedTasksSummary":{}}],"limit":20,"rule":"ALL_DESC"},"parent_msg_id":"e381db30-06a7-42ea-9a63-7a529f286699"},"text/plain":"StatementMeta(, 2511b538-6adf-43c2-8c1f-705b65a98530, 20, Finished, Available)"},"metadata":{}}],"execution_count":7,"metadata":{"jupyter":{"source_hidden":false,"outputs_hidden":false},"nteract":{"transient":{"deleting":false}}},"id":"573b9c81-be7e-4a3b-a0e7-00894d53b117"},{"cell_type":"code","source":["from torch.utils.data import Dataset\r\n","from torch.utils.data import DataLoader\r\n","from torchvision import transforms\r\n","\r\n","import os\r\n","from PIL import Image\r\n","\r\n","class CustomDataset(Dataset):\r\n"," def __init__(self, root_dir, transform=None):\r\n"," self.root_dir = root_dir\r\n"," self.data = data\r\n"," self.transform = transform\r\n","\r\n"," def __len__(self):\r\n"," return len(self.data)\r\n","\r\n"," def __getitem__(self, idx):\r\n"," while True:\r\n"," img_name = os.path.join(self.root_dir, self.data.iloc[idx, 0])\r\n"," if not os.path.exists(img_name):\r\n"," idx = (idx + 1) % len(self.data)\r\n"," continue\r\n"," image = Image.open(img_name)\r\n"," if self.transform:\r\n"," image = self.transform(image)\r\n"," labels = self.data.iloc[idx, 1]\r\n"," return image, labels\r\n","\r\n","transform = transforms.Compose([\r\n"," transforms.Resize((224, 224)),\r\n"," transforms.ToTensor()\r\n","])\r\n","\r\n","train_set = CustomDataset(\"/lakehouse/default/Files/images/train/\", transform=transform)\r\n","test_set = CustomDataset(\"/lakehouse/default/Files/images/test/\", transform=transform)"],"outputs":[{"output_type":"display_data","data":{"application/vnd.livy.statement-meta+json":{"spark_pool":null,"session_id":"2511b538-6adf-43c2-8c1f-705b65a98530","statement_id":22,"state":"finished","livy_statement_state":"available","queued_time":"2023-07-19T15:41:46.7440651Z","session_start_time":null,"execution_start_time":"2023-07-19T15:41:47.4148013Z","execution_finish_time":"2023-07-19T15:41:47.8032129Z","spark_jobs":{"numbers":{"RUNNING":0,"SUCCEEDED":0,"FAILED":0,"UNKNOWN":0},"jobs":[],"limit":20,"rule":"ALL_DESC"},"parent_msg_id":"0eb35d0d-9dc0-4e8a-a504-5cf62bc86c8a"},"text/plain":"StatementMeta(, 2511b538-6adf-43c2-8c1f-705b65a98530, 22, Finished, Available)"},"metadata":{}}],"execution_count":9,"metadata":{"jupyter":{"source_hidden":false,"outputs_hidden":false},"nteract":{"transient":{"deleting":false}}},"id":"cf6314f1-e948-4ded-af9e-382367938537"},{"cell_type":"code","source":["# Load the training and test data\r\n","train_loader = DataLoader(train_set, batch_size=100, shuffle=True, num_workers=2)\r\n","test_loader = DataLoader(test_set, batch_size=100, shuffle=False, num_workers=2)"],"outputs":[{"output_type":"display_data","data":{"application/vnd.livy.statement-meta+json":{"spark_pool":null,"session_id":"2511b538-6adf-43c2-8c1f-705b65a98530","statement_id":23,"state":"finished","livy_statement_state":"available","queued_time":"2023-07-19T15:42:15.0631793Z","session_start_time":null,"execution_start_time":"2023-07-19T15:42:15.7004355Z","execution_finish_time":"2023-07-19T15:42:16.0674669Z","spark_jobs":{"numbers":{"RUNNING":0,"SUCCEEDED":0,"FAILED":0,"UNKNOWN":0},"jobs":[],"limit":20,"rule":"ALL_DESC"},"parent_msg_id":"6bd8709a-6212-4bde-9acf-f1af27b5f45f"},"text/plain":"StatementMeta(, 2511b538-6adf-43c2-8c1f-705b65a98530, 23, Finished, Available)"},"metadata":{}}],"execution_count":10,"metadata":{"jupyter":{"source_hidden":false,"outputs_hidden":false},"nteract":{"transient":{"deleting":false}}},"id":"ce225580-5bba-41b4-874b-45d625c93818"},{"cell_type":"code","source":["# Using mlflow library to activate our ml experiment\r\n","\r\n","import mlflow\r\n","\r\n","mlflow.set_experiment(\"serengeti-experiment\")"],"outputs":[{"output_type":"display_data","data":{"application/vnd.livy.statement-meta+json":{"spark_pool":null,"session_id":"2511b538-6adf-43c2-8c1f-705b65a98530","statement_id":24,"state":"finished","livy_statement_state":"available","queued_time":"2023-07-19T15:42:32.8524728Z","session_start_time":null,"execution_start_time":"2023-07-19T15:42:33.4709955Z","execution_finish_time":"2023-07-19T15:42:42.193492Z","spark_jobs":{"numbers":{"RUNNING":0,"SUCCEEDED":0,"FAILED":0,"UNKNOWN":0},"jobs":[],"limit":20,"rule":"ALL_DESC"},"parent_msg_id":"5cf31587-0c86-4b2c-98fe-2387150cbf0c"},"text/plain":"StatementMeta(, 2511b538-6adf-43c2-8c1f-705b65a98530, 24, Finished, Available)"},"metadata":{}},{"output_type":"stream","name":"stderr","text":["2023/07/19 15:42:33 INFO mlflow.tracking.fluent: Experiment with name 'serengeti-experiment' does not exist. Creating a new experiment.\n"]},{"output_type":"execute_result","execution_count":28,"data":{"text/plain":""},"metadata":{}}],"execution_count":11,"metadata":{"jupyter":{"source_hidden":false,"outputs_hidden":false},"nteract":{"transient":{"deleting":false}}},"id":"a1707c72-9196-483f-bdf7-7b151150a09c"},{"cell_type":"code","source":["import torch\r\n","import torchvision\r\n","import torch.nn as nn\r\n","\r\n","# load the pre-trained DenseNet 201 model\r\n","model = torchvision.models.densenet201(pretrained=True)\r\n","num_ftrs = model.classifier.in_features\r\n","model.classifier = nn.Linear(num_ftrs, 53)\r\n","device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\r\n","model = model.to(device)"],"outputs":[{"output_type":"display_data","data":{"application/vnd.livy.statement-meta+json":{"spark_pool":null,"session_id":"2511b538-6adf-43c2-8c1f-705b65a98530","statement_id":27,"state":"finished","livy_statement_state":"available","queued_time":"2023-07-19T15:43:52.2831787Z","session_start_time":null,"execution_start_time":"2023-07-19T15:43:52.8780256Z","execution_finish_time":"2023-07-19T15:43:53.9090612Z","spark_jobs":{"numbers":{"RUNNING":0,"SUCCEEDED":0,"FAILED":0,"UNKNOWN":0},"jobs":[],"limit":20,"rule":"ALL_DESC"},"parent_msg_id":"9088d9a4-8c62-4dd8-80d4-0b87c47c4cc5"},"text/plain":"StatementMeta(, 2511b538-6adf-43c2-8c1f-705b65a98530, 27, Finished, Available)"},"metadata":{}}],"execution_count":14,"metadata":{"jupyter":{"source_hidden":false,"outputs_hidden":false},"nteract":{"transient":{"deleting":false}}},"id":"d81e2ca7-3610-423f-b937-6bd8a90740ad"},{"cell_type":"code","source":["# define the loss function\r\n","import torch.optim as optim\r\n","criterion = nn.CrossEntropyLoss()\r\n","optimizer = optim.Adam(model.parameters(), lr=0.01)"],"outputs":[{"output_type":"display_data","data":{"application/vnd.livy.statement-meta+json":{"spark_pool":null,"session_id":"2511b538-6adf-43c2-8c1f-705b65a98530","statement_id":29,"state":"finished","livy_statement_state":"available","queued_time":"2023-07-19T15:44:43.914008Z","session_start_time":null,"execution_start_time":"2023-07-19T15:44:44.4992397Z","execution_finish_time":"2023-07-19T15:44:44.8956158Z","spark_jobs":{"numbers":{"RUNNING":0,"SUCCEEDED":0,"FAILED":0,"UNKNOWN":0},"jobs":[],"limit":20,"rule":"ALL_DESC"},"parent_msg_id":"7900f58e-c6b9-4081-9c6c-1bbf52143e69"},"text/plain":"StatementMeta(, 2511b538-6adf-43c2-8c1f-705b65a98530, 29, Finished, Available)"},"metadata":{}}],"execution_count":16,"metadata":{"jupyter":{"source_hidden":false,"outputs_hidden":false},"nteract":{"transient":{"deleting":false}}},"id":"ee18c812-8286-4454-9488-249126e19d6d"},{"cell_type":"code","source":["# train the model\r\n","num_epochs = 5\r\n","for epoch in range(num_epochs):\r\n"," print('Epoch {}/{}'.format(epoch, num_epochs - 1))\r\n"," print('-' * 10)\r\n","\r\n"," # Each epoch has a training and validation phase\r\n"," for phase in [\"train\", \"val\"]:\r\n"," if phase == \"train\":\r\n"," model.train() # Set model to training mode\r\n"," else:\r\n"," model.eval() # Set model to evaluate mode\r\n","\r\n"," running_loss = 0.0\r\n"," running_corrects = 0\r\n"," for i, data in enumerate(train_loader, ):\r\n"," # get the inputs\r\n"," inputs, labels = data[0].to(device), data[1].to(device)\r\n"," inputs = inputs.to(device)\r\n"," labels = labels.to(device)\r\n","\r\n"," # zero the parameter gradients\r\n"," optimizer.zero_grad()\r\n","\r\n"," # forward\r\n"," # track history if only in train\r\n"," with torch.set_grad_enabled(phase == \"train\"):\r\n"," outputs = model(inputs)\r\n"," _, preds = torch.max(outputs, 1)\r\n"," loss = criterion(outputs, labels)\r\n","\r\n"," # backward + optimize only if in training phase\r\n"," if phase == \"train\":\r\n"," loss.backward()\r\n"," optimizer.step()\r\n","\r\n"," # print statistics\r\n"," running_loss += loss.item()\r\n"," if i % 100 == 20: # print every 100 mini-batches\r\n"," print('[%d, %5d] loss: %.3f' %\r\n"," (epoch + 1, i + 1, running_loss / 100))\r\n"," running_loss = 0.0\r\n"," \r\n"," print('Finished Training')"],"outputs":[{"output_type":"display_data","data":{"application/vnd.livy.statement-meta+json":{"spark_pool":null,"session_id":"2511b538-6adf-43c2-8c1f-705b65a98530","statement_id":30,"state":"finished","livy_statement_state":"available","queued_time":"2023-07-19T15:45:31.6353285Z","session_start_time":null,"execution_start_time":"2023-07-19T15:45:32.2015293Z","execution_finish_time":"2023-07-19T15:52:59.7864957Z","spark_jobs":{"numbers":{"RUNNING":0,"SUCCEEDED":0,"FAILED":0,"UNKNOWN":0},"jobs":[],"limit":20,"rule":"ALL_DESC"},"parent_msg_id":"50fc653b-bf1e-44aa-8660-d352b1839848"},"text/plain":"StatementMeta(, 2511b538-6adf-43c2-8c1f-705b65a98530, 30, Finished, Available)"},"metadata":{}},{"output_type":"stream","name":"stdout","text":["Epoch 0/4\n----------\nFinished Training\nEpoch 1/4\n----------\nFinished Training\nEpoch 2/4\n----------\nFinished Training\nEpoch 3/4\n----------\nFinished Training\nEpoch 4/4\n----------\nFinished Training\n"]}],"execution_count":17,"metadata":{"jupyter":{"source_hidden":false,"outputs_hidden":false},"nteract":{"transient":{"deleting":false}}},"id":"fa5d2e4e-4e13-4227-9a55-2b73833ddd38"},{"cell_type":"code","source":["# use an MLflow run and track the results within our machine learning experiment.\r\n","\r\n","with mlflow.start_run() as run:\r\n"," print(\"log pytorch model:\")\r\n"," mlflow.pytorch.log_model(\r\n"," model, \"pytorch-model\",\r\n"," registered_model_name=\"serengeti-pytorch\"\r\n"," )\r\n"," \r\n"," model_uri = \"runs:/{}/pytorch-model\".format(run.info.run_id)\r\n"," print(\"Model saved in run %s\" % run.info.run_id)\r\n"," print(f\"Model URI: {model_uri}\")"],"outputs":[{"output_type":"display_data","data":{"application/vnd.livy.statement-meta+json":{"spark_pool":null,"session_id":"2511b538-6adf-43c2-8c1f-705b65a98530","statement_id":31,"state":"finished","livy_statement_state":"available","queued_time":"2023-07-19T15:53:40.7061951Z","session_start_time":null,"execution_start_time":"2023-07-19T15:53:41.1783907Z","execution_finish_time":"2023-07-19T15:54:01.0757071Z","spark_jobs":{"numbers":{"RUNNING":0,"SUCCEEDED":0,"FAILED":0,"UNKNOWN":0},"jobs":[],"limit":20,"rule":"ALL_DESC"},"parent_msg_id":"f52c72f1-0d70-4d69-92d1-36b48cada37f"},"text/plain":"StatementMeta(, 2511b538-6adf-43c2-8c1f-705b65a98530, 31, Finished, Available)"},"metadata":{}},{"output_type":"stream","name":"stdout","text":["log pytorch model:\nModel saved in run fa021ff1-28bd-421a-bba9-7012a210226a\nModel URI: runs:/fa021ff1-28bd-421a-bba9-7012a210226a/pytorch-model\n"]},{"output_type":"stream","name":"stderr","text":["2023/07/19 15:53:51 WARNING mlflow.utils.environment: Encountered an unexpected error while inferring pip requirements (model URI: /tmp/tmp5cq_lt5t/model/data, flavor: pytorch), fall back to return ['torch==2.0.1', 'cloudpickle==2.2.1']. Set logging level to DEBUG to see the full traceback.\n/home/trusted-service-user/cluster-env/trident_env/lib/python3.10/site-packages/_distutils_hack/__init__.py:33: UserWarning: Setuptools is replacing distutils.\n warnings.warn(\"Setuptools is replacing distutils.\")\nSuccessfully registered model 'serengeti-pytorch'.\n2023/07/19 15:53:58 INFO mlflow.tracking._model_registry.client: Waiting up to 300 seconds for model version to finish creation. Model name: serengeti-pytorch, version 1\nCreated version '1' of model 'serengeti-pytorch'.\n"]}],"execution_count":18,"metadata":{"jupyter":{"source_hidden":false,"outputs_hidden":false},"nteract":{"transient":{"deleting":false}}},"id":"a33b4d88-5136-459b-a012-6fe66a11a29e"},{"cell_type":"code","source":["# load and evaluate the model\r\n","loaded_model = mlflow.pytorch.load_model(model_uri)\r\n","print(type(loaded_model))\r\n","correct_cnt, total_cnt, ave_loss = 0, 0, 0\r\n","for batch_idx, (x, target) in enumerate(test_loader):\r\n"," x, target = x, target\r\n"," out = loaded_model(x)\r\n"," loss = criterion(out, target)\r\n"," _, pred_label = torch.max(out.data, 1)\r\n"," total_cnt += x.data.size()[0]\r\n"," correct_cnt += (pred_label == target.data).sum()\r\n"," ave_loss = (ave_loss * batch_idx + loss.item()) / (batch_idx + 1)\r\n"," \r\n"," if (batch_idx + 1) % 100 == 0 or (batch_idx + 1) == len(test_loader):\r\n"," print(\r\n"," \"==>>> epoch: {}, batch index: {}, test loss: {:.6f}, acc: {:.3f}\".format(\r\n"," epoch, batch_idx + 1, ave_loss, correct_cnt * 1.0 / total_cnt\r\n"," )\r\n"," )"],"outputs":[{"output_type":"display_data","data":{"application/vnd.livy.statement-meta+json":{"spark_pool":null,"session_id":"2511b538-6adf-43c2-8c1f-705b65a98530","statement_id":32,"state":"finished","livy_statement_state":"available","queued_time":"2023-07-19T15:54:11.1794679Z","session_start_time":null,"execution_start_time":"2023-07-19T15:54:11.7647014Z","execution_finish_time":"2023-07-19T15:54:38.3351844Z","spark_jobs":{"numbers":{"RUNNING":0,"SUCCEEDED":0,"FAILED":0,"UNKNOWN":0},"jobs":[],"limit":20,"rule":"ALL_DESC"},"parent_msg_id":"e3fdb800-74de-4b24-bfbc-72922f992c9e"},"text/plain":"StatementMeta(, 2511b538-6adf-43c2-8c1f-705b65a98530, 32, Finished, Available)"},"metadata":{}},{"output_type":"stream","name":"stdout","text":["\n==>>> epoch: 4, batch index: 5, test loss: 31494.506250, acc: 0.373\n"]}],"execution_count":19,"metadata":{"jupyter":{"source_hidden":false,"outputs_hidden":false},"nteract":{"transient":{"deleting":false}}},"id":"6a36b31f-4719-498d-af7d-959d1b77d943"},{"cell_type":"code","source":["# Load a new image from the test data using Pillow\r\n","image = Image.open('/lakehouse/default/Files/images/test/SER_S11/B03/B03_R1/SER_S11_B03_R1_IMAG1021.JPG')\r\n","image"],"outputs":[{"output_type":"display_data","data":{"application/vnd.livy.statement-meta+json":{"spark_pool":null,"session_id":"2511b538-6adf-43c2-8c1f-705b65a98530","statement_id":34,"state":"finished","livy_statement_state":"available","queued_time":"2023-07-19T15:58:03.9984412Z","session_start_time":null,"execution_start_time":"2023-07-19T15:58:04.7322462Z","execution_finish_time":"2023-07-19T15:58:06.526933Z","spark_jobs":{"numbers":{"RUNNING":0,"SUCCEEDED":0,"FAILED":0,"UNKNOWN":0},"jobs":[],"limit":20,"rule":"ALL_DESC"},"parent_msg_id":"02a76afe-ba46-4610-905c-241bc195f82e"},"text/plain":"StatementMeta(, 2511b538-6adf-43c2-8c1f-705b65a98530, 34, Finished, Available)"},"metadata":{}},{"output_type":"execute_result","execution_count":58,"data":{"text/plain":"","image/png":"iVBORw0KGgoAAAANSUhEUgAAAOAAAACoCAIAAAB2RHW3AAD4MklEQVR4nHT9Wa9kWZYeiH1r2PucY3bv9SE8hqwhK5nMGlgUJ5QIaGBDaD20AP0OPUj6A/3eLwKkH6KGCOlB4IPUgLpFqdUQCy2CzWKTrKosFrMqIzMyItzD3e81s3P2XoMe9jFzj2y0PWTeuG7XhnPWXsO3vvUt+t/+x/87jx4RRJSZ4QBARO6emZlgEIBMykwiSgoiogQoiQgAgEzH9x98fYxXY2YiGj+Mn2//ywxm1uvzhQsza2ERERHV8QMDUJHxJ6rKNP5dSAgAcRJREa1VVZVBRCSUIlJKGf85/pBEAbilmWUmgMz06JnZe5cyMWuCwJJBqrU7iPLTT56rxjxREaQHM9dpyYR7EtjME+69AfCMzMxMy4gId4/rg0kpc/xm/C8ARbolwElCRKUUUYowVT7eH5P5/ePl7bund09tWu4AZimtNWSfq8JaFbx4dleUEXY+n3rvZr6tfbNuFq13Y86IZZo5scx127ZS5P7+qKqXba21rs1aswi0ra/r2taLu7v7ed0yU0Raa+MqZSYoAQABgGjYA0cEwAAiIjOHwfz3PQI+LAHYDWD83t0xfk/x8fN1movZbo4AUnIYooiMC4wcBpoYlkqM8RMI2G2UiH/tc4zfj5f6tV9ebTqvX5spMzPH10NyRCRovyJgAJmhIrdTRETYDX68O1GScCane1KaA6r64QyAxtsBUFV3B/L28YQhWt0dAImMA0UiFrFtFynz/fEwVRV2uDU3a32aJuZOLJkws0gTYmY278wyvoiACETEQRwUyQkgMse5Yubee0RYAKJEkhGishwPRXldn453h0w/1Hl+We8OR/vZL9J793jyFhFMjgimmGqFqLmHhyd7chKokJJCg0oVEsqsKoQEca0ViHFllmmelnmacl3X82mFih6WpiRSzGxZ19b6eV1VdTisccxASSRAAARQJmV+sAQGIWnczf/ufSciMAM0bv6w6d1wM6/ejhJ++yt9eHhorfXery7z+tKRERRCAOfHBkRMeTtPt7eXcaS+/5mG+8TVFCRz/0oAMcs4gszMxAQiEBKgAFEmRwRRRqi7I8mBYaDjcweluw8PyswiwkTDMyEYCNoPTRk3A747tv2LfHwJVOs8MfO6rs3MPZOSEvsJyHDv23a5O9b0iOt92rYNSQGKCAZBCJREyZGIjEjsvoQyk3K/EESUsUeqDKJko2Qmj/BwJq21Ho51WYpQdAshr1JkqZ88O142e/fu3OMIj+6+5VqVa609kJ7Ww4KSC4SJLMnTzIKJy7at3W0pfP/s+PLhCERYX5Zl7c3dGTkV9aq1SARq1WVZImJdW+/9m2++ba211kTkg2dJgIbj2AMQAIAzxi/Gpb75NdzMF0jwf8dn8Xjmh5u7O0BOItLPP/+09z5s1MzMLMzzGokiwj3Hjc/kzMzgD84PcXubm4u6ebXbf378gb5/pBgAJ4g/uPqPXWxEhjklIARgPIeZ3T0oyDkiKEhFCAhSGukHgYgiwohqZiYxSzJRBoCtd4yAK5QJIZ7n+XA4SNm/lzG6R+9bEtc6rdvZ3p/6Vpf6CRAj6pkFAREIpKoCkSBkFJbhnpQlryfTzCzg7swcgdZaRGQEAYAwW6ZnjoiZHhujHI+HrZ0IEtZJgOBPX718/+789P59RDCYRTMyiczj/dNq26qFR0CIpJ609bw0771nAkSKmAr9+Cc/hjXh9LbN8/z4+Pj4eMqMInJ/XETk8fFxWg6lFOtxOp167+62B64ICqIgAiV8nPfbbQIADAf3wX0m+GYVw1/imowNowRAFHQ15Wvid/2LIGLWaSq1yDyVqznufnQEoN5827abfwVgtqca1w8XNwOl+DUPCiIK+nVjHS5tREAhYf6QmALg4VaJCUlJ4xgMh3p7o/3d4XAwyAHm4XGFCMxJREQCYPep+0mVkTBR3jITGklqrSoix+NSa+3up9NlW1tmIBLhrJzp67qWIuHBzDGSBKZwCJMhKrNHJnvdvw2Pj+oZRMTuKcRa3T3Tx0XYE9P0SEpKEa2VEb5uF0RPDNuIhCYRi3rm1p0iPYyQhCRigN0iiNcWNIJQRo9sAUskle5FOTryD//u34tIRkT4vFSEC3MtYiB3n2udl3o5P659M2tm8e79d725u2fGSEhG1jjSMIrISACBuIZpHmYL8LhnN5P9nm8SEAGeAEBxs8jMZM6PLAfj9qn1frtVIqKq44WmaRp+dNvqzb+6+8jh9iNyi/uAXPO8X3OlH52hoBHIMwhEu0HGSA/G04go0/ewSPyx9x3n+PbD7nE5CYKkDNw+EokOMwbEIrj3zBxpQGYKUV7T+evrh7sT0bIsy4Kte47Y7ZmZ7vT8xTO+VokJ6RZMOtJXhz2eL0AMD5TI2+cE774kwjNDRJgyGaosQpnaezfLCiKiAAvrNJVSBOHb5gk3M+GyXZpWauabRxAXBoQnLSIkDISf1kvrGxGpKoiTJAkeyJDMiGRPUtEMhLi17bCUIkxK3jXqtGVnomWu93fHtt7/8tu3W2u99741M1OhdDIzIY6R20VkRiJu8TMzI5Bpsdvq8K58cyXfd1sfbimAGEGPk4hGiL6+JpgJgN5i68evNe4ogGHgpaiZeetm1h1mdPWzu4Fyfs8cb9b2se/84Pnye4721yonot3u9xSQd+scOdDHnvv2mrsjNwQlS1ACUKBnJlHZsnf3qZTr90LmXhRWLSIUEZfLRZTnedZSlqkAx1prErvF0/mkpQoTEa3ni4iYwaITCRsF6HLeSpFLj7syRURPH8Y8sthEBiGZejhlZCYxCOTuxKGFKlUWySSQlFJGbTq8lHAJ4q1vqyVIpeizly/Wi2f6pEJELMTMcy2Zh3fv3iHJw7tbErtltw4wRFtfkfHNt18/v9fnx+lumXtfOeV22VXVM87rBUwPDw+Xy2Vd12fPnl0ulwgcDuTuSLbr44ZLZKYTtdbMbOQsI5RmZuTuLz52WJmZQcQfckLmZGbECHgAODNu3o2Ztfd++/sduCEamdPwoPBU4lKnLNXdt61tRuNfM2iPU98v2IerGWFOa7nZbvqAV2xkcgP3ESnMnLdMIPZcQouUUlT1Zo7MH9Lf23t9wB/GqXCCjvKQr5GUmIOZSSQzwxxAxP59R8RofSsorTVmrrUuYPd0z0DW+fB4ukylHI/Hp8vbeTqY0fv35wAx8zRN6xZ3dTpfGouGWymSmYgY8JmMkDi+wi2URIwLWEphVFUFyziP3d09mZkYmWnNKLFtZ5IpE88fDjbluHoRAQqCmBmAWj/tHufzGpetewJcpLLo2lFqKYzvXr/59PlvZPrWLkxkZqUUkWk5JJDn8/lyWc1MRJ89e/bixYvMfPv2/bquy7KIlPP5vG3buq4j97shZevWN5Ft27r7uEERYYbcQyvdbHFEewIhP/hRIh41w6/VKXRFJBV586DwzNjTAiIIcQiIdK+/aIcSV922Ee7NvEdqfAA7ByQxyrpxekRYRGqtw+Yul4uZjW9ys9Hj4W5kvWWqbn3EieEyx7UQkfDMADHdXnm8CIFuT7vW3SBwqdSamZmq1qrjCapqrZVS3DuzHI+Lqq7bhZlVGQiz9vTk87wo4/HxqbWG6U7r9O7947vHS+/m0cyiOTuIky9PG5D2uFZli/Xu7rD2bG0FgDARPh4OQtjahUbqSRhfeRw8Itq6rpszo9ZCDLBSWvfmaxORdM/Mwzw182WqzUJm3bbNvIuISCEiotI9CnNSHO8qlfbtN2+W43Gaptdv3hSWdPNYf+e3f0KIqVZRoiRW8mBhnQqNbNHCw+38eHb3w+GwLMvnn3/+9u1bZp7nOXNkz+XFi/t5nsf9ulwu796v7r6u6/l8vmzN9wNm7h5Je8i/ultKgPiWXGUmEERCnABl+s0T3UB0/bUs4aNcMEe9datshqmVKotNe+Hf/PYJhknVWploIL2MBCClMnMpZZrKNE19a+u63mJEKUVrcc9SdF6mbdtKKTdsbBzHUSvsR8yvWUEmcvwyPhRY19IEgIcNBwzY8E3jO4/XH8ejlHI8HkvV9XQW0IAtRhRTLYe5ttYOh8Pj46MHem9EIjqftnNScfckBjEoPbIZqEdp2cxOp82jz6UcD+V8ad4bMaxtc6miVJVZxcysNSIKnrbWn57eE9H9w7FWRaSQsEyZzsyTCAagKzQzn1Zvrbl7KZMnkNy7r61P02Kep9Nl6661ttbWrdc6e1tVUZhfPH/Wzm8LC2UycyRfzmtgOx7uuACsBnhApKxre3w89e7Pn/M0FTM7n58yfV7KvJRpmob/JoZHOxzmgfeJyDS17jFsozW7FUjubn2vv0HD9yezjGw10iR4FExMLDpCHwkTM+uvheYR/oAEIomUeMBaw/6IqKjUqplT7957HwbRmw+8GunEMs9VWUSImbVOpRRmrlpKkQE3bts2oMR5nj/77Iv3T49v375170QZ8QHhysxhnbczcEtFbh84M5n1eoqG+XpmivM1tRi5gQyAeFSapZRa63jNpU7R2/DxlrGuTwDXaWbW4zK9e/9+4ACqGs5vH5/C4ZkskpmWocRQSWTv/ni6RErr6YZwB7rb5m2bp+K9rbTe3x/nurgTHG55Oj2FZmutbVvAPXqtOpVap3Kcl/A+Gjl9s1GvRUTKZCCIQsvW/Xx+bN0i4MFrt948kO4OkvH1D7O4ra9ePrx98/rTl0dmVhYzO50uX3/7bUCOxy6q3bbz+WLWwgdWY5fLZZr29MzdQamizDwwvUx37621de3u3nuPMBEhlZEZM7eIGPmoE2e2BCIyI1R4T845WZgZmSkqHxckI710dyVOIAnyUQGVt6zRIkaVMj4lERJxS1VvJb9qjygRQYlxiffnpwfAjKno4bCIiCgpY64aEfMXn7797v033/zqsq0iBRgYIYYP2yuw2LPMj+ut7x0nZGbg2pIdX9LdtewZAjMXoakUESFKVR0JaF7P+lzLsM5ShSxLKSLFLS7bqZQJYQwiICNBEd2lTBnuEcysopTjCCWBe/fWm4iWWb1tl9N5Unl2//Ds4W67nDLjMC9E5K0z8zwfAL67e7hcLoG71tqbt9+d17M+PKRw7+Yemvx0aZdtVdWtt9N6Ia69ea01srfWu2UmWeTbx/elTJHmARGO8La1eZ7vD/Kj3/mD+7tFOZaZKSPBrMVju2zNLLZme/prGzO3bQcE3P10Oo3gCcoRnSKiNRsXtrV2Pp+BkjtYMW41KXEolyzDwkZIrilCA9RnIgINFAXKTDxC3yinEplMTDtYD73e8iBCXguxj3qYtHf+6NpDB6WPjj1H7BDjSD3HXY+w1lrCC0uttVn0rSGylBLWrNHIKUUEHo9P73pzIurd13XNzFtf4PruO+oOAGAgApmBQHIyPiBTCKIkMPYMZti0qpYiqioivAMC+CiXt8vlgvBlmUY7PjOrqFQ1jpHmPtwtvdvdYX7/7qlZTpXNm7AAmeSUQCQAFqJw620qxb1Roha6O9w/f7hnyu1yevn8+fGuRs/z+UmvnulweB7bZa55OB61Pr+/K++fziQ77HpeW2aezmt3m4+HrXvrGd6Y2czNzhF7WOutCaivlxFUI3pRvX9++OSTT/7gJ58o49277d3TWyZ89smrp6fvluWYxMwK7qNKLqUMpzWu2zC4bdvGRRjlIHj4hRs3gzJpeJPee+995Jbj2irLeEYA43Zf07pxF4yIRAdShFrr+M34N3ePcAIRRPfqijgzQUl705IGagiABUQM5qAEkveClER2TgmSicijTdOkqplEicEeGQ5sfPOnp6ciNHzVtm0jTHNiBIi85s7DtG7u8+op2T3Ao6WRmTvKdT1FuOXNsrfgdpsupSy1lFLGRWfmUTYRkUhZlkWZxweYl9p7r7W6ezcTKXd3U+99vVyEhNJVkpI6kpksHJyBvdyOCKGRCJOSTZM8u7+blzrXcn9UJC4ThFwoIa5lRw+uVZ3MpODI6MtSN2uX8wpo61tvHaxFNTPTMh1zncLJ3Ue3LwLWOgCmyMyMfjguVbg1fPLqxaeffnp/P63n9v79+8d336UHEX335u00TUTvRYqFE1FSRIQHMhORrBIRHpEUAcrYQWIANauqEqF3b8221dwwEuJbNL/F4YFGXbPQ0XZzBjyjlJJQeMy1LIdpuA/3PugTw2uMFJaI9BouB/jEQDDt7pBotO0pM93t6skyMyWdmQlCQCLdnVndvfVNCIfDYT7M45Wfzm33xxF5dWze97dflqMnRcDCB5Y+TZNZjP5vbzZcKQCt5ZZ0fpR/piOZk3PPOEmIP7pMI3Uel2AUiaVM4yBXLbXWqZTWttYaCwBM0wQA523Qsdy9KgF5uZxmLeU4JzFLebqsb777TlTmeSos27ZZs7mUeZqWBZlZJZW7UkZAkEslVen9EhHCkeGiMmlprbmh1ALAM7joPM9u4Z65WRGttZY6m5ll6JYisq7RPMxdRbiwmbHQ4XDHyMNxLkV62yKsVjq9/Rr9+HR5fHx8FHARbW3cxE1Vt34e3nHPMrtd87rBpLk2YpCJ5ORt68yDTybutq5t27rZtQs/cs0bPJ4fautr2ZoizMwzkYj0vjlhXur9/f00FVU1M1VhlvTYDtsAs0aI39/jFtOJk3YOUYKGA/teITX+MhvTR8yokYIQ0TxPr169ev5wNz7x6+8e379/b9YGr2DbHo/LYVmWWmsgmdvWO7N2N3evdWZGKTTPs5n1ZuODeu4OcqQ112DhmTsXau9HfUAh9pjSe/dahum7R5i3bOOqtdbevHlzmOdpqqNgoivmLCIOjNgX3jPpME8x2ogZp6f3ZnF3XNa2pfe6TMtcvfkO/eaqQoQIh1RWDoRrkYSVSpnsnmbetlNrkpnTfATgyG7erJdan7+c+malTN58W1tg06J38+H+OKvqpXFEnM9nM5umKiLuDgTSvTcFC3nY5uhAZOi6nUsptrVz75ScSarU+jYySzMbFJ9hVemewjdTu8YlCkrK/RYPx4RkJDOpCGUa88dtPyLCrTF5Ra932GSaptPp8enJ53k+HJZBGDoeD+5+BSbHzfK9c4lreZSZSkQARYzcj4lGNxy4UZOoMxErYc9iI4KJ5qkwctu2UuUw1yKUGap6Pp+qQpXX1XJHLskuW/VcSKZSt34xi8xUpApTdOaKhDCVqc4iXvWW4w6wbYueFKQEaKYHkokjnImFxRHCiXB3L1VFBGAbhJ+kSMi1n2bhl8ul9z7btMRSmEopKaRSg2ISyaQmDKbW2rl1InIfOJ8v81xKsSmJiONERMtcKZFpADOYKIuWZDldOlEuSyVS+K2tIelXiJc6M0+1FuLRkItAj67sLS5J3UM5SjqqVu/tqBIR5Ui9I2JFRCJGPGXmbjQ6WG6RmW9fv0MiwtJHz88BHq0K8xxZlpnntXEYCex9ogzfM71RIBLl5XIB4uXLl69evnr/Xp+entZVwuV8Pp/P50uaYa+h53leloWIRhNheLRPPvmEiJalAg9ff/3t5XLpbTOXQmzN4tqp3rpZj3FApMx6K9iJKEYGig9gOLBTVL8H9HtYZkQwU2aSUKY44ng8Hu+WQdZ69+4dEW3blqR+fexcjUbbtp1Op1JKa61vOx9WKJkZnjcom4uO6DwM9OHhYV3X169fb9t2PMzLsgzX31o7n9bhAi3H2eVprtM01VpF9sYYKEbSLLcMhna2UWttOiy11vGZx2UdCeJeCGYuh8NovA3PatYAiJJSAUCUw6HfAtGt7B1Um2maaFBkiAD2NB9Z2jpqxD4yv4FmAHCPAEgKi1hGXDbmDmBEjA90CCDSd0jOPYIyfTTDb4DGHm8SETuB46Ogd731EXltXGfQ+CO6sm9HDkyIyyUeHx+f3d9N0zKu0tYx06HWOi91sPKWZTkej3d3x9PpNCLS9fXDPd6/X1++fPnDH/7W09N5EKYwuHsjYfPovY9ejbuTu8ZH1CQaDhckHzFQRio6fMOern7IAnF7UVUupQDYtm3bNrNOke5OUi/buvXmvhNUAznqKhXZP5X7oKkTkYjL/vsUEYjsx4BIVe/v70VpXVdVXZZlmqbj8Xi5XN69e7dtm8XetVetRLRzL4QxeFZXwsCvPTKTIkfxpCqDZ7RtW2tNlQ6HQybWdT0ej35tukZY7z7oVE4JgEG3JIeZS5HR/RrJtLu65x7yrnXhgNK67e2TyAS4VhcRi5yWha+I2ICczQ053idu1vnBWClutzIiRkl0wysiwvP6Vx96rv6h2MiM3NvXN0slkvEB9pQ0zJ3P5/Pa2/l8Hm3PTCbKMhctd5l5OCzH43GA+WY2z3NEDDhyYAKiOVpT0zQ9PDy8f//+6emJE6M+cXL33s3cBt+3qO/UhKSPHkkk+JB03vLMzKRre2n3FhTDTFX1eH9XSrG+PT09jRscESyttWYW7j5Mc8SazOxXxnt4DgqzR6QZrj0hXIdPgkJJz+ezKInI3d2dqt7d3d3fH92dKEfLoLsPFAmAqoyDS5TJSrR30pjqx2dvfEHLMItt24BJRMbcSK1VVZ3Gq6kI29ooHeGIqMruOfxrLYVvZAdGjEmQcAAe6ZHRzbqrqhYDkOY3F3jlUg0qjK/rSjI+LQ2S436EIASKjMy914UrhO6D0b03//Ykwa7tN5XR2Eu/mu/1MXK2vMEsu/v8HjvOPip0RreCa523tQ3vOEAuIiql8FwLy/39/TzPRPn4+EiUtRZ3F+Fxm5hJhEUpM4e9EmWtCgTzaNDrbDUzMy0yiVIjMBwo3Sj8I/gx50c0pY8sdbRNR4hn4n2G4cWLT0bIdg8tUxK6W5ht1sbxHXcz00YyDBBFcGKAbiKcgLkjPZCBHCdvXLtx9TsQm1HmaFGatdMJ27aNy3dLVTOz9+GWcpy18TM+kKqukeGa14+vMy76PM/zPIsw88S8w5zzVDNAlAMvjDAiBpKJRlBj5st59d5TuLe2bds0LaUU1eqeZpaEwZXZryTluLciRXXHGjfr29oQXaQABnCmue3/SsSAR0Zm2vcBy6S9ncHMTIqkiGtHMDMi9zmB6yhEfkS+zB0S3h25XXm31/uemqCdQsDTfJjn+Xw+b90CKaPJBAyC0nDevW+jjCuqhAjvva2ttWH5laqItL6OSn/kchFh3iyCGcsyieq2tW3bmAfGlpk5DuT40KBMliCiAAj0a2Z6820jXgRFd/vlr766OxynaZrnWko5n89ts7j2gSIirmVBEK5vB8vMSIpxrTMiCeSeo5nWPah1xh7frwHLRGRt7d2jElFVPh6PIqJ151gxc63UWmPe8zmLYHwU8pywj3xoYaplmuY6lzoC2ciK5DqgNy6rCPUIaz0zGTnVoqruGmGqejgcBASPzrzahwlEAASRfQzj2j7JcbD3PFXoOjsmzB4RYRbMQawiQqxJ3W/ZIXKwzUDxIWMMjP8LAhxJ/RrWExitNnwY+EmMZvJAYJjHpEzc7HukjPTROFvPVOZadeDcZrZtGwBh7R6qmthpOlyUaM/pQVHKB864u4+zPXKbiJjqnvSPK1O4xJ72eFEWWYa70R5+q4duYZ2IOPa/HC+hdIu5I5DxrQhwC7deCt72Ry2neZ5LKRFg1nGYdmzIb1R8IiDCk2kMogjxOPFEQvAP2foV0mJmEUMkECLC7L33UkopZduQJKUUdx/E6sHfI+IbdDwQKSA9nQjiREYsFhFQUVUklVLmee59W9fV3cfJnqaJ0jKzdzSPdbtExDRNpZT7++P4apyYixLBfa412uPj4B6MJMwtb65u3EQSHkfuRosMgrn5Fq0197RweAJe63y9G9fCi9S95SCseyRhFNwRMTLLQAA7V39cwg+c2WusuEVzutZGHyWdFPsE3AeGOHPGKNoixhVGeilFZDcAfJ9O3lobLItpmkY2vyeIEZnZrM/TgYlHEjhNEzMPM7j1UwASlloBkIaPEwYiMgAUgxsfwKBegJyIkndQV8G5z6QMNzMo+zmaQ5kavs4zIknL1LrTSH9sB3Gvoyw7jxwJAjxiVMdEFGEAdiP7iKfiFgBYKBASGAZtFoNqMIBPZh4jfiP9uOLGpKoROYhK45mFBZCOPg7PaMr/5g++qHVug2QUsW0bM2thd9+2LiKTFnc/zosIFRYSCd1nUcx8pCI3NlZeOTS38TFmqCp9lPtmJpiR2VpbL5u7B24px/DllFdiYTiYc0S56zg3QXhQxyxtZJYRjtELHtkpsJvw98HsMdaQQCI+LuGFy4inwfTh+UTruu6YQeY8FTOrtQ7uyEA3iSEi7tn77sLdB2WJI+C+93oy6Hw+j+yGiEqZzBoze/QyTfM8i0hrNkjlEfG9Kn586diPbVoGY6TsN7b84Brw7dB8VCcmEH7tFA/fIKJKuDscX79+7XDVsp+nJBAiI5PGaEQGjXGfgbh7pmPvITFlhF/poWB2pxCRyAyEpmybiwwHPErsvb96fbjZON97p8Q9ja3wgJFH7YSI+Osvf3FcZmbO3D3ouq4HmUWEpZvZ8bhEhCofDnMmurXcZ+hsYEmjINjf1X3dzr0NKtpog2FUPSzMzKCduTxuwfC7W7fwgZSQuw9u5PBtdRJ3F9YIc+eBe4+/ev/+fe+D5MW99949OEDD7ijCiUhJ/Do56e5jejY+mtzf/Zz3W34ybn0EWaYIt9ZGUdSaDapaKQXXAfF0RHQlHtjcSEPpSn6/HdrYk41kTiIMPgoQpZTWTCRviLuIEoX6aHJey7v8MKnDY+Ltw1lHCoY9BQBiplFDxM6f/zjhGDmAiByPxx/84Ae/+Zu/+e7du6enp6enp9Eob1u3GLFm4CUYhVF+f6QJY4KYkHuIRIwhgWtLa7j2kAzJK1AlyAz3Ya8fNzlGYMXOdQggiDxCMDiLZr33qSgzT9MuDjCSJ1VlwrAGInJPkf0FI+Jy2c7nM8ClFGGBJJNgolJK732IRETEoB6kk+WYKc3bbbvlfJrI/bKMw4/b1djZC5S1zrXWUooyE9E01Ye740jv1nXbhxz3N53Oq9VaR+Y3RgpvYf3Wlrtd85HiM99YOIOzt39SomRWAtwHujTMF+u6nk4nAHOt0zSNAWKzKKIskhEEYdoTEb9Z2rWJPf7LMjR0IJXMCmJOc88daaOPpA2AIXSQDCYGgUcJfIUwPswvX31nAAiPq12PeQUwSFkoPaxVlVcvnz+7P757e+i9u+fj+dS23lrrHuPbcyIj88MnuVkqiODwq0AImDHiHXOGElGwh3JcT0Xe4A8iGkgKmHhMYXlEYoyAZaZgz8NqkW1Ld19FSlXLOQgFJTcnIuEiY4RDmAlmYb479XVdL5dL7y4C99DpJinBXKiIunvvfDvnA6+55f0fH0hmVkbqqM0/zBSMum0cNNXq3iOIWVl52OjIKwZ9uu6FL7Ztu1wupIWZL5fL5bzt/gwpUm654ziWYwiCgEwwkISrUeLGdS9aaymqPNyQmVFkXIstBrZrrjuca50nAFtvAXhmtyu3d6CH2KlMzMzCtc6qu4FGhHlkklkoAEQOP7qfGyQlj5Lzli/u/dRMXHkAGOlLBgi7XexjUFd+u5AoT5M+Pb03s3k+TNM0zYV4COwsaylyEbna6KgnblAIvv8YX51zb7p+OB65u0lnH5zAncv3UTuHhG8P2usSG7ynoCEkMWqvQeazG/3R3V0IwDwdRig3w1TrCLvM4uHWo/c+LoRFG3MHopo5eOMkoqXIuKm990jwtUM3kqocA5GZmWkW4RmZqsostxScriDa1loplWA7Aqo8QZk5wsx7twZAtYpIKTpN9XwZv+GB5tyc8dbauBw3Rz4eI/O+nQrZHwRgsLz367C1kRw7vFz5wcxMO3ljHGa5oau3CDBQgnHNmZl5Vysy67fY65Hbtk+9K4P2MvAa3IOQmcI8hpsjQq68zByUPAZoyPTsqh5j6Me79d4zQ1RFZJrqPM/zrETUWnv37rsrhW/cXVblOhUAsG5GIy2mD0z4X7fRD5Z6LYMyc+AMt3+8FrA3DIWYmfKDdZKwphBRUIzeWVzT6Jxr0T0RNxkNOkUZ7nPwxl1EahmnVlpvvfkICGtvwipSRql3cyrXfI6SqWcEgZJYWK7DiSPUsnv3Wzro4xINDEREpmlkm8XNcpqHExndTQV3D5EA0M167yKSMjJdTFLA3FpbaEeI6ArY7foRmSIyWhK3ZHGATeMJtI8W8ijJx8cVyqrz+MPWds4orlnZIHlcvezO+diDx0ck9xs/Ka4g1+2ueeTlcslM6/GhFx+RRFCifVR5P9tDR4dGT5yIrG/7UB1yENSnUue5mpnJjp2NpG1kbthRTDHrEYO8Z2bmnp40BEE4GJyDHD94yBF5pVRfv8146dxt9JbI57W5dw0AdHv+h6d9JDCBpORgEMtt+jTp2g4wI0K4S16bq8hhK32kzsxsnmbtVhVt22YZRWuts4jMlVSFB+mbx8DMB4Bm9K9VVRgEZWLSW8y5NRcoEwQIM++c6yKDIRXGcrdu53XdijAoxRPmmYlwEVkOh8ysw5rdt22rtfTeiaMyX4PPPlOaO+RkQLjv6UQRIYQyqEgEeY7KnXrvo4U1jGlY5zRN8zyNOz6I6uMyjkJ+vMto8HommCli1xSJ271LdzfvfB3ZjUxgnwtf102vY855/efhmQKZInSTYWLmaZpqkdboViMzcxEtRUWkVIkos1cAhWUcCGtr1SUzB5o9oJ9xNIeSTgRdj9atUGBk3D7u8CW3pAL/PW5195/0IdG7lQIkTP7BoFkIA+gbU30UCFigjEFeQAuPaUlmrrWK7t5uhIvM0V5f13XFdV6x1rrMh8E0ncqeqUcEj6EwdzCBggUFPBKO8Y2Y2bwPDOHWycgrOW04hTG5Px7u/e3jO3ffhwislyJT0XmeurUxvZ+ZWnjUZxY9HKp7Wc3Idu2yjmAK3HD7K7vF7JpPEY3T6h5GdgVNb59ciJWl2brPzTq1bRtyRWbt/ft1aBZmZrP99ZkZPNwQAeAbMx2jWZ0egwy1HA5zJrXNtLV+80OciKHXQIOIsCMjQLgrAcx8PB5v0BIRyZijSEPwcDlKPNQSh+lv7TJNkyjBkbEbumppbtYj0MkcDAaL0ACIiSj8Q5TP5JtHBH3IgHH1mbfQD4CuqMctFf74abciD0Bi7/QQYbTCiagUPd4th8OBMkY7XoRYVbCTSGz3IqaqY/RPp3me56J16JyYtdswwvVThZAIgUUHW2wfGYAh9KOvuWcXA9B176McHGM5ETbccGBJJk4lMGscj/dVwYxCxbwreFkWFgzxhYgYHIDxt0REGR7GHx0SADnoT24D3B6GOIC2Wx6Cj5KuzPAOL25mTJlhGTbC1wA6bAQWM/lIbmO8necNN9iZgaDMq0ON7BExTXuvJ9K02/f+niIlU5QkkZ4AFSEiCnezpspTmcYRZOYxYr1/6MiBCCVljOgvBJF3WSJZEQoRZiQ7JVdL6+DtTrQ9hp2yh5yzpeRFTxFmtV1Keyvbo7ZNrHNv2jftIS4Z4qleCQqpB79EjZhcc77fnt1fXhzaS7Xnh/bs06fDZ+/4WbR8uDzeXU7FoPLZhVpuMQXd11A1RrIIVMHPynG+L9MBOuXbd+dDeSBnnb5liIgy1SEHNYhad/OM7SwZB+IDhFtmCCdSV4voHKbUECZpDDiiu2apVJFqFj0zSUCidhnDqIiMvqW3BBvh7v7Z2rsnBUv3uLS+rmtr7UHeuaeqPtwfphnEXzJN2e+3c408bOgWjfVyvjxezmadtZiF9/A+1D5YiCU38IpJZjhI5NzWp34xmJGDZkqXDOFGFPcPRyLdoiWTag0zAglz7zYKKc3+4uXL588fWl8/eVhUJZBjns4iT6fL0+MZ6QQuLJkUQtQN6bv01cD9EhawSCLRWi+bu+e6rklFh87HzUAZFLyPn4uSiNaqIlxUh/s5n89xnYLPzFJKkR0vvLm2m4sjogctkRZuRLLMi0p1dydbKZ68O+Xdp8fykrdusxVQevesZGrf+ZN4PkjtGWtvaWjYtuzGSCaBaBYOmvoLLi4ac9b79f5+ez7FMUW7P1k9XV5xvSe6d6r8gu+OfD9326IFd1RQ5WQhLVqmwzIpwbM9Pq3eNwUfi07wYiKiEsWC1nXr1ksp81JZaVoqRYry6PsQuYOQbXSF3a2HBYswgwiSMGvunGqDzEFOLARxNyApaZqWmY9E5BlmUaV2y/eXx5HuZqLqZL1/8cUPDocKMeIWaeEE7sSR0Yiz25q+7k3tIs0GU4YzLYOEmYUhWK1NJZnF3EqRyrqdL8HZ7Kkqs4gU2daN6NCsqWog03tmUBIoapG740FVnx2fHY/Ltm3TXEdCNamWUpbj8enpRCS9OVhqmd1DVcmbEfWWZrbz+EFDdDQzLNKaR/Te9kJFvdstCBIRBEiWYC7lJohIRHTVU/YBNF11hCPCWTMAprx282+wBRHBV6akBBfGQKfdlTFJ9WV5stWkuZMuXEMoMeedFbTiz7EEfcrMl7W/ff/U195736JdKLoKiNV1CpF8rt2r2dz5fl3u2qHwZHPEHRmf9WCXZ9lrIOTF9mzpefjByyOCOEgCiCAGCYm6d1UBwbatte1QDneTVq0L32VkBlfmZVbLCEQSKLyWwkhGWKxCIqzI9KQkToJb9BZOwSpMqUJah4JHl0wYMUudlC+cWYDAmF4LcwyxT0/itXV3r2Xmukvs/uiHv6+FAlvCEyIkyDidTyESaQRYb+5u4d2iKK/dBhNwgP8DE0yJ6TCdzk9TXURJVI48QftpvUhlSv/Dv/X7z+7v/91Pf+ru01Q366N7OtdprsVaJ8Tdod7f398dlt1tpYFIfK8Z1raTQrqbmanslSIJhNmZOXfCiwUSbBndd4fau2/rGEdjZdy0mpKJeUz8EzElEQYJ6CMYEWPObi9oiALZ3cj2UtTpw7zleE6xIE533wY9kVRZp2kq4Ew8ndduvadNKFOqOG2U1iMtCkEjROSg8/FFhcC8W+vnvp17t87MWkKOzz8pGlWzGs0nnc9ajEIcR95kylkvxbraLIcX9eGT891jTx7Z+yi6gHRKR5EZ5gnJ0AQ58an5aX1q7HtDESBAQAwKH6MV+0xBIJIyJQH0KGAywEGJ8AgEQNEpVDw5HQYwC4FZKw76EGHp4dEjUqSqKglnkEVu29YsOJGZxKIso3MDaapwHzWlMMXaTu7dvUdwQtzIOsJ7D6ckIkLytfeRAHQuvXeaiISCrBR+sTzICef19MUXny1H7nb+ye//De/2y6++9vceiipyfzgclgnhlH5YylRpoBMkOF8utVZ3Op8um/VMmqbJPAf6McRatm2jJGTuIHuQp3h4EMzhQUlM4B7S0wJKRLrTlnbtrhAW4cEiCGUtpUxTrWVQMYDYBXFu4NTgN95k9EYulZmDJwrAiUZnvQhNRbQW1TrVKaHt0iRQgSKFLtAVtcvjXU1YUVQhYvPsLaNp9JrBJoc4ON35kqFJlbgavU3FRaKF2Fzkrh5irlS2tR/Ks+V4Rw9TmxLJ9xeeGi4toBihgATEykKcjPCdpEXZwr079a3UuaUdpMySGcYRk6gyJSml24A4iIMzKTplEpyLWU8ISFhnic4EFSE2FvJo4ChFhTScM8z8kYi0UqHM5EiLaL4FmDJpXvROdlJsWPR+eTzF8xd3LGVd197QG9wykk6XpiV7N9EFWa0bkzazFBmeU1kowSP4cbrbdFc9rFkvRZGkBGL/5NO7T1+9IIqEPz6e57rUIsJg8N3huMw106dJlzJVJWurQzz8Bs4ky+hHuGXrvq6ttdY2c7uo9u5eQgCE+RWt1iEKHpzdzDKQFhGeAgIx62g93x77MB4zM0Y3opYyGgnpEYMhSkQZH0SyryMB4+cdEPC9PlyZFDRpWaZaq/JEwXHJVlhU6qEu5/MZnrxqPdWylfs76nlp3p5kc21UkpgDkYmWCPKqXDUlo/HWqSmtRtkB5OQlF1KnJCEuk1tpJxErJhGwczg7D8kFBFEgKBObQBKkDO8dgnmp94dZ6pSihkyWC1vzzhETk4oA4d2JyD2cwMrObDRyqUw8NTdAFBMlyIIoKVm0UJKmAsamlIUtIrDliZkjhYlGlSnCrCIi1iNpR++LMjOFGTi6AYbew7ogq7D0nhlFmF2IUCK4GVQlI6lkuPPesMCVnZ4x3mti5SmsmZuWOlf5jU8/Vcbl/DSi4ABhHh7uendCjLm5ebobLaV0gwiA12+/c89+vjBzLTORbH09v3ufmb05mMD0dD4RkTOGXh1DYvBUk1u35rGZeWDvRTGNOVJ9uDsMo/zYQJk5wmmHxyOCEH61PxBzYuAExLeVCh9hPYj0nWDGOc3MNBSheluZC1VNJDtEyrws/l1kooKICmPyxze6EBa2mc8CUC5Bs81CcuFoHEEWFEbRYY289lK5FJooyuScG538csIFZUGmdhYjFCEKBZwQExMnK0QzCJkBmAoBWO50nucyT6VMxNoGA5JYiUVI4EA2dop0yb6aj/0HoCQKYjCIOPAYGYUXJQ8bFHiKHp4ZqkQMr5sTIiUKklp9UiUCQyT3ibYxnhE61crqcPPNPZSpTAeXS+sX4Vp08o5IH73Vw3LUkmMWr4UlzBNJcDhxEu1Q0uiyAgBjmqdnz++Z8eb1NxX8/NlDKWWZlnVdL5ft6enp2cNzZRCQHnOdTqcTmO7uDsfj/TQXCo+kS9siYoDp1sPTuoGILuft8f3JM5j5cDhcpy08zSNJtYDgSNo5R7CblgwCSAYREyJ1j9/DSQI8hkwjrwd64FUfxL0jbB+O3hODDzs0rmznkXbBwyPJG5h1SIdhYKSs2Skyt752iVZc5rKmZ+upojN1NjOHR6UCUPEybTqnLsSr9E298wb2ReKYtFxeDJZ6JhWHZCSxU3R3FfJM6p2VKySQPXIjEaRmcoDRQaHKOjGY5+MkXCIiuk1SqCGCIgzmKQhF7pPHaQ53ApSdKSiCQCyj+VZKIZl1rphCs/fuIPckEt+CWWWk9wkBCNzp3ppbS2YM/xAEs0iAjR+9J5whlC6C47JME2XQuppbJxQgPNZaoFUiG5Cbn90BQY5WFpyJBJRwchAxUSYRFz0+O+pUT6fHaZpevXhemBmwNd+9OX337lGKmgUr3n331lpzSy76+Q++eHj2jBinS5OMNO9ErTVRPT+egCFJnkQgkTrPp9OJmVTLNM2l1NPp5La1zWIMmHpUraVMQeyggSiZWboPnX8waVEmAiE+CtbjSg+HGYiMjykgY/4EsePcmUnk4WPGd8Dpo6c3Gr4aMRWI8NbbUmvfejVS1MHNPdvaZnS/zPNSLZ38SUKqUIF4HEAgQXBkIvlBp5p08u3sZaplWiFbqziMCcZObmQuaQNi5LI1WzhkohQkxDYjVrtAVHliwBMBTrP+4rNPbaQJiXQG1cu5SahSIeo5OmsBKsxcAoFIIYUzoJzgSMZ1Ai7vHXTZqCPYEyHIMe1BKcUyLZM5hMnT0y1yIaHMcCCSzIZfoUh4twgerQ+3FdCU+endWbio3g1BF1UhTi25WYR7pJlZEjNzwOjq4UHBeVXoyr2AJaIxmLAsR0qcz5ttLbfy9punb777rix1W7swX07nScv9/f1ydyTWx/NFkEyUHra11beRIid4zLcRJQFMqlLnKSOibaY1lmU53t+vT09v3749XS4EKXVi5kxf5sqF3717LATmcLJ5mj979UnVclMI+jCDMsTcIo2ZBxn72lORhHMiIp4/f6Gqb9++HTOWo0tx4/iUSTBBRHrv3k1BYV4mdSTAl8tGrRHxKmYawN5ajLSwmOWQmsErY0s0MCULaTXDJdw8QMyks/Fnsry4m956t83v7+/5bvluPb1bHy89QJyRpSK8R1Ja9JyOPCW0rsaS6cGLcK1U2WA9ViAG2RE0+Jo1mLdEaCGioIhwalACHMWFIUhFgJMpsQu0UG4GECjCELzPC6QBKdzMDMmMiB6+VZV5Lr4N0vve5PQxYsSUQRa9twbkVESZ5qngQugHJ6KQcEdqECEMiERmjBEajRiShUEchGQEginGtCONKj7cT08Xd2fEU0ISGHSVc72cjWneVtv6++cP98ty5AyA162v/X0LByDEkgjzlMHG39u01+YcBaUlHGThaR3nNQKllFK0zrNFjBlxbytEKXpYMKy1TZjLpMdZyTsx9Iqp7005Bo01OCPuD/OttQ6RJzNLsyFGoqqffvrpjfkymtS3kSu56mMxa2Y069NSrQeYtu7pmZ6dItSnVgRSVp3PWlbB1xZ32R8Sc4ZYZGMny8kx53TnhTePtHg4PHy6PL9zTfl6qc+ty6XZi7vnrOX87VdRadOtEfUND/xQo/jqG6L39kokOWmoCGRHWErzWJnhYUhnnjwZwp7hnj2EGRQsyYSgJIZwMCdRJift0M3OEyclUDghhK7byJgT7IlO3Mzbtrl3kBf3k3eczkPSD4PIM6RcmJqZqiYSYdm3uZYAOVrl++EmI2IEuqQhqy1mazghZmQGgoDkUGIkPuhQO2VmEso8revWe59LrSzp2Zuv5ws1nqaji1zswkp1XnRY8+lkwOaxuQFcRSdRBhn1KyJ+s6IPrekrn6avazufz4P4ZmZgFUp3t+7RuyOnaTpMBd6ZMVeZlaJfHKYjVPOty3oF2G8d2BuH0q/zO/M8D5LV/f29mb1//35wT25iBKM8GisBmrEiq4KIOoyKxlic55EWvNJyUjGtbZ4vU+1a8o2lbDJTsmnJvopHkbJJnvhkDC3lSPKslGVZVOrny2dp03qmQGSPz5eH+bP489d/bTWaIJyX3mdwnQ7MxYq17kBKAg7xYM2RpO+bVWCK2vvKFKMJF32WwZohRiIxxgkwqNVMV995fah3AjJtkB5p76ulmyPYLawnc5mmmSXd+2bvD4fD3eFeRFrv5/N6aRslsYqnIbMWHJfj3WEWjnSvs0WO1UoULuGE3LUYrMN9OFEm9GRjdjjv3A8wiMfsMZJPTxtR1lqZtTeLblWmqTLRtEZr1lv6XCaw9N7Is5l3j0u3zS2JW7Y1mYmcr4PUHyrlD/ruREODMQf5i4i2zgCUd9UCEVLiQArlNBWhaVBww/oYlN5JrEkkYCDzo0VK+yICIkrfR8kcDBr8EjP/2HHuagJX4x7UCmbeQnUIUJ3Ons5ToVIGXsoBaSKPRbbKa4HVDPUi8Uhx4qQKFMnDUljukp+ll4vnpUQ7RJ1hnvame31v69aNpnpYrD0dJX/r2fGbRz37BaylHLFWGGnVi9saq0phkAaVCMXM3ohjO3tWItaAGsF7TgUTF2V4FBqUwxz1H0BpSFLklXOcORYDAMAcCgRDkQGMmTgS0FSEevYr0QwebmbeIrP1zts2TRNpkeoS6e7CbDYWhGhVLaKTMDSKnAAQaQZb57Z5RCSl+74nCMTEyZTBt6lkFvAY4gcjxyx5pHvn5LAkcKR0z8en012lx8u7LTtVatm3aIWoW8ukcMTokwU2s7HNzGUQL/cs8YY59rbdmOP7mHCCQNkDw0KZaIxREFGk9zYOzKD3j4UWa4R+oJoSEXgQS65pL26ZZdqeqi7Tzskd/fcBTmWSSAF2lGo8c5h+1cIUYOpuzGw9OT0zka7O4urG3rgbVjPK2AqXrmJy6CJRAPAEuFmcl+RJs3KvTOj5/nT+7ryW9ybz/bv+nfanZ3cyuXGb/8Ynv3n69hdvz+YgpsJaPXrva0i7gCWpQrIRcxSBRDGYQEIkkiJMaSr1wKJhXfeOBMWVaJUEcI7aPYFdagKcYABKFYh9BnikfNcsgCtmyLgjtQqRmnGvtbX29umkl7XUecStTFhPAs11XqaKSF+NDnWuc9I7glpPOIVFZAeNtXeDle87l5yGlqpIll1sliSTfE8Wwz2Q5J7mnu4IgLF1W9evtnSrCHAYzk2rAz3S9xx1mAcDoKSA2xUO58y8kcfzFnJ38+RdkjwZPPKZUjgpEd7MzEgJDTzvhZBFUkaY6eh27m8xNmiBAIgIX9lf9NEw/NqH4ImOwDa40lqruzORDC2QAJBmDkAK+VhwaS5c4AH00Sc0EKw8Rg9KiO1LZa3OkLvkA6hQBtgccVZsk7yDTIfpWbEFv7Lm1DLT4W7xmB5bXpKcVM8y1+Vvf/G3/+Lxl28f3296Zic0poiDlPArWJ8ZTi5UtIBFmDNgHhZEMz3GSh4edsdHhtBwPgRKcABMFsmSzoiEEWzIdoCHXMxwG5Y+bJMzBsbsmt4R4dEFwLZZj8sYOUygtTZYp4flznqPiCJl1qlQFAIsLRrPGcbtwmNCJrKz7sMkvo+hWeZQVKqZYHCAnDAasp4ZiQQxa1jfto0SFFjmmSCi1flcVQK+RQOThXHypLr1Ng7PmL8gd/LIjKQJI8JEAh8GHMZ060fp+c54FJU037XBiIl2ZRpLI6LePRORSGLrFpHq7kGkY+x9h973hlBegdO9/w5kpkqNiDGFfcs83F1VR2k02LwjExWR9B4Unp4JzkSHBNdJPFsXam5vrVk6SSHOTD9eXjT0c1wsL5USrEmVbSltntZjf2zrZu8P5z6tQm1G4O5sVnM5BpdLp3erVfRnz+6p0MQavL2PU+d2V+9flvs52TrTQGMZ81JYjMS7NXTqkUlKLEH97eUJJetcpHfmrFSIIEEcV+1rYMy8+kDnQEGS4AtflWBpDO8HJZhAGbF2RIBKcm4O94iUMYMWkZxg/qA0ISKFpZRClN6tpXOK1hqGdPGuyIk5iHugd9vcR5nCIEsEQQiFUjK3TGRyEueoKMZqv7CXr14dl0VZ3nzzbURYcxFJznM7dSVW7R6WwZH9tDLKTSIiIihCwilhhMHZGEnhCMNE1LvtMPlY9cssIgTp3ui6+E+Ibxk8BzNrsys7Odk8RVQlbZRJoCEokXGFnIjZ3VWn1tZRlTPGHSJmadblOlmWmWEGoHvPiCLkHolM75eSHCG+EYWR9iJsc7sQgy/18vXy/t//5utv+aSFj10fstL7RyEQ2HOCHw5W7jueb48HZ80iJNOKZ+daUIgXSPnsL+Xx2eX18+/Wz9en6fHyQl/3d59sTz9af+sn9PnzcvdX568p+xHt4E+fv/jkNZfwRrxp9aK5HIq1Vonu7p/3nh4qOoOLaUYENhSjUihlTRgJRxlpwLVPlqkeBcpA7mJIU1KGpHFsYYY0kKcgKYIFhOwKeJwrR2R3C4KrcmYIF++FqYSlkHv6+dw6S1GeJ4lFsCA6k3AgmLpHj9yanwJYPYzVk4OU2EGU2SjVyiPnxF0lqzuIteWa3O/uJ5HL5emxbZtHXGx935/OuLyuTneRWNHa4rWsquuB875lTzTxUMtwWUlPyo1xbE+Eve2dRJG7Eh6zRCaGWF5GhETs+61HGugRlo6rBIEIR4QQR4Sbu7swkK7TXCJivA7ig8ZdXDXeW1tvwvLzPI85cTBmnUdM8atQG+3rlK6qRgMEgFAakzjCMyk0wMYAxSp+gV28txqdI8SatfzkSw7UKGSL2DG3Y8l5My1RyIVDEcGRnMHaM/LNdCDSyYqsPRJBPSLe8fs/P395KHe9+ZElk0kmXZ5t0DKFmTMli2thVSo6Makwk3KigJUgAiRlMs21qALaiY10jM9mBNcyD4YEEYmM9Q4e7uQdwQ5wgIl87KhiIBDiShyUSowyK1NEcCFVZZHLZXNP1xzDnN6z1DrXcneoh6XOlSN771tOU5gyB4aOXzKcem+3laeJpJ33KCCSdsdjtznWwuCSrOZp58ullyWD3cTWdjlfclsXcr5vJz/13Mb+R4NpdrSYRA3iEaGQYEmZQD0Tpd4mUgZhiplzfLW8Nq7yw6a1IdeauzL2DhMN7bch9sYf1pkmwGOdgARF9Myr6BIzV9XW2lBrGdz9uU7WutZig+xDVMo05HUGk+WWavB1aAmANIbrLleHoaDqmeixDioaW0zCXbohSPi9fllSppy1LpyndTpu9Xgpx4c1555zj/tOEXkAahjD3hzKnfhkPF0kUHrNlWwVu9DjqVtZeYrCrHI8yP197wH+jtWFDeQsgqQhMBLWkfXqCPvYzcjJ5i0y0o3FBUgmYRYh8zUiWLxMXITcrW1I5MxgVo8cnfoxdCUimZYE5QiAMkAkUoLCrFFCguecIEYlRIa7LXfLYZ5UxIib7iw1OsMxViHuG4VHcTuWYRCQnBgNgqHnp3H0dKJG4oeFS3FEX5s1JzBch8DKJBALyu3S8y0QtVZJaZtvvB3vlns5MBXPaGbeu3dLp95cuvWxy5B3SdehoQcfOEGObdssmUnKQ+q1Z0RSkvJowxehqjoaFcOAxojpDtvtSqcgyJgpu2WgPs+1977PpiQAzMvUzfy66bD3rdZyFXrcc7H86IGr9BS4cFDNKJlAgCJ5jhIFsMu5UZ557dxI8EAHMiDYJLfanw6nx5rfHXPe8tC2+628XOdXF41Gx6TC4fnkAe+mK6mUsICOUenUwMvjMzFJkbvjXWTfok18YcZUiIhrYZATJMIolYnoqldIY+li+NgT4U7u6JYxTvkYnSUnaczpyWYFVhQLsI0LSAgZw8ERkkyJHOse0jMz6Dpwx1M4yDGxcGFQZw4h3TavlIXD+xZ+CQYFuVMjQ7ZITdJMDCiLaNffGIgYJSMHtkcp6h5BTNKPs6iQGmczzrRYLcKisNZ6d4Db2n2ywxSgJhysjuPhXupk48KCwULqkxYmMte2yWUbmMB+x+OK2Kvq8IvCPMSNa6211ozzcKU3AVcAQnQb2x9mE7gZ6Efk4t0FpicGTyKmaQpzovRw82SZM0JVbs7yuoB12rYNH23kvp2AqFsmRxRKrqGSQXDhiDBKecDze9X57miTr7V3Mn489K0/nddHXN4s7e1Ca5UuFNWSguA1aOkyWa0dEjj2lYNDpPeEzRbh0ij7Yvh8ufvk/rk7RcSEjtho8mka8UEGYzXNhXhQtASM/GizaSawwyigAgiNHhoRpxDfAB3P8HS4F0YxtBBOyQRBwRzuAaR1ABwhRBwUUpASFMleh0YGKL13R0f4UPs+rbFuiXQRnooQkacRS++WKWOBW1g0t7X1TL2GTqJMHqP4kV26kTl7Bj1t4RKTZzoWLp0Q6Z4e6VxUny1TxQ+lSlRfPVqWInfToWOo2lcecG92p0hK5+zitU5mFtHGenvsY8Wwvo2KG6xXoaEkwrzMYyhZWUTptoKbCCy7BUYEXceMd7Md1jlCA+iDJsd2WccuiNevX9/d3f3sZz/77d/+Tb5uG1LVf/+XP3v27Nn9w8PYL389AXkz1k5rYGg6DQqPCwVxMJGkFtQiOsehu52srdpe4jMP37b1fTu9Om/vJ5y1OpRDi/lkfOzluJW6FjFihGRNpShEyhHk6CZEREpxvxTORqwEr74dJsg8RHaYkZREnjGaQck05jxz107BVblE9GEcWidDtiRDsKNScEKIVUAySCNGCDc4YWBRlrCgcEpFclWAIoI4M7tUUUmn9CeGZERPikSqji05VidmSrO0nrH5RWkqxPsIMquIZ0YOreYc4MzN6xARQTiQAFkIALBnrGcLzhAtejRSa47IQum5wsGc5WC/7XcZ5YT2ZKubee3g0OSIDEhmxNjrRlDio5SWyOSyC1D6R4EaIz9UVRaAwqN3g3IODRlHUIwtFwFgxyYJN0GH3YMOrhhdtfluDjUiwm2apm+//VZEnj9//vr1N9NUnt6/+au/+vnnn3/+j//xP/4H/+AfvHjx4v5+EK2DeW8b5ECDCZmpxEECIskxPEbE14oPHNBknMM3t7N4Rz7Vcznw3Hjy5aXP7cwexTdVx74AiViTC3LiKImziiAZknBHmrf0jcQb/OxngSoTM4ISlGKWkuLw5jSAOhJOBiuRDMgyhh8FJxJc3HaJB1Yvk3NJZKSjNw6vATdEqVGUZAbiKnUmlCiZSYFBQUGakmYmZZi3/R4DK4zIgztLIJmUOJXBXDBNhVnbOc6nrUcPS4kgSSHUqlt294Awp3BEt/SR+u36ACMGJFsyWB2gEkEpaJVz0oS7ByErEcGJgiXc+xfl5dffvWv9snG3jJlkIs3uBHVCJ26sQUYOcahp516qaOHMElfpuD0b3YtmyvTcRXSFiIcO6NVXXg2Pbs4hiFhu08/MTATmm5BiRnhmCHOtdfTWv/zyrx8eHoZS3nbJWvmP//i/+of/8I9+9KMfXS6Xw6wIw1U1iplDGNjfIFCClFAZwkKMJAYEItwj1+jOcHKfKJW46pvpdJjpTlhK55WPWdNAknkJBBlHE5i2fQ9DAA+TWsycVt0nm8PrylIFjnWLu/vak8/pourZn+WseeGEeSfW0QumwealGMgYiDJlbwGRsISIELMU1klYsW699RW5iNSiUymQYm4ZYWGtiCcYISC28BzLJhHEEEmmZIDWVNAYnWXdWEIJpdTeO6iplmVZ3F21MCmWGJJaHlvvG5Fl7MI5IjRJdU5qHth145IToKAREBDuakQbJkxmxSW7elsc1ZRjYpMwRc5zLfMhIuxtf//0eIrNi27et/RKMzKJKShBzoiMdEdY9sjArku618cig8SeVzWXMTY84J1SSutrKWXsz4jAkIa8YkE7eDfkyYeh640L8iEVYAn3Ud2bmXvftu2rr756+fLlz372s3//F/+6tfajH/3o3/6b//b3f+8n4fpf/r//qar+4Ivf/MlPfqJa58OiXM3Mu8/z7DF2JYLJeWfVgJLcrYi65HldjQEnZtJOlyk3NXnI6ZhEvKYnsQRL1DRukZ2S1AujdFNLovPBqaaiyFTKw2ipBbOWQkdvkrU2DaeWwRy4Z7hbmZakIGYwLG1U2SACKfK66SQHs3dLcAaiuYeTZgTDhUmEMRb3Wo9oFI3Th8hCjr1MoPKhi8KJoLVvynRYDoO3MNXp7p7M7PR0uTyep2maZwV8fXqvMmVK84tZ0yqlFElWXTKBHMpK5tEtw8x8jNVy7q4IY7MRD6adO0855UU4ahZGNJe2LETU7xZldyHdtnSmnkVDn5w25VOsEHXWpy0ExIiEBwyRGeGJjjS42miDxUcmRLc2zQCXAOoWRDbPPFJBu4qSDhWlkRsMDjEx+yBGgc1t54PeOvKj7BrzH733w2EW+ezTTz/9i7/4iz/+4z+e54rIT168/Ff/8k9evXr1f/pP/9M//MM/fPvmTSn1N774wR//s3/2N3784x/96MfdV5ZSq7a2FlUCFMkUQ4eEWRjkKSHclQSchYlIQOLYemVeu1x6NRPbNDobEOQsviBrZApad+MibDJHYagSLzpJWYLVksNRWMEcgiamnBzkoHPylIIM39frgRIZLZBaF8QYJuiECVlocN7JIyNDRIryrBIdGZThnIF0B0cmwjl9opzWLVgG2zsC+zlXrmP7DEWZD/PheEh0z87MlOtUSr1/aFMza7CWmRw4r41p8+juTaec57nWmalGytb6um5m5hHNbWvNLEAy9gXIyKH3tSHEnRCUSY0oQD2yBcIjV3v5MKkizLQuF7fvHrf3F4un/gaXNseFO8wn9IWkonhGoId38ohITzIiwxhSG1ljDuHYYaXuPppOGG1zZvM8XzZe9gOLqwJe5r4knVhBZDYWjsE9tt6Hct+eOogIEzUbHiWZeV3XUap/8skn//Af/tGf//mf/6N/9I/+7M/+7Hd+53devXr19s2bf/2v/tWyLM++uP/lL3/585///O///b8vQkN+MrFnG5wYemxXeAVD1TIojKnX6BWUrD0jsnjN1BDOhKdkJFGATLmIG0DJGMjqwAGdD+bo3shksUrLhLkqc8kkWJdVAXWkk5N2ou/W5Ey1EIEKtIxum/fLE5EIV5FJOAmeyeFIlOvqIHIPIkrHUGjMJCMBlUymlCGpHSyejjSWIAILmBLD5TQb8kNP50fDJjUpcjJRkVKWq/xOikgUWD+7d4/u0W213lspK7MmND3M+tb6uq6b9T6myVlG24bARELJYWk9543JKcm7WheslGt0W7e7Us6tEBCrGLg7b+t6eXrq0bdpfS/bVl2k3EmrG3l3VUmygEUkO3NKBSlx7l3+XQzrAxYEIdBNADli/BOZDYLR3g/KpN7tctmG2hyGmHoiM1vv27bprYSPkQIAw+THFseIqLUej8dS5NWrl59//vkf/3/+6el0+vLLLw+Hw5s3b06n04sXL37xi18Q+G//3b/z9ddfP3v+vDXTWpAQEfCE8MAYcIRnEiETIgSCa/SMpgl4JOqgolNGElw1qHgJUuEuntQ5g0wzhSx5DYpAqhPlBI607pfSiDmzaCNPtAgPkFIlqkJFWI0exv4Gh8WYvhozGRQI92huZyZlFuGJiNzvmZVYkeg9ek8MvQcSgBiMrCAbbFGgD1B6NKxVGIOL78ZEhKilqtBlPTW0iQsRYi2ZG9E21lAXrczYti2ZzFpSik6eGhktkrL1FnNR0ZJbM7NwVtEU9QwmziQCcTI84YBFWrII1aTYlRRAWUUz9OkpfBbB1NZg4DhV2PTabVnmJ2pM8N6sX8K5UIneSMbMJZDMzkgGdp3KIXuJqzB8fqSPmZnpiUwShGXIdT79qr04yBtjdaJ7bNt24yEBvBvo7kdtl8EZHbzXr18/e/bM3b/55puf/ewvB7T0u7/7e9M0/+IXvwTo/dMpQA/PXwTo7/7dv3t//+wHv/EbqpVVenNmNLe0SkkMJ4YHkllyYGUjIjlTXPcTE0TIB06jSSKUyBjLbT28ELGSM3VRR2xCZa6JTQOb8xZ+crs3f7ai+ByThsxBTsydJVLYSTxdp4BFgDw6wjiEQsVr4SI0pNczPdMsGgAwgSamJVMzBv84VSizX3V+8kqkCRBosLpQMxO2a5QRQgDhqMJCTMns6hfatq2iRHZQJ4RwqVrccWkbK1+2s0V6svXwjBRLdM37szIhzNpgORI0nDIJksRMCTaQsVmwUxxCCsvE0khOURwHmep0uPR1VAZQDQy8qT47FJHnl1zbY3QYNdzRwo3OvknRSEZirAtq2SPJk8Lbzk0eUn7X8crMfcX3botEQLj3Kwh1BZsxptvLUGIbZC4GmsUuVZn7QoZ97cEgvQ5twefPH4Yr/fzzz58/f/jpT3/6z/7ZP/t3tRARsf7qV99cLtuzZ/evX7/+8Y9/0pp1s8fHUxIfljuIBpGbRXSkA8GMEACEJM5gAgLKVJmQ7Eh2QkApgyiYnd05QO6USSQq3kLGpiEuKTQ23nP0AC6U56RHWm3b9BLZTPHQizIWgoCxB79IzyQM4UUlEEEIzJS1FKGxUNBtKAKrsEi4JThSCEqQofyX2a+VQA7Rn8yhnSYaRMSeYFKQsHCmR9rYg2OW7r03uFM09A0uJyJnGYup0O1sPS7baXu8dBvqGgoqQQSXpOyRm22cIQyQwGFjIwvAIBnqfg5YshNn8YfWOUMsiaeoEzM6aNs0bbqv0OzoWjiNsnOtquAXdJym2rfW+oVIQkSWuoUlMYcDlojk6JBgzE7uvbWxm4VYd+VNYoqIRML3FrpqLbvIV2YYrhIKyrJM87quQsyqo7Ti1oaeig47HXy58Qcisq7rNE2jh1lKGWz53vsPfvADBt6/fde7//Krr3vvP/iN3/rdn/zkk1ev3r9/fz6vFt8+f/FJELd1rdNUpzla21XZmUiZiCRYkgScEekiDgUnJ/bdN54JTwJFwhKWqYjCm8jF5i0YZWExzRQj9hKzadiUrgHGEzanKNbmi8llmWWe61RrKZVD3EtyDjApETE2DQ3i7UZQJogQsadlZrMAYi4LRrWEoLGLEJGDwRlEKUO1MmHEBto0jzm4N0hRsMIirQcIQrpZEAJJSqzKhWeXjahkOjLC29lOrY2ujBctkWP1mwGaIUlCzAhJSo/0btEzXAmsRRAUlBRjVo3IFIEsZUsjOBewsMzcHy276VyliGVzWAQxMcSDOKOF88w8Q2NZpGoUdOaFpwFxU8yDHB1JDqokvffBFh39zLHc8VbU79U661ynUop5u3XCP3CMBIfDIa+iCoNteDhUIhqL5HeVysvlEhFTrRHxq1/96m/9rd8not77tm3LMn366aeZ+df//q8/efUZq3z51z8/Ho+n0+X942maD8+fv1yW5dWnn9tIv0pprUfkROvYBMrEoJIsQmMJD3uQG3uQk/Z9KoonegzwaEQpggJIIZsP7XDXcFyjGFpkZ0MVLYZcTmqPfsG0pdha4lJCHXedly2wRVldtPCUbcl1CtEDIq7bIVJSkJQR2+pdghkkDBYIiwiIwxaRwiJj3GCnYBBnCMWELEgGNyAAA7eMiQRCYRSWawR5klMIT0gKz8osIkiLbEQp0seiyEwXNqI+T7kc5oxKPK1bfzqv3Y0iE5Ogdu+6K72YiBQqgIanmVEGKNgzjNhZIigknCMjNVKclEpVCKsvc6mWniRFlx5bUIhKi8Z6YletZVvXJBNRUY4EUsHKwoOyTTZ4p7yK10mnuQw3NHoyBbvAOTOPXQC44vICuRKrd9FJ5FVUetdGFTYbW50ySE/b6en101Agf3x/WpZlVO7z3XG1na//y1999eWXX/7e7/1e93j1+Re/+MXPVYQonx7fPXv27Otf/XLbtj/8239nOTz0SB1QYnoVpuiPZRJHNRZiQkA9KJKqJUOrU0p02EULrRRRJqODeyVF8mplc58Ed0Trevf1Y542X2o+VD8z3vdKZ50fzqIpc0xoWnMiQ2U5ylw6V5PizmSdMpKCddKpMRNTH0A8k+fowUphYsqMlMTo0Q+Od6u/KjIVnpAEUuHKWTKDyCn7uCsjhxkwapscAMiG7IUmNDnA5A0ASYLM8xJ0AnUi57gPGCu5m2X21kupSsoJb1t0U2SCDE44WT4VksHxDWIoHBuRcYJt34rkApkpAYMB1sloIKQ+1HN7FuqlN6wjC9SkApUAe6YL7AfUWoksXTvMkxqLcXiGINKTHOGjQ05EwOCAZgLCo+OxVxi256JXUCkzkZlBsS8UToBjJ5QQMXbcduwziVFaQb/6+c9++MMfPj2e7+/uvv3VVz/9s3/r7p988smPf/xjhF0ul1EwffHFZ3/5Fz99eHhI5LO7++Nh/rN/+29ePH/+R3/0R3Wapml58+YNRD+fP08mRLLotjVRgnVxll3qIjONkAhjmikjA6XbHAIqCu3JgTZxuvklkLLAiLNJiZN3mbjCpnaposh76VR7ue9wS6KUZI5cqByy4LE/HJ+TSkjNIlmFlYoSR7brAOstdd/nxGOIVO6Z+05OYE62DGtrhFMEMzWRwqQAK125iUQf6/8QJQY3LCLxwX9kjoGLHtkTICYoZ2Q4EtZ6JyKPpB7WV6LiFu5520GcYybHrps6rqNOGWEECCNzdHHiI52s68fbB3f32nlvmGB4KRoAsGVEULfsfTPPcOcgUndv1lWVMil2zCivxc4YNB9agWPjSsTY4+HXiTl87DJjSI7HDdjf1dmJiIIcSYixEhuBTNKvfv5X3371i4h4/vz5v/yXf/Lll19++tkXvp3/zt/6vae3r1+9evUv/sWfXS6XFy9evHz+8Cd/8ieIfPPmzZib+0f/wX8AYFmWL37jN7e1Q7S1hrIPMKVIjxw67R4OIgwRkzGWjk0oSxCacwJrBkOLtvkMNwAqUzbAjdAb9V5EqJC1wtE3J1JNOaQeTgBzpCuRElfDAro7PAtPWhQHtYlCwABFUuw3Rvbxf2AXGgiLlATBk5ko9sEXokQQ6b7xLRi52+WY9KV9FeS1dI2b5goIA/QZ6GbkmKGDRViOqWUwOY9ulpub7aydQZjIiExKECtzhAIDQph2jSIE8eBQRmC0aLH3OXlvxCDHZtBhmnxzY/suhX0pBTnG5s/oHgFuPbpZb0AaecXgZYEjyRHhOxx/nY0b9n/rHo1NlkQ0kk8MxvpNAD9iLKL2vHFK6PqhPG8CSrnneJlEf/g3f1CXWUTGTOab774jks8///xHP/7JZ599ZmZfffVVrfXx8fHLL7/83d/93dfffHP38NC7/emf/unv/8EfTtP0P/qf/I9fvfps3GlHEknbepnm/bios5M49hVfKsz7JoPCooH2/lyNk7RJ9alux29OSTEdpCh5Z4mY6CJraq7np2fLEa0VpfQtW0jKZ6fns/CBaGIKc5Upodsay/EOSlHQFSkspTBVBLW6d4pv04ecw1qdEpxBCBEWIhESEfBJRFSryjxST6QQyY6rfxgHj6ubHAAK47pvO9MTNshNSZbwCO+x6x1QfLi1t0HyMGeWXSoXbEAPH7XwnQwGOyxj+CEHJY1nImifkb/SBknhu+oWaBhl5hgPSyJGJAWRRUZkC3fXrvDotjlijXb85GFsfsDIg0c1R2NVO1EOKldeL+nYsjD66RhxKR23ksjdSXSo2w3U/RqybksvPHPIyl9r/LY+LhO3tRHkqTd4rtv27x/ffvfNV69evXr1+RdffPHFT3/605/+9KdE9F/87C+U+NnLF3f3z37/93/vR3/jhz//6198/fXXz5492y7r/f09BRJxvpzuVFhrsARljuADIS6cOkiVzOkZZCGdJ54IykyXjJI5l2WTokgOc48WxcskzZ7xJ3ZySPXt3VyC5miRsaYIeNvY4+Hurhc9Jx9ePiModce26sVRKw4cVUOIrkqRmYnkHTrYA15E3JT7BvsGV7HUTDLszA/sXjNvg4u4TTPS/rJEAJOAaP8dPNABd3iMl/PMZEW6R3dn5kwm7AFvSL8O+q9nxj4OTufYMtPiSgynsTtrl9xg7Kgy3xo5I9e7siB2pxUgEBIM2VVIBg4XxIMETRSAM4JAOzs4aQiNXnHN0dccDUIiYoZKGVGHiLZtG4pJO7/J95VLQzeKIIHR4BymeesRMZAf/GtCjyrn797oVLfWIrK19snLV0S02Wrb5euf/+z1L3/++vXr3M4J3E8qIt9+9YtPX77wvj1/ePjP//w//+pXv/jpn//p7/z2b//u3/ybD8+fu/fDrP/8//df/70/+iPzLGUCkpHKweREt8nFpAQ7oUtBSRBDC4HWA6X49kS1lUk5FzOutVg+RW/9ZM/v7lUf/pt//l/9nb/3P0Ck5frm3fmL5S4nPbE1woUY67uH+WEhnnVxyp5iXZKJRZPXcSHiWjnuMWvfIOwBCFPguojuulRxHPQxN3ONXx/yy7F1Dxg0TB2T29e1OBHRki2y99gy0wMjo0Xyxdbda+7rjXdGj0CA9NxXYAyvnIQtfexv2am7uTvx3GvhHAF+7Fkd1rV3Ej56jP7zmBWmsZgoclhcREQOUFMKVxFJirFGaV8jx4OeORRDINchn1Kk7FtBHMD9/T2uG1Bvih4fQg2NhI+uk0w+TfP1e+SNVApAP//s5fm8mtn9cXl8fHy4e9bbutwd3WKp1NpmFveHQqFjozqzPLs//vmf/ZuXn33xW7/1W4d5fvf2dTs/nd+9ntmJ6A/+4A/ZIdkKZa0anYVCxBidqQsahqQ1gUkFJTswmpaEFBZMsp2FV594cwZk8YjzmjWN8Cf//I//o//p//zrv/rWvpXl8uz/9U//iztsx/lw+Hv/wKdqGid76h41VVtnOs5yJK4BCoc0V6WVPYkBZ0gMPzpWfmUGeFcIySGemk4U3cf+iFGVMwZafN19/ZHkyxW4T6KgkYnHaH94sid6ZNsRROdIHWuzzWyaplImVR1U832PVDgRBTKDAolBqoh0vnrL/VwMBHnvF+z2t2dyI/KOUpCADzTgK0ZJ10HhSBnJYlCAQqaqQRAOAY1dl0MgjXYnd32jq2e+ceIidqrH0PXYBcKv6+0A7CqhGSAZU58DEBiH8yOm8jXEr+taivS+/fLLb37wg0+JaJmqCOlSbT2lu4qUqs8/fzWu3Xm7iBS39ubbb/7v/7d/QqzzVJa7+aD05ld//fT09Pqrn5fl7vW7x8c3P3r24kXFC8AVTjAalFUogZL5dpmJkiVZDJR//uW//voXf/nu3XdvN3z93n74wx/+D//eTz774pm6/n//xZ/Yxcj9P/sn/9lvfvHDP/4v/3nbrLxY/ut/8S8vm//NP/iD55+93GhoqsWpwb1f0LTe8bQos1BKdookePLYHBwxpi6veT4SCXZAkiwjw7VyJHnAw4R5eIzvWWTmrTQGIdMZzhxIBzolg5zYh77iCKAAxpbojChc3LO1k6oy6TRNzml22SECYCwWHjPz5tD8sHloL83HxqQPhPodR9g/5G6cGIomV+WdAAlnEoMQY+yNJCOIpaTHrKVHb9HcjZVDGJ5JY//T907IqNDHZfBrc4gob6ozYyT4ViQNw40Y/he34B5xFbCVPYfIocMwkvqnp6dnz45mVkqJtKpK4KLM5MpZC6U3Ag5VluW+edzNNYXnwxSBvp5tK+uJ38Y53N9eHj/97De4nb798i+PBSho/cziT+f3x8PDw/0nRBTBBoiQJcCw3knoq1/99f/5//FP/t35zz7/wWf/8r/507e/7Ig78P/zn3wSf/sPf/Q/+w//F//X/+P/5T/5P/zvv/z6q//2T//1/+Z//b/66S//3bdrOX353X/4H/0vS8iXf/XL569era29395/9vITRM/ktC0aTwLo5JTdA8YQ5kRSDO8Dwq2FQWCmyByb7MnpuhU5Q1X3vjuNLHbo4wWAoL2iJ6IIDwQjBDay2Iz01hNO+9JoziAaAhARSQlPIrIeQBtTisf5aLYPylZlTxqTikJCbsyao8VP7NdYy6C47awa2A0PY9JhkjsmIcSJnYO0/x4xBHMyPX00Z3tb0zoQpdQ+dGwY4/UQuefnOcDPHdSMIB/rn/ax3hDZTz0zE12novcnkF33zA6XLPtqZ3cPEfn/c/UnP7ZtSXon9pmtZjen8fZ27742Il40ZCYzI5NZJIsiKYKqgiYioKGgmSaCBAH6GzTQX6CxAGkkoCYFqCRIQFEghRK7rGQyu2hfE6+7rben23uvxsw02MdvBOsAD7jXr/vz437WWWuZ2ff9PrUqM23pT354wcybzebkZMXMwfO89h0Bqs5x9AGAicwKUXGBHL94+VaIjdjMGh9WfXO2XgRHsz1vTGl5ctF2q+99+ukwFiLZHLZff/ftj37yd37w/Z+sludNu9znKfiu0+DutJ2YmQdX72T///vlv/mPv/grF8NXn72w0Q6HUWh4/Li/vX37j/7pf/F3/9E/+z//X/8v//N//l/+k3/6JwMPQ9TpqxwF0/3u7vrqyUfvff7qq//sH/+DotJUXmtzom2HwOwQvURnjtkFZlZHIDfzM3V+vR4WKUMZBJPZ38qcH/As5DyHB+eKmT0Qu3H0NT3sK6b0cORhPn/1Id4OSiqsaiqsClUzmncgmjVmdAzlJubjF6pqPapXGcZuvjYo2fzMj6HKDzdROh7DeNf6erD4PAjhFGYOBBPosYMBPa5tEanFhWohz/NZc+uuBK6MGTFCalCbKzR54CL+VlKspvaO/HOUgc4/1OwBUNUieT7Wjz/+O3Ol6oNVce5DHEO0vQtBa505tLXWh99LbZrWpKrqlIYYo0ol8iE4UQzDkHM+PT83o3E8tIFNM6P1YMn59nbvYzMxW8k/+4s7Z9IvWiH0lDvKd1dft2xXL3/zxde/+d4PfvTJk49LzlG7yIuW/Zlb/89++s//yz/6Z5+/+qv/j/tvpfpf/ey7aXQ3b+9dj9u7r/9P/8f/ww9/8kf/43/8TzdyY62HaEt0c/v27ZsX//Ev/sz/Tfhf/+//d0NKzN6URaTU5K0EDkQNCGRsR5DrPG01YufYFKz0UGfAqQmBBXOpS7O4ZFaSVDav85VMZgOxmc36ZJkD1mdihfEs23jwBh31oypz4iUdr6umc539UIbzbH4TVXPkCAwyIjcrBpRExb1bf8fW58NDaQ4emNs7eLgwHtOAHR3vjzaHXMrcknZkpDMKbN5D0Tbes3qDCmVVE1WHyvpQNsxzXijB/bb0/p2nZPxO0PQQj3t0dMx/qA/vNjzY4OYv9X72CAkRSAFATGqtvgkhz171o32ERAoRTWnk+bjTh33CDEDTNEbwTHP6gqy64DlPo4kq4e7uru/75aJXqJZJC0inotNuHJrV+md/+adN27959Hi73YLprzffnPy9f/T227cfnn8Ycuf6BWLs921o83sn/T/8+z/9r/7r/yexkhXn5fkHz7787HNXmv2bu1LElv1kQhZO/Hhjstfxk5/+6OLJpQY63I2Xq7OWvZd5oElERAJf5iBGhZtPJlG4mU2J+SD/LebK2dHjCSI3494VykoqsONNbD6/YUfCC8MA1XlkptDjb2/+H2Km5UDVVKFiMyBiLlkx895oVljCDJWMi6ijd5fJh0VMc2/UjnsMjBhm9DuTB7LZ9qXOQERl/kzFQ0zoQwNYlNSUjO3oqJxFxyQZ1VREtaoJm+h833l4HLtpx1U2xx4dl9q78dXx0Jf5bn88bXCcJD0Y+InN5p17Xl1VRehIptH5eiwi3jmK0XvPZhK8NxNzLsYAs5zz3Ao8DFMTYt8taq1F58prPkwoOmbCsu+11qLSL1eLxWJua+z328VikSVbprZZHA5TgRLR1duXJNW8WaR/8W//63TYf7Y4+f73Pj3tHx325XIgx+fXxf79f/jFH//0n/yLf/Evsrv9Z//Tf/xXf/5X6T5EW3351Xd/8Vd/+f2//xPyXYKvdYtA1Lh/+E/+89dXbz//8guX6Lw7rSIBjhw7OFYiMWI48Hw0kc1Hn6lBIaQAO354+RSg37ZvSI67C6mqIxibqRHBze4w8vOmqPNtVuddi+lIXjS8w40Z48hinK+XmBegmSkRjI2O0bwwAuEhXZJMYSBTcs4bF5pfc50LOxLMNI/fPuj4viJABcf3Ef0OZ3fmEcEwEyX0wW6tCl8KxCCKObnEz9MKEpH5NzL/oLN9wGBkM9YL9HDVmcMLTE1NCbNICLP/3dTecRj14Zh/aCGrczQjF2pVEZvtv15qdo68I9PqnDsCsAADYozz3UJEAvuqmkpRRzO7Pzh/1PaK+rY9pJSG6ezsLFeJMXrm1cqJSAEThymJkAeEyQ/bnYe5NijThGmjt9t6980XX4XXnTDOD+Pq6d/607+8++T5n/zs51/tx/2TD07/7D/8683rGuzxvkQN+Ju/+LPf++OPAKV4lqn6tvmH/+SfKstpf/Yf/u2//+H3PhUplcMIYw1OhNSiY0Y1JYKbubNkEJ1tijRD7WU+vYzNoPRweT+eR+8GRQalWUMNgydPjpRMjnB5K6amSsxERxoMMHsxfnsO6rv1b6RzJoCRvdtBMaPzjkueyalnqmZMsIeSwGZFLauCZN63Hp7ewxqROfdwfkP89nQ40g8YjmFk8wR41skDSm7KMBPRqpLYnMYq86DaHZ8ycKwh9WhvfvimM+vm2DV4uLTYu013/rSqerwLz/02kHPsHC2XywdNPohKynW+KvjWMwffd61jCt6ZsXdOzWZ5aEophAhiBZdcQW6aMjOfnJzUWj1jBjPtdrvQNJbrYcxd14lZnibPLFIQm5wE4OA5NMEkr0O0VGQ0gdvUfA9brprCZTjst1M+yWiHbyfr/1//8v+BJE+eXSy75vW3d0SoAbVI4/nN57/mm9cn3Zn4ji8fudM1cYyO7vf3m9e33/8fffDl119978c/rKCpKqt5wFu2OVAhnM6hhgKbO/JzTTRDepQecL743cD6B7gazxk+cylvKlYxv6tZiokIME8uj+6g+UWaV+ncBDKF2PEOqjjyNgAYHOZGq9l8bpKx8XzsE4HYe6tmBhKhuTVm804PB1IcD32bZ+I83/CAGXaOY746MEc+KNk8cFU2VjkOzedNNJiqajWtJELMHseIIgHNLSqCwAzqZlDBu1374cb9rrYjohlri4f6XVXn0tyBHvggcI6851KKaNFZCkbeEYsUhfnZhewJkDrjiWL0qeTZMWeGtm3HMZFjguWcg2/V6nK59I6g9ehBIRqGiV0g9lMqKgVayWzRdSUJVTgyqbWMadWH6XDouWPuxoOEZvXZZzd4nU+fhRo4tKfSdn/+Z684QRUXz7rHT85//u9+6dhnVykeUBcyDrSTz/7037Th8pPnf+/2eYMYQ6ScasPdh08+zPvh6ZNH15ub0PUr3/vgi0kx9aakVuqSmdm5eTADmqmo8/zj2IK2+WSeL6NaiGatk4oJPfyrEkmZHd8e4JLnl2QOd9XjYW08T+3MzI7dqyO0Td+JIeaJI44lOeZqx9hY6DiJUSY/56WKqKTM5JQAcuAZQQpjYudsfm7zlok5YIy4HC8Tv6PKJAI54nkOgKO1jZQBJfewwsSkwkgED0Z3vFuRvzM1fdDd0e98+Lgc+SG66AjoVFVVe9COzNcfeqAzTmmY7zPexXdKHQfyI6uZ+FUnVcBKRIdpbEIAG0i7PkpNXeugZdk1pWjWcn9/b+LakxOAgkPrYkwPNnsSAEWNvPfei4gLRkEP+y3Bs+tRetf0VYtWMWti9qvEFXT9pvjQaZHDcL+WRzlmen/z0U/Hpf7qb/3e+vOfb5M0XOPSrGcL4erlzf+7V918/n+zy6eX5xers8vrwf7dX/7ip3/8J5a62xeT7571l2t/uvjVy68VrnPuk8vlmdfzw00IDTgIB+nayfkMuKxLdiQiTkePxF6ZyJzN9jgyIzUiYpekCvmI+LBBklUSI4OZA4DqxiDozShVRFcZo8K5ltRIM7tSdTQm0+jqykuTwkjGzsAgIV+ZFWAjVmOFsolDZSUygjqmgdYE0BFhp3MYDmba1nzvfXeYG4EQXRGbxSsMm8NJHBvPQhPM2VKwYGqlWKm5VmcIYjBSs0BcACG4mSsHV4/DNjNTK2Lh+Mfj5fZ4lJuZpnp0XQOY8zwBsKkZFSUiCsGFEM0s5ZzzkR5Sq5jV+c1rZp5B4zSp51JKCK5pGs8s1ZjUcZBq5DCHlRBRjO00jNE3RDRNOUbvm6Zpmpk7Ok/qHBwe2gohhEolhLhantaKaRRz1YdYVZ3nAkkp16rPPn6fru7vb1PkmIOxWogteFu19d3ZKIeC+asy+ZiMeHl6Pw3kNbiO0/3rV+MXX1+/vrevv75q8evNy+v2pB/wxTN88O1346vvXv03//d/eXH27H/zv/rf8uNn4+Kk56ahhtVbdplUGqNoQ0ldmPWB3pMpnDFqrXCLKkVFu+CrFA+OLtZcHPG7GxiZ6XHPhSvqnCsgda5KJuMOTtUKAPI1lYYjiqr5QkiBHMBHKRKMDYTZ8ozjxIpmrBXwgCo7VnHzNU/n2yRAZETijKCks4QOLCAV5WOZBIHWagoF5pwGuJnmJ0VVjNWxeHFTMUsqhZEwawwA8KxjVavztyA1JvOtK6XO3bF3d9+H2yfNh8fRbj7vnr+dyMPMRMysqmp96Nr/TpbNcd/3DyupUVUIrJoKVGsTWvJcawW41pI0TZM5TwwfYzuOY9M4yVKcpLTtmtbM5nlr4KPcfz7AJjEyZnZdH5ctAFatJFBU35Dzaoy7+61pcNA6Vlli3XXb7T3u7Wd/Pf7tT8+2WbmNqVQCKnK3WvzixZuXI/7we4txkl4ORrrP7c1NCXTS8nJ7t/326svmtLm5/blMh+jd3//7Z8MkP/+b/+6mf/97/+x/0ta8Vum5bSk6ZuUqUHZWPNdijijAF63wMC91DuHiQOS0VGKUVJj0dxBsc578LNJAJyzAyMogrbVzDqoCEeMYo7M21grRTN68z2S9uuP1lFRJdZasEjC7IXCUh86ScwJ4rsZJj7IsA5EAfMxFIvKYX2eFCRm0zrLAeY6ljg1+bkmpZ3LkffSWFWMpY8lTyu1YGUPQSr4YOYAVDjDjYvXYWoOBShahIoTuSP+aNWDvlpe9A+DMS/Yo9tNjyU9EVEStyizDe/ja+m7lMHsi82XKbdtE5zk2D4N8I0BVZ3xNpUJEQxEHa6gpUmc1Q851vVimNKlVBjHzrAcoWqL3MYb5O7Wh8RzLJGSInc95KlpWq448MmkWu3zWvnizJV04dQaS6KoWOEYNcl3Gp93dzXXNFpwvYhxJHF08+8Gr68//8leHHz15pMPhzdVhN4zj6GvOd3d3yaQGxB4XvTvpGn9mdi5x7cfhl7evvymLN997/oPT849hy4wu9ktnTRWj2MxIEjPnxBGBZF4d1Ucv2dJ0CEzkWWs2Pq5LzHogYtO5wcxgV1R0Xk/OFRRlq+RgviYl4akax5iZhIEqRn7ePoVUAcPxsJRjlfNwvwOUjAFGhc0ieQfAWA2qVOex/KzAcsa+OraWj4QTFSNVVRyb4GrCwmpVJFcVVmM1eKFWqOWscpCS2ZiCOnNFtCrYgVkgqrOFOjhmMX54vvhts3zu+8PcUSOOY8f2QdeMh4889IZ/q3eep7/zG3R21tOffLo8OVkfozXL5I5xx2HuXEipqsqOZlengxlF7/1+P8dPdX3fbrdb76hpgvfeBzcLAqLz8zouyG1cILPU3PXeqNaqPjSpFur8TurXb7e/+uJWU8PiO242bgiuLYcJLQEjKHQ1RIqDlUry6fe+/+uf/dKfL08frTavv/UTJgLUmTQQdVzUxBhoAaCR6FjGpTz9KZ4/xver+7SeD3Ll3aIJZyen7z9770ePnnzSLM+rb6lbCsfQrkNchNCAqJqG4KaHDjPPhZRJrSV6Jpr1aWxGrG6m+wBcgrx68TqSC8FdXK4zZTRcizbc2USeg6gWtgqDog19ETPCsQx/Z0fBbNlRMrwbHSkpAP9Qjj+sAxhXpWosZMoGr+zFuRqCOKgTN9oc6PRufCVKqloqQ51VmIjkKlmkiMkEm5zt2CozhB4tz3sLkUKqInycCZscBQrz2+ahNJpX57EKfFhwc77yb4WzD5PY3/IdMAsbHjoJ9jCdn+sk+qOP25PT9Ww7TmUC0DRNDO00TSHEkvL8qbEJ0zSJFKkcY1StJaX1enUcdZDO/bZ+0cUYQ3AA8jipKjdkGY4CQVxU52DERQAKE+m26K+/evvqKkvy0YIHdkiQmS0ZAXbZ2mKN6w/IiVLTmFXJIAptEwnTfqonqB00goRCJp/gqrIABVAXcfI+vf+TQGn3EfB+bZ+3qQLZ/Gg8STBrVuunF5fPzPXPPvheSuZd98O/9Qc394cXL6/+4A9+6p6eMfn9OAbflSLE3ntWEaAylI7jbHbKME/Aa9zdfPO2yfTp9z7NqNaRwqyQs2gUKpN5EylNcDUXIlJ2RhDWuSXDNuPlZ6StkmGWqBhBZsOyMc1YsP9Uz6+qNAsCZxugHqUF4tLx8na8lxJXBGESolxViqoaaYIWK8VUiYvDSCrRmclpv/RizhwzpyoCcjwT0w1KzDzZ4WH7fHjDPEwE3rWByX47nDByv9tUfnegv5N5PSzx44P+8x+fsqOu64hsNsI3TcPsfQgiOg3j7IqKTTAzkZKTxBgdkao458zEe9/1TUrjNIyxCU3TxOjfoRuZkcccfcNOOSg8xdgS96XS66v7Etq/+uybMfmcEI2spiR5uTgZi6jr7CAOfIaWyR9IBj3ERnI6hLgWF5wjyQdpz7BFi06B7BL1Aj4Y3fo1zp/6Z0/X5yvuQmqovvhirHv8oMPFuesXPiELwcdI6kUI5rRa51tU158+/fiHfzDk8Dc/+yydrJum/0f/+L/4+JMfmzW10J//xV/90R/9kXPOUBhGpvPeR8YAPtt+9R//v//2f/nP/xdUoUzkeBzHVbculYpzgyZ4i6xac2ibJOpI5Di6VDaEOQgWc5OLCHAKNlZCZVTYrOqfQfROj9/3OMfHQ4XEYqhCxUxm7RELcUGooGJcmCq5SpahasTRQphMBqlJ62F/lySNWuKqbRbtxeUZRFDLfPGbm6pmmPVPbDz56d2lk37nrP8frNGH89veaVl+91ZgD8P23/mn43XWn52s7+7ueMFmMrO+ZlJDKZimidl577u+nQMVzHjRBSKKMaaUmiZM0+QDq2oIIbmkD3pbemeoj50qpVJKqu2yE5VXV7f73avN/dQuzh9/+Cztv51SgXN5hrx3jZl46tJEYO/mMQdBRAN3Ke0ir6qSgVVrJE/um87RSsIE25JUVRA++BDPPmzjqnTxvpm0z3y/1W7B9ay9PVQzt04VMJptu5ZU4T3agGhb5zHtrv7iP/6G49mqW3/0ePXk8Uk3vahvebl8ym7Fu9vdi++ePH9uRAJTZgGBjta0j56+/4t+ObHFyJqLiS36dkgH5zwDSy/7vL0a7j/79Rc//vQnjy+fmVZizK7HoOzqEaQrBKUHwwlmKLsZw2DOQAqvx2UKQGDGJMdeJztxpD7USGqkLashVRnLuB229/vd/fYwpnFM45T3+2E3joc8jVUyqTL5mMhxNams2crp5elyuVgt+z6G954+efz4cQisMHZEQCnVjuAio3cuU4P9zjr8ncVnAn13M3m39x+Ha//p4wGUZ34qmfxxKDcnJaqqC34a88xvPhwOrjhmELGqOp7NrGiaUErxjSfniKhp+mEYmqYlohh8zhUg58I0ive+WfQmenW3//rlm2nC5eniZP349Oz5N19eS3bk2aiQMxBdPn9882JfJoU5bkKVYa+J1Qm8qmN0RdVA8BrIrNanZ/i9H6/6atnKrjM6jV3v8zT2HY17WXWoBxToqo9d4Iy8z3UQnLehDy5VcUDKgEMVDAY4iECban5bJVu+efOrL7/5S4M1q8XjP/j9v3d69vy8DG9//vp5//d8vyzOo+kqt0kBF6tgLUGyCWOq1Tnznjc22oJZx/Hmar95+9/99//q16++Xa3P2cq5bxerVa5yTBZT45k8S8eBtogwoYqQ5znX2NVKSgEOoqbmm2Ysgzk5pmlkjtp1srz9evvVz797+83b3dvRqmiuJiJm4i0H5GCZhSKyVUPxLRPBqTpn1TmFKXv1rOKHSnWX7zaDN/r1F9+FENbrpZocDoc2hmXXx649PT29vLxcLpfOOYgaBGBRnZVxdky3fTjNcRzEm5kPTmS2yhyzUEopc3u/VnHOqZpXsXk7XK0WYmoE9sec2jl4gYhSSqJlvVw0TYBqjI1zTgTVqJSSc14ul7FtQ9M9+MwpNm1J2VQcRxeikr++3/3muxvXrH704/cXbVNHefXy7d3dYKbsRCiTYzj37YuXOETYkn2juqOm5po9Witz168xAFyOZATD0uODH5zWuk11PGuN3CSCuwExj2f9IqDSCe9KpSYuvdRd1RYnDK7FO4iBKUTHWTHkFBdh8DqaVMHmXt9bjW6CwgKw6sH2+hf/4b/JWaSSC93P//1/tTp/fPb4fe7OLp59/4Mf/N5idWlwAfSjD96LUimEorw1oaaft7V9TT//5pe/efn5X/zFnz5/9vE/+Ft/6Ly4Sn4Of5YypwnSg8fOcSRTZ44V9VBzyvB8hjaEIGZgr7DD/RCjRyZvTibdvR2//uybL/7qxeFaoy7Y2jicAQpocYWCFK455KGtky8UDVSBbFBDURVmjI4K2eRI2Ymp1GIVLJarqoPlsk8bqFUpzuU3t3uudVZrqOr5+fmHH75/cnKiqrEJy2W/7HpASqmAMjPgZhlkCFFVS1YReB/nXqdUJXipykyOo1Rh9l4x0yz8MCZiG8dxmqa2befmxjRNzDwHlM9PYtE1tRYRUWC5XJZS5v78/f39er2epimEQCZEtDpZX11dqcmy767vdjf3BxdPfNNe3e5u5Hrc7A77HNoTS5N3uHh82jRNGvPbFwdYw741rr4V11StVlOBut85B5TmvCH41uHNYUyhZtQzC6ssnaBdgENvtTYs210KLYIPMeXegzqcKRYVVtE0sQJt9FolOD+VatzcJ6EGQijZFuo2TkLE1bj3hr5xzcK5VJg2EYzx9ptf/Eypv/rive3Xf/7hJz8xOD2/WHKWq9/Q4oQW6y4urToUI7Yn73302atf8aL/kz/447Vfasr7PDWc4TiQVa2qWeeyuFqE053cX23ffPXm9VdXh02asqni3BaP3nt08uzs2SfvZatv3ry6fnU9vNnLXZWNuBTZWk2+i62wTxBoNEfmNEeZmmnoxkMzTs1kvrKrDurnOaSZOhZg7yx57FiV4Q1RJBa0Qsyu0AyxUFJz7EQp59y5kIuYMbO/uz/c3P4cR4iEOUfvv//+j3706fnFmYikOYMYRBzGqYQQ3l5dn56cq0KVhnHftq1zzrmgZiUXZqcGP6VSSmHv0jQxUy4CCLFvmqaq1iqLRWPm58YpoDnnYRi6rquK2eflYyil9H0/5TSTnkJsPQPAycnJdj/txkOBnl1cxuTGVDabt2XctYxHj0/X5492v9mMppvNvRlKAuoJc6d1CmsIdmfLxf4+q3kB4GYFjbLMxl4n4t4eEO6yLpEtKrXDII9YT1eh6hRaYkgT0EZoHloKjfetpUZcrKxqw5Bj34yHQ9PCCJ33Zcpn5JJQqpUaNxXZCBa948ZAuJ2k89IEmNqysTHv2g4gyePnX/3q6y9/+a9Kkfree4+689ftxQcf/dCdPNpNerZ6dtqeKtNNuX/79Zd/9Wd/dt71j3/0R1xc57oiyYxISEuFsjNvCfUgv/rrL1598Wb7+tBI51LTynJJrYmFEt5+Pb2K3/x1+JxbpJQaiTG1TTqNufHaKLu20VGnrR7E67BSo1pdyiHldkrtqG3yLoPE1eoVZGxw4mNhl8wGyIGxoWrOlpE1w03VKYlWA7Ejg9GM/XDsHNXyWynd/J/Z0Y1Qqnz+xdeff/F1jP7s7Kzt4jRNDvHt27fDMPR9P0MUj613ohDC2dnZrL7z3s/QUD8miU3cD7lt4zAN80ColDoz/cnNVNxCcGbWNC1DQxByoQmz38DemWVrrUYopbCDKEGUmeHx9vpqKl5Rb7ZTqsX00HqsV/H8fBUWIURfpE6TzpIZrkGhiFRl3zYo+wNV1gKQBwCqbOpMqEKZxWjD+NHqbLv7dqwy9qs92uhgm2HRIQlCQBvBBcuWc6qjoA+ek/TUDmlaL9qc0oJAAu/gqqAYcVdCjN02Rl+TnHTMTOMovqGmdUo6iAGQaqIIAdGJsgAZRubq5u4+34Uddd/87F86jhZ63yxWy3O2cH17u6n7nzxZ/Ks/f3399r/98Pzj7S+/3ja7wG1wDUkoB9vepKtvbm+/2frc+tS2Ze1KYAlOPZnTouJcoBCstrXqVDssSNlJZGq46yp4yIOQVErBqfe2bW4VtVKpXIpl0Wolo4pjdULOyIgT84Ft8DISCoXsLJGYoxY0s3t9VTMlMyZHOHpdSImYxBwzw0weCMumlnINIYjYnFExjHmc3hKjlKJiRETMh2kqpRA5ImbHU0pZZPfixZFW++BW8ArKRYg45ZpKdc7NqOcpV8l1vremlJ2bYSzWNcE5N45ptVoxc+vD4XDoum4YhjnJU7SWgrkJBYWyE4T7zTBOA3wI3uDobL14fHHexC4JTL0UDyHPLo+jIyZH5mrjqWa4gFrhXCMyWynmku9BXkM6AoT6bOmbLG9ub97cYt/T9xe8PFlcXe+Wp2sm87ZnMmNj76Dm2R3Ggw8hSTEzZ6AK7yEH64WkjIvGo43VQ3pANHoHgR2sWUKNSjUXfVWY0+2oMVYzqGG5DOrNCUoqk5UTj1UMU7kuBbc3WFG/rtSAwsr/o7+/YDr57tf/bvrLv7pfZFIi8SQNSeto7azHFBfdZdvHWuowjRTalEqtNcYoOjpiq4aqgRiA975UrXRs+M8QJa0SZxNmc2BoIDhnFLxFqs4VLqNV78gxKmxysvPl4G1iY4mJZNIRGaPwoG1nNI9xRRQkqkpinlnERFTVVzEicuxVJaXEzE0Ta51le++mRwLBXPoYkNIUY2vsDJimMYTA3pVamd1U0pxcXGoJ0XuwO4zjYtlnKTBm8uxC8L6USkbOhaN1GmRmKRVSc87NcOdcS9M0bdv2i4WPbhxHqAAQE+/cOI6np+vNXos5psY79ZGallfL1bKNKrzZ7KcaxsHlEiBcnfcWKxWYrNu2Fu2bMKbCobXCVEGzzmGOMgKYFSgrasb95vRCLtfu5q1E8q/fVl6bxEF8eHOr3vvT1eLl9f7JAn2FRhxYmoXfWJHgNOGsJVITQdOx7tQDZdgVQrdmUTg4KvJ83RgklUoE18dUC4ir2rKPSlComYyHwowQ4AJaFw77kkvxAc4hFBgGF0GEKPh4FUl2XUoUQz/CgyMHM0qm2ahwY53/eqyuW+9IX9ukccHnK/JexC6WnRUs/Gra1tPuzDIt+5WISMlken9734bWVRepdRoglpFmpqQ4TA57lm2QMUpigRORaiYIlFgmlOLQMoTMiZA6KRB2JTbJjIRUHek81IXR7C2ovpnTi9S0gtR5CGwsE5MjT4HZbF7HBMC5kGvy3se2GYZDCEEVoYlmlkoOwdVa2DsDxjR673Mpfj775/N+N45zj7OK5Jydc82xj0rzOxWAiKkKebfbHURVF2jaMAzDMAxNG3JOTdPUWlOZVOs0TV1/2UQVrSKVio1W+s6bNc77pumyWckM7YDOJAMeYRNiOFktczJDHdIum6moN8zDCiV/BNLQRKTnYbFswt2wU0GA/+TRpxu7N3+7cdKsF7/57D6P+MH324sT7PZgsX1Vrlgs5b5iyLJmsNoisJmlOjcxYIbztUtZGoG6mNJIJRGjdagVznIP5CqLlnPNekzR476lUuroIivSaNY00rhahvPgY60mGDsayILCb3NE3h/Aazx2Z1ozW6UoE9XRoTgkRvMEG2BqMCheCTaK0ZB3eNoQIfLUijW3n+8fLZ6Mn42PV+dR8Hs/+FFZT851pYDaszEp4JduyQ7GVFESpjs33fp8H+TgcraS0sAqHUdHlEtRQtGUCQIH8cg8ON8qauWGGq3mjAEWNREY1PuY0m4WJgOoD4NKojk01OpRn8xEWms1I2NkKSISu7bWCsZUpjnOuChknnoww80NN6Xf/9HZPOdk2HQYmhi95y42+8PWe9/3fVWdqcxiGkLQXGKMqgocVfdd1x0Ou/mD8SG++3A4qOrJyUm1mFJ6+/ZtTurbXoWXy2X0fH52ouzEdf/mP/7NvgpEIHmWW+AMj3+/Uemvf2V4wwEmGJUipuhoTSqGrCizru+jf/Bm0TCqXl5e7Gt6s91//MklDXs3TKfL1dVudyfIFe+v+ufR8W73Z9RfxOHJCtMI9qucdB20x7hybim8KpVGy4R8gmtAuvB0J877Q8ka2YF4lIU5qzp3f80sk2jvM0soGifE1Bw6vqfkG0RoqtgJFideS51xJM65koKYo572ur/PLk6yMpx07DsPx2WY1hQ4S3AkrBUmhiFBGYeEX93jZcI+wjVu2Egsi4YbIkdkL15ff/8Hn5Yh3bx6fXLSrJ51Aw8ZlKRScOzbQHHZn7Sxef367fnqLKK3A/3sT3++Pjt58r3LXbjrHjUf5PatGzZLapZ9vK+f+ifLoe+781FDEZEhdcSOMZSaImVGM/pU4Dj60IjVKR18YBWAw9zvdNEEUqpWYaZIOkfREXsClVyGpmlKqoFaE2ZTYiUWkTzfB/xcQ7WxAZTUVASi0zS1sRERreIcSynOOR+i9z7Xhwgm5pJySolBwUXHrLOh3oU5k26edpIPZPL86bMXr9+oSqnl7dX+ZNGblqbtkptKnaC0ODtJ4yBl8sSrM79uT/cb4O4WsnKMIgAbvJdSApxzIUtVB1Dt2sdPzntNN2W6aVpQRtpcn/atBDK2y5OF3x8Oir61ammxcpdltAHLEw5Vteycwcd5ymNZ6xbWNmCFL1hE3icR00K1RDCbicJBZmunqwBUoR4w4aJUUQ2M5MSvSGNynlxn4CJuh+hDyQWMhkjMSs1pKq1HAw0ejrHL2sVsChdpV2rXu5Srd6xmaiAP59xJG/7uaX+vZUdaEeTC7Tb5zfXt1Q0o4tnz9osXn2EEQOO9vrzfIxK2GjpWUx/2teK1vuxWGAdc4yUISABhu5m2f/ECDjjrlx+ebW1883LHsYt7XqyatYhDySYucOtJnG+7hTnfh0VL3K8zkR8nE8uAlZzzmJxrNCeA2blySGLCvmExs0Q8QpU5mlKV7JhrzmwsUpwxgJISsZFjgq9CnuYoYxHJCWY1Tb5pzMyHmMapbzsp4ojyMC0Wi1KmmvLciuy6zoUGCneMnc3OOQaZaBpz0zTsHJSm4cDMw7h77+nlmPX2dts1vm2dyXh3v6NuzSSnlxdPnr7nHa7evNq9vo0Sr76+328YpfXW5DS5ZiHHQVoRiCkbOUNqlr6U8PK7Vz/6ZHF7j7M1e9UAcM13e2Psz1p+EtGcNffTeBBQgyVBFWWvJw3IsJ3gISQoAvPYO4rOr0ydSgcai1mgRFocrBglRIY4M0MbUAokIkaSUYMgeh6dpoBAtStwowCIwZ/71hnlw+QJ5OBqjVAEOjh2TqkGiiIqy1NHIloxVXMBguoCltFrzsYcu6YYqdFi3HSNv/SUS+Flc0fjyuPRCV7fIsv0wXOIX2xvKk/t9u0eB9+kJQ/wmkQOl08Wjz+8uB7u3uy3YvOEFOSgUtmi1Qavwpf7Fz/84x+8/JtbXZS6ofqh3yV1NIGKcR7dLF3106jB98480W3X9SkrwF3fu9l4KYWUAZbCarXrujSNZOxdKHVzf78bBjl/9KhpGhU1UTaQURXQrMo2ksKqPoTOS6m11jKNUmt0noA5846iOYMHqei8caJKnaZaddYUzhUBAzWVOZS7O+kALkVibIk45xoCmVYDE1SkBs/vPb9kUBudSlHyo3L+Usbbq/u3b2cDZS/+6rt7cYC0qB0JAjW5FHhABMyqYnCIDKSTJ8t9eRVznYqsL9w4ytIjCJfETasUHKssPXab5HoqDd0XTS26GATNbhraYP2atYhXVIUFP7KbioG0FZAIC41kxUCzjd7gA8g7c1KJEkwZszEtEBs8GittcYI5T16O0/Racs1pFtyDZB66I1fA+QsjDT4rMElr4Ip1wEHA3jGURbQgOy02Hgr2B3zQQ7LUCWSIi3q6dv1pfA73t6qv4JfbnbY0rttvf73pOtzfSiqLPjSeW8bi7mZ8u78WX+DavmkhpW3DMOyLSM3w6r021MRf/IfP//Dv/uAv/vvPkYyrokw+xCS5olZVz2ySmEhzUbDydDgMIsYubDZ3c8XCzN7F8/Pzm5tbF/z123x28Wiapv1+uL55wRTOL57pYTocJh9omoau62qtbYwgS3mA4yb2KnyYdvS3P17WWh2D1JoQYRLYqUgbIpGRHY1Rc3fKzHxslI7IqHct1lozOe77XlXHcTw5OTGzedWmOqjq2dnZdr/r+iWHKDnBxHuuaptsv/jqTYKvIM8gSDNyYqkumrQYYhDPzo+a4Agq7DxVEmO4jPb+w7/95Oy9u3R/eP7YT6UGiiHZmWsp+o1OqY6PO7KdjYQNo0acnPafH6xPesFty55IbdqceFsGS4pM5LgNkjtIH8FGUnypRYDQBxShoosmVthktTAABPaaSgsXiEspoWtzzFTVpdlN7A0anNSCxqFmghE7tQYHwSRo+nW72VrjiolVLAAvcIEGMvVsos5ADhbcnkxdGHKRrOzADJ5Zow2MEYzOwuqw32vnkjNQlDHkvb96s//5/eL69aYeuInL6lxBAVU4h4SGgqSpSg4+EAIj5mrLi7KXzcn7C2bcvzr83vs/PHV9TXkijEoKzyo9a2AtkvcpKa+cc8yoteZc2bm581NKcZ5UlcnPMKXDdNhut8tuHWPjqaGZ5YPKnqoIHIsCVImFmaDQAoLzDIrsRAoTEzBbbRwoT6lpwgyoJyJjntODUirAHG8S5km9855jCyBPM1OAa5lF/JxT7brOIHmaoKZSSildG2vJTFgvl2UsjrBaLfaH5BwtuyV7icy7qdZiRMU5FCk+xKoGR1rVmyOoWVqet4u1jcN0crG6Gg79clmqlnE4vTxxTsabsenATaeBX7/a09pz9DeHfHu/vrq65YsmHcpX321/+gePmTeFExibOztrppU3VHCcvTiI5JLNhi9rAc6qkJFtVJQRT1omCRsn6qRhWuQ0mkEBhvMgVCu2CKCAIYMZTjk7m8i2BBHwVKLCipigb0EVcJiyucA2aGA4AxdYFii41bZaCRAjsxA4VMrsXLGa9tXiFLIqaWJkq8sWy8u2WWhX9e1jgyxfvx3f3k5iMEL0cESUAsEvuvOiVKROlp5/9N5t+qxfLu6+PXzv9z5tyg3zGBzGcRv65WYayAc259FoSnDF8QQ+8Z6JbJqy994MUrOZOIapMJGKlJwB7O6vl4uuiTBJTeslTR5mqHUqselLRfStsc+lOmdtDAaVUrzm4r1nds45B9KqZlVKpVlEqqYqIYSc8ly0xrZTVRVVslpFzUZNZjarAWOM7F1KhYhmXWkpk3fkPEfvyfjsdF3LOOdgHYa9aRCBZUlTuTg/JaKwaKf94JyrlNWqudb7ZhKZNeQOxHPEe+8uL0LXWdLHh0nIh9vhsOjcyPjs7bXLeHSOtsGh5oM5PXO3+7p0JEX1zXDaLTY3E6xtY3e7xauSvvdRiAIfCmYhZgUKxsnIS89BTLIoA42HqTERNb7WUgsKqos8OtuIXUZQxpjhHcgBzhFRmgoSAHhG0/SmbF5yGE2tC2iypZYCUdvYfjQfQATXMAtpAYDgYAksiASouGrUQhnmwCKpVkPtImvnUISZ95NaC1JMCeSmbHi/u794BsX4yeM4TXE3HM4uTo3t+na/2dDtlWzuNiWDmuB8ap+UHz65fP1t2d+UL//sm+/96Nnl47Xtd2eni5PT81WK4szg0kar+qolBC552O4Pbeejp+Bptxsd+75vRUpK0+nZmXPNYT+8+O67SHqxXqxW6xBCHnONBBODudCKYrevlkcjF8i8koxJa2Xy9AfvL6L3IkVEGKSlhujm6DTnXEl5pjgTkR09JfTu0C+l8EPwd9G5hHfsj6C9GafWdyHnadG1RhSaSN5pnQjCzMqcrPnN2x3i+upuwyCtOXRtCMEH9+blqy60WlCzEzSgCKXeR82JKfv1tHwm7Vm9GdX72Hae+HB+GRft4nC9f7IIF72oppHcmySTZyEL6PNAy/1yl8ZcNQ+22wyT6Plj/PAHj21792TdkR4cSWs4beNmm7t1Y5MNmg+A91g5+AoxJIfKzMoYqxDC+eLF9WEZcRJQ/dLnYil5T00Xy5QaR1WtqGMKUKKAgcchYUFYG3LgoNqB2aHC1JFlW/kmgtM4Rg8qFgxQKAE+dCiHAiEw4CMnaIkwglSYi5NRZTCZaQFbJfQZmz1ChypYtE0aM9gOFdwjSbfZhpwW9/fy7du31uHJR6t72738Gdy+81gq3f3hTz+46MP4eucrNSet9Q5Nf/VmPFleSjmwz1M2KePl5dk0TdOQ02TL1VpViSFSunYJ+L/4j38tpXz6w4+Cp6ZpiFypslj2RJbLUGuJvi0ZjhvnnGgSEzPLqZiR96Caywy/IzNmlmqOMWuowFRVjDEHMxAcwwFGxGrELgAggtKsrjcx1TqPmnhWou53kyfs6+hiMDgqVUSYJDj1zSJNsrsbqG+Wi4s8JecWh5yIqZQpNqGUAvN6tPcqjJ0LBYcY5cOPHi+f4eXtV5Pgove+0n6L2uq3b+66uDzUcEY15VQJzjqrNbYhDdjclNOmwvw3r2764E7XzfcuLn792XdvXHp6usoluYgKTAUiEld+l1MdoZ3LLDvBpmDdhGlf1ououY61TsB61eWxnkY+ZL0ivLybLsk9788k7yaqMYYpyyHZwSRD2q5NY+IQupamnL3HJNoT16oNgYJndlVzUTaYRj9apQh17jCK61xmO0tUgqFx1WRSLQ7mj7EyjmpHsVREDmxEJEOtVHnZqzXkzFJJqSI4rFscEoIfl3GkKOfr+PS9Ba/9Nm2WIUyrwma7270AzpbTMK1PV9PdNg2pbU+++urbom4ch1JqqjloXSzb3Z7TOIgQcby7vVmslnmcFovFmKfbq00p+dHluWOqkkMGu0ar7Xcj2KrkGGi7vV82C02jcwwU75yPjXdelH0phdSY2bEnPuIf1EhlDgalOaDKjvgJmuGQc7H/7mQ/SlAJOKI3YMcMeiOwqmqtJU/TmMkxQYPDBI2t93Ft6kn85nbQUlWKOq+lMiuqI0SZXTpg55yoVC3stNp+ytAt9d3J957z/beHZbNan+Y2sHThm6/28YO+odw6OmydaMedVdXrFxsvZzewb7+5jWjeuzxftnL1+tW66cfJY3nym9df/vjjOO2k66IyGxcPtG3/3WG4JTSrpkHz1ett56jOb+fgd7VGcz3C7Zu0ftxvpH53X827pdhJ16Zp75wVUefDMJbU4jZNbVhKou0oq45TSStyBfjuSj/5MNRcnFatGA5D0zOxRU+BOReRBgdSmSmbLQokZXQr3metigicReeymJRgTAKCErTrAJxk5IxM8zjZQTLIXOdpPJSzCKHDIR/WHXYj1gGHVP6z31s7ffyn//rrzS4cbrld9dv8pu/8kHBzvbespyt/e/u6aCfWi5TxdjcM4yziJKI05azaLbpXV9dN17+5uRbI5eMLcqJZ7rd3zLHpluOQiYxYJds0HNJ29+TxpXNW0wSjpEXVq7LHQ6YSZtiaHYGOSkeTCc3MUxgwU2JmuwzmJUoPSSLEnnDM6lJVrzxDAWbolPfRzKqpFHOMIkoG0aK1DvtkJZUkYAcLEK5VmcEcDFGrgxgxmdQYY7Uilhdrn2xgbfPEttGz9uLlZ68++NR1vd9P40kHrXUvqBo29zztNVy2OKHd3eaij9d3ebizJyfuPMbd/k2aZHXZxPPlHYbFk7C5zh8s25pFpXCU6BCSdB6l4nAQ2x0eNZdpd7i7HeOSzh+tZdxYtnaxAG13+2kQdYkV5GLMZQwREG19HJKc9M2NJFN3fZ+++aosIlYdPn7/5ISng6T1M96TMXC2XKUx51B2MDJbgSxJHtCumeBKKlt25KhU8T1uB91lbAd8+CjmpFwAEvWWtAqsVhjhvO48Ewdk0kRGEdXQsKekp61LVWIHbnHIiIaAtml0s9lSV9aXpdYm79XaXjXu0kDcBRe+f/68a3Nr09sbHQsjBMCZIuUsJYMdgYvKdjiIaQ9OmjmysDZtGMbSn3c315v9/SFwPFktbm9uiKzrOjAPuex2u/PzUyP1FqYkZJgJdUfvkhzX3rFwB1jxgM85oq/gH+ygc4MJTESe6bcQ4Xc9qXe+UjMNYPKOZdYgQUUJ0KylZKamFMOcNQiFNVYLERmbUpwHNR6FWcBC5IqZOjx++vjN3U27OM2DHfa5a5Cr+C4sFogXTdu26sbNNLBvhumA+9FEnzw6ff3Zm8fvfeyfZJ0OzHuj/OiDiC59/urrT77XKJnrCA37LKkIR0wFHdJl5xG6X7zYLdrT++vDitvO+1dXuzd3mz/822eY9hPt2gUoUBnxxGnaTG/d9P2P1g1k3NQ8ZG7Jg/ev4dZ0tnh0f3J7cz210ZfcVtpXD3VQRsO4uduFwKPpROgC9qNeRG6iMoXdLlnA6yqhYNFQPpgJTrrOo3ppyCoRstUkmgwh8lxLmVU2OHG1Cguqg1ZkmpZNW4bql0GiEaMJ3KK15GqXqmX26Se/f/nv76/v9t8SnT266EFkapKGl/evv/fR40cnZ8tIuTTX48ZMPBPgTVRBqeh2d6ig0DT3tzckslqv6jRws9pvti/upycXT8p+JMnnq8uVv1BCNdzvppc3t0PKmym99+Rxxwjsvvniay8wYqKZKPqfOkH1d4x28+JjuN8i9ghmSjPGkX/r16dZyAFlYuIZsoWiQqaiOn82wEysYBFzHLI4FxopGeDZcQwDhMw5Zu9hZNkzi4o5B+YQm2lMZNS3KxuGftmOOjS9v7mRacO33+xWK7z3o0evNp99cL6OQ3n04cXVcOtKevqo47p//0enYx5vy431WJ3FIe9//wfwSK1f3BwOoWHWcr5sU5qo431RJ3Vt6cOTZiR+mcebl+Mf/+HT7bBbPsOQ9wsqnY+08qPW2POPz87u0z5xynqgKosWDVG2YAWt4fV39eyjpJqrYLOr7oOYknQn7dfX04cfRM6yaB2Mh5LFkBSrpT8catuF60MaGd+8gp3iol82TJETT9mR9nBeIWqjaSXkCjV4D2QVotJAK6iiFEQ/594hZ9yUSQ15tN3W4gIxsM+HCH+fEjyI4MPwd366+uXP7gez2+Gic7HxFgOiX794cff40cWT8460rHVdUlZVBxqGYUhT513jeiGXC6XDQRRni75vYkN0vjp9e/22Vn/Ylx5Goou+N9JJcNYt33759ViqORSrlsppv/zhjz72NkcLz8f4Q94FANMHqq7ZTMA//mW+efLRyzcz+UgRnLfffVQxnnlbpI5MxQwzvJSJCM6zM3OOnIgYTGsFvCfUOTQXQedLLisgJBlVOTRTEQh53373zduLpyfT7hAjXnz99dNHlyZx9zq/+W7H00ILdW9GFkr10J3C/PD48qTuUKuq3bddG07UKFxfl2TjssEK4AoK8TfjIR2GZ3242k+tg2biBWLFtMkni2gYf/jj89WPoxP58XK1LbtlhAyoljxXL1DVpe2zSz6ATLrGI0nsHCm1sVuO4/cvVzukZ4+702V6cv5oN+yftM4bvX/G6XZaLzyPYlZpgiO0CwxTXXXxdp952VYzd5FSoWLuq5eb758uTrsIk2xFTQ9aVM08HKMForJlVbIxOiGBMMCtb5krxypakrlNsUzNAWW4qY/O0EKoyI5gBqe6bKf1Sfz4E/7lLzc50dPz93Ucu+iJacj15dX9aumdbgOCY2V2McZ171RWWXV7qFPmm83BigSCs1qnUdmdLdfu6n57P0hGYd4N5fSkJ7Za8pvrN977oCWX8bMvv24ZTnG+XvmZv8+zah9Cx+LmHXT0t7QdPmIEH0qimRhtxkfSDs2TPLijQPBoPkamOSptZqjbnKvJIjIHSc6Zg9XcA+xUYAR449nSVQw6D8BF1ISBsL/PPkBTUU373RSbriR/+1XebIrKQonqftzfUWybnNT5MGzL6clq2G2Wi8XykUfaLyq4CbXH4uTUdrdRBM6/vNsvFqfB4GqOVgLxdidXhraDLtE27hnz7f72TcbjJ2s6pBPBUuDO+ttxcA4ni+BTCSYnkXdVu+g4GzMVq11H2/H29KK5SQdSveiib5ssw5u3m0+fU6OGrKtAOtVqgOB0GRopvu3u7kbWHLwbxnyz0/Vll96OSy2uQckjmiA1+R6bjFGRKpzBF5wGcONy0EQakpEPVasPXCxxEcfewycOh5ytX95sdsNYY2xu63h+RtH8dijs4Ip6tk/ee+9we/3VN5sXN/zs8ZP7aR88KiiN6Yvv3vzwe894GL0PTEGriKgLLsa44s4nutkcVLFYBo6u1qTURt9cLBd5TKWWkfjN7eZ2f79cLczMRM+Wiyft6f393W63FQXHpobe4yHDhmbu9QNg3BHPxBK2d3fSd/TK30JK+SEfQ0TYAOZZg6d1FqkSSJhYTFRMNQPMoBCO9FIVJRM1ZQ+tR6S1ygN8wioYzOQBT0zsUNn5xmyE0km/9jEMk+SU39xtV80jl0uumXx5/Gx1f/3m5Mx1/Qrgm6vb7dX1e0/Op2G3alNfsMg0jcmryT5HW4ikg+ndVIJQ2CXXp2ULzdohItKGRBsbDuNjwkVEavBd2p4A70fXJmxRcuSUtdPa7zFa5RYrAh2kiTFrdoRaStMgozjoitFLDmo/+83hRz983qTXQdD6OOasDB/ZqpoJg2rKq1WIRqy8CD0w7Kf6bAGRsQlYN76WFAJGwQRMASX6xvfjbt+7mII7qN4XvRzUd5oE1AmrsMKxeUclpzHpOA67jW73yGV89nw5MK1rjg02e5xdLnXPJPHDR+9dX3+3yYc325tFiJpLrUlNrjeleTs+XTfTWCGFmZ0PahinlKpL1W12B2Vw8GKVPXPg1Xr5dNrvIqVhzyF2p6tp2h9Sur6+Ds6XKYXT0/OT03GcpqyFmsK9DzT7r2cKCD9snCbQo1WUeeaNg44BuXOWKJjYeaIZCThHiJpn8k6IqDDmmLQu2VwkVZDOyTWEPK9FsHEQ4jk30hNgQkxGIuagHmihpbIoN1DvOfQtjenWEU7Pz95eXz19dul5td/cmcQQsLpo9H6T8qELfeFehu7uG7c8C+89egI+xL673eSTq8Wgu2+movX05o0N5e7p+3TxqL9/O9UrPP+4YU67xqXYuej2h43XKKgsyC5+Z92S9DTsFhMawhZ06IL6HFRzxSu1++WiM6gMbZgRL7lvSNWSIA3eqF06ub8bDw0clcbhTP2XL56XLLObGaBcNRdJQptNKTUGim3Lq7U+fqSPHjWPL53LueWp99s+5k3BoaIAVnGxgvk6lq1bYqi1FHJSHgdMhEqru2E6twCatqY3W6/yyeY13w9v2uelP8+ffNzTyE0Im93m7UojoXfYlUEO3Db6ZHn6h88ff/7qejttu8Xjmqw6FeOsEe7xWPj17nos427YLUI47fpF2+dShpqKpaJYLBa9bwlGcOz9JZVJtDgnTK1Lj07UWV5etuZ6Iqdl4NEuGhyschwv1qMX03eBuzYjqYlmHDZ+5/GucmczAin/9kvcfLO0OTLAZI6wmDObXfCxznAMDxZQUZvR+Y6dqJhV2JyCrTALxCbHzNbZzQ0imIc5ZhKdjKzrGu+cFDx79txov2g701G1tu3Fzc0Nc3N63gkqQkKgzSZxPPHBuSZ8+9XL3UbWC95u1YDT9WKc3qxOV+enJ9cvX9XiF3232ewWPb1+lR898mrjnE8P5Sjt21fD6UqpOu77EOsw5a5bXm1Hju1+tw8efUOWyiFXZgfzhqmJ2KSZSBw4OPhUc3n8hHJ1JufrU/9v/u2L8b6qwJQJnjkeuYkKoaBWRTEdZHOYXr3aO4cYsWqxWuPZ08X5eb9Yhulws17z9jDuNlifgwq6VZ8sjfuxibHSwjVT4lxdcW1Ltd/e719d51w+u1h+cPfm9gmF8/PF/m5oa3t6cvHi+g7SovFcZRyrbKyehwxdn6w+DO7V/aGWvFqsdxNPeWJH33z7eSOWVfZl8J6pFC4y7A5d34+1VAMxhmHyxKerJbOfchFitbmZSSGEGCyQN8/qGsehpmCSXXTd0sQQwB5HlCAJoApmAoiZjITfJdjZbzHmzjmweTiDzXmSjtl7JpWZzn7kPc+ptm4OODdA3Sz2A0iJzKmCVYrqg+dpvuUyyMEcwDMVnc0MTsUD1QUTHYniYW8GunhE9/vbD947ffmtnJyc3d69ViUfGwTLtP3kx5dE7vbNlJO+fHnzyfcfewvn60Xb1pLKzU21OnrvL84fvfj25rClJrbBI5dpd1+mCZ7rYhX6rt3XoeXFZlOXqxbBBrWo7ZgGkH/95t5cWw/mtTntmjRuT8/irtYsDtZthykKFq1n6qwIdFj0aAP2B2M6/dUvD19+nk3OHDYw7ywSeaskMMV871Exc+QYHrYw9FVyHTGMw+0QX944kbtnj5of/uB0tZLGjU1DrIvWI2+Houo97g8585mTHXcUVzjUQ5TYrlcXTXn9ehrq69U53l6Vj067yOcvv7kxmTTHt1+n0wXXcXJw6aZ+2KCNZb+7W6875f7Vq8P97UZARbHofQzki47j1HfNdn/QgqCoxORDLrVkI0Kuut0dmNl73y6XQq4YZVErUtRUjILrmrBPys6CYyMm9sFITIMnLzAi0gcw1AP7no7xK0pgM8K70x8AExNB59xAIucosFNTwL0j19qcYy5WyciECIwZgULekydYtSxUxBilosAI7GRGsAFz0BPmug1ksCJ1psSHYCbkOKRJ22bpwu1Hn0QY3d4eyMXF6eLRs+bkItxtX61XZ8rTar1umkXNh/WqNY2ecprycnFhGtpm+eWvX0mF42Udpe3p7ORSfHryaKGWYpDt7d3LO/v0e0vIViM2ZXCMw83te6fLu9vJHIilDavpRm5HW52f3O431chF3uRDJpghD9Eh3l7dXD7G9gYMBDz75S+3331XU1oDS09G5CqYHmjEClIouTCLGsUICAIPLB08aFuluU9T3y2/fn0z5Zvvf9I9fnIpdRymfdv5cdLVyt8PlSJuhzcN0JqfAOetlCljsoaffXhW96yhfvLRo6+/u7r5SmwbrzlRT+vFheTx+x99+tXnX7Qtmau1VrJJcj1bnaAsv/32PquvoglTs2xM62rRT1bX637aD0qILij4iAMnkHMp5zdvr+9uN/3bq9PWHVKuYsQ2A/ZrLuQpeF9yKunQerfqOjFLpTomLzZHgpsdgyKOubjHRhITgwkEVp77pSowJbgZnDwf7u+69PPuO7etxKhW3RdjQmTynh2D59YpaRsjqSuFDAUkIJ2lWTCBuRlwiiPausIocDStphHGxDKM+zevrVvwk6dD2zmtXkRFJ9BUNb+9ulHDfTncXOcf/eBJbNLjy9VhX3f3ZbsdlsvlN19d+9Cfnj2dBjLnd7sdrIrYsB1+7/d/nMo2pbQ/7Kto18bpMNxcHc6fLZxfGkoISXN6vPbF61DLtNtobjQsv/721eIpCizYeHbSlAH7AXW0PIzvPX8S2i2MX3xFX322329aAZgbVanWw2bijTiCHONv2NSAADgQA6Tv2MZMqJVjP+bRUXy1sfu/2Xef7/7g71w2vY5pDA3u9jUuF2IJuX77Hd5buN2hWrTGoTtpdts0DkNHp6nI4XAoRRb9+f62SuJ+FULPCvG8bzs5PQ+LdbV9iY1PefKez1aL+uTkxc3kXROcQmrXdbtpn9JUpLZtu1wu95v9quv3wyHGAO+apskGEmHvhzGlQx7Mk2PnnGMfI7OMtRRzZqaOAFitqVYlQwzeV/ndbjwIxx4nMR/5vO+w+AwQ/DGlmNzvEJ9Vj5dSOsatOmPUUmqVAQ0Ugcw7CY4cW+tBjsHeEXsKzpE6I1bHrhYFDFZNYUZ8pLtVM8Ux8KypVUJk1WLabG7L3Z1Ijg5xvboQw5PLx6Vsp6mptX780Q++Hn51e53GvB2G/TRa9GfPL06vbzaf/vC97U4Oh2G73Q6H0nWdY/bkjLDfD/v94e315tHjCNJnH58ebm4vLmPr269f3pyu1zIZeo4rGsahWUaGH253Wab1eqXCy8u62xyqYrlo83ZCbSL39zcp9CDuf/GLuzq8rwiBMygRTYI1rACC2YIxR3XAgTzm/A+tR+D8DHnXCiLNFYD6xTAGW/gyDv/6315fnuOjH3TtymVf3745dAvfNov1SRD1i9WiDGm7Oexfp3YNMreb9u2q4VCfPF2/2Q8XT07YiSPeHq7OF3EcX33yvWbcJeVd3y1C0610CdZhms5Pu9d3E6iHUsmStPZ93y4XuRYA290B4Kvr2yziggPmbGNlM0dMxMkUzCEwwawKwROcao6NI7FcTaR0bXTMtWhJyZdjprk5EM+Lz+CAOYTeEc9CO8Yxt55UH+acv6XoqioczQFPzF6J1LSo5lpGbbUaWfUE5y0yanQWmV01CjN20DlnIOI5hMIRwaw8lG6sygpRy0SOmaWCiJkbqT6nevWq7ZqT/S7d3owhxL/+829Wq9WUvHPhL1+9IDp5+d329Hy1uT+kgtN1e311B/Z3d5t+cbHfp6Z1IlpSRgjNqmm7djhMN1cj6Wpzo4rMT9+eLPpGuKQpCmPbNXROpd7fvvFnfUV/mA6X73Vd8ESnL99Od1eb4NrdXfQkfXOeStwPurkvp5cnv/7sdSkdsUEyKDOZYyeSAZl1DQTPRnNIDYxn2AFgQAUxeC4ZmQBPUZWlGmIzpkFDGxDevD5c3Y/vf7o4f9YPeRpG57RRv7ifbp88X5oFovXL7149+bDzwaLntosp70KbOOr5+qRMGHRcLU6sjk3bMbRtyViFuBxq60LTEdtINrUtNuMo2ToXfEBgZ9GT481mF9teSpEpA8jjJIBjtDEwCLWQmQ8Nc4gGSVNKRTr2zExWc4LjEILkVGs2gVQzM28zj3Sm/cFmh7PNi9UMTARSVZlTQs2YlMwd1STvJE3z+gIILLCZha9GajTlalIJNRNcpckhF0mFhTxRFWdZCQZROcbusjIx5kRNsCkThdkqbSh0LBocE4/DsFovQc00ueubt8whJ428GjeN94uaDuzLo6dnN/dvDvsa+z7n0YXw7dfD06eP7u7utu0mxiWRLVfNxemTcUwvv/v6w48evXlzvezev7uZ0pDWF0vbjvd1d34SvY+PHj398mfbrclrbC+f+tP1+duX45u347PHXpZls92Nh/Z09aik+OLb665x3u210gfv/eQXv/j6139zpeBF6/N0VwTEDTSIeOIEAMZuvuz/Nt5udphDwGaO5rjDY6pSMDibX4E6GtVSTBGCuzDkX//6+n1177//o69+82p7u3NuoG6ClkVcN25hiuFQnr//rAyTQVxk5/Xxe9F2uW+b++uS7rBu1vv7Oo2paVtHaBhadcwTwUcXtvspMKlmGAcXS8kUwqxdX65OpmFsukVVmqZpJhjXnIpJ17SAdW1zlwYzZTr2LIm9c+ac8zFU0AxtbJu2pCy1qNqxSHLEmAPzFNUpicwo0PnkNzOAxMwftzo2MxV9p0o2k6Iz0FGIfa4VxOTYCIIJc66aoYBQLcOmIlPeOo7ccGgWQ6ogLyjROUESYSLHDHYCBiqbRiJhV7yXkgXoCYHZT9NIkYZDYebYuKxDvzjZ3YuIU7XgLNed0bBYnp5erF+8ernf71r/wddfXnerbtrUItd9H0Pkm7tvLy4u+iVevrp6/t6zze3QtssiTZ7q9F08lO2isQNNtS5H40fPHl29ul9i+ZsvD0btYUP6+ORuu20XJ7tp8Gnxy7/5qmTGaVtqWq1Wt5tDySD0AQRR74XJ1apqbBTY7UUsuKAizkdoFbV5tHEU6IIBPoa7HwHhXKGMzEhs6kAKTw6TTZpr3118/avD3ZuXzz+5mOo3421hxdWUr+r1xx/59z9Y9ifdzfUu6GI4DM2K7iuWwULwoW1v70GcJjMn4qM+eb5oej6kqQ8x71ITXck2HmpKdblYHLaDD9Y03TBN8KFd9FOpSXRKBwZ1XZdzMsARmjacrRfIteTknJsOBxfatonX19ePTjpNUww+50whllJCCMOYgvMuwKBeH8LJiMjRzHlmmRuPM2sURg/FuaoK2fEuBOAhdvcos7Mjn3RuC5hZVX2Xm2N4IPASG7SMRpS8eAHHrnXkJI1SRo2F0cGCioKKc5VdMEOtFr1vO+ec5JTnmzMzp3RHaJj5/Pz0MNaL83i/e90vL6dJXQNQqiKxkTbSo7PTm+udK6exXTpXukUkotiE7e7NB5+csds947DoLm6vDxeP159/8blpXPSnaVJz7v6GXL8YB4lN88vPfvH+++vNmKQ0JRWiJjaLF29uLtm34fHLr64fnZ1vdoe+74ssrq+3u939OFUpRuaY/XyPUpCqAMXM5ghwBgwTWM3E4BwHmLM5rHVOZYOAVMwBFVzJlQDzGti8Eu3zjnqHajWHji+Hu7tv8VV/hg+eXZZKVQKbHzZpX/eH8XB++n5J7e3d/snq5PJctjdbsRKITy/cuIvjpja0NNObt4XPK7g6yko2pZKK5mSlCHm5OFt13h+GkZlD0wCsAvaeHNeUg3Nd1x0Og2htYzjsd+u2VzIH8448mamI2P4w9oGs5FJTIBbAcmEQIGbkQvQ6RywQmEjBbo7WFSJSMWMF87tkOtgcW0n0LrePzGYR8xw2NW+o7J2K2hG2PjupMYOgDbPSHsFDCnIZU2E48Y6dE+8kw+Y0ZTM4GDNAVRXMrCqqSlzIKUygJJUXbQB1yezm5ubx0z72NS7GfbmKbXfI0/mjywtvUHr75g1KSLv0+Ik/DFptb+TV7Oz8/Q8+/sHrN7969CQyWU1TkcPj89WHCK9epPVZPFsccrn8zVdv16cLRuqjv/zosQv5ybMnf/lXX+QcXWvN0q1Gfv3ypmxOg5PQjUNKbtjd3Utw6zTymODgpXqCc87rfGligWOam2di3s8SCKdkJrO3m8ycwRTHJQ1SZ61xJicOZNIWnRvSlXxjRVBjLuoiPFy6A2WMsptS1dpBm6rJda5dyaLRmhOUNteV2ROiAcOY0+j7vhtut0NOLrCgPrlYlrobJTe82A45TTqOxVSDw3LVyDQa01ycHIZpN45V0TSdgkvV9fokpdFg++HQshvThFyCc533LgapVmH7KbdNk2tluDRlmROOoVoruQY6x+jNIXcgm03smNMXZ5Hog/X4qAaBHLX17yR5x5gxdiAyJqcwRxCYO95Lj1uoEYzYiGFQzLF4AANUoVpFlcEM4gXIwHVGpCA7I6ek3nOpJSUjEjNjciAvYsNOcz4we0R5e3116U+evP/Iu8VXv3nx+Ml7w6i7jYyxPH92OZThycWz7fhF03Z9s3z23sW3L7/55uWXzRVfPl6M43B2/jjv+5T63TAMY37v/TMtLqzTdOfef+/Tzf314ycLKZuGCDbVpE+f+GprOBaR7VYlISc8+ujxZntrwMnJeUkHKf7+7kCuV3M2EwcsAwIygnNQoiiWVdkQ9d0x7qAyZ13LsSv8kKatUFYjm9/nXuFtzsKGRxUOrUka06YNrsMF3XOJjFICN5OCscQk+3T3m+Htcu2KZObld1/sY8P7w33TTCq0uT0QHJjFKE1ai8+JmZmc24+1pFpFfETrjTSncfK+U0NO9fb2djvlVK2NI6ututY7Wq5Pbu/vp1SEK6p4o2O0vNQiJsA+1QteVqFFF6ZhqtC2baUWNhBpLsWLzV2N2ddi0Lm7Cec8MxgEMn7oJ/12nb6LsaFjdPjx4zTfCI4HPxtgjDl+j9lAIJ6/1TxWJVUQ4I5yv1xhZM4Lu8qOpXKuxMTOmyETq4LYnFRRUiYDXJ58aGLVDJOmWayWl2+vr1La9QvXtGE3Fq197MPd9cE5Wqzar2/LT957WjJ/9vl3i5WvNRPC9o5Tsi7EYU9vX++naeLgSZkdb++xWpxd7bfkDvfT7Scfnp8t2zRhuQolH4aUXDh78+09VXf6aP0ybW73MoxlmjxbW/KNqTBIpAECuMIN5EdHagIqwQoheOejEUoViMEYHMAMKwQjnheoGDyMYGxICnXH4G41FrADNaiFCWwTXK3mS3EBXYu+mDImhRAKs6/Kzs417euUyfv9bZ4G52PjmqY/6e82r8dtYLCfc120sDYm7TjmHHSqEDMhCZ6bwHma2LiohhCzUC0CckYyluJBIVcpdbnufAxTLRVgR+t+YVYgWkSmLJWdMh+qokpTyTlXs2pRmDpHpaQs8Ko2h9+646XOmJmPYiNyxCCDzkvTiEhMZ3kog4jM0bHud8fYFGJHajrHSDueY/1wNIoc49mMAE8QoM5x1HNUCmEOqtNamS2EyKF11ZmyVGU/T7OgRjBWdZgD2TnkUs4fr4dxM+zrr/7mG3PSL1zw7v72ZtiQo7Dd7D755NHrl68c93/8d58ctvjyN6/Y6fnFxfPnJ467t69ed/1qGujnP/+StAmxP1meXN/dEt3vJE/L28UyhrW/eG+5LRsvJR32JlULXr3Y/PCHH/7msxfLVedP/JOPF40tD/tbpva7F29Mi2r1YakFNgtimNjBucJsCljmKuwcihZY5aY9hlvkDDjMrgKaUw6rwcEcKPO8WKHmJjCgHuoJxFRqGck5x50pT0DBMKXJkYcKh6q6BQJhISVOe1muYtMs8i6htMt1s+yaNO0nW0CLX/huGTa3U0m54UiBimklLVrYtI0xOt4dKmsc84GIVajWWhXkvFQTwzgla+KU1bft/W1pPTyQZNYEV5iJoSizYbMbaNwHbVenZ1XGlFIgNfY5ZwFxfbDJvdsg58f8ITy0P9+5OOZr/n/yERC7Y/Y3M6Lzjtgzk+nRN0ezN8Rm0R4ZoCYFDHhmkIcFqFfjKgAXmKmgFDFU5yuxqEJrIHjVOQ+KYUEl1OqULS7aYUrjZCU1qIuG+mEz7e4Op4sTKdb37dlFf3P/Zsj55q6UrC9fv3769LF3EeC+b1YnuHwSFit+c/XN6TkunzQn5w5uaHvrFrU504tnJy9fv3pzdaigF9d1qPFuy/eHBs3TvdDVZnz8+NEH7z+6u7pqQx84ff+jZ4uOqiVu6Mnz91JJBGUUZ8alCaV3lYiqd2NoiicndWIaunV5/lE8uUyhPYBGUH4YDpsjc6RElTgTm5KIVw1GDkzClB0yWYKSp95pI7Uo5RLT5Pe+y741YysiIplImJXhanb7rY07O1ufODesVvTVF195OfGuFZFShylvQLlpnWcxJKVJOAsygDY2EEb1ml0TAhG8903T6EwQAim4qImaEAucazgJqtkwTlp0bt0rUEFwzSHlXFRApBZCdLNWTnWOqfci80ksjpzBQGxzv/13EpkeTnbDO/HoAzmXjkHUZGYMhoGZIXN0KcjA7thkVSMCqr67087mJIYQZoIlzJDBSgSToAK1iX1iXoWwKFkIBFRVPZZxcARuVuSYxrFGfzKNAsZHzz/8+ru/Xq+XpD5ym6eDa1Iqe9FIfll1O05l0WdQefX69d3Gzi+JvT25PF+dnEum+80QvH9zdbU+61frdfLbNGx++KM/GHUDStHzL35xQwOa7nD2weMSTj776qv3HruwL+fLvu5D0w3k89P31qmWrl99/ZtruAjJBDAap30oEW6oksUMPHh/XpP5pl4+6p+/33zz3V2eco3Biid4xizVrcY6y2ca3xTTSqZgthi1kGVDFYqwhcEbMtOgIcMBbVA5aFXnO7IOaKRAtRoKASaujGp6++R9/+LNZ54XL77Zk0QX0LSsNviAtmErOUTdHvbVQATH5FxISdhiFQJQU3aN67oOY4YZmAlMIBHLtSxOl0NJoDKMtVt2qqKKaqZq5KntO09l0WgIfpomF9q2beskRLRYLGQYvSiD2EBixjynUIPma7nOTHSCc87NVZQwMZm5mVg6z4cdgyElRecdqKapcZEdT1qdt+p8MKy72LCvBdupCIcxJ2aF1MDeCFWNwSYCOFI1BazMjVytIN57B42mBrZgEkwCsRcrsNrVfpd3YRGBievY993NcLV8/DQX7O924NItu9PLs29fZThhf//FZ1fr5Wkp+vzZs5cvv1mdh+G2rJbd3SuEpn/zZnty+kEptLm/f/TkItV72j3rTrcnj341vh2/+RXa8PGqi/Gkfvvdl+k3L99bXtyPuFg9r5J3w3h9tX3yYcj5vujwyScfToMdJoOuFZ6QHZekb81ZDJoLyJNJg+4teavm89i8/npbDyloFTPTzhAJnl0FV0EyIs9e7D5wjGileBGrZsREDo5F6hYayJyZ46Ik1VtCKIgFGNjY1BOzFG/qYZ4oFFPWJr2xVBdNS81ZKVcmFu42cnL+qHX3nVPX0G4z+eJTVrCxJ5JMtcD57E2rso9s1rXNSet3Y2GmKsXYKnHHDXJeOTb4ZHXQQ7FOuc1SqKGOtdXDsgkL7kVz1QxJCoS2LQJKphb9u6JnthiZQZn8g7fTkWPGQ4X04DA+9pzmvxKIzHQG2M7JdHPD0zlXxVonq65/cnK26Bo4DClvU7q6SuNY1FS9VVN4qAIuIOM4VWUC6UPcWBUbPHszmoe681MFIvvmsE9N14/7ZGYxtLvtsB923bI7bMdldwq1zd0tx/To/Gxju7TfnX/4OE9yc3V9sv5UEUpavvf86ds31998d7Vej6cXp1+/+HVs2gz96rtvHz9ZdjHtNkq8ZItFNx+8R7/65RddXLR98+HHz3a7Tbq9L+heXr1x3K3Pl0N6c3p6BmrvN8N4YJQMBtQROGtum+iDpLIzB4WpG7k4pgDQOOZcqlY1duTdzCeuJqaFIAomcqqeuDEEWFA4MbO5d2LCxgqvR6WiAzAH2MVZOQGaSwAjeP8QAS9STUkgGZ5c3y7Y6UBcwaJ62NTFsssjYdRcUAUGL2pgPxUqSlMVAXvnPDt2FNl1XSfkDWw5AcAcX1Q5hBC7jg6U8igipgpYG9vonSeGqDGccwIFkNKUsvoYTclUvcIYPFc0zDg2OwFi8seDmECmavO//g8efEzAPS5iHGtOBeC9z2pnXX10sXx2ebboGmUrlIecVkt+8/p2HMpuFPi5iQCUAvNmEVZtTm+eW/wE0+wbypm08hyCDVKYM4nEWYtGBIX0TR+Cq5LyIUUf8jgwc9c2mss+HZ48frxrh+1devz4MSz85svvzMKw7X9+fX1yuoquqpSUDz/6ybNpmk7OuKp78d3ue8/RNk/fflvjwv3BT5/s9t+9/2FDurp+U9cnoet8iOH5h8s3N6+gteuas6ePVOX+ftP3j4dDRugxDkBWWHChyuCjcxzNkgs6FQRTzxDhOqk6gNh77yIblDiriCITCZEzhQpcdAYP+Hd95ocalMBEYFPCrPXhOZJ5bq/iqLslNZgjraXEELmaKZxRKrq/O4TgyJqZiQDoYZfHodbdICJCwciZKfs+l6pKVWbwPFRVSqlwzNwGL+RFZM7O5LmYeWj+SAVJEVOK3LehayNrnd1wh8PITEVLnSflcABU4enoXrfZjGEK8IM49OFxlCkd50EC8maGh/boPObItQQwABGpYspOCSGEJ8vu8rRbL1v2UVi9j91quey702VzezO8eru926aaHAQe0oE28yUbaob5ljmrBqBkMitLHXkFkVYxq77BNIwnJ6djGvfjlpwRamAHZXg3v4hNaOqh1EmqmLcgU877veay7C/2mxJCfP3iNjZCpPdX++XCLdrGBDU3Z61r4/3Fuj/cbmraSuI8FrZSpiB5eP3iq5NVE8lv73d9CI7Wt2/uh2H78Q/WeZLzlTntuSYKKuXgECEWG5iMpsFRm8q2bcWroQgKmYtHgjMMCs8mlNlXQ5q1uFJcUTGpSgSdFyIIwqTzdMWgMH0n3iMIoDMo3XRuV9XjeB/R+aNv0QUvIm1ol4sepKkZpjItlic5JdU6TIcuzGBDg8wmSidzINIsY6nVMYGI2UfvqljKaRwnZnLOe++ZyUzrkcpNDNQKB41MiyZYMWeVYW3fEyMPFkOoVcUsuugcPBGpmVWZf8zGsWPH7HFU0x2LpOPuaXRUJ85SESiRn3vIZsaOH+R2JCJg9MvF+aPTGNsiFhrHHOEccW1bfXT5/6/qTYNt267ysNHMZq2123PO7V4rIR6YxlYAlQ0Ix9i0hlQCAQlHEJzYhJRdFTCO4h8Yl386ThwLC2FsiIoyhMZlguzYqThgKJdNYwzPEhZCTWSEeLrvvfvePe3ee3VzzjFGfsx9ruRTt96999179l17r7lG833f+MaJ9y351c3+leAbx4q5V8vA4TYuVPEjAjjTIgUJPJIXUzMlNGRE4G4TS0njOGRJYRVmndlosz69fHyjhOz9YRpTkuibTz28LKibxbLfD6gWOIAAiBoKmZa5TISb7Z3dBTwaL4rANPTzSG98Bpwffbx47pnNON/oBMvu3sdfev3OabPqFp/8/fOuja8/enT37t3r6xJ8t4xkovfuQr/vy7DVjITmuQ61oHONwgTUG8TgGy3JQS5W0DwUVPamR1N2ASUipPrhAyiiImp1cEMwo2O3SRX+MzMyBaRyvE9VSnvczle9im7jC4IZAkffTvNc5oxMWvBwGNq2LZTI0Zh672joR4V2LIkYzFQ0g7LkEgjMhFFUzRBVFVUNCxE5xsAUPIlqKaVuHGZ2T8IcM9WlqwgSPIuaA5JcmFlMyTkgNgADm0vmGgvNqp7OEMmQVEBBq3zEjjYN/1FOrx5NeFxWZICEqM45uN3wKQY6J0C3XC59d1pEctKFNxcMpFg57oPfnG5gVRb9axfXl5AAV8ACfIDjsL0CYAFkrNipGTkjhLpuR8wQiJ3s++R9LDM47wyxlKnt1k3YLBbhut+DEvGizLkkMGpd9MM4EFkRZQpDUkPKZXYez+49/fj161dele12vT29s99fqU1vfNMbusUr0zQYA/kkU7PdPiW5eeZZfubZ01dfeeWzX/iclx8+bhdhebJAX26uUoYMsNms7r7+8mNR9GGVU6+aEBAhTrNSVOz0j37pyb/59VewbJNdExkQ5nz0+Ef0RYGIARBBQGZFJfNqAZDFRlAECwBceSewDCp45AK5hhUBMWAEBcgAt6fz1vMALKtpkuJ8wMgpZUQHoMMwMHcIAOrmYWr8sojz0CIWUFARMtMi7BFMATVrAm5UVVJCUgFEM8e8aOJ+GFUspeQcATtVFdEa7Kq3r2khKKKFmHPJGTFpIeeHKYlIG6LMidi54wOGT2aPQcFUjZkAAKoPyH/8pWZkdvvXnzBMZNVp5Nb9BoNH5mHMJmpSAhMasjMAYg6CZKzG81N/pFuo7vvpsJf8KnTJiaAAiyVAQyrHj5fEwIDEgUMjVQRThzkPCYhzMQaWMSEQCLz26uuI3hHmaYzeN4t2OIwIIDk51FLEjHy7KBlTmkLD25Pt9fVNSbBc3jnsxvv3t6Pri1y9/OijDvjOvfaFF+69+ui85PWnHj2Oft10dNinYRCR/dm9VdPC5fXLsV34RfasoO3vfOARWrNaN4ebHUCl6go5pwCi5c1vhrvP9yf/Ac4fytEPALMQGoAggxqiN2VRQDM1h1ahX0Kr1qN0+8MADJQBDInqKlpABlAQVXAEyg7r9E3tlY7RFIFBvPMlZ1M9VhApeR80ewPWJGQOi5qEIrNynaE0IgJUAahDEGYgpRgyADhmRhRRERUR731dQVi30tSigIiPqdGglKJFJBcjAgDRIiKiNk2TqqKoRyBk570rpajKXACMkNAFj0xAYHQkb57k3MokmdSB9zrGXrVLqNkQ1CEVsSkV8By8z6J5t2M00qJewHlyUZkKohGSMw/YgVsuTq7a4cGd9pP68uqguXCaZb4VTKmBGRRB0dkbehdjbFBINDkyz8rgGg5AOtvMTgIOhQFg9gjN0i0WzeXjy65FQTXUB3fv9n0vAgi6H4eG5Nn7Dy6vzlfed2vK6TJAPn/lI87Nb3wq5jI8WJ9ikcefvJwGRcswDHEVL16+HC9CCCzp6qDz6wmef8Ma0RQnJ+7q5UdL4jtnZxfn1y32LrZqCVDMdkTuztN3722MZ+MMC5cFIc013wGDiCRQdM47iqaIJqqlWtWRAZgKjQgM4I7Ju9IeJrdNqkOumS0jKTOSldsxLzCFqvatN1Qhe2JCylqYTREIswJYgXX0JsVjApmaJkxTkdqRkQGBmCqighOr/18AgJ2xYw9EDrz3u32fDOphFTB3bHpUnavmpqJQxIoKADBzP40cAhFtT9YAMO4ObYyg6kIIqkVuDeyqzWcVgD4JmbXSqTE221z1TZXRqaVoseKQpMhUrL6C5zDO+eGjT6Dj+6ennnQaDWg5qTcCcMbOouMFdA09lwY5FZmnw9N33+SXzdDnlx9dXV3bXPJcDIwcd2bmHBNmleTY7p6dtE1zGK5aj7FpilBRUeXlqtvv9/50sVmd7neXTz84PYyHVdPs9r1zbrPZ4DytEJhCP04vPH13HPtSbt541jVNQIJpOqhh1zUlw2bb9YM8vYpzKm1z57lNp2DPbefXHl/ef+ZB4xvAIjo07WaWeRnXKeXNqimDhnsOgRmly24N5Fw2MKZQBJab7uzOclEC37jPv7/5yOUnbhK15EoR1qmJrm1o0YVF26IBqklRM3sC9SEqIDMzWN1EKLe9OaoqWPV0yQqmWlSLmmQNqqogxzFuOI57mcGxnwBrmJmJyRORQxVV5hI8ozDqISdDIi1ao7MgsCOxOCcRv7A812qQHUXvEumcipbsA2cVVagG9WDHvCoKLoRpSj7nrEbo0pyJj2wlMooWT+wZPXNsGlcfvqPSA49TnKpqRGagKlVCpwhQE3fFKurSt2p7WxEEEQAqpgDkghfTfprmokbWlzki+mKYzXkEx4wwjyMmosIOY7DABFIYNPU8awPPPX2qli9vJIlgxUYNAJQQgDS4vN7QugsgdvfM7/rLKRsibhbENnucwOSwGyKB07hw6ZB3p9tOs2DuoSSdC7nQUEn92EXfrleEfLM7b1qM7tDENs1p5RfT5TwPZQePY1hG43zYKxZL5XThmwgI0/n5620X2C8+6/6zn3r4cLXa3uz7ZdOWvMtlYF7m/X7BhijDCOCwa5pIUkYYy90YVietrZpPHHIEUVZiVEtTFkg2BN0FxwBKpdSwRwAIjMjeFRJG5KJFDcwEb9W3dhySwRptFEAN1BitVMUZVAV6jaC3Q3hQ/Y7ACJSASDOjBufIFNGKCaI3q2IqRRfYkwAUAHUNGNk8o69MohXJIMYE3nNRc0gJjnUnwtESGclLLsA4zrrbD2erpaTBIXeLxZRmMIjBO2KK0UybEBw7ZMHa3h1VyQh1Ds7MgMnbsbQGuiUpAUHxuNm8cqKAWXJgBwAqgIxFzMyWq4Vsly4unQA6b4Bqs7NiBVr2DXryyIipJEBUQFFvliI58Prg3kkRG+cdO835QASMFaGA5da33nY3Vwx8db0zx8jchJDngaPruujiah4mKKpZS7GzkzvXh8N6sb587WKxcqHhlMpi0YqO3apBsDSn1Wa92792dmf7+PE1aJtGnEZw7jTJvkxlnOeiU2wwCzM1xUoq+9O7nRmrQT8eunUAEmbONhQpuZjKiK5MEzBBjG7OJRCmolTy5dUr2+20PvNn97pXei0qLnLJkguIgSqYSROVjy5DwHoULTMjVYjYVIHEpJq2VsENIqiZVdjPDKym5Zri7Xjjapl66x9jJkeuBQSBTAEkIokzX2RyCFKQ2ImiGQOA8+wcJU1a2ExUIXYtAGBVCauKChI3IU6pEFH1TfLek0HbtmkaZ0NRYMfDVK72w2a1RvRipjkzAQcnUopK27bDbn9xdeWidyZlFpWipmpGSIRMRsC3+Ort/CYeq2zDo6tY7ZluWXoAqFu7RZWcj20X22bfBkAHhqYspWBKAgSqOUMG5wNzw5kKsFPkROBGLpbyPG3W68cXO88ADkzg1uEZoofnn3sw7q8dAoUoDI/O+/VJd7NPC99OCfb9YXXSFkFIcL0b20Uzp7xabfrd1C22Y9mVDOzibsypzKuT8Nqj8/v3n0GCs3tvUEtTOhx2k2Pftasx5wQsiacEWSGjGhIpdNF37XIYdwzddrM5v36tW7v9Yb9c3p/Tzi8W+UCABC6JqkIQ8+iaWaHzC3N+L48gTS3eu/Pg2cXl69fX16XOzJET5L6kMmMy59mYjK0iKsRAjAxSXQzAbg8NVdaZuJgq3s5+oimqqBhKxaPgiDxV1dpxPqKOZABWFbAgYgFHwGqoQhlRjRQ4q4kKsQUmJsUipGqKJQGHun+DwTTnWcSAtI6oIJkVEUEQreN/OUuyhGhpFgWYk+4P03bVoBXPdTItz/OsWXlBRY0RXYi+NlB2O39c5UvOHRUcVdgtVgUajIimCmpcffAQVcFAEVHEGAmIESC0DSMCuxWTU23YeyLPSIiMqiDeo0kpApI4GSObalItRgSisYuqhQxO1ttipYvpcEioYAoP7myLpHE63L9zd3d9EOoWK54TIndTEQCNi3XXLc8PF13ToHP9PPmGL6/P0Zg0tKtVybrf99H5Nq5NHYDLU1YtzpFaOdk+IOhFlNy86JxY6QcD6IZB0BER1PlERpbCIYTHjy8Wy4YMcrLk0pyIMe/3++eeefpw0zt2YBEQkYmJF90qSyF011eHB/efR8Nn72eZhsM+gSJTVPBJAAll1uAgMDCiR6N6zo4tdO0SBIxqfj9SzxVi+YymVrFudgcAqBrKJ/bDYIZoploZF62qSEDwwqjFAEhEAR0rFmBUKI4RQUXUBAjRGSYpN7thtVq0iw5UJCdGMLA5Z8+uqDoH1cWDEIPjkYD0qHMHgyxwfn3jHXVOwVK3CArA3ByGScHIeSnmmNF7b0VA1USfBH9VRagKJbTbSIlHoxzDT/sz2tHUnkBEONQVH+icdyGoWUBiAkeAKgpOihZEEwUrDh2QgxJKVrTCUBqSQacQAhLvb/q2Catlh84/evTKIIkBNtuua+Lu6lpEhnkY0yGDW203j18/N+NpmrquIdXNenl9/lgkz6xqs8e4aOji8tA2692u16yLrgsuljmlMXkEpuIYU5q32+3hcNhu/DQN/bRDCcvNend908YFM3fdYhgGQCWjeZB5soZJkk29asHWLzUBA3etlxwCMRU3DxNiiV27aOLqZM0OpjnJTGl0JN2chrOu1bPtBe5udtNcEjISUVHLmhVQTR1CXayHbIw1gR9FE4AAQPWEyrG8xNvbhYDKCJ+WTMDRv70atFZe+gkPYAYMiIDiJkM1AKI6nwZqIKjIZgRFBIrlZMSsCEAMTOM4ekdNDKGJWqSoBYQxlaOsTjWl5B3VXTEOIas6F1LJotYPcn2zDycLh8jMqMaeiUoqYsi5iPPegyiEUIqIGqopqiIUM6W64Q8Ej8OE9UQ+OZ0ApFrqW1VVMFMFRAMiZgZmdhzBzIwJMxRgFDUsCBpNWLla4o8FpDalwVM2g0LDMO52fdM0y2VjijLPkaGJ9MwzZ5Jmjt0+52kqi9V6+8wbXv6DT2qeFssNUXNzc/jcz37qpd/7vS5GQ7/b7VfLhsTykJxRKdAuFjMO5FEtZ83z9dg0NE7Xm+1azfaHa3JIVmY9+MCr7bIYGxq4KXasgoc+BWeLRTzsd8vFYhinaZqGQbcnm1JKDDbr5To8tVk9OBxUwXnvmDlGv9wstyfLq8ONoZZSVHVOA6JEzU+dbZrgiS+u9ylrVsJyPD9aChxB8YAOHJBTUzU1kONw8m2wND2S8nVMFxAV2NB8DaUGWBE7MJRSBb+1c6q2mwCAyEwMOKGiYzJVQhIlRVQzZDLAJKACagwCRU0QfWhymkop1DZMXgBNiuNAHpVmm3MpRUS8o1KKqkkxqjMYx3QNpWgIDYhM04Ski+WqaWy3n9JcShYXnQevpSgRST1nNWvcyunw1nkejy7g+sSx1tRuMSmolKmIMHMIwTmXzUxBmAFVCLMoQspmpMwaUcOc5l6uqU0uchMCzC5JG5rVYT/v9+mwn5jQO7m+PiCUxsH9u9s8709PT197dJUmRLDTs+3541fn6RDI0tgv1qfDcCgpE8DYT84jgMvZxn2ehtw0a9+dGU+K02qzvLq4zlLmPBvjyaJJpQdmA+6nEcgoeKbQtFudISwOGQa/iMjNanPXLHPAZh2LSFGdSzUFgSSSp745La89voC0WTZrw52yzSlj0zjv+zSKaZICoYxjuh5eXywWC0Zka1o6u3vCzfT4+jAnEQAATACkJgpgxMLOBUavUlQVkGuih9ux29uQcZupAZVMBRwVM6NbBwOzYp+OL4ZAtcAFAEIX0GUFVGUOZmJIsxQjMlMCZ6hqZICKkEWzlIo/LhaL7ckqOidpZiQ2lwUq9vQZCI9XVVBwCKFtkhqTUxMFFVMza2IEzCmPV7ubaRYi7xtfNLk0zyqikk0zkhESERgCklOxDKCVwWCm+h4V5TbyI6sJmAmIOmIiVpOiGB2KiZo5B1LEuQCAjK6kwoAMhjjOaVIrXpXGgBOOqujEWivDNMyHi915NJADwIkRwclp58QW0WWG164utifLcXfV+mDEzrwUVsTNZq2amuCBw07Z0NJwiM537eb6Zh/DZrHapLGPkPvDsDNu2gW2WYGmebq+kfXSg5Y7Z+vrcZ6HOXIIIUbF8/E1b5KGsj1dD+nAYGJWcpgOYKWg44JG3l0f+vund6Z+wJstimYZP3V+k0WJPIaMrCIy7XRSHkacZvUBHTGYJGjNTMrcMtMybNqT3X64uDxkeTLcotl0NqoA/NKmGi0RGRDEDOuwN1avjQqtgJkxgCcqSAhQqozdqmq8qp+0bqcuYFx5f5RkCQWQaLYMjrNKobp7vHJFTS6pFnWeWXNyjtZO1qsN6BisGU3mXKZcmraL7FcSVRIUS/NIDK5pihHmaZ4nH51imQ0cg5FNkg3AciYkU20oGKGYjjpWvQnlnJlZVc3EjJ4ExScDG0dJvdkTdt7M4DP6KgVTU3Kh2jFmFWJOKXninPOxZ1RAgqJqJYOa6HEXLZKrSHNKKaey73sRMedCjDln7/nOnTuYkqoOw6GIJJdWqwUx1921ADDPpRpEnp5sAIAMppxTssbjPI6ABmjn54+7rls1frVahq41YFIjcs8/98Zx2oMWANvvD9M0LdZrFVC1m91VKSW6MME8T7npGk3Zitzc7NEgtg0RDSlXDvBwODik3W53eu/B5dW+pBmQvA/RhxD8OI43/aDkhnESkc1qNU2TZyvENeMzYwRfGBYLC7HdHcZ+SNOUjty6SM4ZEbJBrSJrRKzrqRCwmrjjLeH36X63rgWsbSx+hlDNENSqYJ+QiMhURSx6Z7XM/Yz8aABmltLEzKUUJkppco7b2CxaJCIFG8exXa4MRg4+F3HOLRZtMVXpcy5m1nWd4zCN4M2mlMhxwyZZiWicZvS4apvDfgegIfppnkPTrddrV99GRQqsiIERkaMKMVUrRnvC0xPRE6PQiv3QLdEsIkAOCBerZRYlklK3Z+cCx/xyxAdUiuaCaCYmmkspSGKEiKxmec4ppXpVXddVU5QQAuX86NGjnLNoZfaUmEHKNE11rW2a5lJK07U5ZzBxCL7leZ6b6O/fv//6q48Ou2m1WjlHqcze2jnNWfTqal8xkuggBg+GSWRF1DYt+yAi0mtOybkgYPN+uNrvgbgU2aw2ItYPQ70ANru4ubp/524BSikBqGQ5O1tqEYCy2dy52R8Ow6hE4zyhgXdkOUFhoSSlmBZDJgRSI4QY3NnJ1rkDos1zVoVcDNKsqpGNmQnJQEWlzuIRoCoA6K2DAdbeBxEVjqYbDFTRpareda4qjBTVjBGejOwSgpkoqJZqZmaggGyqiFhybqIfxzH6wMzr9brM1yA4jYm8m6+vx2lC52PTqarzvvokAKGIBN8ED1Lmfd8jeyIqpZA7/rvO+8PQi2rbRUJarVYFMOXZ1ejtvQ/slEotaY+DS2AMR+zz0xGU6qEEqDwwAgFkQ0A2Ah88+pCnkQoqmHNOcr4V2hxrJjMzPXpDVTqASI3YoDBzfdqY2TnnvYfbLzPLOWtOi83KzDabjY8BHJeSnHPr5WK/3wPAPA7b7XbRNq++9hoAnGyXzH63uyaCew+2y2U35+sY45ymcRTgsFmvdzf9om0Q6N69s4vHjwkdodv3ByIKIYSm9eQLFkS8c/fs4maPDh37YRzXqxUiqwIQuxjGYTj0/WZ7utvvuzY+/8ydEP3hcABzw9SLmSHlnNGg69o8jaFxh11ZrjGwA4GSZjMTQE+kAMywXraIsO/7acoqUDKYlt6hB2Y0VFCr5uu3yDwQEDLQcSuLHo9XJa4R0QEW0FtxT1Xo3datt3Gq9hsiVuqeF6t6X0EEVXHOiUjwzgfXdV1sArhVKcUol1LYBRGJscGjLHiyIvVuquqUUwghNE262ceIzkdRiD4uFk3TOTEpCovlAtQWq2U/TjWxEKqhGhnUBZt4DIpgWm6VeGBm9QnFY1w9Im50S3iqKnoGorhc9vNkREXFzOYpHyE706KSi6YsOedcZC5SHwYAMGI1yzn3/VDKcZo0xqgI8zzP82wg6BjJ2rbNc6rnNcb42quPFosFGoQQUC06j2qND22ITQha4Ox0i1Cub65Pz9bDuDuMN6vVigOrqmg+OTmZ5xkAmFzXLl9++Nrh0DvfXu/2IjLPs5imVMxwfzhI0auba/bu7PRut1oisikSUdO1iDiOc/BNjC25ZrM5GceRUWUeAlNsAjo/idQPoWmazXrZRO+ITbXv+yLJQJgIjayIRwLNOY2EEhvuuti2njwIQFbrs/a59ElGsWxQAOV2Bu3WWej2eVbJqlIU9NOjuXwbYkUE1eqIDjN/mlsCMqAjq6hHehFu0XEAMBVmjjEGJiv5cDjUI1hFKm3bksF+f6MqJc9EQAxEqAB934/jqOBibIdxLuWoAfXeg5H3vm1bEQ1NnKapZtEiyc3zXC/X7Da5kxEDKJDBkzf0pKapfhdg8qS7rzHckJtlx8FPw6CqRK4StGpmVlTVFK1mJjWVDMcBTWUGT6pmWaSUEjzX6ykiWQozEZGkPKdcC/y2bQnARy9SDNQ5ymSSZ2IIns0RmIDByWr5zFOLlNJms3KOVqtF3+/nebzea865qDZNc3HxWFXrJ9V1y5SnrmvUct+P7Do1ROBxPLSxibFRMDbabDaHw6HrusPNoXKGOedpLm3jTxdrNZtSZtKuaR1jzsLOAXM/zylnMXXOeceSZk+sKiE22XScMxkQOSKEjKWUNM/IVLcbtY1j5qJ9qp6gBFKEDTwjI3pAq5PcVFMagqGoPdFfamX58Hbx31GFhjUZ1lrgCTnPzJVnrxYb9XtriII6ZFZyE6OZOUfzPJdS6k5hJA1Efd+3bWtm67iepsmAvG+897loDb1TmsGI2IuNu0MfggOTyxu7s10tF8spzcwMxIdDz94hokC1vrnVdBKRUt1+BMxVyX/M7Z8hbrrNERXMMgBE7yLFEJvm8vpmSnMp6llqVFYrIqJH1oMRsXb9hHZ74qGIVILOOQdMKGh29LoPIXrPRVJ9pJxzmos53u/3zrk2xLhctk1T5hScZ0BymOcRvDfJJc3eu5KLluw8rzcrMzObzaxtWwTXH64OfQkManK9u1qv197zPPbzPPd9f//+/evr69Xp6c351clqfX19HZsOBKpQ3DlKKRXTGGNdd1pUULRZLByqc5jyEEPTz2maB+OmqCJicM5K3k997U1bdOjd1A8xtli0AuSm6pyr5ulEhMTkqJ3LNBUpoABZj0aZyHWMW+v6dDAlVr49f7dbMKo7AZTjmManbzdUmhqBn7RViNlUARVJn7yA6fFsqIYQELHrmpQSiNa535vDftG0qrrdbkVEREpOt9WuhOBEi6ohmhapD0+McZ7nlIpjqAD+9e6GQbrF4uLyfLlcjuMIIF3Xuc8EMh0jmKs1aJ0fqGrQit4fzyc9aQ+pmokBOed9s1oms9fPz4vCcrkYptGR5yxIYmZFwQTh6DQEqEJEwXMIkQhFtGhGRO+9IasqIzrnyDsR8Z6Z2RBi2+Sc53mWQqtuoaqnp6d9kRhjmZNIpuiD80QUfZimiRFMhR2dnp7e3NyUUhBxGIbT0zvjOA/DQa20Ae7e3c7zhJQBYtO0L33q95fLpapeX+9K0YuLi0XTTGnuui6XOeec8lRzHBGpqKqKKhOUkqAIDr0wQplXy3YWmbIU8odDP6WMZo6xinlN0fl2Lvjo0bkqPPfMwgEXKcwMosioouwZAdQU0cUYupgTlkmO4wVmqALVxAXNioJyoeOGgdtVAYiOq5+L3g42Yv1+RKs+HMT1bquqmkqxo0Wc3hoWVnr3SSLtFou6mnVI82KxAAVVHeYpT7Pz1HUdoCGi91yKlpQCh8wiUgGxYuZV1USbJqQpIcJmszGzaRxjoEePX3cI4ziGEEQsi7isEtg55wA0a/VfQKomAYiIyPzptbDV1dY7V+YExKCKwL7xcdG13eI/fOTDxQAI+r53RECYiyAJApN3RDSnxFyNHDA4F5qgRVIqqopMZnbYD9wEAGKGnHMpxbcRAOZ53u12qoWZ68bSMc1tiDnnpu32+z2ABueaEEspaZrNbNG2IYR+Gtm7YRimaQCA9Xq924EqNE1DyN77lNJiGR88deeTf/D7xVY3h5sYo3NuN+8jIAjOZd4slmVOZjjOk/FxJR873qxWu8Oe43K1WV9dXHZdB0UK6DwXRrm43sW23U15ynMWJaL1YuHAXAg5Z0E3Fri4uikKTROmjB4B0Y3DEL3j2+IByakhYia0zXrdH8YyzSLFEYsUZkKoNmMAAJqBnZE7TsjVrvaIxDjOWZjZTIlIRAmJ2CmCc5xLORYDakKspmJQQXmiW0IbqgMStW2721+P/QAAOWciYmYywOhqvbRer8dxZAFmTFnEStvGUkouWUQUq8e81Pm2JrouBinTNE2E0XMA1JRKKepiqI8HPcmnzOy9r8H8ibONPcHbzESkDjvXvkwMganruvV28/jx41QMDOa5FiuaUjKzUupwdPHeO+ecc3B8bo/4X70AIprnGfC476Y2/qWOj5UyDbOZTbmM49i2bQjBu9CP0zCMl5eXNa24EEITY4yL1dLMDofDfr9fLpeSS0qp7r+LMW43J1LUFKIPMfi2iSVPOY3OARGI5KZrKxnG7EMIXdfBcaRcmYkYY/RdDMHxMBxijMNwWK+Xi0W73ixznk0LojXdckg5KWfFKYkYMHPbtstlp6pIDogfX+6HDOh9t9ykrIcp9cMI5MZ5mtJshrUJq7nXIaGVNvo2RkIALa5msJr9GBSgAGSFXKAIiEIBEIRcfcpEgbCoWJ3eIwRCZALiLMUQioqCFbRpzinLEUykY0OMFUwlCiFM03R2dnb//v0Y4zyPm+UK1YbxALejPofDIR2/pth4z+wIzrYb7whM52nKaepabyKtxy6Gava5WLTO0ZxTKQWYxjTv93sRcUSEVOdZQO3JgNWnwZ1jnL/9tZrWC0EDZu+Cd03MqewO+za6pNZ1wUQYUYu0TXSeaopv27ZI1UNJhTNK1mnsVa1pIjN3XWdmUxEyIMSKKjAzGooIebfwLo0TkTOzw9CD2rDredkyc3Q+51Qx5GmalsvlarN+9OhVASOi3e5wdnaiCjG211e9qrZtnOapbWPbxpxn1fLgwYN5nodhirFF4rZdHA6Hk/WJ2sjMRqSqWqSIPfv0U7VgaJpGQU+264vz1w207/er9ULM+n4c5wk5coip7FOx1SKulwvTgs6v1tvrXX953U8FmuU6OBunpCJWcmQC9EQ4pUxiiFZUKg2uAmgYvNu4RvNYihIb6pF2J0JmNKvT5FXLC3iroa/LA2pMqWmamJ/8N+fMDosKIqops1Mt1fy93nDHzgd2xKvVkohi4wnw+vqylqRtE7yj4GthRou2Owx9tZ8rJZsokmnRxXpZJEWPftRhSGOemwZOTzer1SLlWaV0XaeSmdmzMzjiWXma3fGibw9ixYwA4MlWmmP/hAiAqlXTTqUUJu9DaNqWvbu4ul6v1+eXl5vVchzHUkoTogHFGEUzYvWAQEI0UMf8BNdIyWKks7Ozx48f+xiapsk2mxQ4irJMFXLJ0zQdplHBNosupZSlGKGKhBDQuZTS1A+rxTL4Zp5nBTsMvYgs15uUkoBJASsGatMw5iTTPKApEvS7vXNOrXTLpUwzMZVSDoeLk+1p0zSLpt3v92PeN84j4unJyZzSxe4aET27pgmqer3fc4kpJc8uTTN6D4zD1HfLbT/t13xXRJZdXDTts08//cpLn5oNLq/3r53P3LIPoYi0MZY0E4GSFck6piYeBeoAOI6zc04RSpHgIljuQqOLdr/vUY/7q1SVyVWPTOcckqmq2BFmRsP/CCU6aimdqj5JYii3A0OqVf2DiERgBgTADj3zer32jpqmAdC+P2y32zmNi8WCQEuamuD6cQzhSEze3NwE3zhmkdy2i8IlpzE6x02z6LxuxXs2s/V6dXH5mIgI1RRrtySmJsaOiSiNE5WsKaUpp2p5fzStu/X7tONwEgIzMKFjAiTAesIUwXfNMM27w36aplAd5HMhQE+8WCxEZBrG2p2YCTObHpvKaoFy9+725ORkmqYQwoN790GPSH7FR4+fWikqME3inPMunm5OmbwopLnkJG3b1k88hHC1u7m4uJinXBTW21MiN85JVdfrjohiaC4vruZ5Pjk5ETErWpXXOcnp5lQVxjGnVNarzTAMAHp+fl7hJwYM3jvnXn/99WXbDYf+cDhM00REXdfNwxhjrK1PVUh478nxnTv39vs9MxLo6Xb16OHLTdOcX95c3sxxwRVr9N6P/cG0zGOPICHQYhlD9KenW1Wdppwz5FzSPJtKkbnIrGlcLcNqGZiAEYgRCdBxjE3TNLH+1DRNiMF5z+ScwyNNfxwXewIOPvmciymQEzOtumg0QkO06HnRtpvV6vRk07WRTA8312gCWkqa5mGchxFUp2Hw3m1WK+/9OPYV8D+OdIqiikNI42BlbjxpmZlMyhw8qcxtDKaplFRb4ZxzSgkRS1ZNulmfOBEhOxoE2C2Ta1wnQZVueze7JYHArJTifDRCH4OL4fGj18Zx5OJOTk5SSlrKPCe1MqdiZiF4ZkfMfGSjjkefmYNn732e05xnM3vttddKKb7rCNlSqp9pSqn0/TRN3bJBtLZtb25uxnFs10sC7thfXd7UYCAKTdM55/q+b53bHfZt23rvRSyEhtmnlBrfLFahgql5HucxURM3q+3l5c3upmfvEFzOEkIYx3GzXZVSpmnKw9TFhgHv3rlTTIuULjZpmtvYaC6r1UrAFqFJ0+xi2PU3LgYRWa03uttZLnfu3zlZLj3AH7z08k0vQlAKoGMGkDQGFMnFMQQHTXQIpW27xaIFONntDi54MBr6vXOcc1YFH2DdLghMtWRRA61B60m3UG+iQ2JnSt4U4ZajRsSqLRExRKu1O1aoj5DZH9tQJkb03kUfgucQQuPd5eXr9+/eO+xvDruiqmK6Wi1SSmDeMe6ub2KMTduO85zSDAAxxhry9vt9E30MTlKOHLkN4zgGz3k8RO7QcmDKaCXnUvKRCVItpVgRRDyu4yYGMqjQbiWPjg/ZLQPxGXI7rL2Ui6FbLcdp6sdBwSzly/OLp5566vL86knJsF6v53E0szmN3vsnEZoBRXIM0Tk39Acw69o25TxNs5bSuYClUCmqakfNGO3HqVt0ZjhNyfuIyG3rbx5fYOfv3Lmz3+/NbE5pnmdiD8gllYv+qsr/pvHQxNj4sN1uL3bX0zSZKGhxzqnaNOUiQhQQ2DHO86zqm+ABLKWZkWL00zStusU0jav1GtOcDQTLPM+I5JwDU+98mWZmTqU89dS9q+vDNA276xsEIJXry/Pd7jCOIgbkwpQL5tx1XZrGAqXxeOfsZJwOzEUkr9atDzyej6UUH5rFclGLs9VqMfb7rgnBs6oP0VvKauxcUMDK9DKTc47RKrNDgOV2EwsiihzDZw1Gt3PrUAvZEIKZRyu1R2yb4NmpljJPN31atN0wHMykH2bnXEXimhhKmrqu6ftR9Wj82YTonJNiKaUYo3eEiNHzdb93MzbtAiKrahN4Hgf2DgxFYKpOzzHURjzGWCCllAgI60ubGbOvufu4ELZiY5/hElqf1JpVQxOJ6Pz8vK4OqYXLNIz37pw1TZzHGRF3u52KVOindjy1mzazEAIAzONUCYnD4QAAzIiIIYQQAh3HSjXnnHOuo/7DMNSr9SF472OMIYR6OlV1t9vV6+z7PudM3i0365xz8L4+XY8fX9TW3nsffQNAi3Y59JMWk2KSdZomMKqtAwCYFjBMKS+6Ze29pmna7XbzPDNzTomZh0P/RAVbn+rD0LeL7vz8XDQzQJknQiulcEBykLIgO+9CmkbHuGjc8889c3KyBRUGu3N6wgiXl+dzGutntVwuQwhmstmumPl0uyYCrEeQmRl9jHA72iAiWkopJaeUUsppfnIrEVmLoIGJ1nFP1SdIIjOg9z6EsN1uV6tV10bnHBEwIJiUUromoBkjMGCe5jbEtm2JwHtfUqoYds5JRDabDaEjIu99t2g2m41JHoZhGjTndOh3bduuFosmRNWSxsF77ofZzGIIqjoMQ2VkmHmxWDjfLtrY1KODiJLyOI6V7K/RznuPalUx5Jyr+gJyvFwupVi7gnYFY5rbSoUBLBYLaCd36CugOCXbLBZqOQSnh904DmpiIKbsfXO5u2hjw0yOWbUg2NYFNsuqkR0UnUrJDjNzIGKi6/0OEAIxi4FqaJrALosg4X5/MOIkakWWyy6lNPfDommdC2royBm7LKlZdIehTH1umpClzLudGaeUXcQ5zQBunjM7VCvTBF23fXTx+mqxKuSGYbgZLjbbpWM0BQKas8RAOs9kRQgPu0PbLu5vt+gW14fUdJth3IvkMWFrPnRtQ/6wS4ABIBPMgdQDP32CZf/wc576/DhvT0/uPL44n677Vdvs5r3CfLZZnTTugHg5pqurIYNHVRFkjg3PYFk4liJFBcB8IEUn7NiEIJthAS82R+d9WLbN2pQUQVVCdDWUMnlEjC6E6ENwzGzW11NbCwALhhRXG68xTmXfnJ2Y2Xa7TSktFgsaxxgipoTjuFqtDofDqmmyyuKsq3FaRMJyuQyrUkp3l0WEKVCMZrbb3UizcBxms+XpWWx80zRjP6w2ARFLKRxNVdz/+89/CeCohyMiNKsJXbV4dnUhYrUxqeGzaHauWnUqM9f9XVKqLxTXfgirzd+REQ0AKlIA9R//45/7+z/yd0WEEL0nROzaZU4Tc/Te39wM3rtqHvEEiLXb3jLn4nyDiIdDsijTNBHgfr936IqVxWLRtu1hHBCxVj+Scgih7/unn7r/yd9/CQDOzy/v3j0jgl3fW5E0mWiJsY3tYhgOZvpZn/VZL/3BK5vNyX53qZY3y9U0Tk0IRygDWDXX8qhrl4guH4amaYaRT09P53l2zu12u3sPTsWqVaVJyszUNM0wHAyBHXZdN0wigszsie9sT8/OYL1aXV9fxxj3+/1ms4H94eGrrz94cIqIjNz3fVXVOOdghlwK0hHBZua6mhcRPWMuCqzM0IRIBmaoBZhAVb/ne77nv/zmbyP06HxKU4VkmNkUSxHnnORU47RpeVLRijxRCdsT6BpuJ9yP2pHbA/OkN3LOVaT8lohyFQWv3yIKIlKzTX1TNe3UF3HOmYmUUn/78OFDJwWY+ZVXX3nve9/bNs3zzz//jne8Q4p636oJkkfEORXngqqCwe9+6P+7c+dOP40//dM/vVwuAeD7vu9/xKqfFTBgxDoexVX6SWQ//dM/883f/F+sNwtEBkBV8NGZ4TCnWhiJmBrUExl8AIAkxYt479WEiWIIi8UipSmLOoS2bVU1xubs7KzvR8t45979j3/844oQgi2Xy+vryzZEEdms133f+8BpHk9PVqol+pYBKYQm+H6YV6vVfjfWvdc3Nzfr9VrEmBlN+753HEMIKeVpLMuuS0qq6jj0fa+KcypN1+qljeM4T7nrOhE7HA5AguDTNBHRZrWWlE3y+mTNAhdXV2bsmEzmxXKxWTaHw+Xp9uzy4mUEGIf5DZ/1WTnnZ599UJmkYRj6vu/7/mSzLXnumnaaZo48z2WeZ1UFQGQFVVUDASRF0yKpC8654I0VeJwyETsXVPAnfuInXn315S/+4i/6wi/8wve+970hxHe+852qYAp/42/8DQD4s9/555555plf/dVfa9v2LW95CwB88IO//Yu/+ItN0/yFv/Df/9iP/fgwDN/wDd/wBV/wBR/72MdfeeWVr/qqr/r+7//+Bw8efMM3fMMLL7yA6FKSR4+uP/zhD3/5l3/5e97znhjjt33btz377LNE8ad+6qdee+3VL/gjf/iFF1742Z/92bZtv+/7vs+MVPVH3vPDwzB8yzd986uvvvqbv/Vv3/SmN33Lt3yLgXNIAZA+8pGPv+lNn/uOd7zjQx/60Cd+75Pve9/7Tk5OvvM7v/Mnf/Inz8/Pv/3bv/3DH/7w137t1/7SL/3Sr/7qrz733HMPHz78gR/4ay743/qt3/qtF9//1FPPiEjf97/2a7/2+PHjt7/97b/5m78RQnjhhRd+6Rf/xe994uNve9u35CSqMOdMxKpQNXLjNO/25bnNZr/vQU1FrsZ90wTnXK10+yEBwHK5nPoDMy/bzhM3TSP5KF0dhgER53mOXXxSLldNwzzNFxcXZ9uT0832tfPHh8NhtVr1fV+LVEdLsyPLxczOc9/3wXeI7L2XLCKyXS9ee/wIzDG5eZ6bpm1C3E3zcrli9vF+u9vtmqZtYqcydF132E+mmR2lLAS4WnbrRTfqQYEeP3p9deeMCXJWJNss2qfu3kUp67PTi6vr5XJpAm2znOf5cDjANLRtCwDDMBDR3bM7V/tDKjl0i+jBgKZcGwgwJCLwjC5QFk2qoFLMKBxlQEjctTH4pmSZkl5f7d75zr/y7nf/YM7l677u61966eHHPvbxz//8z5+LfMe3/9cPHz783Y/83vnl4Zf/5a/9iT/xJ4o6Vd0d0l9+5/f/o3/0D3/9N97/1DOf9Y3f+I0//MM/HNvNP/xH/+Qtb3nL7/3+y3fvP/fWP/6fvumFLwAiVX31tVd+7v/8J123/PKvaL7jO7/rpZde+vBHP/HKo8sv+qIveu4Nn/Nn3vEdf/tv/+3P/Zwv+Evf+85/8JM/cXG5+9jHPvZ5n/d5X/01f/oPfc7n/uiP/ujXf/3Xf89bv+Ld7343+yYVO/Kcf/JP/ammbf/6X//rL7744s/9/Pu+9/v+8pd++Vs/8O8/+PSzz/3ld/5PP/fz7/vQhz9S1D780Y990Ze85Wu+7uvnXFyI/+Jf/PIv/uIv/ca//a1xml5/fPGph6984R/+w1/6ZV/2T//ZP/v4x3/v7W//M7/8y//yr/7Vv/bM08+pgveBiJwLdbqv6g8Oh2G7XVxd3gAAECsgeweECkeIjgFBdOoHVXXEnp1nN49TDS3TlIgotk3bts4FEem67vz8PE9z0zTeu8YHMyGC08362aceNN7Nw1gL7hjjZrMZ9ofKsh4OQ81fJto0Tdu2jvjq6qrqQmpmn6bp5uaGiK+urn7/E6+en18+evX1lNI0TVnKq49eX65Xi27VxeZws9MiJjr0+9Wii8E99eAMpDACgXrGzXrBpAx2fbXb7/fDMHkfxnHc73rv43ZzqgKOeL1ed03b9/1mtbh3716aRzASkTTnnNXMEAwNHOOya+9sl/fvnJ5s16uuHadhnueqGFGtbiVYSmnbhSka0N379/6ff/4LH/jAB062Zwi86Fbdav2BD/57Nfsjb37zV331V2MtUwDe+hVfsdvvP/4fPtG0i9OzuyG27MIb3vim/+od31GJrtPTOw8fvvJ//dP/O2VRw6efee7Pf9d3O+/brmu77oO/8ztI9Me+9EsB8cvf+tZ/94Hffvb5N7zxTZ/9qZdf2e/7vh/f+tY/vt2eft7nfcHP/fz7/tiXffmbXvicf/ubL5as4zAH31Bt23/+53/+S77kS/7nv/k3P/7xjz+pkfu+955Vy5wnchjbsDvcqBZmbJqw2+2+5mu/9rnnnx/HcRzHaRo+9ak/+I3f+I0YIzOvNxtiVrNUsgtRDVNKpjiOsxgCQCpZRMm5eZ7Zh5yEiGKM1fqTiKpwycyaEKN3p9sTMxuGofakpRQAWq5Xq83ae39zcxNj9D4659brdQjN0PddbLzn1XK56LrtZgUmtTlNKUUfVFVzYeY2xO12e3Z2okXSPB8OBzKoHTQAOPKeXXAxzyX6ZrFYKRixdwEMQQycj3POOZdhGC4urqqKjIjatt0sF22IKuIIdjfXaZ6DY8cIqo4xMhAUEUNgF+Ll9RUi931PjmvrDUBjP1R+rwnxsNu3IarCmHJKSQS06j5LApXtehU8E0gb/PPPPdM0jUje7/eHw6EmDXTcNN0wDECMiL/2q//mB37gB/6b//bP//qv/4aIPXz48Pz8/Lu+67s/9LsfNJDYsOgMWIj14vK1H/l7P/Q/fM9feOrpuw9f/mQuI2CpfySS+3H3dX/6q7/yT/3xV159yXlwHkRn0Tnl4eHLn7zanf+5/+7PfuCDL2adyNuv/pt/9eEPf+gd7/gzv/Irv/L8G97wrW972wd++7dzKc77/+OnfuoNb3zjH/2jb/mVX/lXX/mVX/mmFz770euvpZJdkeS9/5qv/ar3vOc9y+Xyq77qTz777NM/+IN/q+u6v/gX/+J7/u4Pv/+3/93b3/6tL7744rve9b/du3fncz/3hV/4hX/+l/7S9/zoj/49F/xqtXrb27/lZ37mZ7bb7Zvf/OZ//a//9Uc++rtveONzIqKSv/Zr/uS73vWu2mYhQ53iKMWmaeq6LucMgM65aZqid22zGMZDVbwCAHvn2UlOpZT9zc6OC5gJARSxbReIeDgcpnG+c+fO6xfnOZf1euV9vLi4WDRtDBFEvXMpJS35tgwXM5MszSKmlAgKAO12O9/4pnVnZ2d9P9dy/mS7ArWLx1dN19YqvmmacRxF57br+sOlc34aU7140bxcLksGIqcA/WFg9tM0MZZN1467w6JrNpvNzTCVkoi4a5jBRIojyjmnKTUhErqUs3MBgWt9eZTXIMX1cr/fB+curq7C2TbP2ZDIabUTIVPvnZapcTzNE/jAhDnnec6CzhOpQBXuINKzzz77rne968u+7I89//yz7373u0vR7/3e73vxxRe/+Iv/kx/5kZ8D1G/6pm8Smds2Itr19eUrr7zyC7/wCyHSP/iJ977tbW/bH67e9YP/6zd/8zf7gLHh07P108/c+/Ef/3FV/e7v/m5inaaBmNtFeOa5Bw+evvtDP/RDIvat3/qt73//i1/4hV/4sz/703/ocz73x37s73/Df/aNP/Sev9M0zXd913e9+O9+8/T09IO/89vnF69fXz0+PT39W3/rf7l3796b3vTGj370o/j+97+ot/RjkeO0Wm3NahsOt/RPZR2esPaI1WfEyPET+FdVqe6aB7kVPvqcc4zeIP+T973vf3/vj4rkJjgziz6cn5+3MYjkrmnP7pxcXl4WMEm5acPJZuuY+t0+OE7TVGtWF3zf99M0+djGGPtxWHTLcRyTFHa0XK3OHz/ebtcgalpASte2eZ6aplGTOuF0dRgIGwCYx/7u3ZOrq5vgO2BIeTAT71uHbrVqcxpVRAUBYJ7znCq4LevtqmnCzc1exNar7ePz17xnH9A5l2btuqX3eL2bm3alWEoat4tmPgxS0vZ0ezWmw4Tn14cm4LP3NzLuI9O+CKhq0gd377Xt4vLmerffk8N5nru2RQOH5Jw7jFNoGzEkhpvDPM6JVHydGXKua5tVR9v15uHr565ZdF338kt/0M/g2tYgp1nf+Vf+6p/++v8cwRtCznMIT3YTgJkxOtXiA8/zDGSVOq48UAU4p2l6Ip/A20mK49Qb0ROt0y2JTyklqmS4au3Tn7T5cGueULLWkY/6moxUSvHh06CBmX3qD176/wFQOo+cctVMVQAAAABJRU5ErkJggg=="},"metadata":{}}],"execution_count":21,"metadata":{"jupyter":{"source_hidden":false,"outputs_hidden":false},"nteract":{"transient":{"deleting":false}}},"id":"998224f0-96dc-415d-8d74-df0d360b0c18"},{"cell_type":"code","source":["# Resize the image to a fixed size\r\n","resize_transform = transforms.Resize((224, 224))\r\n","image = resize_transform(image)\r\n","\r\n","# Convert the image to a PyTorch tensor\r\n","tensor_transform = transforms.ToTensor()\r\n","tensor = tensor_transform(image)\r\n","\r\n","# Add a batch dimension to the tensor\r\n","tensor = tensor.unsqueeze(0)\r\n","\r\n","# Load the model from MLflow\r\n","model = mlflow.pytorch.load_model(model_uri)\r\n","\r\n","# Set the model to evaluation mode\r\n","model.eval()\r\n","\r\n","# Pass the tensor through the model to get the output\r\n","with torch.no_grad():\r\n"," output = model(tensor)\r\n","\r\n","# Get the predicted class\r\n","_, predicted = torch.max(output.data, 1)\r\n","\r\n","print(predicted.item())"],"outputs":[{"output_type":"display_data","data":{"application/vnd.livy.statement-meta+json":{"spark_pool":null,"session_id":"2511b538-6adf-43c2-8c1f-705b65a98530","statement_id":35,"state":"finished","livy_statement_state":"available","queued_time":"2023-07-19T15:59:58.2089018Z","session_start_time":null,"execution_start_time":"2023-07-19T15:59:58.7627372Z","execution_finish_time":"2023-07-19T16:00:02.6712299Z","spark_jobs":{"numbers":{"RUNNING":0,"SUCCEEDED":0,"FAILED":0,"UNKNOWN":0},"jobs":[],"limit":20,"rule":"ALL_DESC"},"parent_msg_id":"7f7317b0-4564-48f1-b95e-e7f0c50451e9"},"text/plain":"StatementMeta(, 2511b538-6adf-43c2-8c1f-705b65a98530, 35, Finished, Available)"},"metadata":{}},{"output_type":"stream","name":"stdout","text":["7\n"]}],"execution_count":22,"metadata":{"jupyter":{"source_hidden":false,"outputs_hidden":false},"nteract":{"transient":{"deleting":false}}},"id":"bb0081a9-4b23-455f-aa29-a8575bf1f90e"}],"metadata":{"language_info":{"name":"python"},"kernelspec":{"name":"synapse_pyspark","language":"Python","display_name":"Synapse PySpark"},"widgets":{},"kernel_info":{"name":"synapse_pyspark"},"save_output":true,"spark_compute":{"compute_id":"/trident/default","session_options":{"enableDebugMode":false,"conf":{}}},"notebook_environment":{},"synapse_widget":{"version":"0.1","state":{}},"trident":{"lakehouse":{"default_lakehouse":"26b7e89e-59a0-4381-876e-ef1d0d15afa0","known_lakehouses":[{"id":"26b7e89e-59a0-4381-876e-ef1d0d15afa0"}],"default_lakehouse_name":"DemoLakehouse","default_lakehouse_workspace_id":"c5a627fd-e48d-405c-8dfd-11a708544ac4"}}},"nbformat":4,"nbformat_minor":5} \ No newline at end of file diff --git a/workshops/fabric-e2e-serengeti/assets/report.png b/workshops/fabric-e2e-serengeti/assets/report.png deleted file mode 100644 index aa69a661..00000000 Binary files a/workshops/fabric-e2e-serengeti/assets/report.png and /dev/null differ diff --git a/workshops/fabric-e2e-serengeti/assets/sample.png b/workshops/fabric-e2e-serengeti/assets/sample.png deleted file mode 100644 index 4c707740..00000000 Binary files a/workshops/fabric-e2e-serengeti/assets/sample.png and /dev/null differ diff --git a/workshops/fabric-e2e-serengeti/assets/saved_sampled.png b/workshops/fabric-e2e-serengeti/assets/saved_sampled.png deleted file mode 100644 index 21b183a0..00000000 Binary files a/workshops/fabric-e2e-serengeti/assets/saved_sampled.png and /dev/null differ diff --git a/workshops/fabric-e2e-serengeti/assets/slicer.png b/workshops/fabric-e2e-serengeti/assets/slicer.png deleted file mode 100644 index e9b93b33..00000000 Binary files a/workshops/fabric-e2e-serengeti/assets/slicer.png and /dev/null differ diff --git a/workshops/fabric-e2e-serengeti/assets/source-settings.png b/workshops/fabric-e2e-serengeti/assets/source-settings.png deleted file mode 100644 index 70064351..00000000 Binary files a/workshops/fabric-e2e-serengeti/assets/source-settings.png and /dev/null differ diff --git a/workshops/fabric-e2e-serengeti/workshop.md b/workshops/fabric-e2e-serengeti/workshop.md deleted file mode 100644 index c2f3c7a1..00000000 --- a/workshops/fabric-e2e-serengeti/workshop.md +++ /dev/null @@ -1,1361 +0,0 @@ ---- -published: true -type: workshop -title: Analyzing Snapshot Serengeti data with Microsoft Fabric -short_title: Analyzing Snapshot Serengeti data with Microsoft Fabric -description: This workshop will walk you through the process of building an end-to-end data analytics solution on Microsoft Fabric using the Snapshot Serengeti dataset. By the end of this workshop, you will have learned how to load data into a Lakehouse, explore the data using SQL, visualize the data using Power BI, and train a machine learning model. -level: beginner -authors: - - Josh Ndemenge - - Bethany Jepchumba - - David Abu -contacts: - - '@Jcardif' - - '@BethanyJep' - - '@DavidAbu' -duration_minutes: 180 -tags: data, analytics, Microsoft Fabric, Power BI, data science, data engineering, data visualization -banner_url: assets/architecture.png -sections_title: - - Introduction - - Pre-requisites - - Loading Data into Lakehouse - - Exploring the SQL endpoint - - Data Visualization using Power BI - - Data Analysis & Transformation with Apache Spark in Fabric - - Download the image files into the Lakehouse - - Preparing your data for Training - - Training and Evaluating the Machine Learning model - - Resources - - Appendix - -wt_id: data-91115-jndemenge - ---- - -## Introduction - -Suppose you are a wildlife researcher who is studying the diversity of wildlife in the African Savannah. You have millions of images captured by camera traps and all the image data store in json metadata files. How do you make sense of all this data? How do you build a data analytics solution that can handle large-scale and complex data sets? - -This workshop will walk you through the process of building an end-to-end data analytics solution thats can help you answer these questions using Microsoft fabric. Microsoft Fabric is a unified data platform that offers a comprehensive suit of services such as data science, data engineering, real-time analytics, and business intelligence. - -You will learn how to: -- Load data into Microsoft Fabric using Data Factory pipelines. -- Leverage SQL queries to explore and analyze the data. -- Create reports & Visualize the data using Power BI. -- Use Apache Spark for data processing and analytics. -- Train & Evaluate a machine learning model using the data science workload in Microsoft Fabric. - -![Snapshot Serengeti](assets/architecture.png) - -By the end of this workshop, you will have a better understanding of how to use Microsoft Fabric to create an end-to-end data analytics solution that can handle large-scale and complex data sets. - -This workshop uses the Snapshot Serengeti Dataset. To learn more about the dataset use the links provided in the citation below. - - -
- -> *The data used in this project was obtained from the [Snapshot Serengeti project.](https://lila.science/datasets/snapshot-serengeti)* - -> Swanson AB, Kosmala M, Lintott CJ, Simpson RJ, Smith A, Packer C (2015) Snapshot Serengeti, high-frequency annotated camera trap images of 40 mammalian species in an African savanna. Scientific Data 2: 150026. DOI: https://doi.org/10.1038/sdata.2015.26 -
- - - ---- - -## Pre-requisites - -To complete this workshop you will need the following: - -1. Familiarity with basic data concepts and terminology. -2. A [Microsoft 365 account for Power BI Service](https://learn.microsoft.com/power-bi/enterprise/service-admin-signing-up-for-power-bi-with-a-new-office-365-trial?WT.mc_id=data-91115-davidabu) -3. A [Microsoft Fabric License](https://learn.microsoft.com/en-us/fabric/enterprise/licenses?WT.mc_id=data-91115-jndemenge) or [Start the Fabric (Preview) trial](https://learn.microsoft.com/en-us/fabric/get-started/fabric-trial?WT.mc_id=data-91115-jndemenge#start-the-fabric-preview-trial) -4. A [Workspace in Microsoft Fabric](https://learn.microsoft.com/fabric/data-warehouse/tutorial-create-workspace?WT.mc_id=data-91115-davidabu) -5. Make sure your Workspace has the Data Model settings activated - - Click **Workspace settings** - - Click **Power BI** - - Open **Power BI** and **Click General** - - **Check** the small box with "Users can edit data models in Power BI service - ---- - -## Loading Data into Lakehouse - -In this section we'll load the data into the Lakehouse. The data is available in a public Blob Storage container. - -To begin, we will create and configure a new Lakehouse. To do this, in your workspace open the `Data Engineering workload` and Click on `Lakehouse` provide a the name `DemoLakehouse` and click `Create`. - -![Create Lakehouse](assets/create-lakehouse.png) - -This will create a new Lakehouse for you. Both the `Files` and `Tables` directory are empty. In the next steps we'll load data into the Lakehouse. There are several ways to achieve this: -1. Using [OneLake shortcuts](https://learn.microsoft.com/en-us/fabric/onelake/onelake-shortcuts/?WT.mc_id=data-91115-jndemenge). -2. Uploading data from your device. -3. Using [Data Factory pipelines](https://learn.microsoft.com/en-gb/training/modules/use-data-factory-pipelines-fabric/?ns-enrollment-type=learningpath&ns-enrollment-id=learn.wwl.get-started-fabric/?WT.mc_id=data-91115-jndemenge). - -For this workshop we will use the Data Factory pipelines to load the data into the Lakehouse. - -### Configure a Data Factory Pipeline to copy data -From the bottom left corner of the workspace switch to the `Data Factory Workload`. On the page that opens up click on `Data pipeline`. Provide a name for your pipeline and click `Create`. - -![Create Data Pipeline](assets/create-data-pipeline.png) - -On the Data Factory pipeline page that opens up click on `Add pipeline activity` and on the dialog that appears scroll down to loops and add the `ForEach` activity. - -This will open the pipeline canvas and you can configure the settings of the activity in the pane underneath. - -![Create Data Pipeline](assets/add-for-each.png) - -in the `General` tab provide a name for the activity. Next click on the `Settings` tab and here we'll provide the items to iterate over. Click in the `Items` text box and click on `Add dynamic content` underneath the text box. - -A new pane appears on the right side of the screen, and inside the text box type the following expression: - -```js -@createArray('SnapshotSerengetiS01.json.zip', -'SnapshotSerengetiS02.json.zip', -'SnapshotSerengetiS03.json.zip', -'SnapshotSerengetiS04.json.zip', -'SnapshotSerengetiS05.json.zip', -'SnapshotSerengetiS06.json.zip', -'SnapshotSerengetiS07.json.zip', -'SnapshotSerengetiS08.json.zip', -'SnapshotSerengetiS09.json.zip', -'SnapshotSerengetiS10.json.zip', -'SnapshotSerengetiS11.json.zip') -``` - -
- -> Learn more about functions and expressions in Data factory [here](https://learn.microsoft.com/en-us/azure/data-factory/control-flow-expression-language-functions?WT.mc_id=data-91115-jndemenge) -
- -We are creating an array of the file names we want to copy from the Blob Storage container. Click ` Ok` to close this pane. - -Next, inside the `ForEach` click on the `+` button to add a new activity. From the dialog that appears select `Copy data`. Click the`Copy data` activity and in the pane underneath we'll configure the settings of this activity. - -In the `General` tab provide a name for the activity. - -Next, click on the `Source` tab. The `Data store type` select `External`. For the Connection click on the `New` button. On the dialog that appears, select `Azure Blob Storage` and click `Continue`. - -For the `Account name` provide the following URL: - -```url -https://lilablobssc.blob.core.windows.net/snapshotserengeti-v-2-0 -``` - -Provide an appropriate connection name, and for the Authentication kind select `Anonymous` and click `Create`. - -![Create Connection](assets/create-connection.png) - -Back on the `Source` tab, in the `File path`, the container name as `snapshotserengeti-v-2-0` leave the directory empty and for the File name, click on the `Add dynamic content` button and from the pane that appears click on `ForEach CurrentItem`. Then click `Ok` to close the pane. - -![Create Data Pipeline](assets/source-settings.png) - -For File format dropdown select `Binary` and click on the `Settings` button next to this dropdown. on the dialog that appears for the `Compression type` select `ZipDeflate` for the `Compression level` select `Fastest` and click `Ok` to close the dialog. - -Next, click on the `Destination` tab to configure the destination settings. For the `Data store type` select `Workspace` and for the `Workspace data store type` select `Lakehouse`. In the Lakehouse dropdown, select the Lakehouse you created earlier. - -For the Root folder select `Files`. For the File path, the directory, text box type in `raw-data`. Click on the File name text box and click on the `Add dynamic content` button and from the pane that appears put in the following expression: - -```js -@replace(item(), '.zip', '') -``` -
- -> This expression will remove the .zip extension from the file name in the current iteration. Remember the file name was in the format `SnapshotSerengetiS01.json.zip` and we want to remove the .zip extension to remain with the file name `SnapshotSerengetiS01.json` after the file has been unzipped and copied to the Lakehouse. -
- -Click `Ok` to close the pane. Back to the destination configuration, for the File format select `Binary`. - -Now that we have finished configuring both activities click on the `Run` button above the canvas. On the dialog, click `Save and run`. The pipeline takes a few seconds to copy and unzip all the specified files from the Blob Storage container to the Lakehouse. -![Create Data Pipeline](assets/complete-copy.png) - -Navigate back to the lakehouse to explore the data. - -### Explore the data in the Lakehouse - -From the Lakehouse right click on the Files Directory and click Refresh. Upon refreshing you’ll find the newly created subdirectory called `raw-data`. - -Clicking this subdirectory will reveal the 11 files that we unzipped and copied from the Blob Storage container. - -![Create Data Pipeline](assets/lakehouse-explorer.png) - -### Convert the json files into Parquet files - -We will need to convert the json files into Parquet files. To do this we will leverage the Fabric Notebooks to perform this task. More about Fabric Notebooks will be covered in [section 6](/?step=5). - -To create a new Notebook, ath the top of the workspace click `Open Notebook` click `New Notebook`. At the top right corner of the workspace click on the Notebook name and rename it to `convert-json-to-parquet`. Click on any empty area to close and rename the Notebook. - -In the first cell of the Notebook paste the following code: - -```python -import json -import os -import pandas as pd - -# Set the path to the directory containing the raw JSON data -raw_data_path = '/lakehouse/default/Files/raw-data' - -# Get the list of JSON files in the raw data path, and select the first 10 for the training set -train_set = os.listdir(raw_data_path)[:10] - -# Select the 11th file for the test set -test_set = os.listdir(raw_data_path)[10] - -# Initialize empty DataFrames to store images, annotations, and categories data -images = pd.DataFrame() -annotations = pd.DataFrame() -categories = pd.DataFrame() - -# Process each JSON file in the training set -for file in train_set: - # Read the JSON file and load its data - path = os.path.join(raw_data_path, file) - with open(path) as f: - data = json.load(f) - - # Extract and concatenate the 'images' and 'annotations' data into their respective DataFrames - images = pd.concat([images, pd.DataFrame(data['images'])]) - annotations = pd.concat([annotations, pd.DataFrame(data['annotations'])]) - - # The 'categories' data is the same for all files, so we only need to do it once - if len(categories) == 0: - categories = pd.DataFrame(data['categories']) - -# Set the path to the directory where the processed data will be saved -data_path = '/lakehouse/default/Files/data' - -# Create the directory if it doesn't exist -if not os.path.exists(data_path): - os.makedirs(data_path) - -# Define the file paths for saving the training data as Parquet files -train_images_file = os.path.join(data_path, 'train_images.parquet') -train_annotations_file = os.path.join(data_path, 'train_annotations.parquet') -categories_file = os.path.join(data_path, 'categories.parquet') - -# Convert the DataFrames to Parquet format using the pyarrow engine with snappy compression -images.to_parquet(train_images_file, engine='pyarrow', compression='snappy') -annotations.to_parquet(train_annotations_file, engine='pyarrow', compression='snappy') -categories.to_parquet(categories_file, engine='pyarrow', compression='snappy') - -# Process the test set, similar to the training set -path = os.path.join(raw_data_path, test_set) -with open(path) as f: - data = json.load(f) - -# Define the file paths for saving the test data as Parquet files -test_images_file = os.path.join(data_path, 'test_images.parquet') -test_annotations_file = os.path.join(data_path, 'test_annotations.parquet') - -# Extract and convert the 'images' and 'annotations' data of the test set into DataFrames, -# then save them as Parquet files with pyarrow engine and snappy compression -test_images = pd.DataFrame(data['images']) -test_annotations = pd.DataFrame(data['annotations']) - -test_images.to_parquet(test_images_file, engine='pyarrow', compression='snappy') -test_annotations.to_parquet(test_annotations_file, engine='pyarrow', compression='snappy') - -``` - -This code will convert the json files into Parquet files and save them in the Lakehouse. To run the code click on the `Run all` button above the Notebook. This will take a few minutes to run. - -Right click on the `Files` directory and click `Refresh`. You will notice that the `data` directory has been created and it contains the Parquet files. We used the json files from season 1 to season 10 to create a training set and season 11 to create a testing set. - - -### Load the Parquet files into Delta Tables - -Now that we already have the data files, we will need to load the data from these files into Delta tables. To do this, navigate back to the Lakehouse and right click on the individual parquet files and click ```Load to Tables``` and then ```Load```. This will load the respective parquet file into a Delta table. - -
- -> Refer to this [learn module](https://learn.microsoft.com/training/modules/work-delta-lake-tables-fabric/?WT.mc_id=data-91115-jndemenge) to learn more about Delta Lake tables and how to work with them in spark. -
- -
- -> You can only load the next file after the previous one has been loaded to Delta Tables. -
- -After successful loading you can see the five tables in the Explorer. - -![Create Data Pipeline](assets/load-to-tables.png) - -Now that we have successfully loaded the data you click on the individual table to view the data. - ---- - -## Exploring the SQL endpoint - -This section covers the SQL-based experience with Microsoft Fabric. The goal of this section is to run SQL scripts, model the data and run measures. - -### Load your dataset - -The [SQL Endpoint](https://learn.microsoft.com/en-us/fabric/data-warehouse/get-started-lakehouse-sql-endpoint?WT.mc_id=data-91115-jndemenge) is created immediately after creating the Lakehouse. - -This autogenerated SQL Endpoint that can be leveraged through familiar SQL tools such as [SQL Server Management Studio](https://learn.microsoft.com/en-us/sql/ssms/download-sql-server-management-studio-ssms?WT.mc_id=data-91115-jndemenge), [Azure Data Studio](https://learn.microsoft.com/en-us/sql/azure-data-studio/what-is-azure-data-studio?WT.mc_id=data-91115-jndemenge), the [Microsoft Fabric SQL Query Editor](https://learn.microsoft.com/en-us/fabric/data-warehouse/sql-query-editor?WT.mc_id=data-91115-jndemenge). - -You can access the SQL endpoint by opening your workspace and Click the Lakehouse name which has the **Type** as SQL endpoint - -![Workspace Interface](assets/Workspace_interface.png) - -Alternatively you can access the SQL endpoint from the Lakehouse by switching the view to **SQL endpoint** from the top right corner and in the drop down menu selecting `SQL endpoint`. - -![Lakehouse Interface](assets/Lakehouse_interface.png) - -You will see all your data loaded and we will be working with the following tables - -- `train_annotations` -- `train_images` -- `categories` - -![SQL Endpoint](assets/SQL_endpoint.png) - -There are 3 views within the Warehouse - -- **Data** - This is where all data are stored -- **Query** - This is where you build your SQL solutions -- **Model** - This is where you connect your tables together - -To learn more on [Model in Power BI](https://learn.microsoft.com/en-us/training/paths/model-data-power-bi/) - -### Create Views with SQL Query - -Based on our `train_annotations` data, we want to create a dimension for **season** column and we will use an SQL Query to do that: - -1. Click **New SQL Query** at the top of your screen -2. Write this code - -```SQL -SELECT DISTINCT season -FROM train_annotations -``` - -1. Highlight the code and click **Run** -You will see the **Results** -1. To save as View, Highlight the code again and Click **Save as View** -1. Give it a name and Click Ok -It automatically create a View in the Views Folder on your left hand side - -![Views](assets/Views.png) - -### Build Model and Define Relationship - -We want to build relationships with the 4 tables we now have - -- `train_annotations` -- `train_images` -- `categories` -- `Season` - -To create relationship : Click the **Model** below the screen where you have Data, Query and Model. You will see all the tables listed above. - -1. Click **Categories[id]** and drag to connect to **train_annotations[category_id]**. -A screen will pop up with Create Relationship. - a. Cardinality : One to Many - b. Cross filter direction : Single - c. Make this relationship active: Ticked - d. Click **Confirm** - -2. Click **Season[season]** and drag to connect to **train_annotation[season]** -A screen will pop up with Create Relationship. - a. Cardinality : One to Many - b. Cross filter direction : Single - c. Make this relationship active: Ticked - d. Click **Confirm** - -3. Click **train_images[id]** and drag to connect to **train_annotation[images_id]** -A screen will pop up with Create Relationship. - a. Cardinality : One to Many - b. Cross filter direction : Single - c. Make this relationship active: Ticked - d. Click **Confirm** - -At the end, you should have a visual like this -![Model](assets/model.png) - -### New Measures - -This section is about building measures for our analysis. Depending on your report, you will have some core measures you need for your report. - -To write our first measure - -1. Click New measure above -2. Change measure to Annotation -3. Type - -```SQL -Annotations = COUNTROWS(train_annotations) -``` - -1. Click **Mark sign** -1. On your right side, check the **properties**, you can change the **Home table** and format the measure - -Apply same steps above for a new measure called **Images** - -```SQL -Images = COUNTROWS(train_images) -``` - -Apply same steps above for a new measure called **Average_Annotation** - - -Next is creating report, if there is a table you do not want to use at the reporting phase, do the following steps. - -For this workshop, we will not be using these tables at the reporting phase - -1. test_annotations -2. test_images - -To hide them, do the following below - -1. Click the **table** -2. Click the **...** -3. Click **Hide in report view** - ---- - -## Data Visualization using Power BI - -### What we will cover - -This section covers the understanding of data analysis within Fabric . The Serengeti dataset is a collection of wildlife images captured by camera traps in the Serengeti National Park in Tanzania. The goal of this project is to analyze the trained data. - -To understand the Power BI interface, Click this [resource](https://learn.microsoft.com/en-us/power-bi/fundamentals/power-bi-service-overview) - -From our previous lesson in SQL endpoint, we have been able to create measures and build relationship, creating a report will be easier. - -Click on New Report and you will see the Power BI interface. - -![report](assets/report.png) - -This report below is what we will build for this workshop - -![dashboard](assets/dashboard.png) - -### Building the Report - -In the **filter pane**, - -![dashboard](assets/filter.png) - -- drag **Categories[id]** to the **Filters on all pages** -- in Filter type, change **Advanced filtering** to **Basic filtering** -- Click **Select all** -- unselect **0** (based on our report, we don't need it) - -To bring in Visuals - -1. For the first card visual - -![dashboard](assets/card.png) - - - Click a **card** visual , - - Click the measure called **annotation** in the **train_annotation** table - - Click the **format icon** in the Visualization pane - -![dashboard](assets/Format_visual.png) - - - Click **Visual** - - Click the Callout Value and increase the font size to 55 - - Click the Category label to increase the font size to 18 - - Click **Effects** and Click **General** - - Click and Open the Background - - On Visual border and increase Rounded corners to 15 - - On Shadow - - -2. For the second card visual - -- Click a **card** visual, click the measure called **images** in the **train_images** table -- You can Format the visual in the **format icon** in the Visualization pane - - - -3. For Slicers - -![dashboard](assets/slicer.png) - -- Click a **slicer** visual, Click **season[season]** - -- Click another **slicer** visual, Click **Category[name]** -- In the Field below Visualization, Right click **name** -- Click **Rename for thsi visual** -- Change **name** to **Animals** - - -- You can Format the visuals in the **format icon** in the Visualization pane - -4. Annotation by Season - -![dashboard](assets/clustered_barchart.png) - -- Click **Clustered bar chart** -- Click **season[season]** and **train_annotation[annotations]** -- You can Format the visual in the **format icon** in the Visualization pane - -5. Top Number of Annotations by Animals - -- Click **Clustered bar chart** -- Click **Category[name]** and **train_annotation[annotations]** -- In the Format Pane, Check the **name**, change **Advanced filtering** to **TopN** -- Show items , **Top N** and write **5** beside -- By Value, drag **train_annotation[annotations]** into the blank space -- you can Format your visual in the **format icon** in the Visualization pane - - -6. Bottom Number of Annotations by Animals - - Click **Clustered bar chart** - - Click **Category[name]** and **train_annotation[annotations]** - - In the Format Pane, Check the **name**, change **Advanced filtering** to **TopN** - - Show items , Change **Top** to **Bottom** and **5** beside - - By Value, drag **train_annotation[annotations]** into the blank space - - you can Format your visual in the **format icon** in the Visualization pane - -![dashboard](assets/dashboard.png) - -Great work in getting to this point. - -I hope you enjoyed this session, You can explore and build more visualizations with the data based on what you have learnt in this session.. - ---- - -## Data Analysis & Transformation with Apache Spark in Fabric - -
- -> You can skip this section (section 6) and Section 7 by downloading the [prep_and_transform notebook](assets/notebooks/prep_and_transform.ipynb) and follow the instructions in the `Appendix Section` to import the notebook into your workspace. Then run all the cells in the notebook. -
- -Now that we have successfully, loaded the data into the Lakehouse and explored how to leverage the SQL endpoint to create views and build relationships, we will now explore how to use Fabric Notebooks to perform data analysis and transformation. - -In this section we will learn how to use Apache Spark for data processing and analytics in a Lakehouse. To learn more about Apache Spark in Fabric see [this learn module](https://learn.microsoft.com/en-gb/training/modules/use-apache-spark-work-files-lakehouse/?WT.mc_id=data-91115-jndemenge). - -### Creating a Fabric Notebook - -To edit and run Spark code in Microsoft Fabric we will use the Notebooks which very similar to Jupyter Notebooks. To create a new Notebook, click on the ```Open Notebook``` from the Lakehouse and from the drop down menu select ```New Notebook```. This will open a new Notebook. On the top right corner of the workspace click on the Notebook name and rename it to ```analyze-and-transform-data```. Click on any empty area to close and rename the Notebook. - -![Rename Notebook](assets/analyze-and-transform-data.png) - -Before we begin the loading of the data let's install some of the libraries that we'll need. - -We will need to install the opencv library using pip. Execute the following code block in the cell to install the opencv library and imutils library which is a set of convenience tools to make working with OpenCV easier. - -```python -%pip install opencv-python imutils -``` - -### Loading data into a Spark Dataframe - -To begin we will load the annotations data from the Lakehouse `train_annotations` table. From this we get information about each season's sequences and labels. - - -We'll then filter out the relevant columns that are we need, *i.e season, seq_id, category_id, image_id and date_time* and also need to filter out all records whose *category_id is greater than 1* to exclude all empty and human images which are not relevant for this training. - -Finally remove any null values in the `image_id` column and drop any duplicate rows, finally convert the spark dataframe to a pandas dataframe for easier manipulation. - -Paste the code below into a cell of the Notebook and run it. Update the select query with the name of the your Lakehouse name. - -```python -# Read all the annotations in the train table from the lakehouse -df = spark.sql("SELECT * FROM DemoLakehouse.train_annotations WHERE train_annotations.category_id > 1") - -# filter out the season, sequence ID, category_id snf image_id -df_train = df.select("season", "seq_id", "category_id", "location", "image_id", "datetime") - -# remove image_id wiTH null and duplicates -df_train = df_train.filter(df_train.image_id.isNotNull()).dropDuplicates() - -# convert df_train to pandas dataframe -df_train = df_train.toPandas() -``` - -### Analyzing data across seasons - -Next we will define a function to plot the number of image sequences in each season. We'll achieve this by using the matplotlib and seaborn libraries. - -```python -import pandas as pd -import matplotlib.pyplot as plt -import seaborn as sns - -def plot_season_counts(df, title="Number of Sequences per Season"): - # Extract the season from the seq_id column using a lambda function - df['season'] = df.seq_id.map(lambda x: x.split('#')[0]) - - # Count the number of sequences in each season, and sort the counts by season - season_counts = df.season.value_counts().sort_index() - - # Replace 'SER_' prefix in season labels with an empty string for easy visibility - season_labels = [s.replace('SER_', '') for s in season_counts.index] - - # Create a bar plot where the x-axis represents the season and the y-axis represents the number of sequences in that season - sns.barplot(x=season_labels, y=season_counts.values) - plt.xlabel('Season') - plt.ylabel('Number of sequences') - plt.title(title) - plt.show() -``` - -This function takes a single argument `df`, which is the pandas DataFrame containing the `seq_id` column. The function first extracts the season from the `seq_id` column using a lambda function, and then counts the number of sequences in each season using the `value_counts` method of the pandas Series object. The counts are sorted by season using the `sort_index` method. - -We then can call the function and pass the `df_train` dataframe as an argument. - -```python -plot_season_counts(df_train, "Original Number of Sequences per Season") -``` - -This will plot the number of sequences in each season. - -![Original Number of Sequences per Season](assets/Original_Number_of_Sequences_per_Season.png) - -### Analyzing & transforming data across image sequences - -Since we are working with camera trap data, it is common to have multiple images in a sequence. - -
- -> A sequence is a group of images captured by a single camera trap in a single location over a short period of time. The images in a sequence are captured in rapid succession, and are often very similar to each other. -
- -We can visualize the number of images we have for each sequence and after executing the code snippet below you will notice that by far most sequences have between 1 and 3 images in them. - -```python -# Create the count plot -ax = sns.countplot(x=df_train.groupby('seq_id').size(), log=True) - -# Set the title and axis labels -ax.set_title('Number of images in each sequence') -ax.set_xlabel('Number of images') -ax.set_ylabel('Count of sequences') - -# Show the plot -plt.tight_layout() -plt.show() -``` - -Next we will load the category names from the Categories table in the lakehouse. We'll then convert the spark dataframe to a pandas dataframe. - -Next the add a new column called *label* in the df_train dataframe which is the category name for each category_id and finally remove the category_id column from df_train and rename the image_id column to filename and append the .JPG extension to the the values - -```python -import numpy as np - -# Load the Categories DataFrame into a pandas DataFrame -category_df = spark.sql("SELECT * FROM DemoLakehouse.categories").toPandas() - -# Map category IDs to category names using a vectorized approach -category_map = pd.Series(category_df.name.values, index=category_df.id) -df_train['label'] = category_map[df_train.category_id].values - -# Drop the category_id column -df_train = df_train.drop('category_id', axis=1) - -# Rename the image_id column to filename -df_train = df_train.rename(columns={'image_id': 'filename'}) - -# Append the .JPG extension to the filename column -df_train['filename'] = df_train.filename + '.JPG' -``` - -Since we are working with a sequence of images we will pick the first image from each sequence, with the assumption that the time period after a camera trap is triggered is the most likely time for an animal to be in the frame. - -```python -# reduce to first frame only for all sequences -df_train = df_train.sort_values('filename').groupby('seq_id').first().reset_index() - -df_train.count() -``` - -The `df_train.count()` method returns the number of rows in the dataframe. Which now reduces to approximately 589758 rows. - -### Analyzing the image labels - -Now that we have handled the image sequences, we will now analyze the labels and as well plot the distribution of labels in the dataset. To do this execute the code snippet below. - -```python -# Create a horizontal bar plot where the y-axis represents the label and the x-axis represents the number of images with that label -plt.figure(figsize=(8, 12)) -sns.countplot(y='label', data=df, order=df_train['label'].value_counts().index) -plt.xlabel('Number of images') -plt.ylabel('Label') - -# Set the x-axis scale to logarithmic -plt.xscale('log') - -plt.show() -``` - -The scale of the x-axis is set to logarithmic to make it easier to read the labels and normalize the distribution. Each bar represents the number of images with that label. - -### Transforming the dataframe - -Now that we have successfully analyzed the labels and sequences we'll perform some transformations on the dataframe to prepare it for downloading the images. - -To do this we will define a function that takes a filename as the input and returns the image url. - -```python -def get_ImageUrl(filename): - return f"https://lilablobssc.blob.core.windows.net/snapshotserengeti-unzipped/{filename}" -``` - -This function is then applied to the `filename` column of the `df_train` dataframe to create a new column called `image_url` which contains the url of the image. - -```python -df_train['image_url'] = df_train['filename'].apply(get_ImageUrl) -``` - -We can test this by selecting a random image and displaying it. To do this define the following two functions: - -```python -import urllib.request - -def display_random_image(label, random_state, width=500): - # Filter the DataFrame to only include rows with the specified label - df_filtered = df_train[df_train['label'] == label] - - # Select a random row from the filtered DataFrame - row = df_filtered.sample(random_state=random_state).iloc[0] - - # Load the image from the URL and display it - url = row['image_url'] - download_and_display_image(url, label) - -# use matplotlib to display the image -def download_and_display_image(url, label): - image = plt.imread(urllib.request.urlopen(url), format='jpg') - plt.imshow(image) - plt.title(f"Label: {label}") - plt.show() -``` - -Now call the `display_random_image` function with the label `leopard` and a random state of 12. - -```python -display_random_image(label='leopard', random_state=12) -``` - -![leopard](assets/leopard.png) - ---- - -## Download the image files into the Lakehouse - -Now that we have successfully analyzed the data and performed some transformations to prepare the data for downloading the images, we will now download the images into the lakehouse. - -
- -> For demo purposes we will not use the entire training dataset. Instead we will use a small percentage of the dataset. -
- -### Proportional allocation of the dataset - -We will select a subset of the data from the main dataset in a way that maintains the same proportions of the `label`, `season` and `location`. - -To do this define a function that takes in the dataset as an input and a percentage and it calculates how many data points to be included based on that percentage. - -```python -def proportional_allocation_percentage(data, percentage): - # Calculate the count of the original sample - original_count = len(data) - - # Calculate the count of the sample based on the percentage - sample_count = int((percentage / 100) * original_count) - - # Perform proportional allocation on the calculated sample count - return proportional_allocation(data, sample_count) -``` - -Notice that this function uses another function to perform the actual proportional allocation. - -```python -def proportional_allocation(data, sample_size): - # Group the data by "label", "season", and "location" columns - grouped_data = data.groupby(["label", "season", "location"]) - - # Calculate the proportion of each group in the original sample - proportions = grouped_data.size() / len(data) - - # Calculate the count of each group in the sample based on proportions - sample_sizes = np.round(proportions * sample_size).astype(int) - - # Calculate the difference between the desired sample size and the sum of rounded sample sizes - size_difference = sample_size - sample_sizes.sum() - - # Adjust the sample sizes to account for the difference - if size_difference > 0: - # If there is a shortage of items, allocate the additional items to the groups with the largest proportions - largest_proportions = proportions.nlargest(size_difference) - for group in largest_proportions.index: - sample_sizes[group] += 1 - elif size_difference < 0: - # If there is an excess of items, reduce the sample sizes from the groups with the smallest proportions - smallest_proportions = proportions.nsmallest(-size_difference) - for group in smallest_proportions.index: - sample_sizes[group] -= 1 - - # Initialize an empty list to store the sample - sample_data = [] - - # Iterate over each group and randomly sample the required count - for group, count in zip(grouped_data.groups, sample_sizes): - indices = grouped_data.groups[group] - sample_indices = np.random.choice(indices, size=count, replace=False) - sample_data.append(data.loc[sample_indices]) - - # Concatenate the sampled dataframes into a single dataframe - sample_data = pd.concat(sample_data) - - # Reset the index of the sample DataFrame - sample_data.reset_index(drop=True, inplace=True) - - return sample_data -``` - -This second function, groups the data based on the `label`, `season` and `location` columns and calculates the proportion of each group in the original sample. It then calculates the count of each group in the sample based on proportions. - -It also adjusts the sample sizes if necessary to make sure the total sample size matches the desired count. Finally, it randomly selects the appropriate number of data points from each group and returns the resulting sample, which is a smaller dataset that represents the original dataset's proportions accurately. - -For purposes of this demo we we will use `0.05%` of the original dataset. - -```python -percent = 0.05 -sampled_train = proportional_allocation_percentage(df_train, percent) -plot_season_counts(sampled_train, f"{percent}% Sample from Original Number of Sequences per Season") -``` -The image below shows a side by side comparison of the output from the execution of the `plot_season_counts` function on the original dataset and the sampled dataset above. - -![sampled](assets/sample.png) - - -### Define functions to download images - -Now that we have a sampled dataset, we will download the images into the lakehouse. - -To do, we will be using the opencv library to download the images. Define a function that takes in the url of the image and the path to download the image to. - -```python -import urllib.request -import cv2 -import imutils - -def download_and_resize_image(url, path, kind): - filename = os.path.basename(path) - directory = os.path.dirname(path) - - directory_path = f'/lakehouse/default/Files/images/{kind}/{directory}/' - - # Create the directory if it does not exist - os.makedirs(directory_path, exist_ok=True) - - # check if file already exists - if os.path.isfile(os.path.join(directory_path, filename)): - return - - # Download the image - urllib.request.urlretrieve(url, filename) - - # Read the image using OpenCV - img = cv2.imread(filename) - - # Resize the image to a reasonable ML training size using imutils - resized_img = imutils.resize(img, width=224, height=224, inter=cv2.INTER_AREA) - - # Save the resized image to a defined filepath - cv2.imwrite(os.path.join(directory_path, filename), resized_img) -``` - -The kind parameter is used to define whether the image is a training image or a validation/testing image. - -We are going to use this `download_and_resize_image` function in another function that will execute the download in parallel using the `concurrent.futures` library. - -```python -import concurrent.futures - -def execute_parallel_download(df, kind): - # Use a process pool instead of a thread pool to avoid thread safety issues - with concurrent.futures.ProcessPoolExecutor() as executor: - # Batch process images instead of processing them one at a time - urls = df['image_url'].tolist() - paths = df['filename'].tolist() - futures = [executor.submit(download_and_resize_image, url, path, kind) for url, path in zip(urls, paths)] - # Wait for all tasks to complete - concurrent.futures.wait(futures) -``` - -### Prepare the test dataset - -Next we will prepare the test data in the same way we have the train data then download both the train and test images. - -```python -df = spark.sql("SELECT * FROM DemoLakehouse.test_annotations WHERE test_annotations.category_id > 1") - -df_test = df.select("season", "seq_id", "category_id", "location", "image_id", "datetime") - -df_test= df_test.filter(df_test.image_id.isNotNull()).dropDuplicates() - -df_test = df_test.toPandas() - -df_test['label'] = category_map[df_test.category_id].values - -df_test = df_test.drop('category_id', axis=1) - -df_test = df_test.rename(columns={'image_id':'filename'}) - -df_test['filename'] = df_test.filename+ '.JPG' - -df_test = df_test.sort_values('filename').groupby('seq_id').first().reset_index() - -df_test['image_url'] = df_test['filename'].apply(get_ImageUrl) - -sampled_test = proportional_allocation_percentage(df_test, 0.27) -``` - -From this code snippet we create a test set using `0.27%` from the original test set. - -### Download the images - -Next we execute the download of the images: this will take approximately 10 minutes to complete. - -```python -import os - -execute_parallel_download(sampled_train, 'train') -execute_parallel_download(sampled_test, 'test') -``` - -### Save the sampled dataframes to parquet files - -Once the download is complete we will then save the sampled train and test dataframes to parquet files in the lakehouse, for use in the next section. We drop all the columns except the filename and label columns, since these are the only required columns for training the model. - -```python -data_dir = '/lakehouse/default/Files/data/' - -train_data_file = os.path.join(data_dir, 'sampled_train.parquet') -test_data_file = os.path.join(data_dir, 'sampled_test.parquet') - -sampled_train.loc[:, ['filename', 'label']].to_parquet(train_data_file, engine='pyarrow', compression='snappy') -sampled_test.loc[:, ['filename', 'label']].to_parquet(test_data_file, engine='pyarrow', compression='snappy') -``` - -You can view the saved parquet files from the Lakehouse explorer. - -![parquet](assets/saved_sampled.png) - -This concludes the data preparation section. The next section covers how to train your ML model using the sampled data. - ---- - -## Preparing your data for Training - -This section covers preparing out data and training a deep learning model on the Serengeti dataset. - -### Load the sample dataset - -From the previous section, our images are already loaded in the lakehouse as `parquet` files contains image details including filename and labels. First we convert the parquet files to Delta tables. In machine learning, Delta tables can be used to store training data for machine learning models, allowing us to easily update the data and retrain the model. - -![Converting parquet files to delta tables](assets/data_to_delta_tables.png) -To convert our data from parquet to delta files we: - -1. Go to our Lakehouse -1. In the Lakehouse, click on `data` -1. Right click on the train and test parquet files. You will do this for both `sample_test.parquet` and `sample_train.parquet` -1. Select **load to Tables** then **create a new table.** -1. Finally, you will see our new delta files in the LakeHouse as shown below: -![Output of our delta files](assets/data_to_delta_tables_output.png) - - Next, create a new notebook and rename it to `train-model` as described in the previous section. - -Before we continue loading our data, we will first install the two libraries we need to train our data using `pip install`. We will be training our model using [Pytorch](https://pytorch.org) which requires two libraries: torch and torchvision. `torch` is the main PyTorch package that provides the core functionality for working with tensors, building neural networks, and training models. `torchvision` is a package that provides tools and utilities for working with computer vision tasks, such as image classification and object detection. - -We will have to install the libraries separately. To install torch we run the command below: - -```python -%pip install torch -``` - -To install torchvision we run the command below: - -```python -%pip install torchvision -``` - -As our datasets are now as delta files, we load our data and convert it to a Pandas dataframe to easily manipulate and visualize our data with inbuilt Pandas tools starting with the train files: - -```python -# load our data -train_df = spark.sql("SELECT * FROM DemoLakehouse.sampled_train LIMIT 1000") - -# import pandas library that will convert our dataset into dataframes -import pandas as pd - -# convert train_df to pandas dataframe -train_df = train_df.toPandas() -``` - -Lastly, we convert our file name to read the image URL as follows: - -```python -# Create a new column in the dataframe to apply to the filename column tor read the image URL -train_df['image_url'] = train_df['filename'].apply(lambda filename: f"/lakehouse/default/Files/images/train/{filename}") - -train_df.head() -``` - -Our output will be as follows, showing the different image urls and their consecutive category labels: -![Our loaded data](assets/load_data.png) - -#### Label encoding - -First, we transform categorical data to numerical data using LabelEncoder which we import from the Scikit-Learn library. It assigns a unique integer to each category in the data, allowing machine learning algorithms to work with categorical data. - -You can do this by: - -```python -from sklearn.preprocessing import LabelEncoder - -# Create a LabelEncoder object -le = LabelEncoder() - -# Fit the LabelEncoder to the label column in the train_df DataFrame -le.fit(train_df['label']) - -# Transform the label column to numerical labels using the LabelEncoder -train_df['labels'] = le.transform(train_df['label']) -``` - -
- -> Ensure you repeat the process for test dataset, by droping the filename column and merge the two dataframes using `pd.concat()` as follows: -
- -```python -# Repeat the process for the test dataset -test_df = spark.sql("SELECT * FROM DemoLakehouse.sampled_test LIMIT 1000") - -# convert test_df to pandas dataframe -test_df = test_df.toPandas() - -# Create a new column in the dataframe using the apply method -test_df['image_url'] = test_df['filename'].apply(lambda filename: f"/lakehouse/default/Files/images/train/{filename}") - -# Fit the LabelEncoder to the label column in the test_df DataFrame -le.fit(test_df['label']) - -# Transform the label column to numerical labels using the LabelEncoder -test_df['labels'] = le.transform(test_df['label']) - -# combine both the train and test dataset -data = pd.concat([test_df, train_df]) - -# drop filename column -data = data[['image_url', 'labels']] -``` - -From this our result will be a combined dataset containing the two features we need: image url and labels. - -### Transforming our dataset - -To train our model, we will be working with Pytorch. To do this, we will need to import our`torch` and `torchvision` libraries. Next, we customize our dataset, transforming our files to tensors with the size 224x224 pixels. This is done to both the train and test dataset as follows: - -```python -from torch.utils.data import Dataset -from torch.utils.data import DataLoader -from torchvision import transforms - -import os -from PIL import Image - -class CustomDataset(Dataset): - def __init__(self, root_dir, transform=None): - self.root_dir = root_dir - self.data = data - self.transform = transform - - def __len__(self): - return len(self.data) - - def __getitem__(self, idx): - while True: - img_name = os.path.join(self.root_dir, self.data.iloc[idx, 0]) - if not os.path.exists(img_name): - idx = (idx + 1) % len(self.data) - continue - image = Image.open(img_name) - if self.transform: - image = self.transform(image) - labels = self.data.iloc[idx, 1] - return image, labels - -transform = transforms.Compose([ - transforms.Resize((224, 224)), - transforms.ToTensor() -]) - -train_set = CustomDataset("/lakehouse/default/Files/images/train/", transform=transform) -test_set = CustomDataset("/lakehouse/default/Files/images/test/", transform=transform) -``` - -`PIL` (Python Imaging Library), is a library that allows us to work with images in Python. For example, using `Image`, we can be able to load and open an image. - -Lastly, we load the training and testing datasets in batches using Dataloader as follows: - -```python -# Load the training and test data -train_loader = DataLoader(train_set, batch_size=100, shuffle=True, num_workers=2) -test_loader = DataLoader(test_set, batch_size=100, shuffle=False, num_workers=2) -``` - -The `batch_size` parameter specifies the number of samples in each batch, the `shuffle` parameter specifies whether to shuffle the data before each epoch, and the `num_workers` parameter specifies the number of subprocesses to use for data loading. - -The purpose of using data loaders is to efficiently load and pre-process large datasets in batches, which can improve the training speed and performance of machine learning models. - -### Setting up mlflow to track our experiments - -`mlflow` is an open source platform for managing the end-to-end machine learning lifecycle. It provides tools for tracking experiments, packaging code into reproducible runs, and sharing and deploying models. - -```python -# Using mlflow library to activate our ml experiment - -import mlflow - -mlflow.set_experiment("serengeti-experiment") -``` - -In the code above, we use the `set_experiment` function from the `mlflow` library to set the active experiment to "serengeti-exp". This will allow us to track the results of our machine learning experiments and compare them across different runs. - -By using `mlflow`, we can easily log and track the parameters, metrics, and artifacts of our machine learning experiments, and visualize and compare the results using the Microsoft Fabric UI. - -![Setting up your mlflow experiment](assets/mlflow_exp.png) - -From the output above, this means, any machine learning runs will be associated with this experiment enabling us to track and compare our runs. - ---- - -## Training and Evaluating the Machine Learning model - -We use a convolutional neural network (CNN) to classify the images in the Serengeti dataset. The CNN consists of several convolutional layers followed by max pooling layers and fully connected layers. - -In our case, we are load a pre-trained DenseNet 201 model from the `torchvision` library and modifying its classifier layer to output 50 classes instead of the default 1000 classes. The DenseNet 201 model is a convolutional neural network (CNN) that has been pre-trained on the ImageNet dataset, which contains millions of images across 1000 classes. The model consists of several convolutional layers followed by dense layers and a softmax output layer. - -Next, we modify the classifier layer of the pre-trained model by replacing it with a new `nn.Linear` layer that has 50 output features. This is done to adapt the pre-trained model to the specific classification task at hand, which is to classify images of wildlife in the Serengeti dataset into 50 different species. - -Additionally we check if a GPU is available and moves the model to the GPU if it is available. This is done to accelerate the training process if a GPU is available. - -After this code is executed, the `model` object will be a pre-trained DenseNet 201 model with a modified classifier layer that can be used to classify images of wildlife in the Serengeti dataset into 50 different species. The code is as follows: - -```python -import torchvision -import torch.nn as nn - -# load the pre-trained DenseNet 201 model -model = torchvision.models.densenet201(pretrained=True) -num_ftrs = model.classifier.in_features -model.classifier = nn.Linear(num_ftrs, 53) -device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") -model = model.to(device) -``` - -### Loss Function - -We use the cross-entropy loss function and the Adam optimizer to train the model. The code is as follows: - -```python -import torch.optim as optim -# define the loss function -criterion = nn.CrossEntropyLoss() -optimizer = optim.Adam(model.parameters(), lr=0.01) -``` - -The `nn.CrossEntropyLoss()` function is used to define the loss function. This loss function is commonly used for multi-class classification problems, such as the Serengeti dataset, where there are multiple classes to predict. The `nn.CrossEntropyLoss()` function combines the `nn.LogSoftmax()` and `nn.NLLLoss()` functions into a single function. - -The `optim.Adam()` function is used to define the optimizer. The Adam optimizer is a popular optimization algorithm for training deep learning models. It is an adaptive learning rate optimization algorithm that is well-suited for large datasets and high-dimensional parameter spaces. - -The `model.parameters()` function is used to specify the parameters that need to be optimized. In this case, it specifies all the parameters of the `model` object, which includes the weights and biases of the convolutional layers and the classifier layer. - -The learning rate for the optimizer is set to 0.01 using the `lr` parameter. This is the step size at which the optimizer updates the model parameters during training. - -### Training our model - -Using the DenseNet Model we just loaded, we go ahead and train our model as shown below. The training will take upto 10 minutes to complete. You can play around with the number of epochs to increase the accuracy of your model, however, the more the epochs the longer it will take for the training to be completed. - -```python -# train the model -num_epochs = 5 -for epoch in range(num_epochs): - print('Epoch {}/{}'.format(epoch, num_epochs - 1)) - print('-' * 10) - - # Each epoch has a training and validation phase - for phase in ["train", "val"]: - if phase == "train": - model.train() # Set model to training mode - else: - model.eval() # Set model to evaluate mode - - running_loss = 0.0 - running_corrects = 0 - for i, data in enumerate(train_loader, ): - # get the inputs - inputs, labels = data[0].to(device), data[1].to(device) - inputs = inputs.to(device) - labels = labels.to(device) - - # zero the parameter gradients - optimizer.zero_grad() - - # forward - # track history if only in train - with torch.set_grad_enabled(phase == "train"): - outputs = model(inputs) - _, preds = torch.max(outputs, 1) - loss = criterion(outputs, labels) - - # backward + optimize only if in training phase - if phase == "train": - loss.backward() - optimizer.step() - - # print statistics - running_loss += loss.item() - if i % 100 == 99: # print every 100 mini-batches - print('[%d, %5d] loss: %.3f' % - (epoch + 1, i + 1, running_loss / 100)) - running_loss = 0.0 - - print('Finished Training') -``` - -The code above shows training of our DenseNet model over 5 epochs using our data for training and validation. At the end of each phase, the code computes the loss and accuracy of our model and once each set is done, it returns, `Finished Training` as the output as shown below: - -![model_training](assets/model_training.png) - -### Saving the model - -We can also use the `mlflow` library to log the trained PyTorch model to the MLflow tracking server and register it as a model version with the name "serengeti-pytorch". Once the model is saved, it can be loaded and used later for inference or further training. - -The code for this is: - -```python -# use an MLflow run and track the results within our machine learning experiment. - -with mlflow.start_run() as run: - print("log pytorch model:") - mlflow.pytorch.log_model( - model, "pytorch-model", - registered_model_name="serengeti-pytorch" - ) - - model_uri = "runs:/{}/pytorch-model".format(run.info.run_id) - print("Model saved in run %s" % run.info.run_id) - print(f"Model URI: {model_uri}") -``` - -The results outputs our `Model URI` and the model version as shown below: -![Output of saving the mlflow model](assets/mlflow_model.png) - -### Evaluating the Machine Learning model - -Once we have trained our model, the next step is to evaluate its performance. We load our PyTorch model from the MLflow tracking server using the `mlflow.pytorch.load_model()` function and evaluating it on the test dataset. - -Once the evaluation is complete, the code prints the final test loss and accuracy. - -```python -# load and evaluate the model -loaded_model = mlflow.pytorch.load_model(model_uri) -print(type(loaded_model)) -correct_cnt, total_cnt, ave_loss = 0, 0, 0 -for batch_idx, (x, target) in enumerate(test_loader): - x, target = x, target - out = loaded_model(x) - loss = criterion(out, target) - _, pred_label = torch.max(out.data, 1) - total_cnt += x.data.size()[0] - correct_cnt += (pred_label == target.data).sum() - ave_loss = (ave_loss * batch_idx + loss.item()) / (batch_idx + 1) - - if (batch_idx + 1) % 100 == 0 or (batch_idx + 1) == len(test_loader): - print( - "==>>> epoch: {}, batch index: {}, test loss: {:.6f}, acc: {:.3f}".format( - epoch, batch_idx + 1, ave_loss, correct_cnt * 1.0 / total_cnt - ) - ) - -``` - -Model evaluation results gives out the epochs, batch index, test loss and model accuracy. To increase our model accuracy, we may need to include more images to our train and test set: -![model_evaluation](assets/model_evaluation.png) - -Next, we test our model with a single image. We use the `PIL` library to load an image from a file as shown below: - -```python -# Load a new image from the test data using Pillow -image = Image.open('/lakehouse/default/Files/images/test/SER_S11/B03/B03_R1/SER_S11_B03_R1_IMAG1021.JPG') -image -``` - -After loading the image we then resize it to a fixed size, convert it to a PyTorch tensor, pass it through our trained PyTorch model, and getting the class label it belongs to as follows: - -```python -# Resize the image to a fixed size -resize_transform = transforms.Resize((224, 224)) -image = resize_transform(image) - -# Convert the image to a PyTorch tensor -tensor_transform = transforms.ToTensor() -tensor = tensor_transform(image) - -# Add a batch dimension to the tensor -tensor = tensor.unsqueeze(0) - -# Load the model from MLflow -model = mlflow.pytorch.load_model(model_uri) - -# Set the model to evaluation mode -model.eval() - -# Pass the tensor through the model to get the output -with torch.no_grad(): - output = model(tensor) - -# Get the predicted class -_, predicted = torch.max(output.data, 1) - -print(predicted.item()) -``` - -Finally the output will be a number representing the class label of our image. By doing this, we have been able to train our model and have our model classify an image. - -As we already have our model logged in Microsoft Fabric using mlflow, we can download the `pkl` files and use it in our applications. Additionally, we can go ahead and visualize our model performance using Power BI. - -This concludes our workshop. The next section covers all the resources you will need to continue your Microsoft Fabric journey. - ---- - -## Resources - -- [Notebook for training our model](assets/notebooks/train_model.ipynb) -- [Notebook for preparing and transforming our data](assets/notebooks/prep_and_transform.ipynb) -- [Get Started with Microsoft Fabric](https://learn.microsoft.com/en-us/fabric/get-started/microsoft-fabric-overview?WT.mc_id=academic-91115-bethanycheum) -- [Explore Lakehouses in Microsoft Fabric](https://learn.microsoft.com/en-us/training/modules/get-started-lakehouses/?WT.mc_id=academic-91115-bethanycheum) -- [Ingest Data with Dataflows Gen2 in Microsoft Fabric](https://learn.microsoft.com/en-us/training/modules/use-dataflow-gen-2-fabric/?WT.mc_id=academic-91115-bethanycheum) -- [Get Started with data science in Microsoft Fabric](https://learn.microsoft.com/en-us/training/modules/get-started-data-science-fabric/?WT.mc_id=academic-91115-bethanycheum) -- [Grow and Learn with the Microsoft Fabric Community](https://community.fabric.microsoft.com/?WT.mc_id=academic-91115-bethanycheum) - ---- - -## Appendix - -### Importing Notebook into the Workspace -To import an existing notebook into the workspace, on the bottom left of your workspace switch to the `Data Engineering` workload. In the page that appears click on `Import Notebook` then click the `Upload` button on the pane that opens. - -![Importing Notebook](assets/import_notebook.png) - -Select the notebook you want to import. After successful import, navigate back to your workspace and you will find the recently imported notebook. - -Open the notebook and if the Lakehouse explorer indicates that `Missing Lakehouse`, click on the arrows to the left of the error and on the dialog that appears click on `Add Lakehouse`. - -![Missing Lakehouse](assets/missing_lakehouse.png) - -On the dialog that appears SELECT `Existing lakehouse` and click `Add`. Select your preferred lakehouse and click `Add`. - diff --git a/workshops/fhir-on-azure/assets/app-id.png b/workshops/fhir-on-azure/assets/app-id.png deleted file mode 100644 index dcecab82..00000000 Binary files a/workshops/fhir-on-azure/assets/app-id.png and /dev/null differ diff --git a/workshops/fhir-on-azure/assets/bearer-test-section.png b/workshops/fhir-on-azure/assets/bearer-test-section.png deleted file mode 100644 index 2698bc3b..00000000 Binary files a/workshops/fhir-on-azure/assets/bearer-test-section.png and /dev/null differ diff --git a/workshops/fhir-on-azure/assets/bearer-token.png b/workshops/fhir-on-azure/assets/bearer-token.png deleted file mode 100644 index 8efdd7b5..00000000 Binary files a/workshops/fhir-on-azure/assets/bearer-token.png and /dev/null differ diff --git a/workshops/fhir-on-azure/assets/capability-statement.png b/workshops/fhir-on-azure/assets/capability-statement.png deleted file mode 100644 index f3cb2380..00000000 Binary files a/workshops/fhir-on-azure/assets/capability-statement.png and /dev/null differ diff --git a/workshops/fhir-on-azure/assets/client-secret.png b/workshops/fhir-on-azure/assets/client-secret.png deleted file mode 100644 index 624ba419..00000000 Binary files a/workshops/fhir-on-azure/assets/client-secret.png and /dev/null differ diff --git a/workshops/fhir-on-azure/assets/create-new-registration.png b/workshops/fhir-on-azure/assets/create-new-registration.png deleted file mode 100644 index 0bfbe588..00000000 Binary files a/workshops/fhir-on-azure/assets/create-new-registration.png and /dev/null differ diff --git a/workshops/fhir-on-azure/assets/create-patient-response.png b/workshops/fhir-on-azure/assets/create-patient-response.png deleted file mode 100644 index 599479ea..00000000 Binary files a/workshops/fhir-on-azure/assets/create-patient-response.png and /dev/null differ diff --git a/workshops/fhir-on-azure/assets/create-patient.png b/workshops/fhir-on-azure/assets/create-patient.png deleted file mode 100644 index e032c5f9..00000000 Binary files a/workshops/fhir-on-azure/assets/create-patient.png and /dev/null differ diff --git a/workshops/fhir-on-azure/assets/fhir-bundles-processed.png b/workshops/fhir-on-azure/assets/fhir-bundles-processed.png deleted file mode 100644 index ed135ca4..00000000 Binary files a/workshops/fhir-on-azure/assets/fhir-bundles-processed.png and /dev/null differ diff --git a/workshops/fhir-on-azure/assets/fhir-endpoint.png b/workshops/fhir-on-azure/assets/fhir-endpoint.png deleted file mode 100644 index fcaf3563..00000000 Binary files a/workshops/fhir-on-azure/assets/fhir-endpoint.png and /dev/null differ diff --git a/workshops/fhir-on-azure/assets/fhir-loader-button.png b/workshops/fhir-on-azure/assets/fhir-loader-button.png deleted file mode 100644 index afd6564e..00000000 Binary files a/workshops/fhir-on-azure/assets/fhir-loader-button.png and /dev/null differ diff --git a/workshops/fhir-on-azure/assets/fhir-loader-deploy-2.png b/workshops/fhir-on-azure/assets/fhir-loader-deploy-2.png deleted file mode 100644 index 9a2b6312..00000000 Binary files a/workshops/fhir-on-azure/assets/fhir-loader-deploy-2.png and /dev/null differ diff --git a/workshops/fhir-on-azure/assets/fhir-loader-deploy.png b/workshops/fhir-on-azure/assets/fhir-loader-deploy.png deleted file mode 100644 index 10e99bd7..00000000 Binary files a/workshops/fhir-on-azure/assets/fhir-loader-deploy.png and /dev/null differ diff --git a/workshops/fhir-on-azure/assets/fhir-service-creation.png b/workshops/fhir-on-azure/assets/fhir-service-creation.png deleted file mode 100644 index fc08410f..00000000 Binary files a/workshops/fhir-on-azure/assets/fhir-service-creation.png and /dev/null differ diff --git a/workshops/fhir-on-azure/assets/fhir-service-search.png b/workshops/fhir-on-azure/assets/fhir-service-search.png deleted file mode 100644 index ccf7ff85..00000000 Binary files a/workshops/fhir-on-azure/assets/fhir-service-search.png and /dev/null differ diff --git a/workshops/fhir-on-azure/assets/member-selection.png b/workshops/fhir-on-azure/assets/member-selection.png deleted file mode 100644 index 16a84255..00000000 Binary files a/workshops/fhir-on-azure/assets/member-selection.png and /dev/null differ diff --git a/workshops/fhir-on-azure/assets/new-collection.png b/workshops/fhir-on-azure/assets/new-collection.png deleted file mode 100644 index 2f9d453e..00000000 Binary files a/workshops/fhir-on-azure/assets/new-collection.png and /dev/null differ diff --git a/workshops/fhir-on-azure/assets/new-environment.png b/workshops/fhir-on-azure/assets/new-environment.png deleted file mode 100644 index 42fbe63c..00000000 Binary files a/workshops/fhir-on-azure/assets/new-environment.png and /dev/null differ diff --git a/workshops/fhir-on-azure/assets/new-registration-button.png b/workshops/fhir-on-azure/assets/new-registration-button.png deleted file mode 100644 index fd826090..00000000 Binary files a/workshops/fhir-on-azure/assets/new-registration-button.png and /dev/null differ diff --git a/workshops/fhir-on-azure/assets/patient-body.png b/workshops/fhir-on-azure/assets/patient-body.png deleted file mode 100644 index 33cb2ba8..00000000 Binary files a/workshops/fhir-on-azure/assets/patient-body.png and /dev/null differ diff --git a/workshops/fhir-on-azure/assets/practitioner-search.png b/workshops/fhir-on-azure/assets/practitioner-search.png deleted file mode 100644 index ef6b404f..00000000 Binary files a/workshops/fhir-on-azure/assets/practitioner-search.png and /dev/null differ diff --git a/workshops/fhir-on-azure/assets/retrieve-patient.png b/workshops/fhir-on-azure/assets/retrieve-patient.png deleted file mode 100644 index 69aa0410..00000000 Binary files a/workshops/fhir-on-azure/assets/retrieve-patient.png and /dev/null differ diff --git a/workshops/fhir-on-azure/assets/role-assignment.png b/workshops/fhir-on-azure/assets/role-assignment.png deleted file mode 100644 index 86dd41ec..00000000 Binary files a/workshops/fhir-on-azure/assets/role-assignment.png and /dev/null differ diff --git a/workshops/fhir-on-azure/assets/role-selection.png b/workshops/fhir-on-azure/assets/role-selection.png deleted file mode 100644 index bccda5a1..00000000 Binary files a/workshops/fhir-on-azure/assets/role-selection.png and /dev/null differ diff --git a/workshops/fhir-on-azure/assets/search-app-registration.png b/workshops/fhir-on-azure/assets/search-app-registration.png deleted file mode 100644 index d71aeb2f..00000000 Binary files a/workshops/fhir-on-azure/assets/search-app-registration.png and /dev/null differ diff --git a/workshops/fhir-on-azure/assets/storage-account.png b/workshops/fhir-on-azure/assets/storage-account.png deleted file mode 100644 index 6708d40e..00000000 Binary files a/workshops/fhir-on-azure/assets/storage-account.png and /dev/null differ diff --git a/workshops/fhir-on-azure/assets/synthea-download.png b/workshops/fhir-on-azure/assets/synthea-download.png deleted file mode 100644 index 83a15c9b..00000000 Binary files a/workshops/fhir-on-azure/assets/synthea-download.png and /dev/null differ diff --git a/workshops/fhir-on-azure/assets/upload-button.png b/workshops/fhir-on-azure/assets/upload-button.png deleted file mode 100644 index e628fe89..00000000 Binary files a/workshops/fhir-on-azure/assets/upload-button.png and /dev/null differ diff --git a/workshops/fhir-on-azure/assets/workspace-creation.png b/workshops/fhir-on-azure/assets/workspace-creation.png deleted file mode 100644 index d82dc71a..00000000 Binary files a/workshops/fhir-on-azure/assets/workspace-creation.png and /dev/null differ diff --git a/workshops/fhir-on-azure/assets/workspace-search.png b/workshops/fhir-on-azure/assets/workspace-search.png deleted file mode 100644 index f99bee28..00000000 Binary files a/workshops/fhir-on-azure/assets/workspace-search.png and /dev/null differ diff --git a/workshops/fhir-on-azure/workshop.md b/workshops/fhir-on-azure/workshop.md deleted file mode 100644 index 8a4032be..00000000 --- a/workshops/fhir-on-azure/workshop.md +++ /dev/null @@ -1,336 +0,0 @@ ---- -published: true -type: workshop -title: FHIR on Azure Health Data Services -short_title: FHIR on Azure -description: Learn how to deploy Azure Health Data Services workspace using Azure portal, generate FHIR resources with Synthea, load them up to the server, query the FHIR server using Postman, and update the FHIR resources using Postman. -level: beginner -authors: - - Martyna Marcinkowska -contacts: - - '@tectonia' -duration_minutes: 120 -tags: FHIR, Azure, Postman, Azure Health Data Services ---- - -# FHIR on Azure Health Data Services - -## Introduction - -In this workshop, you will learn how to: - -- deploy Azure Health Data Services workspace using Azure portal, -- generate FHIR resources with Synthea, -- use the FHIR bulk loader to upload your resources to FHIR server, -- query the FHIR server using Postman, -- update the FHIR resources using Postman. - -**Duration:** 2 hours - -### What is FHIR? - -Fast Healthcare Interoperability Resources (FHIR, pronounced "fire" 🔥) is a standard describing data formats and elements (known as "resources") and an application programming interface (API) for exchanging electronic health records (EHR). The standard was created by the Health Level Seven International (HL7) healthcare standards organization. More information can be found [on the HL7 website](https://www.hl7.org/fhir/summary.html). - -## Prerequisites - -1. **Azure Subscription**: You will need an Azure subscription - you can create one for free [here](https://azure.microsoft.com/en-gb/free/). Ensure that you have the necessary permissions to create resources, app registrations and assign roles within this subscription. -2. **Postman**: You will need to have Postman installed on your machine. You can download it [here](https://www.postman.com/downloads/). - ---- - -## Deploy Azure Health Data Services workspace using Azure portal - -**Duration:** 15 minutes - -1. Go to [Azure portal](https://portal.azure.com) and search for "Health Data Services". - - ![alt text](assets/workspace-search.png) - -2. Click on "Create" to create a new workspace as shown below. Choose your subscription, create new resource group and choose a unique name for the workspace. Choose the region closest to you. The rest of the fields can be left as default. - - ![alt text](assets/workspace-creation.png) - - > Alternative instructions can be found [here](https://learn.microsoft.com/en-us/azure/healthcare-apis/healthcare-apis-quickstart). - -3. Once the resource is created, navigate to the workspace and go to the "FHIR Service" tab. Click on "Add FHIR Service" to create a new FHIR server. - - ![alt text](assets/fhir-service-search.png) - -4. Choose a unique name for the FHIR service and click on "Create". The rest of the fields can be left as default. - - ![alt text](assets/fhir-service-creation.png) - - > Alternative instructions can be found [here](https://learn.microsoft.com/en-us/azure/healthcare-apis/fhir/fhir-portal-quickstart). - ---- - -## Generate FHIR resources with Synthea and load them up to the server - -**Duration:** 45 minutes - -1. Navigate to [FHIR-Bulk Loader](https://github.com/microsoft/fhir-loader) tool and follow the [instructions](https://github.com/microsoft/fhir-loader?tab=readme-ov-file#deployment) to install the tool. The tool is used to load FHIR resources to the FHIR server in bulk. - - - Click on "Deploy to Azure" to deploy the tool to Azure. - - ![alt text](assets/fhir-loader-button.png) - - - Fill in the required fields: - - resource group (create a new one or use an existing one), - - region (closest to you), - - and the name of your previously created FHIR service. - - Click on "Review + create" and then "Create". - - ![alt text](assets/fhir-loader-deploy.png) - ![alt text](assets/fhir-loader-deploy-2.png) - -2. Navigate to the [Synthea](https://github.com/synthetichealth/synthea) GitHub repo and familiarise yourself with the tool. Synthea is a Synthetic Patient Population Simulator. It is used to generate realistic (but not real) patient data and associated health records in FHIR format or CSV. - -
- - > You can either download sample data from the [Synthea website](https://synthea.mitre.org/downloads) or generate your own data following the instructions [here](https://github.com/synthetichealth/synthea/wiki/Basic-Setup-and-Running). - -
- -3. To download sample data, navigate to the [downloads page](https://synthea.mitre.org/downloads) and download the data in FHIR format. - - ![alt text](assets/synthea-download.png) - -4. Back in the Azure portal, navigate to the resource group where the FHIR-Bulk Loader tool was deployed. Click on the Storage account, navigate to Storage browser, then "Blob containers" and upload the downloaded FHIR data (zip folder) to the zip container. - - ![alt text](assets/storage-account.png) - - ![alt text](assets/upload-button.png) - -
- - > If you generated the data using Synthea in a different format, follow the guidance [here](https://github.com/microsoft/fhir-loader?tab=readme-ov-file#importing-fhir-data) on the correct container to upload the data to. - -
- - You should see the zip folder moved to 'zipprocessed' container after a few minutes and as the bundles are processed, they will appear in 'bundles' before being moved to the 'bundlesprocessed' container. - - ![alt text](assets/fhir-bundles-processed.png) - ---- - -## Query & update a FHIR resource using Postman - -**Duration:** 60 minutes - -### Set up the service principal - -1. Go to the Azure portal and search for 'App registrations'. Then select 'New registration' and choose a name unique in your Azure tenant. Click on 'Register'. - - ![alt text](assets/search-app-registration.png) - - ![alt text](assets/new-registration-button.png) - - ![alt text](assets/create-new-registration.png) - -2. Go to the newly created app registration and make note of the 'Application (client) ID' and 'Directory (tenant) ID' - - ![alt text](assets/app-id.png) - -3. Navigate to 'Certificates & secrets' and create a new client secret, make note of the value. - - ![alt text](assets/client-secret.png) - -
- - > This value will only be shown once, so make sure to copy it and store it securely. - -
- -4. Navigate to the FHIR service you created earlier and make note of the 'Subscription ID' and 'FHIR endpoint'. - - ![alt text](assets/fhir-endpoint.png) - -5. Navigate to the 'Access Control (IAM)' section of the FHIR service and click on 'Add role assignment' - - ![alt text](assets/role-assignment.png) - -6. In the 'Role' section choose 'FHIR Data Contributor'. Next, in the 'Members' section, click on 'Select members' and add the app registration you created earlier, then 'Review + assign'. - - ![alt text](assets/role-selection.png) - - ![alt text](assets/member-selection.png) - -### Set up Postman - -1. Open Postman and create a new environment with the following variables: - - ![alt text](assets/new-environment.png) - - Fill out the values with the information you gathered earlier (besides the bearerToken, which will be filled out later). Remember to save! - -2. Create a new empty collection and make sure that the environment you just created is selected. - - ![alt text](assets/new-collection.png) - -3. Create your first request by clicking on the 'New Request' button and filling out the request details. This request should give you the capability statement of the FHIR service. - - ![alt text](assets/capability-statement.png) - -4. Next, create a new request to get the bearer token. This request will use the client ID, tenant ID, and client secret you created earlier. Your request should look like this: - - ![alt text](assets/bearer-token.png) - -5. In the 'Tests' tab of the request, enter the following to save the bearer token to the environment: - - ```javascript - pm.environment.set("bearerToken", pm.response.json().access_token); - ``` - - ![alt text](assets/bearer-test-section.png) - -6. Click on 'Send'. You should get a response with a bearer token as visible on the previous screenshot. You can verify that the token is saved in the environment by clicking on the 'eye' icon next to the 'bearerToken' variable. This token will be used in the Authorization header of all subsequent requests. - -### Query the FHIR server - -1. Create a new request where we will create a new patient. The request should look like this: - - ![alt text](assets/create-patient.png) - - Choose 'POST' as the request type and fill in the 'Request URL' with the FHIR endpoint and the resource type you want to create (in this case 'Patient'). In the 'Headers' tab, add the 'Authorization' header with the value 'Bearer {{bearerToken}}'. - -2. Navigate to the 'Body' tab, select 'raw' and 'JSON' and fill in the request body with the patient data you want to create. You can find the structure of the request body [here](https://www.hl7.org/fhir/patient.html). - - ![alt text](assets/patient-body.png) - - Here is an example patient you can use: - -
- JSON Data (click to expand) - - ```json - { - "resourceType": "Patient", - "meta": { - "profile": [ - "http://hl7.org/fhir/us/core/StructureDefinition/us-core-patient" - ] - }, - "text": { - "status": "generated", - "div": "
JohnTest RadleyTest
" - }, - "extension": [ - { - "url": "http://hl7.org/fhir/us/core/StructureDefinition/us-core-race", - "valueCodeableConcept": { - "coding": [ - { - "system": "http://hl7.org/fhir/v3/Race", - "code": "2106-3", - "display": "White" - } - ], - "text": "race" - } - }, - { - "url": "http://hl7.org/fhir/us/core/StructureDefinition/us-core-ethnicity", - "valueCodeableConcept": { - "coding": [ - { - "system": "http://hl7.org/fhir/v3/Ethnicity", - "code": "2186-5", - "display": "Nonhispanic" - } - ], - "text": "ethnicity" - } - }, - { - "url": "http://hl7.org/fhir/us/core/StructureDefinition/us-core-birthsex", - "valueCode": "M" - } - ], - "identifier": [ - { - "use": "official", - "type": { - "coding": [ - { - "system": "http://terminology.hl7.org/CodeSystem/v2-0203", - "code": "PI", - "display": "Patient internal identifier" - } - ] - }, - "system": "https://www.kirsandinc.com/faye", - "value": "TEST111222", - "period": { - "start": "2002-06-07" - }, - "assigner": { - "display": "EMPI - Enterprise ID system" - } - } - ], - "name": [ - { - "use": "official", - "text": "JohnTest RadleyTest", - "family": "RadleyTest", - "given": [ - "JohnTest" - ] - } - ], - "telecom": [ - { - "system": "phone", - "value": "666-666-6666", - "use": "mobile" - }, - { - "system": "email", - "value": "JohnTestRadleyTest@JohnTestRadleyTest.com" - } - ], - "gender": "male", - "birthDate": "1984-10-07", - "address": [ - { - "use": "home", - "line": [ - "1111 Eveready Drive" - ], - "city": "San Jose", - "state": "CA", - "postalCode": "94134", - "country": "USA" - } - ] - } - ``` - -
- -3. Click on 'Send' and you should get a response with the patient you just created. - - ![alt text](assets/create-patient-response.png) - -4. Create a new request to get the patient you just created. The request should look like this: - - ![alt text](assets/retrieve-patient.png) - -Congratulations! 🎉 You have successfully created a patient and retrieved it from the FHIR server. - -> Alternative instructions on how to interact with the FHIR server through Postman can be found [here](https://learn.microsoft.com/en-us/azure/healthcare-apis/fhir/use-postman). - -### Optional: search capability exploration - -You can now also browse the resources that have been created in the Azure portal. Create some more requests to interact with the FHIR server and explore the different resources you have uploaded. - -For example, search for a practitioner, an observation, or a patient by address or birth date. Here's how to search for a practitioner based in Fitchburg: - -![alt text](assets/practitioner-search.png) - -
- -> Use the [FHIR Search](https://www.hl7.org/fhir/search.html) documentation to help you construct your queries. You can also use the [FHIR Postman collection](https://github.com/Azure-Samples/azure-health-data-and-ai-samples/tree/main/samples/sample-postman-queries) to help you get started. - -
diff --git a/workshops/github-copilot-java/assets/GitHub-Copilot-Chat-Blocked-Public.png b/workshops/github-copilot-java/assets/GitHub-Copilot-Chat-Blocked-Public.png deleted file mode 100644 index 899a6c95..00000000 Binary files a/workshops/github-copilot-java/assets/GitHub-Copilot-Chat-Blocked-Public.png and /dev/null differ diff --git a/workshops/github-copilot-java/assets/GitHub-Copilot-Chat-Code-Explain-1.png b/workshops/github-copilot-java/assets/GitHub-Copilot-Chat-Code-Explain-1.png deleted file mode 100644 index 89d98fbb..00000000 Binary files a/workshops/github-copilot-java/assets/GitHub-Copilot-Chat-Code-Explain-1.png and /dev/null differ diff --git a/workshops/github-copilot-java/assets/GitHub-Copilot-Chat-Code-Explain-2.png b/workshops/github-copilot-java/assets/GitHub-Copilot-Chat-Code-Explain-2.png deleted file mode 100644 index 3e5cde5a..00000000 Binary files a/workshops/github-copilot-java/assets/GitHub-Copilot-Chat-Code-Explain-2.png and /dev/null differ diff --git a/workshops/github-copilot-java/assets/GitHub-Copilot-Chat-Code-Explain-3.png b/workshops/github-copilot-java/assets/GitHub-Copilot-Chat-Code-Explain-3.png deleted file mode 100644 index 9d409cab..00000000 Binary files a/workshops/github-copilot-java/assets/GitHub-Copilot-Chat-Code-Explain-3.png and /dev/null differ diff --git a/workshops/github-copilot-java/assets/GitHub-Copilot-Chat-Generate-Test1.png b/workshops/github-copilot-java/assets/GitHub-Copilot-Chat-Generate-Test1.png deleted file mode 100644 index bfafc66e..00000000 Binary files a/workshops/github-copilot-java/assets/GitHub-Copilot-Chat-Generate-Test1.png and /dev/null differ diff --git a/workshops/github-copilot-java/assets/GitHub-Copilot-Chat-Generate-Test2.png b/workshops/github-copilot-java/assets/GitHub-Copilot-Chat-Generate-Test2.png deleted file mode 100644 index d6eb804e..00000000 Binary files a/workshops/github-copilot-java/assets/GitHub-Copilot-Chat-Generate-Test2.png and /dev/null differ diff --git a/workshops/github-copilot-java/assets/GitHub-Copilot-Chat-Test-Generate-Res-1.png b/workshops/github-copilot-java/assets/GitHub-Copilot-Chat-Test-Generate-Res-1.png deleted file mode 100644 index 5d959e5b..00000000 Binary files a/workshops/github-copilot-java/assets/GitHub-Copilot-Chat-Test-Generate-Res-1.png and /dev/null differ diff --git a/workshops/github-copilot-java/assets/GitHub-Copilot-Chat-Test-Generate-Res-2.png b/workshops/github-copilot-java/assets/GitHub-Copilot-Chat-Test-Generate-Res-2.png deleted file mode 100644 index f8a387f8..00000000 Binary files a/workshops/github-copilot-java/assets/GitHub-Copilot-Chat-Test-Generate-Res-2.png and /dev/null differ diff --git a/workshops/github-copilot-java/assets/GitHub-Copilot-Chat-Test-Generate-Res-3.png b/workshops/github-copilot-java/assets/GitHub-Copilot-Chat-Test-Generate-Res-3.png deleted file mode 100644 index 81c0b6bb..00000000 Binary files a/workshops/github-copilot-java/assets/GitHub-Copilot-Chat-Test-Generate-Res-3.png and /dev/null differ diff --git a/workshops/github-copilot-java/assets/GitHub-Copilot-Chat-Test-Generate-Res-4.png b/workshops/github-copilot-java/assets/GitHub-Copilot-Chat-Test-Generate-Res-4.png deleted file mode 100644 index 71ce712b..00000000 Binary files a/workshops/github-copilot-java/assets/GitHub-Copilot-Chat-Test-Generate-Res-4.png and /dev/null differ diff --git a/workshops/github-copilot-java/assets/GitHub-Copilot-Chat-ask-mistake-1.png b/workshops/github-copilot-java/assets/GitHub-Copilot-Chat-ask-mistake-1.png deleted file mode 100644 index a983664d..00000000 Binary files a/workshops/github-copilot-java/assets/GitHub-Copilot-Chat-ask-mistake-1.png and /dev/null differ diff --git a/workshops/github-copilot-java/assets/GitHub-Copilot-Chat-ask-mistake-2.png b/workshops/github-copilot-java/assets/GitHub-Copilot-Chat-ask-mistake-2.png deleted file mode 100644 index 0c22fec6..00000000 Binary files a/workshops/github-copilot-java/assets/GitHub-Copilot-Chat-ask-mistake-2.png and /dev/null differ diff --git a/workshops/github-copilot-java/assets/GitHub-Copilot-Chat-explanation-code.png b/workshops/github-copilot-java/assets/GitHub-Copilot-Chat-explanation-code.png deleted file mode 100644 index 05affdc3..00000000 Binary files a/workshops/github-copilot-java/assets/GitHub-Copilot-Chat-explanation-code.png and /dev/null differ diff --git a/workshops/github-copilot-java/assets/GitHub-Copilot-Chat-fix-error.mp4 b/workshops/github-copilot-java/assets/GitHub-Copilot-Chat-fix-error.mp4 deleted file mode 100755 index e4091faa..00000000 Binary files a/workshops/github-copilot-java/assets/GitHub-Copilot-Chat-fix-error.mp4 and /dev/null differ diff --git a/workshops/github-copilot-java/assets/GitHub-Copilot-Chat-modify-the-code-1.png b/workshops/github-copilot-java/assets/GitHub-Copilot-Chat-modify-the-code-1.png deleted file mode 100644 index a7dc8f06..00000000 Binary files a/workshops/github-copilot-java/assets/GitHub-Copilot-Chat-modify-the-code-1.png and /dev/null differ diff --git a/workshops/github-copilot-java/assets/GitHub-Copilot-Chat-modify-the-code-2.png b/workshops/github-copilot-java/assets/GitHub-Copilot-Chat-modify-the-code-2.png deleted file mode 100644 index eaf817ed..00000000 Binary files a/workshops/github-copilot-java/assets/GitHub-Copilot-Chat-modify-the-code-2.png and /dev/null differ diff --git a/workshops/github-copilot-java/assets/GitHub-Copilot-Chat-modify-the-code-3.png b/workshops/github-copilot-java/assets/GitHub-Copilot-Chat-modify-the-code-3.png deleted file mode 100644 index d11c883f..00000000 Binary files a/workshops/github-copilot-java/assets/GitHub-Copilot-Chat-modify-the-code-3.png and /dev/null differ diff --git a/workshops/github-copilot-java/assets/GitHub-Copilot-Chat-rejected-prompt.png b/workshops/github-copilot-java/assets/GitHub-Copilot-Chat-rejected-prompt.png deleted file mode 100644 index b12f2d9a..00000000 Binary files a/workshops/github-copilot-java/assets/GitHub-Copilot-Chat-rejected-prompt.png and /dev/null differ diff --git a/workshops/github-copilot-java/assets/GitHub-Copilot-bottom-bar.png b/workshops/github-copilot-java/assets/GitHub-Copilot-bottom-bar.png deleted file mode 100644 index b04dc732..00000000 Binary files a/workshops/github-copilot-java/assets/GitHub-Copilot-bottom-bar.png and /dev/null differ diff --git a/workshops/github-copilot-java/assets/GitHub-Copilot-chat-sample1.png b/workshops/github-copilot-java/assets/GitHub-Copilot-chat-sample1.png deleted file mode 100644 index 33d03974..00000000 Binary files a/workshops/github-copilot-java/assets/GitHub-Copilot-chat-sample1.png and /dev/null differ diff --git a/workshops/github-copilot-java/assets/GitHub-Copilot-chat-sample2.png b/workshops/github-copilot-java/assets/GitHub-Copilot-chat-sample2.png deleted file mode 100644 index 1dd80a90..00000000 Binary files a/workshops/github-copilot-java/assets/GitHub-Copilot-chat-sample2.png and /dev/null differ diff --git a/workshops/github-copilot-java/assets/GitHub-Login-1.png b/workshops/github-copilot-java/assets/GitHub-Login-1.png deleted file mode 100644 index 4e9a2bfe..00000000 Binary files a/workshops/github-copilot-java/assets/GitHub-Login-1.png and /dev/null differ diff --git a/workshops/github-copilot-java/assets/GitHub-Login-2.png b/workshops/github-copilot-java/assets/GitHub-Login-2.png deleted file mode 100644 index f432eafa..00000000 Binary files a/workshops/github-copilot-java/assets/GitHub-Login-2.png and /dev/null differ diff --git a/workshops/github-copilot-java/assets/GitHub-Login-3.png b/workshops/github-copilot-java/assets/GitHub-Login-3.png deleted file mode 100644 index 38b1fd3d..00000000 Binary files a/workshops/github-copilot-java/assets/GitHub-Login-3.png and /dev/null differ diff --git a/workshops/github-copilot-java/assets/Jetbrains-Intellij-IDEA-setup.mp4 b/workshops/github-copilot-java/assets/Jetbrains-Intellij-IDEA-setup.mp4 deleted file mode 100644 index 2416c6c2..00000000 Binary files a/workshops/github-copilot-java/assets/Jetbrains-Intellij-IDEA-setup.mp4 and /dev/null differ diff --git a/workshops/github-copilot-java/assets/VS-Code-code-command-enabled.png b/workshops/github-copilot-java/assets/VS-Code-code-command-enabled.png deleted file mode 100644 index c6ff8657..00000000 Binary files a/workshops/github-copilot-java/assets/VS-Code-code-command-enabled.png and /dev/null differ diff --git a/workshops/github-copilot-java/assets/VS-Code-code-command.png b/workshops/github-copilot-java/assets/VS-Code-code-command.png deleted file mode 100644 index 75147380..00000000 Binary files a/workshops/github-copilot-java/assets/VS-Code-code-command.png and /dev/null differ diff --git a/workshops/github-copilot-java/assets/auto-generation-1.png b/workshops/github-copilot-java/assets/auto-generation-1.png deleted file mode 100644 index 2580925f..00000000 Binary files a/workshops/github-copilot-java/assets/auto-generation-1.png and /dev/null differ diff --git a/workshops/github-copilot-java/assets/banner.jpg b/workshops/github-copilot-java/assets/banner.jpg deleted file mode 100644 index 9c87e96e..00000000 Binary files a/workshops/github-copilot-java/assets/banner.jpg and /dev/null differ diff --git a/workshops/github-copilot-java/assets/create-helloworld-rest-controller-1.png b/workshops/github-copilot-java/assets/create-helloworld-rest-controller-1.png deleted file mode 100644 index ee11797e..00000000 Binary files a/workshops/github-copilot-java/assets/create-helloworld-rest-controller-1.png and /dev/null differ diff --git a/workshops/github-copilot-java/assets/create-helloworld-rest-controller.png b/workshops/github-copilot-java/assets/create-helloworld-rest-controller.png deleted file mode 100644 index 993109ca..00000000 Binary files a/workshops/github-copilot-java/assets/create-helloworld-rest-controller.png and /dev/null differ diff --git a/workshops/github-copilot-java/assets/github-copilot-altenative-code-1.png b/workshops/github-copilot-java/assets/github-copilot-altenative-code-1.png deleted file mode 100644 index 4e12a738..00000000 Binary files a/workshops/github-copilot-java/assets/github-copilot-altenative-code-1.png and /dev/null differ diff --git a/workshops/github-copilot-java/assets/github-copilot-altenative-code-2.png b/workshops/github-copilot-java/assets/github-copilot-altenative-code-2.png deleted file mode 100644 index d9d49f11..00000000 Binary files a/workshops/github-copilot-java/assets/github-copilot-altenative-code-2.png and /dev/null differ diff --git a/workshops/github-copilot-java/assets/github-copilot-altenative-lists.png b/workshops/github-copilot-java/assets/github-copilot-altenative-lists.png deleted file mode 100644 index 36d52eb3..00000000 Binary files a/workshops/github-copilot-java/assets/github-copilot-altenative-lists.png and /dev/null differ diff --git a/workshops/github-copilot-java/assets/github-copilot-chat-start1.png b/workshops/github-copilot-java/assets/github-copilot-chat-start1.png deleted file mode 100644 index 4cde7333..00000000 Binary files a/workshops/github-copilot-java/assets/github-copilot-chat-start1.png and /dev/null differ diff --git a/workshops/github-copilot-java/assets/github-copilot-chat-start2.png b/workshops/github-copilot-java/assets/github-copilot-chat-start2.png deleted file mode 100644 index ffddbaa4..00000000 Binary files a/workshops/github-copilot-java/assets/github-copilot-chat-start2.png and /dev/null differ diff --git a/workshops/github-copilot-java/assets/github-copilot-chat-start3.png b/workshops/github-copilot-java/assets/github-copilot-chat-start3.png deleted file mode 100644 index 99d9910d..00000000 Binary files a/workshops/github-copilot-java/assets/github-copilot-chat-start3.png and /dev/null differ diff --git a/workshops/github-copilot-java/assets/github-copilot-code-generation.mp4 b/workshops/github-copilot-java/assets/github-copilot-code-generation.mp4 deleted file mode 100755 index e32abed5..00000000 Binary files a/workshops/github-copilot-java/assets/github-copilot-code-generation.mp4 and /dev/null differ diff --git a/workshops/github-copilot-java/assets/github-copilot-enable-completion-for-markdown-2.png b/workshops/github-copilot-java/assets/github-copilot-enable-completion-for-markdown-2.png deleted file mode 100644 index 56b10cba..00000000 Binary files a/workshops/github-copilot-java/assets/github-copilot-enable-completion-for-markdown-2.png and /dev/null differ diff --git a/workshops/github-copilot-java/assets/github-copilot-enable-completion-for-markdown.png b/workshops/github-copilot-java/assets/github-copilot-enable-completion-for-markdown.png deleted file mode 100644 index 12bfbfd0..00000000 Binary files a/workshops/github-copilot-java/assets/github-copilot-enable-completion-for-markdown.png and /dev/null differ diff --git a/workshops/github-copilot-java/assets/github-copilot-first-trigger.png b/workshops/github-copilot-java/assets/github-copilot-first-trigger.png deleted file mode 100644 index ff504243..00000000 Binary files a/workshops/github-copilot-java/assets/github-copilot-first-trigger.png and /dev/null differ diff --git a/workshops/github-copilot-java/assets/github-copilot-for-java-properties.png b/workshops/github-copilot-java/assets/github-copilot-for-java-properties.png deleted file mode 100644 index bebed8e9..00000000 Binary files a/workshops/github-copilot-java/assets/github-copilot-for-java-properties.png and /dev/null differ diff --git a/workshops/github-copilot-java/assets/github-copilot-icon.png b/workshops/github-copilot-java/assets/github-copilot-icon.png deleted file mode 100644 index 3e993d8f..00000000 Binary files a/workshops/github-copilot-java/assets/github-copilot-icon.png and /dev/null differ diff --git a/workshops/github-copilot-java/assets/github-copilot-short-cut.png b/workshops/github-copilot-java/assets/github-copilot-short-cut.png deleted file mode 100644 index 545825e0..00000000 Binary files a/workshops/github-copilot-java/assets/github-copilot-short-cut.png and /dev/null differ diff --git a/workshops/github-copilot-java/assets/github-copilot-status.png b/workshops/github-copilot-java/assets/github-copilot-status.png deleted file mode 100644 index 69e03315..00000000 Binary files a/workshops/github-copilot-java/assets/github-copilot-status.png and /dev/null differ diff --git a/workshops/github-copilot-java/assets/github-copilot-toolkit-ext-install-1.png b/workshops/github-copilot-java/assets/github-copilot-toolkit-ext-install-1.png deleted file mode 100644 index 50eea6aa..00000000 Binary files a/workshops/github-copilot-java/assets/github-copilot-toolkit-ext-install-1.png and /dev/null differ diff --git a/workshops/github-copilot-java/assets/github-copilot-toolkit-ext-install-2.png b/workshops/github-copilot-java/assets/github-copilot-toolkit-ext-install-2.png deleted file mode 100644 index c87502f8..00000000 Binary files a/workshops/github-copilot-java/assets/github-copilot-toolkit-ext-install-2.png and /dev/null differ diff --git a/workshops/github-copilot-java/assets/intellij-IDEA-install.png b/workshops/github-copilot-java/assets/intellij-IDEA-install.png deleted file mode 100644 index b5dce3ea..00000000 Binary files a/workshops/github-copilot-java/assets/intellij-IDEA-install.png and /dev/null differ diff --git a/workshops/github-copilot-java/assets/intellij-idea-install-copilot-plugin-1.png b/workshops/github-copilot-java/assets/intellij-idea-install-copilot-plugin-1.png deleted file mode 100644 index a5cd5f7f..00000000 Binary files a/workshops/github-copilot-java/assets/intellij-idea-install-copilot-plugin-1.png and /dev/null differ diff --git a/workshops/github-copilot-java/assets/intellij-idea-install-copilot-plugin-2.png b/workshops/github-copilot-java/assets/intellij-idea-install-copilot-plugin-2.png deleted file mode 100644 index 8bd7de4d..00000000 Binary files a/workshops/github-copilot-java/assets/intellij-idea-install-copilot-plugin-2.png and /dev/null differ diff --git a/workshops/github-copilot-java/assets/intellij-idea-install-copilot-plugin-restart.png b/workshops/github-copilot-java/assets/intellij-idea-install-copilot-plugin-restart.png deleted file mode 100644 index de57aa3d..00000000 Binary files a/workshops/github-copilot-java/assets/intellij-idea-install-copilot-plugin-restart.png and /dev/null differ diff --git a/workshops/github-copilot-java/assets/intellij-idea-login-github-1.png b/workshops/github-copilot-java/assets/intellij-idea-login-github-1.png deleted file mode 100644 index 49c1efa6..00000000 Binary files a/workshops/github-copilot-java/assets/intellij-idea-login-github-1.png and /dev/null differ diff --git a/workshops/github-copilot-java/assets/intellij-idea-login-github-2.png b/workshops/github-copilot-java/assets/intellij-idea-login-github-2.png deleted file mode 100644 index 21e69f38..00000000 Binary files a/workshops/github-copilot-java/assets/intellij-idea-login-github-2.png and /dev/null differ diff --git a/workshops/github-copilot-java/assets/intellij-idea-login-github-3.png b/workshops/github-copilot-java/assets/intellij-idea-login-github-3.png deleted file mode 100644 index c9460359..00000000 Binary files a/workshops/github-copilot-java/assets/intellij-idea-login-github-3.png and /dev/null differ diff --git a/workshops/github-copilot-java/assets/intellij-idea-login-github-4.png b/workshops/github-copilot-java/assets/intellij-idea-login-github-4.png deleted file mode 100644 index 6ac26ac4..00000000 Binary files a/workshops/github-copilot-java/assets/intellij-idea-login-github-4.png and /dev/null differ diff --git a/workshops/github-copilot-java/assets/intellij-idea-login-github-5.png b/workshops/github-copilot-java/assets/intellij-idea-login-github-5.png deleted file mode 100644 index 1c8b8f6a..00000000 Binary files a/workshops/github-copilot-java/assets/intellij-idea-login-github-5.png and /dev/null differ diff --git a/workshops/github-copilot-java/assets/intellij-idea-login-github-6.png b/workshops/github-copilot-java/assets/intellij-idea-login-github-6.png deleted file mode 100644 index 124cee53..00000000 Binary files a/workshops/github-copilot-java/assets/intellij-idea-login-github-6.png and /dev/null differ diff --git a/workshops/github-copilot-java/assets/intellij-idea-login-github-7.png b/workshops/github-copilot-java/assets/intellij-idea-login-github-7.png deleted file mode 100644 index 1d8a6a92..00000000 Binary files a/workshops/github-copilot-java/assets/intellij-idea-login-github-7.png and /dev/null differ diff --git a/workshops/github-copilot-java/assets/intellij-idea-login-github-8.png b/workshops/github-copilot-java/assets/intellij-idea-login-github-8.png deleted file mode 100644 index 23feface..00000000 Binary files a/workshops/github-copilot-java/assets/intellij-idea-login-github-8.png and /dev/null differ diff --git a/workshops/github-copilot-java/assets/intellij-idea-start.png b/workshops/github-copilot-java/assets/intellij-idea-start.png deleted file mode 100644 index 197c38a6..00000000 Binary files a/workshops/github-copilot-java/assets/intellij-idea-start.png and /dev/null differ diff --git a/workshops/github-copilot-java/assets/java-implementation-sample1.png b/workshops/github-copilot-java/assets/java-implementation-sample1.png deleted file mode 100644 index a81daf5b..00000000 Binary files a/workshops/github-copilot-java/assets/java-implementation-sample1.png and /dev/null differ diff --git a/workshops/github-copilot-java/assets/java-implementation-sample2.png b/workshops/github-copilot-java/assets/java-implementation-sample2.png deleted file mode 100644 index dd483e75..00000000 Binary files a/workshops/github-copilot-java/assets/java-implementation-sample2.png and /dev/null differ diff --git a/workshops/github-copilot-java/assets/java-implementation-sample3.png b/workshops/github-copilot-java/assets/java-implementation-sample3.png deleted file mode 100644 index 280ce967..00000000 Binary files a/workshops/github-copilot-java/assets/java-implementation-sample3.png and /dev/null differ diff --git a/workshops/github-copilot-java/assets/java-implementation-sample4.png b/workshops/github-copilot-java/assets/java-implementation-sample4.png deleted file mode 100644 index 6e555594..00000000 Binary files a/workshops/github-copilot-java/assets/java-implementation-sample4.png and /dev/null differ diff --git a/workshops/github-copilot-java/assets/java-implementation-sample5.png b/workshops/github-copilot-java/assets/java-implementation-sample5.png deleted file mode 100644 index 850b9875..00000000 Binary files a/workshops/github-copilot-java/assets/java-implementation-sample5.png and /dev/null differ diff --git a/workshops/github-copilot-java/assets/modernize-java-code-sample1.png b/workshops/github-copilot-java/assets/modernize-java-code-sample1.png deleted file mode 100644 index 951fcff3..00000000 Binary files a/workshops/github-copilot-java/assets/modernize-java-code-sample1.png and /dev/null differ diff --git a/workshops/github-copilot-java/assets/modernize-java-code-sample2.png b/workshops/github-copilot-java/assets/modernize-java-code-sample2.png deleted file mode 100644 index 76936480..00000000 Binary files a/workshops/github-copilot-java/assets/modernize-java-code-sample2.png and /dev/null differ diff --git a/workshops/github-copilot-java/assets/open-spring-boot-app-on-vscode-1.png b/workshops/github-copilot-java/assets/open-spring-boot-app-on-vscode-1.png deleted file mode 100644 index 57a77f14..00000000 Binary files a/workshops/github-copilot-java/assets/open-spring-boot-app-on-vscode-1.png and /dev/null differ diff --git a/workshops/github-copilot-java/assets/vscode-coding-pack-java-install-1.png b/workshops/github-copilot-java/assets/vscode-coding-pack-java-install-1.png deleted file mode 100644 index 9f5ff61a..00000000 Binary files a/workshops/github-copilot-java/assets/vscode-coding-pack-java-install-1.png and /dev/null differ diff --git a/workshops/github-copilot-java/assets/vscode-coding-pack-java-install-2.png b/workshops/github-copilot-java/assets/vscode-coding-pack-java-install-2.png deleted file mode 100644 index 5c4be760..00000000 Binary files a/workshops/github-copilot-java/assets/vscode-coding-pack-java-install-2.png and /dev/null differ diff --git a/workshops/github-copilot-java/assets/vscode-coding-pack-java-install-3.png b/workshops/github-copilot-java/assets/vscode-coding-pack-java-install-3.png deleted file mode 100644 index 540c9989..00000000 Binary files a/workshops/github-copilot-java/assets/vscode-coding-pack-java-install-3.png and /dev/null differ diff --git a/workshops/github-copilot-java/assets/vscode-coding-pack-java-install-4.png b/workshops/github-copilot-java/assets/vscode-coding-pack-java-install-4.png deleted file mode 100644 index 2e7d690a..00000000 Binary files a/workshops/github-copilot-java/assets/vscode-coding-pack-java-install-4.png and /dev/null differ diff --git a/workshops/github-copilot-java/assets/vscode-coding-pack-java-install-5.png b/workshops/github-copilot-java/assets/vscode-coding-pack-java-install-5.png deleted file mode 100644 index 525f911f..00000000 Binary files a/workshops/github-copilot-java/assets/vscode-coding-pack-java-install-5.png and /dev/null differ diff --git a/workshops/github-copilot-java/assets/vscode-github-copilot-setup.mp4 b/workshops/github-copilot-java/assets/vscode-github-copilot-setup.mp4 deleted file mode 100755 index 71c4c9c5..00000000 Binary files a/workshops/github-copilot-java/assets/vscode-github-copilot-setup.mp4 and /dev/null differ diff --git a/workshops/github-copilot-java/workshop.md b/workshops/github-copilot-java/workshop.md deleted file mode 100644 index 67159f64..00000000 --- a/workshops/github-copilot-java/workshop.md +++ /dev/null @@ -1,826 +0,0 @@ ---- -published: true -type: workshop -title: Product Hands-on Lab - GitHub Copilot, your new AI pair programmer for Java Developer -short_title: GitHub Copilot, your new AI pair programmer -description: Discover how to leverage GitHub Copilot to develop your project -level: beginner -authors: - - Yoshio Terada -contacts: - - "@yoshioterada" -duration_minutes: 120 -tags: java, GitHub, copilot, AI, csu -banner_url: assets/banner.jpg -sections_title: - - Introduction - - Install Environment - - Create a new Sample Spring Boot Application - - Github Copilot for Java Development - - Github Copilot Chat for Java Development ---- - -# Enhance Your Efficiency with GitHub Copilot: A Workshop - -This workshop is designed to guide you on how to leverage GitHub Copilot with Java through a hands-on exercise - the creation of a new Spring Boot Application. - -GitHub Copilot is an AI-powered code assistant, aimed at helping developers write better code more efficiently. It uses machine learning models, trained on billions of lines of code, to suggest entire lines or even functions, based on the context of your current work. By utilizing GitHub Copilot, you can improve your coding skills and increase your productivity. - -## Who Can Use GitHub Copilot? - -GitHub Copilot can be accessed through an individual account for personal use, or an Organization account for business use. - -GitHub Copilot is free for verified students, teachers, and maintainers of open-source projects. If you are not a student, teacher, or maintainer of an open-source project, you can try GitHub Copilot for free with a one-time 30-day trial. After the trial period, a paid subscription is required to continue using it. - -## Capabilities of GitHub Copilot: - -With GitHub Copilot, you can: - -- `Generate Code Automatically`: GitHub Copilot can auto-generate the remaining code for you based on a part of your code. -- `Correct Code`: If there are errors in your code, GitHub Copilot can detect them and provide suggested corrections. -- `Predict Code`: GitHub Copilot can anticipate and predict the code you might enter next, even before you input any code. -- `Complete Code`: GitHub Copilot can offer code completion while you are entering your code. -- `Generate Comments Automatically`: GitHub Copilot can auto-generate subsequent comments when you input a comment. -- `Create Comment from Code`: GitHub Copilot can generate an explanation of the comments when you input a comment. - -## Capabilities of GitHub Copilot Chat - -With GitHub Copilot Chat, you can: - -- `Create Unit Tests`: GitHub Copilot Chat can craft unit tests for your code. -- `Explain Code`: GitHub Copilot Chat can provide descriptions and explanations of your code. -- `Propose Code Correction`: If there are areas in your code that need improvement, GitHub Copilot Chat can suggest appropriate corrections. -- `Answer Coding-related Questions`: GitHub Copilot Chat can answer any questions you may have about coding. - -
- -> Please note that GitHub Copilot is a rapidly evolving product, and this workshop may not be 100% up to date with the various features of the different extensions you are going to use. Please use your discretion." - -
- ---- - -# Environment Setup - -To develop Java applications using GitHub Copilot, you need to have certain environments set up. If you haven't installed these environments yet, please do so as necessary. - -## Required Software and Tools - -| Component | Download Location | -| -------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| Editor | [Visual Studio Code](https://code.visualstudio.com/download)
[IntelliJ IDEA](https://www.jetbrains.com/idea/promo/) | -| VS Code for Java Extension | [Coding Pack for Java - Windows](https://aka.ms/vscode-java-installer-win)
[Coding Pack for Java - macOS](https://aka.ms/vscode-java-installer-mac) | -| OpenJDK | [Microsoft Build OpenJDK 21](https://learn.microsoft.com/java/openjdk/download#openjdk-21)　
[Microsoft Build OpenJDK 17](https://learn.microsoft.com/java/openjdk/download#openjdk-17)
[Microsoft Build OpenJDK 11](https://learn.microsoft.com/ja-jp/java/openjdk/download#openjdk-11) | -| GitHub account | [Create free GitHub account](https://github.com/join) | -| GitHub Copilot Access | A 60 day trial can be [requested here](https://github.com/github-copilot/signup) | -| A browser | [Download Microsoft Edge](https://www.microsoft.com/edge) | - - -## Visual Studio Code Setup - -### Install the Extension Pack for GitHub Copilot Tools Pack - -If you have installed the `GitHub Copilot Tools Pack`, the following extensions will be installed: - -- GitHub Copilot -- GitHub Copilot Labs -- GitHub Copilot Chat - -![Install GitHub Copilot Tools Pack Extension 1](assets/github-copilot-toolkit-ext-install-1.png) - -To install the GitHub Copilot Tools Pack Extension, follow these steps: - -1. Select `Extensions` from the VS Code menu. -2. Search for `GitHub Copilot Tools Pack` in the Marketplace -3. Choose `GitHub Copilot Tools Pack` and press the `Install` button. -4. Once completed, a screen that says `Sign in to use GitHub Copilot` will appear. Press the `Sign in to GitHub` button. -5. Next, verify whether you are signed into your GitHub account. If you are not, log in. -6. Once logged in, a screen titled `Authenticate to authorized access` will appear. Choose the necessary organization and click the `Continue` button. -7. Go back to Visual Studio Code and press the `Sign in to GitHub` button to utilize the `GitHub Copilot Lab`. -8. Since you are already logged into GitHub, select the necessary organization and press the `Continue` button. - - - -## IntelliJ IDEA Setup - -To install GitHub Copilot in the IntelliJ IDEA environment, follow the steps below. - -![Start Intellij IDEA](assets/intellij-idea-start.png) - -1. From IntelliJ IDEA's `Settings`, select `Plugins`, search for `GitHub Copilot`, and press the `Install' button. -2. Once the plugin is installed, you will be prompted to restart. Press the `Restart IDE` button. -3. Next, to use GitHub Copilot, sign in to GitHub with your GitHub account. Click the link, or select `Login to GitHub` from the GitHub Copilot Status Menu. -4. Upon requesting to sign in, a window will appear displaying an 8-digit `Device code`. Please remember this code. Then, access https://github.com/login/device from your web browser. -5. Enter the 8-digit Device code generated in the previous step and press the `Continue` button. -6. Next, verify if you are logged in to GitHub with your own GitHub account, and if there are no issues, press the `Authorize GitHub Copilot Plugin` button. -7. Return to the IntelliJ IDEA screen and confirm that you have successfully logged into GitHub and that GitHub Copilot is now available. -8. Check the GitHub Copilot Status displayed at the bottom right of the IntelliJ IDEA screen. If it says `Status: Ready`, it is now available for use. - - - -## Avoid Using Publicly Published Code - -If you set up a Copilot Business subscription for your organization, you can configure GitHub Copilot settings for your organization. - -For instance, if you're coding in a corporate environment, you might not want to use publicly available code due to enterprise rules or code licensing. In such cases, you can configure GitHub Copilot Chat to avoid using publicly published code. - -To do this, you need to set up a [Settings Copilot](https://github.com/settings/copilot) for your organization. - -![GitHub Copilot Rejected Prompt](./assets/GitHub-Copilot-Chat-Blocked-Public.png) - -Once these settings are in place, GitHub Copilot Chat will not use public code. If GitHub Copilot Chat attempts to use public code after entering a prompt, it will display a message and stop processing. - -![GitHub Copilot Rejected Prompt](./assets/GitHub-Copilot-Chat-rejected-prompt.png) - ---- - -# Creating a New Sample Spring Boot Application - -In this section, we will create a sample Java project for testing GitHub Copilot in subsequent chapters. If you already have an existing Java project, feel free to skip this chapter and use your own project instead. - -## Creating a Spring Boot Application - -```bash -> mkdir sample; cd sample -> curl https://start.spring.io/starter.zip \ - -d dependencies=web,devtools \ - -d bootVersion=3.3.0 \ - -d type=maven-project \ - -d applicationName=CopilotSample \ - -d packageName=com.microsoft.sample \ - -d groupId=com.microsoft.sample \ - -d artifactId=CopilotSample \ - -d javaVersion=21 (or 17) \ - -o my-project.zip -> unzip my-project.zip -``` - -After unzipping the file, you will see the following directory structure: - -```bash -. -├── HELP.md -├── mvnw -├── mvnw.cmd -├── my-project.zip -├── pom.xml -├── src -│   ├── main -│   │   ├── java -│   │   │   └── com -│   │   │   └── microsoft -│   │   │   └── sample -│   │   │   └── CopilotSample.java -│   │   └── resources -│   │   ├── application.properties -│   │   ├── static -│   │   └── templates -│   └── test -│   └── java -│   └── com -│   └── microsoft -│   └── sample -│   └── CopilotSampleTests.java -└── target - ├── classes - │   ├── application.properties - │   └── com - │   └── microsoft - │   └── sample - │   └── CopilotSample.class - └── test-classes - └── com - └── microsoft - └── sample - └── CopilotSampleTests.class -``` - -## Opening the Project in VS Code - -Now that the project has been created, it's time to open it in VS Code. -Enter the command 'code .' - -```bash -code . -``` - -Upon opening the project in VS Code, the screen below will appear. Here, click on `Yes, I trust the authors`. - -![Open Spring Boot App on VS Code](assets/create-helloworld-rest-controller.png) - -Opening Spring Boot App on VS Code - -## Creating a Hello World REST Controller - -Next, we will create a REST Controller. Right-click on the `src/main/java/com/microsoft/sample` folder and select `New File`. -Then, type `HelloRestController.java` and press the Enter key. The following screen will appear. - -![Create HelloWorld REST Controller](assets/create-helloworld-rest-controller-1.png) - ---- - -# Github Copilot for Java Development - -## 4.1 Checking if GitHub Copilot is Activated - -The `GitHub Copilot Status Menu` icon is located at the bottom right of VS Code. - -![GitHub Copilot Status menu icon](assets/github-copilot-icon.png) - -By clicking on this icon, you can check the status of GitHub Copilot. -If it displays `Status: Ready` as shown below, GitHub Copilot is ready for use. - -![GitHub Copilot Enabled confirmation](assets/github-copilot-status.png) - -If it does not display `Ready`, please return to the [Install Environment](/workshop/github-copilot-java/?step=1#install-extension-pack-for-github-copilot-tools-pack) section and set up your environment. - -## 4.2 Basic Operations of GitHub Copilot - -Here is a basic guide on how to use GitHub Copilot. It's not just about enabling the features of GitHub Copilot. By executing shortcut commands during program implementation, or changing the content of your comments, you can modify the suggested code. So, please try out these basic operations of GitHub Copilot yourself. - -### 4.2.1 Code Suggestions and Explicit Triggers - -Please open the `HelloRestController.java` file that you created in the previous chapter in your editor. There, by pressing the `Tab` key or entering a newline character, GitHub Copilot will start to suggest code for you. - -![First Trigger of GitHub Copilot](assets/github-copilot-first-trigger.png) - -Alternatively, you can explicitly trigger it. If code suggestions are not being made, please enter the following shortcut key. This will execute the `editor.action.inlineSuggest.trigger` command and display the code that GitHub Copilot recommends. - -| OS | Trigger Inline Suggestion | -| ------- | ------------------------- | -| macOS | `Option (⌥) or Alt + \` | -| Windows | `Alt + \` | - -
- -> GitHub Copilot generates code predictions based on the environment it is being used in, so the content displayed may vary depending on the environment. - -
- -### 4.2.2 Accepting Code Suggestions - -When a code suggestion is displayed, you can confirm it by pressing the `Tab` key. - -If you do not want to accept all the suggestions that Copilot displays and only wish to adopt parts of it, you can do so by pressing the `Command(Ctrl) + Right Arrow` key instead of the `Tab` key. This allows you to adopt suggestions word by word. - -| OS | Determin the proposal | -| ------- | ----------------------- | -| macOS | `Command + right arrow` | -| Windows | `Ctrl + right arrow` | - -### 4.2.3 Displaying Alternate Suggestions (Functionality may slightly vary depending on the environment) - -There may be instances where the source code suggestions given by GitHub Copilot do not match the code you want to implement. In such cases, you can also display alternative suggestions for the initial code displayed. To show alternate suggestions, please press the following shortcut key. - -| OS | See next suggestion | See previous suggestion | -| ------- | ----------------------- | ----------------------- | -| macOS | `Option (⌥) or Alt + ]` | `Option (⌥) or Alt + [` | -| Windows | `Alt + \` | `Alt + [` | - -When you press the shortcut key, it will be displayed as follows. - -![Alternative Code 1](assets/github-copilot-altenative-code-1.png) - -If you press the shortcut key again, a different code will be output from the one above. - -![Alternative Code 2](assets/github-copilot-altenative-code-2.png) - -### 4.2.4 Displaying List of Alternate Suggestions (Functionality may slightly vary depending on the environment) - -Furthermore, if you are using Visual Studio Code, pressing the following shortcut key will display up to 10 alternative suggestions. - -| OS | Next 10 suggestion | -| -------------- | ------------------ | -| macOS, Windows | `Ctrl + Enter` | - -![Alternative Code 3](assets/github-copilot-altenative-lists.png) - -As shown above, not only by simply enabling the GitHub Copilot feature, but also by executing shortcut commands during program implementation, you can display and apply a list of alternative candidates. By all means, please learn the basic operations of GitHub Copilot and give it a try. - -## 4.3 Using GitHub Copilot in Java Application Development - -### 4.3.1 Points where GitHub Copilot can be used during Java application development - -You can use GitHub Copilot in various scenarios during the development of Java applications. - -- Creating source code -- Creating Markdown/HTML documents -- Editing Java property files - -#### Creating Source Code - -As demonstrated in the basic operations above, GitHub Copilot provides various hints during the implementation of source code. If you use Visual Studio Code, you can further enhance your development productivity by utilizing GitHub Copilot Chat. - -#### Creating Markdown/HTML Documents - -When creating documents for your project, you'll likely write them in Markdown. GitHub Copilot can also assist in creating Markdown documents. Open a Markdown file, click on the GitHub Copilot Status Menu icon, and select `Enable Completion for markdown`. - -![Enable Completion for Markdown](assets/github-copilot-enable-completion-for-markdown.png) - -Once enabled, it will provide various hints while you're writing your Markdown document. -For example, if you input `#`, it will prompt you to enter a string following the `#`, and it can even predict what to write next based on the context of your document. - -![Enable Completion for Markdown 2](assets/github-copilot-enable-completion-for-markdown-2.png) - -You can also use GitHub Copilot when writing HTML documents. - -#### Editing Java Property Files - -Additionally, GitHub Copilot can be used when editing Java property files. Properties are set according to the libraries you use, but it can be challenging to remember all the properties. In such cases, when you enter the property keyword, GitHub Copilot will display possible property candidates for you. - -![Enable Completion for Java Properties](assets/github-copilot-for-java-properties.png) - -## Using GitHub Copilot in Java Source Code Implementation - -Let's look at a few specific ways to utilize GitHub Copilot when editing Java source code. - -### 4.4.1 Creating Sample Dummy Data - -When implementing a program, you may need to create sample data and test it locally. -In such cases, GitHub Copilot can help you create sample data easily. For example, if you want to create sample data for stock ticker symbols, GitHub Copilot can generate a list of sample stock symbols for you. - -```text -// Create 20 sample data of Stock Tickers in List -``` - -The list of stock symbols created by GitHub Copilot looks like the following. - -![Java Coding Sample 1](assets/java-implementation-sample1.png) - -Similarly, if you want to create sample data for male names, GitHub Copilot can generate a list of sample male names for you. - -```text -// Create 20 sample data of American men's names in List -``` - -The list of name of male by GitHub Copilot looks like the following. - -![Java Coding Sample 2](assets/java-implementation-sample2.png) - -### 4.4.2 Implementing Check Methods and Utility Methods - -Furthermore, GitHub Copilot can be used in various places during the implementation of a Java program. For example, you might implement check methods or utility methods in Java. In such cases, by writing comments, GitHub Copilot will propose the code for you. - -#### Checking the Format of Email Addresses - -For instance, when implementing a method to determine whether the format of an email address is correct, you would write a comment as shown below. - -```text -/** - * Determine whether the string specified as an argument is - * in the format of a valid email address - * - * @param email - * @return true if the string is a valid email address, false otherwise -*/ -``` - -The actual outputted code looks like the following. - -![Java Coding Sample 3](assets/java-implementation-sample3.png) - -The above regular expression `^[\\w-\\.]+@([\\w-]+\\.)+[\\w-]{2,4}$` is used to validate the format of a typical email address. However, it does not cover some specific cases: - -1. The last part of the domain name (TLD, Top Level Domain) is limited to a range of 2 to 4 characters. However, nowadays there are TLDs with more than 4 characters, such as .info, .museum. -2. The username part does not allow special characters (for example + or =). However, these characters are allowed in some email systems. - -Validating a complete email address is very complex, and regular expressions according to RFC 5322 can be very long and complex. However, for common cases, you can use the simplified regular expression below. If you modify the following code, it allows any alphabetic character of 2 or more characters (e.g. .museum). Also, dot (.) notation is allowed in the username part. - -```java -String emailRegex = "^[\\w-]+(\\.[\\w-]+)*@[\\w-]+(\\.[\\w-]+)*(\\.[a-zA-Z]{2,})$"; -Pattern pat = Pattern.compile(emailRegex); -return email != null && pat.matcher(email).matches(); -``` - -
- -> The code proposed by GitHub Copilot is not always correct. The output code may need to be modified. Please understand the output code and make corrections as necessary. - -
- -#### Leap Year Determination Check - -Next, we will implement a method for checking leap year determination. Please write a comment as shown below: - -```text -/** - * Determine whether the specified year in the argument is a leap year - * - * @param year - * @return true if the year is a leap year, false otherwise -*/ -``` - -When you write a comment, the code will be output as shown below. - -![Java Coding Sample 4](assets/java-implementation-sample4.png) - -Of course, with the above code, you can determine a `leap year`, but by using the switch expression added in Java 14, you can implement it more concisely. To write using the switch expression added in Java 14, modify the comment as shown below. - -```text -/** -* Determine whether the year specified by the argument is a leap year -* The implementation will use the switch expression introduced in Java 14 -* -* @param year -* @return true if the string is leap year, false otherwise -*/ -``` - -![Java Coding Sample 5](assets/java-implementation-sample5.png) - -
- -> The code output will change depending on the comments you write. In some cases, code using older Java language specifications or old libraries may be used. In such cases, by specifically instructing the code you want to write, it will be output as instructed. - -
- -#### Display Files and Directories Under the Specified Directory - -Next, we will implement a method to display the files and directories that exist under the directory specified in the argument. Write a comment as shown below: - -```text -//Show all files and directories recursively under the directory specified by the argument - ``` - -Then, a code like the one below will be suggested. - -![Modernize Java Coding Style 2](assets/modernize-java-code-sample2.png) - -The above code will work without any problems, but using Java NIO.2 or Stream API allows for a more concise description. To write according to the new language specification, let's modify the comments. Write a comment as shown below. - -```text -/** - * In accordance with the Java 17 language specifications, - * using features like NIO.2, Stream API, Files.walk, - * to recursively show the files and directories - * that exist under the specified directory. - * - * @param directoryName - */ -```` - -As a result, a modern code like the one below has been output. - -![Modernize Java Coding Style 1](assets/modernize-java-code-sample1.png) - -
- -> Just like the Leap Year calculation mentioned above, rewriting the comments will prompt it to suggest code using new language specifications or new libraries. - -
- ---- - -# Github Copilot Chat for Java Development - -
- -> Currently, GitHub Copilot Chat does not offer a plugin for IntelliJ IDEA. In this guide, we will demonstrate how to use GitHub Copilot Chat with Visual Studio Code. -
- -## 5.1 About GitHub Copilot Chat - -Similar to GitHub Copilot, GitHub Copilot Chat provides AI-powered code completion. However, it goes beyond code completion by answering questions about code, offering explanations, and suggesting code modifications, thereby enhancing code quality. - -Here are some scenarios where GitHub Copilot Chat can be applied in Java application development: - -- Code generation -- Code explanation -- Code correction -- Answering coding-related questions -- Creating unit tests -- Explaining the error -- Fixing the error - -##　5.2 How to Use GitHub Copilot Chat - -GitHub Copilot can be used in two ways: - -* Through the dedicated GitHub Copilot chat window -* Inline on the editor - -### 5.2.1 Using the dedicated GitHub Copilot chat window - -Click on the GitHub Copilot icon in the lower left of VS Code to display the GitHub Copilot chat window. - -#### Slash Commands - -To help Copilot provide more relevant answers, you can specify a topic for your questions using `slash commands`. Prepend your chat inputs with a specific topic name to guide Copilot towards a more relevant response. When you start typing `/`, a list of possible topics will appear: - -- **/api**: Ask about VS Code extension development. -- **/explain**: Request a step-by-step explanation of the selected code. -- **/fix**: Propose a fix for bugs in the selected code. -- **/new**: Scaffold code for a new workspace. (e.g., /new spring boot) -- **/newNotebook**: Create a new Jupyter Notebook. (Not available for Java) -- **/terminal**: Ask how to perform tasks in the terminal. -- **/tests**: Generate unit tests for the selected code. -- **/vscode**: Ask about VS Code commands and settings. -- **/help**: Get general help about GitHub Copilot. -- **/clear**: Clear the session. - -#### Chat View - -The chat view offers a full chat experience, integrating seamlessly with your IDE. Once the view is open, you can start chatting with Copilot as your personal code coach. It keeps a history of the conversation and provides suggestions for questions along the way. You can: - -- Ask general questions about coding in any language or best practices -- Request code generation or fixes related to the current file and inject the code directly into the file - -![GitHub Copilot Chat Windows](./assets/github-copilot-chat-start1.png) - -### 5.2.2 Using GitHub Copilot Chat Inline on the Editor - -By pressing `Command + i` (or `Ctrl + Shift + i`) on the editor, an inline command window for Copilot Chat will appear within the editor. - -In inline mode, you can: - -- Request code explanations: This helps you understand what the code does and how it works. -- Generate test code: This confirms that the code is working as expected. -- Request code modifications: If there's a problem with the selected code, you can request a modification. -- Generate new code: You can generate code for new features, classes, functions, etc. - -These requests are based on the selected code or the cursor's position. Also, these requests can be made by starting an inline chat session. - -![GitHub Copilot Inline windows1](./assets/github-copilot-chat-start2.png) - -Additionally, typing `/` will display the commands that can be executed in GitHub Copilot Chat. - -![GitHub Copilot Inline windows2](./assets/github-copilot-chat-start3.png) - -## 5.3 Code generation - -GitHub Copilot Chat can generate code for you. - -### 5.3.1 Create CRUD Operation for Spring Boot Application - -When creating a web application, you might need to create an application that performs CRUD operations. This requires implementing the web frontend, backend processing, and configuring property files. In such cases, if you enter a comment like the one below into GitHub Copilot Chat, it will suggest an implementation for a CRUD application. - -```text -> Create a sample application that performs CRUD operations to manage - people (name, id, e-mail, address) in Spring Boot. - The data should be persisted to the DB, and the database operations - should be implemented in Spring Data JPA. - The front should be implemented in Thymeleaf. - Please show all implementation of classes and methods and - application property configuration file. -``` - -Then, the code suggested by GitHub Copilot will be displayed. - - - -### 5.3.2 Create a new Server-Sent Events endpoint with Spring Boot - -While implementing, you may sometimes forget how to implement a method in Java. In such cases, you can ask to generate code by specifying the implementation method using a particular technology. For example, if you forget how to implement using Server-Sent Events in Spring Boot, enter a comment like the one below in GitHub Copilot Chat. - -```text -> Create sample code that periodically sends back data in a Server-Sent Events - to a user who connects to the Spring Boot REST endpoint. -``` - -If you write a comment like this, explanations and code will be generated. - -![GitHub Copilot Saple 1](./assets/GitHub-Copilot-chat-sample1.png) - -And it also shows the sample Java code. - -![GitHub Copilot Saple 2](./assets/GitHub-Copilot-chat-sample2.png) - -## 5.4 Code Explanation - -Next, to understand the contents of the Java source code currently open in VS Code, you can get an explanation of the source code. GitHub Copilot Chat has prearranged commands for elucidating source code. Therefore, you can display the explanations by using these commands. First, input `/` in the console of GitHub Copilot Chat. Then, the commands that are available in GitHub Copilot Chat will appear. - -![GitHub Copilot Chat Explain 1](./assets/GitHub-Copilot-Chat-Code-Explain-1.png) - -At this point, select `/explain`. It will appear as shown. Press the enter key to proceed. - -![GitHub Copilot Chat Explain 2](./assets/GitHub-Copilot-Chat-Code-Explain-2.png) - -For example, select the following Java source code and execute the command. - -![GitHub Copilot Chat Explain 2](./assets/GitHub-Copilot-Chat-explanation-code.png) - -Then, an explanation of the Java source code will be provided. - -![GitHub Copilot Chat Explain 3](./assets/GitHub-Copilot-Chat-Code-Explain-3.png) - -### 5.5 Code Correction - -GitHub Copilot can be used for code modification. For example, in a Java application that was implemented in the past, if there is code that was implemented with an old language specification version, you can modify that code to match the latest language specifications. - -![GitHub Copilot Chat Modify the Code 1](./assets/GitHub-Copilot-Chat-modify-the-code-1.png) - -For example, enter content like the following into the GitHub Copilot Chat prompt. - -```test -> Follow the latest Java 17 language specification and modify your code - to use NIO.2 or the Stream API. Use var to define variables -``` - -Then, it will suggest Java code that conforms to the latest language specifications. - -![GitHub Copilot Chat Modify the Code 2](./assets/GitHub-Copilot-Chat-modify-the-code-2.png) - -When you move the mouse cursor to the top of the window with the suggested code, a menu will appear. - -![GitHub Copilot Chat Modify the Code 3](./assets/GitHub-Copilot-Chat-modify-the-code-3.png) - -You can perform the following actions: - -- `Copy` : Copy the code to the clipboard -- `Insert at Cursor [^Enter]` : Insert the code at the cursor position -- `Insert into New File` : Insert the code into a new file -- `Run in Terminal` : Execute the code (This function cannot be used in Java as you cannot directly execute the code) - -By performing the above `Copy` and Paste or `Insert at Cursor [^Enter]`, you can modify the old Java code to be compliant with the latest language specifications. - -## 5.6 Answering Coding-Related Questions - -Using GitHub Copilot Chat, you can get answers to questions about Java implementation. - -- Questions about the language specifications of the programming language -- Questions about the Java runtime environment -- Questions about library migration -- Questions about how to use a specific library - -### 5.6.1 Questions about the language specifications of the programming language - -When you are programming in Java, there may be times when you want to know more about the language specifications of Java. In such cases, you can ask various questions to GitHub Copilot Chat. - -For example, if you want to know about the language specifications of Java 17, you can ask as follows: - -If you want to know about the language specifications added from Java 11 to Java 17, you can ask as follows. - -```text -> What language specifications have been added from - Java 11 to Java 17? In particular, - please provide detailed information about the new writing style - with sample code. -``` - -### 5.6.2 Questions about the Java runtime environment - -There may be times when you want to understand points of caution in development environments, runtime environments, and production environments. -In such cases, you can also ask questions about points that should be noted. - -```text -> Please explain the differences in Java VM startup options - and VM behavior between Java 11 and 17. -``` - -### 5.6.3 Questions about library migration - -Application development is not only about new development, but also about modernizing source code and migrating from other old frameworks. In such cases, you can inquire about points to note in modernization and specific procedures for migration. - -```text -> What should I keep in mind when migrating from a 2.7.x - project to 3.1.5 with Spring Boot? -``` - -###　5.6.4 Questions about how to use a specific library - -Also, when using a specific library for the first time, you may want to know how to use that library. In such cases, you can ask about how to use the library. - -```text -> Please explain the implementation code for asynchronous - processing using Mono and Flux of Project Reactor - in an easy-to-understand manner with sample code. -``` - -### 5.6.5 Points to Note about GitHub Copilot Chat Questions and Answers - -One thing to note when using GitHub Copilot Chat is that not all inquiries will return the correct answer. For example, in the author's environment, the following inquiry did not return the correct answer. - -```text -> Please explain the details of JEP 450 -``` - -As a result of the inquiry, the following result was returned. - -![GitHub Copilot Chat Mistake 1](./assets/GitHub-Copilot-Chat-ask-mistake-1.png) - -However, the actual JEP 450 is [JEP 450: Compact Object Headers (Experimental)](https://openjdk.org/jeps/450), and it is not a JEP related to Unicode 14 as mentioned above. - -![GitHub Copilot Chat Mistake 2](./assets/GitHub-Copilot-Chat-ask-mistake-2.png) - -The proposed contents by GitHub Copilot Chat are not always correct. Please understand the proposed contents and make corrections as necessary. - -### 5.6.6 Other Inquiry Examples - -In Java development, you can make various inquiries to GitHub Copilot Chat. Here are some examples of inquiries, so please give them a try. - -```text -> Please tell me more about - JEP 394: Pattern Matching for instanceof, - including detailed sample code. - -> What does Erasure mean in Java Generics? Please explain in detail. - -> Please tell me the most recommended way to concatenate strings - in Java 17. Also, please explain the differences between - StringBuffer, StringBuilder, and + in an easy-to-understand manner. - -> Please tell me 10 recommended Extensions that are useful - when developing Java applications in VS Code. - -> Please tell me about the differences in Java VM startup options - and VM behavior between Java 11 and 17. - -> Please tell me how to enable remote debugging for Java - in a container environment. - -> Please explain in detail how to create a custom JRE - using the jdeps and jlink commands. - -> Please tell me 10 items to check before running a Java application - in a production environment. - -> What should I keep in mind when migrating from a 2.7.x project to - 3.1.5 with Spring Boot? - -> I have a Java Web Application implemented with the Facade pattern - in Struts 1.x. Please tell me the specific procedures and - configuration methods to migrate this to Spring Boot 3.1.x, - including sample code. - -> Please tell me the points to be aware of when migrating a - Spring Boot application to a Quarkus application, - and the specific migration method. - -> Please explain the implementation code for asynchronous - processing using Mono and Flux of Project Reactor - in an easy-to-understand manner with sample code. -``` - -## 5.7 Creating Unit Tests - -GitHub Copilot Chat can also generate unit tests for you. - -### 5.7.1 Create a Unit Test for Spring Boot Application - -When implementing a Spring Boot application, you may want to create a unit test for the application. -In such cases, you can select the class or method you want to test and ask GitHub Copilot Chat to create a unit test for you. -For example, if you want to create a unit test for the `ProductController` class, you can select the code and ask GitHub Copilot Chat to create a unit test for you. - -![GitHub Copilot Chat Generate Test Res 1](./assets/GitHub-Copilot-Chat-Test-Generate-Res-1.png) - -After selecting the code, type `/tests` and press `Enter` in GitHub Copilot Chat. - -![GitHub Copilot Chat Generate Test 1](./assets/GitHub-Copilot-Chat-Generate-Test1.png) - -Then, an explanation and the Unit Test code will be generated. - -![GitHub Copilot Chat Generate Test Res 4](./assets/GitHub-Copilot-Chat-Test-Generate-Res-4.png) -![GitHub Copilot Chat Generate Test Res 3](./assets/GitHub-Copilot-Chat-Test-Generate-Res-3.png) -![GitHub Copilot Chat Generate Test Res 2](./assets/GitHub-Copilot-Chat-Test-Generate-Res-2.png) - -## 5.8 Explaining Errors and Fixing Them - -In Java application development, there may be times when errors occur at compile time or runtime. In such cases, you can ask GitHub Copilot Chat about the content of the error, and it will explain the error to you. - -For example, when running a Java application, an exception may occur, and at first glance, the content of the error may be hard to understand. However, by asking GitHub Copilot Chat about the error, it will explain the content of the error to you. - -```text -> When running the application, the following exception was output. - What does the following mean? - -### Exception -Copy & Paste the Java Stack Trace -### -``` - -Furthermore, you can even inquire about how to deal with the problem that occurred. - -```text -> How can I solve the above issues? - -> Can you propose a fix? - -> To solve the issue, please let me know - how to implement *** with sample code. -``` - -When you inquire about an actual exception that occurred and ask for how to deal with it, you will receive an answer as shown below. - - - -In addition to using it to solve actual problems as shown above, you can also use it to find potential bugs or improve the quality of your code by asking about the following contents. - -```text -> Can you check this code for potential bugs or security issues? - -> Do you see any quality improvement to do on this code? -``` - -## 5.9 Conclusion - -GitHub Copilot Chat is a versatile tool in Java application development. By using GitHub Copilot Chat, you can improve your productivity and code quality. Give it a try. - diff --git a/workshops/github-copilot/assets/agents.png b/workshops/github-copilot/assets/agents.png deleted file mode 100644 index cd6c59c1..00000000 Binary files a/workshops/github-copilot/assets/agents.png and /dev/null differ diff --git a/workshops/github-copilot/assets/banner.jpg b/workshops/github-copilot/assets/banner.jpg deleted file mode 100644 index e1ddc32d..00000000 Binary files a/workshops/github-copilot/assets/banner.jpg and /dev/null differ diff --git a/workshops/github-copilot/assets/git-commit.png b/workshops/github-copilot/assets/git-commit.png deleted file mode 100644 index efb08471..00000000 Binary files a/workshops/github-copilot/assets/git-commit.png and /dev/null differ diff --git a/workshops/github-copilot/assets/git-commit2.png b/workshops/github-copilot/assets/git-commit2.png deleted file mode 100644 index bf3574db..00000000 Binary files a/workshops/github-copilot/assets/git-commit2.png and /dev/null differ diff --git a/workshops/github-copilot/assets/quickchat.png b/workshops/github-copilot/assets/quickchat.png deleted file mode 100644 index bbe86c0c..00000000 Binary files a/workshops/github-copilot/assets/quickchat.png and /dev/null differ diff --git a/workshops/github-copilot/assets/src/completesolution.zip b/workshops/github-copilot/assets/src/completesolution.zip deleted file mode 100644 index 42cf9610..00000000 Binary files a/workshops/github-copilot/assets/src/completesolution.zip and /dev/null differ diff --git a/workshops/github-copilot/assets/src/exercisefiles.zip b/workshops/github-copilot/assets/src/exercisefiles.zip deleted file mode 100644 index fa3ead7f..00000000 Binary files a/workshops/github-copilot/assets/src/exercisefiles.zip and /dev/null differ diff --git a/workshops/github-copilot/translations/workshop.ja.md b/workshops/github-copilot/translations/workshop.ja.md deleted file mode 100644 index 125d4297..00000000 --- a/workshops/github-copilot/translations/workshop.ja.md +++ /dev/null @@ -1,1147 +0,0 @@ ---- -published: true -type: workshop -title: Product Hands-on Lab - GitHub Copilot, 新たな AI ペアプログラマー -short_title: GitHub Copilot, 新たな AI ペアプログラマー -description: GitHub Copilotを活用してプロジェクトを開発する方法をご紹介 -level: beginner -authors: - - Philippe DIDIERGEORGES - - Louis-Guillaume MORAND -contacts: - - '@philess' - - '@lgmorand' -duration_minutes: 240 -tags: javascript, .net, GitHub, IA, copilot, AI, csu -banner_url: ../assets/banner.jpg -sections_title: - - はじめに - - Github Copilot - - Github Copilot Chat - - 高度なプロンプトエンジニアリング - - チャレンジ 1 - NodeJS server - - チャレンジ 2 - .Net Core API - - チャレンジ 3 - Infra as Code - - 解答 - - Credits ---- - -# GitHub Copilot を活用して効率を上げる -このワークショップの目的は、Nodejs を使って様々な機能を持つウェブサーバーを構築し、さらに .NET Web API を作成するという課題を通じて GitHub Copilot の使い方を学ぶことです。第二部では、Infrastructure as Code での活用方法や、セキュリティ上好ましくないコードの修正方法を学びます。 - -GitHub Copilot は、開発者がより良いコードをより早く書くのを助ける AI コードアシスタントです。何十億行ものコードに対して訓練された機械学習モデルを使用して、対象のコンテキストに基づいて行全体または関数レベルの提案を行います。GitHub Copilot を使用することで、より良いコードを書く方法を学び、生産性を向上させることができます。 - -
- -> GitHub Copilot は急速に進化している製品であるため、このワークショップは、使用する拡張機能のさまざまな機能に関して 100% 最新ではない可能性があります。コンテンツと異なる場合は状況に応じて推察して進めてください。参考までに、このページは 2023 年 12 月に更新されました。 - -
- -## 前提条件 - -| | | -|----------------|-----------------| -| Node.js v16+ | [Node.js のダウンロード](https://nodejs.org) | -| .Net Core | [.Net Core のダウンロード](https://dotnet.microsoft.com/download) | -| GitHub アカウント | [GitHub 無料アカウントの作成](https://github.com/join) | -| GitHub Copilot のアクセス | 60 日間の試用版は[こちらから申請](https://github.com/github-copilot/signup) | -| コードエディター | [VS Code のダウンロード](https://code.visualstudio.com/Download) | -| VSCode 拡張機能 | [GitHub Copilot](https://marketplace.visualstudio.com/items?itemName=GitHub.copilot), [GitHub Copilot Chat](https://marketplace.visualstudio.com/items?itemName=GitHub.copilot-chat)| -| ブラウザー | [Microsoft Edge のダウンロード](https://www.microsoft.com/edge)もしくはその他| - -
- -> また、いくつかのアセットをダウンロードする必要があります。[ここからダウンロード](../assets/src/exercisefiles.zip)できます。 - -
- -## GitHub Codespaces で作業 - -ハンズオン環境は既に [GitHub Codespaces](https://github.com/features/codespaces) で動作するように構成されており、構成ファイルは *.devcontainer* フォルダーにあります。 - -新しい codespace を作成するだけでプログラミングを開始する準備が整うため、何もインストールする必要はありません。 - -## ローカルで作業 - -また、ローカルのコンピュータ上で作業することもできます。 - -1. [Visual Studio Code](https://code.visualstudio.com/) のインストール -2. [GitHub Copilot](https://marketplace.visualstudio.com/items?itemName=GitHub.copilot) 拡張機能のインストール -3. [GitHub Copilot Chat](https://marketplace.visualstudio.com/items?itemName=GitHub.copilot-chat) 拡張機能のインストール -4. [Node および npm](https://docs.npmjs.com/downloading-and-installing-node-js-and-npm) のインストール -5. mocha のインストール - ``` bash - npm install --global mocha - npm install axios - ``` - -7. [Docker](https://docs.docker.com/engine/install/) のインストール -8. [.NET Core](https://dotnet.microsoft.com/download) のインストール - ---- - -# Github Copilot の最初のステップ - -このセクションでは、GitHub Copilot を使い始めるための最初の手順について説明します。GitHub Copilot でできること、そして GitHub Copilot の可能性を最大限に活用する方法を学びます。既に GitHub Copilot の操作に慣れている場合は、NodeJS での最初の課題にスキップできます。 - -## 準備 - -この最初のチャレンジでは、次の GitHub リポジトリをクローンする必要があります: [Github Copilot Demo](https://github.com/Philess/gh-copilot-demo) - -このリポジトリは、GitHub Copilot の全ての機能を試すのに役立つコードスターターです。ページに表示されているアーキテクチャ設計をよく確認し、準備ができたら、コマンドラインからリポジトリをクローンして VS Code で開きます。 - -``` bash -git clone https://github.com/Philess/gh-copilot-demo -cd gh-copilot-demo -code . -``` - -## はじめての GitHub Copilot - -プロンプトの入力を開始し、Copilot が提案を生成したら、次のショートカットを使用して Copilot を操作できます。 -
    -
  • `tab` : 現在の提案を完全に受け入れる (`最も一般的`)
    • -
  • `ctrl + →` : 単語単位で提案を受け入れる (`部分的に利用`)
    • -
  • `alt + ^` : 次の候補を表示
    • -
  • `shift + tab` : 前の候補を表示
    • -
  • `ctrl + enter` : Copilot ペインの表示
    • -
- -
- -> これらのショートカットを思い出せない場合でも、候補の上にポインターを置くだけで表示されます。 - -
- -## 自然言語翻訳 - -**文字列の自動補完** - -`album-viewer/lang/translations.json` ファイルを開きます。 - -```json -[ - { - "language": "en", - "values": { - "main-title": "Welcome to the world of the future", - "main-subtitle": "The future is now with copilot", - "main-button": "Get started" - } - } -] -``` - -- 最後の "}" の後に "," を追加して新しいブロックを追加し、Enter キーを押します - -
- -## コード生成 - -**プロンプトとは?** -Copilot のコンテキストでは、プロンプトはコード候補を生成するために使用される自然言語の説明の一部です。これは、Copilot がコードを生成するために使用する入力です。1 行または複数行のプロンプトを利用することができます。 - -**プロンプトからコードを生成** - -新しい `album-viewer/utils/validators.ts` ファイルを作成し、プロンプトから始めます: - -```ts -// フランス語形式のテキスト入力から日付の検証をおこない、date オブジェクトに変換 -``` - -Copilot は正規表現を書くのにも役立ちます。これらを試してください: - -```ts -// GUID 文字列の形式を検証する関数 - -// IPV6 アドレス文字列の形式を検証する関数 -``` - -
- -**Copilot で新しいツールやライブラリを発見する** - -同じ `album-viewer/utils/validators.ts` ファイルに、次のプロンプトを追加します: - -```ts -// テキスト入力から電話番号の検証をおこない、国コードを抽出 -``` - -
- -> この例では、おそらく未定義のメソッドを呼び出すという提案がされるでしょう。これは、`ctrl+enter` ショートカットを使用して Copilot ペインを表示し、代替案を検討する良い機会です。 -
外部ライブラリを参照しているものを選択し、Copilot を使用してインポートしてみてください。新しいものを発見するのにこのツールが役に立つことがわかるでしょう。 - -
- -**複雑なアルゴリズムの生成** - -`albums-api/Controllers/AlbumController.cs` ファイルで、`GetByID` メソッドの現在の実装を置き換えて完成させてみてください: - -```cs -// GET api//5 -[HttpGet("{id}")] -public IActionResult Get(int id) -{ - //here -} -``` - -同じファイルで、次のような他のプロンプトを試してみてください: - -```cs -// アルバムを name, artist ないし genre で検索する関数 - -// アルバムを name, artist ないし genre でソートする関数 -``` - -## 大きなタスクと小さなタスク - -### 大きなプロンプトと短いプロンプト - -Copilotは多くの場合で、クラス全体を生成するための複数行のプロンプトではなく、小さいながらも正確に記述されたコードを生成するプロンプトの方がより効果的です。 - -
- -> 大きなコードを生成するための最良の戦略は、簡単なプロンプトでコードの基本的な構造の生成から始めて、単純なプロンプトで小さな部分を 1 つずつ追加していくことです。 - -
- -**動作する*可能性のある*大きなプロンプト** - -- `albums-viewer/utils` に戻り、新しいファイル `viz.ts` を追加して、グラフを生成する関数を作成します。これをおこなうプロンプトの例を次に示します: - -```ts -// D3.js でアルバムの販売価格を年ごとにプロットする -// X 軸は月、Y 軸はアルバム販売数を示す -// アルバムの売上のデータは、外部ソースから読み込まれ、JSON 形式 -``` - -
- ->Copilot はおそらく、詳細を追加してプロンプトを完成させようとするでしょう。自分で詳細を追加するか、Copilot の提案に従ってください。提案を停止してコード生成を開始したい場合は、別の行にジャンプして、Copilot に任せてください。 - -
- -- チャートのコードを生成すると、IDE が d3 オブジェクトについて警告します。これにも Copilot が役立ちます。 -ファイルの先頭に戻り、`import d3` と入力して Copilot に自動補完させます。 - -```ts -import d3 from "d3"; -``` - -Copilot が生成したものを確認してください。コードが正常に動作し、要求した全てを実装している場合もありますが、トークンの制限に達し、Copilot がコード全体を生成できなかった可能性もあります。 - -これは、自動補完用の Copilot が、大きなコードを一度に作成するために作られているのではなく、小さな断片を段階的に生成することに特化しているためです。 - -**ステップバイステップで再チャレンジ** - -以下の手順に従って、プロンプトを細かく分割してコードを生成してみてください: - -```ts -import * as d3 from 'd3'; - -// json ファイルからデータを読み込み、then 関数のコールバック内で d3 svg を作成 -``` - -then 関数のコールバック内で、プロットの基本条件を設定することから始めます - -```ts -// svg の作成 -``` - -```ts -// X 軸と Y 軸のスケールを作成 -// X 軸は月、Y 軸はアルバムの販売数を示す -``` - -```ts -// X 軸と Y 軸の作成 -``` - -あとは、Copilot にチャートを完成させるように依頼するだけです - -```ts -// アルバムの売上データに基づいて折れ線グラフを生成 -``` - -
- ->Copilot の自動補完で大きなタスクを小さなチャンクに分割することで、**常に**より良い結果を得ることができます。また、Copilot は魔法ではなく、他の IDE 機能や開発者のロジックと共に使用する必要があることも示されています。 - -
- -## テスト - -Copilot は、コードで記述されたあらゆる種類のテストを生成するのに役立ちます。これには、たとえば JMeter スクリプトを使用した `単体テスト、統合テスト、エンドツーエンドテスト、ロードテスト` などのテストが含まれます。 - -- `albums-viewer/tests` フォルダーに新しいファイル `validators.test.ts` を追加します - -- 適切なテストの提案を得るには、使用するテストフレームワークなどの基本情報を Copilot に提供する必要があります: - -```ts -import { describe } -``` - -`describe` 関数を入力し始めると、Copilot は開発者が TS のテストファイル内で作業していることを確認し、JS / TS の有名なテストフレームワークである Mocha から `describe` 関数と `it` 関数をインポートすることを提案します。 -提案を受け入れると、Chai からの `expect` 関数も自動的に提案されますので、それも受け入れてください: - -```ts -import {describe, it} from 'mocha'; -import {expect} from 'chai'; -``` - -これでテストフレームワークの準備が整いました。ここで、`import` キーワードで新しい行を開始し、テスト対象の関数をインポートするだけで、Copilot はテストファイル内で作業していると判断し、いくつかの `validators` をテストするために、次のようなものを提案します: - -```ts -import {validateAlbumId} from '../src/validators'; -``` - -一見問題ないように見えますが、Copilot は全てのコードにアクセスできるわけではなく、開いているタブと限られた情報しかアクセスできないため、パスと関数名の両方が間違っていることがわかります。 -
-... 少なくとも Copilot は試してみました ... -
-しかし、Copilotは魔法ではなく、他の IDE 機能やあなたの脳と一緒に使用する必要があることを示す良い例です。 - -- 提案を受け入れ、パスを変更します。`ctrl + space` ショートカットを使うことで、実際に使用可能な関数を VS Code が提案してくれます。 - -- テストしたい最初の関数にコメントを追加して、魔法を起こさせます。: - -```ts -import {describe, it} from 'mocha'; -import {expect} from 'chai'; - -import {validateDate, validateIPV6} from '../utils/validators'; - -// validataDate 関数のテスト -``` - -Boom! - -```ts -describe('validateDate', () => { - it('should return a date object when given a valid date string', () => { - const date = '01/01/2019'; - const expectedDate = new Date(2019, 0, 1); - expect(validateDate(date)).to.deep.equal(expectedDate); - }); - - it('should throw an error when given an invalid date string', () => { - const date = '01/01/2019'; - expect(() => validateDate(date)).to.throw(); - }); -}); -``` - -*他の `it` ブロックを追加して、テストケースを追加したり、他の関数のテストを追加したりできます。たとえば、新しい `it` ブロックを追加して、空の文字列が与えられたときにエラーがスローされることをテストしてみてください。* - -## CI パイプラインの記述 - -*Copilot は、さまざまなステップやタスクのコードを生成することで、パイプライン定義ファイルの作成に役立ちます。ここでは、その機能の例をいくつか紹介します:* - -- *パイプライン定義ファイルを `ゼロから` 生成します* -- *さまざまな `ステップ、タスク、スクリプトの一部` の `コードを生成する` ことで、パイプライン定義ファイルの記述を高速化します* -- *あなたのニーズに合った `Marketplace のタスクと拡張機能を見つける` のを手伝ってくれます* - -### ステップ 1: ゼロから生成 - -- プロジェクトの `.github/workflows` フォルダーに新しいファイル `pipeline.yml` を作成し、次のプロンプトの入力を開始します: - -```yml -# main ブランチへの push 時に起動する GitHub Actions のパイプライン -# album-api イメージを Docker build し、ACR に push -``` - -*Copilot はブロックごとにパイプラインを生成します。パイプラインの Yaml ファイル生成では、他のタイプのコードよりも頻繁に次のブロックの生成をトリガーするために、新しい行にジャンプする必要がある場合があります。* - -*多くの場合、インデントが正しくなかったり、タスク名の周りの引用符が欠落していたりして、いくつかのエラーが発生するタスクが生成されます。これらは、IDE と開発者のスキルで簡単に修正できます。* - -### ステップ 2: プロンプトからタスクを追加 - -- 少なくともコンテナーレジストリへの "ログイン" タスクと "docker ビルドとデプロイ" タスクを含む GitHub Actions ワークフローが生成されているでしょう。これらのタスクの後に新しいコメントを追加して、Docker イメージに GitHub Actions run-id のタグを付け、レジストリにプッシュします: - -```yml -# GitHub Actions の run-id のタグをイメージに付与し、Docker Hub に push -``` - -次のような他のプロンプトを試すことができます: - -```yml -# album-api イメージでテストを実行 - -# album-api を dev 環境の AKS クラスターにデプロイ -``` - -### ステップ 3: プロンプトからスクリプトを追加 - -- Copilot は、次の例のようなカスタムスクリプトを記述する必要がある場合にも非常に便利です: - -```yml -# 全ての appmanifest.yml ファイルで %%VERSION%% を見つけて GitHub Actions の run-id に置換 -``` - -## Infra As Code - -Copilot は、インフラをコードとして記述するのにも役立ちます。`Terraform、ARM、Bicep、Pulumi` や、`Kubernetes マニフェスト ファイル` などのコードを生成することも可能です。 - -### Bicep - -`iac/bicep` フォルダー内の `main.bicep` ファイルを開き、ファイルの最後にプロンプトを入力して新しいリソースを追加します: - -```js -// Container Registry - -// Azure Cognitive Services Custom Vision resource -``` - -### Terraform - -`iac/terraform` フォルダー内の `app.tf` ファイルを開き、ファイルの最後にプロンプトを入力して新しいリソースを追加します: - -```yml -# Container Registry - -# Azure Cognitive Services Custom Vision resource -``` - -## Git Commit コメントの生成 - -コメントを書くことは必須であるべきですが、開発者は怠りがちです。GitHub Copilot がお手伝いします。 - -1. 適当なファイルに何かしらのコンテンツを追加します。 - -2. Git コミットパネルで、右側にある小さな魔法のボタンをクリックします - - ![Github Copilot Git コメントジェネレーター](../assets/git-commit.png) - -3. あなたに代わってコメントを生成しくれた Copilot を褒めてあげましょう - - ![生成されたコメント](../assets/git-commit2.png) - -## ドキュメントの記述 - -Copilot は自然言語のプロンプトを理解してコードを生成することができます。また、`コードを理解し、自然言語で説明する`こともでき、コードのドキュメントを作成するのに役立ちます。 -そのため、全てのドキュメント作成タスクに役立ちます。単純なドキュメントコメントや、JavaDoc、JsDoc などの標準化されたドキュメントコメントを生成できます。また、ドキュメントをさまざまな言語に翻訳するのにも役立ちます。それがどのように機能するか見てみましょう。 - -### 単純なドキュメントコメント - -これを確認するには、クラス、メソッド、または任意のコード行の上にポインターを置き、選択した言語のコメントハンドラーの入力を開始して Copilot をトリガーします。たとえば、Java、C#、TS などの言語では、 `//` と入力するだけで魔法が起こります。 - -以下は `albums-viewer/routes/index.js` ファイルの例です。行を挿入し、`try ブロック` 内の 13 行目の入力を開始します - -```js -router.get("/", async function (req, res, next) { - try { - // Invoke the album-api via Dapr - const url = `http://127.0.0.1:${DaprHttpPort}/v1.0/invoke/${AlbumService}/method/albums`; - -``` - -他のコードでも何が起こるかを試してみてください。 - -### 標準化されたドキュメントコメント (JavaDoc、JsDoc など) - -この場合、ドキュメントコメントの生成をトリガーするには、特定のコメント形式を意識する必要があります: - -- `/**` JS/TS: たとえば `index.js` ファイル内 -- `///` C#: たとえば AlbumApi の `AlbumController.cs` ファイル内 - -```cs -/// -/// アルバムを取得する関数 -/// -/// アルバムのID -/// アルバムの情報 -[HttpGet("{id}")] -public IActionResult Get(int id) -``` - -### マークダウンと HTML ドキュメントの記述 - -Copilot は、ドキュメントの記述にも非常に強力です。`マークダウン` と `html` コードを生成し、例えばこのような readme.md ファイルの記述を加速するのに役立ちます。 - -これを試すために、プロジェクトのルートに新しいファイル `demo.md` を作成し、次のプロンプトを入力してください: - -```md -# Github Copilot ドキュメント -このドキュメントは、ツールが何をできるかを示すために GitHub Copilot で作成されました。 - -## -``` - -次に、2 番目のレベルのタイトルで新しい行を開始することで、ドキュメントのコンテンツを生成し、ドキュメントの記述プロセスを加速する方法を示します。 - ---- - -# コード品質を向上させるために Copilot Chat を使用する - -GitHub Copilot は生成 AI であり、そのため、コードを生成するのに最適ですが、コードの分析機能も強力で、コードの品質を向上させるために使用できます。たとえば、セキュリティの問題を見つけ、コードの品質を向上させるための提案を生成し、レガシーコードにコメントを追加し、リファクタリングをおこない、テストを生成するなど、さまざまなケースでコード品質を向上させるために使用できます。 - -既に慣れている場合は、次のセクションに進むことができます。 - -## はじめに - -GitHub Copilot Chat を使用するには、まず次のことが必要です: - -- 有効な GitHub Copilot ライセンス (Individual、Business、Enterprise) を持っていること。 -- IDE に拡張機能をインストールすること。VS Code の場合、拡張機能タブで `Github Copilot Chat` を検索して直接見つけることができます。 - -### リポジトリのクローン - -前のセクションと同じリポジトリを使用し、Copilot Chat を使用してコード品質を向上させる方法を示します。既にお持ちの場合は、この手順をスキップできます。 - -次の GitHub リポジトリをクローンする必要があります: [Github Copilot Demo](https://github.com/Philess/gh-copilot-demo) - -このリポジトリは、GitHub Copilot で全ての機能を試すのに役立つコードスターターです。リポジトリのページに表示されているアーキテクチャー設計をよく確認し、準備ができたら、コマンドラインからリポジトリをクローンして VS Code で開きます。 - -``` bash -git clone https://github.com/Philess/gh-copilot-demo -cd gh-copilot-demo -code . -``` - -## はじめての Copilot Chat - -Copilot Chat をインストールしたら、次の手順で使い始めることができます。 - -- IDE の左側のツールバー (チャットアイコン) から**チャットビュー**にアクセスします。 -- `Ctrl` + `Shift` + `i` ショートカットを押すと、チャットに簡単な**インライン質問**が表示されます。 - -最初のものは固定のバージョンで、チャットを開いたままにして Copilot に質問するのに非常に便利です。 -2 つ目は、質問をし、回答を得てコマンドを起動するための簡単な方法です。 - -### チャットビュー - -チャットビューは、IDE の他のツールビューと同様に統合され、完全なチャットエクスペリエンスを提供します。ビューが開いたら、Copilot をコードコーチとしてチャットを開始できます。会話の履歴を保持し、前の回答に関連する質問をすることができます。また、途中で質問に対する提案も提供します。次のことが可能です: - -- 任意の言語でのコーディングやベストプラクティスに関する一般的な質問をする -- 現在のファイルに関連するコードを生成または修正し、そのコードをファイルに直接挿入するように依頼する - -これは、コード補完の提供に特化した普通の Copilot よりも高レベルの Copilot です。 - -次のようないくつかの質問で試してみてください: - -```text -> C#で乱数を生成するには? -> ASP.NET Core でルートをセキュアにする最良の方法は何ですか? -> NodeJS で静的 Web サイトを生成する最も簡単な方法は何ですか? -``` - -次に、リポジトリ内のコード ファイルのいくつかで試してみてください。ファイルを開いて、次の質問をしてみてください: - -```text -> このコードが何をするのか説明していただけますか? -> (コードの一部のみを選択した状態) 選択したコードが何をするのか説明していただけますか? -> 1 から 10 までの乱数を返す関数を生成できますか? -> この関数にドキュメントコメントを追加できますか? -``` - -途中で表示される質問の提案も使用してみてください。 - -### インラインの質問 - -インライン質問は、Copilot に質問して回答を得るための簡単な方法です。これは、特定のコードについて質問するのに適した方法です。また、Copilot のコマンドを起動するのにも適しています。コードの生成、コードの修正、テストの生成などを依頼できます。 - -`Ctrl` + `Shift` + `i` を押して、チャットビューで試したのと同じコマンドを入力して試してみてください。 - -### スラッシュコマンド - -Copilot がより関連性の高い回答を得られるように、`スラッシュコマンド` を使用して質問のトピックを選択できます。 - -チャット入力の先頭に特定のトピック名を付加すると、Copilot がより関連性の高い応答を返すことができます。 `/` と入力し始めると、考えられるトピックのリストが表示されます。 - -- **/explain**: 選択したコードがどのように機能するかを段階的に説明します。 -- **/fix**: 選択したコードのバグの修正を提案します。 -- **/help**: GitHub Copilot に関する一般的なヘルプを出力します。 -- **/tests**: 選択したコードの単体テストを生成します。 -- **/vscode**: VS Code のコマンドと設定に関する質問に答えます。 -- **/clear**: セッションをクリアします。 - -## コードをセキュリティで保護する - -Copilotは、コード内のセキュリティ問題を見つけて修正するのに役立ちます。また、コード内の悪いプラクティスを見つけて修正するのにも役立ちます。それがどのように機能するか見てみましょう。 - -`album-api/Controllers/UnsecuredController.cs` ファイルを開き、次のような質問をチャットに入力します: - -```text -> このコードにセキュリティ上の問題がないか確認できますか? -> このコードの品質を改善する方法はありますか? -``` - -答えが得られたら、次のように入力して問題の修正を依頼できます。 - -```text -> 修正を提案してもらえますか? -``` - -コードの修正が提案された場合は、チャットのコードブロックにカーソルを合わせ、左上の適切なオプションを選択して、**コピーするか、ファイルに直接挿入する**かを選択します。 - -## コードの説明とドキュメント - -Copilot Chat を使用してコードを説明してもらうことができます。`自然言語でコードを説明したり、ドキュメントコメントを生成したり`することができます。次のコマンドで試してみましょう。: - -```test -> /explain -> このコードのドキュメントコメントを生成して -``` - -## コードのリファクタリング - -さらに印象的なのは、Copilot チャットがコードのリファクタリングに役立つことです。`変数の名前変更、メソッドの抽出、クラスの抽出など`に役立ちます。 - -これらのコマンドのいくつかは、`album-api/Controllers/UnsecuredController.cs` ファイルで試すことができます: - -```test -> メソッドを抽出して -> 非同期処理が適切な場合、各メソッドの非同期バージョンを作成して -``` - -## コード翻訳 - -*Copilot は自然言語とプログラミング言語の両方を理解して生成できるため、それらを組み合わせることで、`コードの断片をある言語から別の言語に翻訳する`のに使用できます* - -特定の言語のコードを翻訳するには、そのコードを開き、チャットで別の言語への翻訳を依頼します。たとえば、Copilot 自動補完専用の最初のセクションで作成した `validators.ts` ファイルを開き、C への翻訳を依頼します。 - -COBOL のようなレガシーコードを扱う場合も非常に便利です。`legacy/albums.cbl` ファイルを開き、コードを Python に変換してみてください。 - -## テスト生成 - -Copilot は、コードのテストを生成するのにも役立ちます。たとえば、JMeter スクリプトを使用して `単体テスト、統合テスト、エンドツーエンドテスト、およびロードテスト`を生成できます。 - -`album-api/Controllers/UnsecuredController.cs` ファイルを開き、次のような質問をチャットに入力します: - -```test -> このコードの単体テストクラスを生成して -``` - -また、Copilot を使用して、テスト用のスタブとモックを生成することもできます。 - -```text -> FileStream クラスのモックを生成して -> そのモックを単体テストで使用して -``` - -
- -> Copilot chat は会話の前の Q & A を追跡しているため、以前に生成されたモックを参照して簡単にテストできることを覚えておいてください。 - -
- -## Chat participants の利用 - -Chat participants は、特定のタスクを支援できる専門の専門家のようなものです。チャットで @ 記号を使用してメンションできます。現在、次の 3 つの Chat participants を使用できます: - -- **@workspace**: この chat participant は、ワークスペース内のコードに関する知識を持っており、関連するファイルまたはクラスを見つけることで、ワークスペースの遷移を支援できます。@workspace chat participant は、メタプロンプトを使用して、質問への回答に役立てるためにワークスペースから収集する情報を決定します。 -- **@vscode**: この chat participant は、VS Code エディター自体のコマンドと機能に関する知識があり、それらの使用を支援できます。 -- **@terminal**: この chat participant は Visual Studio Code のターミナルとそのコンテンツに関するコンテキストを持っています。 - -今のところはそれほどリッチではないかもしれませんが、これらの機能は時間の経過とともに成長し続けます。次に例をいくつか示します - -サイドチャットパネルを開き、 `@workspace /new` と入力して、新しいプロジェクトを作成することを指定します。たとえば、ASP.NET プロジェクトを作成してみてください - -```text -> @workspace /new Index、Users、および Product の 3 つのビューをもった ASP.NET Core 6.0 のプロジェクト -``` - -構造化されたプロジェクトと、ファイルを作成するための新しいボタンが表示されるはずです。`Create workspace` をクリックして、作成中のファイルを確認します。 - -![GitHub Copilot Chat Participants](../assets/agents.png) - -## Tips - -GitHub Copilot Chat は非常に便利ですが、開発者にとっては、キーボードを離れ、マウスを動かしてチャットタブを開くのは面倒な作業です。そのためコードエディター内でチャットを直接呼び出すことができます。 - -1- コードを含むファイルを開きます - -2- ショートカット **Ctrl + i** を使用します。クイックチャットのポップアップ、即ちカーソルを移動させることができる小さなチャットウィンドウが開きます。 - -![GitHub Copilot クイックチャット](../assets/quickchat.png) - -3- コードを生成するコマンドを入力します (たとえば、`Toto という名前の C# クラスを作成して`)。生成されたコードは、現在のファイル内に挿入されます。これはあなたが望むものかもしれません。 - ---- - -# Copilot Chat におけるプロンプトエンジニアリング - -前のセクションでは、基本的なプロンプトを使用して Copilot Chat にコードを生成してもらう方法について説明しました。このセクションでは、プロンプトエンジニアリング手法を使用して、より正確な結果を得るための手法を学習します。 - -**プロンプトエンジニアリングとは？** -プロンプトエンジニアリングは、高品質のコード提案を生成するために、高品質のプロンプトを設計するプロセスです。より良いプロンプトを書くための良いプラクティスとヒントがあります。それらのいくつかを見てみましょう。 - -## 例を挙げる: one-shot と few-shots プログラミング - -プロンプトエンジニアリングを利用し、チャットを使って Copilot に例を提供することもできます。これは、Copilot が何をしたいのかを理解し、より良いコードを生成するのに役立つ良い方法です。チャットで例を提供するには、validator.ts ファイルを開いて次のように入力します: - -```bash -# one-shot プログラミング - -現在のファイルで mocha と chai を使用して、電話番号検証メソッドの単体テストを書いてください。 -ポジティブテスト (true を返すテスト) には、次の例を使用します: -it('電話番号が有効な国際番号である場合、trueを返す', () => { expect(validatePhoneNumber('+33606060606')).to.be.true; }); -ロジックのセットでテストを整理し、メソッドごとに少なくとも 4 つのポジティブテストと 2 つのネガティブテストを生成してください。 -``` - -```bash -# few-shot プログラミング - -現在のファイルで mocha と chai を使用して、全ての検証メソッドの単体テストを書いてください。 -ポジティブテスト (true を返すテスト) には、次の例を使用します。 -it('電話番号が有効な国際番号の場合は true を返す', () => { expect(validatePhoneNumber('+33606060606')).to.be.true; }); -it('電話番号が有効なアメリカのローカル番号の場合は true を返す', () => { expect(validatePhoneNumber('202-939-9889')).to.be.true; }); -it('指定された電話番号が空の場合、エラーをスローする', () => { expect(validatePhoneNumber('')).to.throw(); }); -ロジックのセットでテストを整理し、メソッドごとに少なくとも 4 つのポジティブテストと 2 つのネガティブテストを生成してください。 -``` - -この手法を使用して、**別のファイルのスタイルに従うコードを生成**できます。たとえば、albums-api>Models>Album.cs と同じように音楽スタイルのサンプルのレコードを作成する場合は、ファイルを開いて次のように入力します: - -```bash -Album.cs ファイルのように、6 つのサンプル値を List の形で持つ MusicStyle レコードを作成してください。 -``` - -## 外部参照の提供 - -Copilot Chat は、外部参照を使用して、より正確な提案を作成できます。たとえば、API にリクエストを送信するコードを生成する場合は、チャットで API レスポンスの例を提供するか、API リファレンスの URL を提供できます。Copilot はそれを使用して、より良いコードを生成します。 - -```bash -次の API から全ての犬種を取得し、Breed オブジェクトの配列を返す TS 関数を書いてください: HTTP GET https://dog.ceo/api/breeds/list/all -``` - -Copilot は、指定された外部参照を使用してコードを生成します。subBreeds プロパティをもつ Breef インターフェイス (またはクラス) が生成されます。これは、外部参照によって指定された API から取得されます。 - -```ts -interface Breed { - name: string; - subBreeds: string[]; -} -``` - -
- -> SDK やライブラリなどの外部ドキュメントへのリンクを提供することもできます。または RFC などのイベント規範文書など... - -
- -## ロールプロンプト - -foundational prompt とも呼ばれ、Copilot Chat の行動をパーソナライズし、Copilot のフレーバーを設定するために与える一般的なプロンプトです。 - -Copilot Chat で新しいタスクを開始するときに最初におこなうべきことは、**何を構築したいのか、Copilot にどのように支援してもらいたいのかを明確に説明すること**です。 - -
- -> **これは適切に処理された場合に非常に強力です**。そのため、全てのコーディングセッションをロールプロンプトで開始し、将来的な使用のために最適なプロンプトを保存してください。 - -
- -***ロールプロンプトの構造*** - -ロールプロンプトに何を含めることができるか: - -- 構築したいものに関する確かなコンテキストと背景情報を提供します。 -- GitHub Copilot の役割を定義し、どのようなフィードバックを求めているかについて期待値を設定します。 -- 回答の質を具体的にし、より多くのことを学び、受け取った回答が正しいことを確認するために、参考資料や追加のリソースを求めてください。 -- タスクを再開し、指示が明確かどうかを尋ねます。 - -***ロールプロンプトの例*** - -新しい会話を開始し、次のプロンプトを入力します: - -```bash -私は React Native で構築された新しいモバイルアプリケーションに取り組んでいます。 -ユーザーが犬の写真をアップロードして犬の品種を取得できる新しい機能を構築する必要があります。 -品種を操作するには、https://dog.ceo/api/breeds の API セットを使用する必要があります。コードが少なくとも OWASP Top 10 (https://owasp.org/Top10/) に対応していることを確認する必要があります。 -コードの単体テストが必要であり、また、https://www.w3.org/TR/WCAG21/ で定義されている WCAG 2.1 レベル A および AA の達成基準に準拠したアクセシビリティにしたいと考えています。 -私のコードがこれら全ての要件を満たしていることを確認するために、私自身のコードコーチとして行動してもらう必要があります。 -可能であれば、追加の学習のためのリンクと参照を提供してください。 -これらの指示を理解していますか? -``` - -そこから質問を開始し、時々、Copilot が指示に従っていることを確認するには、次の質問をします: - -```bash -私の指示をまだ使用していますか? -``` - -***ロールプロンプトをテストする*** - -React Native Apps と OWASP Top 10 に関するアクセシビリティのベストプラクティスについて質問をすることで、ロールプロンプトをテストできます。また、アップロード機能のコードを生成するように依頼し、生成されたコードがセキュリティで保護され、アクセシビリティの基準を満たしているかどうかを確認することもできます。 - -たとえば、次の質問を試してみてください: - -```bash -React Native で実装されたアプリのアクセシビリティを向上させるにはどうすればいいですか? - -アプリから写真をアップロードする最も安全な方法は何ですか? -``` - ---- - -# NodeJS サーバーの開発 - -この最初の演習では、機能要件に従って実際のプロジェクトを開発します。自分自身で進めることもできますし、GitHub Copilot の助けを借りることも可能です。 - -## 手順 - -- [exercicefile](../assets/src/exercisefiles.zip) フォルダーをローカルにダウンロードします -- `nodeserver.js` を開き、Nodejs サーバーを作成することから始め、テキストに初めから書いてある指示を基にした Copilot の提案を確認します -- `test.js` ファイルを開き、現在のテストを分析します -- コマンドプロンプトを開き、テストを実行します (`mocha test.js`) -- 結果を見ると、次のように表示されるはずです: - -``` bash -mocha test.js -server is listening on port 3000 - - Node Server - - √ should return "key not passed" if key is not passed - - 1 passing (34ms) - -``` - -- `nodeserver.js` では、演習セクションで残りのメソッドを開発します - -> 全てのコンテキストを与えてよりよい提案が生成されるように、Visual Studio Codeで `color.json` ファイルを開くことを忘れないでください。 - -- test.js ファイルに、機能をテストするメソッドを追加します。 -- テストを実行して、全てが機能していることを確認します。 -- `dockerfile` ファイルを開いてプロンプトを入力し、Web サーバーを実行できる Node イメージを含む Docker コンテナーを作成します。 -- ポート 4000 で docker を実行するコマンドを作成します。 -- アプリケーションがポート 4000 で動作していることをテストします。 -- **nodeserver.js** ファイルに `//サーバーをテストするために curl コマンドを実行` のような新しい行を入力すると、現在のファイルに基づいて GitHub Copilot がコマンドラインで実行するための curl コマンドをどのように生成するかを確認できます。 -- 注: `//daysBetweenDatesをテストするための curl コマンドを実行` といったコメントのように、より具体的に指定できます。この場合は特定のメソッドのテストが生成されるはずです。 - -## 演習 - -次に、新しい機能を開発し、サーバーに追加してください。サーバーが実装すべき要件は次のとおりです。 - -
- -> 入力途中で GitHub Copilot が提案を表示します。提案を受け入れるには Tab キーを押します。数行書いた後 GitHub Copilot が何も表示しない場合は、Enter キーを押して数秒待ちます。Windows または Linux では、Ctrl + Enter キーを押します。 - -
- -
- -> 書くべきコードはたくさんありますが、完成までにかかる時間に驚かれるかもしれません。また、必要に応じて 7 つか 8 つだけ書いても構いませんが、この演習は退屈なものではありません。 - -
- -| メソッド | 要件 | -|---|---| -|**/Get**|hello world メッセージを返します| -|**/DaysBetweenDates**|2 つの日付の間の日数を計算します
2 つのクエリ文字列パラメーター date1 と date 2 で受け取り、これら 2 つの日付の間の日数を計算します| -|**/Validatephonenumber**|クエリ文字列で `phoneNumber` というパラメーターを受け取ります
`phoneNumber` をスペイン語のフォーマットで検証します (例: +34666777888)
`phoneNumber` が有効であれば "valid" を返します
`phoneNumber` が有効でなければ "invalid" を返します| -|**/ValidateSpanishDNI**|クエリ文字列で `dni` というパラメーターを受け取り、DNI の文字列を検証します
DNI が有効であれば "valid" を返します
DNI が有効でなければ "invalid" を返します
機能が正しく実装されていることを確認するための自動テストを作成します| -|**/ReturnColorCode**|クエリ文字列で `color`というパラメーターを受け取ります
`colors.json` ファイルを読み取り、該当する色の `rgba` フィールドを返します| -|**/TellMeAJoke**|axios を使って joke API を呼び出し、ランダムにジョークを返します| -|**/MoviesByDirector**|(この実装には [https://www.omdbapi.com/apikey.aspx](https://www.omdbapi.com/apikey.aspx) を参照し、無料の API キーを取得する必要があります)
クエリ文字列で `director` というパラメーターを受け取ります
axios を使って映画の API を呼び出し、その `director` の映画一覧を返します| -|**/ParseUrl**|クエリ文字列で `someurl` というパラメーターを受け取ります
URL を Parse し、プロトコル、ホスト、ポート、パス、クエリ文字列、およびハッシュを返します| -|**/ListFiles**|カレントディレクトリを取得します
カレントディレクトリの全てのファイルを取得します
ファイルの一覧を返します| -|**/GetFullTextFile**|`sample.txt` を読み込み、"Fusce" という単語を含む行を返します (この実装には注意してください。なぜならば普通は解析前にファイルのコンテンツを全て読み込むことになるため、メモリー使用率が上昇し、ファイルがあまりにも大きいとエラーになる可能性があるためです。)| -|**/GetLineByLinefromtTextFile**|`sample.txt` を行ごとに読み込みます
ファイルを行ごとに読み込む promise を作成し、"Fusce" という単語を含む行の一覧を返します| -|**/CalculateMemoryConsumption**|プロセスのメモリー消費量を GB 単位で返します (小数点以下 2 桁に丸めて)| -|**/MakeZipFile**|zlib を利用して、`sample.txt` を含む `sample.gz` ファイルを作成します| -|**/RandomEuropeanCountry**|ヨーロッパの国とその ISO コードの配列を作成します
配列からランダムな国を返します
国とその ISO コードを返します| - -## GitHub Copilot Chat 演習 - -以下のタスクは、[GitHub Copilot Chat](https://marketplace.visualstudio.com/items?itemName=GitHub.copilot-chat) 拡張機能で実行できます。 - -- **説明** - -validatePhoneNumber メソッドで正規表現の書かれている行を選択し、`/explain` コマンドを使用します。正規表現中のそれぞれの異なる表記法が何を意味するかについて詳しく説明してくれます。 - -- **プログラミング言語変換** - -たとえば、次のソースコードを選択します: - -``` js -var randomCountry = countries[Math.floor(Math.random() * countries.length)]; -``` - -Copilot Chat に別の言語 (Python など) への変換を依頼します。**python** で新しいコードが表示されます。 - -- **読みやすさ** - -MakeZipFile メソッドを選択します - -コードを読みやすくするよう Copilot Chat に依頼します。どのようにコメントが追加されるか、また、短い名前の変数の名前がどのようなわかりやすい名前に変更されるかを確認してください。 - -- **バグの修正** - -この演習では、ほとんどのコードが GitHub Copilot によって書かれるため、バグはほとんどありません。いくつかのバグをわざと作りこむことによって、デバッグ機能をテストできます。 - -次のようなバグを作りこみます: - -for ループで、先頭を次のように変更します (0 を 1 に変更します)。 - -``` js - for (var i = 1 -``` - -当該コードを選択し、Copilot Chat にバグの修正を依頼します。 - -- **堅牢にする** - -ユーザー入力由来のテキスト (クエリ文字列から取得される変数など) を選択します。 - -``` js - var queryData = url.parse(req.url, true).query; - var date1 = queryData.date1; - var date2 = queryData.date2; -``` - -Copilot Chat にコードを堅牢にするよう依頼すると、検証ロジックが追加されることがわかります。 - -- **ドキュメント** - -以下のような行を選択します (例: メソッドや if 句の先頭) - -``` js - else if (req.url.startsWith('/GetFullTextFile')) -``` - -Copilot Chat にドキュメントの作成を依頼します。Copilot Chat がコードの動作を説明し、コメントが追加されます。 - ---- - -# .Net Core - -目標は、GitHub Copilot の助けを借りて、.NET 6.0 と Docker を使用した単純な WebAPI を作成することです。 -以下の手順に従って、可能な限り GitHub Copilot を使用してみてください。 -さまざまなことを試してみて、Dockerfile やクラスの生成、コメントの追加など、GitHub Copilot で何ができるかを確認してください。 - -注意: - -VS Code の右下隅にあるステータス バーを確認し、GitHub Copilot が現在の言語用に構成され、有効になっていることを確認します。 - -## dotnet WebAPI プロジェクトの作成 - -- 新規 .NET プロジェクトの作成 - -``` powershell -dotnet new webapi -``` - -- フォルダーに新しいファイル `User.cs` を作成し、Copilot にクラスを生成するように指示します。 - -- 新しいファイル `UserController.cs` を Controllers フォルダーに追加し、Copilot にコントローラーを ControllerBase から継承して生成するように指示します。 - -- クラスに `ApiController` 属性と `Route` 属性を追加します。 - -- `IUserService` という新しいファイルを Abstractions フォルダーに追加し、Copilot にインターフェースを生成するように指示します。 - -- 次のコマンドを利用してアプリケーションを実行します (GitHub Codespaces を使用している場合は、`Program.cs` から HTTPS リダイレクトを削除する必要がある場合があります) - -``` powershell -dotnet run -``` - -- Services フォルダーの `UserService.cs` にインターフェイス IUserService を実装し、GitHub Copilot が実装を生成できるようにコメントを追加します。 - -- Copilot に、ユーザーのリストと、そのリストを使用した追加メソッドと取得メソッドを生成するように指示します。 - -- アプリをビルドする前に、`Program.cs` で `IUserService` を DI コンテナーに登録します。 - -``` csharp -builder.Services.AddSingleton(); -``` - -- 次のコマンドを利用してアプリケーションを実行します。 - -``` powershell -dotnet run -``` - -> "No server certificate was specified..." エラーが発生した場合、次のコマンドを実行してください。 - -``` powershell -dotnet dev-certs https -``` - -- 必要に応じてポートフォワーディングの設定をおこなってください - -- /swagger エンドポイントにアクセスします。e.g. [https://leomicheloni-supreme-space-invention-q74pg569452ggq-5164.preview.app.github.dev/swagger/index.html](https://leomicheloni-supreme-space-invention-q74pg569452ggq-5164.preview.app.github.dev/swagger/index.html) - -## アプリケーションを Docker コンテナーで実行 - -- アプリケーションを publish し、_publish_ フォルダーに出力します - -``` dotnet -dotnet publish -c Release -o publish -``` - -- 既存の `Dockerfile` ファイルを利用し、アプリケーションをコンテナーに配置して実行します (指示を追加するか、もしくはコードを書き、GitHub Copilot に保管させます) - -- イメージをビルドし、ポート 8080 で実行します - -``` powershell -docker build -t dotnetapp . -docker run -d -p 8080:80 --name dotnetapp dotnetapp -``` - ---- - -# Infrastructure as Code - -プログラミング言語からコードを生成するのは 1 つの使用例ですが、GitHub Copilot は Terraform、Bicep などの構成ファイルを生成するのに役立つでしょうか？ - -この演習では、前のセクションで開発した Web アプリケーションをデプロイし、Azure でホストします。要件は次のとおりです。 - -- アプリケーションは、Azure Web アプリ名 `my-webapp` でホスト -- App Service プラン (CPU とメモリ) の名前は `my-plan` で、SKU (サイズ) `B1` を使用 -- Web アプリは、リソース グループ名 `oh-yes` で West Europe でホスト - -
- -> 上記のケースで GitHub Copilot を使用する方法はいくつかあります。たとえば、GitHub Copilot に提案を生成してもらう前に、数行の連続したコメントを書くことができます。さらに、結果が意図したものでない場合、サイドパネルを開いて 10 個の代替案を生成できます。これをおこなうには、`ctrl` + `Enter` をクリックします - -
- ---- - -# 解答 - -ここでは、いくつかの演習の解決策を確認することができます。 - -## コーディング - -コーディング演習の解答は [ここからダウンロード](../assets/src/completesolution.zip) できます。 - -## Infrastructure As Code - -この部分が最も簡単ですが、GitHub Copilot は不適切なコードやコメント付きのコードをランダムに生成する可能性があります。 - -``` bicep -param webAppName string = 'my-webapp' -param webAppPlanName string = 'my-plan' -param webAppPlanSku string = 'B1' -param webAppPlanLocation string = 'westeurope' -param resourceGroupName string = 'oh-yeah' - -resource appServicePlan 'Microsoft.Web/serverfarms@2021-02-01' = { - name: webAppPlanName - location: webAppPlanLocation - kind: 'app' - sku: { - name: webAppPlanSku - tier: 'Basic' - size: 'B1' - } -} - -resource webApp 'Microsoft.Web/sites@2021-02-01' = { - name: webAppName - location: webAppPlanLocation - kind: 'app' - properties: { - serverFarmId: appServicePlan.id - } -} - -resource resourceGroup 'Microsoft.Resources/resourceGroups@2021-04-01' = { - name: resourceGroupName - location: webAppPlanLocation -} -``` - -## DevSecOps - -GitHub Copilotは、全てのコードを修正およびリファクタリングできない場合がありますが (たとえば、`バグの修正` プロンプトを使用)、チャットで質問すると、Code smells や悪いプラクティスを認識してくれるのは非常に良いことです。 - -以下の短いコードには、いくつかのセキュリティ上の欠陥があります。少なくとも 4 つの重大な悪いプラクティスが見つかるはずです。 - -このコードは無害に見えますが、[パスインジェクション](https://learn.microsoft.com/en-us/dotnet/fundamentals/code-analysis/quality-rules/ca3003)を許可する可能性があります。これは、誰かがディスク上の別のファイルにアクセスしようとする可能性があることを意味します。 - -``` csharp -using (FileStream fs = File.Open(userInput, FileMode.Open)) - { - // 可能であれば、ユーザー入力に基づいてファイルパスを明示的に既知のセーフリストに制限します。たとえば、アプリケーションが "red.txt"、"green.txt"、または "blue.txt" にのみアクセスする必要がある場合は、これらの値のみを許可します。 - // 信頼できないファイル名をチェックし、名前が問題ないことを確認します。 - // パスを指定するときは、絶対パス名を使用します。 - // PATH 環境変数などの潜在的に危険な構造は避けてください。 - // 長いファイル名のみを受け入れ、ユーザーが短い名前を送信する場合にのみ長い名前を検証します。 - // エンドユーザー入力を有効な文字に制限します。 - // MAX_PATH の長さを超えた名前をリジェクトします。 - // ファイル名を解釈せずにリテラルとして扱います。 - // ファイル名がファイルとデバイスのどちらを表しているかを確認します。 - - byte[] b = new byte[1024]; - UTF8Encoding temp = new UTF8Encoding(true); - - while (fs.Read(b, 0, b.Length) > 0) - { - return temp.GetString(b); - } - } - - return null; -} -``` - -これは [SQL インジェクション](https://learn.microsoft.com/en-us/dotnet/fundamentals/code-analysis/quality-rules/ca3001)の有名な例です。一番良いのは、エスケープコードや間違ったパラメーターボックス（型）の試みに対処できるパラメータを使うことです。 - -``` csharp -public int GetProduct(string productName) -{ - using (SqlConnection connection = new SqlConnection(connectionString)) - { - SqlCommand sqlCommand = new SqlCommand() - { - CommandText = "SELECT ProductId FROM Products WHERE ProductName = '" + productName + "'", - CommandType = CommandType.Text, - }; - - // セキュアな方式 - // SqlCommand sqlCommand = new SqlCommand() - // { - // CommandText = "SELECT ProductId FROM Products WHERE ProductName = @productName", - // CommandType = CommandType.Text, - // }; - // sqlCommand.Parameters.AddWithValue("@productName", productName); - - SqlDataReader reader = sqlCommand.ExecuteReader(); - return reader.GetInt32(0); - } - -} -``` - -一般的な良いプラクティスは、エンドユーザーに技術的なエラーを表示しないことです ([情報の開示](https://learn.microsoft.com/en-us/dotnet/fundamentals/code-analysis/quality-rules/ca3004))。使用されているテクノロジー、プラグインのバージョン、さらには攻撃者がアプリケーションに対して使用できる追加情報であるスタックトレースが表示される可能性があります。 - -``` csharp -public void GetObject() -{ - try - { - object o = null; - o.ToString(); - } - catch (Exception e) - { - this.Response.Write(e.ToString()); - // よりよい方式 - // myLogger.Log(e.ToString()); // 例外のログ出力 - // this.Response.Write("例外が発生しました"); // 一般的なメッセージを返す - } - -} -``` - -以下の例はトリッキーであると同時にシンプルです。connectionString には資格情報を含めることができ、ハードコーディングしてはなりません。簡単に変更することができないというのもありますが、さらに重要なことは、ソースコードにアクセスする人なら誰でもシークレットにアクセスできることです。 - -``` csharp -private string connectionString = ""; -``` - ---- - -# Credits - -This workshop's challenges are a fork from the original Hackaton [accessible here](https://github.com/microsoft/CopilotHackathon). We just wanted to integrate it into the [MOAW](https://github.com/microsoft/moaw) format and add some exercises. - -Role Prompts described in the Prompt engineering section are inspired by this [great blog post](https://github.blog/2023-10-09-prompting-github-copilot-chat-to-become-your-personal-ai-assistant-for-accessibility/) from Github's engineers [Ed Summers](https://github.com/edsummersnc) and [Jesse Dugas](https://github.com/jadugas). - -A big thanks to them <3 diff --git a/workshops/github-copilot/workshop.md b/workshops/github-copilot/workshop.md deleted file mode 100644 index a91ced82..00000000 --- a/workshops/github-copilot/workshop.md +++ /dev/null @@ -1,1148 +0,0 @@ ---- -published: true -type: workshop -title: Product Hands-on Lab - GitHub Copilot, your new AI pair programmer -short_title: GitHub Copilot, your new AI pair programmer -description: Discover how to leverage GitHub Copilot to develop your project -level: beginner -authors: - - Philippe DIDIERGEORGES - - Louis-Guillaume MORAND -contacts: - - '@philess' - - '@lgmorand' -duration_minutes: 240 -tags: javascript, .net, GitHub, IA, copilot, AI, csu -banner_url: assets/banner.jpg -sections_title: - - Introduction - - Github Copilot - - Github Copilot Chat - - Advanced Prompt Engineering - - Challenge 1 - A NodeJS server - - Challenge 2 - A .Net Core API - - Challenge 3 - Infra as Code - - Solutions - - Credits ---- - -# Activate GitHub Copilot to become more efficient - -The goal of this workshop is to learn how to use GitHub Copilot, using an exercise that consists of building a web server using Nodejs with different functionalities and a .NET Web API. In the second part, you'll learn how to use it for infrastructure as code but also to fix bad practices in terms of security. - -GitHub Copilot is an AI-powered code assistant that helps developers write better code faster. It uses machine learning models trained on billions of lines of code to suggest whole lines or entire functions based on the context of what you’re working on. By using GitHub Copilot, you can learn how to write better code and improve your productivity. - -
- -> GitHub Copilot is a quickly evolving product and thus this workshop may not be 100% up to date with the differentes features of the different extensions you are going to use. Please be clever if it's not exactly the same. For info, this page has been updated in December 2023. - -
- -## Pre-requisites - -| | | -|----------------|-----------------| -| Node.js v16+ | [Download Node.js](https://nodejs.org) | -| .Net Core | [Download .Net Core](https://dotnet.microsoft.com/download) | -| GitHub account | [Create free GitHub account](https://github.com/join) | -| GitHub Copilot Access | A 60 day trial can be [requested here](https://github.com/github-copilot/signup) | -| A code editor | [Download VS Code](https://code.visualstudio.com/Download) | -| some VSCode extensions | The first one [GitHub Copilot](https://marketplace.visualstudio.com/items?itemName=GitHub.copilot), and the other one allows you to have [GitHub Copilot Chat](https://marketplace.visualstudio.com/items?itemName=GitHub.copilot-chat).| -| A browser | [Download Microsoft Edge](https://www.microsoft.com/edge) or any other one ;-)| - -
- -> You also have to download some assets. They can be [downloaded here](assets/src/exercisefiles.zip). - -
- -## Work with GitHub Codespaces - -The environment is already configured to work with [GitHub Codespaces](https://github.com/features/codespaces), you can find the configuration files in the *.devcontainer* folder. - -To start programming just start a new codespace and you are ready to go, don't need to install anything. - -## Work locally - -You can also choose to work locally on your computer. - -1. Install [Visual Studio Code](https://code.visualstudio.com/) -2. Install the [GitHub Copilot](https://marketplace.visualstudio.com/items?itemName=GitHub.copilot) extension -3. Install the [GitHub Copilot Chat](https://marketplace.visualstudio.com/items?itemName=GitHub.copilot-chat) extension -4. Install [Node and npm](https://docs.npmjs.com/downloading-and-installing-node-js-and-npm) -5. Install mocha - - ``` bash - npm install --global mocha - npm install axios - ``` - -7. Install [Docker](https://docs.docker.com/engine/install/) -8. Install [.NET Core](https://dotnet.microsoft.com/download) - ---- - -# First steps with Github Copilot - -This section will guide you through the first steps with GitHub Copilot. You will learn what you can do and how to use it at his full potential. If you already feel confortable with it you can jump to the first challenge with NodeJS. - -## Get ready - -This first challenges needs you to clone the following GitHub Repository: [Github Copilot Demo](https://github.com/Philess/gh-copilot-demo) - -This repository is a code starter that will help you experiment all capabilities with GitHub Copilot. Take the time to look at the architecture design displayed on the page and when you're ready, clone the repository from the command line and open it in VS Code. - -``` bash -git clone https://github.com/Philess/gh-copilot-demo -cd gh-copilot-demo -code . -``` - -## Start playing with GitHub Copilot - -Once you start typing a prompt and copilot generate proposals, you can use the following shortcuts to interact with Copilot: -
    -
  • `tab` to accept the current suggestion entirely (`most common`)
    • -
  • `ctrl + right arrow` to accept word by word the suggestion (`for partial use`)
    • -
  • `alt + ^` to move to next suggestion
    • -
  • `shift + tab` to go back to the previous suggestion
    • -
  • `ctrl+enter` to display the copilot pane
    • -
- -
- -> If you can't remember it, just hover your pointer on top of a suggestion to make them appear. - -
- -## Natural Language Translations - -**Automate text completion** - -Open file `album-viewer/lang/translations.json` - -```json -[ - { - "language": "en", - "values": { - "main-title": "Welcome to the world of the future", - "main-subtitle": "The future is now with copilot", - "main-button": "Get started" - } - } -] -``` - -- Start adding a new block by adding a "," after the last "}" and press enter - -
- -## Code Generation - -**What is a prompt?** -In the context of Copilot, a prompt is a piece of natural language description that is used to generate code suggestions. It's the input that Copilot uses to generate code. It can be a single line or a multiple lines description. - -**Generate code from prompt** - -Create a new `album-viewer/utils/validators.ts` file and start with the prompt: - -```ts -// validate date from text input in french format and convert it to a date object -``` - -Copilot can help you also to write `RegExp patterns`. Try these: - -```ts -// function that validates the format of a GUID string - -// function that validates the format of a IPV6 address string -``` - -
- -**Discover new tool and library on the job with Copilot** - -Still on the same `album-viewer/utils/validators.ts` file add the following prompt: - -```ts -// validate phone number from text input and extract the country code -``` - -
- -> For this one it will probably give you proposal that call some methods not defined here and needed to be defined. It's a good opportunity to explore the alternatives using the `ctrl+enter` shortcut to display the copilot pane. -
You can choose one that uses something that looks like coming for an external library and use copilot to import it showing that the tool helps you discover new things. - -
- -**Complex algoritms generation** - -In the `albums-api/Controllers/AlbumController.cs` file try to complete the `GetByID` method by replace the current return: - -```cs -// GET api//5 -[HttpGet("{id}")] -public IActionResult Get(int id) -{ - //here -} -``` - -In the same file you can show other prompts like: - -```cs -// function that search album by name, artist or genre - -// function that sort albums by name, artist or genre -``` - -## Big tasks vs small tasks - -### Big Prompts and Short Prompts - -Copilot will probably will always more effective with prompt to generate small but precisely described pieces of code rather than a whole class with a unique multiple lines prompt. - -
- -> The best strategy to generate big piece of code, is starting by the basic shell of your code with a simple prompt and then adding small pieces one by one. - -
- -**Big prompts that *could* works** - -- Back in the `albums-viewer/utils` add a new file `viz.ts` to create a function that generates a graphe. Here is a sample of prompt to do that: - -```ts -// generate a plot with D3.js of the selling price of the album by year -// x-axis are the month series and y-axis show the numbers of album selled -// data from the sales of album are loaded in from an external source and are in json format -``` - -
- ->Copilot will probably try to complete the prompt by adding more details. You can try to add more details yourself or follow copilot's suggestions. When you want it to stop and start generating the code just jump on another line and let the copilot do its work. - -
- -- Once you achieved to generate the code for the chart you probably see that your IDE warn you about the d3 object that is unknown. For that also Copilot helps. -Return on top of the file and start typing `import d3` to let copilot autocomplete - -```ts -import d3 from "d3"; -``` - -Look at what Copilot has been able to generate. It's possible that the code is working fine and does everything you asked for but also you probably hit the token limit and Copilot was not able to generate the whole code. - -It's because Copilot for autocompletion is not made for creating big pieces of code at once, but is more specialized in generating small pieces step by step. - -**Try again by build it step by step** - -Try to generate the code for the plot by cutting it into small pieces following the steps below: - -```ts -import * as d3 from 'd3'; - -// load the data from a json file and create the d3 svg in the then function -``` - -Inside the then function, starts by setting up the basics of the plot - -```ts -// create the svg -``` - -```ts -// create the scales for the x and y axis -// x-axis are the month series and y-axis show the numbers of album selled -``` - -```ts -// create axes for the x and y axis -``` - -From there you can just ask to copilot to complete the chart - -```ts -// generate a line chart based on the albums sales data -``` - -
- ->You will **always** get better results by cutting big task into small chunks with copilot autocomplete. It's also a good way to show that copilot is not magic and you have to use it with your other IDE feature and your developer logic. - -
- -## Tests - -Copilot can help generate all kind of tests that are written with code. It Includes `unit tests, integration tests, end to end tests, and load testing` tests with JMeter scripts for example. - -- Add a new file `validators.test.ts` in the `albums-viewer/tests` folder - -- To have good test suggestion, you hould provide some basic informations to Copilot such as the test framework you want to use: - -```ts -import { describe } -``` - -When you start typing the `describe` function, copilot will see you're in test file in TS and suggest you to import the `describe` and `it` functions from Mochai which is a famous test framework for JS/TS. -Accept the suggestion and it will automatically suggest also the `expect` function from Chai: accept it also. - -```ts -import {describe, it} from 'mocha'; -import {expect} from 'chai'; -``` - -You have your test framework in place! Now just import the functions you want to test by starting a new line by `import` keyword copilot will see you are in a test file, to test some `validators` because of the name and it will suggest something like that: - -```ts -import {validateAlbumId} from '../src/validators'; -``` - -It looks ok but because Copilot doesn't have access to all your code, only the open tab and limited informations, you can see that both the path and the function name are wrong. -
-... At least he tried ... -
-but it's a good way to show that Copilot is not magic and you have to use it with your other IDE feature and your brain :) - -- Accept the suggestion and change the path. You will be able to have VS Code to give you the available function with the `ctrl+space` shortcut. - -- Add a comment with the first function you want to test and let the magic happen: - -```ts -import {describe, it} from 'mocha'; -import {expect} from 'chai'; - -import {validateDate, validateIPV6} from '../utils/validators'; - -// test the validataDate function -``` - -Boom! - -```ts -describe('validateDate', () => { - it('should return a date object when given a valid date string', () => { - const date = '01/01/2019'; - const expectedDate = new Date(2019, 0, 1); - expect(validateDate(date)).to.deep.equal(expectedDate); - }); - - it('should throw an error when given an invalid date string', () => { - const date = '01/01/2019'; - expect(() => validateDate(date)).to.throw(); - }); -}); -``` - -*You can add other `it` block to add more test cases and also add the tests for the other functions. For example try add a new `it` block for the validateDate function to test that it throws and error when given en empty string.* - -## Writing CI pipelines - -*Copilot will help you in writing your pipeline definition files to generate the code for the different steps and tasks. Here are some examples of what it can do:* - -- *generate a pipeline definition file `from scratch`* -- *accelerate the writing of a pipeline definition file by `generating the code` for the different `steps, tasks and pieces of script`* -- *help `discover marketplace tasks and extensions` that match your need* - -### Step 1: generate from scratch - -- Create a new file `pipeline.yml` in the `.github/workflows` folder of the project and start typing the following prompt: - -```yml -# Github Action pipeline that runs on push to main branch -# Docker build and push the album-api image to ACR -``` - -*Copilot will generate the pipeline block by block. Generation pipelines Yaml, you will sometimes need to jump to a new line to trigger the generation of the next block more often than with other type of code.* - -*It will often generate a task with a few errores coming from bad indentation or missing quote around a task name. You can easily fix these with your IDE and your developer skills :)* - -### Step 2: add tasks from prompts - -- You probably have a github action workflow with at least a "login" task to your container registry and a "docker build and deploy" task. Add a new comment after those tasks to tag the docker image with the github run id and push it to the registry: - -```yml -# tag the image with the github run id and push to docker hub -``` - -you can play with other prompts like: - -```yml -# run tests on the album-api image - -# deploy the album-api image to the dev AKS cluster -``` - -### Step 3: add scripts from prompts - -- Copilot is also very usefull when you need to write custom script like the following example: - -```yml -# find and replace the %%VERSION%% by the github action run id in every appmanifest.yml file -``` - -## Infra As Code - -Copilot can also help you write Infrastructure as code. It can generate code for `Terraform, ARM, Bicep, Pulumi, etc...` and also `Kubernetes manifest files`. - -### Bicep - -Open the `main.bicep`file in `iac/bicep` folder and start typing prompts at the end of the file to add new resources: - -```js -// Container Registry - -// Azure Cognitive Services Custom Vision resource -``` - -### Terraform - -Open the `app.tf`file in `iac/terraform` folder and start typing prompts at the end of the file to add new resources: - -```yml -# Container Registry - -# Azure Cognitive Services Custom Vision resource -``` - -## Generate Git Commit comment - -Yes, writing a comment should be mandatory and developers tend to be lazy. GitHub Copilot can help with that. - -1. Just edit any file by adding some relevant content into it. - -2. On the Git commit panel, click the small magical button on the right - - ![GitHub Copilot Git comment generator](assets/git-commit.png) - -3. Admire Copilot having generated a comment for you - - ![Generated comment(assets/git-commit2.png) - -## Writing documentation - -Copilot can understand a natural language prompt and generate code and because it's just language to it, it can also `understand code and explain it in natural language` to help you document your code. -So it can help you in all your documentation tasks. It can generate simple documentation comment or standardized documentation comment like JavaDoc, JsDoc, etc... it can also help you translate your documentation in different languages. Let's see how it works. - -### simple documentation comment - -To see that just put you pointer on top of a Class, a method or any line of code and start typing the comment handler for the selected language to trigger copilot. In language like Java, C# or TS for example, just type `// `and let the magic happen. - -Here is an example in the `albums-viewer/routes/index.js` file. Insert a line and start typing on line 13 inside the `try block` - -```js -router.get("/", async function (req, res, next) { - try { - // Invoke the album-api via Dapr - const url = `http://127.0.0.1:${DaprHttpPort}/v1.0/invoke/${AlbumService}/method/albums`; - -``` - -Continue to play with it and see what happens on other pieces of code. - -### standardized documentation comment (JavaDoc, JsDoc, etc...) - -For this one, to trigger the documentation comment generation, you need to respect the specific comment format: - -- `/**` (for JS/TS) in the `index.js` file for example -- `///` for C# in the `AlbumController.cs` of the AlbumApi file for example - -```cs -/// -/// function that returns a single album by id -/// -/// -/// -[HttpGet("{id}")] -public IActionResult Get(int id) -``` - -### Writing markdown and html documentation - -Copilot is also very powerfull to help you write documentation. It can generate `markdown` and `html` code and accelerate the writing of your readme.md files like for this one for example. - -You can show that by creating a new file `demo.md` in the root of the project and start typing the following prompt: - -```md -# Github Copilot documentation -This documentation is generated with Github Copilot to show what the tool can do. - -## -``` - -From there by starting a new line with a secondary level title it will start generating the content of the documentation and it will showcase how it will accelerate the documentation writing process. - ---- - -# Use Copilot Chat to improve code quality - -GitHub Copilot is a generative AI and thus, perfect to generate code, but it has powerfull analysis capabilities on your code that can be used in several case to improve code quality like: find security issues, bad practices in your code and générate a fix, refactor and add comment to legacy code, generate tests, etc... - -If you already feel confortable with it you can jump to the next section. - -## Let's Start - -To start using Github Copilot Chat, you first need to: - -- Have a valid GitHub Copilot license (personal, business or enterprise). -- Install the extension in your IDE. For VS Code, you can find it directly by searching for `Github Copilot Chat` in the extensions tab. - -### Clone the repository - -We will use the same repository as the previous section to show how to use Copilot Chat to improve code quality. If you already have it, you can skip this step. - -You need to clone the following GitHub Repository: [Github Copilot Demo](https://github.com/Philess/gh-copilot-demo) - -This repository is a code starter that will help you experiment all capabilities with GitHub Copilot. Take the time to look at the architecture design displayed on the page and when you're ready, clone the repository from the command line and open it in VS Code. - -``` bash -git clone https://github.com/Philess/gh-copilot-demo -cd gh-copilot-demo -code . -``` - -## Start playing with the Chat - -Once Copilot Chat is setup, you can start using it: - -- by accessing the **chat view** from the left toolbar of your IDE (chat icon) -- by pressing `Ctrl` + `Shift` + `i` shortcut for a quick **inline question** to the chat - -The first one is a sticky version, very usefull to keep the chat open and ask questions to copilot. -The second one is a quick way to ask a question and get an answer and launch commands. - -### Chat View - -The chat view gives you a full chat experience, integrate as any other tool view in your IDE. Once the view is open you can start chatting with Copilot as your personnal code coach. It keeps the history of the conversation and you can ask question related to the previoius answers. It also provides suggestions for questions along the way. You can: - -- ask general question about coding on any language or best practice -- ask to generate or fix code related to the current file and inject the code directly in the file - -It's a more high level copilot than the vanilla copilot which is specialized on providing code completion. - -Try it with a few questions like: - -```text -> How to generate a random number in C#? -> What is the best way to secure a route is ASP.NET Core? -> What is the easiest way to generate a static website with NodeJS? -``` - -Try it then with some of your code files in the repository. Open a file a try asking: - -```text -> Can you explain me what this code does? -> (with only part of the code selected) Can you explain me what the selected code does? -> Can you generate a function that returns a random number between 1 and 10? -> Can you add documentation commentes to this function? -``` - -Try also using the questions suggestions that appears along the way. - -### Inline question - -The inline question is a quick way to ask a question to Copilot and get an answer. It's a good way to ask a question about a specific piece of code. It's also a good way to launch commands to Copilot. You can ask it to generate code, fix code, generate tests, etc... - -try it by pressing `Ctrl` + `Shift` + `i` and type the same type of commands you tried in the chat view. - -### Slash Commands - -To further help Copilot give you more relevant answers, you can choose a topic for your questions through "slash commands." - -You can prepend your chat inputs with a specific topic name to help Copilot give you a more relevant response. When you start typing /, you’ll see the list of possible topics: - -- **/explain**: Explain step-by-step how the selected code works. -- **/fix**: Propose a fix for the bugs in the selected code. -- **/help**: Prints general help about GitHub Copilot. -- **/tests**: Generate unit tests for the selected code. -- **/vscode**: Questions about VS Code commands and settings. -- **/clear**: Clear the session. - -## Secure your code - -Copilot can help you find security issues in your code and fix them. It can also help you find bad practices in your code and fix them. Let's see how it works. - -Open the `album-api/Controllers/UnsecuredController.cs` file and type questions like these to the chat: - -```text -> Can you check this code for security issues? -> Do you see any quality improvement to do on this code? -``` - -Once you have the answer, you can ask to fix the issues by typing: - -```text -> Can you propose a fix? -``` - -When you have the fix in the code you choose to **copy it or inject it directy in the file** by hovering the code block in the chat and selecting the right option on the top left. - -## Code Explanation and documentation - -You can use Copilot Chat to explain code to you. It can `explain you the code in natural language or generate documentation comments for you`. Let's try that with the following commands: - -```test -> /explain -> Generate documentation comments for this code -``` - -## Code Refactoring - -More impressive, Copilot chat can help you refactor your code. It can help you `rename variables, extract methods, extract classes, etc...`. - -You can try some of these commands on the `album-api/Controllers/UnsecuredController.cs` file: - -```test -> extract methods -> create Async version of each methods when it makes sense -``` - -## Code Translation - -*Copilot can understand and generate natural languages and code language in both way so by combining everything you can use it to `translate code pieces from a language to another one`* - -To translate a piece of code in a specific language, open it and ask to the chat to translate it to another language. For example open the `validators.ts` file created in the first section dedicated to Copilot autocompletion and ask to translate it to C for example. - -In case of dealing with Legacy code like COBOL for example it can be very useful. Open the `legacy/albums.cbl` file and try translating the code to Python. - -## Tests generation - -Copilot can also help you generate tests for your code. It can generate `unit tests, integration tests, end to end tests, and load testing` tests with JMeter scripts for example. - -Open the `album-api/Controllers/UnsecuredController.cs` file and type questions like these to the chat: - -```test -> Generate a unit tests class for this code -``` - -You can also use copilot to help you generate Stubs and Mocks for your tests. - -```text -> Generate a mock for FileStream class -> Use that mock in the unit tests -``` - -
- -> Remember that Copilot chat is keeping track of the previous Q & A in the conversation, that's why you can reference the previously generated mock and test easily. - -
- -## Use Chat participants - -Chat participants are like specialized experts who can assist you with specific tasks. You can mention them in the chat using the @ symbol. Currently, there are three Chat participants available for Visual Studio Code: - -- **@workspace**: This chat participant has knowledge about the code in your workspace and can help you navigate it by finding relevant files or classes. The @workspace chat participant uses a meta prompt to determine what information to collect from the workspace to help answer your question. -- **@vscode**: This chat participant is knowledgeable about commands and features in the VS Code editor itself, and can assist you in using them. -- **@terminal**: This chat participant has context about the Visual Studio Code terminal shell and its contents. - -They may not be super rich for the moment but their features will continue to grow over the time. Here are some example - -Open the side Chat panel and type **@workspace /New* to specify that you want to create a new project. For instance, try to create an Asp.Net project - -```text -> @workspace /new create a new asp.net core 6.0 project, with three views Index, Users and products. -``` - -It should create a structured project and even a new button to create the file. Click on "Create workspace" to see files being created. - -![GitHub Copilot Chat Participants](assets/agents.png) - -## Tips - -GitHub Copilot Chat is very handful but for a developer, leaving the keyboard and having to take the mouse to open the Chat tab can be boring. You can directly call the Chat inside the code editor. - -1- Open any file containing code - -2- Use the shortcut **Ctrl + i**. It should open the Quick chat popup, a small chat windows where you put your cursor - -![GitHub Copilot Quick Chat](assets/quickchat.png) - -3- Type any command to generate some code (i.e. "Create a C# class named Toto). The generated code is injected inside the current file which may be what you want - ---- - -# Prompt engineering in Copilot Chat - -In the previous section you discovered how to use basic prompts to get code from Copilot Chat. In this section you will learn techniques to get more accurate results using prompt engineering techniques. - -**What is prompt engineering?** -Prompt engineering is the process of designing high quality prompts to generate high quality code suggestions. There are good practices and tips to write better prompts. Let's see some of them. - -## Provide examples: one-shot and few-shots programming - -Talking about prompt engineering, you can also use the chat to provide examples to Copilot. It's a good way to help Copilot understand what you want to do and generate better code. You can provide examples in the chat by typing with the validator.ts file open: - -```bash -# one-shot programming - -Write me unit tests for phone number validators methods using mocha and chai in the current file. -Use the following examples for positive test (test that should return true): -it('should return true if the phone number is a valid international number', () => { expect(validatePhoneNumber('+33606060606')).to.be.true; }); -Organize test in logic suites and generate at least 4 positives tests and 2 negatives tests for each method. -``` - -```bash -# few-shot programming - -Write me unit tests for all validators methods using mocha and chai in the current file. -Use the following examples for positive test (test that should return true): -it('should return true if the phone number is a valid international number', () => { expect(validatePhoneNumber('+33606060606')).to.be.true; }); -it('should return true if the phone number is a valid local american number', () => { expect(validatePhoneNumber('202-939-9889')).to.be.true; }); -it('should throw an error if the given phone number is empty', () => { expect(validatePhoneNumber('')).to.throw(); }); -Organize test in logic suites and generate at least 4 positives tests and 2 negatives tests for each method. -``` - -You can use this technique to **generate code that keeps the styling code from another file**. For example if you want to create sample records for music style like the Albums in albums-api>Models>Album.cs file, open it and type: - -```bash -Write a MusicStyle record that conatins a List with 6 sample values like in the Album.cs file. -``` - -## Provide external references - -The chat copilot can use external references to build more accurate suggestions. For exemple if you want to generate a code that make a request to an API you can provide an example of the API response in the chat or the url to the API reference. Copilot will use it to generate better code. - -```bash -Write a TS function that retreiev all dog breeds from the following API and return an array of Breed objects Request: HTTP GET https://dog.ceo/api/breeds/list/all -``` - -Copilot will use the given external reference to generate the code. You will see that he wil generate the Breef interface (or class) with a subBreeds property. It's coming from the API given by the external reference. - -```ts -interface Breed { - name: string; - subBreeds: string[]; -} -``` - -
- -> You can also provide links to external documentations like SDK, libraries, etc... or event normative documents like RFCs, etc... - -
- -## Role Prompting - -Also called foundational prompt, it's a general prompt you're giving to Copilot Chat to personnalise his behavior and setup your flavour of Copilot. - -This is probably the first thing to do when you start a new task with Copilot Chat: **provide a clear description of what you want to build and how do you want copilot to help you**. - -
- -> **This is very powerfull when handled properly** so be sure to start every coding sessions with a role prompt and save your best prompt for future use. - -
- -***Structure of a role prompt*** - -What can you include in a role prompt: - -- Provide solid context and background information on what you want to build. -- Define GitHub Copilot’s role and setting expectations about what feedback we are looking for. -- Be specific in the quality of answers and ask for reference and additional resources to learn more and ensure the answers you receive are correct -- Resume the task and ask if the instructions are clear - -***Example of a role prompt*** - -Start a new conversation and type the following prompt: - -```bash -I'm working on a new mobile application that is built on React Native. -I need to build a new feature that will allow the user to upload a picture of a dog and get the breed of the dog. -I will need to use the following set of APIs to work on the breeds: https://dog.ceo/api/breeds. I need to be sure that my code is secured againt at least the OWASP Top 10 treats (https://owasp.org/Top10/). -I need to have unit tests for the code and i want my application to be fully accessible and conforms with the WCAG 2.1 level A and AA success criteria defined at https://www.w3.org/TR/WCAG21/. -I need you to act as my own code coach to ensure that my code fits all these requirements. -When possible, please provide links and references for additional learning. -Do you understand these instructions? -``` - -From there you can start asking questions and from time to time, ensure Copilot still follows the instructions by asking: - -```bash -Are you still using the instructions I provided? -``` - -***Test your role prompt*** - -You can test your role prompt by asking questions about best practices for accessibility on React Native Apps and OWASP Top 10 treats. You can also ask to generate code for the upload feature and check if the generated code is secured and accessible. - -Try these questions for example: - -```bash -how can i make my app accessible with react native? - -what is the most secure way to upload a photo from my app? -``` - ---- - -# Develop a NodeJS server - -In this first exercise, you are going to develop a real project following functional requirements. You can do it by yourself or...with the help of GitHub Copilot. - -## Instructions - -- Download to local the [exercicefile](assets/src/exercisefiles.zip) folder -- Open `nodeserver.js` and begin by writing a Nodejs server, check the first suggestions based on the initial text -- Open `test.js` file and analyze the current test -- Open a command prompt and run the test (`mocha test.js`) -- See the result, it should display something like: - -``` bash -mocha test.js -server is listening on port 3000 - - Node Server - - √ should return "key not passed" if key is not passed - - 1 passing (34ms) - -``` - -- In the `nodeserver.js` develop the rest of the methods described in the Exercise described in the section below - -> Do not forget to open `color.json` file in Visual Studio Code, so GitHub Copilot get all the context to make better recommendations - -- In the test.js file add the methods to test the functionality -- Run the tests to verify that all is working -- Open the `dockerfile` file, and fill it, to create a docker container with a node image that can run the web server -- Create a command to run docker in port 4000 -- Test that the application is working in port 4000 -- In the **nodeserver.js** file, you can type a new line like `//run a curl command to test the server` so we can see how GitHub Copilot based on the current file produces a curl command, to be executed in the command line. -- Note: you can be more specific like `//run a curl command to test the daysBetweenDates` method. It should generate a test for a specific method - -## Exercise - -You must now develop and add new features to your server. The requests that the server must attend are the following: - -
- -> As you type GitHub Copilot will make suggestions, you can accept them by pressing Tab. If nothing shows up after GitHub Copilot write some lines, press enter and wait a couple of seconds. On Windows or Linux, press Ctrl + Enter. - -
- -
- -> There are a lot of code to write but you may be surprised by the time it will take to you to complete it. You can also only write 7 or 8 of them if you want, the exercise is not meant to be boring. - -
- -| Method | Requirements| -|---|---| -|**/Get**|Return a hello world message| -|**/DaysBetweenDates**|Calculate days between two dates.
Receive by query string 2 parameters date1 and date 2 , and calcualte the days that are between those two dates.| -|**/Validatephonenumber**|Receive by querystring a parameter called `phoneNumber`.
Validate `phoneNumber` with Spanish format, for example +34666777888
if `phoneNumber` is valid return "valid"
if `phoneNumber` is not valid return "invalid"| -|**/ValidateSpanishDNI**|Receive by querystring a parameter called `dni`. calculate DNI letter
if DNI is valid return "valid"
if DNI is not valid return "invalid"
We will create automated tests to check that the functionality is correctly implemented.
When the development is completed, we will build a container using Docker| -|**/ReturnColorCode**|Receive by querystring a parameter called `color`
read `colors.json` file and return the `rgba` field
get color var from querystring
iterate for each color in colors.json to find the color
return the code.hex field| -|**/TellMeAJoke**|Make a call to the joke api and return a random joke using axios| -|**/MoviesByDirector**|(this will require to browse to [https://www.omdbapi.com/apikey.aspx](https://www.omdbapi.com/apikey.aspx) and request a FREE API Key)
Receive by querystring a parameter called director
Make a call to the movie api and return a list of movies of that director using axios
Return the full list of movies| -|**/ParseUrl**|Retrieves a parameter from querystring called someurl
Parse the url and return the protocol, host, port, path, querystring and hash
Return the parsed host| -|**/ListFiles**|Get the current directory
Get the list of files in the current directory
Return the list of files| -|**/GetFullTextFile**|Read `sample.txt` and return lines that contains the word "Fusce". (becareful with this implementation, since this normally reads the full content of the file before analizing it, so memory usage is high and may fail when files are too big)| -|**/GetLineByLinefromtTextFile**|Read `sample.txt` line by line
Create a promise to read the file line by line, and return a list of lines that contains the word "Fusce"
Return the list of lines| -|**/CalculateMemoryConsumption**|Return the memory consumption of the process in GB, rounded to 2 decimals| -|**/MakeZipFile**|Using zlib create a zip file called `sample.gz` that contains `sample.txt`| -|**/RandomEuropeanCountry**|Make an array of european countries and its ISO codes
Return a random country from the array
Return the country and its ISO code| - -## GitHub Copilot Chat exercises - -These tasks can be performed with the [GitHub Copilot Chat](https://marketplace.visualstudio.com/items?itemName=GitHub.copilot-chat) add-in. - -- **Explain** - -Select the line that has the regex in the validatePhoneNumber method, and use `/explain` comamand. You will see an explanation detailing what each different notation does in the regular expression. - -- **Language translation** - -Select some source code, like this line: - -``` js -var randomCountry = countries[Math.floor(Math.random() * countries.length)]; -``` - -Ask to the chat to translate it to another language, for example Python. You should see new code generated in **python** - -- **Readable** - -Select the content of MakeZipFile - -Ask to the chat to make it more readable. See how comments are added and also variables that have short names are renamed to more understandable names. - -- **Fix Bug** - -In the exercise, there should be no bugs, since most of the code will be done by GitHub Copilot. We can force some errors and then test the debug functionality. - -Force some errors like: - -In a for loop change the beginning to (change the 0 for a 1): - -``` js - for (var i = 1 -``` - -select the text and ask to the chat to fix the bug. - -- **Make robust** - -Select some text that comes from input, for example, variables that come from query string: - -``` js - var queryData = url.parse(req.url, true).query; - var date1 = queryData.date1; - var date2 = queryData.date2; -``` - -Ask to the chat to make it robust, and you will see that additional validation is added. - -- **Document** - -Select some line (e.g. a method or the beginning of the if clause) - -``` js - else if (req.url.startsWith('/GetFullTextFile')) -``` - -Ask to the chat to document it. You will see that Copilot chat will explain what the code does and add comments to the code. - ---- - -# .Net Core - -The goal is to create a simple WebAPI using .NET 6.0 and Docker with the help of GitHub Copilot. -Follow the instructions below and try to use GitHub Copilot as much as possible. -Try different things and see what GitHub Copilot can do for you, like generating a Dockerfile or a class, adding comments, etc. - -Remember: - -Make sure GitHub Copilot is configured and enabled for the current language, just check the status bar on the bottom right corner of VS Code. - -## Create dotnet WebAPI project - -- Create a new NET project using - -``` powershell -dotnet new webapi -``` - -- Create a new file `User.cs` in the Models folder, and instruct Copilot to generate a class for you. - -- Add a new file `UserController.cs` in the Controllers folder that inherits from ControllerBase, and instruct Copilot to generate a controller for you. - -- Add `ApiController` and Route attributes to the class. - -- Add a new file `IUserService` in the Abstractions folder, and instruct Copilot to generate an interface for you. - -- Run the app using (if you are working with GitHub Codespaces you may need to remove HTTPS redirection from `Program.cs` ) - -``` powershell -dotnet run -``` - -- Implement the interface IUserService in `UserService.cs` in the Services folder and add some comments so GitHub Copilot be able to generate the implementation for you. - -- Instruct Copilot to generate a List for Users and the Add and Get Methods using the list. - -- Go to `Program.cs` a inject the `IUserService` before building the app. - -``` csharp -builder.Services.AddSingleton(); -``` - -- Run the app using - -``` powershell -dotnet run -``` - -> If you run into and "No server certificate was specified..." error, run the following command - -``` powershell -dotnet dev-certs https -``` - -- Forward port if needed - -- Navigate to your address /swagger. Example: [https://leomicheloni-supreme-space-invention-q74pg569452ggq-5164.preview.app.github.dev/swagger/index.html](https://leomicheloni-supreme-space-invention-q74pg569452ggq-5164.preview.app.github.dev/swagger/index.html) - -## Put the application into a Docker container - -- Publish the app and put it in a folder called _publish_ - -``` dotnet -dotnet publish -c Release -o publish -``` - -- Using the existing `Dockerfile`, put the app in a container and run it (add more instructions or start to write code and let GitHub Copilot complete it for you) -- Build the image and run the app on port 8080 - -``` powershell -docker build -t dotnetapp . -docker run -d -p 8080:80 --name dotnetapp dotnetapp -``` - ---- - -# Infrastructure as Code - -Generating code from a programming language is one thing, but can GitHub Copilot help to generate configurations such as Terraform, Bicep, etc? - -For this exercise, you want to deploy your previously developed Web application and want to host it on Azure. Here are the requirements: - -- The application will be hosted on an Azure Web App name `my-webapp` -- The App Service Plan (CPU & Memory) is named `my-plan` and is using the SKU (size) `B1` -- The Web app is hosted in West Europe in a resource group name `oh-yeah` - -
- -> There are several ways of using GitHub Copilot for that. For instance, you can write several consecutive lines of comments before letting GitHub Copilot giving recommandations. Furthermore, if the result is not conclusive, you can open a side panel to generate 10 alternatives suggestions. To do so, click `ctrl` + `Enter` - -
- ---- - -# Solutions - -Here you can find the solution to the different exercises. - -## Coding - -The solution of the coding exercise can be [downloaded here](assets/src/completesolution.zip) - -## Infrastructure As Code - -This part is the easiest one but GitHub Copilot can randomly generate bad or commented code. - -``` bicep -param webAppName string = 'my-webapp' -param webAppPlanName string = 'my-plan' -param webAppPlanSku string = 'B1' -param webAppPlanLocation string = 'westeurope' -param resourceGroupName string = 'oh-yeah' - -resource appServicePlan 'Microsoft.Web/serverfarms@2021-02-01' = { - name: webAppPlanName - location: webAppPlanLocation - kind: 'app' - sku: { - name: webAppPlanSku - tier: 'Basic' - size: 'B1' - } -} - -resource webApp 'Microsoft.Web/sites@2021-02-01' = { - name: webAppName - location: webAppPlanLocation - kind: 'app' - properties: { - serverFarmId: appServicePlan.id - } -} - -resource resourceGroup 'Microsoft.Resources/resourceGroups@2021-04-01' = { - name: resourceGroupName - location: webAppPlanLocation -} -``` - -## DevSecOps - -GitHub Copilot may not be able to fix and refactor all the code (for instance using the `fix bug` prompt) but it is pretty good to recognize code smells and bad practices if you ask through the chat. - -Several security flaws are present in this short piece of code. You should have at least found 4 major bad practices: - -This code seems innocent but it could allow [Path injection](https://learn.microsoft.com/en-us/dotnet/fundamentals/code-analysis/quality-rules/ca3003). It means someone can try to access another file on the disk. - -``` csharp -using (FileStream fs = File.Open(userInput, FileMode.Open)) - { - // If possible, limit file paths based on user input to an explicitly known safe list. For example, if your application only needs to access "red.txt", "green.txt", or "blue.txt", only allow those values. - // Check for untrusted filenames and validate that the name is well formed. - // Use full path names when specifying paths. - // Avoid potentially dangerous constructs such as path environment variables. - // Only accept long filenames and validate long name if user submits short names. - // Restrict end user input to valid characters. - // Reject names where MAX_PATH length is exceeded. - // Handle filenames literally, without interpretation. - // Determine if the filename represents a file or a device. - - byte[] b = new byte[1024]; - UTF8Encoding temp = new UTF8Encoding(true); - - while (fs.Read(b, 0, b.Length) > 0) - { - return temp.GetString(b); - } - } - - return null; -} -``` - -This one is a famous example of [SQL Injection](https://learn.microsoft.com/en-us/dotnet/fundamentals/code-analysis/quality-rules/ca3001). The best thing to do is to use a parameter that will handle any attempt of escape code or wrong parameter boxing (type). - -``` csharp -public int GetProduct(string productName) -{ - using (SqlConnection connection = new SqlConnection(connectionString)) - { - SqlCommand sqlCommand = new SqlCommand() - { - CommandText = "SELECT ProductId FROM Products WHERE ProductName = '" + productName + "'", - CommandType = CommandType.Text, - }; - - // The secure way - // SqlCommand sqlCommand = new SqlCommand() - // { - // CommandText = "SELECT ProductId FROM Products WHERE ProductName = @productName", - // CommandType = CommandType.Text, - // }; - // sqlCommand.Parameters.AddWithValue("@productName", productName); - - SqlDataReader reader = sqlCommand.ExecuteReader(); - return reader.GetInt32(0); - } - -} -``` - -A general good practice is to never display ([Disclosing information](https://learn.microsoft.com/en-us/dotnet/fundamentals/code-analysis/quality-rules/ca3004)) any technical error to the end-user. It can show the used technology, the version of a plugin or even the stack trace which are additional information an attacker could use against your application. - -``` csharp -public void GetObject() -{ - try - { - object o = null; - o.ToString(); - } - catch (Exception e) - { - this.Response.Write(e.ToString()); - // better to do - // myLogger.Log(e.ToString()); // or any way to log the exception information - // this.Response.Write("An exception has occured"); // return a generic message - } - -} -``` - -This one is tricky and simple at the same time. The connectionString may contain credentials and must NEVER be hard coded. You can't change it easily but more important, anyone accessing the source code can get access to the secret. - -``` csharp -private string connectionString = ""; -``` - ---- - -# Credits - -This workshop's challenges are a fork from the original Hackaton [accessible here](https://github.com/microsoft/CopilotHackathon). We just wanted to integrate it into the [MOAW](https://github.com/microsoft/moaw) format and add some exercises. - -Role Prompts described in the Prompt engineering section are inspired by this [great blog post](https://github.blog/2023-10-09-prompting-github-copilot-chat-to-become-your-personal-ai-assistant-for-accessibility/) from Github's engineers [Ed Summers](https://github.com/edsummersnc) and [Jesse Dugas](https://github.com/jadugas). - -A big thanks to them <3 diff --git a/workshops/iot-hub-mxchip/assets/60516369-e7ab9d00-9cdd-11e9-905d-3808b1ad7e4e.png b/workshops/iot-hub-mxchip/assets/60516369-e7ab9d00-9cdd-11e9-905d-3808b1ad7e4e.png deleted file mode 100644 index 4b7b8465..00000000 Binary files a/workshops/iot-hub-mxchip/assets/60516369-e7ab9d00-9cdd-11e9-905d-3808b1ad7e4e.png and /dev/null differ diff --git a/workshops/iot-hub-mxchip/assets/connect-ssid.png b/workshops/iot-hub-mxchip/assets/connect-ssid.png deleted file mode 100644 index b4bbf6ff..00000000 Binary files a/workshops/iot-hub-mxchip/assets/connect-ssid.png and /dev/null differ diff --git a/workshops/iot-hub-mxchip/assets/creation-function.gif b/workshops/iot-hub-mxchip/assets/creation-function.gif deleted file mode 100644 index d1c4b05f..00000000 Binary files a/workshops/iot-hub-mxchip/assets/creation-function.gif and /dev/null differ diff --git a/workshops/iot-hub-mxchip/assets/creation-iotdevice.gif b/workshops/iot-hub-mxchip/assets/creation-iotdevice.gif deleted file mode 100644 index 681b1a8b..00000000 Binary files a/workshops/iot-hub-mxchip/assets/creation-iotdevice.gif and /dev/null differ diff --git a/workshops/iot-hub-mxchip/assets/creation-iothub.gif b/workshops/iot-hub-mxchip/assets/creation-iothub.gif deleted file mode 100644 index 1ff5b37d..00000000 Binary files a/workshops/iot-hub-mxchip/assets/creation-iothub.gif and /dev/null differ diff --git a/workshops/iot-hub-mxchip/assets/creation-projetIoT.gif b/workshops/iot-hub-mxchip/assets/creation-projetIoT.gif deleted file mode 100644 index 9245c130..00000000 Binary files a/workshops/iot-hub-mxchip/assets/creation-projetIoT.gif and /dev/null differ diff --git a/workshops/iot-hub-mxchip/assets/creation-rg.gif b/workshops/iot-hub-mxchip/assets/creation-rg.gif deleted file mode 100644 index fdb7bbc1..00000000 Binary files a/workshops/iot-hub-mxchip/assets/creation-rg.gif and /dev/null differ diff --git a/workshops/iot-hub-mxchip/assets/envvar-function.gif b/workshops/iot-hub-mxchip/assets/envvar-function.gif deleted file mode 100644 index 3c896cc9..00000000 Binary files a/workshops/iot-hub-mxchip/assets/envvar-function.gif and /dev/null differ diff --git a/workshops/iot-hub-mxchip/assets/install-sdk.png b/workshops/iot-hub-mxchip/assets/install-sdk.png deleted file mode 100644 index ad39cf67..00000000 Binary files a/workshops/iot-hub-mxchip/assets/install-sdk.png and /dev/null differ diff --git a/workshops/iot-hub-mxchip/assets/install-workbench.png b/workshops/iot-hub-mxchip/assets/install-workbench.png deleted file mode 100644 index 8e18f17f..00000000 Binary files a/workshops/iot-hub-mxchip/assets/install-workbench.png and /dev/null differ diff --git a/workshops/iot-hub-mxchip/assets/iot-banner.jpg b/workshops/iot-hub-mxchip/assets/iot-banner.jpg deleted file mode 100644 index 33549790..00000000 Binary files a/workshops/iot-hub-mxchip/assets/iot-banner.jpg and /dev/null differ diff --git a/workshops/iot-hub-mxchip/assets/iot-workbench-setting.png b/workshops/iot-hub-mxchip/assets/iot-workbench-setting.png deleted file mode 100644 index b51c8052..00000000 Binary files a/workshops/iot-hub-mxchip/assets/iot-workbench-setting.png and /dev/null differ diff --git a/workshops/iot-hub-mxchip/assets/redeempass-1.jpg b/workshops/iot-hub-mxchip/assets/redeempass-1.jpg deleted file mode 100644 index da8a649b..00000000 Binary files a/workshops/iot-hub-mxchip/assets/redeempass-1.jpg and /dev/null differ diff --git a/workshops/iot-hub-mxchip/assets/redeempass-2.jpg b/workshops/iot-hub-mxchip/assets/redeempass-2.jpg deleted file mode 100644 index 77b9829d..00000000 Binary files a/workshops/iot-hub-mxchip/assets/redeempass-2.jpg and /dev/null differ diff --git a/workshops/iot-hub-mxchip/assets/redeempass-3.jpg b/workshops/iot-hub-mxchip/assets/redeempass-3.jpg deleted file mode 100644 index 102a6a37..00000000 Binary files a/workshops/iot-hub-mxchip/assets/redeempass-3.jpg and /dev/null differ diff --git a/workshops/iot-hub-mxchip/assets/redeempass-4.jpg b/workshops/iot-hub-mxchip/assets/redeempass-4.jpg deleted file mode 100644 index c3568ccc..00000000 Binary files a/workshops/iot-hub-mxchip/assets/redeempass-4.jpg and /dev/null differ diff --git a/workshops/iot-hub-mxchip/assets/redeempass-5.jpg b/workshops/iot-hub-mxchip/assets/redeempass-5.jpg deleted file mode 100644 index 76875382..00000000 Binary files a/workshops/iot-hub-mxchip/assets/redeempass-5.jpg and /dev/null differ diff --git a/workshops/iot-hub-mxchip/assets/redeempass-6.jpg b/workshops/iot-hub-mxchip/assets/redeempass-6.jpg deleted file mode 100644 index ee8834bc..00000000 Binary files a/workshops/iot-hub-mxchip/assets/redeempass-6.jpg and /dev/null differ diff --git a/workshops/iot-hub-mxchip/assets/redeempass-7.jpg b/workshops/iot-hub-mxchip/assets/redeempass-7.jpg deleted file mode 100644 index 12a38c25..00000000 Binary files a/workshops/iot-hub-mxchip/assets/redeempass-7.jpg and /dev/null differ diff --git a/workshops/iot-hub-mxchip/assets/vscode-com.jpg b/workshops/iot-hub-mxchip/assets/vscode-com.jpg deleted file mode 100644 index 6b7aaeaf..00000000 Binary files a/workshops/iot-hub-mxchip/assets/vscode-com.jpg and /dev/null differ diff --git a/workshops/iot-hub-mxchip/assets/wifi-ap.jpg b/workshops/iot-hub-mxchip/assets/wifi-ap.jpg deleted file mode 100644 index b08e92d1..00000000 Binary files a/workshops/iot-hub-mxchip/assets/wifi-ap.jpg and /dev/null differ diff --git a/workshops/iot-hub-mxchip/assets/wifi-ip.jpg b/workshops/iot-hub-mxchip/assets/wifi-ip.jpg deleted file mode 100644 index 80ce5031..00000000 Binary files a/workshops/iot-hub-mxchip/assets/wifi-ip.jpg and /dev/null differ diff --git a/workshops/iot-hub-mxchip/assets/wifi-portal.png b/workshops/iot-hub-mxchip/assets/wifi-portal.png deleted file mode 100644 index 1daa8191..00000000 Binary files a/workshops/iot-hub-mxchip/assets/wifi-portal.png and /dev/null differ diff --git a/workshops/iot-hub-mxchip/workshop.fr.md b/workshops/iot-hub-mxchip/workshop.fr.md deleted file mode 100644 index 19851ba7..00000000 --- a/workshops/iot-hub-mxchip/workshop.fr.md +++ /dev/null @@ -1,433 +0,0 @@ ---- -short_title: Workshop IoT -description: Discover how to build connected IoT experiences using an Arduino board and Azure services. -type: workshop -authors: - - Jim Bennet - - Christopher Maneu - - Wassim Chegham - - Olivier Leplus - - Yohan Lasorsa -contacts: - - '@jimbobbennett' - - '@cmaneu' - - '@manekinekko' - - '@olivierleplus' - - '@sinedied' -banner_url: assets/iot-banner.jpg -duration_minutes: 180 -audience: students, pro devs -level: beginner -tags: iot, mxchip, azure, vs code -published: true -sections_title: - - Introduction ---- - -# Développez un projet IoT connecté avec la carte MXChip et Azure - -Ce workshop, accessible à **tous les développeurs même sans connaissance en IoT ou sur Azure -**, vous permettra de -découvrir la programmation sur des devices IoT avec ([Arduino](https://www.arduino.cc)) et [Visual Studio Code](https://code.visualstudio.com/)), ainsi que -la création d'expériences connectées avec les services [Azure](https://azure.microsoft.com/)). - -## Pré-requis - -Afin de réaliser ce workshop, vous aurez besoin: - -- D'un PC (ou Mac) de développement, sur lequel il faudra installer un certain nombre d'outils et de drivers, -- D'un abonnement Azure (d'essai, payant ou MSDN), -- Dans l'idéal, d'une carte de développement [MXChip](https://learn.microsoft.com/azure/iot-develop/quickstart-devkit-mxchip-az3166) ([acheter](https://www.seeedstudio.com/AZ3166-IOT-Developer-Kit.html)), ou de l'émulateur. - ---- - -## Préparer sa machine de dev - -Afin de pouvoir développer, puis déployer à la fois sur le board MXChip et sur Azure, il vous faudra plusieurs outils -(gratuits): - -- [.NET Core 3.1](https://dotnet.microsoft.com/download) -- [Visual Studio Code](https://code.visualstudio.com/) ainsi que quelques extensions - - L'extension [Azure IoT tools](https://marketplace.visualstudio.com/items?itemName=vsciot-vscode.azure-iot-tools) ([Installer](vscode:extension/vsciot-vscode.azure-iot-tools)), qui contient notamment _IoT Workbench_, - - L'extension [Arduino](https://marketplace.visualstudio.com/items?itemName=vsciot-vscode.vscode-arduino) de l'éditeur Microsoft, - - L'extension [Azure Tools](https://marketplace.visualstudio.com/items?itemName=ms-vscode.vscode-node-azure-pack), - - Les extensions pour les langages que vous allez utiliser - - [C#](https://marketplace.visualstudio.com/items?itemName=ms-vscode.csharp), - - JavaScript est déjà inclus :) -- [Arduino IDE](https://www.arduino.cc/en/Main/Software): il contient les outils de builds et de déploiment pour la carte MXChip. **Attention:** Installez la version "standalone", et non pas la version du Store. -- Le driver _ST-Link_: - - Windows - Télécharger et installer le driver depuis le site [STMicro](http://www.st.com/en/development-tools/stsw-link009.html). - - - macOS - Pas de driver nécessaire - - - Ubuntu - Exécuter la commande suivante dans votre terminal, puis déconnectez/reconnectez-vous afin d'appliquer le changement - de permissions - - ```bash - # Copy the default rules. This grants permission to the group 'plugdev' - sudo cp ~/.arduino15/packages/AZ3166/tools/openocd/0.10.0/linux/contrib/60-openocd.rules /etc/udev/rules.d/ - sudo udevadm control --reload-rules - - # Add yourself to the group 'plugdev' - # Logout and log back in for the group to take effect - sudo usermod -a -G plugdev $(whoami) - ``` - -L'installation d'une extension Visual Studio peut se faire par ligne de commande, ou directement dans l'interface via -l'onglet "extensions" (le 5ème icône sur la gauche). - -![Installer IoT Device Workbench](assets/install-workbench.png) - -Si vous souhaitez installer l'ensemble des extensions, voici un script pour Windows: -``` -code --install-extension vsciot-vscode.azure-iot-tools -code --install-extension vsciot-vscode.vscode-arduino -code --install-extension ms-vscode.vscode-node-azure-pack -code --install-extension ms-vscode.csharp -``` - -Une fois l'ensemble de ces composants installés, il faudra s'assurer que Visual Studio Code puisse utiliser l'installation -d'Arduino. Ouvrir **File > Preference > Settings** et ajouter les lignes suivantes à votre configuration. - -![Configurer Arduino path](assets/iot-workbench-setting.png) - -Voici les valeurs par défaut à ajouter à cette configuration: - -* Windows - - ```JSON - "arduino.path": "C:\\Program Files (x86)\\Arduino", - "arduino.additionalUrls": "https://raw.githubusercontent.com/VSChina/azureiotdevkit_tools/master/package_azureboard_index.json" - ``` - -* macOS - - ```JSON - "arduino.path": "/Applications", - "arduino.additionalUrls": "https://raw.githubusercontent.com/VSChina/azureiotdevkit_tools/master/package_azureboard_index.json" - ``` - -* Ubuntu - - ```JSON - "arduino.path": "/home/{username}/Downloads/arduino-1.8.5", - "arduino.additionalUrls": "https://raw.githubusercontent.com/VSChina/azureiotdevkit_tools/master/package_azureboard_index.json" - ``` - -**Pensez à sauvegarder vos paramètres avant de passer à l'étape suivante !** - -Enfin il faudra ajouter le SDK spécifique pour la board Arduino MXChip. Pour cela, via la palette de commande (`Ctrl+Shift+P` - ou `Cmd+Shif+P`), ouvrir la page **Arduino: Board Manager**, et rechercher **AZ3166**, puis installer la version **1.6.0** (les autres versions suppérieures 1.6.0+ ne sont pas encore compatibles avec le SDK). - -![Installer le SDK MXChip](assets/install-sdk.png) - ---- - -## Préparer son environnement Azure - -Afin de réaliser cet atelier, vous aurez besoin d'une souscription Azure. Il y a plusieurs moyens d'en obtenir une: - -- (**Obligation**) Si vous lisez cet atelier durant le Roadshow, vous pouvez utiliser l'Azure Pass que nous vous fournissons, -- Ou si vous êtes abonnés MSDN, utiliser les crédits offerts par votre abonnement. -- Ou créer un [abonnement d'essai](https://azure.microsoft.com/en-us/free/), - -### Utiliser votre Azure Pass - -1. Rendez-vous sur [microsoftazurepass.com](https://www.microsoftazurepass.com/) et cliquez sur **Start**, -![Démarrer l'utilisation du pass](assets/redeempass-1.jpg) -2. Connectez vous avec un compte Microsoft Live **Vous devez utiliser un compte Microsoft qui n'est associé à aucune - autre souscription Azure** -3. Vérifiez l'email du compte utilisé et cliquez sur **Confirm Microsoft Account** -![Confirmer le compte](assets/redeempass-2.jpg) -4. Entrez le code que nous vous avons communiqués, puis cliquez sur **Claim Promo Code** (et non, le code présent sur la - capture d'écran n'est pas valide ;) ), -![Indiquer son code](assets/redeempass-3.jpg) -5. Nous validons votre compte, cela prend quelques secondes -![Validation du code](assets/redeempass-4.jpg) -6. Nous serez ensuite redirigé vers une dernière page d'inscrption. Remplissez les informations, puis cliquez sur **Suivant** -![Entrer les informations](assets/redeempass-5.jpg) -7. Il ne vous restera plus que la partie légale: accepter les différents contrats et déclarations. Cochez les cases que -vous acceptez, et si c'est possible, cliquez sur le bouton **Inscription** -![Accepter les conditions légales](assets/redeempass-6.jpg) - -Encore quelques minutes d'attente, et voilà, votre compte est créé ! Prenez quelques minutes afin d'effectuer la -visite et de vous familiariser avec l'interface du portail Azure. - -![Accueil du portail Azure](assets/redeempass-7.jpg) - ---- - -## Configurez votre board - -
- -Votre board est normalement déjà configurée. Vous n'avez rien à faire, sauf en cas de souci de connection. - -
- -### Configurer le Wi-Fi de votre MX Chip - -Si vous avec besoin de reconnecter votre board au WiFi, suivez ces instructions. - -1. Maintenir appuyé le bouton **B**, appuyer et relacher le bouton **Reset** , puis relâcher le bouton **B**. La board va alors passer en mode _configuration WiFi_. Pour se faire, il va lui-même diffuser un point d'accès auquel se connecter. L'écran affiche ainsi le SSID, ainsi que l'adresse IP à utiliser. - - ![Reset button, button B, and SSID](assets/wifi-ap.jpg) - -2. Connectez-vous au réseau WiFi indiqué sur la board. Si votre appareil demande un mot de passe, laissez-le vide. - - ![Network info and Connect button](assets/connect-ssid.png) - -3. Ouvrez **192.168.0.1** dans un navigateur. Sélectionnez le réseau sur lequel vous souhaitez vous connecter. Indiquez la clé WEP/WPA, puis cliquez sur **Connect**. - - ![Password box and Connect button](assets/wifi-portal.png) - -4. La board va redémarrer quelques secondes après. Elle affichera alors le nom du wifi ainsi que son adresse IP directement sur l'écran: - - ![Wi-Fi name and IP address](assets/wifi-ip.jpg) - ---- - -## Créer vos services dans Azure - -Nous allons maintenant utiliser le [portail Azure](https://portal.azure.com/?feature.customportal=false) afin de créer l'ensemble des services dans Azure - -appelée _ressources_. Si vous êtes plus bash que clic, vous pouvez utiliser la [Azure CLI](https://learn.microsoft.com/cli/azure/?view=azure-cli-latest) (on vous laisse -chercher comment faire 😉. - -### Créer un resource group - -Nous allons commencer par créer un groupe de ressources (_resource group_). C'est un conteneur logique pour l'ensemble -des services que vous allez créer ensuite. Chaque service doit absolument être dans un resource group. - -Depuis le portail Azure, vous avez trois moyens de créer une nouvelle ressource : - -- Le bouton **Créer une ressource** en haut à gauche, -- Naviguer vers un type de service, puis cliquer sur le bouton **Ajouter** -- Depuis la page du groupe de ressources, cliquer sur le bouton **Ajouter** - -La vidéo suivante vous montre comment créer votre premier groupe de ressources. Le nom du groupe de ressource doit être -unique au sein de votre compte Azure. Pour ce qui est de la région, nous choisirons tout au long de cet atelier _Europe -Occidentale_ ou _West Europe_. - -![Video - Création d'un ressource group](assets/creation-rg.gif) - -Une fois créé, vous pouvez vous rendre sur la page de la ressource via l'icône de notifications en haut. - -### Créer un IoT Hub - -L'IoT Hub est un service qui vous permet de gérer la connexion entre vos devices IoT et vos services hébergés sur Azure -(ou ailleurs). Plus concrètement, il vous permet : - -- D'identifier et de recevoir des données de vos périphériques IoT - on appelle cela le _Device To Cloud_, -- D'envoyer ces données à différents applicatifs, -- De transmettre des commandes ou des données du cloud vers vos périphériques - c'est le _Cloud To _Device_, -- De mettre à jour les micrologiciels à distance de vos périphériques, voire de déployer du code à distance. - -La vidéo suivante nous montre comment créer un nouveau IoT Hub. Choisissez bien le groupe de ressources créé à l'étape -précédente, puis choisissez la région (Europe occidentale) puis un nom. - -
- -> Comme beaucoup de ressources dans Azure, leur nom devient une partie d'une adresse Internet - ici -`monhub.wassim-ioth.azure-devices.net`. Il doit donc être unique à tous les utilisateurs d'Azure ! - -
- -A l'étape d'après, vous serez amené à choisir un niveau de tarification (_tier_) et une mise à l'échelle. Pour cet -atelier, nous choisirons la taille **S1: Niveau Standard**. - -
- -> Il existe à aujourd'hui trois tiers. Le tiers gratuit est limité en nombre de messages, alors que le tiers basique ne -dispose pas des fonctionnalités _Cloud to Device_ ou _Device Twins_ que nous allons utiliser plus loin. Le nombre -d'unités permet quand à lui de supporter un plus grand nombre de périphériques IoT. - -
- -![Video - Création d'un IoT Hub](assets/creation-iothub.gif) - -### Créer un IoT Device - -Au sein du IoT Hub, chacun de vos périphériques IoT se doit d'être déclaré afin de pouvoir le gérer et accepter des -données. Pour cet atelier simple, nous allons ajouter le périphérique à la main. Si nous avions à déployer des milliers -de périphériques, il y a bien évidemment [une solution](https://learn.microsoft.com/fr-fr/azure/iot-dps/)] :) - -La création d'un device IoT dans le portail est assez simple. Naviguez jusqu'à l'onglet **Appareils IoT**, puis cliquez -sur **Ajouter**. Vous avez alors simplement à donner un nom à votre périphérique. - -![Video - Création d'un périphérique IoT Hub](assets/creation-iotdevice.gif) - -Lorsque vous vous rendez sur l'écran de votre appareil IoT, vuos verrez alors deux clés : **Ce sont elles qui permettent -de sécuriser la connexion entre votre appareil et Azure**. Il est important **de ne pas les diffuser ou les mettre dans -votre code source (ou repository Github)**. Nous verrons plus tard comment la déployer sur la carte. - -
- -> **Notez cette clé d'accès quelque part** ou gardez la fenêtre ouverte, nous allons l'utiliser dans quelques étapes. - -
- -Nous en avons pour l'instant fini avec IoT Hub, mais nous reviendrons plus tard sur cette partie. - -### Créer une Azure Function - -Notre site web sera simplement un "Front HTML". Il lui faudra communiquer avec l'IoT Hub, et pour cela utiliser un - _secret_. Afin de protéger ce secret et de limiter ce qu'il est possible de faire, nous allons créer une API contenant - cette logique et ce secret. Nous pourrions l'héberger dans un site web - comme celui que nous avons créé il y a - quelques instants - mais Azure Functions est un service intéressant pour cet usage : il vous permet d'héberger non pas - un site web, mais simplement une méthode de code ! - -Lors de la création, il vous faudra indiquer un nom et un emplacement. Hormis ces informations et le groupe de -ressources, laissez tous les autres paramètres à leurs valeurs par défaut. - -![Video - Création d'une Azure Function](assets/creation-function.gif) - ---- - -## Déployez du code sur votre board et connectez-là à Azure - -Si vous avez installés tous les prérequis, et que votre board est [connectée à Internet](docs/configurer-wifi.md), alors - nous pouvons continuer. Notre première étape est de créer un projet **Azure IoT Workbench Visual Studio Code**. Ce type de projet va nous apporter toutes les fonctionnalités nécessaires pour travailler: builder le code, configurer la carte, déployer le code sur la carte, etc... - - Dans la suite de cet atelier, nous allons utiliser beaucoup de commandes. Celles-ci sont accessibles via le raccourci clavier `Ctrl+Shift+P` (ou `Cmd+Shift+P` sous Mac). Pour créer votre projet: - - 1. Recherchez `Workbench create`, et choisissez `Azure IoT Device Workbench: Create Project` - 2. Sélectionnez `IoT DevKit` - 3. Selectionnez le template `With Azure IoT Hub` - 4. Choisissez un dossier dans lequel enregistrer vos fichiers sources - -> Nous allons créer plusieurs projets lors de cet atelier. Je vous suggère la hiérarchie de dossiers suivante : -> ``` -> MonProjet -> | - device -> | - fonction -> | - web -> ``` -> - -![Video - Création d'une Azure Function](assets/creation-projetIoT.gif) - -Nous pouvons désormais copier-coller le [contenu du fichier `.ino`](https://raw.githubusercontent.com/themoaw/DevRoadShow-IOT/master/src/arduino/DevKitState.ino) . A cette étape, il est possible de compiler le code avec la commande `Azure IoT Device Workbench: Compile Device Code`. - -Il nous reste désormais deux choses à faire: connecter la board à notre IoT Hub, et déployer le code sur le device. Pour la connection, nous allons simplement envoyer la chaîne de connexion - créée au début du tutoriel - sur la board. Au préalable, assurez-vous que Visual Studio a bien sélectionné votre type de board ainsi que le port série (émulé via l'USB). - -![Sélecteur Visual Studio Code de board et de port série](assets/vscode-com.jpg) - -1. Maintenez appuyé le bouton **A** puis appuyez et relâchez le bouton **reset** pour passer en mode configuration -2. A l'aide de la commande `Azure IoT Device Workbench: Configure Device Settings`, choisissez `Config Device Connection String`, puis `Input IoT Hub Device Connection String`, et collez la connection string complète générée au début de l'atelier. - -Nous pouvons maintenant déployer notre code. Toujours à l'aide de la commande de palettes, sélectionnez `Azure IoT Device Workbench: Upload Device Code`. -L'opération peut prendre quelques minutes. Pendant ce temps-là, la LED "programming" sur la board devrait clignoter. - -
- -> **Vous n'avez pas de MXChip sous la main et vous voulez tout de même tester cela ?** -> C'est possible, avec [l'émulateur](https://azure-samples.github.io/iot-devkit-web-simulator/) ! Copiez-collez le code Arduino dans l'émulateur, indiquez votre ->chaîne de connexion à l'IoT Hub et c'est parti. - -
- -### Créer une Azure Function pour communiquer avec IoT Hub - -1. Lancez la commande **Azure Functions: Create new project**, -2. Sélectionnez un répertoire, -3. Choisissez le langage C#, -4. Sélectionnez la runtime **Azure Functions v3** -5. Sélectionnez **Skip for now** lors du choix de type de fonction -6. Ajoutez le projet a votre workspace courant - -
- -> Si vous êtes plus à l'aise avec Python ou avec NodeJS, vous devriez pouvoir porter le code avec les SDKs correspondants. Mais dites-le nous avant de vous lancer! - -
- -Vous pouvez maintenant copier le code C# de [ce repository GitHub](https://github.com/themoaw/DevRoadShow-IOT/tree/master/src) dans votre dossier de travail. - -Afin que le code fonctionne, nous devons inclure le SDK _Azure Devices_ dans notre projet. Cela s'effectue -au niveu du fichier `.csproj`. Assurez-vous que vous retrouvez les lignes de code suivantes (la ligne `PackageReference` qui inclut `Microsoft.Azure.Devices`). - -```csharp - - - - -``` - -Dans le fichier `cs`, à la ligne 22, il vous faudra remplacer la référence _DeviceMX_ par celle du nom du device IoT créé au début de l'atelier. - -Enfin pour déployer le code de votre fonction, faites un clic-droit sur le dossier de votre projet fonction, et cliquez sur **Deploy to Function App**. Vous pourrez alors choisir la souscription, puis l'application fonction que vous aviez créé tout au début. - -Il vous restera une dernière petite chose : faire communiquer votre Azure Function avec l'IoT Hub. Pour se faire, vous devez indiquer la chaîne de connexion à l'IoT Hub à la fonction. Nous allons passer par une variable d'environnement `iotHubConnectionString`, qui est lue par la méthode `Environment.GetEnvironmentVariable`. Nous pouvons faire cela directement via le portail Azure. - -![Configuration des variables d'environnement dans Azure Function](assets/envvar-function.gif) - -
- -> Nous parlons ici de la _connection string_ à l'IoT Hub, et non pas celle du Device lui-même ! Vous trouverez cette clé dans le portail Azure, sur votre IoT Hub : allez sur **Shared Access Policy**, puis cliquez sur **iothubowner*. - -
- -### Le moment de vérité ! - -Lancez https://mxchip-workshop.netlify.com/, puis indiquez l'URL complète de votre Azure Function. - -![Image - Récupértion de l'URL de la fonction](https://user-images.githubusercontent.com/1699357/60516369-e7ab9d00-9cdd-11e9-905d-3808b1ad7e4e.png) - -Si tout se passe bien, vous devriez voir le status de votre carte, et en cliquant sur le bouton "RGB LED", la LED de votre carte devrait s'allumer ! - ---- - -## Changer la couleur de la LED - -Maintenant que votre projet fonctionne, et que vous pouvez allumer et éteindre la LED à distance, essayons d'ajouter -un peu de disco ! Nous allons maintenant faire en sorte de pouvoir choisir la couleur de la LED RGB. Jusqu'à présent, -le code permet uniquement de sélectionner les tons de rouge. Il faut donc compléter le code pour supporter le vert et -le bleu. - -Pour réaliser cela, il vous faudra retrouver les `TODO` dans le code dans le fichier `State.cs`: - -On vous laisse trouver ce qu'il faut changer dans le code. - -N'hésitez pas à nous appeler à l'aide en cas de soucis ! - ---- - -## Bonus - -### [Etape Bonus] Changez à distance le message - -Regardez donc la méthode `DeviceTwinCallback`, c'est elle qui est appelée quand le device reçoit un message du cloud. - - -### [Etape cachée] Envoyez un dessin à distance - -Le MXChip est équipé d'un écran LCD de 128x64px qui est capable d'afficher des dessins sommaires. Avant de tenter un envoi à distance, -essayer de l'afficher en modifiant le code du projet Arduino. Pour vous aider, le site [pixelcraft](https://pixelcraft.cc/) vous -permet de générer le code correspondant à votre dessin. - -La méthode `Screen.draw` vous permet de dessiner cette matrice à l'écran. -Enfin, il vous faudra envoyer en JSON ce dessin depuis l'IoT Hub, et décrypter le JSON sur l'arduino. Pour tout cela, -il vous faudra probablement les include suivants: - -```cpp -#include "RGB_LED.h" -#include -``` - -### [Premier de la classe] Testez l'un des autres projets - -La commande **Azure IoT Device Workbench: Open Examples** vous permet d'accéder à un ensemble d'exemples préassemblés. Testez-en un, comme par exemple le _DevKit Translator_. - ---- - -## Conclusion - -Bravo, vous avez fini le workshop! - -### Crédit - -Ce workshop a été créé par [Jim Bennett](https://github.com/jimbobbennett/MXChip-Workshop) puis traduit en français par [Christopher Maneu](https://twitter.com/cmaneu) et ré-arrangé par [Wassim Chegham](https://twitter.com/manekinekko), [Olivier Leplus](https://twitter.com/olivierleplus) et [Yohan Lasorsa](https://twitter.com/sinedied). - -Vous pouvez trouver la version anglaise à [cette adresse](https://github.com/jimbobbennett/MXChip-Workshop). diff --git a/workshops/ml-custom-vision/assets/github-actions.png b/workshops/ml-custom-vision/assets/github-actions.png deleted file mode 100644 index 4c37662a..00000000 Binary files a/workshops/ml-custom-vision/assets/github-actions.png and /dev/null differ diff --git a/workshops/ml-custom-vision/assets/githubtemplate.png b/workshops/ml-custom-vision/assets/githubtemplate.png deleted file mode 100644 index 0b8ce69a..00000000 Binary files a/workshops/ml-custom-vision/assets/githubtemplate.png and /dev/null differ diff --git a/workshops/ml-custom-vision/assets/ml-banner.jpg b/workshops/ml-custom-vision/assets/ml-banner.jpg deleted file mode 100644 index c51831a7..00000000 Binary files a/workshops/ml-custom-vision/assets/ml-banner.jpg and /dev/null differ diff --git a/workshops/ml-custom-vision/assets/new-project.png b/workshops/ml-custom-vision/assets/new-project.png deleted file mode 100644 index 711ea378..00000000 Binary files a/workshops/ml-custom-vision/assets/new-project.png and /dev/null differ diff --git a/workshops/ml-custom-vision/assets/new-resourcegroup.png b/workshops/ml-custom-vision/assets/new-resourcegroup.png deleted file mode 100644 index 74c081aa..00000000 Binary files a/workshops/ml-custom-vision/assets/new-resourcegroup.png and /dev/null differ diff --git a/workshops/ml-custom-vision/assets/resource-overview.png b/workshops/ml-custom-vision/assets/resource-overview.png deleted file mode 100644 index 59a0fded..00000000 Binary files a/workshops/ml-custom-vision/assets/resource-overview.png and /dev/null differ diff --git a/workshops/ml-custom-vision/assets/result-frontend.png b/workshops/ml-custom-vision/assets/result-frontend.png deleted file mode 100644 index e1aa6d76..00000000 Binary files a/workshops/ml-custom-vision/assets/result-frontend.png and /dev/null differ diff --git a/workshops/ml-custom-vision/assets/result.png b/workshops/ml-custom-vision/assets/result.png deleted file mode 100644 index 0c998fb5..00000000 Binary files a/workshops/ml-custom-vision/assets/result.png and /dev/null differ diff --git a/workshops/ml-custom-vision/assets/settingsinportal.png b/workshops/ml-custom-vision/assets/settingsinportal.png deleted file mode 100644 index 0aca93b0..00000000 Binary files a/workshops/ml-custom-vision/assets/settingsinportal.png and /dev/null differ diff --git a/workshops/ml-custom-vision/assets/swa-github.png b/workshops/ml-custom-vision/assets/swa-github.png deleted file mode 100644 index 1a270ad5..00000000 Binary files a/workshops/ml-custom-vision/assets/swa-github.png and /dev/null differ diff --git a/workshops/ml-custom-vision/assets/swa-workshop.zip b/workshops/ml-custom-vision/assets/swa-workshop.zip deleted file mode 100644 index c229080a..00000000 Binary files a/workshops/ml-custom-vision/assets/swa-workshop.zip and /dev/null differ diff --git a/workshops/ml-custom-vision/assets/testing-images.zip b/workshops/ml-custom-vision/assets/testing-images.zip deleted file mode 100644 index 366a5096..00000000 Binary files a/workshops/ml-custom-vision/assets/testing-images.zip and /dev/null differ diff --git a/workshops/ml-custom-vision/assets/testinwebsite.png b/workshops/ml-custom-vision/assets/testinwebsite.png deleted file mode 100644 index 3ba81900..00000000 Binary files a/workshops/ml-custom-vision/assets/testinwebsite.png and /dev/null differ diff --git a/workshops/ml-custom-vision/assets/training-images.zip b/workshops/ml-custom-vision/assets/training-images.zip deleted file mode 100644 index 189e0bc0..00000000 Binary files a/workshops/ml-custom-vision/assets/training-images.zip and /dev/null differ diff --git a/workshops/ml-custom-vision/assets/uploadsettings.png b/workshops/ml-custom-vision/assets/uploadsettings.png deleted file mode 100644 index f46e12ec..00000000 Binary files a/workshops/ml-custom-vision/assets/uploadsettings.png and /dev/null differ diff --git a/workshops/ml-custom-vision/workshop.md b/workshops/ml-custom-vision/workshop.md deleted file mode 100644 index fcff8840..00000000 --- a/workshops/ml-custom-vision/workshop.md +++ /dev/null @@ -1,622 +0,0 @@ ---- -short_title: Custom Vision Workshop -description: Train a Machine learning model using Custom Vision and use it in a web application deployer on Azure Static Web Apps using Azure Functions as a backend. -type: workshop -authors: - - Olivier Leplus - - Christopher Harrison -contacts: - - '@olivierleplus' - - '@GeekTrainer' -banner_url: assets/ml-banner.jpg -duration_minutes: 180 -audience: students, pro devs -level: beginner -tags: machine learning, azure, custom vision, static web apps, python, javascript -published: true -sections_title: - - Introduction ---- - -# Machine Learning with Custom Vision - -In this workshop, you will learn how to build a model to detect dog breeds. You'll start by installing and configuring the necessary tools, then creating the custom model by uploading and tagging images. And finally, you will use the model using the Custom Vision REST API and the Software Development Kit (SDK). - -
Train your model on the Custom Vision portal
- -![Add a Dev Container](assets/result.png) - -
Deploy an application to detect dog breeds
- -![Add a Dev Container](assets/result-frontend.png) - -## Prerequisites - -To do this workshop, you will need: -* Basic JavaScript or Python knowledge -* [A Microsoft Azure account](https://azure.microsoft.com/free/) -* [A GitHub account](http://github.com/) -* [Visual Studio Code](https://code.visualstudio.com/) (VSCode) -* [Node.js 14 or 16 installed](https://nodejs.org/) -* Python 3.8 or greater with pip installed -* [Azure Functions Core Tools](https://learn.microsoft.com/azure/azure-functions/functions-run-local?tabs=v4%2Clinux%2Ccsharp%2Cportal%2Cbash) - ---- - -## Create the Azure resources - -As with any project, a few tools are going to be needed. In particular you'll need a code editor, an Azure subscription, and a couple of keys for Custom Vision. - -Start by opening the [Custom Vision dashboard](https://ms.portal.azure.com/#blade/Microsoft_Azure_ProjectOxford/CognitiveServicesHub/CustomVision) in the Azure Portal. Then, click on the `+ Create` button or the `Create custom vision` button if you don't have any custome vision yet. This link will take you to the Custom Vision creation form. You may be prompted to log into your Azure Subscription. If you don't have one you can create a Free trial. - -Let's fill it out! - -* Select `Both` for `Create options` as we are going to use our resource to both train our model and make predictions. -* Select your Subscription. -* Create a new Resource Group. - -
- -> In Azure, a Resource Group is a logical structure that holds resources usually related to an application or a project. A Resource Group can contain virtual machines, storage accounts, web applications, databases and more. - -
- -* Select `West Europe` as Region if you are in Europe or choose the region closest to you. -* Give a Name to your project, for instance "simplon-workshop". -* Select the Free `F0 Plan` for the training and prediction pricing tiers. - -
- -> It is recommended to host your resource in a region close to your users. - -
- -* Click on `Review + Create` (not on `Next`) and then on `Create`. - -After a few minutes, your Custom Vision resource will be created on Azure. - -You now have all the necesary tools for creating your custom vision model. - ---- - -## Train your model - -### Create your Custom Vision Project - -Custom Vision lives in another platform than the Azure Portal. Now that you have created your resource, let's use the Custom Vision Portal to train your model. - -First, navigate to [Custom Vision](https://www.customvision.ai) and sign-in with your account. - -* Select `New Project` -* Enter `Dog Classification` for the project name -* Select the resource you created in the previous step of the workshop -* Select `Classification` as `Project Types` - -
- -> Image classification tags whole images where Object Detection finds the location of content within an image. - -
- -* Select `Multiclass` for `Classification Types` as our dogs will only have one breed -* Select `General [A2]` for `Domain` as our dogs are not related to any other domain -* Click on the `Create project` button - -
- -> You can find more information about the differences between Domains in the [Custom Vision documentation](https://learn.microsoft.com/azure/cognitive-services/custom-vision-service/select-domain). - -
- -### Upload your images - -Once the project is created, it's time to upload images. These images will be used to train your model. - -
- -> As a general rule, the more images you can use to train a model the better. You want to include as much variety in the images as possible, including different lightning, angles, and settings. - -
- -We will provide you with a few images of dogs to train your model. Download this [Training images set](assets/training-images.zip) and extract the zip file. - -Now, go in the Custom Vision portal: -* Click on `Add images` -* Navigate to the `training-images` folder you just downloaded -* Select all the images marked as `american-staffordshire-terrier` in the folder, and click `Open`. -* Enter `american-staffordshire-terrier` for the tag and click `Upload 8 files` -* Click `Done` -* Click on `Add images` and repeat the above steps for the other breeds. - * `australian-shepherd` - * `buggle` - * `german-wirehaired-pointer` - * `shorkie` - -### Train your model - -Now that you have uploaded and tagged your images, it's time to train your model. - -* Click on the `Train` button to open the training dialog. -* Leave `Quick Training` selected and click `Train` to begin the training process. - -It will take between 2 to 5 minutes to train your model. - ---- - -## Test & Publish your model - -With the model trained, it's time to turn our attention to using it. We'll start by testing it in the Custom Vision website. Then we'll explore how we can call the model from our code by using the REST API or the SDK. - -### Test your model in the Custom Vision portal - -Let's see how well our model works. **It's important to use images which weren't used to train the model**. After all, if the model has already seen the image it's going to know the answer. Start by downloading the [Testing images set](assets/testing-images.zip) and extract the zip file. - -Then, in the Custom Vision portal -* Click on the `Quick Test` button -* Select `Browse local files` -* Navigate to the `testing-images` folder you just downloaded and select one of the dog images -* Click `Open` - -Notice the `tag` and `probability` scores. The `tag` is the breed of the dog, and the `probability` is the confidence that the model has in its prediction. Try with another images :) - -![Test result in portal](assets/testinwebsite.png) - -### Publish your model - -Playing with your model in the Custom Vision portal is funny. But, the goal of creating a model in Custom Vision is to use it in different applications. To access it from outside of the Custom Vision website it needs to be published. - -* Go to the `Performance` tab and click `Publish` -* Enter `dogs` as `Model name` -* Select the resource in your subscription named `myresource-Prediction` -* Click `Publish` - ---- - -## Azure Static Web App - -### What is Azure Static Web Apps (SWA) - -Azure SWA is a service that enables developers to deploy their web applications on Azure in a seamless way. This means developers can focus on their code and spend less time administrating servers! - -So, with Azure SWA, developers can not only deploy frontend static apps and their serverless backends but they can also benefit from features like `Custom domains`, `pre-prod environments`, `authentication providers`, `custom routing` and more. - -### Who is it for? -Azure SWA is for every developer who wants to spend more time in their code editor than they do managing resources in the cloud. - -If you are a so-called "Full stack developer", then you probably want to deploy both your frontend and your backend to the cloud, ideally with limited administration and configuration management. - -### Frontend developers -Azure SWA supports many frameworks and static site generators out of the box. If you are using a framework like `Angular`, `React`, `Vue.js`, a site generator like `Gatsby`, `Hugo` or one of the many solutions Azure SWA supports, then you don't have to take care of the deployment. If you have a specific need to customise the build for your app, you can configure it yourself very easily! - -### Backend developers -Azure SWA relies on `Azure Functions` for your application backend. So, if you are developing in `JavaScript`, `Java`, `Python` or `.NET`, SWA makes it very easy to deploy your backend as well! - -
- -> You can find the official SWA documentation here: [https://aka.ms/swadocs](https://aka.ms/swadocs) - -
- -Oh, and did I forget to mention there is a Free tier for SWA? You can start using it for free and only pay once your application gets popular! - ---- - -## Get the website template - -Once upon a time, there was a website that needed a place to live, be visible to the world and have a backend to be more interactive. - -For this workshop, we won't ask you to create a website from scratch. As we want you to focus on the Custom Vision integration, we've built one for you. - -Go to [this GitHub repository](https://github.com/themoaw/template-customvision-workshop) and click on `Use this template`. - -![GitHub template](assets/githubtemplate.png) - -You will be redirected to the repository creation page. Just enter `customvision-workshop` as name for your new repository and click on `Create repository from template`. - -
- -> Create the repository from the GitHub Template and clone it on your computer. - -
- -You now have your baseline project. Open the `customvision-workshop` folder in VSCode. - ---- - -## Create an Azure Static Web App - -Start by opening the [Create Static Web App form](https://portal.azure.com/#create/Microsoft.StaticApp) in the Azure Portal. This link will take you directly to the Static Web App creation form. You may be prompted to log into your Azure Subscription. If you don't have one you can create a Free trial. - -Let's fill it out! - -* Select your Subscription. -* Select your Resource Group. - -
- -> In Azure, a Resource Group is a logical structure that holds resources usually related to an application or a project. A Resource Group can contain virtual machines, storage accounts, web applications, databases and more. - -
- -* Give a Name to your Static Web App, for example `my-swa`. -* Select the Free Plan (we won't need any feature in the Standard Plan for this workshop). -* Select `West Europe` for your backend. - -
- -> It is recommended to host your backend in a Region closest to your users. - -
- -* Select `GitHub` as a deployment source. - -We are going to host our website source code on GitHub. Later in the workshop, we will see how Static Web Apps will automaticalLy deploy our website every time we push new code to our repository. - -* Sign in with your GitHub account -* Select the organization (your account), the repository(customvision-workshop) and the branch of your project (main). - -As we mentioned at the beginning of the workshop, our app will have a backend and a frontend. In order for Static Web App to know what to deploy and where, we need to tell it where our apps are located in our repository. - -Azure Static Web Apps can handle several well-known frontend frameworks and can "compile" your Angular, React or Hugo application before deploying them. - -In our case, we have a very simple JavaScript application which does not require anything to run. So in the Build Presets option, let's choose `Custom`. -* In the `App location`, enter the `/www` folder as this is where our frontend is. -* In the `Api location`, enter the `/api` or `api` (depending on your preference) folder as this is where our backend is. -* In the `Output`, enter the `/www` folder as your frontend does not need any build system to run. - -![Enter GitHub information when creating SWA](assets/swa-github.png) - -* Click on `Review + Create` and then on `Create`. - -After a few minutes, your Static Web App will be created on Azure and your website deployed. - -Once the resource is created, you should `pull` your repository in VSCode as a few files have been added to your GitHub repo by Azure. - -## So, what just happened? - -### On GitHub - -When Azure created your Static Web App, it pushed a new YAML file to the `.github/workflow` folder of your repository. - -The files in this folder describe GitHub Actions, which are event-based actions that can be triggered by events like a `push`, a `new pull request`, a `new issue`, a `new collaborator` and more. - -You can see the complete list of triggers [here](https://docs.github.com/en/actions/reference/events-that-trigger-workflows) - -
- -> If you are not familiar with GitHub Actions, you can read about them [here](https://github.com/features/actions). - -
- -Let's have a look at the beginning of the YAML file Azure created for us: - -```yaml -on: - push: - branches: - - main - pull_request: - types: [opened, synchronize, reopened, closed] - branches: - - main -``` - -Here, you can see that the GitHub Action is going to be triggered every time there is a `push` on the `main` branch or every time a Pull Request is `opened`, `synchronize`, `reopened` or `closed`. - -As we want our website to be redeployed automaticalLy every time we push on our main branch, this is perfect! - -Take a few minutes to read the YAML file and understand what exactly happens when the GitHub Action is triggered. You can see that most of the information you entered when you created your Static Web App on Azure is here. - -
- -> The YAML file is in your GitHub repository so you can edit it! Your frontend site folder name changed? No problem, just edit the file and push it to GitHub! - -
- -Now, go to your GitHub repository in a web browser and click on the `Actions` tab. Here, you will see the list of all the GitHub Actions that have been triggered so far. Click on the last one to see your application being deployed. - -![Check your GitHub Actions](assets/github-actions.png) - -### On Azure - -Once your Static Web App is created, go to the Resource page in your Azure portal. You can find the list of all your Static Web Apps [here](https://portal.azure.com/#blade/HubsExtension/BrowseResource/resourceType/Microsoft.Web%2FStaticSites). - -In the Overview panel of your Static Web App, look for the `URL` parameter. This is the url of your website. - -![Resource overview of your project](assets/resource-overview.png) - -Open the link and you can see that your app has been deployed and is accessible to the world! - -Congratulations, you just deployed your first Static Web App on Azure! 🥳 - ---- - -## Test your project locally - -There are two ways to test your project. You can either push your code on GitHub every time you need to test it (not recommended), or use the `Static Web Apps CLI`. - -### The Static Web App CLI - -The Static Web Apps Command Line Interface, also known as the `SWA CLI`, serves as a local development tool for Azure Static Web Apps. In our case, we will use the CLI to: - -* Serve static app assets -* Serve API requests -* Emulate authentication and authorization -* Emulate Static Web Apps configuration, including routing - -You can install the CLI via npm. - -```bash -npm install -g @azure/static-web-apps-cli -``` - -We are only going to use a few features of the CLI so if you want to become a SWA CLI expert, you can find all the features the CLI provides [here](https://github.com/Azure/static-web-apps-cli). - -### Run your project locally - -The CLI offers many options, but in our case we want it to serve both our API and our web application located in our `www` folder. - -Right now, we don' have an API yet. In your terminal, type the following command to start your project: - -```bash -swa start www -``` - -This CLI gives you two urls: - -* [http://localhost:4280](http://localhost:4280) corresponding to your frontend. - -Congratulations, you now have everything you need to test your app on your computer! 🥳 - ---- - -## Let's add a backend - -Now that our frontend is deployed, we want to make it interactive and need to add a backend! - -Azure Static Web Apps relies on Azure Functions for your application backend. Azure Functions is an Azure service which enables you to deploy simple code-based microservices triggered by events. In our case, events will be HTTP requests. - -
- -> Ever heard of Serverless or FaaS (Function as a Service)? Well, you get it, this is what Azure Functions is ^^. - -
- -### Installation - -You can create an Azure Function from the [Azure portal](https://portal.azure.com/) but let's be honest, it's so much easier to stay in VSCode and use the Azure Functions extension. - -So, start by installing the Azure Function extension from VSCode. - -You can download the extension either directly from the `Extension panel (Ctrl + Shift + X)` in VSCode or by going [here](https://marketplace.visualstudio.com/items?itemName=ms-azuretools.vscode-azurefunctions)and clicking on the `Install` button. - -### Create your Function app - -Start by creating an `api` folder at the root of your project. This is where we are going to host our backend. - -Azure Functions live in an Azure Functions App. When you create your Function App, a single Azure Function will be created by default. You can then add more Functions as required. - -So, let's create our Functions App and a Function to retrieve our task list for our TODO frontend. - -* In VSCode, open the Command panel using `Ctrl + Shift + p` and search for `Azure Functions: Create new project`. -* Select the `api` folder. This is where our Function App will be created. -* Choose `JavaScript` or `Python` depending on the language you prefer as this is the langage we are going to use to write our Function. - -
- -> Azure Static Web Apps don't support all the languages you can develop Azure Functions with. Supported backends can be developed using JavaScript, TypeScript, Python or C#. - -
- -* As we are creating a REST API that will be called by our website, the trigger we are looking for is the `HTTP trigger`. -* Our first Function will be used to retrieve our task list. Let's call it `dogs`. -* Select the `Anonymous` authorization level. - -
- -> If you want to learn more about the different authorization levels for Functions and how to secure your API, check out the docs [here](https://learn.microsoft.com/azure/azure-functions/security-concepts). - -
- -### Start your backend - -The `Static Web App` CLI allows you to start both your frontend and your backend. - -In your terminal, type the following command to start your project: - -```bash -swa start www --api-location api -``` - -You can test your api here: [http://localhost:7071/api/dogs](http://localhost:7071/api/dogs). -A "Hello world" Function has been created for you so you don't start with a blank project. Let's modify it for our needs. - - ---- - -## Call your model using the REST API - -There are two ways to call your model. You can either use the `REST API` or use the `Custom Vision SDK`. - -Let's start by using the `REST API`. -* In your custom vision portal, go to the `Performance` screen and click select the `Prediction URL` - -We are going to use the web interface we deployed in the previous step of this tutorial. You don't have to change anything in the front end. All we will do is going to be in the `index.js` or `__init__.py` file of the Azure Function. - -### Using Node.js (skip this part if you chose Python) - -The first thing to do is to retrieve the image in the function. You can use a 3rd party library to do so. We recommand you use [azure-function-multipart](https://www.npmjs.com/package/@anzp/azure-function-multipart) but any library will do. -The `fetch` API was only recently added to node.js. Therefore, you may also need to use a 3rd party library to make the API call to the Custom Vision REST API. We recommand that you use [node-fetch](https://www.npmjs.com/package/node-fetch). - -Once you have installed these libraries using `npm install `, you can start coding. -Add these two lines at the top of your `index.js` file to use the libraries. - -```js -const parseMultipartFormData = require('@anzp/azure-function-multipart').default; -const fetch = require('node-fetch'); - -``` - -You can then easily get the file sent by the frontend by doing: - -```js -const file = files[0].bufferFile -``` - -Don't hesitate to read the documentation of these two libraries to understand how they work and how to use them. - -
- -> Use parseMultipartFormData to get the file and fetch to call the REST API using the prediction URL. -> Return the response to the frontend. - -
- - -### Using Python - -Make sure you installed all the requirements in your api folder. - -``` -pip install -r requirements.txt -``` - -To call an API in Python, you will need a 3rd party library. We recommand that you use [requests](https://pypi.org/project/requests/). Go check the documentation to see how to make a POST request using requests [here](https://docs.python-requests.org/en/latest/user/quickstart/#more-complicated-post-requests). - -You will need to send the file to the Custom Vision API. Here is how you can do to get the content of the file sent to the Azure function. Add these three lines in the function of your `__init__.py` file. -```python -file = req.files["file"] -filename = file.filename -content = file.stream.read() -``` - -
- Use request to call the REST API using the prediction URL. - Return the response to the frontend. -
- -Use one of the images in the `testing-images` folder to test your code. You should see the prediction like in the screenshot below. - -![Enter GitHub information when creating SWA](assets/result-frontend.png) - - ---- - -## Call your model using the SDK - -Using the `REST API` is a bit cumbersome. You can use the `Custom Vision SDK` to call your model. - -The SDK for Custom Vision uses a slightly different URL than the one you copied earlier. The value you copied will look something like the following: -``` -https://customvisionworkshop-prediction.cognitiveservices.azure.com/customvision/v3.0/Prediction/0dd3299b-6a41-40fe-ab06-dd20e886ccd4/classify/iterations/Iteration1/image -``` -To create the endpoint value, remove everything after azure.com. Your endpoint value should look like this: -``` -https://customvisionworkshop-prediction.cognitiveservices.azure.com/ -``` - -Now, let's change our Azure Function to use the SDK. - -### Using Node.js - -Microsoft provides several SDKs for Custom Vision. In our case, as we are only using our code to make prediction, we will only need the [cognitiveservices-customvision-prediction](https://www.npmjs.com/package/@azure/cognitiveservices-customvision-prediction) package. - -### Using Python - -Microsoft provides several SDKs for Custom Vision. In our case, as we are only using our code to make prediction, we will only need the [azure-cognitiveservices-vision-customvision](https://pypi.org/project/azure-cognitiveservices-vision-customvision/) package. - -
- -> Replace the code you wrote in the previous step to use the Custom Vision SDK instead. - -
- ---- - -## Deploy your project - -### Start by securing your sensitive information - -To deploy a Static Web App application, you need to push it on GitHub. -Before pushing any code to a public repository, you need to make sure there is no sensitive information in your code. Publishing things like credentials, API keys, passwords, etc. is a bad idea... - -APIs in Azure Static Web Apps are powered by Azure Functions, which allows you to define application settings in the `local.settings.json` file when you're running the application locally. This file defines application settings in the Values property of the configuration. - -The following sample local.settings.json shows how to add a value for the `PROJECT_ID`. -```json -{ - "IsEncrypted": false, - "Values": { - "AzureWebJobsStorage": "", - "FUNCTIONS_WORKER_RUNTIME": "node", - "PROJECT_ID": "" - } -} -``` - -Settings defined in the Values property can be referenced from code as environment variables. In Node.js functions, for example, they're available in the process.env object. - -```javascript -const projectId = process.env.PROJECT_ID; -``` - -In python, they are accessible as environment variables. - -```python -projectId = os.environ["PROJECT_ID"] -``` - -The `local.settings.json` file should be in your `.gitignore` file and therefore not pushed to your GitHub repository. Since the local settings remain on your machine, you need to manually configure your settings in Azure. - - -
- -> Move the project Id, Iteration name and prediction key to the local.settings.json file. If you are still using the API (of if you have just commented your code), also add the Prediction url used for calling the REST API
-> Test your project using the swa CLI to make sure everything works. - -
- - -### Publish your sensitive information to Azure - -There is two ways to create settings for your project. You can go to your project in the `Azure Portal` or you can do it directly from `VSCode` using the [Static Web Apps extension](https://marketplace.visualstudio.com/items?itemName=ms-azuretools.vscode-azurestaticwebapps). -Install the extension and login with your Azure Account if you haven't done it before. -* Click on the Azure icon on the sidenav and then on the `Static Web Apps` tab. -* Open your project and right click on `Application Settings` in your branch -* Click on `Upload Local Settings` - -![Upload Local Settings menu](assets/uploadsettings.png) - -Your local settings should now appear in the `Application Settings` section of your project. Check your project using the CLI and it should now work again. - -You could have also added the settings in the portal by going to your Static Web App resource in the `Configuration` menu. - -![Application Settings in Azure Portal](assets/settingsinportal.png) - - -### Deploy your project - -Now that you have secured your code, you only need to push it to your GitHub repository. This will trigger a GitHub Action and your project will be automaticalLy deployed on Azure. - -Go back to your Custom Vision resource in the Azure portal. - -![Resource overview of your project](assets/resource-overview.png) -In the `Overview` menu, you can find the public url of your website. Click on it. - -You've made it. You have published your website on Azure. - ---- - -## Conclusion - -Congratulations, you've reach the end of this workshop! 🚀 - -## Solution - -Did you know? You had the solution all along ;) -You can checkout the `solution branch` of the template repository to get the complete code of this workshop. -Or, you can download it from [here](https://github.com/themoaw/template-customvision-workshop/tree/solution). - -## Credits ❤️ - -This workshop is inspired from [this workshop](https://github.com/jlooper/workshop-library/tree/main/full/ml-model-custom-vision) created by [Christopher Harrison](https://twitter.com/GeekTrainer). diff --git a/workshops/semantic-kernel-ws/assets/chat-playground.png b/workshops/semantic-kernel-ws/assets/chat-playground.png deleted file mode 100644 index 017e510d..00000000 Binary files a/workshops/semantic-kernel-ws/assets/chat-playground.png and /dev/null differ diff --git a/workshops/semantic-kernel-ws/assets/copilot-stack.png b/workshops/semantic-kernel-ws/assets/copilot-stack.png deleted file mode 100644 index 53e9df1e..00000000 Binary files a/workshops/semantic-kernel-ws/assets/copilot-stack.png and /dev/null differ diff --git a/workshops/semantic-kernel-ws/assets/kernel-flow.png b/workshops/semantic-kernel-ws/assets/kernel-flow.png deleted file mode 100644 index 04048a6b..00000000 Binary files a/workshops/semantic-kernel-ws/assets/kernel-flow.png and /dev/null differ diff --git a/workshops/semantic-kernel-ws/assets/mind-and-body-of-semantic-kernel.png b/workshops/semantic-kernel-ws/assets/mind-and-body-of-semantic-kernel.png deleted file mode 100644 index f5b5d850..00000000 Binary files a/workshops/semantic-kernel-ws/assets/mind-and-body-of-semantic-kernel.png and /dev/null differ diff --git a/workshops/semantic-kernel-ws/assets/nav-to-portal.png b/workshops/semantic-kernel-ws/assets/nav-to-portal.png deleted file mode 100644 index 381d8708..00000000 Binary files a/workshops/semantic-kernel-ws/assets/nav-to-portal.png and /dev/null differ diff --git a/workshops/semantic-kernel-ws/assets/oai-portal.png b/workshops/semantic-kernel-ws/assets/oai-portal.png deleted file mode 100644 index 0131b565..00000000 Binary files a/workshops/semantic-kernel-ws/assets/oai-portal.png and /dev/null differ diff --git a/workshops/semantic-kernel-ws/workshop.md b/workshops/semantic-kernel-ws/workshop.md deleted file mode 100644 index 6951c5ae..00000000 --- a/workshops/semantic-kernel-ws/workshop.md +++ /dev/null @@ -1,493 +0,0 @@ ---- -published: true # Optional. Set to true to publish the workshop (default: false) -type: workshop # Required. -title: Build intelligent apps with Azure OpenAI & Semantic Kernel # Required. Full title of the workshop -short_title: Build intelligent apps with Azure OpenAI & Semantic Kernel # Optional. Short title displayed in the header -description: This is a workshop to discover how to use the power of Azure OpenAI services & Semantic Kernel to build intelligent apps in the form of a series of progressive challenges # Required. -level: advanced # Required. Can be 'beginner', 'intermediate' or 'advanced' -authors: # Required. You can add as many authors as needed - - Philippe Didiergeorges - - Vivien Londe - - Maxime Villeger -contacts: # Required. Must match the number of authors - - '@Philess' - - '@vivienlonde' - - '@mavilleg' -duration_minutes: 480 # Required. Estimated duration in minutes -tags: C#, Python, GenAI, OpenAI # Required. Tags for filtering and searching -#banner_url: assets/banner.jpg # Optional. Should be a 1280x640px image -#video_url: https://youtube.com/link # Optional. Link to a video of the workshop -#audience: students # Optional. Audience of the workshop (students, pro devs, etc.) -#wt_id: # Optional. Set advocacy tracking code for supported links -#oc_id: # Optional. Set marketing tracking code for supported links -#navigation_levels: 2 # Optional. Number of levels displayed in the side menu (default: 2) -sections_title: # Optional. Override titles for each section to be displayed in the side bar - - Introduction - - - Prerequisites - - - Challenge 1 - Azure OpenAi Playground - - - Challenge 2 - Semantic Kernel Samples - - - Challenge 3 - Build your first plugins - - - Challenge 4 - Plugins orchestration - - - Challenge 5 - Extra-challenges ---- - -# Semantic Kernel Workshop - -## Introduction - -This is an envisioning workshop, based on Microsoft's Copilot stack [Microsoft's Copilot stack](https://learn.microsoft.com/en-us/semantic-kernel/overview/#semantic-kernel-is-at-the-center-of-the-copilot-stack), to rethink user experience, architecture, and app development by leveraging the intelligence of foundation models. This workshop will use Semantic Kernel (SK), along with SK's Design thinking material, to guide you through the lifecycle of intelligent app development. - -![Copilot Stack](assets/copilot-stack.png) - -
- -> Semantic Kernel and the Azure OpenAI Services are quickly evolving products and thus this workshop may not be 100% up to date with the differentes features of the different extensions you are going to use. Please be clever. - -
- -**Semantic Kernel** is an open-source SDK that lets you easily combine AI services like [OpenAI](https://platform.openai.com/docs/), [Azure OpenAI](https://azure.microsoft.com/products/cognitive-services/openai-service/), and [Hugging Face](https://huggingface.co/) with conventional programming languages like C# and Python. By doing so, you will create AI apps that combine the best of both worlds. - -The Semantic Kernel has been engineered to allow developers to flexibly integrate AI services into their existing apps. To do so, Semantic Kernel provides a set of connectors that make it easy to add [memories](https://learn.microsoft.com/en-us/semantic-kernel/memories/) and models. In this way, Semantic Kernel is able to add a simulated "brain" to your app. - -Why use an AI orchestrator you may ask ? - -If you wanted, you could use the APIs for popular AI services directly and feed the results into your existing apps and services. This, however, requires you to learn the APIs for each service and then integrate them into your app. Using the APIs directly also does not allow you to easily draw from the recent advances in AI research that require solutions on top of these services. For example, the existing APIs do not provide planning or AI memories out-of-the-box. To simplify the creation of AI apps, open source projects have emerged. Semantic Kernel is Microsoft's contribution to this space and is designed to support enterprise app developers who want to integrate AI into their existing apps. - -By using multiple AI models, plugins, and memory all together within Semantic Kernel, you can create sophisticated pipelines that allow AI to automate complex tasks for users. -For example, with Semantic Kernel, you could create a pipeline that helps a user send an email to their marketing team. With memory, you could retrieve information about the project and then use planner to autogenerate the remaining steps using available plugins (e.g., ground the user's ask with Microsoft Graph data, generate a response with GPT-4, and send the email). Finally, you can display a success message back to your user in your app using a custom plugin. - -![ImageSK](assets/kernel-flow.png) - -Additionally, Semantic Kernel makes it easy to add skills to your applications with AI plugins that allow you to interact with the real world. These plugins are composed of prompts and native functions that can respond to triggers and perform actions. In this way, plugins are like the "body" of your AI app. - -Because of the extensibility Semantic Kernel provides with connectors and plugins, you can use it to orchestrate AI plugins from both OpenAI and Microsoft on top of nearly any model. For example, you can use Semantic Kernel to orchestrate plugins built for ChatGPT, Bing chat, and Microsoft 365 Copilot on top of models from OpenAI, Azure, or even Hugging Face. - -In a nutshell it will allow you to orchestrate AI plugins from any provider. - -## Running this workshop as a challenge - -Are you running this workshop in the form of a challenge? Either if you're doing it bt teams or individually, here is a proposal for counting points along the way with the challenges. - -There is very precise rules for challenges: -- challenges are composed of different **steps** that weight **1 point** each -- when **all the steps** of a challenge are completed, **5 more points** will be awarded -- if you complete the **bonus steps** of the challenges, it will give you **2 extra points** each - -But that's not it ! - -In addition, during the challenges some extra points may be awarded by your beloved coach regarding the **creativity** of you application, its **humor** or **hallucinativity**. Each of those item weighting **2 additional points**. - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Team nameChallenge 1Challenge 2Challenge 3Challenge 4Challenge 5Coach BonusTotal Score
scorebonusscorebonusscorebonusscorebonusscorebonus
Team1
Team2
Team3
- ---- - -# Prerequisites - -In just a few steps, you can start running the getting started guides for Semantic Kernel in either C# or Python. After completing the guides, you'll know how to... -- Configure your local machine to run Semantic Kernel -- Run AI prompts from the kernel -- Make AI prompts dynamic with variables -- Create a simple AI agent -- Automatically combine functions together with planners -- Store and retrieve memory with embeddings - -Before running the guides in C#, make sure you have the following installed on your local machine. -- `git` or the GitHub app -- VSCode or Visual Studio -- An OpenAI key -- NodeJS and yarn to run the sample apps of the Semantic Kernel - -If you prefer to do the Workshop in C#: -- .Net 7 SDK - for C# notebook guides -- In VS Code the Polyglot Notebook - for notebook guides - -If you are using Python, you just need python. - ---- - - -# Challenge 1: Azure OpenAi Playground - -## Prerequisites - -- An Azure account with an active subscription. [Create an account for free](https://azure.microsoft.com/free/?WT.mc_id=academic-0000-jabenn). -- An Azure OpenAI Services Instance. [Create an instance](https://learn.microsoft.com/en-us/azure/ai-services/openai/how-to/create-resource?pivots=web-portal). -- Some models deployments. [Deploy a model](https://learn.microsoft.com/en-us/azure/ai-services/openai/how-to/create-resource?pivots=web-portal#deploy-a-model). - - -To access the Azure OpenAI portal, go to [Azure OpenAI portal](https://oai.azure.com/) or by clicking on the link on your Azure OpenAI instance on the Azure portal. -![NavToPortal](assets/nav-to-portal.png) - - -Once in the Azure OpenAI portal, you'll find everything to take the most advantage of the service: -- You'll be able to manage your **models and your deployments (1)**. -- You will have interractive playgrounds to customize and test your models with the **playgrounds (2)** -- You will find very usefull resources like **samples (3)** to accelerate your development - -![Oai Portal](assets/oai-portal.png) - -
- -> The models avalailable are not only OpenAI models but a lot of other models from different providers. -You can deploy models like Ada (text embeddings), GPT 3.5 and 4, Davinci, Dall-E, and others models from providers or your own models in the future... -

-For the upcomming Hands-on-lab we recommend that you deploy at least a **GPT model** and a **Ada text embeddings model**. - -
- -## Step 1: Set Up a Conversational Agent - -In the Azure OpenAI Studio, open the Chat playground by clicking on the link in the left pane. -![Chat Playground](assets/chat-playground.png) -

-In this playground you have 3 main panels: -- **Assistant Setup**: you define your assistant goal, and how it should behave. There is a few sample available to help you get started, give it a try to start understanding how it works. This is also in this panel that you will train your assistant with you own data but we'll see that later. -- **Chat**: this is the place where you can test your assistant. You can type your question and see the answer from your assistant. -- **Configuration**: this is where you can define the parameters of you assistant regarding the deployed model you use. You can select the deployment and define the parameters you send to the model depending on what kind of result you want to get. - -There's a few parameters that will have big impact on the kind of answer that will be generated. Keep in mind that the two most important parameters are the **temperature** and the **max response** for you to define: -- **Temperature**: The higher the temperature, the more random the response. The lower the temperature, the more predictable the response. The default temperature is 0.7. **Top P** is an alternative to temperature that sets the probability that the model will choose from the most likely tokens. The default is 1.0. -- **Max response**: The maximum number of tokens to generate. The default is 64. - -
- -Try to play with the parameters and see how it impacts the answer. - -## Step 2: Build, Train and Deploy you assistant with your own data - -Now that you started building your assistant, you can start training it with your own data. As the OpenAI service is a constantly evolving service, the way you can train your assistant can change over time. - -You will find a step by step guide on how to train your assistant with your own data from the playground [here](https://learn.microsoft.com/en-us/azure/ai-services/openai/use-your-data-quickstart). - - -## Validate the challenge: - -In order to validate the challenge, you should be able to: -- demonstrate a working conversational agent that can answer questions based on your own data. -- ***Bonus***: demonstrate a deployed version of your agent (Webapp or Power Virtual Agent) - ---- - -# Challenge 2: Semantic Kernel Samples - - -We are about to embark on a remarkable journey. However, instead of a DeLorean, our vehicle will be our collective creativity, innovation and the semantic Kernel. - -As we buckle up and ignite our flux capacitors for this workshop, let's channel the spirit of the future and let's dive into the world of innovation and creativity. - -## Prerequisites - -It's time to play with Semantic Kernel! Let's start by cloning the repository and open it in VS Code (or your favorite IDE). - -```bash -git clone https://github.com/microsoft/semantic-kernel.git -cd semantic-kernel -code . -``` - -You'll find a few folders containing everything you need to build AI Apps with semantic kernel in this repository: -- **/docs**: you'll find all the documentation you need to get started with Semantic Kernel -- **/samples**: you'll find a few samples to get started with Semantic Kernel -- **/dotnet , /python, ...**: you'll find the source code of Semantic Kernel in the given language and a bunch of samples and skills for each language -- **/*.md files**: you'll find this documentation that will help you get started with Semantic Kernel, explaining concepts and how to run the samples - - -## Simple Chat Summary sample - -Let's start by the most simple sample: the Simple Chat Summary sample. This sample is a simple chatbot that will summarize the conversation. -You'll find it in the **samples/apps** folder. - -Take your time to build and run this sample and explore the skills and the code. You'll find a lot of comments in the code to help you understand how it works. - -Your goal here is simply to run the sample and try to understand how it works. Once you're pretty confident with it, you can go to the next step: the **Book Generator sample** to start seeing another powerful feature of Semantic Kernel: the planner, that helps you orchestrate skills together to build more complex scenarios. - -## Book Generator sample - -The power of semantic kernel is the ability to orchestrate skills together to build more complex scenarios. You can create static plans by assembling skills together, but you can also leverage the power of semantic kernel to dynamically create plans based on the context of the conversation. - -In this sample, you'll find a simple book generator that will generate a book based on the conversation. You'll find it in the **samples/apps** folder. Try it and see how it works. - -## Validate the Challenge - -In order to validate the challenge, you should be able to: -- Demonstrate the Sample "Chat summary" running -- Demonstrate the Sample "Book Generator" running -- ***Bonus***: Now that you have an app that can generate book for childrens, try to add a call to the Azure OpenAI Dall-E endpoint that will generate a book cover based on the story generated. Try adding this to the react app first, using the ***Javascript Azure OpenAI Client***. You'll be able to integrate it as a Skill later on this workshop. - -
- -> If you don't have a Dall-E deployment and you're running this Hands-On-Lab in a proctored session, ask to your proctor to provide a Dall-E endpoint for you. - -
- ---- - - -# Challenge 3 : build your first plugins - -## Step 1 : define a semantic function and a native function - -Semantic Kernel makes the difference between **semantic functions** and **native functions**. Semantic functions make use of a prompt to call a Large Language Model (LLM). Native functions don't need LLMs and can be written in C#. - -The full power of Semantic Kernel comes from combining semantic and native functions. - -
- -> To learn more about semantic and native functions, and how you can combine them, you can read the Semantic Kernel documentation [here](https://learn.microsoft.com/en-us/semantic-kernel/) - -
- -### Step 1 goals - -1 - **Write a semantic function** that generates an excuse email for your boss to avoid work and watch the next ***[your favorite team and sport]*** game. The function takes as input the day and time of the game, which you provide manually. -The function generates: - - the body of the email - - its topic - - its recipient. - -2 - **Write a native function** that calls a REST API (e.g. Bing search) to automatically retrieve the day and time of the next ***[your favorite team and sport]*** game in order to be integrated in the email. - - -
- -> If you don't have a REST API to retrieve the date and time of the next ***[your favorite team and sport]*** game, why not trying using Bing Chat or Github Copilot to help you build a solution to retrieve it. - -
- - -### Step 1 useful concepts and syntax - -Functions (both native and semantic) are grouped into plugins. In other words, a plugin is a collection of functions. Some plugins are already defined by the Semantic Kernel package. See the source code [here](https://github.com/microsoft/semantic-kernel/tree/main/dotnet/src/Plugins). You can use these plugins in your code with the following syntax: -```csharp -var timePlugin = new Timeskill(); // instantiate an out-of-the-box plugin -var daysAgo = time.DaysAgo; // instantiate a native function of this plugin -``` - -In the above snippet, Timeskill is the name of the plugin and DaysAgo is the name of one of its functions. - -Other plugins are defined by developers. In the semantic kernel repository, you will find examples of such plugins [here](https://github.com/microsoft/semantic-kernel/tree/main/samples/plugins). If you want to use these plugins in your solution, you have to copy paste them into your codebase. You can also create your own plugins. The documentation explains how to [organise your plugins folder](https://learn.microsoft.com/en-us/semantic-kernel/ai-orchestration/plugins/native-functions/using-the-skfunction-decorator?tabs=Csharp#finding-a-home-for-your-native-functions). Note that within a plugin, each semantic function has its own folder whereas native functions are all defined in a single C# file. Take a look at the [documentation](https://learn.microsoft.com/en-us/semantic-kernel/ai-orchestration/plugins/semantic-functions/serializing-semantic-functions?tabs=Csharp) to see how to organize your folders to create a semantic function. - -To use a developer defined plugin, you can use the following syntax: -```csharp -var emailPlugin = kernel.ImportSemanticSkillFromDirectory("./Plugins", "Email"); -var generateExcuse = emailPlugin["GenerateExcuse"]; -``` -In the above snippet, `Plugins` is the name of the folder containing all the developer defined plugins. `Email` is the name of the folder containing the email plugin. `GenerateExcuse` is the name of the folder that defines the semantic function that generates an excuse email. `GenerateExcuse` contains two files: `config.json` and `skprompt.txt`. - -It is possible to parametrize semantic functions by adding a parameter in `config.json`: -```json -{ - "input": { - "parameters": [ - { - "name": "myParameter", - "description": "", - "defaultValue": "" - } - ] - } -} -``` -and giving the parameter in `skprompt.txt`: -```txt -this prompt depends on the following parameter {{$myParameter}}. -``` - -The value of a parameter can be defined in the main program through context variables: -```csharp -var contextVariables = new ContextVariables - { - ["myParameter"] = "myValueForThisExecution", - }; -``` - -## Step 2 : chain functions - -### Step 2 goal - -Integrate your semantic and native functions in order to generate your email excuse. For now we encourage you to do the integration by writing native code. In the next part, we will see how **planners** leverage LLMs to deal with the orchestration of functions. - -### Step 2 concepts and syntax - -A semantic function must be called through a kernel: -```csharp -var result = await kernel.RunAsync(contextVariables, mySemanticFunction); -``` - -A kernel can be instantiated as follows: -```csharp -var builder = new KernelBuilder(); -builder.WithAzureChatCompletionService( - // Azure OpenAI Deployment Name, - // Azure OpenAI Endpoint, - // Azure OpenAI Key, - ); -var kernel = builder.Build(); -``` - -A native function can be called directly from the main program: -```csharp -var timePlugin = new Timeskill(); // instantiate an out-of-the-box plugin -var daysAgo = time.DaysAgo; // instantiate a native function of this plugin -var dateInAWeek = daysAgo(-7); // call a native function -``` - -## Validate the challenge - -In order to validate the challenge, you should: -- demonstrate your plugins by generating an email excuse and retrieving the date of the next ***[your favorite team and sport]*** game. -- demonstrate a manual orchestration that uses function chaining between semantic and native functions to achieve the goal of generating an email excuse. -- ***Bonus***: find the way to pass multiple variables thru the chain of functions using context. -- ***Bonus***: use a native function that calls a REST API to retrieve the date of the next ***[your favorite team and sport]*** game. - ---- - -# Challenge 4 : plugins orchestration - -## Orchestrate functions with a Planner - -### Goal - -Leverage a planner to orchestrate the previously written semantic and native functions: date retrieval, email generation. - -### Concepts and syntax - -Planners use LLMs to orchestrate the usage of semantic and native functions. At step 2, we integrated the email generation and API calls with native code. By using a planner, we can rely on an LLM to integrate semantic and native functions in order to achieve a goal. It's a declarative way of programming. - -The planner is given the goal that the developper wants to achieve and the functions (semantic and native) that are available to him. The planner leverages the natural language understanding and generation capabilities of LLMs to propose a plan, i.e. a step by step usage of the available functions to achieve the goal. The proposed plan is then executed. - -In Semantic Kernel, the 3 following planners are available: - - Action Planner: chooses a single function in order to achieve the given goal. - - Sequential Planner: chains several functions (semantic and native) to achieve the given goal. - - Stepwise Planner: iteratively and adaptively produces a plan: the next step may depend on the result of the execution of the previous steps. - -For a planner to know which plugins it can use to generate a plan, plugins must be registered to a kernel. - -Developer-defined plugins are registered with the syntax that we already used previously: -```csharp -var emailPlugin = kernel.ImportSemanticSkillFromDirectory("./Plugins", "Email"); -var generateExcuse = emailPlugin["GenerateExcuse"]; -``` - -For an out-of-the-box plugin, you can use the following syntax to register it to a kernel: -```csharp -kernel.ImportSkill(time); -``` - -To use the sequential planner, you can use the following syntax: -```csharp -var planner = new SequentialPlanner(kernel); // instantiate a planner -var plan = await planner.CreatePlanAsync("ask for the goal to achieve"); // create a plan -var result = await plan.InvokeAsync(); // execute the plan -``` -Other planners (e.g. ActionPlanner and StepwisePlanner) follow a similar syntax. - - -### Validate the challenge - -In order to validate the challenge, you should demonstrate: -- a planner orchestration of semantic and native functions to achieve the goal of generating an email excuse. -- ***Bonus***: using a stepwise planner that can adaptively generate a plan depending on the result of retrieving the date of the next ***[your favorite team and sport]*** game. - - ---- - -# Extra-challenge : add nice-to-haves - -Feel free to add any nice-to-have feature that you think makes sense. - -## Examples of goals to achieve - -- Add memory : keep track of the previuosly sent excuses to make sure that you don't use the same excuse twice. See [Semantic Memory](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/KernelSyntaxExamples/Example14_SemanticMemory.cs) for a sample. - -- Add telemetry : See [ApplicationInsights](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/ApplicationInsightsExample/Program.cs) for a sample. - -- Build a nicer user interface. - -- ... diff --git a/workshops/swa-cosmosdb/assets/Function_execute.png b/workshops/swa-cosmosdb/assets/Function_execute.png deleted file mode 100644 index ba9e4ade..00000000 Binary files a/workshops/swa-cosmosdb/assets/Function_execute.png and /dev/null differ diff --git a/workshops/swa-cosmosdb/assets/UserId_inspect.png b/workshops/swa-cosmosdb/assets/UserId_inspect.png deleted file mode 100644 index 2dd895cc..00000000 Binary files a/workshops/swa-cosmosdb/assets/UserId_inspect.png and /dev/null differ diff --git a/workshops/swa-cosmosdb/assets/banner.jpg b/workshops/swa-cosmosdb/assets/banner.jpg deleted file mode 100644 index 9c87e96e..00000000 Binary files a/workshops/swa-cosmosdb/assets/banner.jpg and /dev/null differ diff --git a/workshops/swa-cosmosdb/assets/connection-string.png b/workshops/swa-cosmosdb/assets/connection-string.png deleted file mode 100644 index c97abdce..00000000 Binary files a/workshops/swa-cosmosdb/assets/connection-string.png and /dev/null differ diff --git a/workshops/swa-cosmosdb/assets/create-db.png b/workshops/swa-cosmosdb/assets/create-db.png deleted file mode 100644 index 5993e892..00000000 Binary files a/workshops/swa-cosmosdb/assets/create-db.png and /dev/null differ diff --git a/workshops/swa-cosmosdb/assets/dev-container.png b/workshops/swa-cosmosdb/assets/dev-container.png deleted file mode 100644 index 080c5d46..00000000 Binary files a/workshops/swa-cosmosdb/assets/dev-container.png and /dev/null differ diff --git a/workshops/swa-cosmosdb/assets/finish.png b/workshops/swa-cosmosdb/assets/finish.png deleted file mode 100644 index 1cecbe78..00000000 Binary files a/workshops/swa-cosmosdb/assets/finish.png and /dev/null differ diff --git a/workshops/swa-cosmosdb/assets/function_local_run.png b/workshops/swa-cosmosdb/assets/function_local_run.png deleted file mode 100644 index eb364589..00000000 Binary files a/workshops/swa-cosmosdb/assets/function_local_run.png and /dev/null differ diff --git a/workshops/swa-cosmosdb/assets/github-actions.png b/workshops/swa-cosmosdb/assets/github-actions.png deleted file mode 100644 index 4c37662a..00000000 Binary files a/workshops/swa-cosmosdb/assets/github-actions.png and /dev/null differ diff --git a/workshops/swa-cosmosdb/assets/githubtemplate.png b/workshops/swa-cosmosdb/assets/githubtemplate.png deleted file mode 100644 index 0b8ce69a..00000000 Binary files a/workshops/swa-cosmosdb/assets/githubtemplate.png and /dev/null differ diff --git a/workshops/swa-cosmosdb/assets/login_window.png b/workshops/swa-cosmosdb/assets/login_window.png deleted file mode 100644 index 8ddfe54a..00000000 Binary files a/workshops/swa-cosmosdb/assets/login_window.png and /dev/null differ diff --git a/workshops/swa-cosmosdb/assets/resource-overview.png b/workshops/swa-cosmosdb/assets/resource-overview.png deleted file mode 100644 index 59a0fded..00000000 Binary files a/workshops/swa-cosmosdb/assets/resource-overview.png and /dev/null differ diff --git a/workshops/swa-cosmosdb/assets/swa-github.png b/workshops/swa-cosmosdb/assets/swa-github.png deleted file mode 100644 index 1a270ad5..00000000 Binary files a/workshops/swa-cosmosdb/assets/swa-github.png and /dev/null differ diff --git a/workshops/swa-cosmosdb/assets/swa-tips-video-environment.png b/workshops/swa-cosmosdb/assets/swa-tips-video-environment.png deleted file mode 100644 index fe2d06b0..00000000 Binary files a/workshops/swa-cosmosdb/assets/swa-tips-video-environment.png and /dev/null differ diff --git a/workshops/swa-cosmosdb/assets/swa-workshop-final.zip b/workshops/swa-cosmosdb/assets/swa-workshop-final.zip deleted file mode 100644 index 7f34e2a9..00000000 Binary files a/workshops/swa-cosmosdb/assets/swa-workshop-final.zip and /dev/null differ diff --git a/workshops/swa-cosmosdb/assets/todo-banner.jpg b/workshops/swa-cosmosdb/assets/todo-banner.jpg deleted file mode 100644 index 79746ec0..00000000 Binary files a/workshops/swa-cosmosdb/assets/todo-banner.jpg and /dev/null differ diff --git a/workshops/swa-cosmosdb/assets/todo.png b/workshops/swa-cosmosdb/assets/todo.png deleted file mode 100644 index a96f4223..00000000 Binary files a/workshops/swa-cosmosdb/assets/todo.png and /dev/null differ diff --git a/workshops/swa-cosmosdb/assets/userid-copy.png b/workshops/swa-cosmosdb/assets/userid-copy.png deleted file mode 100644 index 16323dfa..00000000 Binary files a/workshops/swa-cosmosdb/assets/userid-copy.png and /dev/null differ diff --git a/workshops/swa-cosmosdb/workshop.md b/workshops/swa-cosmosdb/workshop.md deleted file mode 100644 index afe5c3a4..00000000 --- a/workshops/swa-cosmosdb/workshop.md +++ /dev/null @@ -1,878 +0,0 @@ ---- -short_title: Static Web Apps Workshop -description: Learn how to develop an end to end web application with a frontend, a backend, a database and some user authentication using Azure Static Web Apps, Cosmos DB, VSCode and GitHub Actions. -type: workshop -authors: - - Olivier Leplus - - Yohan Lasorsa - - Rohit Turambekar -contacts: - - '@olivierleplus' - - '@sinedied' - - '@rohit2git' -banner_url: assets/todo-banner.jpg -duration_minutes: 180 -audience: students, pro devs -level: beginner -tags: azure, static web apps, cosmos db, github actions, vs code -published: true -sections_title: - - Introduction ---- - -# Develop and deploy your fullstack web app with Azure Static Web Apps and Cosmos DB - -In this workshop, you will learn how to create a full application with a `frontend`, a `serverless backend`, `user authentication` and a `database` to store your data. - -Don't worry if you are not familiar with Microsoft Azure. This workshop will walk you through some of the most important services if you are a web developer. We will see how to use `Cosmos DB` with the `MongoDB` API, `Azure Static Web Apps` to host your client application and `Azure Functions` for your backend and `GitHub Actions` to automate your app deployment! - -At the end of this workshop, you will have a full understanding on how to develop and deploy a simple web app on Azure. - -### Prerequisites - -To do this workshop, you will need: -* Basic JavaScript knowledge -* [A Microsoft Azure account](https://azure.microsoft.com/free/) -* [A GitHub account](http://github.com/) -* [Visual Studio Code](https://code.visualstudio.com/) (VSCode) -* [Node.js 20 installed](https://nodejs.org/) - - ---- - -## What's Azure Static Web Apps? - -Azure Static Web Apps is a service that enables developers to deploy their web applications on Azure in a seamless way. This means developers can focus on their code and spend less time administering servers! - -So, with Azure Static Web Apps (aka SWA), developers can not only deploy frontend static apps and their serverless backends but they can also benefit from features like `Custom domains`, `pre-prod environments`, `authentication providers`, `custom routing` and more. - -### Who is it for? -Azure Static Web Apps is for every developer who wants to spend more time in their code editor than they do managing resources in the cloud. - -If you are a so-called "Full stack developer", then you probably want to deploy both your frontend and your backend to the cloud, ideally with limited administration and configuration management. - -### Front end developers -Azure Static Web Apps supports many frameworks and static site generators out of the box. If you are using a framework like `Angular`, `React`, `Vue.js`, a site generator like `Gatsby`, `Hugo` or one of the many solutions Azure Static Web Apps supports, then you don't have to take care of the deployment. If you have a specific need to customize the build for your app, you can configure it yourself very easily! - -### Backend developers -Azure Static Web Apps relies on `Azure Functions` for your application backend. So, if you are developing in `JavaScript`, `Java`, `Python` or `.NET`, SWA makes it very easy to deploy your backend as well! - -
- -> You can find the official Static Web Apps documentation here: [learn.microsoft.com/azure/static-web-apps/](https://learn.microsoft.com/azure/static-web-apps/) - -
- -Oh, and did I forget to mention there is a Free tier for Static Web Apps? You can start using it for free and only pay once your application gets popular! - ---- - -## Start with a website - -Once upon a time, there was a website that needed a place to live, be visible to the world and have a backend to be more interactive. - -Go to [this GitHub repository](https://github.com/themoaw/swa-workshop) and click on `Use this template`. - -![Create from template](assets/githubtemplate.png) - -You will be redirected to the repository creation page. Enter a name for your new repository, set the repo visibility to `public` and click on `Create repository from template`. - -Once the repository is created, clone it locally using git. - -You now have your baseline project. Open your repository folder in VSCode. - ---- - -## Create an Azure Static Web App - -Start by opening the [Create Static Web App form](https://portal.azure.com/#create/Microsoft.StaticApp) in the Azure Portal. This link will take you directly to the Static Web App creation form. You may be prompted to log into your Azure Subscription. If you don't have one you can create a Free trial. - -Let's fill it out! - -* Select your Subscription. -* Create a new Resource Group. - -
- -> In Azure, a Resource Group is a logical structure that holds resources usually related to an application or a project. A Resource Group can contain virtual machines, storage accounts, web applications, databases and more. - -
- -* Give a Name to your Static Web App. -* Select the Free Plan (we won't need any feature in the Standard Plan for this workshop). -* Select `West Europe` for your backend. - -
- -> It's recommended to host your backend in a Region closest to your users. - -
- -* Select `GitHub` as a deployment source. - -We are going to host our website source code on GitHub. Later in the workshop, we will see how Static Web Apps will automaticaly deploy our website every time we push new code to our repository. - -* Sign in with your GitHub account and select the repository and the branch of your project. - -As we mentioned at the beginning of the workshop, our app will have a backend and a frontend. In order for Static Web App to know what to deploy and where, we need to tell it where our apps are located in our repository. - -Azure Static Web Apps can handle several well-known frontend frameworks and can "compile" your Angular, React or Hugo application before deploying them. - -In our case, we have a very simple JavaScript application which does not require anything to run. So, let's choose `Custom`. -* In the `App location`, enter the `/www` folder as this is where our frontend is. -* In the `Api location`, enter the `/api` folder as this is where our backend is. -* In the `Output`, enter the `/www` folder as your frontend does not need any build system to run. - -![Enter GitHub information when creating SWA](assets/swa-github.png) - -* Click on `Review + Create` and then on `Create`. - -After a few minutes, your Static Web App will be created on Azure and your website deployed. - -Once the resource is created, you should `pull` your repository in VSCode as a few files have been added to your GitHub repo by Azure. - -### So, what just happened? - -#### On GitHub - -When Azure created your Static Web App, it pushed a new YAML file to the `.github/workflow` folder of your repository. - -The files in this folder describe GitHub Actions, which are event-based actions that can be triggered by events like a `push`, a `new pull request`, a `new issue`, a `new collaborator` and more. - -You can see the complete list of triggers [here](https://docs.github.com/en/actions/reference/events-that-trigger-workflows) - -
- -> If you are not familiar with GitHub Actions, you can read about them [here](https://github.com/features/actions). - -
- -Let's have a look at the YAML file Azure created for us: - -```yaml -on: - push: - branches: - - main - pull_request: - types: [opened, synchronize, reopened, closed] - branches: - - main -``` - -Here, you can see that the GitHub Action is going to be triggered every time there is a `push` on the `main` branch or every time a Pull Request is `opened`, `synchronize`, `reopened` or `closed`. - -As we want our website to be redeployed automatically every time we push on our main branch, this is perfect! - -Take a few minutes to read the YAML file and understand what exactly happens when the GitHub Action is triggered. You can see that most of the information you entered when you created your Static Web App on Azure is here. - -
- -> The YAML file is in your GitHub repository so you can edit it! Your frontend site folder name changed? No problem, just edit the file and push it to GitHub! - -
- -In your folder find the YAML file `.github\workflows\.yml` and change the `actions/checkout@` section to adopt new nodejs version(i.e. 20). - -```yaml -steps: - - uses: actions/checkout@v3 -``` -to - -```yaml -steps: - - uses: actions/checkout@v4 -``` - -Save the changes and push the file in repo. - -Go to Terminal in VS Code and type below command in your working app directory - -```bash -git add . -git commit -m "changed the checkout action version" -git push -``` -This will trigger the GitHub Action to deploy the changes. - -Now, go to your GitHub repository in a web browser and click on the `Actions` tab. Here, you will see the list of all the GitHub Actions that have been triggered so far. Click on the last one to see your application being deployed. - -![Check your GitHub Actions](assets/github-actions.png) - -#### On Azure - -Once your Static Web App is created, go to the Resource page. You can find the list of all your Static Web Apps [here](https://portal.azure.com/#blade/HubsExtension/BrowseResource/resourceType/Microsoft.Web%2FStaticSites). - -In the Overview panel of your Static Web App, look for the `URL` parameter. This is the url of your website. - -![Resource overview of your project](assets/resource-overview.png) - -Open the link and you can see that your TODO list app has been deployed and is accessible to the world! - -Congratulations, you deployed your first Static Web App on Azure! 🥳 - ---- - -## Test your project locally - -There are two ways to test your project. You can either push your code on GitHub every time you need to test it (not recommended), or use the `Static Web Apps CLI`. - -### The Static Web App CLI - -The Static Web Apps Command Line Interface, also known as the `SWA CLI`, serves as a local development tool for Azure Static Web Apps. In our case, we will use the CLI to: - -* Serve static app assets -* Serve API requests -* Emulate authentication and authorization -* Emulate Static Web Apps configuration, including routing - -You can install the CLI via npm. - -```bash -npm install -g @azure/static-web-apps-cli -``` - -We are only going to use a few features of the CLI so if you want to become a SWA CLI expert, you can find all the features the CLI provides [here](https://github.com/Azure/static-web-apps-cli) - -### Run your project locally - -The CLI offers many options, but in our case we want it to serve both our API located in our `api` folder and our web application located in our `www` folder. - -In your terminal, type the following command to start your project: - -```bash -swa start www --open -``` - -A local web server will be started at [http://localhost:4280](http://localhost:4280) to run your app. - -
- -> Later on, you will need to add the option `--api-location api` to also start the Azure Functions API server. You'll be able to test your API at [http://localhost:7071/api/tasks](http://localhost:7071/api/tasks). -> -> The CLI may take more time than usual to launch your Azure Functions, especially the first time as it may need to download the needed tools. - -
- -Congratulations, you now have everything you need to test your app on your computer! 🥳 - ---- - -## Let's add a backend - -Now that our TODO frontend is deployed, we want to make it interactive and need to add a backend! - -Azure Static Web Apps relies on Azure Functions for your application backend. Azure Functions is an Azure service which enables you to deploy simple code-based microservices triggered by events. In our case, events will be HTTP requests. - -
- -> Ever heard of Serverless or FaaS (Function as a Service)? Well, you get it, this is what Azure Functions is ^^. - -
- -### Installation - -You can create an Azure Function from the [Azure portal](https://portal.azure.com/) but let's be honest, it's so much easier to stay in VSCode and use the Azure Functions extension. - -So, start by installing the Azure Function extension from VSCode. - -You can download the extension either directly from the `Extension panel (Ctrl + Shift + X)` in VSCode or by going [here](https://marketplace.visualstudio.com/items?itemName=ms-azuretools.vscode-azurefunctions) and clicking on the `Install` button. - -Install the [Azure Functions extension](https://marketplace.visualstudio.com/items?itemName=ms-azuretools.vscode-azurefunctions&WT.mc_id=javascript-76678-cxa) v1.10.4 or above for Visual Studio Code. This extension installs [Azure Functions Core Tools](https://learn.microsoft.com/en-us/azure/azure-functions/functions-run-local) for you the first time you locally run your functions. - -> Functions runtime v4 requires version 4.0.5382, or a later version of Core Tools. - -Go to VSCode terminal or windows cmd and check the version of Core tools as - -```bash -func -v -> 4.0.5801 -``` - -### Create your Function app - -Azure Functions live in an Azure Functions App. When you create your Function App, a single Azure Function will be created by default. You can then add more Functions as required. - -So, let's create our Functions App and a Function to retrieve our task list for our TODO frontend. - -* In VSCode, open the Command panel using `Ctrl + Shift + p` and search for `Azure Functions: Create new project`. -* Select the `api` folder. This is where our Function App will be created. -* Choose `JavaScript` as this is the language we are going to use to write our Function. - -
- -> Azure Static Web Apps don't support all the languages you can develop Azure Functions with. Supported backends can be developed using JavaScript, TypeScript, Python or C#. - -
- -* Select `Model v4`. -* As we are creating a REST API that will be called by our website, the trigger we are looking for is the `HTTP trigger`. -* Our first Function will be used to retrieve our task list. Let's call it `tasks-get`. - - -
- -> If you want to learn more about the different authorization levels for Functions and how to secure your API, check out the docs [here](https://learn.microsoft.com/azure/azure-functions/security-concepts). - -
- -A function will be created for you so you don't start with a blank project. Let's modify the scaffold for our needs. - -Right now, we don't have a database to store our users or our tasks so let's use an array as a "fake" database. - -```json -const tasks = [ - { - id: 1, - label: "🍔 Eat", - status: "" - }, - { - id: 2, - label: "🛏 Sleep", - status: "" - }, - { - id: 3, - label: " Code", - status: "" - } -]; -``` - -
- -> Modify the Azure Function (api\src\functions\tasks-get.js) so it returns the list of tasks. - -
- -Once deployed, you will be able to call your function like any REST API. - -### Configure your endpoints - -By default, when an Azure Function is created using the VSCode extension, the function will support `GET` and `POST` requests and the URL will end with your function name. In our case, the `tasks-get` function can either be called using -* `GET https://your-website-url.com/api/tasks-get` -* `POST https://your-website-url.com/api/tasks-get` - -This is not exactly what we need. - -As we only need our Function to retrieve a list of tasks, let's remove the `POST` method in the `methods` array from the file `api\src\functions\tasks-get.js` so our Function can only be called using `GET` requests. - -```javascript -... -app.http('tasks-get', { - methods: ['GET'], - authLevel: 'anonymous', -... -``` - -The other thing you may want to change is the URL of your Function. Having a route called `/api/tasks-get` is not very standard. You can easily change your endpoint name in the `api\src\functions\tasks-get.js` file by adding a `route` parameter as below. - -```javascript -app.http('tasks-get', { - methods: ['GET'], - authLevel: 'anonymous', - route: 'tasks' -``` - -Now, your Function is only accessible using a `GET /api/tasks` request. - -### Run the function locally - -Visual Studio Code integrates with Azure Functions Core tools to let you run this project on your local development computer before you publish to Azure. - -1. To start the function locally, press `F5` or the **Run and Debug** icon in the left-hand side Activity bar. - -If you get the Storage pop-up message, select the local Emulator option. - -The **Terminal** panel displays the Output from Core Tools. Your app starts in the **Terminal** panel. You can see the URL endpoint of your HTTP-triggered function running locally. - -![Function Run terminal](assets/function_local_run.png) - -If you have trouble running on Windows, make sure that the default terminal for Visual Studio Code isn't set to **WSL Bash**. - -2. With Core Tools still running in Terminal, choose the Azure icon in the activity bar. In the **Workspace** area, expand **Local Project > Functions**. Right-click (Windows) or Ctrl - click (macOS) the new function and choose **Execute Function Now....** - -![Execute function](assets/Function_execute.png) - -3. In **Enter request body** you see the request message body value of `{ "name": "Azure" }`. Press Enter to send this request message to your function. - -4. When the function executes locally and returns a response, a notification is raised in Visual Studio Code. Information about the function execution is shown in **Terminal** panel. - -5. With the **Terminal** panel focused, press `Ctrl + C` to stop Core Tools and disconnect the debugger. - -You've done it! You wrote your first Azure Function. Congratulations! 🥳 - ---- - -## Add authentication - -Azure Static Web Apps manages authentication out of the box. There are pre-configured providers and you can add you own custom providers if needed. Among the pre-configured ones are `Twitter` are `GitHub` - -
- -> If you are using the CLI, you won't really be connecting to the selected provider. The CLI offers a proxy to simulate the connection to the provider and gives you a fake userId. - -
- -### Sign in & Sign out - -When I said "out of the box", I really meant it. You don't need to do anything for most of the providers. Let's use the GitHub one for our application. The only thing you will have to do is add a button in your frontend that redirects to `/.auth/login/github`. - -
- -> Update the Login button in the www/login.html so your users can sign in using GitHub. - -
- - -By default, once logged in, your users are redirected to the same page. However, we would like our users to be redirected to our TODO page after successfully logging in. You can do that by using the `post_login_redirect_uri` query param at the end of the url. -Eg. `?post_login_redirect_uri=/index.html` - -Change the login button code as below - -```html - - -``` -Then, navigate to [http://localhost:4280/login.html](http://localhost:4280/login.html) to test. - -
- -> If you are building a React app, go [check the Microsoft Learn module](https://learn.microsoft.com/learn/modules/publish-static-web-app-authentication/) that will show you how to do it. - -
- -### Getting user information - -Once your user is authenticated, you can retrieve the user's information by fetching the url `/.auth/me`. This will return some JSON containing a clientPrincipal object. If the object is null, the user is not authenticated. Otherwise, the object contains data like the provider, the roles and the username. - -```json -{ - "identityProvider": "github", - "userId": "d75b260a64504067bfc5b2905e3b8182", - "userDetails": "", - "userRoles": ["anonymous", "authenticated"] -} -``` -The `userId` is unique and can be used to identify the user. We will use it later in our database. - -
- -> Complete the `getUser()` method in `www/app.js` to retrieve the logged-in user information and display the username in the `
- -> You need to restart the CLI each time your make a change in the staticwebapp.config.json file. - -
- -### Secure our website and APIs - -There are many properties available to configure your Static Web App but let's concentrate only on the few we need in our app. - -The `routes` parameter is an array of all the rules for your routes. For example, in our case, we would like to prevent unauthenticated users from calling our API. We can do that very easily. - -```json -{ - "routes": [ - { - "route": "/api/tasks/*", - "allowedRoles": [ - "authenticated" - ] - } - ] -} -``` - -What if we want to restrict a part of our frontend to authenticated users? We can do exactly the same with any frontend route. - -```json -{ - "routes": [ - { - "route": "/", - "allowedRoles": [ - "authenticated" - ] - } - ] -} -``` - -Now you have made this change your website root will only be accessible to logged in (authenticated) users. - -### Manage HTTP error codes - - You will notice that your unauthenticated users are redirected to a default 401 (HTTP Unauthorized) web page. We can customize that using another property in the config file. The `responseOverrides` property enables you to redirect a user to a specific page when an HTTP code is returned. Let's redirect all non-authenticated users to the `login.html` page. - -```json -{ - "responseOverrides": { - "401": { - "rewrite": "/login.html" - } - }, -} -``` - -Here, we simply tell our Static Web App to redirect every 401 response to the `login.html` page. - -
- -> Create a `custom-404.html` page in your www folder and add a rule to redirect users to this page when they enter a URL which does not exist (HTTP error code 404). - -
- -So final file `staticwebapp.config.json` should be look like below -```json -{ - "routes": [ - { - "route": "/", - "allowedRoles": [ - "authenticated" - ] - }, - { - "route": "/api/tasks/*", - "allowedRoles": [ - "authenticated" - ] - } - ], - "responseOverrides": { - "401": { - "rewrite": "/login.html" - }, - "404": { - "rewrite": "custom-404.html" - } - } -} -``` - -Now, try to go to a non-existent page on your website like `/hello.html`. You should be redirected to the `custom-404.html` page you just created. - -
- -> This is also very useful if you are doing a Single Page Application (SPA) where the routing is managed on the client. You may then need to redirect all your URLs to `index.html`. Check the `navigationFallback` property in the documentation [here](https://learn.microsoft.com/azure/static-web-apps/configuration). - -
- -Congratulations, your website and your APIs are now secured and you have a seamless workflow for your users. 🥳 - ---- - -## Store your data in a database - -There are several databases available on Azure. One of the most powerful ones is `Cosmos DB`. Azure Cosmos DB is a fully managed NoSQL database which supports several APIs. When you create your Cosmos DB Database, you can choose which API you want to use. Among the most popular ones are `MongoDB`, `SQL` and `Cassandra`. - -### Setup your dev environment - -Let's go back to our Azure Function in VSCode. We will be using the Cosmos DB MongoDB API, so we need a library to connect to our database. In the `/api` folder, open the `package.json` file and add `"mongodb": "^4.0.1"` to the `dependencies` property as shown below. - -```json -... - "dependencies": { - "mongodb": "^4.0.1" - }, -... -``` - -In a terminal, type `npm install` and hit enter. This will download the dependencies for you to the `node_modules` folder of your Azure Functions App. - -While you are in VSCode, let's install the `Azure Databases` extension. This will allow you to explore your database and make queries from VSCode without having to go to the Azure Portal. - -In the extension menu of VSCode, search for `Azure Databases` or [here](https://marketplace.visualstudio.com/items?itemName=ms-azuretools.vscode-cosmosdb) and click on `Install`. - -### Create your database - -Start by opening the [Create Azure Cosmos DB form](https://portal.azure.com/#create/Microsoft.DocumentDB) in the Azure Portal. For our application, we are going to use the `MongoDB` API so select `Azure Cosmos DB for MongoDB` and click `Create`. - -* Select `Request unit (RU) database account` and click `Create` -* Select your Subscription. -* Select the same Resource Group you used earlier in this workshop for Static Apps. -* Enter a name to identify your Cosmos DB Account. This name must be unique. -* Select the Location where your Account is going to be hosted. We recommend using the same location as your Static Web Apps. -* Availability Zones, keep them default i.e. `Disable` as this is for workshop. -* Select `Provisioned throughput` there and next ensure you select `Apply` for `Apply Free Tier Discount`. -* For `Version` keep the default i.e. `6.0`. -* Click on `Review + Create` and then on `Create`. - -Creating the Cosmos DB Accounts may take some time so grab a cup of hot chocolate and be ready to deal with Cosmos DB! - -☕ - -OK, let's go back to VSCode. In the Azure Extension, in Resources area, when you click on your Azure Subscription, you now have the `Database` tab where you should see the Cosmos DB Account you just created. If you don't see it, just click the refresh button. - -Once you see your Account, right click on it, select `Create Database` and enter a name(in VS Code top pallet). Then it will ask for Collection and name it `tasks` as it will be used to store our tasks. - -![Create a Cosmos DB database](assets/create-db.png) - -### Add some data - -Let's focus on our existing Azure Function. We will see later how to create a Function to add new tasks in our database. - -Right now, we just want to get our tasks from the database instead of the static JSON array we created earlier in our Function. - -In VSCode, right click on your `tasks` collection and select `Create Document`. To get the userId, just log into your web application(http://localhost:4280/.auth/me) and copy from userId field. - -![Get User ID from login](assets/userid-copy.png) - -Edit the generated document to include the below json and make sure to replace the **userId** by the one of your logged in user copied from earlier. - -```json -{ - "_id": { - "$oid": "" - }, - "userId": "", - "label": "Buy tomatoes", - "status": "checked" -} -``` - -Do it again for the two following tasks: Create new Document and edit to add below. In the end you will have 3 separate documents in `tasks` collection. - -```json -{ - "_id": { - "$oid": "" - }, - "userId": "", - "label": "Learn Azure", - "status": "" -} -``` - -```json -{ - "_id": { - "$oid": "" - }, - "userId": "", - "label": "Go to space", - "status": "" -} -``` - -### Let's code - -Now that we have our database set up and have added some data to it, let's make sure our user interface displays it! - -In your `tasks-get` Azure Function, start by importing the `mongoClient` from the MongoDB library we installed earlier. - -```javascript -const mongoClient = require("mongodb").MongoClient; -``` - -When your Static Web App calls the API, the user information is sent to the Function in the `x-ms-client-principal` HTTP header. - -You can use the code below to retrieve the same user JSON you get in the `clientPrincipal` property when you go to `/.auth/me`. - -```javascript -const header = request.headers.get('x-ms-client-principal'); -const encoded = Buffer.from(header, 'base64'); -const decoded = encoded.toString('ascii'); -const user = JSON.parse(decoded); -``` - -Let's see how the MongoDB API works: - -* First, your need to connect to your server (in our case our Cosmos DB Accounts) - -In order to connect your application to your database, you will need a connection string. - -You can find your server connection string in the Azure Portal. But, as always, you can stay in your VSCode. In the Azure Database extension, right click on your database server and select `Copy Connection String`. - -![Retrieve your Cosmos DB connection string](assets/connection-string.png) - -Add your connection string to your ```local.settings.json``` file -```javascript -"values": { - ... - "COSMOSDB_CONNECTION_STRING": "", - ... -} -``` - -Now, let's connect! - - -```javascript -const client = await mongoClient.connect(process.env.COSMOSDB_CONNECTION_STRING); -``` - -### Request your data - -Once you are connected to your Cosmos DB server using the MongoDB API, you can use this connection to select a database. - -```javascript -const database = client.db("YOUR_DB_NAME"); -``` - -Replace `YOUR_DB_NAME` by the name you entered when you created your database. - -Then, query the document where the userId property is the same as the userId sent in the headers when your Function is called. - -```javascript -const response = await database.collection("tasks").find({ - userId: user.userId -}); - -const tasks = await response.toArray(); -``` - -
- -> Update your Azure Function so it returns the tasks in the database associated to your logged in user. - -
- -**Final function should look like this** - -```javascript -app.http('tasks-get', { - methods: ['GET'], - authLevel: 'anonymous', - route: 'tasks', - handler: async (request, context) => { - - context.log(`Http function processed request for url "${request.url}"`); - - const header = request.headers.get('x-ms-client-principal'); - const encoded = Buffer.from(header, 'base64'); - const decoded = encoded.toString('ascii'); - const user = JSON.parse(decoded); - - const client = await mongoClient.connect(process.env.COSMOSDB_CONNECTION_STRING); - const database = client.db(""); - - const response = await database.collection("tasks").find({ - userId: user.userId - }); - - const tasks = await response.toArray(); - - return { - body: JSON.stringify(tasks), - headers: { - 'Content-Type': 'application/json' - } - } - } -}); -``` - -Test your website locally and push your changes to GitHub. The GitHub Action will be triggered and your website deployed. - -Last step, setting up environment variable for Static App for CosmosDB connection string. - -Go to portal, search for 'Static Web App' and click your app, the in left panel, select `Environment variables`. -Click `+ Add` and give the name `COSMOSDB_CONNECTION_STRING` and copy the value as we did earlier and Apply. - -Go check the public URL, you should see the tasks of your database. - -![Your TODO app up and running](assets/finish.png) - -If you are not able to see the tasks, see if the userId(from login data) is same as you saved earlier in CosmosDB document. This userId can be seen in browser, right click Inspect -> Network -> Me attribute. If they are not, then you have to update the user Id again in CosmosDB. -![Inspect the UserId](assets/UserId_inspect.png) - ---- - -## Bonus - -### Finish your application - -You now know how to create and publish an app with a frontend, a backend, a database and some user authentication. - -However, in order to make your TODO app fully functional, you need to add a few more features. The good news? You have everything your need to do it! - -#### Add a new task - -We have already added the source code to call the API in the frontend so all you need to do is to create the Azure Function and connect it to the database. - -
- -> Create one Azure Function to add a new task to the database. You don't need to create a new project, just create new Function in your existing Function App project in VSCode. - -
- -#### Update a task - -You may have noticed that there is a `status` attribute in every task in your database. The value can be either `""` or `"checked"`. Right now, there is no way to change this status. - -
- -> Write an Azure Function and the JavaScript code in your frontend to update this status. - -
- -### Monitor your app - -You now know how to create resources on Azure and how connection string work. - -You may have noticed that, once deployed, we don't have any logs for our app which means we have no way to know what happens if there are any issues. - -
- -> Create an [Application Insights](https://learn.microsoft.com/azure/azure-monitor/app/app-insights-overview) resource in your Resource Group and connect it to your Static Web App. - -
- ---- - -## Conclusion - -Congratulations, you've reached the end of this workshop! - - -### Solution - -You can download the completed app code with the features to add and update a task [here](assets/swa-workshop-final.zip). diff --git a/workshops/swa-gatsby-portfolio/assets/gatsby.svg b/workshops/swa-gatsby-portfolio/assets/gatsby.svg deleted file mode 100644 index 0fa69e68..00000000 --- a/workshops/swa-gatsby-portfolio/assets/gatsby.svg +++ /dev/null @@ -1,6 +0,0 @@ - - - - - - diff --git a/workshops/swa-gatsby-portfolio/assets/gh-git.png b/workshops/swa-gatsby-portfolio/assets/gh-git.png deleted file mode 100644 index c6500bee..00000000 Binary files a/workshops/swa-gatsby-portfolio/assets/gh-git.png and /dev/null differ diff --git a/workshops/swa-gatsby-portfolio/assets/portfolio-banner.jpg b/workshops/swa-gatsby-portfolio/assets/portfolio-banner.jpg deleted file mode 100644 index f6058419..00000000 Binary files a/workshops/swa-gatsby-portfolio/assets/portfolio-banner.jpg and /dev/null differ diff --git a/workshops/swa-gatsby-portfolio/assets/prerequisites/check-01.png b/workshops/swa-gatsby-portfolio/assets/prerequisites/check-01.png deleted file mode 100644 index a5b472ac..00000000 Binary files a/workshops/swa-gatsby-portfolio/assets/prerequisites/check-01.png and /dev/null differ diff --git a/workshops/swa-gatsby-portfolio/assets/prerequisites/check-02.png b/workshops/swa-gatsby-portfolio/assets/prerequisites/check-02.png deleted file mode 100644 index fc7d4d7f..00000000 Binary files a/workshops/swa-gatsby-portfolio/assets/prerequisites/check-02.png and /dev/null differ diff --git a/workshops/swa-gatsby-portfolio/assets/prerequisites/redeempass-1.jpg b/workshops/swa-gatsby-portfolio/assets/prerequisites/redeempass-1.jpg deleted file mode 100644 index da8a649b..00000000 Binary files a/workshops/swa-gatsby-portfolio/assets/prerequisites/redeempass-1.jpg and /dev/null differ diff --git a/workshops/swa-gatsby-portfolio/assets/prerequisites/redeempass-2.jpg b/workshops/swa-gatsby-portfolio/assets/prerequisites/redeempass-2.jpg deleted file mode 100644 index 77b9829d..00000000 Binary files a/workshops/swa-gatsby-portfolio/assets/prerequisites/redeempass-2.jpg and /dev/null differ diff --git a/workshops/swa-gatsby-portfolio/assets/prerequisites/redeempass-3.jpg b/workshops/swa-gatsby-portfolio/assets/prerequisites/redeempass-3.jpg deleted file mode 100644 index 102a6a37..00000000 Binary files a/workshops/swa-gatsby-portfolio/assets/prerequisites/redeempass-3.jpg and /dev/null differ diff --git a/workshops/swa-gatsby-portfolio/assets/prerequisites/redeempass-4.jpg b/workshops/swa-gatsby-portfolio/assets/prerequisites/redeempass-4.jpg deleted file mode 100644 index c3568ccc..00000000 Binary files a/workshops/swa-gatsby-portfolio/assets/prerequisites/redeempass-4.jpg and /dev/null differ diff --git a/workshops/swa-gatsby-portfolio/assets/prerequisites/redeempass-5.jpg b/workshops/swa-gatsby-portfolio/assets/prerequisites/redeempass-5.jpg deleted file mode 100644 index 76875382..00000000 Binary files a/workshops/swa-gatsby-portfolio/assets/prerequisites/redeempass-5.jpg and /dev/null differ diff --git a/workshops/swa-gatsby-portfolio/assets/prerequisites/redeempass-6.jpg b/workshops/swa-gatsby-portfolio/assets/prerequisites/redeempass-6.jpg deleted file mode 100644 index ee8834bc..00000000 Binary files a/workshops/swa-gatsby-portfolio/assets/prerequisites/redeempass-6.jpg and /dev/null differ diff --git a/workshops/swa-gatsby-portfolio/assets/prerequisites/redeempass-7.jpg b/workshops/swa-gatsby-portfolio/assets/prerequisites/redeempass-7.jpg deleted file mode 100644 index 12a38c25..00000000 Binary files a/workshops/swa-gatsby-portfolio/assets/prerequisites/redeempass-7.jpg and /dev/null differ diff --git a/workshops/swa-gatsby-portfolio/assets/prerequisites/student-1.png b/workshops/swa-gatsby-portfolio/assets/prerequisites/student-1.png deleted file mode 100644 index 6ed7bca0..00000000 Binary files a/workshops/swa-gatsby-portfolio/assets/prerequisites/student-1.png and /dev/null differ diff --git a/workshops/swa-gatsby-portfolio/assets/react.svg b/workshops/swa-gatsby-portfolio/assets/react.svg deleted file mode 100644 index 138d99ba..00000000 --- a/workshops/swa-gatsby-portfolio/assets/react.svg +++ /dev/null @@ -1,37 +0,0 @@ - - - - - - - - - - - diff --git a/workshops/swa-gatsby-portfolio/assets/static-web-apps.png b/workshops/swa-gatsby-portfolio/assets/static-web-apps.png deleted file mode 100644 index 3d4fc791..00000000 Binary files a/workshops/swa-gatsby-portfolio/assets/static-web-apps.png and /dev/null differ diff --git a/workshops/swa-gatsby-portfolio/outline.md b/workshops/swa-gatsby-portfolio/outline.md deleted file mode 100644 index 55931586..00000000 --- a/workshops/swa-gatsby-portfolio/outline.md +++ /dev/null @@ -1,45 +0,0 @@ -PWA WORKSHOP -============ - -Sandbox backup in case of no Azure account: https://aka.ms/aka/swa-sandbox - -# Intro: Objectives - - Bootstrap a Gatsby app from a template - - Create a GitHub repository to push your code - - Deploy your website on Azure Static Web Apps with a CI/CD pipeline - - Understand how Gatsby works with React and Markdown - - Create your own content - -0. Check Azure and GitHub accounts, Git, Node.js version, VS Code ~10min - -1. Bootstrap Gatsby app ~35min - - What's Gatsby? - - CLI Installation - - Generate app from template: https://github.com/sinedied/gatsby-portfolio - - Test app locally - - Create GitHub repository https://github.com/new => my-portfolio - - Create local repo, commit and push code - - Check code - -2. Deploy on SWA ~25min - - What's SWA? - - Go to the Azure portal, create SWA - - Steps / connect your GitHub repo - - Show app on Azure, then show GitHub actions - - Open YAML workflow and explain CI/CD - - View result - -3. Create content ~35min - - Explain app structure - - What's markdown? .mdx? - - Example content - - Make it your own :) - * name - * projects - * resume - -5. Bonus ~5-10min (early or conclusion) - - Create a custom React component: https://reactjs.org/tutorial/tutorial.html - - Homework - - Show off! - - Resource to learn more diff --git a/workshops/swa-gatsby-portfolio/translations/prerequisites.fr.md b/workshops/swa-gatsby-portfolio/translations/prerequisites.fr.md deleted file mode 100644 index 5440e1c0..00000000 --- a/workshops/swa-gatsby-portfolio/translations/prerequisites.fr.md +++ /dev/null @@ -1,126 +0,0 @@ -# Préparez votre environnement - -Afin de réaliser ce workshop, vous aurez besoin de: - -- **Node.js**: https://nodejs.org (v12.13 minimum) -- **Git**: https://git-scm.com -- **Un compte GitHub**: https://github.com/join -- **Un éditeur de code**, par exemple: https://aka.ms/get-vscode -- **Un navigateur**, par exemple: https://www.microsoft.com/edge -- La **CLI Gastby**, à installer en copiant la commande suivante dans un terminal: `npm install -g gatsby-cli` (nécessite d'avant installé Node.js avant) -- **Une souscription Azure**, voir ci-dessous pour les détails - -## Configurer son compte Azure - -Il existe différentes manières d'obtenir une souscription à Microsoft Azure. -Ce compte est nécessaire afin de créer les ressources Azure pour ce workshop. -Les ressources utilisées le seront dans les limites des tiers gratuits, il se peut cependant que des frais soient occasionnés en suivant ce workshop. - -Afin de vous aider à créer votre compte Azure, choisissez l'option qui -correspond le mieux à votre situation : - -- [J'ai déjà un abonnement](#already-sub) -- [Je suis étudiant](#student) -- [Je suis un abonné MSDN/Visual Studio](#vss) -- [J'ai un Azure Pass](#azure-pass) (Si vous n'avez pas de compte étudiant, un Azure Pass vous sera donné au début du workshop) -- [Je n'ai rien de tout cela](#nothing) - -### J'ai déjà un abonnement Azure :id=already-sub - -C'est une excellente nouvelle ! Il faudra toutefois veiller à vérifier que vous avez les autorisations nécessaires -afin de pouvoir créer des ressources sur cet abonnement. - -Vous pouvez maintenant [vérifier si tout est prêt pour la prochaine étape](#self-check). - -### Je suis étudiant :id=student - -En tant qu'étudiant, vous avez probablement accès à l'offre **Azure For Students**. -Pour le savoir, rendez-vous sur la [page dédiée][azure-student], et cliquez sur **Activate Now**. -On vous demandera alors de confirmer vos informations personnelles, ainsi que votre numéro de téléphone afin de recevoir -un SMS de validation. - -!> Si, à un moment dans le parcours d'inscription, vos informations de carte bleue vous sont demandées, c'est probablement qu'il y a eu une erreur dans le parcours. - -Il se peut que votre portail étudiant vous amène directement sur le portail Azure, sans toutefois avoir de souscription -Azure. Dans ce cas, recherchez "Education" dans la barre de recherche en haut à droite. Sur cette page éducation, -cliquez sur le bouton **Claim your Azure credit now** afin de démarrer le processus de création d'abonnement. -![](../assets/prerequisites/student-1.png) - -Dans le cas où votre établissement d'enseignement ne serait pas reconnu, vous pouvez toujours -[créer un abonnement d'essai](#nothing). - -Vous pouvez maintenant [vérifier si tout est prêt pour la prochaine étape](#self-check). - -### Un employé Microsoft m'a communiqué un _Azure Pass_ :id=azure-pass - -Vous êtes sur un événement et un employé vous a communiqué un code _Azure Pass_? Dans ce cas -vous pouvez l'utiliser afin de créer un abonnement. Avant de démarrer, assurez-vous : - -- d'avoir un compte Microsoft (anciennement Live). Vous pouvez en créer un sur [account.microsoft.com](https://account.microsoft.com), -- que ce compte n'a jamais été utilisé pour un autre abonnement Azure. Si vous avez déjà eu un compte d'essai ou payant -avec la même adresse, il vous sera alors impossible d'utiliser l'Azure Pass. - -!> Si, à un moment dans le parcours d'inscription, vos informations de carte bleue vous sont demandées, c'est probablement qu'il y a eu une erreur dans le parcours. Demandez de l'aide à l'employé Microsoft. - -1. Rendez-vous sur [microsoftazurepass.com][azurepass] et cliquez sur **Start**, -![Démarrer l'utilisation du pass](../assets/prerequisites/redeempass-1.jpg) -2. Connectez vous avec un compte Microsoft Live **Vous devez utiliser un compte Microsoft qui n'est associé à aucune - autre souscription Azure** -3. Vérifiez l'email du compte utilisé et cliquez sur **Confirm Microsoft Account** -![Confirmer le compte](../assets/prerequisites/redeempass-2.jpg) -4. Entrez le code que nous vous avons communiqués, puis cliquez sur **Claim Promo Code** (et non, le code présent sur la - capture d'écran n'est pas valide ;) ), -![Indiquer son code](../assets/prerequisites/redeempass-3.jpg) -5. Nous validons votre compte, cela prend quelques secondes -![Validation du code](../assets/prerequisites/redeempass-4.jpg) -6. Nous serez ensuite redirigé vers une dernière page d'inscrption. Remplissez les informations, puis cliquez sur **Suivant** -![Entrer les informations](../assets/prerequisites/redeempass-5.jpg) -7. Il ne vous restera plus que la partie légale: accepter les différents contrats et déclarations. Cochez les cases que -vous acceptez, et si c'est possible, cliquez sur le bouton **Inscription** -![Accepter les conditions légales](../assets/prerequisites/redeempass-6.jpg) - -Encore quelques minutes d'attente, et voilà, votre compte est créé ! Prenez quelques minutes afin d'effectuer la -visite et de vous familiariser avec l'interface du portail Azure. - -![Accueil du portail Azure](../assets/prerequisites/redeempass-7.jpg) - -Vous pouvez maintenant [vérifier si tout est prêt pour la prochaine étape](#self-check). - -### Je suis un abonné Visual Studio / MSDN :id=vss - -Vous avez accès à un crédit mensuel gratuit dans le cadre de votre abonnement. Si vous ne l'avez pas déjà activé, -il suffit d'aller sur la [page dédiée](https://azure.microsoft.com/pricing/member-offers/credit-for-visual-studio-subscribers/?WT.mc_id=javascript-19816-yolasors) -puis de cliquer sur le bouton **activer**. - -Vous pouvez maintenant [vérifier si tout est prêt pour la prochaine étape](#self-check). - -### Je n'ai rien de tout cela :id=nothing - -Vous pouvez toujours créer un [abonnement d'essai][azure-free-trial]. Les informations de carte bleue vous seront -demandées afin de s'assurer que vous êtes une personne physique. - -Vous pouvez maintenant [vérifier si tout est prêt pour la prochaine étape](#self-check). - -### ✅ Vérifions si votre compte Azure a bien été créé :id=self-check - -Avant de passer à l'étape suivante, nous allons nous assurer que votre souscription -a bien été créée. Pour cela, quelques étapes suffisent : - -1. Rendez-vous sur [le portail Azure][azure-portal], -2. Dans la barre de recherche en haut de la page web, entrez "Subscriptions", puis cliquez sur -l'élément ![](../assets/prerequisites/check-01.png) -3. Une liste apparaît, dans laquelle vous devez avoir un élément avec un status Actif ![](../assets/prerequisites/check-02.png) - ->La capture d'écran indique un nom d'abonnement _Azure for Students_. Ce nom ->peut différer en fonction du type d'abonnement Azure, ainsi que de qui l'a créé. ->Il est en effet possible de rennomer son abonnement avec un nom plus de - -**Félicitations**, vous êtes prêt pour le workshop! 🥳 - -[azurepass]: https://www.microsoftazurepass.com/?WT.mc_id=javascript-19816-yolasors -[azure-portal]: https://portal.azure.com/?feature.customportal=false&WT.mc_id=javascript-19816-yolasors -[azure-free-trial]: https://azure.microsoft.com/free/?WT.mc_id=javascript-19816-yolasors -[azure-student]: https://azure.microsoft.com/free/students/?WT.mc_id=javascript-19816-yolasors - ---- -Merci à [Christopher Maneu](https://twitter.com/cmaneu) pour ces instructions détaillées en français. diff --git a/workshops/swa-gatsby-portfolio/workshop.md b/workshops/swa-gatsby-portfolio/workshop.md deleted file mode 100644 index 9d89614b..00000000 --- a/workshops/swa-gatsby-portfolio/workshop.md +++ /dev/null @@ -1,301 +0,0 @@ ---- -short_title: Portfolio Workshop -description: Discover how to create and deploy your personal portfolio website using Gatsby, GitHub and Azure Static Web Apps. -type: workshop -authors: Yohan Lasorsa -contacts: '@sinedied' -banner_url: assets/portfolio-banner.jpg -duration_minutes: 120 -audience: students -level: beginner -tags: github, gatsby, azure, static web apps, javascript, markdown, react -published: true -sections_title: - - Introduction - - Bootstrap your app - - Deploy your app - - Add your own content - - Go further (Bonus) ---- - -# Create and deploy your online portfolio with Gatsby, GitHub and Azure Static Web Apps - -In this workshop we'll learn how to create and deploy your personal portfolio website and customize its content. - -## Workshop Objectives -- Bootstrap a Gatsby app from a template -- Create a GitHub repository to push your code -- Deploy your website on Azure Static Web Apps with a CI/CD pipeline -- Understand how Gatsby works with React and Markdown -- Create your own content 😎 - -## Prerequisites - -| | | -|---------------------|-----------------------------------------------| -| Node.js v12.13+ | https://nodejs.org | -| Git | https://git-scm.com | -| GitHub account | https://github.com/join | -| Azure account | https://aka.ms/student/azure | -| A code editor | https://aka.ms/get-vscode | -| A browser | https://www.microsoft.com/edge | -| Gastby CLI | Run `npm install -g gatsby-cli` in a terminal | - -You can test your setup by opening a terminal and typing: - -```sh -npm --version -git --version -gatsby --version -``` - ---- - -## Bootstrap Gatsby app - -### What's [Gatsby](http://www.gatsbyjs.com)? - -It's an **open-source** static website generator and **framework**, allowing you to create website content from various data sources like JSON, **Markdown**, a database, an API and more, all based on **React** components. - -#### Features -- Themes and starter templates -- Plugins -- Built-in GraphQL support -- Ideal for portfolios, blogs, e-shops, company homepages, web apps... - -### Create Gatsby app - -```sh -# See available commands -gatsby --help - -# Create new app -# (package install might take a while...) -gatsby new my-portfolio https://github.com/themoaw/gatsby-portfolio - -# Start development server -cd my-portfolio -gatsby develop -``` - -### Push to GitHub - -1. Create a new repo with the name `my-portfolio`: https://github.com/new -2. Push your code to the repo, by copy/pasting the commands shown on GitHub: - ![Screenshot showing git commands on GitHub](assets/gh-git.png) -3. After the push is finished, check that your code is visible on GitHub - ---- - -## Deploy your app on Azure Static Web Apps - -### What's the cloud? ☁️ - -It's just someone else's computer. 🙂 -More seriously, we call "the cloud" software and services that you can access remotely over the internet. - -There are 3 main categories of cloud computing services: - -| | | | -|----------------------|----------------------|----------------------| -| **IaaS** | **PaaS** | **SaaS** | -| Infrastructure as a Service
(Storage, computing, networking) | Platform as a Service
(Focus on your code/data) | Software as a Service
(Ready-to-use software) | - -### [Azure Static Web Apps](https://aka.ms/docs/swa) - -#### What's Azure Static Web Apps (SWA)? - -It's an all-inclusive **hosting service** for web apps with **serverless APIs**, based on a continuous integration and deployment pipeline from a GitHub repository. - -#### Features -CI/CD, assets hosting, APIs, SSL certificate, route control, authentication, authorization, CDN, staging environments... - -### Deploy to Azure Static Web Apps - -1. Open Azure portal: [aka.ms/create/swa](https://aka.ms/create/swa) -2. Create new resource group `my-portfolio` -3. Enter a name and choose a region -4. Sign in to GitHub and select your GitHub repo and `main` branch -5. In **Build Details**, choose the `Gatsby` build preset -6. Click **Review + create**, then **Create** - -### The deployment workflow - -In the Azure portal, in the created resource, select **GitHub Actions runs**. - -> You can see the **C**ontinous **I**ntegration (CI) and **C**ontinuous **D**eployment (CD) jobs running. - -#### How the process works? - -- Azure made a new commit in your repo with `.github/workflows/.yml` -- The workflow is built with [GitHub Actions](https://github.com/features/actions) -- Every **new commit** triggers a new build and deployment -- You can preview changes separately using a **pull request** - ---- - -## Create your own content - -### Gatsby app structure - -```sh -|-- config/ # Website config: title, description, URL, links... -|-- content/ # Pages content: markdown, images - |-- imprint/ - |-- index/ - |-- about/ - |-- contact/ - |-- ... -|-- src/ # React source code and components -|-- static/ # Static assets, like your resume in PDF -|-- public/ # Built version of your website (with gatsby build) -|-- gatsby-config.js # Gatsby plugins configuration -|-- gatsby-browser.js # Browser events hooks -|-- package.json # Your Node.js app details - -``` - -### The `config/index.js` file - -#### It contains the website settings: -- Metadata about the site -- Theme colors -- Optional RSS feed to display -- Social media links -- Navigation menu settings - -#### Task -- Open a terminal and run `gatsby develop` -- Edit `/config/index.js` to make it yours 🙂 - -
- -> - Change the website title and info -> - Personalize the colors -> - Change or hide RSS feed -> - Set social media links - -
- -### What's [Markdown](https://commonmark.org/help/)? - -Markdown is a lightweight markup language for creating formatted text (HTML) using a plain-text editor. - -#### Example code vs result - -```md -# Header - -Some text in **bold** or *italic* with a [Link](http://link.com). - -- Item 1 -- Item 2 -- Item 3 -``` - -Will be displayed as: - -# Header - -Some text in **bold** or *italic* with a [Link](http://link.com). - -- Item 1 -- Item 2 -- Item 3 - -### What's [MDX](https://mdxjs.com/)? - -MDX is markdown with JSX* support. - -```mdx -# Hello, *world* - -Below is an example of JSX embedded in Markdown.
-Try and change the background color! - -
-

This is JSX

-
-``` - -*JSX: a syntax extension to JavaScript used in React to produce HTML. [More info](https://reactjs.org/docs/introducing-jsx.html) - -### `/content` files - -- Used by Gatsby to create your website content -- They use MDX with [Front Matter](https://jekyllrb.com/docs/front-matter/) (metadata in text format at the top) - ```yaml - --- - title: "I'm a Gatsby Starter" - icon: "waving-hand.png" - subtitle: "one-page portfolios" - --- - ``` - -- You can put images next to .mdx files -- You can add or create as many as you need - -#### Task - -- Put your own content there! 🌈 - -
- -> To redeploy, commit and push
your changes - -
- -
- -> - Change hero content -> - Put your story in about section -> - Change contact details -> - Feature your skills in interests section -> - Show your projects - -
- ---- - -## Go Further (Bonus) - -### React components - -The `src` folder contains the React app behind your website. - -You can also create custom components to change the visual look of the content. -See `src/components/styles/underlining.js` for example. - -
- -> The [Styled Components](https://styled-components.com/) library is used for CSS styling. - -
- - -
- -> - Create a custom component -> - Use it in a content a file - -
- -### For further study - -#### GraphQL - -Gatbsy has built-in support for consuming data from [GraphQL](https://graphql.org/). You can follow this tutorial to learn about GraphQL and how to use it in Gatsby: - -**👉 https://aka.ms/learn/gatsby-graphql** - -#### API - -Gatbsy and Azure Static Web Apps are not limited to static data. You can follow this tutorial to learn how to create your own serverless API with JavaScript and deploy it on with Static Web Apps: - -**👉 https://aka.ms/learn/swa-api** - -#### Resources -- Gatbsy: https://www.gatsbyjs.com/ -- Learn Markdown: https://commonmark.org/help/ -- More Static Web Apps tutorials: https://aka.ms/learn/swa -- Intro to React: https://reactjs.org/tutorial/tutorial.html diff --git a/workshops/swa-pwa-angular/assets/gh-git.png b/workshops/swa-pwa-angular/assets/gh-git.png deleted file mode 100644 index d24bbcc5..00000000 Binary files a/workshops/swa-pwa-angular/assets/gh-git.png and /dev/null differ diff --git a/workshops/swa-pwa-angular/assets/prerequisites/check-01.png b/workshops/swa-pwa-angular/assets/prerequisites/check-01.png deleted file mode 100644 index a5b472ac..00000000 Binary files a/workshops/swa-pwa-angular/assets/prerequisites/check-01.png and /dev/null differ diff --git a/workshops/swa-pwa-angular/assets/prerequisites/check-02.png b/workshops/swa-pwa-angular/assets/prerequisites/check-02.png deleted file mode 100644 index fc7d4d7f..00000000 Binary files a/workshops/swa-pwa-angular/assets/prerequisites/check-02.png and /dev/null differ diff --git a/workshops/swa-pwa-angular/assets/prerequisites/redeempass-1.jpg b/workshops/swa-pwa-angular/assets/prerequisites/redeempass-1.jpg deleted file mode 100644 index da8a649b..00000000 Binary files a/workshops/swa-pwa-angular/assets/prerequisites/redeempass-1.jpg and /dev/null differ diff --git a/workshops/swa-pwa-angular/assets/prerequisites/redeempass-2.jpg b/workshops/swa-pwa-angular/assets/prerequisites/redeempass-2.jpg deleted file mode 100644 index 77b9829d..00000000 Binary files a/workshops/swa-pwa-angular/assets/prerequisites/redeempass-2.jpg and /dev/null differ diff --git a/workshops/swa-pwa-angular/assets/prerequisites/redeempass-3.jpg b/workshops/swa-pwa-angular/assets/prerequisites/redeempass-3.jpg deleted file mode 100644 index 102a6a37..00000000 Binary files a/workshops/swa-pwa-angular/assets/prerequisites/redeempass-3.jpg and /dev/null differ diff --git a/workshops/swa-pwa-angular/assets/prerequisites/redeempass-4.jpg b/workshops/swa-pwa-angular/assets/prerequisites/redeempass-4.jpg deleted file mode 100644 index c3568ccc..00000000 Binary files a/workshops/swa-pwa-angular/assets/prerequisites/redeempass-4.jpg and /dev/null differ diff --git a/workshops/swa-pwa-angular/assets/prerequisites/redeempass-5.jpg b/workshops/swa-pwa-angular/assets/prerequisites/redeempass-5.jpg deleted file mode 100644 index 76875382..00000000 Binary files a/workshops/swa-pwa-angular/assets/prerequisites/redeempass-5.jpg and /dev/null differ diff --git a/workshops/swa-pwa-angular/assets/prerequisites/redeempass-6.jpg b/workshops/swa-pwa-angular/assets/prerequisites/redeempass-6.jpg deleted file mode 100644 index ee8834bc..00000000 Binary files a/workshops/swa-pwa-angular/assets/prerequisites/redeempass-6.jpg and /dev/null differ diff --git a/workshops/swa-pwa-angular/assets/prerequisites/redeempass-7.jpg b/workshops/swa-pwa-angular/assets/prerequisites/redeempass-7.jpg deleted file mode 100644 index 12a38c25..00000000 Binary files a/workshops/swa-pwa-angular/assets/prerequisites/redeempass-7.jpg and /dev/null differ diff --git a/workshops/swa-pwa-angular/assets/prerequisites/student-1.png b/workshops/swa-pwa-angular/assets/prerequisites/student-1.png deleted file mode 100644 index 6ed7bca0..00000000 Binary files a/workshops/swa-pwa-angular/assets/prerequisites/student-1.png and /dev/null differ diff --git a/workshops/swa-pwa-angular/assets/pwa-banner.jpg b/workshops/swa-pwa-angular/assets/pwa-banner.jpg deleted file mode 100644 index 9b81f678..00000000 Binary files a/workshops/swa-pwa-angular/assets/pwa-banner.jpg and /dev/null differ diff --git a/workshops/swa-pwa-angular/prerequisites.md b/workshops/swa-pwa-angular/prerequisites.md deleted file mode 100644 index f8cc056e..00000000 --- a/workshops/swa-pwa-angular/prerequisites.md +++ /dev/null @@ -1,112 +0,0 @@ -# Prepare your environment - -To follow this workshop, you'll need: - -- **Node.js**: https://nodejs.org (v12.15 minimum) -- **Git**: https://git-scm.com -- **A GitHub account**: https://github.com/join -- **A code editor**, for example: https://aka.ms/get-vscode -- **A browser**, for example: https://www.microsoft.com/edge -- **An Azure subscription**, see below for details - -## Configure your Azure account - -There are different ways to get a Microsoft Azure subscription. -This account is necessary to create Azure resources for this workshop. -The resources used should all be within the limits of free tiers, still it may be possible that fees are caused by following this workshop. - -To help you create your Azure account, choose the option that best match your situation: - -- [I already have a subscription](#already-sub) -- [I have a MSDN/Visual Studio subscription](#vss) -- [I have an Azure Pass](#azure-pass) -- [I'm a student](#student) -- [I have nothing of these](#nothing) - -### I already have an Azure subscription :id=already-sub - -That's excellent news! However, it will be necessary to ensure that you have the necessary authorizations in order to create resources on this subscription. - -You can now [check if everything is ready for the next step](#self-check). - -### I have an _Azure Pass_ :id=azure-pass - -You are taking part in an event and you were provided an _azure pass_ code? -In that case, you can use it to create a subscription. -Before starting, make sure: - -- To have a Microsoft account (formerly live). You can create one on [account.microsoft.com](https://account.microsoft.com), -- That this account has never been used for another Azure subscription. If you have already had a test or paying account with the same address, it will be impossible for you to use the Azure Pass. In that case, you need to create a new Microsoft account. - -!> If, at any time during the registration path, your credit card information is required, it's probably that there was a mistake in the process. Ask for help of a Microsoft employee. - -1. Go to [microsoftazurepass.com][azurepass] and click **Start**, -![Redeem pass](assets/redeempass-1.jpg) -2. Connect with a Microsoft Live account. **You must use a Microsoft account that is not associated with any other Azure subscription** -3. Check the email used for the account and click on **Confirm Microsoft Account** -![Confirm account](assets/redeempass-2.jpg) -4. Enter the Azure Pass code that you received, and then click **Claim Promo Code** (and no, the code present on the - screenshot is not valid ;) ), -![Enter your code](assets/redeempass-3.jpg) -5. We are validing your account, it takes a few seconds -![Code Validation](assets/redeempass-4.jpg) -6. You will then be redirected to a last registration page. Fill out the information, and then click **Next** -![Entrer les informations](assets/redeempass-5.jpg) -7. It will only remain the legal part: accept the various contracts and declarations. Check the boxes that you accept, and if possible, click on the button **Subscribe** -![Accept legal contractss](assets/redeempass-6.jpg) - -Another a few minutes of waiting, and that's it, your account is created! Take a few minutes to perform the visit and familiarize yourself with the Azure portal interface. - -![Azure portal](assets/redeempass-7.jpg) - -You can now [check if everything is ready for the next step](#self-check). - -### I have a MSDN/Visual Studio subscription :id=vss - -You have access to a free monthly credit as part of your subscription. If you have not already activated it, Just go on the [dedicated page](https://azure.microsoft.com/pricing/member-offers/credit-for-visual-studio-subscribers/?WT.mc_id=javascript-32417-yolasors) -then click on the **activate** button. - -You can now [check if everything is ready for the next step](#self-check). - -### I'm a student :id=student - -As a student, you may have access to the **Azure For Students** offer. -To find out, go to the [dedicated page][azure-student], and click on **Activate Now**. -You will then be asked to confirm your personal information, as well as your phone number to receive a SMS validation. - -!> If, at any time during the registration path, your credit card information is required, it's probably that there was a mistake in the process. Ask for help of a Microsoft employee. - -Your student portal may take you directly to the Azure portal, without having any Azure subscription. -In this case, search for "Education" in the search bar at the top right. On this Education page, click on **Claim your Azure credit now** in order to start the subscription creation process. -![](assets/student-1.png) - -In the case where your educational institution is not recognized, you can still [create a trial subscription](#nothing). - -You can now [check if everything is ready for the next step](#self-check). - -### I have nothing of these :id=nothing - -You can always create a [free trial subscription][azure-free-trial]. Credit card information will be requested to make sure you are a physical person. - -You can now [check if everything is ready for the next step](#self-check). - -### ✅ Check if your Azure account has been created :id=self-check - -Before moving on to the next step, we will ensure that your subscription -has been created. For this, follow these steps: - -1. Go to [Azure portal][azure-portal], -2. In the search bar at the top of the web page, enter "Subscriptions", then click on the item ![](assets/check-01.png) -3. A list appears, where you must have an element with an active status ![](assets/check-02.png) - ->The screenshot indicates a subscription name _Azure for Students_. This name may differ depending on the type of Azure subscription, as well as who created it. - -**Congratulations**, You are ready for the Workshop! 🥳 - -[azurepass]: https://www.microsoftazurepass.com/?WT.mc_id=javascript-32417-yolasors -[azure-portal]: https://portal.azure.com/?feature.customportal=false&WT.mc_id=javascript-32417-yolasors -[azure-free-trial]: https://azure.microsoft.com/free/?WT.mc_id=javascript-32417-yolasors -[azure-student]: https://azure.microsoft.com/free/students/?WT.mc_id=javascript-32417-yolasors - ---- -Thanks to [Christopher Maneu](https://twitter.com/cmaneu) for these detailed instructions. diff --git a/workshops/swa-pwa-angular/translations/prerequisites.fr.md b/workshops/swa-pwa-angular/translations/prerequisites.fr.md deleted file mode 100644 index 2f5eadc3..00000000 --- a/workshops/swa-pwa-angular/translations/prerequisites.fr.md +++ /dev/null @@ -1,125 +0,0 @@ -# Préparez votre environnement - -Afin de réaliser ce workshop, vous aurez besoin de: - -- **Node.js**: https://nodejs.org (v14.17 minimum) -- **Git**: https://git-scm.com -- **Un compte GitHub**: https://github.com/join -- **Un éditeur de code**, par exemple: https://aka.ms/get-vscode -- **Un navigateur**, par exemple: https://www.microsoft.com/edge -- **Une souscription Azure**, voir ci-dessous pour les détails - -## Configurer son compte Azure - -Il existe différentes manières d'obtenir une souscription à Microsoft Azure. -Ce compte est nécessaire afin de créer les ressources Azure pour ce workshop. -Les ressources utilisées le seront dans les limites des tiers gratuits, il se peut cependant que des frais soient occasionnés en suivant ce workshop. - -Afin de vous aider à créer votre compte Azure, choisissez l'option qui -correspond le mieux à votre situation : - -- [J'ai déjà un abonnement](#already-sub) -- [Je suis étudiant](#student) -- [Je suis un abonné MSDN/Visual Studio](#vss) -- [J'ai un Azure Pass](#azure-pass) (👉 [Cliquez ici](https://thankful-forest-09176b503.azurestaticapps.net/event/swc210622) pour en récupérer un) -- [Je n'ai rien de tout cela](#nothing) - -### J'ai déjà un abonnement Azure :id=already-sub - -C'est une excellente nouvelle ! Il faudra toutefois veiller à vérifier que vous avez les autorisations nécessaires -afin de pouvoir créer des ressources sur cet abonnement. - -Vous pouvez maintenant [vérifier si tout est prêt pour la prochaine étape](#self-check). - -### Je suis étudiant :id=student - -En tant qu'étudiant, vous avez peut-être accès à l'offre **Azure For Students**. -Pour le savoir, rendez-vous sur la [page dédiée][azure-student], et cliquez sur **Activate Now**. -On vous demandera alors de confirmer vos informations personnelles, ainsi que votre numéro de téléphone afin de recevoir -un SMS de validation. - -!> Si, à un moment dans le parcours d'inscription, vos informations de carte bleue vous sont demandées, c'est probablement qu'il y a eu une erreur dans le parcours. - -Il se peut que votre portail étudiant vous amène directement sur le portail Azure, sans toutefois avoir de souscription -Azure. Dans ce cas, recherchez "Education" dans la barre de recherche en haut à droite. Sur cette page éducation, -cliquez sur le bouton **Claim your Azure credit now** afin de démarrer le processus de création d'abonnement. -![](../assets/student-1.png) - -Dans le cas où votre établissement d'enseignement ne serait pas reconnu, vous pouvez toujours -[créer un abonnement d'essai](#nothing). - -Vous pouvez maintenant [vérifier si tout est prêt pour la prochaine étape](#self-check). - -### Un employé Microsoft m'a communiqué un _Azure Pass_ :id=azure-pass - -Vous êtes sur un événement et un employé vous a communiqué un code _Azure Pass_? Dans ce cas -vous pouvez l'utiliser afin de créer un abonnement. Avant de démarrer, assurez-vous : - -- d'avoir un compte Microsoft (anciennement Live). Vous pouvez en créer un sur [account.microsoft.com](https://account.microsoft.com), -- que ce compte n'a jamais été utilisé pour un autre abonnement Azure. Si vous avez déjà eu un compte d'essai ou payant -avec la même adresse, il vous sera alors impossible d'utiliser l'Azure Pass. - -!> Si, à un moment dans le parcours d'inscription, vos informations de carte bleue vous sont demandées, c'est probablement qu'il y a eu une erreur dans le parcours. Demandez de l'aide à l'employé Microsoft. - -1. Rendez-vous sur [microsoftazurepass.com][azurepass] et cliquez sur **Start**, -![Démarrer l'utilisation du pass](../assets/redeempass-1.jpg) -2. Connectez vous avec un compte Microsoft Live **Vous devez utiliser un compte Microsoft qui n'est associé à aucune - autre souscription Azure** -3. Vérifiez l'email du compte utilisé et cliquez sur **Confirm Microsoft Account** -![Confirmer le compte](../assets/redeempass-2.jpg) -4. Entrez le code que nous vous avons communiqués, puis cliquez sur **Claim Promo Code** (et non, le code présent sur la - capture d'écran n'est pas valide ;) ), -![Indiquer son code](../assets/redeempass-3.jpg) -5. Nous validons votre compte, cela prend quelques secondes -![Validation du code](../assets/redeempass-4.jpg) -6. Nous serez ensuite redirigé vers une dernière page d'inscrption. Remplissez les informations, puis cliquez sur **Suivant** -![Entrer les informations](../assets/redeempass-5.jpg) -7. Il ne vous restera plus que la partie légale: accepter les différents contrats et déclarations. Cochez les cases que -vous acceptez, et si c'est possible, cliquez sur le bouton **Inscription** -![Accepter les conditions légales](../assets/redeempass-6.jpg) - -Encore quelques minutes d'attente, et voilà, votre compte est créé ! Prenez quelques minutes afin d'effectuer la -visite et de vous familiariser avec l'interface du portail Azure. - -![Accueil du portail Azure](../assets/redeempass-7.jpg) - -Vous pouvez maintenant [vérifier si tout est prêt pour la prochaine étape](#self-check). - -### Je suis un abonné Visual Studio / MSDN :id=vss - -Vous avez accès à un crédit mensuel gratuit dans le cadre de votre abonnement. Si vous ne l'avez pas déjà activé, -il suffit d'aller sur la [page dédiée](https://azure.microsoft.com/pricing/member-offers/credit-for-visual-studio-subscribers/?WT.mc_id=javascript-32417-yolasors) -puis de cliquer sur le bouton **activer**. - -Vous pouvez maintenant [vérifier si tout est prêt pour la prochaine étape](#self-check). - -### Je n'ai rien de tout cela :id=nothing - -Vous pouvez toujours créer un [abonnement d'essai][azure-free-trial]. Les informations de carte bleue vous seront -demandées afin de s'assurer que vous êtes une personne physique. - -Vous pouvez maintenant [vérifier si tout est prêt pour la prochaine étape](#self-check). - -### ✅ Vérifions si votre compte Azure a bien été créé :id=self-check - -Avant de passer à l'étape suivante, nous allons nous assurer que votre souscription -a bien été créée. Pour cela, quelques étapes suffisent : - -1. Rendez-vous sur [le portail Azure][azure-portal], -2. Dans la barre de recherche en haut de la page web, entrez "Subscriptions", puis cliquez sur -l'élément ![](../assets/check-01.png) -3. Une liste apparaît, dans laquelle vous devez avoir un élément avec un status Actif ![](../assets/check-02.png) - ->La capture d'écran indique un nom d'abonnement _Azure for Students_. Ce nom ->peut différer en fonction du type d'abonnement Azure, ainsi que de qui l'a créé. ->Il est en effet possible de rennomer son abonnement avec un nom plus de - -**Félicitations**, vous êtes prêt pour le workshop! 🥳 - -[azurepass]: https://www.microsoftazurepass.com/?WT.mc_id=javascript-32417-yolasors -[azure-portal]: https://portal.azure.com/?feature.customportal=false&WT.mc_id=javascript-32417-yolasors -[azure-free-trial]: https://azure.microsoft.com/free/?WT.mc_id=javascript-32417-yolasors -[azure-student]: https://azure.microsoft.com/free/students/?WT.mc_id=javascript-32417-yolasors - ---- -Merci à [Christopher Maneu](https://twitter.com/cmaneu) pour ces instructions détaillées en français. diff --git a/workshops/swa-pwa-angular/workshop.md b/workshops/swa-pwa-angular/workshop.md deleted file mode 100644 index 90e10200..00000000 --- a/workshops/swa-pwa-angular/workshop.md +++ /dev/null @@ -1,442 +0,0 @@ ---- -short_title: PWA Workshop -description: Build and deploy your first PWA from scratch, adding a serverless API, handling offline mode, app install and updates. -type: workshop -authors: Yohan Lasorsa -contacts: '@sinedied' -banner_url: assets/pwa-banner.jpg -duration_minutes: 120 -audience: students -level: beginner -tags: pwa, serverless, azure, static web apps, javascript, service worker, offline -published: true -sections_title: - - Introduction - - Bootstrap your app - - Audit and upgrade to PWA - - Add API and update SW config - - Manage app updates (Bonus) ---- - -# Build, deploy and audit a serverless PWA with Azure Static Web Apps - -In this workshop we'll learn how to build and deploy your first PWA with Angular, from scratch! -Let's find out in practice the changes necessary to transform your web app into a PWA, and manage offline mode, app install and updates. - -After setting up continuous deployment for your app using Azure Static Web Apps and GitHub Actions, we'll gradually improve your app by checking the useful metrics to keep in mind. And even if you've never touched Angular, you can still follow this workshop as it's not the focus here 😉. - -## Workshop Objectives -- Create an Angular app and turn it into a PWA (Progressive Web App) -- Set up continuous deployment on Azure Static Web Apps -- Configure service worker to add offline support -- Audit your app performance with Lighthouse -- Create a serverless API with Azure Functions -- Review caching and updating strategies -- Handle app updates - -## Prerequisites - -| | | -|---------------|-----------------| -| Node.js v14.17+ | https://nodejs.org | -| Git | https://git-scm.com | -| GitHub account | https://github.com/join | -| Azure account | https://aka.ms/student/azure | -| A code editor | https://aka.ms/get-vscode | -| A chromium-based browser | https://www.microsoft.com/edge | - -You can test your setup by opening a terminal and typing: - -```sh -node --version -git --version -``` - ---- - -## Bootstrap app, setup repository & deploy - -### Create Angular app - -Open a terminal and type these commands: - -```sh -# Install Angular CLI -npm install -g @angular/cli@latest - -# Create Angular app -ng new my-pwa --minimal --defaults -cd my-pwa - -# Test it -ng serve --open -``` - -### Push your app to GitHub - -1. Create new repo: https://github.com/new - -
- - > With GitHub CLI (https://cli.github.com) you can do it directly from command line: `gh repo create --public` - -
- -2. Push your code to the repo, by copy/pasting the commands shown on GitHub: - - ![](assets/gh-git.png) - -### What's Azure Static Web Apps? - -Static Web Apps (or SWA for short) is an all-inclusive **hosting service** for web apps with **serverless APIs**, based on a continuous integration and deployment pipeline from a GitHub or Azure DevOps repository. - -It provides a lot of features out of the box, like: -- CI/CD -- Assets hosting -- APIs -- SSL certificate -- Route control -- Authentication -- Authorization -- CDN -- Staging environments -- And more... - -### Deploy to Azure Static Web Apps - -1. Open Azure portal: [aka.ms/create/swa](https://aka.ms/create/swa) - -2. Create new resource group `my-pwa` - -3. Enter a name and choose a region - -4. Sign in to GitHub and select your GitHub repo and `main` branch - -5. In **Build Details**, choose the `Angular` build preset - -6. For the **API location**, enter `/api` - -7. For the **Output location**, enter `dist/my-pwa` - -8. Click **Review + create**, then **Create** - -### The deployment workflow - -In the Azure portal, in the created resource, select **GitHub Actions runs**. - -> You can see the **C**ontinous **I**ntegration (CI) and **C**ontinuous **D**eployment (CD) jobs running. - -#### How the process works? - -- Azure made a new commit in your repo with `.github/workflows/.yml` -- The workflow is built with [GitHub Actions](https://github.com/features/actions) -- Every **new commit** triggers a new build and deployment -- You can preview changes separately using a **pull request** - ---- - -## 2. Audit your app and update it to a PWA - -### Audit your app - -1. Open your deployed website URL - -2. Open DevTools with `F12` or `OPTION+COMMAND+I` (Mac) - -3. Navigate to **Lighthouse** tab - -4. Click on **Generate report** - -### Update your app to a PWA - -```sh -# Make your app a PWA -ng add @angular/pwa - -# See the changes -git diff -``` - -
- -> There is no `ngsw-worker.js` file, it will be automatically generated during build based on `ngsw-config.json`. - -
- -
- -> If you're not using Angular, [WorkBox](https://developers.google.com/web/tools/workbox) is an alternative service worker library that works with any website. - -
- -### Test your app offline - -Commit and push your changes: -```sh -git add . && git commit -m "add PWA support" && git push -``` - -Then wait for the updated version of your app to be deployed. - -Once your new version is deployed: - -- Open your web app again then **generate an audit report again**. - -- Take a look at **Application** tab. - -- Go to **Network** tab, switch from `online` to `offline` and refresh. - ---- - -## 3. App API and update service worker config - -### Add API - -The API is based on Azure Functions: [aka.ms/go/functions](https://aka.ms/go/functions) - -```sh -# Create API folder -mkdir api && cd api - -# Create functions config file -echo "{ \"version\": \"2.0\" }" > host.json - -# Create function folder -mkdir hello && cd hello -``` - -
- -> If you install Azure Functions Core Tools ([aka.ms/tools/func](https://aka.ms/tools/func)) you can just use `func init` instead. You can also test your functions locally with `func start`. - -
- -#### Create new function - -Create a file `index.js` with this content: - -```js -async function GetHello(context, req) { - context.res = { - body: 'Hello from API at ' + new Date().toLocaleTimeString() - }; -}; - -module.exports = GetHello; -``` - -Create a file `function.json` with this content: -```json -{ - "bindings": [ - { - "authLevel": "anonymous", - "type": "httpTrigger", - "direction": "in", - "name": "req", - "methods": ["get"] - }, - { - "type": "http", - "direction": "out", - "name": "res" - } - ] -} -``` - -### Update API in Angular app - -Edit `src/app/app.component.ts` to add this: -```ts -export class AppComponent { - hello = ''; - - async ngOnInit() { - try { - const response = await fetch('api/hello'); - if (!response.ok) { - throw new Error(response.statusText); - } - this.hello = await response.text(); - } catch (err) { - this.hello = 'Error: ' + err.message; - } - } -} -``` - -Edit `src/app/app.component.html` and replace all content with this: -```ts -Message: {{ hello }} -``` - -#### Deploy and test API - -Commit and push your changes: -```sh -git add . && git commit -m "add API" && git push -``` - -#### Once your new version is deployed - -- Open your web app again and hit `shift` + refresh (don't forget to switch back to `online` first!). - -- Go to **Network** tab, switch from `online` to `offline` and refresh. - -😱 - -### Update service worker config - -Edit `ngsw-config.json`: -```json -{ - ... - "dataGroups": [ - { - "name": "api", - "urls": ["/api/hello"], - "cacheConfig": { - "maxSize": 1, - "maxAge": "1m", - "strategy": "freshness" - } - } - ] -} -``` - -
- -> Read more about Angular service worker configuration [here](https://angular.io/guide/service-worker-config) - -
- -#### Deploy and test API (again) - -Commit and push your changes: -```sh -git add . && git commit -m "update SW config" && git push -``` - -#### Once your new version is deployed - -- Open your web app again and hit `shift` + refresh (don't forget to switch back to `online` first!). - -- Go to **Network** tab, switch from `online` to `offline` and refresh. - -😎 - -### Test home screen install - -Open your app on your mobile phone. - -#### Android - -Refresh a few times to see the install banner, or tap menu and choose **Add to home screen**. - -
- -> You can customize install UI using `beforeinstallprompt` event, see https://web.dev/customize-install - -
- -#### iOS - -Tap the share button and choose **Add to home screen**. - ---- - -## 4. Manage app updates - -### Using `SwUpdate` service - -Have a look at the docs here: https://angular.io/guide/service-worker-communications - -This service allows you to: -- Get notified of available updates -- Get notified of update activation -- Check for updates -- Activate latest updates - -### Add update button - -Edit `src/app/app.component.ts`, and add this: -```ts -import { SwUpdate } from '@angular/service-worker'; -... - -export class AppComponent { - ... - updateAvailable$ = this.swUpdate.available; - - constructor(private swUpdate: SwUpdate) {} - - async update() { - await this.swUpdate.activateUpdate(); - document.location.reload(); - } -} -``` - -Edit `src/app/app.component.html` and replace its content with: -```html -Message: {{ hello }} - -

- An update is available! - Click here to apply: - -

- -

No update available.

- -``` - -#### Commit and redeploy - -Commit and push your changes: -```sh -git add . && git commit -m "add SW updates" && git push -``` - -#### Once your new version is deployed - -- Open your web app again and hit `shift` + refresh in your browser. - -- Check that you see the message `No update available.`, meaning that your app is up-to-date. - -### Create a new version of your app - -- Add `

New version!

` to the top of `src/app/app.component.html`. - -- Commit, push and wait for the new version to be deployed: -```sh -git add . && git commit -m "minor update" && git push -``` - -#### Once your new version is deployed - -- Hit refresh **ONCE** and **WITHOUT** holding shift. - You should see the **Update** button appear after waiting a bit. - -- Click **Update**, and the new version should appear. - ---- - -## Go further - -Now that you got all the basics, you can go further and build your own PWA! -You can use these resources to help you in your journey: - -### Public APIs (to get some app ideas) -- https://github.com/public-apis/public-apis -- https://apilist.fun - -### Additional resources -- Angular Service Worker: https://angular.io/guide/service-worker-intro -- Workbox: https://developers.google.com/web/tools/workbox -- Azure Static Web Apps: https://aka.ms/docs/swa -- Web Capabilities (Project Fugu 🐡): https://web.dev/fugu-status/ -- Ngx Rocket: https://github.com/ngx-rocket/generator-ngx-rocket - * Responsive PWA app generator with Material/Ionic/Bootstrap templates, i18n and more