Media connector updates (#112)

Updates to media connector sample including those from #111

---------

Co-authored-by: Sam Chang <[email protected]>

samples/media-connector-invoke-test/.config_aio_connectors_namespace

@@ -0,0 +1 @@
+azure-iot-operations

samples/media-connector-invoke-test/.config_aio_namespace

@@ -0,0 +1 @@
+azure-iot-operations

samples/media-connector-invoke-test/.config_resources_path

@@ -0,0 +1 @@
+${PSScriptRoot}/resources

samples/media-connector-invoke-test/Install-ResourceFile.ps1

@@ -0,0 +1,34 @@
+#Requires -Version 7
+<#
+    Install a resource file.
+#>
+param (
+    [Parameter(
+        Mandatory=$true,
+        HelpMessage="The resource file.")]
+    [string]$resourceFile
+)
+
+Write-Host "`n"
+Write-Host (Split-Path -Path $PSCommandPath -Leaf).ToUpper() -ForegroundColor White
+
+$aioConnectorsNamespace = (Get-Content -Path (Join-Path -Path $PSScriptRoot -ChildPath ".config_aio_connectors_namespace") -Raw).Trim()
+Write-Host "AIO connectors namespace: $aioConnectorsNamespace"
+
+$resourcesDirectory = (Get-Content -Path (Join-Path -Path $PSScriptRoot -ChildPath ".config_resources_path") -Raw).Trim()
+$resourcesDirectory = $ExecutionContext.InvokeCommand.ExpandString(${resourcesDirectory})
+Write-Host "Resources directory: $resourcesDirectory"
+
+Write-Host "Resource file: $resourceFile"
+
+$fileFullPath=(Join-Path -Path $resourcesDirectory -ChildPath $resourceFile)
+Write-Host "Resource file full path: ${fileFullPath}"
+
+Write-Host "Applying the resource..."
+. kubectl apply -n ${aioConnectorsNamespace} -f ${fileFullPath}
+If ($LastExitCode -ne 0) {
+    Write-Host "Error: The resource could not be applied."
+    Exit $LastExitCode
+} Else {
+    Write-Host "The resource was applied successfully.`n"
+}

samples/media-connector-invoke-test/Invoke-ResourceInstallMonitorAndUninstall.ps1

@@ -0,0 +1,60 @@
+#Requires -Version 7
+<#
+    Install a resource pair (aep/asset), monitor it, and uninstall it.
+#>
+param (
+    [Parameter(
+        Mandatory,
+        HelpMessage="The AEP name.")]
+    [string]$aepName = "",
+    [Parameter(
+        Mandatory,
+        HelpMessage="The asset name.")]
+    [string]$assetName = "",
+    [Parameter(
+        Mandatory,
+        HelpMessage="The datapoint name.")]
+    [string]$datapointName = "",
+    [Parameter(
+        Mandatory,
+        HelpMessage="The monitor expression.")]
+    [string]$monitorExpresion = ""
+)
+
+Write-Host "`n"
+Write-Host (Split-Path -Path $PSCommandPath -Leaf).ToUpper() -ForegroundColor White
+
+. (Join-Path $PSScriptRoot "Test-Prerequisites.ps1") | Out-Null
+
+$aioNamespace = (Get-Content -Path (Join-Path $PSScriptRoot ".config_aio_namespace") -Raw).Trim()
+Write-Host "AIO namespace: $aioNamespace"
+
+$aioConnectorsNamespace = (Get-Content -Path (Join-Path $PSScriptRoot ".config_aio_connectors_namespace") -Raw).Trim()
+Write-Host "AIO connectors namespace: $aioConnectorsNamespace"
+
+Write-Host "AEP Name: $aepName"
+
+Write-Host "Asset Name: $assetName"
+
+Write-Host "Datapoint Name: $datapointName"
+
+Write-Host "Installing the resource files..."
+. (Join-Path $PSScriptRoot "Install-ResourceFile.ps1") -resourceFile "${aepName}.yaml"
+. (Join-Path $PSScriptRoot "Install-ResourceFile.ps1") -resourceFile "${assetName}.yaml"
+
+Write-Host "Waiting for the snapshot-to-mqtt task to be ready...`n"
+Start-Sleep -Seconds 3
+
+try {
+    Write-Host "Starting the monitor command...`n"
+    $monitorExpresion = $ExecutionContext.InvokeCommand.ExpandString(${monitorExpresion})
+    Invoke-Expression "${monitorExpresion}"
+
+} finally {
+
+    Write-Host "Uninstalling the resource files...`n"
+    . (Join-Path $PSScriptRoot "Uninstall-ResourceFile.ps1") -resourceFile "${assetName}.yaml"
+    . (Join-Path $PSScriptRoot "Uninstall-ResourceFile.ps1") -resourceFile "${aepName}.yaml"
+
+    Write-Host "Snapshot-to-mqtt task test completed.`n"
+}

samples/media-connector-invoke-test/Invoke-TestClipToFsAutostart.ps1

@@ -0,0 +1,20 @@
+#Requires -Version 7
+
+$aioConnectorsNamespace = (Get-Content -Path (Join-Path -Path $PSScriptRoot -ChildPath ".config_aio_connectors_namespace") -Raw).Trim()
+Write-Host "AIO connectors namespace: $aioConnectorsNamespace"
+
+$aepName = "aep-public-http-anonymous-1"
+Write-Host "AEP name: $aepName"
+
+$assetName = "asset-public-http-anonymous-1-clip-to-fs-autostart"
+Write-Host "Asset name: $assetName"
+
+$datapointName = "clip-to-fs"
+Write-Host "Datapoint name: $datapointName"
+
+. (Join-Path -Path $PSScriptRoot -ChildPath "Invoke-ResourceInstallMonitorAndUninstall.ps1") `
+    -aepName $aepName `
+    -assetName $assetName `
+    -datapointName $datapointName `
+    -monitorExpresion `
+        "& (Join-Path -Path $PSScriptRoot -ChildPath `"Start-FileSystemMonitor.ps1`") -pathToMonitor /tmp/$aioConnectorsNamespace/data/$assetName/"

samples/media-connector-invoke-test/Invoke-TestSnapshotToFsAutostart.ps1

@@ -0,0 +1,20 @@
+#Requires -Version 7
+
+$aioConnectorsNamespace = (Get-Content -Path (Join-Path -Path $PSScriptRoot -ChildPath ".config_aio_connectors_namespace") -Raw).Trim()
+Write-Host "AIO connectors namespace: $aioConnectorsNamespace"
+
+$aepName = "aep-public-http-anonymous-1"
+Write-Host "AEP name: $aepName"
+
+$assetName = "asset-public-http-anonymous-1-snapshot-to-fs-autostart"
+Write-Host "Asset name: $assetName"
+
+$datapointName = "snapshot-to-fs"
+Write-Host "Datapoint name: $datapointName"
+
+. (Join-Path -Path $PSScriptRoot -ChildPath "Invoke-ResourceInstallMonitorAndUninstall.ps1") `
+    -aepName $aepName `
+    -assetName $assetName `
+    -datapointName $datapointName `
+    -monitorExpresion `
+        "& (Join-Path -Path $PSScriptRoot -ChildPath `"Start-FileSystemMonitor.ps1`") -pathToMonitor /tmp/$aioConnectorsNamespace/data/$assetName/"

samples/media-connector-invoke-test/Invoke-TestSnapshotToMqttAutostart.ps1

@@ -0,0 +1,20 @@
+#Requires -Version 7
+
+$aioConnectorsNamespace = (Get-Content -Path (Join-Path -Path $PSScriptRoot -ChildPath ".config_aio_connectors_namespace") -Raw).Trim()
+Write-Host "AIO connectors namespace: $aioConnectorsNamespace"
+
+$aepName = "aep-public-http-anonymous-1"
+Write-Host "AEP name: $aepName"
+
+$assetName = "asset-public-http-anonymous-1-snapshot-to-mqtt-autostart"
+Write-Host "Asset name: $assetName"
+
+$datapointName = "snapshot-to-mqtt"
+Write-Host "Datapoint name: $datapointName"
+
+. (Join-Path -Path $PSScriptRoot -ChildPath "Invoke-ResourceInstallMonitorAndUninstall.ps1") `
+    -aepName $aepName `
+    -assetName $assetName `
+    -datapointName $datapointName `
+    -monitorExpresion `
+        "& (Join-Path -Path $PSScriptRoot -ChildPath `"Start-MqttListener.ps1`") -listenTopics $aioConnectorsNamespace/data/$assetName/#"

samples/media-connector-invoke-test/Invoke-TestStreamToRtspAutostart.ps1

@@ -0,0 +1,21 @@
+#Requires -Version 7
+
+$aioConnectorsNamespace = (Get-Content -Path (Join-Path -Path $PSScriptRoot -ChildPath ".config_aio_connectors_namespace") -Raw).Trim()
+Write-Host "AIO connectors namespace: $aioConnectorsNamespace"
+
+$aepName = "aep-public-http-anonymous-1"
+Write-Host "AEP name: $aepName"
+
+$assetName = "asset-public-http-anonymous-1-stream-to-rtsp-autostart"
+Write-Host "Asset name: $assetName"
+
+$datapointName = "stream-to-rtsp"
+Write-Host "Datapoint name: $datapointName"
+
+. (Join-Path -Path $PSScriptRoot -ChildPath "Invoke-ResourceInstallMonitorAndUninstall.ps1") `
+    -aepName $aepName `
+    -assetName $assetName `
+    -datapointName $datapointName `
+    -monitorExpresion `
+        "& (Join-Path -Path $PSScriptRoot -ChildPath `"Start-RtspStreamViewer.ps1`") -assetName $assetName ; `
+        try { Write-Host `"`nHit Ctrl+C to terminate`" ; Start-Sleep -Seconds 3600 } finally { Write-Host `"`nContinuing...`n`" }"

samples/media-connector-invoke-test/Overview.md

@@ -0,0 +1,29 @@
+# What is the media connector?
+
+The media connector makes media from media sources such as edge attached cameras available to other Azure IoT Operations components. The media connector is secure, performant, and can connect to the following media sources:
+
+| Media source | Example URLs | Notes |
+|--------------| ---------------|-------|
+| Edge attached camera | `file://host/dev/video0`<br/>`file://host/dev/usb0` | No authentication required. The URL refers to the device file. Connects to a node using USB, FireWire, MIPI, or proprietary interface. |
+| IP camera | `rtsp://192.168.178.45:554/stream1` | JPEG over HTTP for snapshots, RTSP/RTCP/RTP/MJPEG-TS for video streams. An IP camera might also expose a standard ONVIF control interface. |
+| Media server | `rtsp://192.168.178.45:554/stream1` | JPEG over HTTP for snapshots, RTSP/RTCP/RTP/MJPEG-TS for video streams. A media server can also serve images and videos using URLs such as `ftp://host/path` or `smb://host/path` |
+| Media file | `http://camera1/snapshot/profile1`<br/>`nfs://server/path/file.extension`<br/>` file://localhost/media/path/file.mkv`  | Any media file with a URL accessible from the cluster. |
+| Media folder | `file://host/path/to/folder/`<br/>`ftp://server/path/to/folder/` | A folder, accessible from the cluster, that contain media files such as snapshots or clips. |
+
+Example uses of the media connector include:
+
+- Capture snapshots from a video stream or from an image URL and publish them to an MQTT topic. A subscriber to the MQTT topic can use the captured images for further processing or analysis.
+
+- Save video streams to a local file system on your cluster. [Edge Storage Accelerator](https://learn.microsoft.com/azure/azure-arc/edge-storage-accelerator/) can provide a reliable and fault-tolerant solution for uploading the captured video to the cloud for storage or processing.
+
+- Proxy a live video stream from a camera to an endpoint that an operator can access. For security and performance reasons, only the media connector should have direct access to an edge camera. The media connector uses a separate media server component to stream video to an operator's endpoint. This media server can transcode to a variety of protocols such as RTSP, RTCP, SRT, and HLS.
+
+## How does it relate to Azure IoT Operations?
+
+The media connector component is part of Azure IoT Operations. The connector deploys to an Arc-enabled Kubernetes cluster on the edge as part of an Azure IoT Operations deployment. The connector interacts with other Azure IoT Operations elements, such as:
+
+- _Asset endpoints_, which are custom resources in your Kubernetes cluster that define connections to resources such as cameras. An asset endpoint configuration includes the URL of the media source, the type of media source, and any credentials that are needed to access the media source. The media connector uses an asset endpoint to access the media source.
+
+- _Assets_, which in Azure IoT Operations Preview are logical entities that you create to represent a real assets such as cameras. An Azure IoT Operations camera asset can have properties, tags, and video streams.
+
+- The Azure IoT Operations portal, which is a web UI that provides a unified experience for you to manage assets such as cameras. You can use the portal to configure the assets and asset endpoints that the media connector uses to access media sources.

samples/media-connector-invoke-test/README.md

@@ -1,8 +1,102 @@
-# Azure IoT Operations (AIO) Media Connector Demo
-
-> [!NOTE]  
-> Ensure you're using Azure IoT Operations version 1.0.0a3 or later before moving forward.
+# Azure IoT Operations media connector demo
 
 ## Introduction
 
-This collection consists of .yaml files that could be utilized by the sample commands found in the introductory document. 
+This document describes the Azure IoT Operations media connector demo package.
+
+The Azure IoT Operations media connector is part of the Azure IoT Operations platform:
+
+- It is designed to be secure, scalable and fault-tolerant.
+- It is responsible for the ingestion, storage, and distribution of media content.
+- It takes care of the management of media metadata and the generation of media thumbnails.
+
+## Demo environment diagram
+
+The demo environment consists of the following components:
+
+![Demo environment diagram](media-connector-demo.png)
+
+The dotted lines represent connections and components that are possible but not show in the demo package.
+
+## Getting started
+
+1. The scripts are designed to run on PowerShell 7 or newer.
+   - You can test if you have the necessary prerequisites by running `Test-Prerequisites.ps1`.
+   - The *Installation of prerequisites* section below has useful relevant information.
+   - This demo folder should be in a path without spaces.
+1. [Deploy Azure IoT Operations](https://aka.ms/getAzure IoT Operations).
+   - The Azure IoT Operations Kubernetes cluster should be configured as the `kubectl` current context.
+1. Upgrade the public preview components by running the script `Update-Azure IoT OperationsMediaConnector.ps1` in the `update-Azure IoT Operations/` folder.
+1. Install the media server using the files in the `media-server/` folder.
+1. You should have a listener without TLS configured on port 1883.
+   - You can verify by calling the `Update-Azure IoT OperationsMqEndpointFile.ps1` script.
+   - You can use the files under `broker-listener/` to deploy this.
+1. Then you can run each of the test scripts `Invoke-Test*.ps1` to run different test scenarios:
+   - `Invoke-TestSnapshotToMqttAutostart.ps1` takes snapshots from the demo stream and publishes them to the MQTT broker.
+   - `Invoke-TestSnapshotToFsAutostart.ps1` takes snapshots from the demo stream and writes them as files to the file system.
+   - `Invoke-TestClipToFsAutostart.ps1` creates clips at regular intervals from the demo stream and writes them as files to the file system.
+   - `Invoke-TestStreamToRtspAutostart.ps1` pushes the demo stream to a media server, from where it can be retrieved.
+
+   The scripts deploy the endpoint and asset and monitors their activity. Use `Ctrl+C` to end the monitoring, remove the endpoint and asset and terminate the script.
+
+## Installation of prerequisites
+
+To install the prerequisites you can follow the instructions below.
+
+These might not be the preferred installation procedure for your system and IT environment.
+
+Check with your administrator before installing tools.
+
+### Windows
+
+Run these commands from the command line:
+
+`winget install -e --id Microsoft.PowerShell`
+`winget install -e --id Kubernetes.kubectl`
+`winget install -e --id Microsoft.AzureCLI`
+`winget install -e --id Helm.Helm`
+`winget install -e --id EclipseFoundation.Mosquitto`
+
+You might need to add `C:\Program Files\mosquitto` and `helm.exe` to your PATH.
+
+### Ubuntu GNU/Linux
+
+Follow the official documentation to install:
+- [PowerShell](https://learn.microsoft.com/en-us/powershell/scripting/install/install-ubuntu)
+- [Azure CLI](https://learn.microsoft.com/en-us/cli/azure/install-azure-cli-linux)
+- [Helm](https://helm.sh/docs/intro/install/)
+- kubectl(depends on your system)
+
+You can install the mosquitto clients by running:
+`sudo apt install mosquitto-clients`
+
+## Changes since Azure IoT Operations public preview
+
+- Additional test scripts have been added to demonstrate the capabilities of the Azure IoT Operations media connector.
+- Now using PowerShell scripts instead of Polyglot notebooks.
+
+## Limitations
+
+The Azure IoT Operations media connector has the following limitations:
+
+- No discovery, this will be implemented by the ONVIF connector, currently under development.
+- Limits on the number of concurrent connections and the file system use are not enforced.
+- Performance and footprint are not optimized.
+- The mRPC API is not publicly documented yet and is subject to change.
+
+## Description of package contents:
+
+- **README.md**: This file.
+- **Overview.md**: General information about Azure IoT Operations and the Azure IoT Operations media connector.
+- **media-connector-demo.mermaid and produced .png and .svg**: Diagram of the demo environment.
+- **Broker listener (broker-listener/)**: This folder contains PowerShell scripts and kuberentes resources that show how to deploy an open (non-TLS) listener for MQ.
+- **Media Server (media-server/)**: This folder contains scripts and yaml files that demonstrates how to deploy a media server in a kubernetes cluster.
+- **resources/aep-*.yaml**: Example Asset Endpoint Profiles (AEPs) that can be used to configure the media connector.
+- **resources/asset-*.yaml**: Example assets that can be used to configure the media connector.
+- **Install-ResourceFile.ps1**: This PowerShell script installs a kubernetes resource file for the Azure IoT Operations media connector.
+- **Uninstall-ResourceFile.ps1**: This PowerShell script uninstalls a kubernetes resource file for the Azure IoT Operations media connector.
+- **Invoke-Test*.ps1**: These PowerShell scripts run different test scenarios for the Azure IoT Operations media connector.
+- **Start-InteractiveSession.ps1**: This PowerShell script starts an interactive session in the Azure IoT Operations media connector container.
+- **Start-MqttListener.ps1**: This PowerShell script starts an MQTT listener on the Azure IoT Operations MQTT broker.
+- **Start-FileSystemMonitor.ps1**: This PowerShell script start monitoring for file changes on the Azure IoT Operations media connector container.
+- **Start-RtspStreamViewer.ps1**: This PowerShell script start the default browser to look at streaming RTPS from the default media server.

samples/media-connector-invoke-test/Start-FileSystemMonitor.ps1

@@ -0,0 +1,50 @@
+#Requires -Version 7
+<#
+    Start a file monitor session in the AIO Media Connector pod.
+#>
+param (
+    [Parameter(
+        HelpMessage="The path to monitor.")]
+    [string]$pathToMonitor = ""
+)
+
+Write-Host (Split-Path -Path $PSCommandPath -Leaf).ToUpper() -ForegroundColor White
+
+. (Join-Path $PSScriptRoot "Test-Prerequisites.ps1")
+
+$aioNamespace = (Get-Content -Path (Join-Path $PSScriptRoot ".config_aio_namespace") -Raw).Trim()
+Write-Host "AIO namespace: $aioNamespace"
+
+$aioConnectorsNamespace = (Get-Content -Path (Join-Path $PSScriptRoot ".config_aio_connectors_namespace") -Raw).Trim()
+Write-Host "AIO connectors namespace: $aioConnectorsNamespace"
+
+If ($pathToMonitor -eq "") {
+    $pathToMonitor = "/tmp/${aioNamespace}"
+}
+Write-Host "Path to monitor: $pathToMonitor"
+
+$podName = (kubectl get pods -n $aioConnectorsNamespace -l app.kubernetes.io/component=aio-opc-media-1 --output=jsonpath='{.items[*].metadata.name}')
+if ($null -eq $podName) {
+    Write-Host "No pod found"
+    Exit 1
+}
+Write-Host "Pod name: $podName"
+
+try {
+    Write-Host "Checking if inotifywait is installed..."
+    $iNotifyWaitPath = "/usr/bin/inotifywait"
+    $commandString = "kubectl exec --stdin --tty $podName -n $aioConnectorsNamespace -- ls ${iNotifyWaitPath}"
+    $result = Invoke-Expression "${commandString}"
+    If ($result -eq $iNotifyWaitPath) {
+        Write-Host "Starting the file system monitor based on inotifywait..."
+        $commandString = "kubectl exec --stdin --tty $podName -n $aioConnectorsNamespace -- inotifywait -m -r -e create -e delete -e modify --timefmt `"%Y-%m-%d %H:%M:%S`" --format `"[%T] [%e] [%w] [%f]`" ${pathToMonitor}"
+        Invoke-Expression "${commandString}"
+    } Else {
+        Write-Host "inotifywait is not installed, pooling with find..."
+        $commandString = "kubectl exec --stdin --tty $podName -n $aioConnectorsNamespace -- sh -c `'while true ; do find ${pathToMonitor} -cmin 0.05 ; sleep 3 ; done `'"
+        Invoke-Expression "${commandString}"
+    }
+
+} finally {
+    Write-Host "`nThe file system monitor ended."
+}