From 2eada5aeeec7b852b3030b9661b8380d8974f292 Mon Sep 17 00:00:00 2001 From: ayakoakasaka <98828539+ayakoakasaka@users.noreply.github.com> Date: Wed, 29 Nov 2023 16:13:09 +0100 Subject: [PATCH] Import ideas (#2) * Import ideas from other repo --- .clang-format | 12 + .devcontainer/Dockerfile | 9 + .devcontainer/devcontainer.json | 32 +++ .devcontainer/setup.sh | 30 ++ .gitattributes | 2 + .github/workflows/main.yml | 6 +- .github/workflows/wasmtime.yml | 130 +++++++++ .gitignore | 2 + LICENSE.md | 8 - README.md | 279 +++++++++++++++---- frame_management.md | 99 +++++++ guest/Cargo.toml | 16 ++ guest/README.md | 15 + guest/build.sh | 18 ++ guest/src/lib.rs | 210 ++++++++++++++ guest_c/build.sh | 58 ++++ guest_c/main.c | 182 ++++++++++++ host/Cargo.toml | 22 ++ host/README.md | 10 + host/src/dummy_device.rs | 146 ++++++++++ host/src/main.rs | 321 +++++++++++++++++++++ host/src/nokhwa.rs | 137 +++++++++ host/src/pool.rs | 59 ++++ host/src/traits.rs | 23 ++ imports.md | 72 ----- renovate.json | 12 + sensing.md | 402 +++++++++++++++++++++++++++ test/README.md | 11 - wit/deps/buffer-pool/buffer-pool.wit | 100 +++++++ wit/deps/buffer-pool/data-types.wit | 66 +++++ wit/deps/example-dep/example-api.wit | 8 - wit/example.wit | 59 ---- wit/property.wit | 36 +++ wit/sensor.wit | 62 +++++ wit/world.wit | 10 +- 35 files changed, 2441 insertions(+), 223 deletions(-) create mode 100644 .clang-format create mode 100644 .devcontainer/Dockerfile create mode 100644 .devcontainer/devcontainer.json create mode 100644 .devcontainer/setup.sh create mode 100644 .gitattributes create mode 100644 .github/workflows/wasmtime.yml create mode 100644 .gitignore delete mode 100644 LICENSE.md create mode 100644 frame_management.md create mode 100644 guest/Cargo.toml create mode 100644 guest/README.md create mode 100755 guest/build.sh create mode 100644 guest/src/lib.rs create mode 100755 guest_c/build.sh create mode 100644 guest_c/main.c create mode 100644 host/Cargo.toml create mode 100644 host/README.md create mode 100644 host/src/dummy_device.rs create mode 100644 host/src/main.rs create mode 100644 host/src/nokhwa.rs create mode 100644 host/src/pool.rs create mode 100644 host/src/traits.rs delete mode 100644 imports.md create mode 100644 renovate.json create mode 100644 sensing.md delete mode 100644 test/README.md create mode 100644 wit/deps/buffer-pool/buffer-pool.wit create mode 100644 wit/deps/buffer-pool/data-types.wit delete mode 100644 wit/deps/example-dep/example-api.wit delete mode 100644 wit/example.wit create mode 100644 wit/property.wit create mode 100644 wit/sensor.wit diff --git a/.clang-format b/.clang-format new file mode 100644 index 0000000..c7d85c3 --- /dev/null +++ b/.clang-format @@ -0,0 +1,12 @@ +BasedOnStyle: LLVM +PointerAlignment: Right +ColumnLimit: 79 +IndentWidth: 8 +TabWidth: 8 +ContinuationIndentWidth: 8 +ConstructorInitializerIndentWidth: 8 +AlwaysBreakAfterReturnType: TopLevelDefinitions +BreakBeforeBraces: Linux +IndentGotoLabels: false +ForEachMacros: ["ARRAY_FOREACH", "LIST_FOREACH", "LIST_FOREACH_REVERSE", "VEC_FOREACH", "VEC_FOREACH_IDX"] +SpaceBeforeParens: ControlStatementsExceptControlMacros diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile new file mode 100644 index 0000000..b319790 --- /dev/null +++ b/.devcontainer/Dockerfile @@ -0,0 +1,9 @@ +FROM ubuntu:22.04 + +WORKDIR /home/ + +COPY . . + +ENV PATH="/root/.cargo/bin:$PATH" + +RUN bash ./setup.sh \ No newline at end of file diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 0000000..89423e4 --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,32 @@ +{ + "name": "Codespaces Rust Starter", + "extensions": [ + "matklad.rust-analyzer", + "serayuzgur.crates", + "vadimcn.vscode-lldb", + "ms-vscode.makefile-tools", + "ms-vscode.cpptools-extension-pack" + ], + "dockerFile": "Dockerfile", + "initCommands": [ + "settings set target.disable-aslr false" + ], + "settings": { + "editor.formatOnSave": true, + "terminal.integrated.profiles.linux": { + "zsh": { + "path": "/usr/bin/zsh", + "environment": [ + "RUST_BACKTRACE=1", + "RUST_LOG=debug", + "RUST_LOG_STYLE=always" + ] + } + }, + "terminal.integrated.defaultProfile.linux": "zsh", + "files.exclude": { + "**/CONTRIBUTING.md": true, + "**/LICENSE": true + } + } +} \ No newline at end of file diff --git a/.devcontainer/setup.sh b/.devcontainer/setup.sh new file mode 100644 index 0000000..af2af92 --- /dev/null +++ b/.devcontainer/setup.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +## update and install 1st level of packages +apt-get update +apt-get install -y \ + curl \ + git \ + gnupg2 \ + jq \ + sudo \ + zsh \ + build-essential \ + cmake \ + libssl-dev \ + openssl \ + unzip \ + g++-12 \ + +update-alternatives --install /usr/bin/c++ c++ /usr/bin/g++-12 50 + +## Add package directories to PATH +export PATH="/usr/bin:/usr/local/bin:$PATH" + +## update and install 2nd level of packages +apt-get install -y pkg-config + +## install rustup and common components +curl https://sh.rustup.rs -sSf | sh -s -- -y + +source $HOME/.cargo/env diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..33e8c66 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,2 @@ +# A hack to enable some degree of syntax highlighting on GitHub. Should be removed if GitHub ever receives native support for Wit files. +*.wit linguist-language=Rust \ No newline at end of file diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index e210671..f5cae17 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -10,5 +10,7 @@ jobs: name: Check ABI files are up-to-date runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - uses: WebAssembly/wit-abi-up-to-date@v15 + - uses: actions/checkout@v4 + - uses: WebAssembly/wit-abi-up-to-date@v16 + with: + wit-bindgen: '0.14.0' diff --git a/.github/workflows/wasmtime.yml b/.github/workflows/wasmtime.yml new file mode 100644 index 0000000..fcdf613 --- /dev/null +++ b/.github/workflows/wasmtime.yml @@ -0,0 +1,130 @@ +name: Wasmtime + +on: + push: + branches: [ "main" ] + tags: + - "v*" + pull_request: + branches: [ "main" ] + +concurrency: + group: cmake-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + build-and-run: + env: + CC: ${{matrix.compiler}} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest] + runs-on: ${{matrix.os}} + + steps: + - name: Install dependencies (rustup) + run: | + rustup target add wasm32-wasi + + - name: Install dependencies (cargo) + run: | + cargo install --version 1.0.51 wasm-tools + cargo install --version 0.14.0 wit-bindgen-cli + + - name: Install dependencies (ubuntu) + if: startsWith(matrix.os, 'ubuntu-') + run: sudo apt-get update && sudo apt-get install -y pax + + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Install wasmtime (ubuntu) + if: startsWith(matrix.os, 'ubuntu-') + run: | + curl -O -L https://github.com/bytecodealliance/wasmtime/releases/download/v14.0.4/wasmtime-v14.0.4-x86_64-linux.tar.xz + tar xvJf wasmtime-v14.0.4-x86_64-linux.tar.xz + echo "WASMTIME=$(pwd -P)/wasmtime-v14.0.4-x86_64-linux/wasmtime" >> ${GITHUB_ENV} + + - name: Install wasmtime (macOS) + if: startsWith(matrix.os, 'macos-') + run: | + curl -O -L https://github.com/bytecodealliance/wasmtime/releases/download/v14.0.4/wasmtime-v14.0.4-x86_64-macos.tar.xz + tar xvzf wasmtime-v14.0.4-x86_64-macos.tar.xz + echo "WASMTIME=$(pwd -P)/wasmtime-v14.0.4-x86_64-macos/wasmtime" >> ${GITHUB_ENV} + + - name: Build guest (Rust) + run: | + ./build.sh + working-directory: ${{github.workspace}}/guest + + - name: Build guest (C) + run: | + ./build.sh + working-directory: ${{github.workspace}}/guest_c + + - name: Build host + run: | + cargo build + working-directory: ${{github.workspace}}/host + + - name: Run (component) + run: | + cargo run ../guest/guest-component.wasm + working-directory: ${{github.workspace}}/host + + - name: Check the output (component) + run: | + test $(ls *.jpg|wc -l) -eq 60 + working-directory: ${{github.workspace}}/host + + - name: Clean up the output (component) + run: | + rm *.jpg + working-directory: ${{github.workspace}}/host + + - name: Run (wasmtime precompiled component) + run: | + cargo run ../guest/guest-component.cwasm + working-directory: ${{github.workspace}}/host + + - name: Check the output (wasmtime precompiled component) + run: | + test $(ls *.jpg|wc -l) -eq 60 + working-directory: ${{github.workspace}}/host + + - name: Clean up the output (wasmtime precompiled component) + run: | + rm *.jpg + working-directory: ${{github.workspace}}/host + + - name: Run (C, component) + run: | + cargo run ../guest_c/guest-component.wasm + working-directory: ${{github.workspace}}/host + + - name: Check the output (C, component) + run: | + test $(ls *.jpg|wc -l) -eq 60 + working-directory: ${{github.workspace}}/host + + - name: Clean up the output (C, component) + run: | + rm *.jpg + working-directory: ${{github.workspace}}/host + + - name: Run (C, wasmtime precompiled component) + run: | + cargo run ../guest_c/guest-component.cwasm + working-directory: ${{github.workspace}}/host + + - name: Check the output (C, precompiled component) + run: | + test $(ls *.jpg|wc -l) -eq 60 + working-directory: ${{github.workspace}}/host + + - name: Clean up the output (C, precompiled component) + run: | + rm *.jpg + working-directory: ${{github.workspace}}/host diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..9cea445 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +*/target/* +*.lock \ No newline at end of file diff --git a/LICENSE.md b/LICENSE.md deleted file mode 100644 index 4753095..0000000 --- a/LICENSE.md +++ /dev/null @@ -1,8 +0,0 @@ -Copyright © 2019-2023 the Contributors to the WASI Specification, published -by the [WebAssembly Community Group][cg] under the -[W3C Community Contributor License Agreement (CLA)][cla]. A human-readable -[summary][summary] is available. - -[cg]: https://www.w3.org/community/webassembly/ -[cla]: https://www.w3.org/community/about/agreements/cla/ -[summary]: https://www.w3.org/community/about/agreements/cla-deed/ diff --git a/README.md b/README.md index 733036b..00d747c 100644 --- a/README.md +++ b/README.md @@ -1,112 +1,271 @@ -# [Example WASI proposal] +# WASI-Sensor Idea Discussion + +A proposed [WebAssembly System Interface](https://github.com/WebAssembly/WASI) API from Midokura. + +Caution: This is a draft proposal and is subject to change. + + + +## Table of Contents + +- [WASI-Sensor Idea Discussion](#wasi-sensor-idea-discussion) + - [Table of Contents](#table-of-contents) + - [Introduction](#introduction) + - [Goals](#goals) + - [Non-goals, Out-of-scope](#non-goals-out-of-scope) + - [API walk-through](#api-walk-through) + - [Sensor lifecycle management](#sensor-lifecycle-management) + - [Sensor data quality control](#sensor-data-quality-control) + - [Sensor](#sensor) + - [\[Use case 1: GPS/GNSS, IMU (1-3D data) sensor control\]](#use-case-1-gpsgnss-imu-1-3d-data-sensor-control) + - [Candidate sensor framework (under wasi-sensor): RTKLIB](#candidate-sensor-framework-under-wasi-sensor-rtklib) + - [\[Use case 2: Image/Depth sensor (2/3D data) control\]](#use-case-2-imagedepth-sensor-23d-data-control) + - [Candidate sensor(camera) framework: v4l2, libcamera, GeniCam](#candidate-sensorcamera-framework-v4l2-libcamera-genicam) + - [Buffer-pool](#buffer-pool) + - [\[Use case 1: Buffer overwrite mode\]](#use-case-1-buffer-overwrite-mode) + - [\[Use case 2: Sensor virtualization\]](#use-case-2-sensor-virtualization) + - [Detailed design discussion](#detailed-design-discussion) + - [\[Relationship between wasi-i2c, wasi-spi #1\]](#relationship-between-wasi-i2c-wasi-spi-1) + - [\[Relationship between wasi-io #2\]](#relationship-between-wasi-io-2) + - [Stakeholder Interest and Feedback](#stakeholder-interest-and-feedback) + - [References and acknowledgements](#references-and-acknowledgements) + +## Introduction + +This interface helps reduce the effort required on the host side (platform vendor) to deal with a series of fragmented sensor frameworks. + +In regard to the interface required for subscribing to sensor data, this document also provides a buffer-pool interface as a complementary interface. +In the future this could be merged into other specifications, such as **wasi-messaging** or **wasi-io**, but for now, it will serve to enable unified control over systems with different scales of devices and OSs. + +## Goals + +The primary goal is to provide a platform-agnostic, unified sensor control interface to be used either above or below typical sensor/camera frameworks. + +This covers: + + - Sensor lifecycle management. + - Sensor data quality control. + - Sensor data type definition. + +```mermaid +flowchart TD + + subgraph case 2 + sensor_framework1--> wasi-sensor1; + wasi-sensor1 --> sensor1["sensor"] + end + subgraph case 1 + wasi-sensor --> sensor_framework; + sensor_framework --> sensor + sensor_framework1["sensor_framework"] + wasi-sensor1["wasi-sensor"] + end +``` -This template can be used to start a new proposal, which can then be proposed in the WASI Subgroup meetings. +> When the **wasi-sensor** is positioned above the **sensor_framework**, sensor data should be kept in host memory and processed by **sensor_framework** or other native processing. -The sections below are recommended. However, every proposal is different, and the community can help you flesh out the proposal, so don't block on having something filled in for each one of them. +> When the **wasi-sensor** is positioned below the **sensor_framework**, sensor data should be copied to linear memory and processed by **sensor_framework** or **webassembly module**. -Thank you to the W3C Privacy CG for the [inspiration](https://github.com/privacycg/template)! +The secondary goal is to provide the buffer-pool management interface for connecting various application use cases and various embedded or IoT environments. + +This covers: -# [Title] + - Buffer overwrite mode. + - Sensor virtualization. + +## Non-goals, Out-of-scope -A proposed [WebAssembly System Interface](https://github.com/WebAssembly/WASI) API. +- Support Media framework, such as live video/audio streaming. +- Support Signal/Data processing, such as filtering or converting the format. -### Current Phase +## API walk-through -[Fill in the current phase, e.g. Phase 1] +The full API documentation can be found [here](sensing.md). -### Champions +### Sensor lifecycle management -- [Champion 1] -- [Champion 2] -- [etc.] +- This interface provides access to sensor controllers owned by vendors and other sensor frameworks. +Here, sensor (device) is treated as the resource, which the **webassembly module** can access via a handle. Exclusive access to the device would be set by defining ownership to the handle. -### Phase 4 Advancement Criteria +The concept here is similar to [Sensor object](https://www.w3.org/TR/generic-sensor/#sensor) -TODO before entering Phase 2. +```rust -## Table of Contents [if the explainer is longer than one printed page] +interface sensor { + use property.{property-key, property-value}; -- [Introduction](#introduction) -- [Goals [or Motivating Use Cases, or Scenarios]](#goals-or-motivating-use-cases-or-scenarios) -- [Non-goals](#non-goals) -- [API walk-through](#api-walk-through) - - [Use case 1](#use-case-1) - - [Use case 2](#use-case-2) -- [Detailed design discussion](#detailed-design-discussion) - - [[Tricky design choice 1]](#tricky-design-choice-1) - - [[Tricky design choice 2]](#tricky-design-choice-2) -- [Considered alternatives](#considered-alternatives) - - [[Alternative 1]](#alternative-1) - - [[Alternative 2]](#alternative-2) -- [Stakeholder Interest & Feedback](#stakeholder-interest--feedback) -- [References & acknowledgements](#references--acknowledgements) + // Sensor device + resource device { + /// open the device. + /// this might power on the device. + open: static func(name: string) -> result; -### Introduction + /// get a list of names of devices available on the system. + list-names: static func() -> result, device-error>; -[The "executive summary" or "abstract". Explain in a few sentences what the goals of the project are, and a brief overview of how the solution works. This should be no more than 1-2 paragraphs.] + /// start sending the data to buffer + start: func( + buffer-pool: string + )->result<_, device-error>; -### Goals [or Motivating Use Cases, or Scenarios] + /// stop sending the data to buffer + stop: func( + )->result<_, device-error>; -[What is the end-user need which this project aims to address?] + /// set property + set-property: func( + key: property-key, + value: property-value + ) ->result<_, device-error>; -### Non-goals + /// get property + get-property: func( + property: property-key + )->result; + } +} +``` -[If there are "adjacent" goals which may appear to be in scope but aren't, enumerate them here. This section may be fleshed out as your design progresses and you encounter necessary technical and other trade-offs.] +### Sensor data quality control -### API walk-through +The settings for improving the data quality depends on the device, but by using generalized interface named as property it is possible to control this using the above sensor interface. -The full API documentation can be found [here](wasi-proposal-template.md). +This is the example of the kind of setting that is prepared: -[Walk through of how someone would use this API.] +```rust + record dimension { + /// Image width. + width: u32, + /// Image height. + height: u32, + /// Image stride + stride-bytes: u32, + /// The format of a pixel. + pixel-format: pixel-format, + } +``` -#### [Use case 1] +```rust + variant property { -[Provide example code snippets and diagrams explaining how the API would be used to solve the given problem] + dimension(dimension), + sampling-rate(sampling-rate), -#### [Use case 2] + ... + } +``` -[etc.] +### Sensor + +#### [Use case 1: GPS/GNSS, IMU (1-3D data) sensor control] -### Detailed design discussion +> Once the sensor is activated via **wasi-sensor**, raw data is published from **GPS**/**GNSS**/**IMU** sensor with each vendor's original format and converted to a generic position and posture data through some software components provided by each vendor. +**Wasi-sensor** supports passing this general data format in unified way. +Later on, these information would be processed to be used for localization, mapping, posture control, and so on, but for now it is out of scope for the **wasi-sensor**. -[This section should mostly refer to the .wit.md file that specifies the API. This section is for any discussion of the choices made in the API which don't make sense to document in the spec file itself.] +### Candidate sensor framework (under wasi-sensor): RTKLIB -#### [Tricky design choice #1] +#### [Use case 2: Image/Depth sensor (2/3D data) control] -[Talk through the tradeoffs in coming to the specific design point you want to make.] +> Once the sensor is activated, raw data is published from the Image/Depth sensor with each vendor's original format such as Bayer + alpha, and converted to a general data format, such as RGB24, through some software components provided by each vendor. +**Wasi-sensor** supports passing this general data format in a unified way. +```rust + variant frame-data { + /// data passed by value + by-value(data-type), + + /// a reference to host memory + host-memory(memory), + } ``` -// Illustrated with example code. + +These data generally have the dimensions of width and height and are handled with these information. +Later on, these information would be processed to be used for object recognision, video streaming, and so on, but for now it is out of the scope for the **wasi-sensor**. + +### Candidate sensor(camera) framework: v4l2, libcamera, GeniCam + +```rust + record image { + dimension: dimension, + payload: list, + } ``` -[This may be an open question, in which case you should link to any active discussion threads.] +```rust + record dimension { + /// Image width. + width: u32, + /// Image height. + height: u32, + /// Image stride + stride-bytes: u32, + /// The format of a pixel. + pixel-format: u8, + } +``` -#### [Tricky design choice 2] +### Buffer-pool -[etc.] +#### [Use case 1: Buffer overwrite mode] + +Realtime usecase --- Buffering off -### Considered alternatives +> Real-time performance (low latency) is important for highly interactive applications that do not require time-series data, such as object recognition, which only requires a single image for processing. In that case, buffering must be disabled and the latest data must always be supplied to the application. -[This section is not required if you already covered considered alternatives in the design discussion above.] +Sensing usecase --- Buffering on -#### [Alternative 1] +> Reliable data (usually time-series) is critical for sensing applications such as object tracking and behavior analysis. Unlike in a realtime use case, it is not necessary to provide real-time data to the application. The reliability is the main priority. In that case, buffering must be enabled to store a certain amount of time-series data with no room for error. -[Describe an alternative which was considered, and why you decided against it.] +Hence, the below buffering-mode interface setting is presented for the application. -#### [Alternative 2] +```rust + enum buffering-mode { + buffering-off, + buffering-discard, /**< Discard the latest frame. behave like queue */ + buffering-overwrite, /**< Overwrite the oldest frame. behave like ring */ + } +``` + +#### [Use case 2: Sensor virtualization] + +There are two motivations for virtualizing the **sensor**. + + 1. [Sensor Fusion](https://www.w3.org/TR/generic-sensor/#sensor-fusion) + + Produce another sensing data by fusioning several sensor data (for example, SLAM sensor). + + 2. [Platform-sensor](https://www.w3.org/TR/generic-sensor/#concept-platform-sensor) + + Use the same sensor data from multiple applications simultaneously. + +For this purpose, **buffer-pool** can work as a virtualized sensor. For this purpose, the **buffer-pool* part of the wasi-sensor might be improved as Post MVP. + +## Detailed design discussion + +Relationship between **sensor interface** and **buffer-pool interface** [here](frame_management.md). + +### [Relationship between [wasi-i2c](https://github.com/WebAssembly/wasi-i2c), wasi-spi #1] + +wasi-i2c is effective for special settings that cannot be covered by Wasi-sensor, so they will coexist. + +### [Relationship between [wasi-io](https://github.com/WebAssembly/wasi-io) #2] + +First, we would like to confirm the effectiveness in an environment with few resources, and then consider whether we can incorporate important essences into wasi-io. [etc.] -### Stakeholder Interest & Feedback + +## Stakeholder Interest and Feedback TODO before entering Phase 3. [This should include a list of implementers who have expressed interest in implementing the proposal] -### References & acknowledgements +## References and acknowledgements -Many thanks for valuable feedback and advice from: +Many thanks for valuable feedback and advice: -- [Person 1] -- [Person 2] +- [@username] (https://www.github.com/username) +- [@username] (https://www.github.com/username) - [etc.] diff --git a/frame_management.md b/frame_management.md new file mode 100644 index 0000000..a743056 --- /dev/null +++ b/frame_management.md @@ -0,0 +1,99 @@ +## Relationship between sensor interface and buffer-pool interface + +Sensors are always data publishers and will continue to provide the data at regular intervals. + +We assume sensor data would be temporarily stored in the buffer on the host side. This is because: + +- Sensor data frequency is relatively high or sensor data is relatively large in many cases to be used for sensing purpose +- Memory management in an environment with low resources should be performed on the host side + +Therefore, buffer-pool interface is proposed as an independent interface to manage the buffer-pool on the host side. +However, since the contents of the buffer pool are sensor data, the contents can be cast to sensor data type on the wasm side when data is passed to the wasm side. + +When connecting to other interfaces, such as the [Generic Sensor API](https://www.w3.org/TR/generic-sensor/), the [Sampling frequencies](https://www.w3.org/TR/generic-sensor/#concepts-sampling-and-reporting-frequencies) is connected to the sensor and the reporting frequency is connected to the buffer-pool. + +```mermaid +flowchart + + subgraph Legend + wasm + host + style wasm fill:#ffffff + end + + sensor -.->|publish|buffer-pool; + subgraph buffer-pool-wit + allocator_handle --> |create\ndestroy| allocator; + allocator --> |alloc\nfree| buffer-pool; + frame_handle --> |poll-read\nblock-read\nrelease|frame; + frame --> |peek| buffer-pool; + end + + subgraph sensor-wit + sensor_handle -->|open\nclose\nstart\nstop| sensor; + data_type --> |cast read|frame; + sensor --> |i2c/spi accesss| device; + end + + style sensor_handle fill:#ffffff + style data_type fill:#ffffff + style frame_handle fill:#ffffff + style allocator_handle fill:#ffffff + +``` + +## Considered alternatives + +### No Data Type definition? + +It may not be necessary to specify the data type of the sensor here, as WASI-NN does not specify a definition for the content of the inference results. +At a minimum, entensibility for this interface is needed. + +### No Buffer-pool definition? + +It is also possible to put buffer on the linear memory side. In that case, if the host memory and linear memory are not shared, copying will occur, but this method must be adopted if multiple processes handle the same data. + +```mermaid + + +flowchart TD + + subgraph Legend + wasm + host + style wasm fill:#ffffff + end + + subgraph sensor-wit + sensor_handle --->|open\nclose\nstart\nstop| sensor; + sensor --> device; + end + + + buffer-pool --> |qbuf|frame + frame --> |dbuf\ncopy| buffer-pool + frame-handler --> |poll-read\nblock-read\nrelease| frame + frame -.- |publish| sensor + + style sensor_handle fill:#ffffff + style buffer-pool fill:#ffffff + style frame-handler fill:#ffffff + +``` + +### Frame + +Frame has the below data structure. In order to be passed to other processing at host environment or Webassembly environment, handle or data itself should be stored together with the timestamp and other identifier information. + + record frame-info { + sequence-number: u64, + rawdata: list + } + + record raw-data { + /// when data is used from wasm (copied to linear memory) + wasm: option, + /// when data is kept at host and passed to other native processing + host: option, + timestamp: u64 + } diff --git a/guest/Cargo.toml b/guest/Cargo.toml new file mode 100644 index 0000000..210fb4d --- /dev/null +++ b/guest/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "guest" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[lib] +crate-type = ["cdylib"] + +[dependencies] +anyhow = "1.0.75" +fraction = "0.14" +getopts = "0.2" +wit-bindgen = "0.14.0" +image = { version = "0.24.7", default-features = false, features = ["jpeg"] } diff --git a/guest/README.md b/guest/README.md new file mode 100644 index 0000000..4e553b8 --- /dev/null +++ b/guest/README.md @@ -0,0 +1,15 @@ +# guest-component.wasm + +a test component which uses wasi-sensor and wasi preview2. + +## prerequisites + +``` +curl -L -O https://github.com/bytecodealliance/wasmtime/releases/download/dev/wasi_snapshot_preview1.reactor.wasm +``` + +## build + +``` +./build.sh +``` diff --git a/guest/build.sh b/guest/build.sh new file mode 100755 index 0000000..da793b9 --- /dev/null +++ b/guest/build.sh @@ -0,0 +1,18 @@ +#! /bin/sh +set -e +if [ ! -f wasi_snapshot_preview1.reactor.wasm ]; then + curl --fail -L -O https://github.com/bytecodealliance/wasmtime/releases/download/v14.0.4/wasi_snapshot_preview1.reactor.wasm +fi + +release_opt=--release +release=release + +#release_opt= +#release=debug + +cargo build ${release_opt} --target wasm32-wasi +wasm-tools component new \ +./target/wasm32-wasi/${release}/guest.wasm \ +-o guest-component.wasm \ +--adapt ./wasi_snapshot_preview1.reactor.wasm +WASMTIME_BACKTRACE_DETAILS=1 ${WASMTIME:-wasmtime} compile --wasm component-model guest-component.wasm diff --git a/guest/src/lib.rs b/guest/src/lib.rs new file mode 100644 index 0000000..f3b09cd --- /dev/null +++ b/guest/src/lib.rs @@ -0,0 +1,210 @@ +use anyhow::bail; +use anyhow::ensure; +use anyhow::Context; +use anyhow::Result; +use image::flat::FlatSamples; +use image::flat::SampleLayout; +use image::ImageBuffer; +use image::Rgb; + +use fraction::Fraction; +use getopts::Options; +use std::env; + +wit_bindgen::generate!({ + world: "sensing", + path: "../wit", + exports:{ + world: T, + "wasi:sensor/interface": T, + }, +}); + +struct T; + +use crate::exports::wasi::sensor::interface::Guest; + +// a naive implementation. maybe it's better to use fix point arithmetic. +// https://hk.interaction-lab.org/firewire/yuv.html +fn yuv_to_rgb(y: i32, u: i32, v: i32) -> Vec { + let y = 1.164 * ((y as f32) - 16.0); + let u = u as f32 - 128.0; + let v = v as f32 - 128.0; + let r = y + 1.596 * v; + let g = y - 0.293 * u - 0.813 * v; + let b = y + 2.018 * u; + let r = r.clamp(0.0, 255.0) as u8; + let g = g.clamp(0.0, 255.0) as u8; + let b = b.clamp(0.0, 255.0) as u8; + vec![r, g, b] +} + +fn convert_yuy2_to_rgb(width: u32, height: u32, stride: u32, yuv: &Vec) -> Vec { + let size = (3 * width * height) as usize; + let mut vec = Vec::with_capacity(size); + yuv.as_slice() + .chunks_exact(stride as usize) + .for_each(|row| { + row.chunks_exact(4).take(width as usize).for_each(|px| { + let y1 = i32::from(px[0]); + let u = i32::from(px[1]); + let y2 = i32::from(px[2]); + let v = i32::from(px[3]); + let rgb = yuv_to_rgb(y1, u, v); + vec.extend(rgb); + let rgb = yuv_to_rgb(y2, u, v); + vec.extend(rgb); + }) + }); + vec +} + +fn process_pixel_image(image: &wasi::buffer_pool::data_types::Image) -> Result<()> { + let dimension = &image.dimension; + let payload = &image.payload; + println!( + "guest: received a frame: dimension {:?} payload len {}", + dimension, + payload.len() + ); + let channels = 3; + let height_stride = dimension.stride_bytes; + + // convert to rgb + let converted; + let (height_stride, payload) = match dimension.pixel_format { + wasi::buffer_pool::data_types::PixelFormat::Rgb24 => (height_stride, payload), + wasi::buffer_pool::data_types::PixelFormat::Yuy2 => { + converted = convert_yuy2_to_rgb( + dimension.width, + dimension.height, + dimension.stride_bytes, + payload, + ); + (dimension.width * channels, &converted) + } + _ => { + println!( + "guest: dropping a frame with unimplemented format {:?}", + dimension.pixel_format + ); + return Ok(()); + } + }; + + let layout = SampleLayout { + channels: channels as u8, + channel_stride: 1, + width: dimension.width, + width_stride: channels as usize, + height: dimension.height, + height_stride: height_stride as usize, + }; + println!("layout {:?} payload len {}", layout, payload.len()); + let flat = FlatSamples { + samples: &payload[..], + layout: layout, + color_hint: None, + }; + + let buffer: ImageBuffer, &[u8]> = flat.try_into_buffer().unwrap(); + let unixtime_ns = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .expect("unix time") + .as_nanos(); + buffer.save(format!("{}.jpg", unixtime_ns))?; + Ok(()) +} + +fn process_frame(frame: &wasi::buffer_pool::buffer_pool::FrameInfo) -> Result<()> { + println!( + "got a frame with sequence number {} timestamp {}", + frame.sequence_number, frame.timestamp + ); + for (i, ref data) in frame.data.iter().enumerate() { + println!("frame-data {}", i); + match data { + wasi::buffer_pool::buffer_pool::FrameData::ByValue(ref data) => match data { + wasi::buffer_pool::data_types::DataType::Image(ref image) => { + process_pixel_image(image) + } + _ => todo!(), + }, + _ => todo!(), + } + .context("data type")? + } + Ok(()) +} + +fn main2() -> Result<()> { + let args: Vec = env::args().collect(); + let mut opts = Options::new(); + opts.optopt("", "pool", "pool name", "POOL"); + opts.optopt("", "sensor", "device name", "DEVICE"); + opts.optopt("", "sampling-rate", "samples per sec", "RATE"); + let matches = match opts.parse(&args[1..]) { + Ok(m) => m, + Err(f) => { + println!("{}", opts.usage(&format!("guest: {}", f.to_string()))); + return Err(f.into()); + } + }; + let pool_name = matches.opt_str("pool").unwrap_or("my-pool".to_string()); + let device_name = matches.opt_str("sensor").unwrap_or("dummy".to_string()); + let sampling_rate = matches.opt_get::("sampling-rate")?; + println!("pool name {}", pool_name); + println!("sensor device name {}", device_name); + println!("sampling rate {:?}", sampling_rate); + let frame_size_in_bytes = 0; // XXX + let number_of_frames = 16; + let pool = wasi::buffer_pool::buffer_pool::Pool::create( + wasi::buffer_pool::buffer_pool::BufferingMode::BufferingDiscard, + frame_size_in_bytes, + number_of_frames, + &pool_name, + )?; + + let device_names = wasi::sensor::sensor::Device::list_names()?; + println!("available devices: {:?}", device_names); + + let sensor = wasi::sensor::sensor::Device::open(&device_name)?; + println!("opened sensor {:?}", sensor); + + let value = sensor.get_property(wasi::sensor::property::PropertyKey::SamplingRate)?; + println!("sensor default sampling rate {:?}", value); + + if let Some(rate) = sampling_rate { + let rate = Fraction::from(rate); + let Fraction::Rational(sign, ratio) = rate else { + bail!("failed to process sampling rate {}", rate); + }; + ensure!(sign.is_positive(), "negative sampling rate {}", rate); + let value = + wasi::sensor::property::PropertyValue::Fraction(wasi::sensor::property::Fraction { + numerator: *ratio.numer() as u32, + denominator: *ratio.denom() as u32, + }); + println!("setting sampling rate to {:?}", value); + sensor.set_property(wasi::sensor::property::PropertyKey::SamplingRate, &value)?; + // confirm the result + let value = sensor.get_property(wasi::sensor::property::PropertyKey::SamplingRate)?; + println!("sensor sampling rate {:?}", value); + } + + println!("starting sensor {:?}", sensor); + sensor.start(&pool_name)?; + for _ in 0..60 { + let frame = pool.block_read_frame()?; + process_frame(&frame)?; + } + Ok(()) +} + +impl Guest for T { + fn main() -> Result<(), ()> { + println!("Hello, world!"); + main2().expect("main2"); + Ok(()) + } +} diff --git a/guest_c/build.sh b/guest_c/build.sh new file mode 100755 index 0000000..3185e5d --- /dev/null +++ b/guest_c/build.sh @@ -0,0 +1,58 @@ +#! /bin/sh +set -e +if [ ! -f wasi_snapshot_preview1.reactor.wasm ]; then + curl --fail -L -O https://github.com/bytecodealliance/wasmtime/releases/download/v14.0.4/wasi_snapshot_preview1.reactor.wasm +fi + +if [ ! -d libjpeg ]; then + mkdir libjpeg + cd libjpeg + curl --fail -L https://github.com/yamt/libjpeg/releases/download/wasm32-wasi-20231128/libjpeg-wasm32-wasi.tgz | pax -rvz + cd .. +fi + +# prepare wasi-sdk. this is copy-and-paste from: +# https://github.com/yamt/toywasm/blob/master/build-wasm32-wasi.sh + +MAJOR=${WASI_SDK_MAJOR:-20} +MINOR=${WASI_SDK_MINOR:-0} +WASI_SDK_DIR=${WASI_SDK_DIR:-$(pwd)/.wasi-sdk-${MAJOR}.${MINOR}} +DIST_DIR=.dist + +mkdir -p ${DIST_DIR} + +fetch_wasi_sdk() +{ + UNAME=$(uname -s) + case ${UNAME} in + Darwin) + PLATFORM=macos + ;; + Linux) + PLATFORM=linux + ;; + *) + echo "Unknown uname ${UNAME}" + exit 1 + ;; + esac + TAR=wasi-sdk-${MAJOR}.${MINOR}-${PLATFORM}.tar.gz + if [ ! -f ${DIST_DIR}/${TAR} ]; then + URL=https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-${MAJOR}/${TAR} + curl -L -o ${DIST_DIR}/${TAR} ${URL} + fi + pax -rz \ + -f ${DIST_DIR}/${TAR} \ + -s"!^wasi-sdk-${MAJOR}\.${MINOR}!${WASI_SDK_DIR}!" +} + +test -d "${WASI_SDK_DIR}" || fetch_wasi_sdk + +CC=${CC:-${WASI_SDK_DIR}/bin/clang} +wit-bindgen c ../wit +${CC} -I./libjpeg/include -mexec-model=reactor -o guest.wasm main.c sensing.c sensing_component_type.o -L./libjpeg/lib -ljpeg +wasm-tools component new \ +guest.wasm \ +-o guest-component.wasm \ +--adapt ./wasi_snapshot_preview1.reactor.wasm +WASMTIME_BACKTRACE_DETAILS=1 ${WASMTIME:-wasmtime} compile --wasm component-model guest-component.wasm diff --git a/guest_c/main.c b/guest_c/main.c new file mode 100644 index 0000000..336b9c0 --- /dev/null +++ b/guest_c/main.c @@ -0,0 +1,182 @@ +#include +#include +#include +#include +#include +#include + +#include + +#include "sensing.h" + +#define STRING_LITERAL(str) \ + { \ + .ptr = (uint8_t *)str, .len = sizeof(str) - 1 \ + } + +sensing_string_t device_name = STRING_LITERAL("dummy"); +sensing_string_t pool_name = STRING_LITERAL("my-pool"); + +bool +process_pixel_image(const wasi_buffer_pool_data_types_image_t *image) +{ + const wasi_buffer_pool_data_types_dimension_t *dimension = + &image->dimension; + const uint8_t *payload = image->payload.ptr; + const size_t payload_len = image->payload.len; + + switch (dimension->pixel_format) { + case WASI_BUFFER_POOL_DATA_TYPES_PIXEL_FORMAT_RGB24: + break; + default: + fprintf(stderr, "unimplemented pixel format %u\n", + dimension->pixel_format); + return false; + } + + struct timespec tv; + if (clock_gettime(CLOCK_REALTIME, &tv)) { + fprintf(stderr, "clock_gettime failed\n"); + return false; + } + uintmax_t timestamp_ns = + (uintmax_t)tv.tv_sec * 1000000000 + tv.tv_nsec; + char filename[PATH_MAX]; + snprintf(filename, sizeof(filename), "%ju.jpg", timestamp_ns); + + FILE *fp = fopen(filename, "w"); + if (fp == NULL) { + fprintf(stderr, "failed to open a file %s: %s\n", filename, + strerror(errno)); + return false; + } + + struct jpeg_compress_struct cinfo; + struct jpeg_error_mgr jerr; + cinfo.err = jpeg_std_error(&jerr); + jpeg_create_compress(&cinfo); + jpeg_stdio_dest(&cinfo, fp); + cinfo.image_width = dimension->width; + cinfo.image_height = dimension->height; + cinfo.in_color_space = JCS_RGB; + cinfo.input_components = 3; + jpeg_set_defaults(&cinfo); + jpeg_start_compress(&cinfo, TRUE); + uint32_t i; + for (i = 0; i < dimension->height; i++) { + /* we assume 8-bit JSAMPLE */ + JSAMPROW row = (JSAMPROW)&payload[dimension->stride_bytes * i]; + jpeg_write_scanlines(&cinfo, &row, 1); + } + jpeg_finish_compress(&cinfo); + jpeg_destroy_compress(&cinfo); + fclose(fp); + return true; +} + +bool +process_frame_info(wasi_buffer_pool_buffer_pool_frame_info_t *frame) +{ + fprintf(stderr, "sequence_number %" PRIu64 ", timestamp %ju\n", + frame->sequence_number, (uintmax_t)frame->timestamp); + size_t i; + for (i = 0; i < frame->data.len; i++) { + wasi_buffer_pool_buffer_pool_frame_data_t *data = + &frame->data.ptr[i]; + wasi_buffer_pool_data_types_data_type_t *data_type; + wasi_buffer_pool_data_types_image_t *image; + switch (data->tag) { + case WASI_BUFFER_POOL_BUFFER_POOL_FRAME_DATA_BY_VALUE: + data_type = &data->val.by_value; + switch (data_type->tag) { + case WASI_BUFFER_POOL_DATA_TYPES_DATA_TYPE_IMAGE: + image = &data_type->val.image; + if (!process_pixel_image(image)) { + return false; + } + break; + default: + fprintf(stderr, + "unimplemented data-type tag %u\n", + data_type->tag); + return false; + } + break; + default: + fprintf(stderr, "unimplemented frame-data tag %u\n", + data->tag); + return false; + } + } + return true; +} + +bool +exports_wasi_sensor_interface_main() +{ + /* + * a hack alert! + * see https://github.com/bytecodealliance/wasmtime/issues/7592 + */ + void __wasm_call_ctors(); + __wasm_call_ctors(); + + fprintf(stderr, "C guest started\n"); + + wasi_buffer_pool_buffer_pool_buffer_error_t buffer_error; + sensing_own_pool_t pool; + if (!wasi_buffer_pool_buffer_pool_static_pool_create( + WASI_BUFFER_POOL_BUFFER_POOL_BUFFERING_MODE_BUFFERING_DISCARD, + 0, 1, &pool_name, &pool, &buffer_error)) { + fprintf(stderr, "pool.create failed (error %u)\n", + (unsigned int)buffer_error); + return false; + } + fprintf(stderr, "pool.create succeeded\n"); + + wasi_sensor_sensor_device_error_t device_error; + sensing_own_device_t device; + if (!wasi_sensor_sensor_static_device_open(&device_name, &device, + &device_error)) { + fprintf(stderr, "device.open failed (error %u)\n", + (unsigned int)device_error); + return false; + } + fprintf(stderr, "device.open succeeded\n"); + + sensing_borrow_device_t borrowed_device = + wasi_sensor_sensor_borrow_device(device); + if (!wasi_sensor_sensor_method_device_start( + borrowed_device, &pool_name, &device_error)) { + fprintf(stderr, "device.start failed (error %u)\n", + (unsigned int)device_error); + return false; + } + fprintf(stderr, "device.start succeeded\n"); + + sensing_borrow_pool_t borrowed_pool = + wasi_buffer_pool_buffer_pool_borrow_pool(pool); + int n = 60; + int i; + for (i = 0; i < n; i++) { + wasi_buffer_pool_buffer_pool_frame_info_t frame; + if (!wasi_buffer_pool_buffer_pool_method_pool_block_read_frame( + borrowed_pool, &frame, &buffer_error)) { + fprintf(stderr, "block-read-frame failed (error %u)\n", + (unsigned int)buffer_error); + return false; + } + fprintf(stderr, "got a frame (%u/%u)\n", i + 1, n); + if (!process_frame_info(&frame)) { + return false; + } + wasi_buffer_pool_buffer_pool_frame_info_free(&frame); + } + + fprintf(stderr, "cleaning up\n"); + wasi_sensor_sensor_device_drop_own(device); + wasi_buffer_pool_buffer_pool_pool_drop_own(pool); + + fprintf(stderr, "succeeded\n"); + return true; +} diff --git a/host/Cargo.toml b/host/Cargo.toml new file mode 100644 index 0000000..5faeea2 --- /dev/null +++ b/host/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "host" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +anyhow = "1.0.75" +fraction = "0.14" +tracing = { version = "0.1.40", features = ["max_level_trace"] } +tracing-subscriber = "0.3.17" +image = { version = "0.24.7", default-features = false } + +# released versions of nokhwa is a bit broken for avfoundation. +# https://github.com/l1npengtul/nokhwa/pull/151 +# https://github.com/l1npengtul/nokhwa/pull/152 +nokhwa = {git = "https://github.com/yamt/nokhwa", rev = "0.10+fixes", features = ["input-native", "output-threaded"]} + +# preview2 and component-model are still moving targets. +wasmtime = { version = "14.0.4", default-features = false, features = ["component-model", "cranelift"]} +wasmtime-wasi = { version = "14.0.4", default-features = false, features = ["preview2", "sync"] } diff --git a/host/README.md b/host/README.md new file mode 100644 index 0000000..389a377 --- /dev/null +++ b/host/README.md @@ -0,0 +1,10 @@ +# host + +a program which loads and runs `../guest-component.wasm`. +it provides a world with wasi-sensor and wasi preview2. + +## build and run + +``` +cargo run guest-component.wasm guestargs... +``` diff --git a/host/src/dummy_device.rs b/host/src/dummy_device.rs new file mode 100644 index 0000000..67b49a6 --- /dev/null +++ b/host/src/dummy_device.rs @@ -0,0 +1,146 @@ +use super::wasi; +use fraction::Fraction; +use std::sync::Arc; +use std::sync::Mutex; +use std::thread; +use std::time::Duration; +use std::time::Instant; + +use wasi::sensor::property::PropertyKey; +use wasi::sensor::property::PropertyValue; +use wasi::sensor::sensor::DeviceError; + +use crate::traits::BufferPool; +use crate::traits::SensorDevice; + +#[derive(Clone)] +pub struct DummyDeviceConfig { + pub width: u32, + pub height: u32, + pub frame_duration: Duration, +} + +pub struct DummyDevice { + pub pool: Arc>>>, + pub config: Arc>, +} + +impl DummyDevice { + pub fn new() -> Result { + Ok(Self { + pool: Arc::new(Mutex::new(None)), + config: Arc::new(Mutex::new(DummyDeviceConfig { + width: 640, + height: 480, + frame_duration: Duration::from_millis(33), + })), + }) + } +} + +fn generate_dummy_image(frame_no: u64, xsz: u32, ysz: u32) -> (u32, u32, Vec) { + let mut img = image::RgbImage::new(xsz, ysz); + let thresh = (xsz as f32 * (frame_no % 100) as f32 / 100 as f32) as u32; + for (x, y, p) in img.enumerate_pixels_mut() { + let r = (x as f32 / xsz as f32 * 255 as f32) as u8; + let g = if x < thresh { 255 } else { 0 }; + let b = (y as f32 / ysz as f32 * 255 as f32) as u8; + *p = image::Rgb([r, g, b]); + } + (img.width(), img.height(), img.into_raw()) +} + +impl SensorDevice for DummyDevice { + fn start_streaming( + &mut self, + pool: Arc, + ) -> Result<(), DeviceError> { + let _ = self.pool.lock().unwrap().insert(pool); + let pool = self.pool.clone(); + let config_mutex = self.config.clone(); + thread::spawn(move || { + let mut frame_no = 0; + println!("DummyDevice thread started"); + let mut next_frame = Instant::now(); + while let Some(ref pool) = *pool.lock().unwrap() { + let config_locked = config_mutex.lock().unwrap(); + let config = config_locked.clone(); + drop(config_locked); + // dummy image data + let (width, height, samples) = + generate_dummy_image(frame_no, config.width, config.height); + frame_no += 1; + let image = wasi::buffer_pool::data_types::Image { + dimension: wasi::buffer_pool::data_types::Dimension { + width, + height, + stride_bytes: width * 3, + pixel_format: wasi::buffer_pool::data_types::PixelFormat::Rgb24, + //pixel_format: wasi::buffer_pool::data_types::PixelFormat::Yuy2, + }, + payload: samples, + }; + let data = wasi::buffer_pool::buffer_pool::FrameData::ByValue( + wasi::buffer_pool::data_types::DataType::Image(image), + ); + loop { + let now = Instant::now(); + if now >= next_frame { + break; + } + thread::sleep(next_frame - now); + } + next_frame += config.frame_duration; + match pool.enqueue(Box::new(data), None) { + Ok(_) => println!("DummyDevice generated frame enqueued"), + _ => println!("DummyDevice generated frame dropped"), + } + } + println!("DummyDevice thread finished"); + }); + Ok(()) + } + fn stop_streaming(&mut self) -> Result<(), DeviceError> { + self.pool.lock().unwrap().take(); + Ok(()) + } + fn set_property(&mut self, key: PropertyKey, value: PropertyValue) -> Result<(), DeviceError> { + match key { + PropertyKey::SamplingRate => { + let PropertyValue::Fraction(frac) = value else { + return Err(DeviceError::InvalidArgument); + }; + if frac.numerator == 0 || frac.denominator == 0 { + return Err(DeviceError::InvalidArgument); + } + let frame_duration_sec = frac.denominator as f32 / frac.numerator as f32; + let mut config = self.config.lock().unwrap(); + config.frame_duration = Duration::from_secs_f32(frame_duration_sec); + } + _ => return Err(DeviceError::NotSupported), + }; + Ok(()) + } + fn get_property(&mut self, key: PropertyKey) -> Result { + let value = match key { + PropertyKey::SamplingRate => { + let config = self.config.lock().unwrap(); + let frame_duration_sec = config.frame_duration.as_secs_f32(); + let frame_duration_sec = Fraction::from(frame_duration_sec); + let Fraction::Rational(sign, ratio) = frame_duration_sec else { + return Err(DeviceError::OutOfRange); + }; + if sign.is_negative() { + return Err(DeviceError::OutOfRange); + } + let ratio = ratio.recip(); + PropertyValue::Fraction(wasi::sensor::property::Fraction { + numerator: *ratio.numer() as u32, + denominator: *ratio.denom() as u32, + }) + } + _ => return Err(DeviceError::NotSupported), + }; + Ok(value) + } +} diff --git a/host/src/main.rs b/host/src/main.rs new file mode 100644 index 0000000..0824e5f --- /dev/null +++ b/host/src/main.rs @@ -0,0 +1,321 @@ +use anyhow::anyhow; +use anyhow::bail; +use anyhow::Result; +use std::collections::HashMap; +use std::sync::Arc; +use tracing::trace; +use wasmtime::component::*; +use wasmtime::Precompiled; +use wasmtime::{Config, Engine, Store}; +use wasmtime_wasi::ambient_authority; +use wasmtime_wasi::preview2::DirPerms; +use wasmtime_wasi::preview2::FilePerms; +use wasmtime_wasi::preview2::Table; +use wasmtime_wasi::preview2::WasiCtx; +use wasmtime_wasi::preview2::WasiCtxBuilder; +use wasmtime_wasi::preview2::WasiView; +use wasmtime_wasi::Dir; + +mod dummy_device; +mod nokhwa; +mod pool; +mod traits; + +use dummy_device::DummyDevice; +use nokhwa::NokhwaDevice; +use pool::SimplePool; +use traits::SensorDevice; + +wasmtime::component::bindgen!({ + path: "../wit", + tracing: true, + with: { + "wasi:buffer-pool/buffer-pool/pool": Pool, + "wasi:sensor/sensor/device": Device, + }, +}); + +trait WasiSensorView { + fn table(&mut self) -> &mut Table; + fn pools(&mut self) -> &mut HashMap>; +} + +pub struct Pool { + name: String, + pool: Arc, +} +pub struct Device { + device: Box, +} + +impl wasi::buffer_pool::data_types::Host for T {} +impl wasi::buffer_pool::buffer_pool::Host for T {} + +impl wasi::buffer_pool::buffer_pool::HostMemory for T { + fn address( + &mut self, + res: Resource, + ) -> wasmtime::Result { + bail!("not implemented"); + } + fn size( + &mut self, + res: Resource, + ) -> wasmtime::Result { + bail!("not implemented"); + } + fn invalidate( + &mut self, + res: Resource, + ) -> Result> { + bail!("not implemented"); + } + fn drop( + &mut self, + res: Resource, + ) -> wasmtime::Result<()> { + bail!("not implemented"); + } +} + +impl wasi::buffer_pool::buffer_pool::HostPool for T { + fn create( + &mut self, + mode: wasi::buffer_pool::buffer_pool::BufferingMode, + size: u32, + buffer_num: u32, + name: String, + ) -> Result< + Result< + Resource, + wasi::buffer_pool::buffer_pool::BufferError, + >, + > { + let pool = SimplePool::new(mode, size as usize, buffer_num as usize)?; + let pool = Arc::new(pool); + let idx = self.table().push_resource(Pool { + name: name.clone(), + pool: pool.clone(), + })?; + self.pools().insert(name, pool); + Ok(Ok(idx)) + } + + fn block_read_frame( + &mut self, + res: Resource, + ) -> Result< + Result< + wasi::buffer_pool::buffer_pool::FrameInfo, + wasi::buffer_pool::buffer_pool::BufferError, + >, + > { + let pool = self.table().get_resource_mut(&res)?; + let (sequence_number, timestamp, data) = pool.pool.dequeue(); + let frame = wasi::buffer_pool::buffer_pool::FrameInfo { + sequence_number: sequence_number, + timestamp: timestamp, + data: vec![*data], + }; + Ok(Ok(frame)) + } + fn poll_read_frame( + &mut self, + res: Resource, + ) -> Result< + Result< + wasi::buffer_pool::buffer_pool::FrameInfo, + wasi::buffer_pool::buffer_pool::BufferError, + >, + > { + Ok(Err( + wasi::buffer_pool::buffer_pool::BufferError::NotSupported, + )) + } + fn drop( + &mut self, + res: Resource, + ) -> wasmtime::Result<()> { + let pool = self.table().get_resource(&res)?; + let name = pool.name.clone(); + self.table().delete_resource(res)?; + self.pools().remove(&name); + Ok(()) + } +} + +impl wasi::sensor::sensor::HostDevice for T { + fn open( + &mut self, + device_name: String, + ) -> Result, wasi::sensor::sensor::DeviceError>> + { + trace!("opening a device {}", device_name); + let device_impl: Box = match &*device_name { + "dummy" => Box::new(match DummyDevice::new() { + Err(e) => return Ok(Err(e)), + Ok(i) => i, + }), + "nokhwa" => Box::new(match NokhwaDevice::new() { + Err(e) => return Ok(Err(e)), + Ok(i) => i, + }), + _ => return Ok(Err(wasi::sensor::sensor::DeviceError::NotFound)), + }; + let device = Device { + device: device_impl, + }; + let idx = self.table().push_resource(device)?; + Ok(Ok(idx)) + } + + fn list_names(&mut self) -> Result, wasi::sensor::sensor::DeviceError>> { + Ok(Ok(vec!["dummy".to_string(), "nokhwa".to_string()])) + } + + fn start( + &mut self, + res: Resource, + buffer_pool: String, + ) -> Result> { + let pool = match self.pools().get(&buffer_pool) { + Some(pool) => pool, + _ => return Ok(Err(wasi::sensor::sensor::DeviceError::NotFound)), + }; + let pool = Arc::clone(pool); + let device = self.table().get_resource_mut(&res)?; + Ok(device.device.start_streaming(pool)) + } + fn stop( + &mut self, + res: Resource, + ) -> Result> { + Ok(Err(wasi::sensor::sensor::DeviceError::NotSupported)) + } + fn set_property( + &mut self, + res: Resource, + key: wasi::sensor::property::PropertyKey, + value: wasi::sensor::property::PropertyValue, + ) -> Result> { + let device = self.table().get_resource_mut(&res)?; + Ok(device.device.set_property(key, value)) + } + fn get_property( + &mut self, + res: Resource, + key: wasi::sensor::property::PropertyKey, + ) -> Result> + { + let device = self.table().get_resource_mut(&res)?; + Ok(device.device.get_property(key)) + } + fn drop(&mut self, res: Resource) -> wasmtime::Result<()> { + trace!("dropping {:?}", res); + self.table().delete_resource(res)?; + Ok(()) + } +} + +impl wasi::sensor::sensor::Host for T {} + +impl wasi::sensor::property::Host for T {} + +struct State { + wasi: WasiCtx, + table: Table, + pools: HashMap>, +} + +impl WasiView for State { + fn table(&self) -> &Table { + &self.table + } + fn table_mut(&mut self) -> &mut Table { + &mut self.table + } + fn ctx(&self) -> &wasmtime_wasi::preview2::WasiCtx { + &self.wasi + } + fn ctx_mut(&mut self) -> &mut wasmtime_wasi::preview2::WasiCtx { + &mut self.wasi + } +} + +impl WasiSensorView for State { + fn table(&mut self) -> &mut Table { + &mut self.table + } + fn pools(&mut self) -> &mut HashMap> { + &mut self.pools + } +} + +fn main() -> Result<()> { + println!("start"); + tracing_subscriber::fmt::init(); + + let mut config = Config::new(); + config.wasm_component_model(true); + config.wasm_backtrace_details(wasmtime::WasmBacktraceDetails::Enable); + let engine = Engine::new(&config)?; + + let args = std::env::args().collect::>(); + let args = args.iter().map(|x| &**x).collect::>(); + println!("args: {:?}", args); + let filename = &args[1]; + let guest_args = &args[1..]; + + // Note: precompiled modules should have a configuration matching + // the host. as we enable backtrace details above, you should + // compile them as: + // WASMTIME_BACKTRACE_DETAILS=1 wasmtime compile --wasm component-model guest-component.wasm + let component = match engine.detect_precompiled_file(filename) { + Ok(Some(Precompiled::Component)) => { + println!("load a precompiled component"); + unsafe { Component::deserialize_file(&engine, filename) }? + } + _ => { + println!("load a component"); + Component::from_file(&engine, filename)? + } + }; + println!("loaded"); + + let dir = Dir::open_ambient_dir(".", ambient_authority())?; + let wasi_ctx = WasiCtxBuilder::new() + .inherit_stdio() + .args(guest_args) + .preopened_dir(dir, DirPerms::all(), FilePerms::all(), ".") + .build(); + + println!("prepare a linker"); + let mut linker = Linker::new(&engine); + println!("add sensing"); + Sensing::add_to_linker(&mut linker, |s: &mut State| s)?; + println!("add wasi"); + wasmtime_wasi::preview2::command::sync::add_to_linker(&mut linker)?; + + println!("prepare a store"); + let mut store = Store::new( + &engine, + State { + wasi: wasi_ctx, + table: Table::new(), + pools: HashMap::new(), + }, + ); + + println!("instantiate"); + let (bindings, _) = Sensing::instantiate(&mut store, &component, &linker)?; + + println!("calling the entry point"); + let result = bindings.wasi_sensor_interface().call_main(&mut store)?; + + println!("done with result {:?}", result); + println!( + "table dump (including 3 entries for stdio): {:#?}", + store.data().table + ); + result.or(Err(anyhow!("guest failed"))) +} diff --git a/host/src/nokhwa.rs b/host/src/nokhwa.rs new file mode 100644 index 0000000..9bd900c --- /dev/null +++ b/host/src/nokhwa.rs @@ -0,0 +1,137 @@ +use super::wasi; +use std::sync::Arc; + +use wasi::sensor::property::PropertyKey; +use wasi::sensor::property::PropertyValue; +use wasi::sensor::sensor::DeviceError; + +use nokhwa::error::NokhwaError; +use nokhwa::pixel_format::RgbFormat; +use nokhwa::utils::CameraIndex; +use nokhwa::utils::FrameFormat; +use nokhwa::utils::RequestedFormat; +use nokhwa::utils::RequestedFormatType; +use nokhwa::CallbackCamera; + +use crate::traits::BufferPool; +use crate::traits::SensorDevice; + +pub struct NokhwaDevice { + camera: CallbackCamera, +} + +impl NokhwaDevice { + pub fn new() -> Result { + nokhwa::nokhwa_initialize(|granted| { + println!("granted: {}", granted); + }); + println!("NokhwaDevice granted"); + let index = CameraIndex::Index(0); // XXX should not hardcode + let requested = + RequestedFormat::new::(RequestedFormatType::AbsoluteHighestFrameRate); + println!("NokhwaDevice creating a threaded camera"); + let mut camera = CallbackCamera::new(index, requested, |buffer| { + println!("NokhwaDevice dummy callback (this should not be called)"); + })?; + Ok(Self { camera: camera }) + } +} + +impl From for DeviceError { + fn from(error: NokhwaError) -> Self { + println!("converting nokhwa error: {}", error); + DeviceError::Unknown // XXX do a more appropriate conversion + } +} + +impl NokhwaDevice { + fn set_frame_rate(&mut self, frame_rate: f32) -> Result<(), DeviceError> { + println!("NokhwaDevice set_frame_rate {}", frame_rate); + if frame_rate < 1.0 { + return Err(DeviceError::NotSupported); + } + let frame_rate = frame_rate as u32; + self.camera.set_frame_rate(frame_rate)?; + Ok(()) + } + fn get_frame_rate(&mut self) -> Result { + let frame_rate = self.camera.frame_rate()?; + Ok(frame_rate as f32) + } +} + +impl SensorDevice for NokhwaDevice { + fn start_streaming( + &mut self, + pool: Arc, + ) -> Result<(), DeviceError> { + let ref mut camera = self.camera; + camera.set_callback(move |buffer| { + println!("NokhwaDevice callback"); + let resolution = buffer.resolution(); + let width = resolution.width(); + let height = resolution.height(); + let (pixel_format, channels) = match buffer.source_frame_format() { + FrameFormat::YUYV => (wasi::buffer_pool::data_types::PixelFormat::Yuy2, 4), + _ => { + println!("NokhwaDevice dropping a frame with unimplemented format"); + return; + } + }; + let image = wasi::buffer_pool::data_types::Image { + dimension: wasi::buffer_pool::data_types::Dimension { + width, + height, + stride_bytes: width * channels, + pixel_format: pixel_format, + }, + payload: buffer.buffer().to_vec(), + }; + let data = wasi::buffer_pool::buffer_pool::FrameData::ByValue( + wasi::buffer_pool::data_types::DataType::Image(image), + ); + match pool.enqueue(Box::new(data), None) { + Ok(_) => println!("NokhwaDevice frame enqueued"), + _ => println!("NokhwaDevice frame dropped"), + } + })?; + println!("NokhwaDevice calling open_stream"); + camera.open_stream()?; + println!("NokhwaDevice started"); + Ok(()) + } + fn stop_streaming(&mut self) -> Result<(), DeviceError> { + self.camera.stop_stream()?; + Ok(()) + } + fn set_property(&mut self, key: PropertyKey, value: PropertyValue) -> Result<(), DeviceError> { + match key { + PropertyKey::SamplingRate => { + let PropertyValue::Fraction(frac) = value else { + return Err(DeviceError::InvalidArgument); + }; + if frac.numerator == 0 || frac.denominator == 0 { + return Err(DeviceError::InvalidArgument); + } + let frame_rate = frac.numerator as f32 / frac.denominator as f32; + self.set_frame_rate(frame_rate)?; + } + _ => return Err(DeviceError::NotSupported), + }; + Ok(()) + } + fn get_property(&mut self, key: PropertyKey) -> Result { + let value = match key { + PropertyKey::SamplingRate => { + let frame_rate = self.get_frame_rate()?; + // note: nokhwa support only u32 frame rate. + PropertyValue::Fraction(wasi::sensor::property::Fraction { + numerator: frame_rate as u32, + denominator: 1, + }) + } + _ => return Err(DeviceError::NotSupported), + }; + Ok(value) + } +} diff --git a/host/src/pool.rs b/host/src/pool.rs new file mode 100644 index 0000000..0083208 --- /dev/null +++ b/host/src/pool.rs @@ -0,0 +1,59 @@ +use anyhow::Error; +use anyhow::Result; +use std::sync::mpsc::sync_channel; +use std::sync::mpsc::Receiver; +use std::sync::mpsc::SyncSender; +use std::sync::Mutex; +use std::time::Instant; + +use super::*; +use traits::BufferPool; +use wasi::buffer_pool::buffer_pool::BufferError; +use wasi::buffer_pool::buffer_pool::BufferingMode; +use wasi::buffer_pool::buffer_pool::FrameData; + +struct SimplePoolSequencer { + sequence_number: u64, + boottime: Instant, + sender: SyncSender<(u64, u64, Box)>, +} + +pub struct SimplePool { + sequencer: Mutex, + receiver: Mutex)>>, +} + +impl SimplePool { + pub fn new(mode: BufferingMode, sz: usize, num: usize) -> Result { + match mode { + BufferingMode::BufferingDiscard => mode, + _ => return Err(BufferError::NotSupported), + }; + let (sender, receiver) = sync_channel(num); + Ok(Self { + sequencer: Mutex::new(SimplePoolSequencer { + sequence_number: 0, + boottime: Instant::now(), + sender, + }), + receiver: Mutex::new(receiver), + }) + } +} + +impl BufferPool for SimplePool { + fn enqueue(&self, frame: Box, timestamp: Option) -> Result<(), Error> { + let mut seq = self.sequencer.lock().unwrap(); + let timestamp = match timestamp { + Some(t) => t, + _ => seq.boottime.elapsed().as_nanos() as u64, + }; + let seqno = seq.sequence_number; + seq.sequence_number += 1; + seq.sender.try_send((seqno, timestamp, frame))?; + Ok(()) + } + fn dequeue(&self) -> (u64, u64, Box) { + self.receiver.lock().unwrap().recv().unwrap() + } +} diff --git a/host/src/traits.rs b/host/src/traits.rs new file mode 100644 index 0000000..d1f6a0f --- /dev/null +++ b/host/src/traits.rs @@ -0,0 +1,23 @@ +use anyhow::Error; +use std::sync::Arc; + +use super::*; +use wasi::buffer_pool::buffer_pool::FrameData; +use wasi::sensor::property::PropertyKey; +use wasi::sensor::property::PropertyValue; +use wasi::sensor::sensor::DeviceError; + +pub trait BufferPool { + fn enqueue(&self, frame: Box, timestamp: Option) -> Result<(), Error>; + fn dequeue(&self) -> (u64, u64, Box); +} + +pub trait SensorDevice { + fn start_streaming( + &mut self, + pool: Arc, + ) -> Result<(), DeviceError>; + fn stop_streaming(&mut self) -> Result<(), DeviceError>; + fn set_property(&mut self, key: PropertyKey, value: PropertyValue) -> Result<(), DeviceError>; + fn get_property(&mut self, key: PropertyKey) -> Result; +} diff --git a/imports.md b/imports.md deleted file mode 100644 index 8c60710..0000000 --- a/imports.md +++ /dev/null @@ -1,72 +0,0 @@ -

World imports

- -

Import interface wasi:example-api-package/example-dep-interface

-
-

Types

-

type example-dep-type

-

u32

-

-## Import interface wasi:example-package/example-interface -

Short interface description.

-

Explanation for developers using the interface API. It should include an -overview of the API as a whole as well as call out notable items in it, -for example example-api-type and example-api-function.

-
-

Types

-

type example-dep-type

-

example-dep-type

-

-#### `record example-api-type` -

Short type description

-

Explanation for developers using this type. It may be useful to give -some examples of places in the API where the type is used, such as in -the arguments and return type of example-api-function.

-
-Detailed specification -More rigorous specification details for implementers go here, if needed. -The intention is to keep the developer-oriented docs focused on things that -most developers will need to be aware of, while putting bulkier descriptions -of precise behavior here. -
-
Record Fields
-
    -
  • -

    field0: u64

    -

    A description of a field. -

    • -
  • -

    field1: string

    -

    A description of another field. -

    • -
-
-

Functions

-

example-api-function: func

-

Short function description

-

Explanation for developers using the API. This should describe the -arguments which in this function are arg0, arg1, and arg2, and the -return value.

-
-Detailed specification -Similar to the details section above, this is meant for more rigorous -specification details for implementors. This may explain what a compliant -implementation MUST do, such as never returning an earlier result from a -later call, for example. -
-
Params
- -
Return values
- diff --git a/renovate.json b/renovate.json new file mode 100644 index 0000000..f3346d1 --- /dev/null +++ b/renovate.json @@ -0,0 +1,12 @@ +{ + "$schema": "https://docs.renovatebot.com/renovate-schema.json", + "extends": [ + "config:base" + ], + "packageRules": [ + { + "matchPackagePatterns": ["^wasmtime"], + "enabled": false + } + ] +} diff --git a/sensing.md b/sensing.md new file mode 100644 index 0000000..0960c61 --- /dev/null +++ b/sensing.md @@ -0,0 +1,402 @@ +

World sensing

+ +

Import interface wasi:buffer-pool/data-types

+

WASI Sensor is an Sensor abstraction API

+
+

Types

+

record vector3f

+

sensor data type

+
Record Fields
+
    +
  • x: float32
    • +
  • y: float32
    • +
  • z: float32
    • +
+

record quaternion-f

+
Record Fields
+
    +
  • x: float32
    • +
  • y: float32
    • +
  • z: float32
    • +
  • w: float32
    • +
+

enum pixel-format

+
Enum Cases
+
    +
  • +

    grey

    +

    greyscale, bpp=8 +

    • +
  • +

    rgb24

    +

    r,g,b bpp=24 +

    • +
  • +

    bgr24

    +

    b,g,r bpp=24 +

    • +
  • +

    argb32

    +

    a,r,g,b bpp=32 +

    • +
  • +

    abgr32

    +

    a,b,g,r bpp=32 +

    • +
  • +

    yuy2

    +

    YUV422 (Y1,Cb,Y2,Cr) bpp=32 +

    • +
+

record dimension

+
Record Fields
+ +

record image

+
Record Fields
+ +

record depth

+
Record Fields
+
    +
  • payload: list<u8>

    dimension of depth image is updated later here +

    • +
+

variant data-type

+
Variant Cases
+ +

Import interface wasi:sensor/property

+
+

Types

+

type dimension

+

dimension

+

+#### `record fraction` +

Record Fields
+ +

record exposure

+
Record Fields
+
    +
  • mode: string

    Mode of exposure. +

    • +
+

enum property-key

+
Enum Cases
+ +

variant property-value

+
Variant Cases
+ +

Import interface wasi:sensor/sensor

+
+

Types

+

type property-key

+

property-key

+

+#### `type property-value` +[`property-value`](#property_value) +

+#### `enum device-error` +

Enum Cases
+ +

resource device

+
+

Functions

+

[static]device.open: func

+

open the device. +this might power on the device.

+
Params
+ +
Return values
+ +

[static]device.list-names: func

+

get a list of names of devices available on the system.

+
Return values
+ +

[method]device.start: func

+

start sending the data to buffer

+
Params
+ +
Return values
+ +

[method]device.stop: func

+

stop sending the data to buffer

+
Params
+ +
Return values
+ +

[method]device.set-property: func

+

set property

+
Params
+ +
Return values
+ +

[method]device.get-property: func

+

get property

+
Params
+ +
Return values
+ +

Import interface wasi:buffer-pool/buffer-pool

+

sensor frame/buffer management I/F

+
+

Types

+

type data-type

+

data-type

+

+#### `enum buffer-error` +

Enum Cases
+ +

type size

+

u32

+

+#### `type timestamp` +`u64` +

timestamp is the elasped time in nanoseconds since a fixed point +in the past. it's supposed to increase monotonically. +

resource memory

+

variant frame-data

+
Variant Cases
+ +

record frame-info

+
Record Fields
+
    +
  • +

    sequence-number: u64

    +

    sequence number within the pool. it increases monotonically. +a user of this api might observe discontiguous values when some +of frames are discarded within the pool. +

    • +
  • +

    timestamp: timestamp

    +

    timestamp of the frame. +usually the time when it was read from the underlying hardware. +

    • +
  • +

    data: list<frame-data>

    +

    1 or more raw-data for this frame. +

    • +
+

enum buffering-mode

+
Enum Cases
+ +

resource pool

+

a pool consists of a set of buffers. +the number of buffers in a pool is static. +when data (a frame) comes in from an associated device, +it's stored in one of free buffers. +when a user app request data either by block-read or poll-read, +the oldest frame is returned. +when the user app is done with the frame, it should notify it to +the pool by dropping the frame-info and associated resources +like "memory".

+

Functions

+

[method]memory.address: func

+
Params
+ +
Return values
+
    +
  • u64
    • +
+

[method]memory.size: func

+
Params
+ +
Return values
+ +

[method]memory.invalidate: func

+
Params
+ +
Return values
+ +

[static]pool.create: func

+

create a pool.

+

size: the max size of each buffer in bytes. if frame-info::data +has exactly one data and its type is data-types::image, +this value controls the max payload size. otherwise, it's +implementation-defined.

+

buffer-num: the max number of buffers in the pool. +for buffering-discard and buffering-overwrite, this controls +how many frames can be in the pool. +for other buffering modes, this is ignored.

+

name: the name of the pool. you can use this for device.start().

+
Params
+ +
Return values
+ +

[method]pool.block-read-frame: func

+
Params
+ +
Return values
+ +

[method]pool.poll-read-frame: func

+
Params
+ +
Return values
+ +

Export interface wasi:sensor/interface

+
+

Functions

+

main: func

+
Return values
+
    +
  • result
    • +
diff --git a/test/README.md b/test/README.md deleted file mode 100644 index c274acd..0000000 --- a/test/README.md +++ /dev/null @@ -1,11 +0,0 @@ -# Testing guidelines - -TK fill in testing guidelines - -## Installing the tools - -TK fill in instructions - -## Running the tests - -TK fill in instructions diff --git a/wit/deps/buffer-pool/buffer-pool.wit b/wit/deps/buffer-pool/buffer-pool.wit new file mode 100644 index 0000000..ea05295 --- /dev/null +++ b/wit/deps/buffer-pool/buffer-pool.wit @@ -0,0 +1,100 @@ +/// WASI Sensor is an Sensor abstraction API +package wasi:buffer-pool; + +/// sensor frame/buffer management I/F +interface buffer-pool { + use data-types.{data-type}; + + enum buffer-error { + not-found, + invalid-argument, + resource-exhausted, + permission-denied, + busy, + timeout, + cancelled, + aborted, + already-exists, + invalid-operation, + out-of-range, + data-loss, + hardware-error, + not-supported, + unknown + } + + type size = u32; + + /// timestamp is the elasped time in nanoseconds since a fixed point + /// in the past. it's supposed to increase monotonically. + type timestamp = u64; + + record frame-info { + /// sequence number within the pool. it increases monotonically. + /// a user of this api might observe discontiguous values when some + /// of frames are discarded within the pool. + sequence-number: u64, + + /// timestamp of the frame. + /// usually the time when it was read from the underlying hardware. + timestamp: timestamp, + + /// 1 or more raw-data for this frame. + data: list + } + + variant frame-data { + /// data passed by value + by-value(data-type), + + /// a reference to host memory + host-memory(memory), + } + + resource memory { + address: func()-> u64; + size: func()-> size; + invalidate: func()-> result<_, buffer-error>; + } + + enum buffering-mode { + buffering-off, + buffering-discard, /**< Discard the latest frame. behave like queue */ + buffering-overwrite, /**< Overwrite the oldest frame. behave like ring */ + buffering-unlimited, + } + + /// a pool consists of a set of buffers. + /// the number of buffers in a pool is static. + /// when data (a frame) comes in from an associated device, + /// it's stored in one of free buffers. + /// when a user app request data either by block-read or poll-read, + /// the oldest frame is returned. + /// when the user app is done with the frame, it should notify it to + /// the pool by dropping the frame-info and associated resources + /// like "memory". + resource pool { + /// create a pool. + /// + /// size: the max size of each buffer in bytes. if frame-info::data + /// has exactly one data and its type is data-types::image, + /// this value controls the max payload size. otherwise, it's + /// implementation-defined. + /// + /// buffer-num: the max number of buffers in the pool. + /// for buffering-discard and buffering-overwrite, this controls + /// how many frames can be in the pool. + /// for other buffering modes, this is ignored. + /// + /// name: the name of the pool. you can use this for device.start(). + create: static func(mode:buffering-mode, size:u32, buffer-num:u32, name:string) ->result; + + block-read-frame: func()-> result; + poll-read-frame: func()-> result; + } +} + +world buffering { + import buffer-pool; + import data-types; +} diff --git a/wit/deps/buffer-pool/data-types.wit b/wit/deps/buffer-pool/data-types.wit new file mode 100644 index 0000000..d3f9676 --- /dev/null +++ b/wit/deps/buffer-pool/data-types.wit @@ -0,0 +1,66 @@ +/// WASI Sensor is an Sensor abstraction API + +interface data-types { + /// sensor data type + record vector3f { + x: float32, + y: float32, + z: float32, + } + + record quaternion-f { + x: float32, + y: float32, + z: float32, + w: float32, + } + + + + enum pixel-format { + /// greyscale, bpp=8 + grey, + /// r,g,b bpp=24 + rgb24, + /// b,g,r bpp=24 + bgr24, + /// a,r,g,b bpp=32 + argb32, + /// a,b,g,r bpp=32 + abgr32, + /// YUV422 (Y1,Cb,Y2,Cr) bpp=32 + yuy2, + + /// how to express yuv, compress, planar + } + + record dimension { + /// Image width. + width: u32, + /// Image height. + height: u32, + /// Image stride + stride-bytes: u32, + /// The format of a pixel. + pixel-format: pixel-format, + } + + record image { + dimension: dimension, + payload: list, + } + + record depth { + /// dimension of depth image is updated later here + payload: list, + } + + + variant data-type { + image(image), + /// other sensor such as depth, angular velocity, linear acceleration etc is comming here + + } + +} + diff --git a/wit/deps/example-dep/example-api.wit b/wit/deps/example-dep/example-api.wit deleted file mode 100644 index e5fb235..0000000 --- a/wit/deps/example-dep/example-api.wit +++ /dev/null @@ -1,8 +0,0 @@ -// An example dependency, showing how these look. Actual proposals should -// delete this file and add their actual dependencies in the `deps` directory. - -package wasi:example-api-package - -interface example-dep-interface { - type example-dep-type = u32 -} diff --git a/wit/example.wit b/wit/example.wit deleted file mode 100644 index 55701be..0000000 --- a/wit/example.wit +++ /dev/null @@ -1,59 +0,0 @@ -// Instructions for filling in this file: -// -// - Delete all these `//` comments, up to the first `///` comment. -// -// - Replace the remaining contents below with [Wit] code describing -// `interface`s and/or `world`s, using the same formatting style. -// -// If you want to include examples of the API in use, these should be in the -// README.md at the root of the repository and linked to from this file. -// -// [Wit]: https://github.com/WebAssembly/component-model/blob/main/design/mvp/WIT.md - -/// Short interface description. -/// -/// Explanation for developers using the interface API. It should include an -/// overview of the API as a whole as well as call out notable items in it, -/// for example `example-api-type` and `example-api-function`. -interface example-interface { - use wasi:example-api-package/example-dep-interface.{example-dep-type} - - /// Short type description - /// - /// Explanation for developers using this type. It may be useful to give - /// some examples of places in the API where the type is used, such as in - /// the arguments and return type of `example-api-function`. - /// - ///
- /// Detailed specification - /// More rigorous specification details for implementers go here, if needed. - /// The intention is to keep the developer-oriented docs focused on things that - /// most developers will need to be aware of, while putting bulkier descriptions - /// of precise behavior here. - ///
- record example-api-type { - /// A description of a field. - field0: u64, - /// A description of another field. - field1: string, - } - - /// Short function description - /// - /// Explanation for developers using the API. This should describe the - /// arguments which in this function are `arg0`, `arg1`, and `arg2`, and the - /// return value. - /// - ///
- /// Detailed specification - /// Similar to the details section above, this is meant for more rigorous - /// specification details for implementors. This may explain what a compliant - /// implementation MUST do, such as never returning an earlier result from a - /// later call, for example. - ///
- example-api-function: func( - arg0: example-api-type, - arg1: string, - arg2: example-dep-type, - ) -> result -} diff --git a/wit/property.wit b/wit/property.wit new file mode 100644 index 0000000..39bec04 --- /dev/null +++ b/wit/property.wit @@ -0,0 +1,36 @@ + +interface property { + use wasi:buffer-pool/data-types.{dimension}; + + record fraction { + numerator: u32, + denominator: u32 + } + + + + record exposure { + /// Mode of exposure. + mode: string, + /// More parameters for exposure. + } + + + enum property-key { + /// The number of samples in a second. (fraction) + /// Eg. frame rate for image sensors. + sampling-rate, + + dimension, + exposure, + } + + variant property-value { + fraction(fraction), + + dimension(dimension), + exposure(exposure), + } +} + + diff --git a/wit/sensor.wit b/wit/sensor.wit new file mode 100644 index 0000000..5454ba0 --- /dev/null +++ b/wit/sensor.wit @@ -0,0 +1,62 @@ +/// WASI Sensor is an Sensor abstraction API +package wasi:sensor; + +interface sensor { + use property.{property-key, property-value}; + + enum device-error { + not-found, + invalid-argument, + resource-exhausted, + permission-denied, + busy, + timeout, + cancelled, + aborted, + already-exists, + invalid-operation, + out-of-range, + data-loss, + hardware-error, + not-supported, + unknown + } + + // Sensor device + resource device { + /// open the device. + /// this might power on the device. + open: static func(name: string) -> result; + + /// get a list of names of devices available on the system. + list-names: static func() -> result, device-error>; + + /// start sending the data to buffer + start: func( + buffer-pool: string + )->result<_, device-error>; + + /// stop sending the data to buffer + stop: func( + )->result<_, device-error>; + + /// set property + set-property: func( + key: property-key, + value: property-value + ) ->result<_, device-error>; + + /// get property + get-property: func( + property: property-key + )->result; + } +} + +interface %interface { + main: func() -> result; +} + + + + diff --git a/wit/world.wit b/wit/world.wit index 53ca2e3..758f9a5 100644 --- a/wit/world.wit +++ b/wit/world.wit @@ -5,8 +5,12 @@ // Proposals should remove these `//` commments, and edit the `world` name and // imports below to pull in their own `interface`s. -package wasi:example-package +package wasi:sensor; -world imports { - import example-interface +world sensing { + import sensor; + import property; + import wasi:buffer-pool/buffer-pool; + import wasi:buffer-pool/data-types; + export %interface; }