Add Dart API for audio tagging (#1181)

.github/scripts/test-dart.sh

@@ -4,6 +4,14 @@ set -ex
 
 cd dart-api-examples
 
+pushd audio-tagging
+echo '----------zipformer----------'
+./run-zipformer.sh
+
+echo '----------ced----------'
+./run-ced.sh
+popd
+
 pushd vad-with-non-streaming-asr
 echo '----------TeleSpeech CTC----------'
 ./run-telespeech-ctc.sh

.github/workflows/test-dart.yaml

@@ -110,6 +110,7 @@ jobs:
           cp scripts/dart/tts-pubspec.yaml dart-api-examples/tts/pubspec.yaml
           cp scripts/dart/kws-pubspec.yaml dart-api-examples/keyword-spotter/pubspec.yaml
           cp scripts/dart/vad-non-streaming-asr-pubspec.yaml dart-api-examples/vad-with-non-streaming-asr/pubspec.yaml
+          cp scripts/dart/audio-tagging-pubspec.yaml dart-api-examples/audio-tagging/pubspec.yaml
 
           cp scripts/dart/sherpa-onnx-pubspec.yaml flutter/sherpa_onnx/pubspec.yaml
 

CHANGELOG.md

@@ -1,3 +1,7 @@
+## 1.10.20
+
+* Add Dart API for audio tagging
+
 ## 1.10.19
 
 * Prefix all C API functions with SherpaOnnx

README.md

@@ -4,9 +4,13 @@
 |------------------|------------------|----------------------|------------------------|
 |   ✔️              |         ✔️        |          ✔️           |                ✔️       |
 
-| Spoken Language identification | Audio tagging | Voice activity detection | Keyword spotting |
-|--------------------------------|---------------|--------------------------|------------------|
-|                 ✔️              |          ✔️    |                ✔️         |         ✔️        |
+| Spoken Language identification | Audio tagging | Voice activity detection |
+|--------------------------------|---------------|--------------------------|
+|                 ✔️              |          ✔️    |                ✔️         |
+
+| Keyword spotting | Add punctuation |
+|------------------|-----------------|
+|     ✔️            |       ✔️         |
 
 ### Supported platforms
 

dart-api-examples/README.md

@@ -5,7 +5,7 @@ This directory contains examples for Dart API.
 You can find the package at
 https://pub.dev/packages/sherpa_onnx
 
-## Descirption
+## Description
 
 | Directory | Description |
 |-----------|-------------|
@@ -15,6 +15,7 @@ https://pub.dev/packages/sherpa_onnx
 | [./tts](./tts)| Example for text to speech|
 | [./vad](./vad)| Example for voice activity detection|
 | [./vad-with-non-streaming-asr](./vad-with-non-streaming-asr)| Example for voice activity detection with non-streaming speech recognition. You can use it to generate subtitles.|
+| [./audio-tagging](./audio-tagging)| Example for audio tagging.|
 
 ## How to create an example in this folder
 

dart-api-examples/audio-tagging/.gitignore

@@ -0,0 +1,3 @@
+# https://dart.dev/guides/libraries/private-files
+# Created by `dart pub`
+.dart_tool/

dart-api-examples/audio-tagging/README.md

@@ -0,0 +1,8 @@
+# Introduction
+
+This example shows how to use the Dart API from sherpa-onnx for audio tagging.
+
+| File | Description|
+|------|------------|
+|[./bin/zipformer.dart](./bin/zipformer.dart)| Use a Zipformer model for audio tagging. See [./run-zipformer.sh](./run-zipformer.sh)|
+|[./bin/ced.dart](./bin/ced.dart)| Use a [CED](https://github.com/RicherMans/CED) model for audio tagging. See [./run-ced.sh](./run-ced.sh)|

dart-api-examples/audio-tagging/analysis_options.yaml

@@ -0,0 +1,30 @@
+# This file configures the static analysis results for your project (errors,
+# warnings, and lints).
+#
+# This enables the 'recommended' set of lints from `package:lints`.
+# This set helps identify many issues that may lead to problems when running
+# or consuming Dart code, and enforces writing Dart using a single, idiomatic
+# style and format.
+#
+# If you want a smaller set of lints you can change this to specify
+# 'package:lints/core.yaml'. These are just the most critical lints
+# (the recommended set includes the core lints).
+# The core lints are also what is used by pub.dev for scoring packages.
+
+include: package:lints/recommended.yaml
+
+# Uncomment the following section to specify additional rules.
+
+# linter:
+#   rules:
+#     - camel_case_types
+
+# analyzer:
+#   exclude:
+#     - path/to/excluded/files/**
+
+# For more information about the core and recommended set of lints, see
+# https://dart.dev/go/core-lints
+
+# For additional information about configuring this file, see
+# https://dart.dev/guides/language/analysis-options

dart-api-examples/audio-tagging/bin/ced.dart

@@ -0,0 +1,54 @@
+// Copyright (c)  2024  Xiaomi Corporation
+import 'dart:io';
+
+import 'package:args/args.dart';
+import 'package:sherpa_onnx/sherpa_onnx.dart' as sherpa_onnx;
+import './init.dart';
+
+void main(List<String> arguments) async {
+  await initSherpaOnnx();
+
+  final parser = ArgParser()
+    ..addOption('model', help: 'Path to the zipformer model')
+    ..addOption('labels', help: 'Path to class_labels_indices.csv')
+    ..addOption('top-k', help: 'topK events to be returned', defaultsTo: '5')
+    ..addOption('wav', help: 'Path to test.wav to be tagged');
+
+  final res = parser.parse(arguments);
+  if (res['model'] == null || res['labels'] == null || res['wav'] == null) {
+    print(parser.usage);
+    exit(1);
+  }
+
+  final model = res['model'] as String;
+  final labels = res['labels'] as String;
+  final topK = int.tryParse(res['top-k'] as String) ?? 5;
+  final wav = res['wav'] as String;
+
+  final modelConfig = sherpa_onnx.AudioTaggingModelConfig(
+    ced: model,
+    numThreads: 1,
+    debug: true,
+    provider: 'cpu',
+  );
+
+  final config = sherpa_onnx.AudioTaggingConfig(
+    model: modelConfig,
+    labels: labels,
+  );
+
+  final at = sherpa_onnx.AudioTagging(config: config);
+
+  final waveData = sherpa_onnx.readWave(wav);
+
+  final stream = at.createStream();
+  stream.acceptWaveform(
+      samples: waveData.samples, sampleRate: waveData.sampleRate);
+
+  final events = at.compute(stream: stream, topK: topK);
+
+  print(events);
+
+  stream.free();
+  at.free();
+}

dart-api-examples/audio-tagging/bin/init.dart

@@ -0,0 +1 @@
+../../vad/bin/init.dart

dart-api-examples/audio-tagging/bin/zipformer.dart

@@ -0,0 +1,59 @@
+// Copyright (c)  2024  Xiaomi Corporation
+import 'dart:io';
+
+import 'package:args/args.dart';
+import 'package:sherpa_onnx/sherpa_onnx.dart' as sherpa_onnx;
+import './init.dart';
+
+void main(List<String> arguments) async {
+  await initSherpaOnnx();
+
+  final parser = ArgParser()
+    ..addOption('model', help: 'Path to the zipformer model')
+    ..addOption('labels', help: 'Path to class_labels_indices.csv')
+    ..addOption('top-k', help: 'topK events to be returned', defaultsTo: '5')
+    ..addOption('wav', help: 'Path to test.wav to be tagged');
+
+  final res = parser.parse(arguments);
+  if (res['model'] == null || res['labels'] == null || res['wav'] == null) {
+    print(parser.usage);
+    exit(1);
+  }
+
+  final model = res['model'] as String;
+  final labels = res['labels'] as String;
+  final topK = int.tryParse(res['top-k'] as String) ?? 5;
+  final wav = res['wav'] as String;
+
+  final zipformerModelConfig =
+      sherpa_onnx.OfflineZipformerAudioTaggingModelConfig(
+    model: model,
+  );
+
+  final modelConfig = sherpa_onnx.AudioTaggingModelConfig(
+    zipformer: zipformerModelConfig,
+    numThreads: 1,
+    debug: true,
+    provider: 'cpu',
+  );
+
+  final config = sherpa_onnx.AudioTaggingConfig(
+    model: modelConfig,
+    labels: labels,
+  );
+
+  final at = sherpa_onnx.AudioTagging(config: config);
+
+  final waveData = sherpa_onnx.readWave(wav);
+
+  final stream = at.createStream();
+  stream.acceptWaveform(
+      samples: waveData.samples, sampleRate: waveData.sampleRate);
+
+  final events = at.compute(stream: stream, topK: topK);
+
+  print(events);
+
+  stream.free();
+  at.free();
+}

dart-api-examples/audio-tagging/pubspec.yaml

@@ -0,0 +1,17 @@
+name: audio_tagging
+
+description: >
+  This example demonstrates how to use the Dart API for audio tagging.
+
+version: 1.0.0
+
+environment:
+  sdk: ^3.4.0
+
+dependencies:
+  sherpa_onnx: ^1.10.19
+  path: ^1.9.0
+  args: ^2.5.0
+
+dev_dependencies:
+  lints: ^3.0.0

dart-api-examples/audio-tagging/run-ced.sh

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+
+set -ex
+
+dart pub get
+
+if [[ ! -f ./sherpa-onnx-ced-mini-audio-tagging-2024-04-19/model.onnx ]]; then
+  curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/audio-tagging-models/sherpa-onnx-ced-mini-audio-tagging-2024-04-19.tar.bz2
+  tar xvf sherpa-onnx-ced-mini-audio-tagging-2024-04-19.tar.bz2
+  rm sherpa-onnx-ced-mini-audio-tagging-2024-04-19.tar.bz2
+fi
+
+for w in 1 2 3 4 5 6; do
+  dart run \
+    ./bin/ced.dart \
+    --model ./sherpa-onnx-ced-mini-audio-tagging-2024-04-19/model.int8.onnx \
+    --labels ./sherpa-onnx-ced-mini-audio-tagging-2024-04-19/class_labels_indices.csv \
+    --wav ./sherpa-onnx-ced-mini-audio-tagging-2024-04-19/test_wavs/$w.wav
+done

dart-api-examples/audio-tagging/run-zipformer.sh

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+
+set -ex
+
+dart pub get
+
+if [[ ! -f ./sherpa-onnx-zipformer-audio-tagging-2024-04-09/model.onnx ]]; then
+  curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/audio-tagging-models/sherpa-onnx-zipformer-audio-tagging-2024-04-09.tar.bz2
+  tar xvf sherpa-onnx-zipformer-audio-tagging-2024-04-09.tar.bz2
+  rm sherpa-onnx-zipformer-audio-tagging-2024-04-09.tar.bz2
+fi
+
+for w in 1 2 3 4 5 6; do
+  dart run \
+    ./bin/zipformer.dart \
+    --model ./sherpa-onnx-zipformer-audio-tagging-2024-04-09/model.int8.onnx \
+    --labels ./sherpa-onnx-zipformer-audio-tagging-2024-04-09/class_labels_indices.csv \
+    --wav ./sherpa-onnx-zipformer-audio-tagging-2024-04-09/test_wavs/$w.wav
+done

dart-api-examples/non-streaming-asr/bin/nemo-ctc.dart

@@ -1,6 +1,5 @@
 // Copyright (c)  2024  Xiaomi Corporation
 import 'dart:io';
-import 'dart:typed_data';
 
 import 'package:args/args.dart';
 import 'package:sherpa_onnx/sherpa_onnx.dart' as sherpa_onnx;

dart-api-examples/non-streaming-asr/bin/nemo-transducer.dart

@@ -1,6 +1,5 @@
 // Copyright (c)  2024  Xiaomi Corporation
 import 'dart:io';
-import 'dart:typed_data';
 
 import 'package:args/args.dart';
 import 'package:sherpa_onnx/sherpa_onnx.dart' as sherpa_onnx;

dart-api-examples/non-streaming-asr/bin/paraformer-itn.dart

@@ -1,6 +1,5 @@
 // Copyright (c)  2024  Xiaomi Corporation
 import 'dart:io';
-import 'dart:typed_data';
 
 import 'package:args/args.dart';
 import 'package:sherpa_onnx/sherpa_onnx.dart' as sherpa_onnx;

dart-api-examples/non-streaming-asr/bin/paraformer.dart

@@ -1,6 +1,5 @@
 // Copyright (c)  2024  Xiaomi Corporation
 import 'dart:io';
-import 'dart:typed_data';
 
 import 'package:args/args.dart';
 import 'package:sherpa_onnx/sherpa_onnx.dart' as sherpa_onnx;

dart-api-examples/non-streaming-asr/bin/sense-voice.dart

@@ -1,6 +1,5 @@
 // Copyright (c)  2024  Xiaomi Corporation
 import 'dart:io';
-import 'dart:typed_data';
 
 import 'package:args/args.dart';
 import 'package:sherpa_onnx/sherpa_onnx.dart' as sherpa_onnx;

dart-api-examples/non-streaming-asr/bin/telespeech-ctc.dart

@@ -1,6 +1,5 @@
 // Copyright (c)  2024  Xiaomi Corporation
 import 'dart:io';
-import 'dart:typed_data';
 
 import 'package:args/args.dart';
 import 'package:sherpa_onnx/sherpa_onnx.dart' as sherpa_onnx;

dart-api-examples/non-streaming-asr/bin/whisper.dart

@@ -1,6 +1,5 @@
 // Copyright (c)  2024  Xiaomi Corporation
 import 'dart:io';
-import 'dart:typed_data';
 
 import 'package:args/args.dart';
 import 'package:sherpa_onnx/sherpa_onnx.dart' as sherpa_onnx;

dart-api-examples/non-streaming-asr/bin/zipformer-transducer.dart

@@ -1,6 +1,5 @@
 // Copyright (c)  2024  Xiaomi Corporation
 import 'dart:io';
-import 'dart:typed_data';
 
 import 'package:args/args.dart';
 import 'package:sherpa_onnx/sherpa_onnx.dart' as sherpa_onnx;

dart-api-examples/tts/bin/coqui.dart

@@ -1,6 +1,5 @@
 // Copyright (c)  2024  Xiaomi Corporation
 import 'dart:io';
-import 'dart:typed_data';
 
 import 'package:args/args.dart';
 import 'package:sherpa_onnx/sherpa_onnx.dart' as sherpa_onnx;
@@ -65,5 +64,5 @@ void main(List<String> arguments) async {
     samples: audio.samples,
     sampleRate: audio.sampleRate,
   );
-  print('Saved to ${outputWav}');
+  print('Saved to $outputWav');
 }

dart-api-examples/tts/bin/piper.dart

@@ -80,5 +80,5 @@ void main(List<String> arguments) async {
     samples: audio.samples,
     sampleRate: audio.sampleRate,
   );
-  print('Saved to ${outputWav}');
+  print('Saved to $outputWav');
 }

dart-api-examples/tts/bin/zh.dart

@@ -1,6 +1,5 @@
 // Copyright (c)  2024  Xiaomi Corporation
 import 'dart:io';
-import 'dart:typed_data';
 
 import 'package:args/args.dart';
 import 'package:sherpa_onnx/sherpa_onnx.dart' as sherpa_onnx;
@@ -82,5 +81,5 @@ void main(List<String> arguments) async {
     samples: audio.samples,
     sampleRate: audio.sampleRate,
   );
-  print('Saved to ${outputWav}');
+  print('Saved to $outputWav');
 }

dart-api-examples/vad/bin/vad.dart

@@ -77,5 +77,5 @@ void main(List<String> arguments) async {
   sherpa_onnx.writeWave(
       filename: outputWav, samples: s, sampleRate: waveData.sampleRate);
 
-  print('Saved to ${outputWav}');
+  print('Saved to $outputWav');
 }

flutter/sherpa_onnx/lib/sherpa_onnx.dart

@@ -2,6 +2,7 @@
 import 'dart:io';
 import 'dart:ffi';
 
+export 'src/audio_tagging.dart';
 export 'src/feature_config.dart';
 export 'src/keyword_spotter.dart';
 export 'src/offline_recognizer.dart';