Skip to content

Commit

Permalink
Dart API for adding punctuations to text (#1182)
Browse files Browse the repository at this point in the history
  • Loading branch information
csukuangfj authored Jul 29, 2024
1 parent cd1feda commit 646f99c
Show file tree
Hide file tree
Showing 30 changed files with 327 additions and 22 deletions.
5 changes: 5 additions & 0 deletions .github/scripts/test-dart.sh
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,11 @@ set -ex

cd dart-api-examples

pushd add-punctuations
echo '----------CT Transformer----------'
./run-ct-transformer.sh
popd

pushd audio-tagging
echo '----------zipformer----------'
./run-zipformer.sh
Expand Down
1 change: 1 addition & 0 deletions .github/workflows/test-dart.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -111,6 +111,7 @@ jobs:
cp scripts/dart/kws-pubspec.yaml dart-api-examples/keyword-spotter/pubspec.yaml
cp scripts/dart/vad-non-streaming-asr-pubspec.yaml dart-api-examples/vad-with-non-streaming-asr/pubspec.yaml
cp scripts/dart/audio-tagging-pubspec.yaml dart-api-examples/audio-tagging/pubspec.yaml
cp scripts/dart/add-punctuations-pubspec.yaml dart-api-examples/add-punctuations/pubspec.yaml
cp scripts/dart/sherpa-onnx-pubspec.yaml flutter/sherpa_onnx/pubspec.yaml
Expand Down
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
## 1.10.20

* Add Dart API for audio tagging
* Add Dart API for adding punctuations to text

## 1.10.19

Expand Down
2 changes: 1 addition & 1 deletion CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ project(sherpa-onnx)
# ./nodejs-addon-examples
# ./dart-api-examples/
# ./CHANGELOG.md
set(SHERPA_ONNX_VERSION "1.10.19")
set(SHERPA_ONNX_VERSION "1.10.20")

# Disable warning about
#
Expand Down
1 change: 1 addition & 0 deletions dart-api-examples/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ https://pub.dev/packages/sherpa_onnx
| [./vad](./vad)| Example for voice activity detection|
| [./vad-with-non-streaming-asr](./vad-with-non-streaming-asr)| Example for voice activity detection with non-streaming speech recognition. You can use it to generate subtitles.|
| [./audio-tagging](./audio-tagging)| Example for audio tagging.|
| [./add-punctuations](./add-punctuations)| Example for adding punctuations to text.|

## How to create an example in this folder

Expand Down
3 changes: 3 additions & 0 deletions dart-api-examples/add-punctuations/.gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
# https://dart.dev/guides/libraries/private-files
# Created by `dart pub`
.dart_tool/
8 changes: 8 additions & 0 deletions dart-api-examples/add-punctuations/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
# Introduction

This example shows how to use the Dart API from sherpa-onnx to add punctuations to text.

| File | Description|
|------|------------|
|[./bin/punctuations.dart](./bin/punctuations.dart)| Use a [CT Transformer model](https://modelscope.cn/models/iic/punc_ct-transformer_zh-cn-common-vocab272727-pytorch/summary) to add punctuations to text. See [./run-ct-transformer.sh](./run-ct-transformer.sh)|

30 changes: 30 additions & 0 deletions dart-api-examples/add-punctuations/analysis_options.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
# This file configures the static analysis results for your project (errors,
# warnings, and lints).
#
# This enables the 'recommended' set of lints from `package:lints`.
# This set helps identify many issues that may lead to problems when running
# or consuming Dart code, and enforces writing Dart using a single, idiomatic
# style and format.
#
# If you want a smaller set of lints you can change this to specify
# 'package:lints/core.yaml'. These are just the most critical lints
# (the recommended set includes the core lints).
# The core lints are also what is used by pub.dev for scoring packages.

include: package:lints/recommended.yaml

# Uncomment the following section to specify additional rules.

# linter:
# rules:
# - camel_case_types

# analyzer:
# exclude:
# - path/to/excluded/files/**

# For more information about the core and recommended set of lints, see
# https://dart.dev/go/core-lints

# For additional information about configuring this file, see
# https://dart.dev/guides/language/analysis-options
1 change: 1 addition & 0 deletions dart-api-examples/add-punctuations/bin/init.dart
46 changes: 46 additions & 0 deletions dart-api-examples/add-punctuations/bin/punctuations.dart
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
// Copyright (c) 2024 Xiaomi Corporation
import 'dart:io';

import 'package:args/args.dart';
import 'package:sherpa_onnx/sherpa_onnx.dart' as sherpa_onnx;
import './init.dart';

void main(List<String> arguments) async {
await initSherpaOnnx();

final parser = ArgParser()..addOption('model', help: 'Path to model.onnx');

final res = parser.parse(arguments);
if (res['model'] == null) {
print(parser.usage);
exit(1);
}

final modelFile = res['model'] as String;
final modelConfig = sherpa_onnx.OfflinePunctuationModelConfig(
ctTransformer: modelFile,
numThreads: 1,
provider: 'cpu',
debug: false,
);

final config = sherpa_onnx.OfflinePunctuationConfig(model: modelConfig);

final punct = sherpa_onnx.OfflinePunctuation(config: config);

final texts = [
'这是一个测试你好吗How are you我很好thank you are you ok谢谢你',
'我们都是木头人不会说话不会动',
'The African blogosphere is rapidly expanding bringing more voices online in the form of commentaries opinions analyses rants and poetry',
];

for (final t in texts) {
final textWithPunct = punct.addPunct(t);
print('----------');
print('Before: $t');
print('After: $textWithPunct');
}
print('----------');

punct.free();
}
17 changes: 17 additions & 0 deletions dart-api-examples/add-punctuations/pubspec.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
name: add_punctuations

description: >
This example demonstrates how to use the Dart API to add punctuations to text.
version: 1.0.0

environment:
sdk: ^3.4.0

dependencies:
sherpa_onnx: ^1.10.20
path: ^1.9.0
args: ^2.5.0

dev_dependencies:
lints: ^3.0.0
15 changes: 15 additions & 0 deletions dart-api-examples/add-punctuations/run-ct-transformer.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
#!/usr/bin/env bash

set -ex

dart pub get

if [[ ! -f ./sherpa-onnx-punct-ct-transformer-zh-en-vocab272727-2024-04-12/model.onnx ]]; then
curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/punctuation-models/sherpa-onnx-punct-ct-transformer-zh-en-vocab272727-2024-04-12.tar.bz2
tar xvf sherpa-onnx-punct-ct-transformer-zh-en-vocab272727-2024-04-12.tar.bz2
rm sherpa-onnx-punct-ct-transformer-zh-en-vocab272727-2024-04-12.tar.bz2
fi

dart run \
./bin/punctuations.dart \
--model ./sherpa-onnx-punct-ct-transformer-zh-en-vocab272727-2024-04-12/model.onnx
2 changes: 1 addition & 1 deletion dart-api-examples/audio-tagging/pubspec.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ environment:
sdk: ^3.4.0

dependencies:
sherpa_onnx: ^1.10.19
sherpa_onnx: ^1.10.20
path: ^1.9.0
args: ^2.5.0

Expand Down
2 changes: 1 addition & 1 deletion dart-api-examples/keyword-spotter/pubspec.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ environment:
sdk: ^3.4.0

dependencies:
sherpa_onnx: ^1.10.19
sherpa_onnx: ^1.10.20
# sherpa_onnx:
# path: ../../flutter/sherpa_onnx
path: ^1.9.0
Expand Down
2 changes: 1 addition & 1 deletion dart-api-examples/non-streaming-asr/pubspec.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ environment:

# Add regular dependencies here.
dependencies:
sherpa_onnx: ^1.10.19
sherpa_onnx: ^1.10.20
path: ^1.9.0
args: ^2.5.0

Expand Down
2 changes: 1 addition & 1 deletion dart-api-examples/streaming-asr/pubspec.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ environment:

# Add regular dependencies here.
dependencies:
sherpa_onnx: ^1.10.19
sherpa_onnx: ^1.10.20
path: ^1.9.0
args: ^2.5.0

Expand Down
2 changes: 1 addition & 1 deletion dart-api-examples/tts/pubspec.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ environment:

# Add regular dependencies here.
dependencies:
sherpa_onnx: ^1.10.19
sherpa_onnx: ^1.10.20
path: ^1.9.0
args: ^2.5.0

Expand Down
2 changes: 1 addition & 1 deletion dart-api-examples/vad-with-non-streaming-asr/pubspec.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ environment:
sdk: ^3.4.0

dependencies:
sherpa_onnx: ^1.10.19
sherpa_onnx: ^1.10.20
path: ^1.9.0
args: ^2.5.0

Expand Down
2 changes: 1 addition & 1 deletion dart-api-examples/vad/pubspec.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ environment:
sdk: ^3.4.0

dependencies:
sherpa_onnx: ^1.10.19
sherpa_onnx: ^1.10.20
path: ^1.9.0
args: ^2.5.0

Expand Down
4 changes: 2 additions & 2 deletions flutter-examples/streaming_asr/pubspec.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ description: >
publish_to: 'none'

version: 1.10.19
version: 1.10.20

topics:
- speech-recognition
Expand All @@ -30,7 +30,7 @@ dependencies:
record: ^5.1.0
url_launcher: ^6.2.6

sherpa_onnx: ^1.10.19
sherpa_onnx: ^1.10.20
# sherpa_onnx:
# path: ../../flutter/sherpa_onnx

Expand Down
4 changes: 2 additions & 2 deletions flutter-examples/tts/pubspec.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ description: >
publish_to: 'none' # Remove this line if you wish to publish to pub.dev

version: 1.10.19
version: 1.10.20

environment:
sdk: '>=3.4.0 <4.0.0'
Expand All @@ -17,7 +17,7 @@ dependencies:
cupertino_icons: ^1.0.6
path_provider: ^2.1.3
path: ^1.9.0
sherpa_onnx: ^1.10.19
sherpa_onnx: ^1.10.20
url_launcher: ^6.2.6
audioplayers: ^5.0.0

Expand Down
1 change: 1 addition & 0 deletions flutter/sherpa_onnx/lib/sherpa_onnx.dart
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ export 'src/offline_recognizer.dart';
export 'src/offline_stream.dart';
export 'src/online_recognizer.dart';
export 'src/online_stream.dart';
export 'src/punctuation.dart';
export 'src/speaker_identification.dart';
export 'src/tts.dart';
export 'src/vad.dart';
Expand Down
91 changes: 91 additions & 0 deletions flutter/sherpa_onnx/lib/src/punctuation.dart
Original file line number Diff line number Diff line change
@@ -0,0 +1,91 @@
// Copyright (c) 2024 Xiaomi Corporation
import 'dart:ffi';
import 'package:ffi/ffi.dart';

import './sherpa_onnx_bindings.dart';

class OfflinePunctuationModelConfig {
OfflinePunctuationModelConfig(
{required this.ctTransformer,
this.numThreads = 1,
this.provider = 'cpu',
this.debug = true});

@override
String toString() {
return 'OfflinePunctuationModelConfig(ctTransformer: $ctTransformer, numThreads: $numThreads, provider: $provider, debug: $debug)';
}

final String ctTransformer;
final int numThreads;
final String provider;
final bool debug;
}

class OfflinePunctuationConfig {
OfflinePunctuationConfig({
required this.model,
});

@override
String toString() {
return 'OfflinePunctuationConfig(model: $model)';
}

final OfflinePunctuationModelConfig model;
}

class OfflinePunctuation {
OfflinePunctuation._({required this.ptr, required this.config});

// The user has to invoke OfflinePunctuation.free() to avoid memory leak.
factory OfflinePunctuation({required OfflinePunctuationConfig config}) {
final c = calloc<SherpaOnnxOfflinePunctuationConfig>();

final ctTransformerPtr = config.model.ctTransformer.toNativeUtf8();
c.ref.model.ctTransformer = ctTransformerPtr;
c.ref.model.numThreads = config.model.numThreads;
c.ref.model.debug = config.model.debug ? 1 : 0;

final providerPtr = config.model.provider.toNativeUtf8();
c.ref.model.provider = providerPtr;

final ptr =
SherpaOnnxBindings.sherpaOnnxCreateOfflinePunctuation?.call(c) ??
nullptr;

calloc.free(providerPtr);
calloc.free(ctTransformerPtr);
calloc.free(c);

return OfflinePunctuation._(ptr: ptr, config: config);
}

void free() {
SherpaOnnxBindings.sherpaOnnxDestroyOfflinePunctuation?.call(ptr);
ptr = nullptr;
}

String addPunct(String text) {
final textPtr = text.toNativeUtf8();

final p = SherpaOnnxBindings.sherpaOfflinePunctuationAddPunct
?.call(ptr, textPtr) ??
nullptr;

calloc.free(textPtr);

if (p == nullptr) {
return '';
}

final ans = p.toDartString();

SherpaOnnxBindings.sherpaOfflinePunctuationFreeText?.call(p);

return ans;
}

Pointer<SherpaOnnxOfflinePunctuation> ptr;
final OfflinePunctuationConfig config;
}
Loading

0 comments on commit 646f99c

Please sign in to comment.