diff --git a/3rdparty/voicevox/.gitignore b/3rdparty/voicevox/.gitignore new file mode 100644 index 000000000..8cb8e60a3 --- /dev/null +++ b/3rdparty/voicevox/.gitignore @@ -0,0 +1,6 @@ +build +dict +lib +node_scripts/voicevox_engine +requirements.txt +!.gitignore diff --git a/3rdparty/voicevox/CMakeLists.txt b/3rdparty/voicevox/CMakeLists.txt new file mode 100644 index 000000000..631126026 --- /dev/null +++ b/3rdparty/voicevox/CMakeLists.txt @@ -0,0 +1,73 @@ +cmake_minimum_required(VERSION 2.8.3) +project(voicevox) + +find_package(catkin REQUIRED + COMPONENTS + catkin_virtualenv +) + +catkin_python_setup() + +set(INSTALL_DIR ${PROJECT_SOURCE_DIR}) + +catkin_package() + +catkin_generate_virtualenv( + INPUT_REQUIREMENTS requirements.in + PYTHON_INTERPRETER python3 + USE_SYSTEM_PACKAGES FALSE +) + +add_custom_command( + OUTPUT voicevox_model_installed + COMMAND make -f ${PROJECT_SOURCE_DIR}/Makefile.model + MD5SUM_DIR=${PROJECT_SOURCE_DIR}/md5sum + INSTALL_DIR=${INSTALL_DIR} +) + + +add_custom_command( + OUTPUT voicevox_core_installed + COMMAND make -f ${PROJECT_SOURCE_DIR}/Makefile.core + MD5SUM_DIR=${PROJECT_SOURCE_DIR}/md5sum + INSTALL_DIR=${INSTALL_DIR} +) + +add_custom_command( + OUTPUT voicevox_engine_installed + COMMAND make -f ${PROJECT_SOURCE_DIR}/Makefile.engine + MD5SUM_DIR=${PROJECT_SOURCE_DIR}/md5sum + INSTALL_DIR=${INSTALL_DIR} +) + +add_custom_command( + OUTPUT open_jtalk_dic_installed + COMMAND make -f ${PROJECT_SOURCE_DIR}/Makefile.open_jtalk_dic + MD5SUM_DIR=${PROJECT_SOURCE_DIR}/md5sum + INSTALL_DIR=${INSTALL_DIR} +) + +add_custom_target(all_installed ALL DEPENDS + voicevox_model_installed + voicevox_core_installed + voicevox_engine_installed + open_jtalk_dic_installed) + +file(GLOB NODE_SCRIPTS_FILES node_scripts/*.py) +catkin_install_python( + PROGRAMS ${NODE_SCRIPTS_FILES} + DESTINATION ${CATKIN_PACKAGE_BIN_DESTINATION}/node_scripts/ +) +install(DIRECTORY node_scripts/voicevox_engine + DESTINATION ${CATKIN_PACKAGE_SHARE_DESTINATION}/catkin_virtualenv_scripts/ + USE_SOURCE_PERMISSIONS) +install(DIRECTORY launch dict + DESTINATION ${CATKIN_PACKAGE_SHARE_DESTINATION} + USE_SOURCE_PERMISSIONS) +install(PROGRAMS bin/text2wave + DESTINATION ${CATKIN_PACKAGE_SHARE_DESTINATION}/bin) + +install(DIRECTORY + ${INSTALL_DIR}/lib + DESTINATION ${CATKIN_PACKAGE_SHARE_DESTINATION} + USE_SOURCE_PERMISSIONS) diff --git a/3rdparty/voicevox/Makefile b/3rdparty/voicevox/Makefile new file mode 100644 index 000000000..a2c90f3bb --- /dev/null +++ b/3rdparty/voicevox/Makefile @@ -0,0 +1,11 @@ +all: + make -f Makefile.core + make -f Makefile.model + make -f Makefile.engine + make -f Makefile.open_jtalk_dic +clean: + make -f Makefile.core clean + make -f Makefile.model clean + make -f Makefile.engine clean + make -f Makefile.open_jtalk_dic clean + rm -rf build diff --git a/3rdparty/voicevox/Makefile.core b/3rdparty/voicevox/Makefile.core new file mode 100644 index 000000000..bac21eb0f --- /dev/null +++ b/3rdparty/voicevox/Makefile.core @@ -0,0 +1,28 @@ +# -*- makefile -*- + +all: installed.viocevox_core + +VERSION = 0.11.4 +FILENAME = core.zip +TARBALL = build/$(FILENAME) +TARBALL_URL = "https://github.com/VOICEVOX/voicevox_core/releases/download/$(VERSION)/core.zip" +SOURCE_DIR = build/core +UNPACK_CMD = unzip +MD5SUM_DIR = $(CURDIR)/md5sum +MD5SUM_FILE = $(MD5SUM_DIR)/$(FILENAME).md5sum +SCRIPT_DIR = $( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +include $(shell rospack find mk)/download_unpack_build.mk +INSTALL_DIR = './' + + +installed.viocevox_core: $(SOURCE_DIR)/unpacked + mkdir -p $(INSTALL_DIR)/lib + cp build/core/lib*.so $(INSTALL_DIR)/lib/ + cp build/core/*.bin $(INSTALL_DIR)/lib/ + cp build/core/metas.json $(INSTALL_DIR)/lib/metas.json + +clean: + rm -rf $(TARBALL) + rm -rf $(SOURCE_DIR) + rm -rf $(INSTALL_DIR)/lib + rm -rf build diff --git a/3rdparty/voicevox/Makefile.engine b/3rdparty/voicevox/Makefile.engine new file mode 100644 index 000000000..b3d6899fa --- /dev/null +++ b/3rdparty/voicevox/Makefile.engine @@ -0,0 +1,24 @@ +# -*- makefile -*- + +all: installed.voicevox_engine + +VERSION = 0.11.4 +FILENAME = $(VERSION).tar.gz +TARBALL = build/$(FILENAME) +TARBALL_URL = "https://github.com/VOICEVOX/voicevox_engine/archive/refs/tags/$(FILENAME)" +SOURCE_DIR = build/voicevox_engine-$(VERSION) +UNPACK_CMD = tar xvzf +MD5SUM_DIR = $(CURDIR)/md5sum +MD5SUM_FILE = $(MD5SUM_DIR)/voicevox_engine.tar.gz.md5sum +include $(shell rospack find mk)/download_unpack_build.mk +INSTALL_DIR = './' + + +installed.voicevox_engine: $(SOURCE_DIR)/unpacked + cp -r build/voicevox_engine-$(VERSION) $(INSTALL_DIR)/node_scripts/voicevox_engine + +clean: + rm -rf $(TARBALL) + rm -rf $(SOURCE_DIR) + rm -rf $(INSTALL_DIR)/node_scripts/voicevox_engine + rm -rf build diff --git a/3rdparty/voicevox/Makefile.model b/3rdparty/voicevox/Makefile.model new file mode 100644 index 000000000..004028105 --- /dev/null +++ b/3rdparty/voicevox/Makefile.model @@ -0,0 +1,26 @@ +# -*- makefile -*- + +all: installed.voicevox_model + +VERSION = 1.10.0 +FILENAME = onnxruntime-linux-x64-$(VERSION).tgz +TARBALL = build/$(FILENAME) +TARBALL_URL = "https://github.com/microsoft/onnxruntime/releases/download/v$(VERSION)/$(FILENAME)" +SOURCE_DIR = build/onnxruntime-linux-x64-$(VERSION) +UNPACK_CMD = tar xvzf +MD5SUM_DIR = $(CURDIR)/md5sum +MD5SUM_FILE = $(MD5SUM_DIR)/$(FILENAME).md5sum +SCRIPT_DIR = $( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +include $(shell rospack find mk)/download_unpack_build.mk +INSTALL_DIR = './' + + +installed.voicevox_model: $(SOURCE_DIR)/unpacked + mkdir -p $(INSTALL_DIR)/lib + cp build/onnxruntime-linux-x64-$(VERSION)/lib/* $(INSTALL_DIR)/lib + +clean: + rm -rf $(TARBALL) + rm -rf $(SOURCE_DIR) + rm -rf $(INSTALL_DIR)/lib + rm -rf build diff --git a/3rdparty/voicevox/Makefile.open_jtalk_dic b/3rdparty/voicevox/Makefile.open_jtalk_dic new file mode 100644 index 000000000..646921159 --- /dev/null +++ b/3rdparty/voicevox/Makefile.open_jtalk_dic @@ -0,0 +1,25 @@ +# -*- makefile -*- + +all: installed.open_jtalk_dic + +VERSION = 1.11.1 +FILENAME = open_jtalk_dic_utf_8-1.11.tar.gz +TARBALL = build/$(FILENAME) +TARBALL_URL = "https://github.com/r9y9/open_jtalk/releases/download/v$(VERSION)/$(FILENAME)" +SOURCE_DIR = build/open_jtalk_dic_utf_8-1.11 +UNPACK_CMD = tar xvzf +MD5SUM_DIR = $(CURDIR)/md5sum +MD5SUM_FILE = $(MD5SUM_DIR)/open_jtalk_dic.tar.gz.md5sum +include $(shell rospack find mk)/download_unpack_build.mk +INSTALL_DIR = './' + + +installed.open_jtalk_dic: $(SOURCE_DIR)/unpacked + mkdir -p $(INSTALL_DIR)/dict + cp -r build/open_jtalk_dic_utf_8-1.11 $(INSTALL_DIR)/dict + +clean: + rm -rf $(TARBALL) + rm -rf $(SOURCE_DIR) + rm -rf $(INSTALL_DIR)/dict/open_jtalk_dic_utf_8-1.11 + rm -rf build diff --git a/3rdparty/voicevox/README.md b/3rdparty/voicevox/README.md new file mode 100644 index 000000000..d5602db71 --- /dev/null +++ b/3rdparty/voicevox/README.md @@ -0,0 +1,103 @@ +# voicevox + +ROS Interface for [VOICEVOX](https://voicevox.hiroshiba.jp/) (AI speech synthesis) + +## TERM + +[VOICEVOX](https://voicevox.hiroshiba.jp/) is basically free to use, but please check the terms of use below. + +[TERM](https://voicevox.hiroshiba.jp/term) + +Each voice synthesis character has its own rules. Please use this package according to those terms. + +| Character name | term link | +| ---- | ---- | +| 四国めたん | https://zunko.jp/con_ongen_kiyaku.html | +| ずんだもん | https://zunko.jp/con_ongen_kiyaku.html | +| 春日部つむぎ | https://tsukushinyoki10.wixsite.com/ktsumugiofficial/利用規約 | +| 波音リツ | http://canon-voice.com/kiyaku.html | +| 雨晴はう | https://amehau.com/?page_id=225 | +| 玄野武宏 | https://virvoxproject.wixsite.com/official/voicevoxの利用規約 | +| 白上虎太郎 | https://virvoxproject.wixsite.com/official/voicevoxの利用規約 | +| 青山龍星 | https://virvoxproject.wixsite.com/official/voicevoxの利用規約 | +| 冥鳴ひまり | https://kotoran8zunzun.wixsite.com/my-site/利用規約 | +| 九州そら | https://zunko.jp/con_ongen_kiyaku.html | + +## Installation + +Build this package. + +```bash +cd /path/to/catkin_workspace +catkin build voicevox +``` + +## Usage + +### Launch sound_play with VOICEVOX Text-to-Speech + +```bash +roslaunch voicevox voicevox_texttospeech.launch +``` + + +### Say something + +#### For python users + +```python +import rospy +from sound_play.libsoundplay import SoundClient + +rospy.init_node('say_node') + +client = SoundClient(sound_action='robotsound_jp', sound_topic='robotsound_jp') + +client.say('こんにちは', voice='四国めたん-あまあま') +``` + +You can change the voice by changing the voice_name. +You can also specify the speaker id. +Look at the following tables for further details. + +| speaker_id | voice_name | +| ---- | ---- | +| 0 | 四国めたん-あまあま | +| 1 | ずんだもん-あまあま | +| 2 | 四国めたん-ノーマル | +| 3 | ずんだもん-ノーマル | +| 4 | 四国めたん-セクシー | +| 5 | ずんだもん-セクシー | +| 6 | 四国めたん-ツンツン | +| 7 | ずんだもん-ツンツン | +| 8 | 春日部つむぎ-ノーマル | +| 9 | 波音リツ-ノーマル | +| 10 | 雨晴はう-ノーマル | +| 11 | 玄野武宏-ノーマル | +| 12 | 白上虎太郎-ノーマル | +| 13 | 青山龍星-ノーマル | +| 14 | 冥鳴ひまり-ノーマル | +| 15 | 九州そら-あまあま | +| 16 | 九州そら-ノーマル | +| 17 | 九州そら-セクシー | +| 18 | 九州そら-ツンツン | +| 19 | 九州そら-ささやき | + +#### For roseus users + +``` +$ roseus +(load "package://pr2eus/speak.l") + +(ros::roseus "say_node") + +(speak "JSKへようこそ。" :lang "波音リツ" :wait t :topic-name "robotsound_jp") +``` + +### Tips + +Normally, the server for speech synthesis starts up at `http://localhost:50021`. +You can change the url and port by setting values for `VOICEVOX_TEXTTOSPEECH_URL` and `VOICEVOX_TEXTTOSPEECH_PORT`. + +You can also set the default character by setting `VOICEVOX_DEFAULT_SPEAKER_ID`. +Please refer to [here](#saysomething) for the speaker id. diff --git a/3rdparty/voicevox/bin/text2wave b/3rdparty/voicevox/bin/text2wave new file mode 100755 index 000000000..ca9630f39 --- /dev/null +++ b/3rdparty/voicevox/bin/text2wave @@ -0,0 +1,153 @@ +#!/usr/bin/env python3 +# -*- coding:utf-8 -*- + +import argparse +import os +import shutil +import sys + +import requests + +from voicevox.filecheck_utils import checksum_md5 +from voicevox.filecheck_utils import get_cache_dir + + +speaker_id_to_name = { + '0': '四国めたん-あまあま', + '1': 'ずんだもん-あまあま', + '2': '四国めたん-ノーマル', + '3': 'ずんだもん-ノーマル', + '4': '四国めたん-セクシー', + '5': 'ずんだもん-セクシー', + '6': '四国めたん-ツンツン', + '7': 'ずんだもん-ツンツン', + '8': '春日部つむぎ-ノーマル', + '9': '波音リツ-ノーマル', + '10': '雨晴はう-ノーマル', + '11': '玄野武宏-ノーマル', + '12': '白上虎太郎-ノーマル', + '13': '青山龍星-ノーマル', + '14': '冥鳴ひまり-ノーマル', + '15': '九州そら-あまあま', + '16': '九州そら-ノーマル', + '17': '九州そら-セクシー', + '18': '九州そら-ツンツン', + '19': '九州そら-ささやき', +} + +name_to_speaker_id = { + b: a for a, b in speaker_id_to_name.items() +} + + +DEFAULT_SPEAKER_ID = os.environ.get( + 'VOICEVOX_DEFAULT_SPEAKER_ID', '2') +if not DEFAULT_SPEAKER_ID.isdigit(): + DEFAULT_SPEAKER_ID = name_to_speaker_id[DEFAULT_SPEAKER_ID] +VOICEVOX_TEXTTOSPEECH_URL = os.environ.get( + 'VOICEVOX_TEXTTOSPEECH_URL', 'localhost') +VOICEVOX_TEXTTOSPEECH_PORT = os.environ.get( + 'VOICEVOX_TEXTTOSPEECH_PORT', 50021) +cache_enabled = os.environ.get( + 'ROS_VOICEVOX_TEXTTOSPEECH_CACHE_ENABLED', True) +cache_enabled = cache_enabled is True \ + or cache_enabled == 'true' # for launch env tag. + + +def determine_voice_name(voice_name): + if len(voice_name) == 0: + speaker_id = DEFAULT_SPEAKER_ID + else: + if voice_name.isdigit(): + if voice_name in speaker_id_to_name: + speaker_id = voice_name + else: + print( + '[Text2Wave] Invalid speaker_id ({}). Use default voice.' + .format(speaker_id_to_name[DEFAULT_SPEAKER_ID])) + speaker_id = DEFAULT_SPEAKER_ID + else: + candidates = list(filter( + lambda name: name.startswith(voice_name), + name_to_speaker_id)) + if candidates: + speaker_id = name_to_speaker_id[candidates[0]] + else: + print('[Text2Wave] Invalid voice_name ({}). Use default voice.' + .format(speaker_id_to_name[DEFAULT_SPEAKER_ID])) + speaker_id = DEFAULT_SPEAKER_ID + print('[Text2Wave] Speak using voice_name ({})..'.format( + speaker_id_to_name[speaker_id])) + return speaker_id + + +def convert_to_str(x): + if isinstance(x, str): + pass + elif isinstance(x, bytes): + x = x.decode('utf-8') + else: + raise ValueError( + 'Invalid input x type: {}' + .format(type(x))) + return x + + +def request_synthesis( + sentence, output_path, speaker_id='1'): + headers = {'accept': 'application/json'} + + sentence = convert_to_str(sentence) + speaker_id = convert_to_str(speaker_id) + params = { + 'speaker': speaker_id, + 'text': sentence, + } + base_url = 'http://{}:{}'.format( + VOICEVOX_TEXTTOSPEECH_URL, + VOICEVOX_TEXTTOSPEECH_PORT) + url = '{}/audio_query'.format(base_url) + response = requests.post(url, headers=headers, + params=params) + data = response.json() + url = '{}/synthesis'.format(base_url) + response = requests.post(url, headers=headers, + params=params, + json=data) + with open(output_path, 'wb') as f: + f.write(response.content) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='') + parser.add_argument('-eval', '--evaluate') + parser.add_argument('-o', '--output') + parser.add_argument('text') + args = parser.parse_args() + + with open(args.text, 'rb') as f: + speech_text = f.readline() + + speaker_id = determine_voice_name( + args.evaluate.lstrip('(').rstrip(')')) + + if cache_enabled: + cache_dir = get_cache_dir() + md5 = checksum_md5(args.text) + cache_filename = os.path.join( + cache_dir, + '--'.join([md5, speaker_id]) + + '.wav') + if os.path.exists(cache_filename): + print('[Text2Wave] Using cached sound file ({}) for {}' + .format(cache_filename, speech_text.decode('utf-8'))) + shutil.copy(cache_filename, args.output) + sys.exit(0) + + request_synthesis(speech_text, + args.output, + speaker_id) + if cache_enabled: + text_cache_filename = os.path.splitext(cache_filename)[0] + '.txt' + shutil.copy(args.text, text_cache_filename) + shutil.copy(args.output, cache_filename) diff --git a/3rdparty/voicevox/launch/voicevox_texttospeech.launch b/3rdparty/voicevox/launch/voicevox_texttospeech.launch new file mode 100644 index 000000000..c701a1bd4 --- /dev/null +++ b/3rdparty/voicevox/launch/voicevox_texttospeech.launch @@ -0,0 +1,28 @@ + + + + + + + + + + + + + + + + + + + diff --git a/3rdparty/voicevox/md5sum/core.zip.md5sum b/3rdparty/voicevox/md5sum/core.zip.md5sum new file mode 100644 index 000000000..f5b5ac439 --- /dev/null +++ b/3rdparty/voicevox/md5sum/core.zip.md5sum @@ -0,0 +1 @@ +96149a074d8ee093039321a88e00076d core.zip diff --git a/3rdparty/voicevox/md5sum/onnxruntime-linux-x64-1.10.0.tgz.md5sum b/3rdparty/voicevox/md5sum/onnxruntime-linux-x64-1.10.0.tgz.md5sum new file mode 100644 index 000000000..817b68d89 --- /dev/null +++ b/3rdparty/voicevox/md5sum/onnxruntime-linux-x64-1.10.0.tgz.md5sum @@ -0,0 +1 @@ +9ca61e2009a16cf8a1e9ab9ad0655009 onnxruntime-linux-x64-1.10.0.tgz diff --git a/3rdparty/voicevox/md5sum/open_jtalk_dic.tar.gz.md5sum b/3rdparty/voicevox/md5sum/open_jtalk_dic.tar.gz.md5sum new file mode 100644 index 000000000..8ce4bb07b --- /dev/null +++ b/3rdparty/voicevox/md5sum/open_jtalk_dic.tar.gz.md5sum @@ -0,0 +1 @@ +ba02dac4143492c3790f949be224dfdf open_jtalk_dic_utf_8-1.11.tar.gz diff --git a/3rdparty/voicevox/md5sum/voicevox_engine.tar.gz.md5sum b/3rdparty/voicevox/md5sum/voicevox_engine.tar.gz.md5sum new file mode 100644 index 000000000..5947e3633 --- /dev/null +++ b/3rdparty/voicevox/md5sum/voicevox_engine.tar.gz.md5sum @@ -0,0 +1 @@ +997bf9e915f7d6288c923ab1ff5f4ff6 0.11.4.tar.gz diff --git a/3rdparty/voicevox/node_scripts/server.py b/3rdparty/voicevox/node_scripts/server.py new file mode 100644 index 000000000..add596aff --- /dev/null +++ b/3rdparty/voicevox/node_scripts/server.py @@ -0,0 +1,573 @@ +#!/usr/bin/env python3 + +# This code was created based on the following link's code. +# https://github.com/VOICEVOX/voicevox_engine/blob/0.11.4/run.py + +import base64 +from distutils.version import LooseVersion +from functools import lru_cache +import imp +import json +import multiprocessing +import os +import os.path as osp +from pathlib import Path +from tempfile import NamedTemporaryFile +from tempfile import TemporaryFile +from typing import Dict +from typing import List +from typing import Optional +import zipfile + +from fastapi import FastAPI +from fastapi import HTTPException +from fastapi.middleware.cors import CORSMiddleware +from fastapi.params import Query +from fastapi import Response +import rospkg +import rospy +import soundfile +from starlette.responses import FileResponse +import uvicorn + + +PKG_NAME = 'voicevox' +abs_path = osp.dirname(osp.abspath(__file__)) +voicevox_engine = imp.load_package( + 'voicevox_engine', osp.join(abs_path, 'voicevox_engine/voicevox_engine')) +rospack = rospkg.RosPack() +voicevox_dir = rospack.get_path(PKG_NAME) +voicevox_lib_dir = osp.join(voicevox_dir, 'lib') +# set pyopenjtalk's dic.tar.gz file +os.environ['OPEN_JTALK_DICT_DIR'] = osp.join( + voicevox_dir, 'dict', 'open_jtalk_dic_utf_8-1.11') + + +from voicevox_engine import __version__ +from voicevox_engine.kana_parser import create_kana +from voicevox_engine.kana_parser import parse_kana +from voicevox_engine.model import AccentPhrase +from voicevox_engine.model import AudioQuery +from voicevox_engine.model import ParseKanaBadRequest +from voicevox_engine.model import ParseKanaError +from voicevox_engine.model import Speaker +from voicevox_engine.model import SpeakerInfo +from voicevox_engine.model import SupportedDevicesInfo +from voicevox_engine.morphing import \ + synthesis_morphing_parameter as _synthesis_morphing_parameter +from voicevox_engine.morphing import synthesis_morphing +from voicevox_engine.preset import Preset +from voicevox_engine.preset import PresetLoader +from voicevox_engine.synthesis_engine import make_synthesis_engines +from voicevox_engine.synthesis_engine import SynthesisEngineBase +from voicevox_engine.user_dict import user_dict_startup_processing +from voicevox_engine.utility import connect_base64_waves +from voicevox_engine.utility import ConnectBase64WavesException +from voicevox_engine.utility import engine_root + + +def b64encode_str(s): + return base64.b64encode(s).decode("utf-8") + + +def generate_app( + synthesis_engines: Dict[str, SynthesisEngineBase], latest_core_version: str +) -> FastAPI: + root_dir = engine_root() + + default_sampling_rate = synthesis_engines[latest_core_version].default_sampling_rate + + app = FastAPI( + title="VOICEVOX ENGINE", + description="VOICEVOXの音声合成エンジンです。", + version=__version__, + ) + + app.add_middleware( + CORSMiddleware, + allow_origins=["*"], + allow_credentials=True, + allow_methods=["*"], + allow_headers=["*"], + ) + + preset_loader = PresetLoader( + preset_path=root_dir / "presets.yaml", + ) + + # キャッシュを有効化 + # モジュール側でlru_cacheを指定するとキャッシュを制御しにくいため、HTTPサーバ側で指定する + # TODO: キャッシュを管理するモジュール側API・HTTP側APIを用意する + synthesis_morphing_parameter = lru_cache(maxsize=4)(_synthesis_morphing_parameter) + + # @app.on_event("startup") + # async def start_catch_disconnection(): + # if args.enable_cancellable_synthesis: + # loop = asyncio.get_event_loop() + # _ = loop.create_task(cancellable_engine.catch_disconnection()) + + @app.on_event("startup") + def apply_user_dict(): + user_dict_startup_processing() + + def get_engine(core_version: Optional[str]) -> SynthesisEngineBase: + if core_version is None: + return synthesis_engines[latest_core_version] + if core_version in synthesis_engines: + return synthesis_engines[core_version] + raise HTTPException(status_code=422, detail="不明なバージョンです") + + @app.post( + "/audio_query", + response_model=AudioQuery, + tags=["クエリ作成"], + summary="音声合成用のクエリを作成する", + ) + def audio_query(text: str, speaker: int, core_version: Optional[str] = None): + """ + クエリの初期値を得ます。ここで得られたクエリはそのまま音声合成に利用できます。各値の意味は`Schemas`を参照してください。 + """ + engine = get_engine(core_version) + accent_phrases = engine.create_accent_phrases(text, speaker_id=speaker) + return AudioQuery( + accent_phrases=accent_phrases, + speedScale=1, + pitchScale=0, + intonationScale=1, + volumeScale=1, + prePhonemeLength=0.1, + postPhonemeLength=0.1, + outputSamplingRate=default_sampling_rate, + outputStereo=False, + kana=create_kana(accent_phrases), + ) + + @app.post( + "/audio_query_from_preset", + response_model=AudioQuery, + tags=["クエリ作成"], + summary="音声合成用のクエリをプリセットを用いて作成する", + ) + def audio_query_from_preset( + text: str, preset_id: int, core_version: Optional[str] = None + ): + """ + クエリの初期値を得ます。ここで得られたクエリはそのまま音声合成に利用できます。各値の意味は`Schemas`を参照してください。 + """ + engine = get_engine(core_version) + presets, err_detail = preset_loader.load_presets() + if err_detail: + raise HTTPException(status_code=422, detail=err_detail) + for preset in presets: + if preset.id == preset_id: + selected_preset = preset + break + else: + raise HTTPException(status_code=422, detail="該当するプリセットIDが見つかりません") + + accent_phrases = engine.create_accent_phrases( + text, speaker_id=selected_preset.style_id + ) + return AudioQuery( + accent_phrases=accent_phrases, + speedScale=selected_preset.speedScale, + pitchScale=selected_preset.pitchScale, + intonationScale=selected_preset.intonationScale, + volumeScale=selected_preset.volumeScale, + prePhonemeLength=selected_preset.prePhonemeLength, + postPhonemeLength=selected_preset.postPhonemeLength, + outputSamplingRate=default_sampling_rate, + outputStereo=False, + kana=create_kana(accent_phrases), + ) + + @app.post( + "/accent_phrases", + response_model=List[AccentPhrase], + tags=["クエリ編集"], + summary="テキストからアクセント句を得る", + responses={ + 400: { + "description": "読み仮名のパースに失敗", + "model": ParseKanaBadRequest, + } + }, + ) + def accent_phrases( + text: str, + speaker: int, + is_kana: bool = False, + core_version: Optional[str] = None, + ): + """ + テキストからアクセント句を得ます。 + is_kanaが`true`のとき、テキストは次のようなAquesTalkライクな記法に従う読み仮名として処理されます。デフォルトは`false`です。 + * 全てのカナはカタカナで記述される + * アクセント句は`/`または`、`で区切る。`、`で区切った場合に限り無音区間が挿入される。 + * カナの手前に`_`を入れるとそのカナは無声化される + * アクセント位置を`'`で指定する。全てのアクセント句にはアクセント位置を1つ指定する必要がある。 + * アクセント句末に`？`(全角)を入れることにより疑問文の発音ができる。 + """ + engine = get_engine(core_version) + if is_kana: + try: + accent_phrases = parse_kana(text) + except ParseKanaError as err: + raise HTTPException( + status_code=400, + detail=ParseKanaBadRequest(err).dict(), + ) + accent_phrases = engine.replace_mora_data( + accent_phrases=accent_phrases, speaker_id=speaker + ) + + return accent_phrases + else: + return engine.create_accent_phrases(text, speaker_id=speaker) + + @app.post( + "/mora_data", + response_model=List[AccentPhrase], + tags=["クエリ編集"], + summary="アクセント句から音高・音素長を得る", + ) + def mora_data( + accent_phrases: List[AccentPhrase], + speaker: int, + core_version: Optional[str] = None, + ): + engine = get_engine(core_version) + return engine.replace_mora_data(accent_phrases, speaker_id=speaker) + + @app.post( + "/mora_length", + response_model=List[AccentPhrase], + tags=["クエリ編集"], + summary="アクセント句から音素長を得る", + ) + def mora_length( + accent_phrases: List[AccentPhrase], + speaker: int, + core_version: Optional[str] = None, + ): + engine = get_engine(core_version) + return engine.replace_phoneme_length( + accent_phrases=accent_phrases, speaker_id=speaker + ) + + @app.post( + "/mora_pitch", + response_model=List[AccentPhrase], + tags=["クエリ編集"], + summary="アクセント句から音高を得る", + ) + def mora_pitch( + accent_phrases: List[AccentPhrase], + speaker: int, + core_version: Optional[str] = None, + ): + engine = get_engine(core_version) + return engine.replace_mora_pitch( + accent_phrases=accent_phrases, speaker_id=speaker + ) + + @app.post( + "/synthesis", + response_class=FileResponse, + responses={ + 200: { + "content": { + "audio/wav": {"schema": {"type": "string", "format": "binary"}} + }, + } + }, + tags=["音声合成"], + summary="音声合成する", + ) + def synthesis( + query: AudioQuery, + speaker: int, + enable_interrogative_upspeak: bool = Query( # noqa: B008 + default=True, + description="疑問系のテキストが与えられたら語尾を自動調整する", + ), + core_version: Optional[str] = None, + ): + engine = get_engine(core_version) + wave = engine.synthesis( + query=query, + speaker_id=speaker, + enable_interrogative_upspeak=enable_interrogative_upspeak, + ) + + with NamedTemporaryFile(delete=False) as f: + soundfile.write( + file=f, data=wave, samplerate=query.outputSamplingRate, format="WAV" + ) + + return FileResponse(f.name, media_type="audio/wav") + + @app.post( + "/multi_synthesis", + response_class=FileResponse, + responses={ + 200: { + "content": { + "application/zip": { + "schema": {"type": "string", "format": "binary"} + } + }, + } + }, + tags=["音声合成"], + summary="複数まとめて音声合成する", + ) + def multi_synthesis( + queries: List[AudioQuery], + speaker: int, + core_version: Optional[str] = None, + ): + engine = get_engine(core_version) + sampling_rate = queries[0].outputSamplingRate + + with NamedTemporaryFile(delete=False) as f: + + with zipfile.ZipFile(f, mode="a") as zip_file: + + for i in range(len(queries)): + + if queries[i].outputSamplingRate != sampling_rate: + raise HTTPException( + status_code=422, detail="サンプリングレートが異なるクエリがあります" + ) + + with TemporaryFile() as wav_file: + + wave = engine.synthesis(query=queries[i], speaker_id=speaker) + soundfile.write( + file=wav_file, + data=wave, + samplerate=sampling_rate, + format="WAV", + ) + wav_file.seek(0) + zip_file.writestr(f"{str(i + 1).zfill(3)}.wav", wav_file.read()) + + return FileResponse(f.name, media_type="application/zip") + + @app.post( + "/synthesis_morphing", + response_class=FileResponse, + responses={ + 200: { + "content": { + "audio/wav": {"schema": {"type": "string", "format": "binary"}} + }, + } + }, + tags=["音声合成"], + summary="2人の話者でモーフィングした音声を合成する", + ) + def _synthesis_morphing( + query: AudioQuery, + base_speaker: int, + target_speaker: int, + morph_rate: float = Query(..., ge=0.0, le=1.0), # noqa: B008 + core_version: Optional[str] = None, + ): + """ + 指定された2人の話者で音声を合成、指定した割合でモーフィングした音声を得ます。 + モーフィングの割合は`morph_rate`で指定でき、0.0でベースの話者、1.0でターゲットの話者に近づきます。 + """ + engine = get_engine(core_version) + + # 生成したパラメータはキャッシュされる + morph_param = synthesis_morphing_parameter( + engine=engine, + query=query, + base_speaker=base_speaker, + target_speaker=target_speaker, + ) + + morph_wave = synthesis_morphing( + morph_param=morph_param, + morph_rate=morph_rate, + output_stereo=query.outputStereo, + ) + + with NamedTemporaryFile(delete=False) as f: + soundfile.write( + file=f, + data=morph_wave, + samplerate=morph_param.fs, + format="WAV", + ) + + return FileResponse(f.name, media_type="audio/wav") + + @app.post( + "/connect_waves", + response_class=FileResponse, + responses={ + 200: { + "content": { + "audio/wav": {"schema": {"type": "string", "format": "binary"}} + }, + } + }, + tags=["その他"], + summary="base64エンコードされた複数のwavデータを一つに結合する", + ) + def connect_waves(waves: List[str]): + """ + base64エンコードされたwavデータを一纏めにし、wavファイルで返します。 + """ + try: + waves_nparray, sampling_rate = connect_base64_waves(waves) + except ConnectBase64WavesException as err: + return HTTPException(status_code=422, detail=str(err)) + + with NamedTemporaryFile(delete=False) as f: + soundfile.write( + file=f, + data=waves_nparray, + samplerate=sampling_rate, + format="WAV", + ) + + return FileResponse(f.name, media_type="audio/wav") + + @app.get("/presets", response_model=List[Preset], tags=["その他"]) + def get_presets(): + """ + エンジンが保持しているプリセットの設定を返します + + Returns + ------- + presets: List[Preset] + プリセットのリスト + """ + presets, err_detail = preset_loader.load_presets() + if err_detail: + raise HTTPException(status_code=422, detail=err_detail) + return presets + + @app.get("/version", tags=["その他"]) + def version() -> str: + return __version__ + + @app.get("/core_versions", response_model=List[str], tags=["その他"]) + def core_versions() -> List[str]: + return Response( + content=json.dumps(list(synthesis_engines.keys())), + media_type="application/json", + ) + + @app.get("/speakers", response_model=List[Speaker], tags=["その他"]) + def speakers( + core_version: Optional[str] = None, + ): + engine = get_engine(core_version) + return Response( + content=engine.speakers, + media_type="application/json", + ) + + @app.get("/speaker_info", response_model=SpeakerInfo, tags=["その他"]) + def speaker_info(speaker_uuid: str, core_version: Optional[str] = None): + """ + 指定されたspeaker_uuidに関する情報をjson形式で返します。 + 画像や音声はbase64エンコードされたものが返されます。 + + Returns + ------- + ret_data: SpeakerInfo + """ + speakers = json.loads(get_engine(core_version).speakers) + for i in range(len(speakers)): + if speakers[i]["speaker_uuid"] == speaker_uuid: + speaker = speakers[i] + break + else: + raise HTTPException(status_code=404, detail="該当する話者が見つかりません") + + try: + policy = (root_dir / f"speaker_info/{speaker_uuid}/policy.md").read_text( + "utf-8" + ) + portrait = b64encode_str( + (root_dir / f"speaker_info/{speaker_uuid}/portrait.png").read_bytes() + ) + style_infos = [] + for style in speaker["styles"]: + id = style["id"] + icon = b64encode_str( + ( + root_dir / f"speaker_info/{speaker_uuid}/icons/{id}.png" + ).read_bytes() + ) + voice_samples = [ + b64encode_str( + ( + root_dir + / "speaker_info/{}/voice_samples/{}_{}.wav".format( + speaker_uuid, id, str(j + 1).zfill(3) + ) + ).read_bytes() + ) + for j in range(3) + ] + style_infos.append( + {"id": id, "icon": icon, "voice_samples": voice_samples} + ) + except FileNotFoundError: + import traceback + + traceback.print_exc() + raise HTTPException(status_code=500, detail="追加情報が見つかりませんでした") + + ret_data = {"policy": policy, "portrait": portrait, "style_infos": style_infos} + return ret_data + + @app.get("/supported_devices", response_model=SupportedDevicesInfo, tags=["その他"]) + def supported_devices( + core_version: Optional[str] = None, + ): + supported_devices = get_engine(core_version).supported_devices + if supported_devices is None: + raise HTTPException(status_code=422, detail="非対応の機能です。") + return Response( + content=supported_devices, + media_type="application/json", + ) + + return app + + +if __name__ == "__main__": + multiprocessing.freeze_support() + rospy.init_node('voicevox_server') + + voicelib_dir = [Path(voicevox_lib_dir)] + use_gpu = False + host = rospy.get_param('~host', "127.0.0.1") + port = rospy.get_param('~port', 50021) + cpu_num_threads = rospy.get_param('~cpu_num_threads', None) + if cpu_num_threads is None: + cpu_num_threads = multiprocessing.cpu_count() + + synthesis_engines = make_synthesis_engines( + use_gpu=use_gpu, + voicelib_dirs=voicelib_dir, + cpu_num_threads=cpu_num_threads, + ) + if len(synthesis_engines) == 0: + rospy.logerr("音声合成エンジンがありません。") + latest_core_version = str(max([LooseVersion(ver) + for ver in synthesis_engines])) + + uvicorn.run( + generate_app(synthesis_engines, latest_core_version), + host=host, + port=port, + ) diff --git a/3rdparty/voicevox/package.xml b/3rdparty/voicevox/package.xml new file mode 100644 index 000000000..5240c3468 --- /dev/null +++ b/3rdparty/voicevox/package.xml @@ -0,0 +1,36 @@ + + + + voicevox + 0.0.1 + VOICEVOX: AI speech synthesis + Iori Yanokura + + MIT + + http://ros.org/wiki/voicevox + + Iori Yanokura + + catkin + catkin_virtualenv + + mk + roslib + rospack + unzip + wget + + python3 + python3-requests + sound_play + unzip + wget + + + requirements.txt + + + diff --git a/3rdparty/voicevox/python/voicevox/__init__.py b/3rdparty/voicevox/python/voicevox/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/3rdparty/voicevox/python/voicevox/filecheck_utils.py b/3rdparty/voicevox/python/voicevox/filecheck_utils.py new file mode 100644 index 000000000..6c881b5f5 --- /dev/null +++ b/3rdparty/voicevox/python/voicevox/filecheck_utils.py @@ -0,0 +1,43 @@ +import hashlib +import os + + +def get_cache_dir(): + """Return cache dir. + + Returns + ------- + cache_dir : str + cache directory. + """ + ros_home = os.getenv('ROS_HOME', os.path.expanduser('~/.ros')) + pkg_ros_home = os.path.join(ros_home, 'voicevox_texttospeech') + default_cache_dir = os.path.join(pkg_ros_home, 'cache') + cache_dir = os.environ.get( + 'ROS_VOICEVOX_TEXTTOSPEECH_CACHE_DIR', + default_cache_dir) + if not os.path.exists(cache_dir): + os.makedirs(cache_dir) + return cache_dir + + +def checksum_md5(filename, blocksize=8192): + """Calculate md5sum. + + Parameters + ---------- + filename : str or pathlib.Path + input filename. + blocksize : int + MD5 has 128-byte digest blocks (default: 8192 is 128x64). + Returns + ------- + md5 : str + calculated md5sum. + """ + filename = str(filename) + hash_factory = hashlib.md5() + with open(filename, 'rb') as f: + for chunk in iter(lambda: f.read(blocksize), b''): + hash_factory.update(chunk) + return hash_factory.hexdigest() diff --git a/3rdparty/voicevox/requirements.in b/3rdparty/voicevox/requirements.in new file mode 100644 index 000000000..c9cfd223a --- /dev/null +++ b/3rdparty/voicevox/requirements.in @@ -0,0 +1,11 @@ +PyYAML +aiofiles +appdirs +fastapi +git+https://github.com/VOICEVOX/pyopenjtalk@a85521a0a0f298f08d9e9b24987b3c77eb4aaff5#egg=pyopenjtalk +numpy +python-multipart +pyworld +scipy +soundfile +uvicorn diff --git a/3rdparty/voicevox/setup.py b/3rdparty/voicevox/setup.py new file mode 100644 index 000000000..939174bc8 --- /dev/null +++ b/3rdparty/voicevox/setup.py @@ -0,0 +1,12 @@ +from distutils.core import setup + +from catkin_pkg.python_setup import generate_distutils_setup +from setuptools import find_packages + + +d = generate_distutils_setup( + packages=find_packages('python'), + package_dir={'': 'python'}, +) + +setup(**d) diff --git a/dialogflow_task_executive/node_scripts/task_executive.py b/dialogflow_task_executive/node_scripts/task_executive.py index 686327a55..e826d50d7 100644 --- a/dialogflow_task_executive/node_scripts/task_executive.py +++ b/dialogflow_task_executive/node_scripts/task_executive.py @@ -7,7 +7,9 @@ import rospy from app_manager.msg import AppList +from app_manager.msg import KeyValue from app_manager.srv import StartApp +from app_manager.srv import StartAppRequest from app_manager.srv import StopApp from std_srvs.srv import Empty @@ -90,12 +92,16 @@ def available_apps(self): return map(lambda a: a.name, self._latest_msg.available_apps) - def start_app(self, name): + def start_app(self, name, launch_args): if name in self.running_apps: raise RuntimeError("{} is already running".format(name)) elif name not in self.available_apps: raise RuntimeError("{} is not available".format(name)) - res = self._srv_start_app(name=name) + req = StartAppRequest() + req.name = name + for key, value in launch_args.items(): + req.args.append(KeyValue(key=key, value=value)) + res = self._srv_start_app(req) if res.started: rospy.loginfo("{} successfully started".format(name)) return True @@ -221,6 +227,12 @@ def dialog_cb(self, msg): try: params = json.loads(msg.parameters) rospy.set_param("/action/parameters", params) + # set launch_args + launch_args = {} + for key, value in params.items(): + launch_args[key.encode('utf-8')] = value.encode('utf-8') + except AttributeError as e: + rospy.logerr(e) except ValueError: rospy.logerr( "Failed to parse parameters of action '{}'".format(msg.action)) @@ -228,7 +240,7 @@ def dialog_cb(self, msg): rospy.loginfo( "Starting '{}' with parameters '{}'" .format(msg.action, msg.parameters)) - self.app_manager.start_app(action) + self.app_manager.start_app(action, launch_args) def app_start_cb(self, name): rospy.loginfo("{} started".format(name)) diff --git a/google_chat_ros/CMakeLists.txt b/google_chat_ros/CMakeLists.txt new file mode 100644 index 000000000..300d8e27e --- /dev/null +++ b/google_chat_ros/CMakeLists.txt @@ -0,0 +1,75 @@ +cmake_minimum_required(VERSION 2.8.3) +project(google_chat_ros) + +find_package( + catkin REQUIRED COMPONENTS + catkin_virtualenv REQUIRED + rospy + actionlib_msgs + std_msgs + message_generation + rostest + ) + +catkin_python_setup() + +add_message_files( + DIRECTORY msg + ) + +add_action_files( + FILES + SendMessage.action + ) + +generate_messages( + DEPENDENCIES + std_msgs + actionlib_msgs + ) + +catkin_package() + +include_directories() + +# generate the virtualenv +catkin_generate_virtualenv( + PYTHON_INTERPRETER python3 + USE_SYSTEM_PACKAGES FALSE + EXTRA_PIP_ARGS + -vvv + ) + +# import test +if(CATKIN_ENABLE_TESTING) + catkin_install_python( + PROGRAMS test/test_import.py + DESTINATION ${CATKIN_PACKAGE_BIN_DESTINATION} + ) + add_rostest(test/import.test + DEPENDENCIES ${PROJECT_NAME}_generate_virtualenv + ) +endif() + +# install +# euslisp +file(GLOB EUSLISP_SCRIPTS scripts/*.l) +install(FILES ${EUSLISP_SCRIPTS} + DESTINATION ${CATKIN_PACKAGE_BIN_DESTINATION} + ) + +# python +file(GLOB PYTHON_SCRIPTS scripts/*.py) +catkin_install_python( + PROGRAMS ${PYTHON_SCRIPTS} + DESTINATION ${CATKIN_PACKAGE_BIN_DESTINATION} + ) +# python requirements +install(FILES requirements.txt + DESTINATION ${CATKIN_PACKAGE_SHARE_DESTINATION} + ) + +# launch +install(DIRECTORY launch + DESTINATION ${CATKIN_PACKAGE_SHARE_DESTINATION} + ) diff --git a/google_chat_ros/README.md b/google_chat_ros/README.md new file mode 100644 index 000000000..815bf4116 --- /dev/null +++ b/google_chat_ros/README.md @@ -0,0 +1,309 @@ +# Google Chat ROS +The ROS wrapper for Google Chat API +1. [Installation Guide](#install) +1. [Sending the message](#send) +1. [Receiving the message](#recieve) +1. [Handling the event](#event) +1. [Optional functions](#optional) +1. [Helper nodes](#helper) + + +## 1. Installation Guide +### 1.1 Get the API KEY +At first, you should have the permission to access the Google Chat API. +See [Google Official Document](https://developers.google.com/chat/how-tos/service-accounts#step_1_create_service_account_and_private_key). Please ensure to get JSON credetial file and save it. DO NOT LOST IT! +For JSK members, all keys are available at [Google Drive](https://drive.google.com/drive/folders/1Enbbta5QuZ-hrUWdTjVDEjDJc3j7Abxo?usp=sharing). If you make new API keys, please upload them here. + +### 1.2 Select the way how to recieve Google Chat event +The way you recieve Google Chat event from API server depends on your system. If your system has static IP and is allowed to recieve https request with specific port, please see [HTTPS mode](#https). If not, please see [Pub/Sub mode](#pubsub). + + +#### HTTPS mode +When you send the message, the node uses Google REST API. +When you recieve the message, Google Chat API sends https request to your machine and the node handles it. + +![google_chat_https_system](https://user-images.githubusercontent.com/27789460/166410618-6ae286bd-86d8-47e8-87c9-bae0c66493ed.png) + +You have to prepare SSL certificate. Self-signed one is not available because of Google security issue. Please use the service like Let's Encrypt. In Google Cloud console, please choose `App URL` as connection settings and fill the URL in the App URL form. + +![google_chat_https](https://user-images.githubusercontent.com/27789460/166408349-09520454-4c55-4ca7-bbdc-1d3f27e243b9.png) + + +#### Pub/Sub mode +When you send the message, the node uses Google REST API. +When you recieve the message, the node uses Google Pub/Sub API's subscription. The node has already established its connection to Google Pub/Sub API when you launch it. + +![google_chat_pubsub_system](https://user-images.githubusercontent.com/27789460/166410714-03f16096-eea4-4eeb-9487-5ab3df309332.png) + +The way how to set up in Google Cloud console shows below. +##### 1. Authorize the existing Google Chat API project to access Google Cloud Pub/Sub service +In IAM settings in the console, please add the role `Pub/Sub Admin` to service account. + +![pubsub_admin_mosaic](https://user-images.githubusercontent.com/27789460/166408915-832a279f-da9e-463b-86e4-8a18ad5e4f5a.png) + +##### 2. Create Pub/Sub topic and subscriber +In Pub/Sub settings in the console, please add the topic and subscriptions. +In the figure, we set the topic name `chat`, the subscription name `chat-sub` as an example. + +![pubsub_topic_mosaic](https://user-images.githubusercontent.com/27789460/166409434-8f7fa329-1ae1-4cc4-aba2-82f43c2de16f.png) + +![pubsub_subscription](https://user-images.githubusercontent.com/27789460/166409454-cc59ec43-f59e-4b63-a1b8-fadefcdd46ce.png) + +Note that if you set the topic name `chat`, the full name of it becomes `projects//topics/chat`. Please confirm the subsciptions subscribes the full name not short one. + +##### 3. Grant publish rigts on your topic +In order for Google Chat to publish messages to your topic, it must have publishing rights to the topic. To grant Google Chat these permissions, assign the Pub/Sub Publisher role to the following service account +``` +chat-api-push@system.gserviceaccount.com +``` +![google_chat_pubsub_permission](https://user-images.githubusercontent.com/27789460/173894738-cc169b21-0873-4def-9179-f686a2ae68ec.png) + + +##### 4. Set Google Chat API Connection settings +Please choose `Cloud Pub/Sub` as connection settings and fill the full topic name in the Topic Name form. + +![google_chat_pubsub](https://user-images.githubusercontent.com/27789460/166408478-b662b73c-35a8-43e8-aaaa-93efdd48e486.png) + + +### 1.3 Install/Build the ROS node +If you want to build from the source +```bash +source /opt/ros/${ROS_DISTRO}/setup.bash +mkdir -p ~/catkin_ws/src && cd ~/catkin_ws/src +git clone https://github.com/jsk-ros-pkg/jsk_3rdparty +rosdep install --ignore-src --from-paths . -y -r +cd .. +catkin build +``` +### 1.4 Launch the node +#### HTTPS mode +You have to set rosparams `receiving_mode=https`, `google_cloud_credentials_json`, `host`, `port`, `ssl_certfile`, `ssl_keyfile`. +#### Pub/Sub mode +You have to set rosparams `receiving_mode=pubsub`, `google_cloud_credentials_json`, `project_id`, `subscription_id`. `subscription_id` would be `chat-sub` if you follow [Pub/Sub mode](#pubsub) example. +##### Example +```bash +roslaunch google_chat_ros google_chat.launch receiving_mode:=pubsub google_cloud_credentials_json:=/path/to/-XXXXXXXX.json project_id:= subscription_id:=chat-sub +``` + +## 2. Sending the message +### 2.1 Understanding Google Chat Room +When you see Google Chat UI with browsers or smartphone's apps, you may see `space`, `thread`. If you send new message, you must specify the space or thread you want to send the message to. You can get the space name from chat room's URL. If it is `https://mail.google.com/chat/u/0/#chat/space/XXXXXXXXXXX`, `XXXXXXXXXXX` becomes the space name. + +### 2.2 Message format +There are 2 types of messages, text and card. The card basically follows [the original json structure](https://developers.google.com/chat/api/guides/message-formats/events#event_fields). As the node covers all the units in here with ros action msgs, it may be complicated for you if you want to use all of them. So in Examples sections, we'll show you simple ones. + +### 2.3 Sending the message by actionlib +All you have to do is send Actionlib goal to `~send/goal`. + +### 2.4 Examples +Showing the message examples with `rostopic pub -1` command on `bash`. +#### Sending a text message +``` bash +rostopic pub -1 /google_chat_ros/send/goal google_chat_ros/SendMessageActionGoal "goal: + text: 'Hello!' + space: 'spaces/'" +``` +![google_chat_text](https://user-images.githubusercontent.com/27789460/166410345-4ba29050-a83d-42c7-9dfe-0babc0486001.png) + + +#### Sending a message with KeyValue card +``` bash +rostopic pub -1 /google_chat_ros/send/goal google_chat_ros/SendMessageActionGoal "goal: + text: 'Something FATAL errors have happened in my computer, please fix ASAP' + cards: + - + sections: + - + widgets: + - + key_value: + top_label: 'Process ID' + content: '1234' + bottom_label: 'rospy' + icon: 'DESCRIPTION' + space: 'spaces/'" +``` +![google_chat_keyvalue](https://user-images.githubusercontent.com/27789460/166410374-f94a9da7-45fb-4915-929e-3181891d7293.png) + + + +#### Sending an Interactive button +``` bash +rostopic pub -1 /google_chat_ros/send/goal google_chat_ros/SendMessageActionGoal "goal: + cards: + - + header: + title: 'What do you want to eat?' + subtitle: 'Please choose the food shop!' + sections: + - + widgets: + - + buttons: + - + text_button_name: 'STARBUCKS' + text_button_on_click: + action: + action_method_name: 'vote_starbucks' + parameters: + - + key: 'shop' + value: 'starbucks' + - + text_button_name: 'SUBWAY' + text_button_on_click: + action: + action_method_name: 'vote_subway' + parameters: + - + key: 'shop' + value: 'subway' + + space: 'spaces/'" +``` +![google_chat_interactive_button](https://user-images.githubusercontent.com/27789460/166410386-395daab4-158c-4f47-b0c3-324b2f258ffd.png) + + +#### Sending a message with an image +See [Here](#image). + + +## 3. Receiving the messages +### 3.1 ROS Topic +When the bot was mentioned, the node publishes `~message_activity` topic. + +### 3.2 Examples + +#### Receiving a text message +```yaml +event_time: "2022-04-28T06:25:26.884623Z" +space: + name: "spaces/" + display_name: '' + room: False + dm: True +message: + name: "spaces//messages/" + sender: + name: "users/" + display_name: "Yoshiki Obinata" + avatar_url: "" + avatar: [] + email: "" + bot: False + human: True + create_time: "2022-04-28T06:25:26.884623Z" + text: "Hello!" + thread_name: "spaces//threads/" + annotations: [] + argument_text: "Hello!" + attachments: [] +user: + name: "users/" + display_name: "Yoshiki Obinata" + avatar_url: "" + avatar: [] + email: "" + bot: False + human: True +``` + +#### Receiving a message with an image or gdrive file and download it + + + +## 4. Handling the interactive event +If you've already sent the interactive card like [Interactive card example](#interactive), you can receive the activity of buttons. Suppose someone pressed the button `STARBUCKS`, the node publishes a `~card_activity` topic like + +``` yaml +event_time: "2022-05-02T00:23:47.855023Z" +space: + name: "spaces/" + display_name: "robotroom_with_thread" + room: True + dm: False +message: + name: "spaces//messages/Go__sDfIdec.Go__sDfIdec" + sender: + name: "users/100406614699672138585" + display_name: "Fetch1075" + avatar_url: "https://lh4.googleusercontent.com/proxy/hWEAWt6fmHsFAzeiEoV5FMOx5-jmU3OnzQxCtrr9unyt73NNwv0lh7InFzOh-0yO3jOPgtColHBywnZnJvl4SVqqqrYkyT1uf18k_hDIVYrAv87AY7lM0hp5KtQ1m9br-aPFE98QwNnSTYc2LQ" + avatar: [] + email: '' + bot: True + human: False + create_time: "2022-05-02T00:23:47.855023Z" + text: '' + thread_name: "spaces//threads/Go__sDfIdec" + annotations: [] + argument_text: '' + attachments: [] +user: + name: "users/103866924487978823908" + display_name: "Yoshiki Obinata" + avatar_url: "https://lh3.googleusercontent.com/a-/AOh14GgexXiq8ImuKMgOq6QG-4geIzz5IC1-xa0Caead=k" + avatar: [] + email: "" + bot: False + human: True +action: + action_method_name: "vote_starbucks" + parameters: + - + key: "shop" + value: "starbucks" +``` +After the node which handles the chat event subscribed the topic, it can respond with text message like + +``` bash +rostopic pub -1 /google_chat_ros/send/goal google_chat_ros/SendMessageActionGoal "goal: + cards: + - + sections: + - + widgets: + - + key_value: + top_label: 'The shop accepted!' + content: 'You choose STARBUCKS!!' + icon: 'DESCRIPTION' + space: 'spaces/' + thread_name: 'spaces//threads/'" +``` +![google_chat_interact](https://user-images.githubusercontent.com/27789460/166410418-c2bdc2e5-9916-4b50-a705-f838d86681aa.png) + + +The important point is that the client node has to remember the `thread_name` which the card event was occured at and send response to it. + + +## 5. Optional functions + +### 5.1 Sending a message with an image +To send an image, you have to use `card` type message. If you want to add the image uploaded to a storage server available for everyone, you just add its URI like +``` yaml +rostopic pub -1 /google_chat_ros/send/goal google_chat_ros/SendMessageActionGoal "goal: + cards: + - + sections: + - + widgets: + - + image: + image_url: 'https://media-cdn.tripadvisor.com/media/photo-s/11/fb/90/e4/dsc-7314-largejpg.jpg' + space: 'spaces/'" +``` +If you want to attach image saved at your host, you have to launch (gdrive_ros)[https://github.com/jsk-ros-pkg/jsk_3rdparty/tree/master/gdrive_ros] at first and set `~gdrive_upload_service` param with `gdrive_ros/Upload` service name. Then publish topic like +``` yaml +rostopic pub -1 /google_chat_ros/send/goal google_chat_ros/SendMessageActionGoal "goal: + cards: + - + sections: + - + widgets: + - + image: + localpath: '/home/user/Pictures/image.png' + space: 'spaces/' +``` +### 5.2 Receiving a message with images or gdrive file +You have to set rosparam `~download_data` True, `~download_directory`. If the node recieved the message with image or google drive file, it automatically downloads to `~donwload_directory` path. diff --git a/google_chat_ros/action/SendMessage.action b/google_chat_ros/action/SendMessage.action new file mode 100644 index 000000000..4b1d2bb35 --- /dev/null +++ b/google_chat_ros/action/SendMessage.action @@ -0,0 +1,14 @@ +# Define the goal +string text +google_chat_ros/Card[] cards +bool update_message # Not creating new message, but rewrite existing message +string thread_name +string space +--- +# Define the result +google_chat_ros/Message message_result +google_chat_ros/Card[] cards_result +bool done +--- +# Define a feedback message +string status diff --git a/google_chat_ros/launch/google_chat.launch b/google_chat_ros/launch/google_chat.launch new file mode 100644 index 000000000..8ef7a56ac --- /dev/null +++ b/google_chat_ros/launch/google_chat.launch @@ -0,0 +1,55 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + receiving_mode: $(arg receiving_mode) + gdrive_upload_service: $(arg gdrive_upload_service) + upload_data_timeout: $(arg upload_data_timeout) + download_data_timeout: $(arg download_data_timeout) + download_data: $(arg download_data) + download_directory: $(arg download_directory) + download_avatar: $(arg download_avatar) + google_cloud_credentials_json: $(arg google_cloud_credentials_json) + host: $(arg host) + port: $(arg port) + ssl_certfile: $(arg ssl_certfile) + ssl_keyfile: $(arg ssl_keyfile) + project_id: $(arg project_id) + subscription_id: $(arg subscription_id) + + + + + + + to_dialogflow_client: $(arg to_dialogflow_client) + debug_sound: $(arg debug_sound) + + + diff --git a/google_chat_ros/msg/ActionParameter.msg b/google_chat_ros/msg/ActionParameter.msg new file mode 100644 index 000000000..6f825d5f6 --- /dev/null +++ b/google_chat_ros/msg/ActionParameter.msg @@ -0,0 +1,2 @@ +string key +string value diff --git a/google_chat_ros/msg/Annotation.msg b/google_chat_ros/msg/Annotation.msg new file mode 100644 index 000000000..720f6a16f --- /dev/null +++ b/google_chat_ros/msg/Annotation.msg @@ -0,0 +1,8 @@ +int32 length +int32 start_index +google_chat_ros/User user +## START AnnotationType ## +bool mention +bool slash_command +### END AnnotationType ### +google_chat_ros/SlashCommand command diff --git a/google_chat_ros/msg/Attachment.msg b/google_chat_ros/msg/Attachment.msg new file mode 100644 index 000000000..8b4be8f10 --- /dev/null +++ b/google_chat_ros/msg/Attachment.msg @@ -0,0 +1,10 @@ +string name +string content_name +string content_type +string thumnail_uri +string download_uri +string localpath +bool drive_file +bool uploaded_content +string attachment_resource_name +string drive_field_id diff --git a/google_chat_ros/msg/Button.msg b/google_chat_ros/msg/Button.msg new file mode 100644 index 000000000..f83b9d200 --- /dev/null +++ b/google_chat_ros/msg/Button.msg @@ -0,0 +1,10 @@ +### FIELD DATA CAN BE ONLY ONE OF THE TEXT BUTTON OR IMAGE BUTTON ### +# A button with text and onclick action +string text_button_name +google_chat_ros/OnClick text_button_on_click +# An image button with an onclick action +string image_button_name +google_chat_ros/OnClick image_button_on_click +string icon # see https://developers.google.com/chat/api/reference/rest/v1/cards#icon +string original_icon_url # Original icon image's url +string original_icon_filepath # Original icon diff --git a/google_chat_ros/msg/Card.msg b/google_chat_ros/msg/Card.msg new file mode 100644 index 000000000..be826b8d1 --- /dev/null +++ b/google_chat_ros/msg/Card.msg @@ -0,0 +1,4 @@ +google_chat_ros/CardHeader header # The header of the card +google_chat_ros/Section[] sections # Sections are separated by a line divider +google_chat_ros/CardAction[] card_actions # The actions of this card +string name # Name of the card diff --git a/google_chat_ros/msg/CardAction.msg b/google_chat_ros/msg/CardAction.msg new file mode 100644 index 000000000..1b74d469d --- /dev/null +++ b/google_chat_ros/msg/CardAction.msg @@ -0,0 +1,2 @@ +string action_label # The label used to be displayed in the action menu item +google_chat_ros/OnClick on_click diff --git a/google_chat_ros/msg/CardEvent.msg b/google_chat_ros/msg/CardEvent.msg new file mode 100644 index 000000000..c3ff7821b --- /dev/null +++ b/google_chat_ros/msg/CardEvent.msg @@ -0,0 +1,5 @@ +string event_time +google_chat_ros/Space space +google_chat_ros/Message message # TODO:check is it required +google_chat_ros/User user +google_chat_ros/FormAction action diff --git a/google_chat_ros/msg/CardHeader.msg b/google_chat_ros/msg/CardHeader.msg new file mode 100644 index 000000000..4aa453bbb --- /dev/null +++ b/google_chat_ros/msg/CardHeader.msg @@ -0,0 +1,5 @@ +string title +string subtitle +bool image_style_circular # The image's type, makes border +string image_url +string image_filepath # The image in the card header diff --git a/google_chat_ros/msg/FormAction.msg b/google_chat_ros/msg/FormAction.msg new file mode 100644 index 000000000..01177d90e --- /dev/null +++ b/google_chat_ros/msg/FormAction.msg @@ -0,0 +1,3 @@ +# The form action data associated with an interactive card that was clicked. Only populated for CARD_CLICKED events. +string action_method_name +google_chat_ros/ActionParameter[] parameters diff --git a/google_chat_ros/msg/Image.msg b/google_chat_ros/msg/Image.msg new file mode 100644 index 000000000..62412143a --- /dev/null +++ b/google_chat_ros/msg/Image.msg @@ -0,0 +1,4 @@ +string image_url +string localpath # If you want to upload new image on google drive, please set this element, not image_uri. +google_chat_ros/OnClick on_click +float64 aspect_ratio # The aspect ratio of this image (width/height). If unset, the server fills it by prefetching the image diff --git a/google_chat_ros/msg/KeyValue.msg b/google_chat_ros/msg/KeyValue.msg new file mode 100644 index 000000000..9f1fa75d3 --- /dev/null +++ b/google_chat_ros/msg/KeyValue.msg @@ -0,0 +1,9 @@ +string top_label # The text of the top label +string content # The text of the content. Always required +bool content_multiline # If the content should be multiline +string bottom_label # The text of the bottom label +google_chat_ros/OnClick on_click +string icon # see https://developers.google.com/chat/api/reference/rest/v1/cards#icon +string original_icon_url # Original icon image's url +string original_icon_localpath # If you want to upload new image on google drive, please set this element, not original_icon_url. +google_chat_ros/Button button # A button that can be clicked to trigger an action diff --git a/google_chat_ros/msg/Message.msg b/google_chat_ros/msg/Message.msg new file mode 100644 index 000000000..37b5b9e55 --- /dev/null +++ b/google_chat_ros/msg/Message.msg @@ -0,0 +1,8 @@ +string name # Resource name in form spaces/*/messages/* +google_chat_ros/User sender # The user who created the message +string create_time # The time at which the message was created in Google Chat server +string text # Plain-text body of the message +string thread_name # The thread the message belongs to +google_chat_ros/Annotation[] annotations # Annotations associated with the plain-text body of the message +string argument_text # Plain-text body of the message with all bot mentions stripped out +google_chat_ros/Attachment[] attachments # User uploaded attachment diff --git a/google_chat_ros/msg/MessageEvent.msg b/google_chat_ros/msg/MessageEvent.msg new file mode 100644 index 000000000..b50f7454f --- /dev/null +++ b/google_chat_ros/msg/MessageEvent.msg @@ -0,0 +1,5 @@ +# ROS message for receiving Google Chat message +string event_time +google_chat_ros/Space space +google_chat_ros/Message message +google_chat_ros/User user diff --git a/google_chat_ros/msg/OnClick.msg b/google_chat_ros/msg/OnClick.msg new file mode 100644 index 000000000..a13e01268 --- /dev/null +++ b/google_chat_ros/msg/OnClick.msg @@ -0,0 +1,3 @@ +### FIELD DATA CAN BE ONLY ONE OF THE FOLLOWING ### +google_chat_ros/FormAction action # A form action will be triggered by this onclick if specified +string open_link_url # A link that opens a new window diff --git a/google_chat_ros/msg/Section.msg b/google_chat_ros/msg/Section.msg new file mode 100644 index 000000000..102d26350 --- /dev/null +++ b/google_chat_ros/msg/Section.msg @@ -0,0 +1,2 @@ +string header # The header of the section +google_chat_ros/WidgetMarkup[] widgets diff --git a/google_chat_ros/msg/SlashCommand.msg b/google_chat_ros/msg/SlashCommand.msg new file mode 100644 index 000000000..cd1dad57f --- /dev/null +++ b/google_chat_ros/msg/SlashCommand.msg @@ -0,0 +1,6 @@ +google_chat_ros/User user +bool added +bool invoke +string command_name +string command_id +bool triggers_dialog diff --git a/google_chat_ros/msg/Space.msg b/google_chat_ros/msg/Space.msg new file mode 100644 index 000000000..692b40cd2 --- /dev/null +++ b/google_chat_ros/msg/Space.msg @@ -0,0 +1,4 @@ +string name +string display_name +bool room # is chat room +bool dm # is direct message diff --git a/google_chat_ros/msg/SpaceEvent.msg b/google_chat_ros/msg/SpaceEvent.msg new file mode 100644 index 000000000..079edb4bb --- /dev/null +++ b/google_chat_ros/msg/SpaceEvent.msg @@ -0,0 +1,5 @@ +bool added +bool removed +string event_time +google_chat_ros/Space space +google_chat_ros/User user diff --git a/google_chat_ros/msg/User.msg b/google_chat_ros/msg/User.msg new file mode 100644 index 000000000..268c4c5e4 --- /dev/null +++ b/google_chat_ros/msg/User.msg @@ -0,0 +1,7 @@ +string name +string display_name +string avatar_url +uint8[] avatar +string email +bool bot +bool human diff --git a/google_chat_ros/msg/WidgetMarkup.msg b/google_chat_ros/msg/WidgetMarkup.msg new file mode 100644 index 000000000..4c8e48506 --- /dev/null +++ b/google_chat_ros/msg/WidgetMarkup.msg @@ -0,0 +1,4 @@ +google_chat_ros/Button[] buttons +string text_paragraph # Display a text paragraph in this widget +google_chat_ros/Image image # Display an image in this widget +google_chat_ros/KeyValue key_value # Display a key value item in this widget diff --git a/google_chat_ros/package.xml b/google_chat_ros/package.xml new file mode 100644 index 000000000..a043f4155 --- /dev/null +++ b/google_chat_ros/package.xml @@ -0,0 +1,27 @@ + + + google_chat_ros + 2.1.25 + Use Google Chat API clients via ROS + + Yoshiki Obinata + Kei Okada + + BSD + + catkin + python3-setuptools + + message_generation + catkin_virtualenv + + message_runtime + rospy + std_msgs + gdrive_ros + dialogflow_task_executive + + + requirements.txt + + diff --git a/google_chat_ros/requirements.txt b/google_chat_ros/requirements.txt new file mode 100644 index 000000000..e11fffb78 --- /dev/null +++ b/google_chat_ros/requirements.txt @@ -0,0 +1,36 @@ +beautifulsoup4==4.11.1 +cachetools==4.2.4 +certifi==2022.5.18.1 +charset-normalizer==2.0.12 +filelock==3.4.1 +gdown==4.4.0 +google-api-core[grpc]==2.8.2 +google-api-python-client==2.51.0 +google-auth-httplib2==0.1.0 +google-auth-oauthlib==0.5.2 +google-auth==2.8.0 +google-cloud-pubsub==2.12.1 +googleapis-common-protos[grpc]==1.56.2 +grpc-google-iam-v1==0.12.4 +grpcio-status==1.46.3 +grpcio==1.46.3 +httplib2==0.20.4 +idna==3.3 +importlib-resources==5.4.0 +oauth2client==4.1.3 +oauthlib==3.2.0 +proto-plus==1.20.6 +protobuf==3.19.4 +pyasn1-modules==0.2.8 +pyasn1==0.4.8 +pyparsing==3.0.9 +pysocks==1.7.1 +requests-oauthlib==1.3.1 +requests[socks]==2.27.1 +rsa==4.8 +six==1.16.0 +soupsieve==2.3.2.post1 +tqdm==4.64.0 +uritemplate==4.1.1 +urllib3==1.26.9 +zipp==3.6.0 diff --git a/google_chat_ros/scripts/google-chat.l b/google_chat_ros/scripts/google-chat.l new file mode 100755 index 000000000..f146737e5 --- /dev/null +++ b/google_chat_ros/scripts/google-chat.l @@ -0,0 +1,98 @@ +#!/usr/bin/env roseus + +(ros::load-ros-manifest "google_chat_ros") +(ros::roseus "google_chat_eus_client") + +(defun send-google-chat-text (space content + &key (thread-name nil) (action-goal-name "google_chat_ros/send") (wait t)) + (wait (boundp 'google_chat_ros::SendMessageAction) + (let ((goal (instance google_chat_ros::SendMessageActionGoal :init)) + (ac (instance ros::simple-action-client :init + action-goal-name google_chat_ros::SendMessageAction))) + (when (send ac :wait-for-server 1) + (when (eq (send ac :get-state) actionlib_msgs::GoalStatus::*active*) + (send ac :cancel-goal) + (send ac :wait-for-result :timeout 5)) + (send goal :goal :space space) + (when thread-name + (send goal :goal :thread_name thread-name)) + (send goal :goal :text content) + (send ac :send-goal goal) + (if wait + (return-from send-google-chat-text (send ac :wait-for-result :timeout 5)) + (return-from send-google-chat-text t)))))) + +(defun send-google-chat-image + (space image-path + &key (image-header "") (thread-name nil) + (action-goal-name "google_chat_ros/send") (wait t)) + (when (boundp 'google_chat_ros::SendMessageAction) + (let ((goal (instance google_chat_ros::SendMessageActionGoal :init)) + (ac (instance ros::simple-action-client :init + action-goal-name google_chat_ros::SendMessageAction)) + (card (instance google_chat_ros::Card :init)) + (section (instance google_chat_ros::Section :init)) + (widget (instance google_chat_ros::WidgetMarkup :init)) + (image (instance google_chat_ros::Image :init))) + (when (send ac :wait-for-server 1) + (when (eq (send ac :get-state) actionlib_msgs::GoalStatus::*active*) + (send ac :cancel-goal) + (send ac :wait-for-result :timeout 5)) + (send image :localpath image-path) + (send widget :image image) + (send section :widgets (list widget)) + (send section :header image-header) + (send card :sections (list section)) + (send goal :goal :cards (list card)) + (send goal :goal :space space) + (when thread-name + (send goal :goal :thread_name thread-name)) + (send ac :send-goal goal) + (if wait + (return-from send-google-chat-image (send ac :wait-for-result :timeout 5)) + (return-from send-google-chat-image t)))))) + +(defun create-google-chat-button + (button-name button-action-name + &key (button-action-key) (button-action-value)) + (let ((button (instance google_chat_ros::Button :init)) + (text-button-on-click (instance google_chat_ros::OnClick :init)) + (action (instance google_chat_ros::FormAction :init)) + (parameter (instance google_chat_ros::ActionParameter :init))) + (send button :text_button_name button-name) + (send action :action_method_name button-action-name) + (send parameter :key button-action-key) + (send parameter :value button-action-value) + (send action :parameters (list parameter)) + (send text-button-on-click :action action) + (send button :text_button_on_click text-button-on-click) + button)) + +(defun send-google-chat-buttons + ;; buttons should be list + (space buttons + &key (buttons-header "") (thread-name nil) + (action-goal-name "google_chat_ros/send") (wait nil)) + (when (boundp 'google_chat_ros::SendMessageAction) + (let ((goal (instance google_chat_ros::SendMessageActionGoal :init)) + (ac (instance ros::simple-action-client :init + action-goal-name google_chat_ros::SendMessageAction)) + (card (instance google_chat_ros::Card :init)) + (section (instance google_chat_ros::Section :init)) + (widget (instance google_chat_ros::WidgetMarkup :init))) + (when (send ac :wait-for-server 1) + (when (eq (send ac :get-state) actionlib_msgs::GoalStatus::*active*) + (send ac :cancel-goal) + (send ac :wait-for-result :timeout 5)) + (send widget :buttons buttons) + (send section :widgets (list widget)) + (send section :header buttons-header) + (send card :sections (list section)) + (send goal :goal :cards (list card)) + (send goal :goal :space space) + (when thread-name + (send goal :goal :thread_name thread-name)) + (send ac :send-goal goal) + (if wait + (return-from send-google-chat-buttons (send ac :wait-for-result :timeout 5)) + (return-from send-google-chat-buttons t)))))) diff --git a/google_chat_ros/scripts/google_chat_ros_node.py b/google_chat_ros/scripts/google_chat_ros_node.py new file mode 100644 index 000000000..8f98eafd6 --- /dev/null +++ b/google_chat_ros/scripts/google_chat_ros_node.py @@ -0,0 +1,474 @@ +#!/usr/bin/env python3 +import gdown +import json +import os +import requests +from requests.exceptions import Timeout +from requests.exceptions import ConnectionError + +# ROS libraries +import actionlib +import ast +from dialogflow_task_executive.msg import DialogResponse +from gdrive_ros.srv import * +from google_chat_ros.google_chat import GoogleChatRESTClient +from google_chat_ros.google_chat import GoogleChatHTTPSServer +from google_chat_ros.google_chat import GoogleChatPubSubClient +from google_chat_ros.msg import * +import rospy + + +class GoogleChatROS(object): + """ + Send request to Google Chat REST API via ROS + """ + def __init__(self): + receiving_chat_mode = rospy.get_param('~receiving_mode') # select from 'url', 'pubsub', 'none' + self.gdrive_ros_srv = rospy.get_param('~gdrive_upload_service') + google_credentials = rospy.get_param('~google_cloud_credentials_json') + + # For sending message + self._client = GoogleChatRESTClient(google_credentials) + rospy.loginfo("Starting Google Chat REST service...") + try: + self._client.build_service() # Start google chat authentication and service + rospy.loginfo("Succeeded in starting Google Chat REST service") + # ROS ActionLib + self._as = actionlib.SimpleActionServer( + '~send', SendMessageAction, + execute_cb=self.rest_cb, auto_start=False + ) + self._as.start() + except Exception as e: + rospy.logwarn("Failed to start Google Chat REST service") + rospy.logerr(e) + + # For receiving message + if receiving_chat_mode in ("url", "dialogflow", "pubsub"): + # rosparams + self.upload_data_timeout = rospy.get_param('~upload_data_timeout') + self.download_data = rospy.get_param('~download_data') + self.download_directory = rospy.get_param('~download_directory') + self.download_avatar = rospy.get_param('~download_avatar') + # ROS publisher + self._message_activity_pub = rospy.Publisher("~message_activity", MessageEvent, queue_size=1) + self._space_activity_pub = rospy.Publisher("~space_activity", SpaceEvent, queue_size=1) + self._card_activity_pub = rospy.Publisher("~card_activity", CardEvent, queue_size=1) + + if receiving_chat_mode == "url": + rospy.loginfo("Expected to get Google Chat Bot URL request") + rospy.loginfo("Starting Google Chat HTTPS server...") + self.host = rospy.get_param('~host') + self.port = int(rospy.get_param('~port')) + self.ssl_certfile = rospy.get_param('~ssl_certfile') + self.ssl_keyfile = rospy.get_param('~ssl_keyfile') + try: + self._server = GoogleChatHTTPSServer( + self.host, self.port, self.ssl_certfile, self.ssl_keyfile, callback=self.event_cb, user_agent='Google-Dynamite') + rospy.on_shutdown(self.killhttpd) # shutdown https server TODO is this okay in try ? + self._server.run() + except ConnectionError as e: + rospy.logwarn("The error occurred while starting HTTPS server") + rospy.logerr(e) + elif receiving_chat_mode == "pubsub": + rospy.loginfo("Expected to use Google Cloud Pub Sub service") + self.project_id = rospy.get_param("~project_id") + self.subscription_id = rospy.get_param("~subscription_id") + self._pubsub_client = GoogleChatPubSubClient( + self.project_id, self.subscription_id, self.event_cb, google_credentials) + rospy.on_shutdown(self.killpubsub) + self._pubsub_client.run() + + elif receiving_chat_mode == "none": + rospy.logwarn("You cannot recieve Google Chat event because HTTPS server or Google Cloud Pub/Sub is not running.") + + else: + rospy.logerr("Please choose receiving_mode param from dialogflow, https, pubsub, none.") + + def killhttpd(self): + self._server.kill() + + def killpubsub(self): + self._pubsub_client.kill() + + def rest_cb(self, goal): + """Get ROS SendMessageAction Goal and send request to Google Chat API. + :param goal: ROS SendMessageAction Goal + :rtype: none + """ + feedback = SendMessageFeedback() + result = SendMessageResult() + r = rospy.Rate(1) + success = True + + json_body = {} + json_body['text'] = goal.text + json_body['thread'] = {'name': goal.thread_name} + + # Card + json_body['cards'] = [] + if goal.cards: + if goal.update_message: + json_body['actionResponse'] = {"type": "UPDATE_MESSAGE"} + for card in goal.cards: + card_body = {} + # card/header + if card.header: + header = {} + header['title'] = card.header.title + header['subtitle'] = card.header.subtitle + header['imageStyle'] = 'AVATAR' if card.header.image_style_circular else 'IMAGE' + card_body['header'] = header + if card.header.image_url: + header['imageUrl'] = card.header.image_url + elif card.header.image_filepath: + header['imageUrl'] = self._upload_file(card.header.image_filepath) + # card/sections + sections = [] + sections = self._make_sections_json(card.sections) + # card/actions + card_actions = [] + for card_action_msg in card.card_actions: + card_action = {} + card_action['actionLabel'] = card_action_msg.action_label + card_action['onClick'] = self._make_on_click_json(card_action_msg.on_click) + card_actions.append(card_action) + card_body['sections'] = sections + if card_actions: + card_body['cardActions'] = card_actions + card_body['name'] = card.name + json_body['cards'].append(card_body) + + try: + rospy.logdebug("Send json") + rospy.logdebug(str(json_body)) + self._client.build_service() + res = self._client.message_create( + space=goal.space, + json_body=json_body + ) + result.message_result = self._make_message_msg({'message': res}) + # TODO add the result of what card was sent + # result.cards_result = + except Exception as e: + rospy.logerr(str(e)) + feedback.status = str(e) + success = False + finally: + self._as.publish_feedback(feedback) + r.sleep() + result.done = success + self._as.set_succeeded(result) + + def event_cb(self, event: dict, publish_topic=True): + """Parse Google Chat API json content and publish as a ROS Message. + See https://developers.google.com/chat/api/reference/rest + to check what contents are included in the json. + :param event: A google Chat API POST request json content. + See https://developers.google.com/chat/api/guides/message-formats/events#event_fields for details. + :rtype: ros message + """ + rospy.logdebug("GOOGLE CHAT ORIGINAL JSON EVENT") + rospy.logdebug(json.dumps(event, indent=2)) + # GET EVENT TYPE + # event/eventTime + event_time = event.get('eventTime', '') + # event/space + space = Space() + space.name = event.get('space', {}).get('name', '') + space.room = True if event.get('space', {}).get('type', '') == "ROOM" else False + if space.room: + space.display_name = event.get('space', {}).get('displayName', '') + space.dm = True if event.get('space', {}).get('type', '') == "DM" else False + # event/user + user = self._get_user_info(event.get('user', {})) + + if event['type'] == 'ADDED_TO_SPACE' or event['type'] == 'REMOVED_FROM_SPACE': + msg = SpaceEvent() + msg.event_time = event_time + msg.space = space + msg.user = user + msg.added = True if event['type'] == "ADDED_TO_SPACE" else False + msg.removed = True if event['type'] == "REMOVED_FROM_SPACE" else False + if publish_topic: + self._space_activity_pub.publish(msg) + return msg + + elif event['type'] == 'MESSAGE': + msg = MessageEvent() + msg.event_time = event_time + msg.space = space + msg.user = user + msg.message = self._make_message_msg(event) + if publish_topic: + self._message_activity_pub.publish(msg) + return msg + + elif event['type'] == 'CARD_CLICKED': + msg = CardEvent() + msg.event_time = event_time + msg.space = space + msg.user = user + msg.message = self._make_message_msg(event) + if event.get('action'): + action = event.get('action') + msg.action.action_method_name = action.get('actionMethodName') + if action.get('parameters'): + parameters = [] + for param in action.get('parameters'): + action_parameter = ActionParameter() + action_parameter.key = param.get('key') + action_parameter.value = param.get('value') + parameters.append(action_parameter) + msg.action.parameters = parameters + if publish_topic: + self._card_activity_pub.publish(msg) + return msg + + else: + rospy.logerr("Got unknown event type.") + return + + def dialogflow_cb(self, msg): + if msg.source == "hangouts": + ast_data = ast.literal_eval(msg.payload) + json_dumped = json.dumps(ast_data) + json_data = json.loads(json_dumped) + self.event_cb(json_data.get('data', {}).get('event', {})) + + def _make_message_msg(self, event): + message = Message() + message_content = event.get('message', '') + message.name = message_content.get('name', '') + + # event/message/sender + message.sender = self._get_user_info(message_content.get('sender')) + message.create_time = message_content.get('createTime', '') + message.text = message_content.get('text', '') + message.thread_name = message_content.get('thread', {}).get('name', '') + + # event/messsage/annotations + if 'annotations' in message_content: + for item in message_content['annotations']: + annotation = Annotation() + annotation.length = int(item.get('length', 0)) + annotation.start_index = int(item.get('startIndex', 0)) + annotation.mention = True if item.get('type') == 'USER_MENTION' else False + if annotation.mention: + annotation.user = self._get_user_info(item.get('userMention').get('user')) + annotation.slash_command = True if item.get('type') == 'SLASH_COMMAND' else False + message.annotations.append(annotation) + message.argument_text = message_content.get('argumentText', '') + + # event/message/attachment + if 'attachment' in message_content: + for item in message_content['attachment']: + message.attachments.append(self._get_attachment(item)) + + return message + + def _make_sections_json(self, sections_msg): + """ + :type msg: list of google_chat_ros.msgs/Section + :rtype json_body: list of json + """ + json_body = [] + for msg in sections_msg: + section = {} + if msg.header: + section['header'] = msg.header + section['widgets'] = self._make_widget_markups_json(msg.widgets) + json_body.append(section) + return json_body + + def _make_widget_markups_json(self, widgets_msg): + """Make widget markup json lists. + See https://developers.google.com/chat/api/reference/rest/v1/cards#widgetmarkup for details. + :rtype widgets_msg: list of google_chat_ros.msgs/WidgetMarkup + :rtype json_body: list of json + """ + json_body = [] + for msg in widgets_msg: + is_text = bool(msg.text_paragraph) + is_image = bool(msg.image.image_url) or bool(msg.image.localpath) + is_keyval = bool(msg.key_value.content) + # make buttons + buttons = [] + buttons_msg = msg.buttons + for button_msg in buttons_msg: + buttons.append(self._make_button_json(button_msg)) + if buttons: + json_body.append({'buttons':buttons}) + + if (is_text & is_image) | (is_image & is_keyval) | (is_keyval & is_text): + rospy.logerr("Error happened when making widgetMarkup json. Please fill in one of the text_paragraph, image, key_value. Do not fill in more than two at the same time.") + elif is_text: + json_body.append({'textParagraph':{'text':msg.text_paragraph}}) + elif is_image: + image_json = {} + if msg.image.image_url: + image_json['imageUrl'] = msg.image.image_url + elif msg.image.localpath: + image_json['imageUrl'] = self._upload_file(msg.image.localpath) + if msg.image.on_click.action.action_method_name or msg.image.on_click.open_link_url: + image_json['onClick'] = self._make_on_click_json(msg.image.on_click) + if msg.image.aspect_ratio: + image_json['aspectRatio'] = msg.image.aspect_ratio + json_body.append({'image':image_json}) + elif is_keyval: + keyval_json = {} + keyval_json['topLabel'] = msg.key_value.top_label + keyval_json['content'] = msg.key_value.content + keyval_json['contentMultiline'] = msg.key_value.content_multiline + keyval_json['bottomLabel'] = msg.key_value.bottom_label + if msg.key_value.on_click.action.action_method_name or msg.key_value.on_click.open_link_url: + keyval_json['onClick'] = self._make_on_click_json(msg.key_value.on_click) + if msg.key_value.icon: + keyval_json['icon'] = msg.key_value.icon + elif msg.key_value.original_icon_url: + keyval_json['iconUrl'] = msg.key_value.original_icon_url + elif msg.key_value.original_icon_localpath: + keyval_json['iconUrl'] = self._upload_file(msg.key_value.original_icon_localpath) + if msg.key_value.button.text_button_name or msg.key_value.button.image_button_name: + keyval_json['button'] = self._make_button_json(msg.key_value.button) + json_body.append({'keyValue':keyval_json}) + return json_body + + def _make_on_click_json(self, on_click_msg): + """Make onClick json. + See https://developers.google.com/chat/api/reference/rest/v1/cards#onclick for details. + :rtype on_click_msg: google_chat_ros.msg/OnClick.msg + :rtype json_body: json + """ + json_body = {} + if on_click_msg.action.action_method_name and on_click_msg.open_link_url: + rospy.logerr("Error happened when making onClick json. Please fill in one of the action, open_link_url. Do not fill in more than two at the same time.") + elif on_click_msg.action.action_method_name: + action = {} + action['actionMethodName'] = on_click_msg.action.action_method_name + parameters = [] + for parameter in on_click_msg.action.parameters: + parameters.append({'key':parameter.key, 'value':parameter.value}) + action['parameters'] = parameters + json_body['action'] = action + elif on_click_msg.open_link_url: + json_body['openLink'] = {'url': on_click_msg.open_link_url} + return json_body + + def _make_button_json(self, button_msg): + """Make button json. + See https://developers.google.com/chat/api/reference/rest/v1/cards#button for details. + :rtype button_msg: google_chat_ros.msg/Button.msg + :rtype json_body: json + """ + json_body = {} + if button_msg.text_button_name and button_msg.image_button_name: + rospy.logerr("Error happened when making Button json. Please fill in one of the text_button_name or image_button_name. Do not fill in more than two at the same time.") + elif button_msg.text_button_name: + rospy.loginfo("Build text button:{}".format(button_msg.text_button_name)) + text_button = {} + text_button['text'] = button_msg.text_button_name + text_button['onClick'] = self._make_on_click_json(button_msg.text_button_on_click) + json_body['textButton'] = text_button + elif button_msg.image_button_name: + rospy.loginfo("Build image button:{}".format(button_msg.image_button_name)) + image_button = {} + image_button['onClick'] = self._make_on_click_json(button_msg.image_button_on_click) + if button_msg.icon: + image_icon['icon'] = button_msg.icon + elif button_msg.original_icon_url: + image_icon['iconUrl'] = button_msg.original_icon_url + elif button_msg.original_icon_filepath: + image_icon['iconUrl'] = self._upload_file(button_msg.original_icon_filepath) + return json_body + + def _get_user_info(self, item): + user = User() + user.name = item.get('name', '') + user.display_name = item.get('displayName', '') + user.avatar_url = item.get('avatarUrl', '') + if self.download_avatar: + user.avatar = self._get_image_from_uri(user.avatar_url) + user.email = item.get('email', '') + user.bot = True if item.get('type') == "BOT" else False + user.human = True if item.get('type') == "HUMAN" else False + return user + + def _upload_file(self, filepath, return_id=False): + """Get local filepath and upload to Google Drive + :param filepath: local file's path you want to upload + :type filepath: string + :returns: URL file exists + :rtype: string + """ + # ROS service client + try: + rospy.wait_for_service(self.gdrive_ros_srv, timeout=self.upload_data_timeout) + gdrive_upload = rospy.ServiceProxy(self.gdrive_ros_srv, Upload) + except rospy.ROSException as e: + rospy.logerr("No Google Drive ROS upload service was found. Please check gdrive_ros is correctly launched and service name is correct.") + rospy.logerr(e) + return + # upload + try: + res = gdrive_upload(file_path=filepath) + except rospy.ServiceException as e: + rospy.logerr("Failed to call Google Drive upload service, status:{}".format(str(e))) + else: + if return_id: + drive_id = res.file_id + rospy.loginfo("Google drive ID:{}".format(drive_id)) + return drive_id + else: + url = res.file_url + rospy.loginfo("Google drive URL:{}".format(url)) + return url + + def _get_attachment(self, item): + attachment = Attachment() + attachment.name = item.get('name', '') + attachment.content_name = item.get('contentName', '') + attachment.content_type = item.get('contentType', '') + attachment.thumnail_uri = item.get('thumnailUri', '') + attachment.download_uri = item.get('downloadUri', '') + attachment.drive_file = True if item.get('source') == 'DRIVE_FILE' else False + attachment.uploaded_content = True if item.get('source') == 'UPLOADED_CONTENT' else False + attachment.attachment_resource_name = item.get('attachmentDataRef', {}).get('resourceName', '') + attachment.drive_field_id = item.get('driveDataRef', {}).get('driveFileId', '') + if self.download_data and attachment.download_uri: + self._download_content(uri=attachment.download_uri, filename=attachment.content_name) + if self.download_data and attachment.drive_field_id: + self._download_content(drive_id=attachment.drive_field_id) + return attachment + + def _get_image_from_uri(self, uri): + try: + img = requests.get(uri, timeout=10).content + except Timeout: + rospy.logerr("Exceeded timeout when downloading {}.".format(uri)) + except Exception as e: + rospy.logwarn("Failed to get image from {}".format(uri)) + rospy.logerr(e) + else: + return img + + def _download_content(self, uri=None, drive_id=None, filename=''): + try: + if drive_id: + path = os.path.join(self.download_directory, drive_id) + gdown.download(id=drive_id, output=path) + elif uri: + path = os.path.join(self.download_directory, filename) + gdown.download(url=uri, output=path) + except Exception as e: + rospy.logwarn("Failed to download the attatched file.") + rospy.logerr(e) + else: + rospy.loginfo("Suceeded in downloading the attached file. Saved at {}".format(path)) + finally: + return path + +if __name__ == '__main__': + rospy.init_node('google_chat') + node = GoogleChatROS() + rospy.spin() diff --git a/google_chat_ros/scripts/helper.py b/google_chat_ros/scripts/helper.py new file mode 100644 index 000000000..4b5a33247 --- /dev/null +++ b/google_chat_ros/scripts/helper.py @@ -0,0 +1,80 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +import queue + +import rospy +import actionlib + +from std_msgs.msg import String +from google_chat_ros.msg import * +from dialogflow_task_executive.msg import * +from sound_play.msg import * + +class GoogleChatROSHelper(object): + """ + Helper node for google chat ROS + """ + def __init__(self): + # Get configuration params + self.to_dialogflow_task_executive = rospy.get_param("~to_dialogflow_client") + self.sound_play_jp = rospy.get_param("~debug_sound") + self._message_sub = rospy.Subscriber("google_chat_ros/message_activity", MessageEvent, callback=self._message_cb) + self.recent_message_event = None + + # GOOGLE CHAT + def send_chat_client(self, goal): + client = actionlib.SimpleActionClient('google_chat_ros/send', SendMessageAction) + client.wait_for_server() + client.send_goal(goal) + client.wait_for_result() + return client.get_result() + + def dialogflow_action_client(self, query): + """ + :rtype: DialogTextActionResult + """ + client = actionlib.SimpleActionClient('dialogflow_client/text_action', DialogTextAction) + client.wait_for_server() + goal = DialogTextActionGoal() + goal.goal.query = query + client.send_goal(goal.goal) + client.wait_for_result() + return client.get_result() + + # SOUND + def sound_client(self, goal): + client = actionlib.SimpleActionClient('robotsound_jp', SoundRequestAction) + client.wait_for_server() + client.send_goal(goal) + client.wait_for_result() + return client.get_result() + + def _message_cb(self, data): + """ + Callback function for subscribing MessageEvent.msg + """ + sender_id = data.message.sender.name + sender_name = data.message.sender.display_name + space = data.space.name + thread_name = data.message.thread_name + text = data.message.argument_text + if self.to_dialogflow_task_executive: + chat_goal = SendMessageGoal() + chat_goal.space = space + chat_goal.thread_name = thread_name + dialogflow_res = self.dialogflow_action_client(text) + content = "<{}> {}".format(sender_id, dialogflow_res.response.response) + chat_goal.text = content + self.send_chat_client(chat_goal) + if self.sound_play_jp: + sound_goal = SoundRequestGoal() + sound_goal.sound_request.sound = sound_goal.sound_request.SAY + sound_goal.sound_request.command = sound_goal.sound_request.PLAY_ONCE + sound_goal.sound_request.volume = 1.0 + sound_goal.sound_request.arg = "{}さんから，{}というメッセージを受信しました".format(sender_name, text) + self.sound_client(sound_goal) + +if __name__ == '__main__': + rospy.init_node('google_chat_helper') + node = GoogleChatROSHelper() + rospy.spin() diff --git a/google_chat_ros/setup.py b/google_chat_ros/setup.py new file mode 100644 index 000000000..7953ab089 --- /dev/null +++ b/google_chat_ros/setup.py @@ -0,0 +1,9 @@ +from catkin_pkg.python_setup import generate_distutils_setup +from setuptools import setup + +d = generate_distutils_setup( + packages=['google_chat_ros'], + package_dir={'': 'src'} +) + +setup(**d) diff --git a/google_chat_ros/src/google_chat_ros/__init__.py b/google_chat_ros/src/google_chat_ros/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/google_chat_ros/src/google_chat_ros/google_chat.py b/google_chat_ros/src/google_chat_ros/google_chat.py new file mode 100644 index 000000000..f704bf67d --- /dev/null +++ b/google_chat_ros/src/google_chat_ros/google_chat.py @@ -0,0 +1,140 @@ +from apiclient.discovery import build +import base64 +from concurrent.futures import TimeoutError +from google.cloud import pubsub_v1 +from google.oauth2.service_account import Credentials +from httplib2 import Http +import http.server as s +import json +from oauth2client.service_account import ServiceAccountCredentials +import rospy # for logging +import socket +import ssl + +class GoogleChatRESTClient(): + def __init__(self, keyfile): + self._auth_scopes = "https://www.googleapis.com/auth/chat.bot" + self.keyfile = keyfile + self.__credentials = None + self._chat = None + + def build_service(self): + """Authenticate Googel REST API and start service by json key file. + Please see https://developers.google.com/chat/how-tos/service-accounts#step_1_create_service_account_and_private_key for details. + :param keyfile_path: str, the file path of json key file + """ + self.__credentials = ServiceAccountCredentials.from_json_keyfile_name(self.keyfile, self._auth_scopes) + self._chat = build('chat', 'v1', http=self.__credentials.authorize(Http())) + + def message_create(self, space, json_body): + if not space.startswith('spaces/'): + raise RuntimeError("Space name must begin with spaces/") + # returns same 403 error both authenticate error and not connected error + return self._chat.spaces().messages().create(parent=space, body=json_body).execute() + + def list_members(self, space): + """Show member list in the space. + """ + if not space.startswith('spaces/'): + raise RuntimeError("Space name must begin with spaces/") + return self._chat.spaces().members().list(parent=space).execute() + +class GoogleChatHTTPSServer(): + """The server for getting https request from Google Chat + """ + def __init__(self, host, port, certfile, keyfile, callback, user_agent): + """ + :param host: str, hostname + :param port: int, port number + :param certfile: str, ssl certfile path + :param keyfile: str, ssl keyfile path + """ + self._host = host + self._port = port + self._certfile = certfile + self._keyfile = keyfile + self._callback = callback + self.user_agent = user_agent + self.__RUN = True + + def __handler(self, *args): + try: + GoogleChatHTTPSHandler(self._callback, self.user_agent, *args) + except socket.error as e: + if e.errno == 104: # ignore SSL Connection reset error + rospy.logdebug(e) + else: + raise e + + def run(self): + self._httpd = s.HTTPServer((self._host, self._port), self.__handler) + self._httpd.socket = ssl.wrap_socket(self._httpd.socket, certfile=self._certfile, keyfile=self._keyfile) + while self.__RUN: + self._httpd.handle_request() + + def kill(self): + self.__RUN = False + self._httpd.server_close() + +class GoogleChatHTTPSHandler(s.BaseHTTPRequestHandler): + """The handler for https request from Google chat API. Mainly used for receiving messages, events. + """ + def __init__(self, callback, user_agent, *args): + self._callback = callback + self.user_agent = user_agent + s.BaseHTTPRequestHandler.__init__(self, *args) + + def do_POST(self): + """Handles an event from Google Chat. + Please see https://developers.google.com/chat/api/guides/message-formats/events for details. + """ + user_agent = self.headers.get("User-Agent") + print('user_agent ' + str(user_agent)) + if user_agent == self.user_agent: + self._parse_json() + self._callback(self.json_content) + self._response() + + def _parse_json(self): + content_len = int(self.headers.get("content-length")) + request_body = self.rfile.read(content_len).decode('utf-8') + self.json_content = json.loads(request_body) # json.loads returns unicode by default + + def _response(self): + self.send_response(200) + self.end_headers() + + def _bad_request(self): + self.send_response(400) + self.end_headers() + +class GoogleChatPubSubClient(): + def __init__(self, project_id, subscription_id, callback, keyfile): + self._callback = callback + self.__credentials = Credentials.from_service_account_file(keyfile) + self._sub = pubsub_v1.SubscriberClient(credentials=self.__credentials) + sub_path = self._sub.subscription_path(project_id, subscription_id) + self._streaming_pull_future = self._sub.subscribe(sub_path, callback=self._pubsub_cb) + + def _pubsub_cb(self, message): + rospy.logdebug("Recieved {message}") + rospy.logdebug(message.data) + try: + json_content = json.loads(message.data) + self._callback(json_content) + except Exception as e: + rospy.logerr("Failed to handle the request from Cloud PubSub.") + rospy.logerr("It might be caused because of invalid type message from GCP") + rospy.logerr("{}".str(e)) + finally: + message.ack() + + def run(self): + with self._sub: + try: + self._streaming_pull_future.result() + except KeyboardInterrupt: + self._streaming_pull_future.cancel() + + def kill(self): + self._streaming_pull_future.cancel() diff --git a/google_chat_ros/test/import.test b/google_chat_ros/test/import.test new file mode 100644 index 000000000..4715913fe --- /dev/null +++ b/google_chat_ros/test/import.test @@ -0,0 +1,6 @@ + + + diff --git a/google_chat_ros/test/test_import.py b/google_chat_ros/test/test_import.py new file mode 100644 index 000000000..4f718ee68 --- /dev/null +++ b/google_chat_ros/test/test_import.py @@ -0,0 +1,30 @@ +import unittest + +PKG = 'google_chat_ros' +NAME = 'test_import' + +class TestBlock(unittest.TestCase): + def __init__(self, *args): + super(TestBlock, self).__init__(*args) + + def test_import(self): + try: + from apiclient.discovery import build + import base64 + from concurrent.futures import TimeoutError + import gdown + from google.cloud import pubsub_v1 + from google.oauth2.service_account import Credentials + from httplib2 import Http + import http.server as s + import json + from oauth2client.service_account import ServiceAccountCredentials + import requests + import socket + import ssl + except Exception as e: + assert False + +if __name__ == "__main__": + import rostest + rostest.rosrun(PKG, NAME, TestBlock) diff --git a/respeaker_ros/CMakeLists.txt b/respeaker_ros/CMakeLists.txt index 59c6034b0..f6b63469b 100644 --- a/respeaker_ros/CMakeLists.txt +++ b/respeaker_ros/CMakeLists.txt @@ -18,5 +18,7 @@ catkin_install_python(PROGRAMS ${PYTHON_SCRIPTS} if(CATKIN_ENABLE_TESTING) find_package(rostest REQUIRED) + find_package(roslaunch REQUIRED) add_rostest(test/sample_respeaker.test) + roslaunch_add_file_check(launch/sample_respeaker.launch) endif() diff --git a/respeaker_ros/README.md b/respeaker_ros/README.md index e42ba1202..247168ba5 100644 --- a/respeaker_ros/README.md +++ b/respeaker_ros/README.md @@ -92,6 +92,151 @@ A ROS Package for Respeaker Mic Array a: 0.3" ``` +## Parameters for respeaker_node.py + + - ### Publishing topics + + - `audio` (`audio_common_msgs/AudioData`) + + Processed audio for ASR. 1 channel. + + - `audio_info` (`audio_common_msgs/AudioInfo`) + + Audio info with respect to `~audio`. + + - `audio_raw` (`audio_common_msgs/AudioData`) + + Micarray audio data has 4-channels. Maybe you need to update respeaker firmware. + + If the firmware isn't supported, this will not be output. + + - `audio_info_raw` (`audio_common_msgs/AudioInfo`) + + Audio info with respect to `~audio_raw`. + + If the firmware isn't supported, this will not be output. + + - `speech_audio` (`audio_common_msgs/AudioData`) + + Audio data while a person is speaking using the VAD function. + + - `speech_audio_raw` (`audio_common_msgs/AudioData`) + + Audio data has 4-channels while a person is speaking using the VAD function. + + If the firmware isn't supported, this will not be output. + + - `audio_merged_playback` (`audio_common_msgs/AudioData`) + + Data that combines the sound of mic and speaker. + + If the firmware isn't supported, this will not be output. + + For more detail, please see https://wiki.seeedstudio.com/ReSpeaker_Mic_Array_v2.0/ + + - `~is_speeching` (`std_msgs/Bool`) + + Using VAD function, publish whether someone is speaking. + + - `~sound_direction` (`std_msgs/Int32`) + + Direction of sound. + + - `~sound_localization` (`geometry_msgs/PoseStamped`) + + Localized Sound Direction. The value of the position in the estimated direction with `~doa_offset` as the radius is obtained. + + - ### Parameters + + - `~update_rate` (`Double`, default: `10.0`) + + Publishing info data such as `~is_speeching`, `~sound_direction`, `~sound_localization`, `~speech_audio` and `~speech_audio_raw`. + + - `~sensor_frame_id` (`String`, default: `respeaker_base`) + + Frame id. + + - `~doa_xy_offset` (`Double`, default: `0.0`) + + `~doa_offset` is a estimated sound direction's radius. + + - `~doa_yaw_offset` (`Double`, default: `90.0`) + + Estimated DoA angle offset. + + - `~speech_prefetch` (`Double`, default: `0.5`) + + Time to represent how long speech is pre-stored in buffer. + + - `~speech_continuation` (`Double`, default: `0.5`) + + If the time between the current time and the time when the speech is stopped is shorter than this time, + it is assumed that someone is speaking. + + - `~speech_max_duration` (`Double`, default: `7.0`) + + - `~speech_min_duration` (`Double`, default: `0.1`) + + If the speaking interval is within these times, `~speech_audio` and `~speech_audio_raw` will be published. + + - `~suppress_pyaudio_error` (`Bool`, default: `True`) + + If this value is `True`, suppress error from pyaudio. + +## Parameters for speech_to_text.py + + - ### Publishing topics + + - `~speech_to_text` (`speech_recognition_msgs/SpeechRecognitionCandidates`) + + Recognized text. + + - ### Subscribing topics + + - `audio` (`audio_common_msgs/AudioData`) + + Input audio. + + - ### Parameters + + - `~audio_info` (`String`, default: ``) + + audio_info (`audio_common_msgs/AudioInfo`) topic. If this value is specified, `~sample_rate`, `~sample_width` and `~channels` parameters are obtained from the topic. + + - `~sample_rate` (`Int`, default: `16000`) + + Sampling rate. + + - `~sample_width` (`Int`, default: `2`) + + Sample with. + + - `~channels` (`Int`, default: `1`) + + Number of channels. + + - `~target_channel` (`Int`, default: `0`) + + Target number of channel. + + - `~language` (`String`, default: `ja-JP`) + + language of speech to text service. For English users, you can specify `en-US`. + + - `~self_cancellation` (`Bool`, default: `True`) + + ignore voice input while the robot is speaking. + + - `~tts_tolerance` (`String`, default: `1.0`) + + time to assume as SPEAKING after tts service is finished. + + - `~tts_action_names` (`List[String]`, default: `['sound_play']`) + + If `~self_chancellation` is `True`, this value will be used. + + When the actions are active, do nothing with the callback that subscribes to `audio`. + ## Use cases ### Voice Recognition diff --git a/respeaker_ros/launch/sample_respeaker.launch b/respeaker_ros/launch/sample_respeaker.launch index 31d083608..e2c43c557 100644 --- a/respeaker_ros/launch/sample_respeaker.launch +++ b/respeaker_ros/launch/sample_respeaker.launch @@ -13,6 +13,8 @@ + + + respawn="true" respawn_delay="10" > + @@ -30,6 +33,7 @@ + audio_info: $(arg audio_info) language: $(arg language) self_cancellation: $(arg self_cancellation) tts_tolerance: 0.5 diff --git a/respeaker_ros/package.xml b/respeaker_ros/package.xml index a8d193b72..41a08fa4f 100644 --- a/respeaker_ros/package.xml +++ b/respeaker_ros/package.xml @@ -15,6 +15,7 @@ flac geometry_msgs std_msgs + sound_play speech_recognition_msgs tf python-numpy diff --git a/respeaker_ros/scripts/respeaker_node.py b/respeaker_ros/scripts/respeaker_node.py index b773b3cb7..42c38a4a8 100755 --- a/respeaker_ros/scripts/respeaker_node.py +++ b/respeaker_ros/scripts/respeaker_node.py @@ -16,6 +16,13 @@ import sys import time from audio_common_msgs.msg import AudioData +enable_audio_info = True +try: + from audio_common_msgs.msg import AudioInfo +except Exception as e: + rospy.logwarn('audio_common_msgs/AudioInfo message is not exists.' + ' AudioInfo message will not be published.') + enable_audio_info = False from geometry_msgs.msg import PoseStamped from std_msgs.msg import Bool, Int32, ColorRGBA from dynamic_reconfigure.server import Server @@ -265,7 +272,6 @@ def __init__(self, on_audio, channel=0, suppress_error=True): if self.channels != 6: rospy.logwarn("%d channel is found for respeaker" % self.channels) rospy.logwarn("You may have to update firmware.") - self.channel = min(self.channels - 1, max(0, self.channel)) self.stream = self.pyaudio.open( input=True, start=False, @@ -295,9 +301,8 @@ def stream_callback(self, in_data, frame_count, time_info, status): data = np.frombuffer(in_data, dtype=np.int16) chunk_per_channel = int(len(data) / self.channels) data = np.reshape(data, (chunk_per_channel, self.channels)) - chan_data = data[:, self.channel] # invoke callback - self.on_audio(chan_data.tobytes()) + self.on_audio(data) return None, pyaudio.paContinue def start(self): @@ -333,14 +338,24 @@ def __init__(self): self.pub_doa_raw = rospy.Publisher("sound_direction", Int32, queue_size=1, latch=True) self.pub_doa = rospy.Publisher("sound_localization", PoseStamped, queue_size=1, latch=True) self.pub_audio = rospy.Publisher("audio", AudioData, queue_size=10) + if enable_audio_info is True: + self.pub_audio_info = rospy.Publisher("audio_info", AudioInfo, + queue_size=1, latch=True) + self.pub_audio_raw_info = rospy.Publisher("audio_info_raw", AudioInfo, + queue_size=1, latch=True) self.pub_speech_audio = rospy.Publisher("speech_audio", AudioData, queue_size=10) # init config self.config = None self.dyn_srv = Server(RespeakerConfig, self.on_config) # start self.respeaker_audio = RespeakerAudio(self.on_audio, suppress_error=suppress_pyaudio_error) + self.n_channel = self.respeaker_audio.channels + self.speech_prefetch_bytes = int( - self.speech_prefetch * self.respeaker_audio.rate * self.respeaker_audio.bitdepth / 8.0) + 1 + * self.speech_prefetch + * self.respeaker_audio.rate + * self.respeaker_audio.bitdepth / 8.0) self.speech_prefetch_buffer = b"" self.respeaker_audio.start() self.info_timer = rospy.Timer(rospy.Duration(1.0 / self.update_rate), @@ -348,6 +363,58 @@ def __init__(self): self.timer_led = None self.sub_led = rospy.Subscriber("status_led", ColorRGBA, self.on_status_led) + # processed audio for ASR + if enable_audio_info is True: + info_msg = AudioInfo( + channels=1, + sample_rate=self.respeaker_audio.rate, + sample_format='S16LE', + bitrate=self.respeaker_audio.rate * self.respeaker_audio.bitdepth, + coding_format='WAVE') + self.pub_audio_info.publish(info_msg) + + if self.n_channel > 1: + # The respeaker has 4 microphones. + # Multiple microphones can be used for + # beam forming (strengthening the sound in a specific direction) + # and sound localization (the respeaker outputs the azimuth + # direction, but the multichannel can estimate + # the elevation direction). etc. + + # Channel 0: processed audio for ASR + # Channel 1: mic1 raw data + # Channel 2: mic2 raw data + # Channel 3: mic3 raw data + # Channel 4: mic4 raw data + # Channel 5: merged playback + # For more detail, please see + # https://wiki.seeedstudio.com/ReSpeaker_Mic_Array_v2.0/ + # (self.n_channel - 2) = 4 channels are multiple microphones. + self.pub_audio_raw = rospy.Publisher("audio_raw", AudioData, + queue_size=10) + self.pub_audio_merged_playback = rospy.Publisher( + "audio_merged_playback", AudioData, + queue_size=10) + if enable_audio_info is True: + info_raw_msg = AudioInfo( + channels=self.n_channel - 2, + sample_rate=self.respeaker_audio.rate, + sample_format='S16LE', + bitrate=(self.respeaker_audio.rate * + self.respeaker_audio.bitdepth), + coding_format='WAVE') + self.pub_audio_raw_info.publish(info_raw_msg) + + self.speech_audio_raw_buffer = b"" + self.speech_raw_prefetch_buffer = b"" + self.pub_speech_audio_raw = rospy.Publisher( + "speech_audio_raw", AudioData, queue_size=10) + self.speech_raw_prefetch_bytes = int( + (self.n_channel - 2) + * self.speech_prefetch + * self.respeaker_audio.rate + * self.respeaker_audio.bitdepth / 8.0) + def on_shutdown(self): try: self.respeaker.close() @@ -385,14 +452,30 @@ def on_status_led(self, msg): oneshot=True) def on_audio(self, data): - self.pub_audio.publish(AudioData(data=data)) + # take processed audio for ASR. + processed_data = data[:, 0].tobytes() + self.pub_audio.publish(AudioData(data=processed_data)) + if self.n_channel > 1: + raw_audio_data = data[:, 1:5].reshape(-1).tobytes() + self.pub_audio_raw.publish( + AudioData(data=raw_audio_data)) + self.pub_audio_merged_playback.publish( + AudioData(data=data[:, 5].tobytes())) if self.is_speeching: if len(self.speech_audio_buffer) == 0: self.speech_audio_buffer = self.speech_prefetch_buffer - self.speech_audio_buffer += data + if self.n_channel > 1: + self.speech_audio_raw_buffer = self.speech_raw_prefetch_buffer + self.speech_audio_buffer += processed_data + if self.n_channel > 1: + self.speech_audio_raw_buffer += raw_audio_data else: - self.speech_prefetch_buffer += data + self.speech_prefetch_buffer += processed_data self.speech_prefetch_buffer = self.speech_prefetch_buffer[-self.speech_prefetch_bytes:] + if self.n_channel > 1: + self.speech_raw_prefetch_buffer += raw_audio_data + self.speech_raw_prefetch_buffer = self.speech_raw_prefetch_buffer[ + -self.speech_raw_prefetch_bytes:] def on_timer(self, event): stamp = event.current_real or rospy.Time.now() @@ -432,13 +515,15 @@ def on_timer(self, event): elif self.is_speeching: buf = self.speech_audio_buffer self.speech_audio_buffer = b"" + buf_raw = self.speech_audio_raw_buffer + self.speech_audio_raw_buffer = b"" self.is_speeching = False duration = 8.0 * len(buf) * self.respeaker_audio.bitwidth - duration = duration / self.respeaker_audio.rate / self.respeaker_audio.bitdepth + duration = duration / self.respeaker_audio.rate / self.respeaker_audio.bitdepth / self.n_channel rospy.loginfo("Speech detected for %.3f seconds" % duration) if self.speech_min_duration <= duration < self.speech_max_duration: - self.pub_speech_audio.publish(AudioData(data=buf)) + self.pub_speech_audio_raw.publish(AudioData(data=buf_raw)) if __name__ == '__main__': diff --git a/respeaker_ros/scripts/speech_to_text.py b/respeaker_ros/scripts/speech_to_text.py index 439652ba8..a3a481919 100755 --- a/respeaker_ros/scripts/speech_to_text.py +++ b/respeaker_ros/scripts/speech_to_text.py @@ -2,6 +2,10 @@ # -*- coding: utf-8 -*- # Author: Yuki Furuta +from __future__ import division + +import sys + import actionlib import rospy try: @@ -9,8 +13,16 @@ except ImportError as e: raise ImportError(str(e) + '\nplease try "pip install speechrecognition"') +import numpy as np from actionlib_msgs.msg import GoalStatus, GoalStatusArray from audio_common_msgs.msg import AudioData +enable_audio_info = True +try: + from audio_common_msgs.msg import AudioInfo +except Exception as e: + rospy.logwarn('audio_common_msgs/AudioInfo message is not exists.' + ' AudioInfo message will not be published.') + enable_audio_info = False from sound_play.msg import SoundRequest, SoundRequestAction, SoundRequestGoal from speech_recognition_msgs.msg import SpeechRecognitionCandidates @@ -18,8 +30,32 @@ class SpeechToText(object): def __init__(self): # format of input audio data - self.sample_rate = rospy.get_param("~sample_rate", 16000) - self.sample_width = rospy.get_param("~sample_width", 2) + audio_info_topic_name = rospy.get_param('~audio_info', '') + if len(audio_info_topic_name) > 0: + if enable_audio_info is False: + rospy.logerr( + 'audio_common_msgs/AudioInfo message is not exists.' + ' Giving ~audio_info is not valid in your environment.') + sys.exit(1) + rospy.loginfo('Extract audio info params from {}'.format( + audio_info_topic_name)) + audio_info_msg = rospy.wait_for_message( + audio_info_topic_name, AudioInfo) + self.sample_rate = audio_info_msg.sample_rate + self.sample_width = audio_info_msg.bitrate // self.sample_rate // 8 + self.channels = audio_info_msg.channels + else: + self.sample_rate = rospy.get_param("~sample_rate", 16000) + self.sample_width = rospy.get_param("~sample_width", 2) + self.channels = rospy.get_param("~channels", 1) + if self.sample_width == 2: + self.dtype = 'int16' + elif self.sample_width == 4: + self.dtype = 'int32' + else: + raise NotImplementedError('sample_width {} is not supported' + .format(self.sample_width)) + self.target_channel = rospy.get_param("~target_channel", 0) # language of STT service self.language = rospy.get_param("~language", "ja-JP") # ignore voice input while the robot is speaking @@ -78,7 +114,11 @@ def audio_cb(self, msg): if self.is_canceling: rospy.loginfo("Speech is cancelled") return - data = SR.AudioData(msg.data, self.sample_rate, self.sample_width) + + data = SR.AudioData( + np.frombuffer(msg.data, dtype=self.dtype)[ + self.target_channel::self.channels].tobytes(), + self.sample_rate, self.sample_width) try: rospy.loginfo("Waiting for result %d" % len(data.get_raw_data())) result = self.recognizer.recognize_google( diff --git a/respeaker_ros/test/sample_respeaker.test b/respeaker_ros/test/sample_respeaker.test index 5d51c220c..61f10fb7b 100644 --- a/respeaker_ros/test/sample_respeaker.test +++ b/respeaker_ros/test/sample_respeaker.test @@ -3,6 +3,7 @@ + diff --git a/rostwitter/CMakeLists.txt b/rostwitter/CMakeLists.txt index 7f323bafc..18df95ea5 100644 --- a/rostwitter/CMakeLists.txt +++ b/rostwitter/CMakeLists.txt @@ -40,7 +40,7 @@ else() ) endif() -install(DIRECTORY test +install(DIRECTORY test launch DESTINATION ${CATKIN_PACKAGE_SHARE_DESTINATION} USE_SOURCE_PERMISSIONS ) diff --git a/rostwitter/README.md b/rostwitter/README.md new file mode 100644 index 000000000..81750bf5b --- /dev/null +++ b/rostwitter/README.md @@ -0,0 +1,116 @@ +# rostwitter + +This package is a ROS wrapper for Twitter. You can tweet via ROS. + +# How to use + +## Get access key for API. + +Please get access to the Twitter API. Please refer to the following URL. + +https://developer.twitter.com/en/docs/twitter-api/getting-started/getting-access-to-the-twitter-api + +After that, save the yaml file in the following format. + +``` +CKEY: +CSECRET: +AKEY: +ASECRET: +``` + +## Launch tweet node + +``` +roslaunch rostwitter tweet.launch account_info:= +``` + +## Tweet text + +You can tweet by simply publish on the `/tweet` topic. + +``` +rostopic pub /tweet std_msgs/String "Hello. Tweet via rostwitter (https://github.com/jsk-ros-pkg/jsk_3rdparty)" +``` + +![](./doc/tweet-string.jpg) + +If the string to be tweeted exceeds 140 full-width characters or 280 half-width characters, it will be tweeted in the "thread" display. + +``` +rostopic pub /tweet std_msgs/String """The Zen of Python, by Tim Peters + +Beautiful is better than ugly. +Explicit is better than implicit. +Simple is better than complex. +Complex is better than complicated. +Flat is better than nested. +Sparse is better than dense. +Readability counts. +Special cases aren't special enough to break the rules. +Although practicality beats purity. +Errors should never pass silently. +Unless explicitly silenced. +In the face of ambiguity, refuse the temptation to guess. +There should be one-- and preferably only one --obvious way to do it. +Although that way may not be obvious at first unless you're Dutch. +Now is better than never. +Although never is often better than *right* now. +If the implementation is hard to explain, it's a bad idea. +If the implementation is easy to explain, it may be a good idea. +Namespaces are one honking great idea -- let's do more of those! +""" +``` + +![](./doc/tweet-string-thread.jpg) + +## Tweet text with image + +You can also tweet along with your images. + +If a base64 or image path is inserted in the text, it will jump to the next reply in that section. + +### Image path + +``` +wget https://github.com/k-okada.png -O /tmp/k-okada.png +rostopic pub /tweet std_msgs/String "/tmp/k-okada.png" +``` + +![](./doc/tweet-image-path.jpg) + +### Base64 + +You can even tweet the image by encoding in base64. The following example is in python. + +Do not concatenate multiple base64 images without spaces. + + +```python +import rospy +import cv2 +import std_msgs.msg +import numpy as np +import matplotlib.cm + +from rostwitter.cv_util import extract_media_from_text +from rostwitter.cv_util import encode_image_cv2 + +rospy.init_node('rostwitter_sample') +pub = rospy.Publisher('/tweet', std_msgs.msg.String, queue_size=1) +rospy.sleep(3.0) + +colormap = matplotlib.cm.get_cmap('hsv') + +text = 'Tweet with images. (https://github.com/jsk-ros-pkg/jsk_3rdparty/pull/375)\n' +N = 12 +for i in range(N): + text += str(i) + color = colormap(1.0 * i / N)[:3] + img = color * np.ones((10, 10, 3), dtype=np.uint8) * 255 + img = np.array(img, dtype=np.uint8) + text += encode_image_cv2(img) + ' ' +pub.publish(text) +``` + +[The result of the tweet.](https://twitter.com/pr2jsk/status/1561995909524705280) diff --git a/rostwitter/doc/tweet-image-path.jpg b/rostwitter/doc/tweet-image-path.jpg new file mode 100644 index 000000000..dffc9baec Binary files /dev/null and b/rostwitter/doc/tweet-image-path.jpg differ diff --git a/rostwitter/doc/tweet-string-thread.jpg b/rostwitter/doc/tweet-string-thread.jpg new file mode 100644 index 000000000..13783eaef Binary files /dev/null and b/rostwitter/doc/tweet-string-thread.jpg differ diff --git a/rostwitter/doc/tweet-string.jpg b/rostwitter/doc/tweet-string.jpg new file mode 100644 index 000000000..c41daa779 Binary files /dev/null and b/rostwitter/doc/tweet-string.jpg differ diff --git a/rostwitter/launch/tweet.launch b/rostwitter/launch/tweet.launch new file mode 100644 index 000000000..1d202a05e --- /dev/null +++ b/rostwitter/launch/tweet.launch @@ -0,0 +1,12 @@ + + + + + + + + + + diff --git a/rostwitter/python/rostwitter/cv_util.py b/rostwitter/python/rostwitter/cv_util.py new file mode 100644 index 000000000..ad284bc63 --- /dev/null +++ b/rostwitter/python/rostwitter/cv_util.py @@ -0,0 +1,80 @@ +import base64 +import imghdr +import os.path +import re + +import cv2 +import numpy as np +import rospy + + +base64_and_filepath_image_pattern = re.compile(r'((?:/9j/)(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)? ?|/\S+\.(?:jpeg|jpg|png|gif))') + + +def encode_image_cv2(img, quality=90): + encode_param = [int(cv2.IMWRITE_JPEG_QUALITY), quality] + result, encimg = cv2.imencode('.jpg', img, encode_param) + b64encoded = base64.b64encode(encimg).decode('ascii') + return b64encoded + + +def decode_image_cv2(b64encoded): + bin = b64encoded.split(",")[-1] + bin = base64.b64decode(bin) + bin = np.frombuffer(bin, np.uint8) + img = cv2.imdecode(bin, cv2.IMREAD_COLOR) + return img + + +def is_base64_image(b64encoded): + try: + decode_image_cv2(b64encoded) + except Exception as e: + rospy.logerr(str(e)) + return False + return True + + +def get_image_from_text(text): + if base64_and_filepath_image_pattern.match(text) is None: + return None + + if os.path.exists(text): + path = text + if imghdr.what(path) in ['jpeg', 'png', 'gif']: + with open(path, 'rb') as f: + return f.read() + else: + succ = is_base64_image(text) + if succ: + bin = text.split(",")[-1] + bin = base64.b64decode(bin) + bin = np.frombuffer(bin, np.uint8) + return bin + + +def extract_media_from_text(text): + texts = base64_and_filepath_image_pattern.split(text) + target_texts = list(filter(lambda x: x is not None and len(x.strip()) > 0, texts)) + + split_texts = [''] + imgs_list = [] + + texts = [] + imgs = [] + for text in target_texts: + img = get_image_from_text(text) + if img is None: + split_texts.append(text) + imgs_list.append(imgs) + imgs = [] + else: + imgs.append(img) + + if len(imgs) > 0: + imgs_list.append(imgs) + if len(split_texts) > 0: + if len(split_texts[0]) == 0 and len(imgs_list[0]) == 0: + split_texts = split_texts[1:] + imgs_list = imgs_list[1:] + return imgs_list, split_texts diff --git a/rostwitter/python/rostwitter/twitter.py b/rostwitter/python/rostwitter/twitter.py index cdb020e15..c56cf5289 100644 --- a/rostwitter/python/rostwitter/twitter.py +++ b/rostwitter/python/rostwitter/twitter.py @@ -1,16 +1,20 @@ # originally from https://raw.githubusercontent.com/bear/python-twitter/v1.1/twitter.py # NOQA +import math import json as simplejson import requests -from requests_oauthlib import OAuth1 -# https://stackoverflow.com/questions/11914472/stringio-in-python3 try: - from StringIO import StringIO ## for Python 2 + from itertools import zip_longest except ImportError: - from io import StringIO ## for Python 3 + from itertools import izip_longest as zip_longest +from requests_oauthlib import OAuth1 import rospy +from rostwitter.util import count_tweet_text +from rostwitter.util import split_tweet_text +from rostwitter.cv_util import extract_media_from_text + class Twitter(object): def __init__( @@ -54,24 +58,80 @@ def _request_url(self, url, verb, data=None): ) return 0 # if not a POST or GET request - def post_update(self, status): - if len(status) > 140: - rospy.logwarn('tweet is too longer > 140 characters') - status = status[:140] - url = 'https://api.twitter.com/1.1/statuses/update.json' - data = {'status': StringIO(status)} - json = self._request_url(url, 'POST', data=data) - data = simplejson.loads(json.content) + def _check_post_request(self, request): + valid = True + data = simplejson.loads(request.content) + if request.status_code != 200: + rospy.logwarn('post tweet failed. status_code: {}' + .format(request.status_code)) + if 'errors' in data: + for error in data['errors']: + rospy.logwarn('Tweet error code: {}, message: {}' + .format(error['code'], error['message'])) + valid = False + if valid: + return data + + def _post_update_with_reply(self, texts, media_list=None, + in_reply_to_status_id=None): + split_media_list = [] + media_list = media_list or [] + for i in range(0, int(math.ceil(len(media_list) / 4.0))): + split_media_list.append(media_list[i * 4:(i + 1) * 4]) + for text, media_list in zip_longest(texts, split_media_list): + text = text or '' + media_list = media_list or [] + url = 'https://api.twitter.com/1.1/statuses/update.json' + data = {'status': text} + media_ids = self._upload_media(media_list) + if len(media_ids) > 0: + data['media_ids'] = media_ids + if in_reply_to_status_id is not None: + data['in_reply_to_status_id'] = in_reply_to_status_id + r = self._request_url(url, 'POST', data=data) + data = self._check_post_request(r) + if data is not None: + in_reply_to_status_id = data['id'] + return data + + def _upload_media(self, media_list): + url = 'https://upload.twitter.com/1.1/media/upload.json' + media_ids = [] + for media in media_list: + data = {'media': media} + r = self._request_url(url, 'POST', data=data) + if r.status_code == 200: + rospy.loginfo('upload media success') + media_ids.append(str(r.json()['media_id'])) + else: + rospy.logerr('upload media failed. status_code: {}' + .format(r.status_code)) + media_ids = ','.join(media_ids) + return media_ids + + def post_update(self, status, in_reply_to_status_id=None): + media_list, status_list = extract_media_from_text(status) + for text, mlist in zip_longest(status_list, media_list): + text = text or '' + texts = split_tweet_text(text) + data = self._post_update_with_reply( + texts, + media_list=mlist, + in_reply_to_status_id=in_reply_to_status_id) + if data is not None: + in_reply_to_status_id = data['id'] return data - def post_media(self, status, media): - # 116 = 140 - len("http://t.co/ssssssssss") - if len(status) > 116: - rospy.logwarn('tweet wit media is too longer > 116 characters') - status = status[:116] + def post_media(self, status, media, in_reply_to_status_id=None): + texts = split_tweet_text(status) + status = texts[0] url = 'https://api.twitter.com/1.1/statuses/update_with_media.json' - data = {'status': StringIO(status)} + data = {'status': status} data['media'] = open(str(media), 'rb').read() - json = self._request_url(url, 'POST', data=data) - data = simplejson.loads(json.content) + r = self._request_url(url, 'POST', data=data) + data = self._check_post_request(r) + if len(texts) > 1: + data = self._post_update_with_reply( + texts[1:], + in_reply_to_status_id=data['id']) return data diff --git a/rostwitter/python/rostwitter/util.py b/rostwitter/python/rostwitter/util.py index 36a613b46..f5e51471c 100644 --- a/rostwitter/python/rostwitter/util.py +++ b/rostwitter/python/rostwitter/util.py @@ -1,4 +1,6 @@ import os +import sys +import unicodedata import yaml import rospy @@ -16,9 +18,47 @@ def load_oauth_settings(yaml_path): rospy.logerr("EOF") return None, None, None, None with open(yaml_path, 'r') as f: - key = yaml.load(f) + key = yaml.load(f, Loader=yaml.SafeLoader) ckey = key['CKEY'] csecret = key['CSECRET'] akey = key['AKEY'] asecret = key['ASECRET'] return ckey, csecret, akey, asecret + + +def count_tweet_text(text): + count = 0 + if sys.version_info.major <= 2: + text = text.decode('utf-8') + for c in text: + if unicodedata.east_asian_width(c) in 'FWA': + count += 2 + else: + count += 1 + return count + + +def split_tweet_text(text, length=280): + texts = [] + split_text = '' + count = 0 + if sys.version_info.major <= 2: + text = text.decode('utf-8') + for c in text: + if count == 281: + # last word is zenkaku. + texts.append(split_text[:-1]) + split_text = split_text[-1:] + count = 2 + elif count == 280: + texts.append(split_text) + split_text = '' + count = 0 + split_text += c + if unicodedata.east_asian_width(c) in 'FWA': + count += 2 + else: + count += 1 + if count != 0: + texts.append(split_text) + return texts diff --git a/rostwitter/scripts/tweet.py b/rostwitter/scripts/tweet.py index d4b666959..50c44cf48 100755 --- a/rostwitter/scripts/tweet.py +++ b/rostwitter/scripts/tweet.py @@ -32,29 +32,9 @@ def tweet_cb(self, msg): rospy.loginfo(rospy.get_name() + " sending %s", ''.join([message] if len(message) < 128 else message[0:128]+'......')) - # search word start from / and end with {.jpeg,.jpg,.png,.gif} - m = re.search('/\S+\.(jpeg|jpg|png|gif)', message) - ret = None - if m: - filename = m.group(0) - message = re.sub(filename, "", message) - if os.path.exists(filename): - rospy.loginfo( - rospy.get_name() + " tweet %s with file %s", - message, filename) - # 140 - len("http://t.co/ssssssssss") - ret = self.api.post_media(message[0:116], filename) - if 'errors' in ret: - rospy.logerr('Failed to post: {}'.format(ret)) - # ret = self.api.post_update(message) - else: - rospy.logerr(rospy.get_name() + " %s could not find", filename) - else: - ret = self.api.post_update(message[0:140]) - if 'errors' in ret: - rospy.logerr('Failed to post: {}'.format(ret)) - # seg faults if message is longer than 140 byte ??? - rospy.loginfo(rospy.get_name() + " receiving %s", ret) + ret = self.api.post_update(message) + if ret is not None: + rospy.loginfo(rospy.get_name() + " receiving %s", ret) if __name__ == '__main__':