Skip to content

Commit

Permalink
Add joss-paper.
Browse files Browse the repository at this point in the history
  • Loading branch information
mamaheux committed Sep 14, 2023
0 parents commit 314ac75
Show file tree
Hide file tree
Showing 3 changed files with 287 additions and 0 deletions.
25 changes: 25 additions & 0 deletions .github/workflows/draft-joss-paper-pdf.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
name: JOSS Paper Draft PDF

on: [push]

jobs:
paper:
runs-on: ubuntu-latest
name: Paper Draft
steps:
- name: Checkout
uses: actions/checkout@v3
- name: Build draft PDF
uses: openjournals/openjournals-draft-action@master
with:
journal: joss
# This should be the path to the paper within your repo.
paper-path: joss-paper/paper.md
- name: Upload
uses: actions/upload-artifact@v3
with:
name: paper
# This is the output path where Pandoc will write the compiled
# PDF. Note, this should be the same directory as the input
# paper.md
path: joss-paper/paper.pdf
154 changes: 154 additions & 0 deletions joss-paper/paper.bib
Original file line number Diff line number Diff line change
@@ -0,0 +1,154 @@

@article{saltzman_loneliness_2020,
title = {Loneliness, isolation, and social support factors in post-{COVID}-19 mental health.},
volume = {12},
journal = {Psychological Trauma: Theory, Research, Practice, and Policy},
author = {Saltzman, Leia Y. and Hansel, Tonya Cross and Bordnick, Patrick S.},
year = {2020},
pages = {S55--S57},
doi = {10.1037/tra0000703}
}

@inproceedings{ramirez_gomez_socially_2022,
address = {USA},
series = {{CHI} {PLAY} '22},
title = {Socially Distanced Games: Exploring the Future Opportunities of Remote Play},
isbn = {978-1-4503-9211-2},
booktitle = {Extended {Abstracts} of the {Annual} {Symposium} on {Computer}-{Human} {Interaction} in {Play}},
publisher = {Association for Computing Machinery},
author = {Ramirez Gomez, Argenis and Stawarz, Katarzyna},
year = {2022},
pages = {93--102},
doi = {10.1145/3505270.3558338}
}

@inproceedings{yuan_tabletop_2021,
address = {USA},
series = {{CHI} '21},
title = {Tabletop Games in the Age of Remote Collaboration: Design Opportunities for a Socially Connected Game Experience},
isbn = {978-1-4503-8096-6},
abstract = {Prior research has highlighted opportunities for technology to better support the tabletop game experience in offline and online settings, but little work has focused on the social aspect of tabletop gaming. We investigated the social and collaborative aspects of tabletop gaming in the unique context of “social distancing” during the 2020 COVID-19 pandemic to shed light on the experience of remote tabletop gaming. With a multi-method qualitative approach (including digital ethnography and in-depth interviews), we empirically studied how people appropriate existing technologies and adapt their offline practices to play tabletop games remotely. We identify three themes that describe people's game and social experience during remote play: creating a shared tabletop environment (shared space), enabling a collective understanding (shared information and awareness), and facilitating a communal temporal experience (shared time). We reflect on challenges and design opportunities for a better experience in the age of remote collaboration.},
booktitle = {Proceedings of the {CHI} {Conference} on {Human} {Factors} in {Computing} {Systems}},
publisher = {Association for Computing Machinery},
author = {Yuan, Ye and Cao, Jan and Wang, Ruotong and Yarosh, Svetlana},
year = {2021},
doi = {10.1145/3411764.3445512}
}

@article{kristoffersson_review_2013,
title = {A Review of Mobile Robotic Telepresence},
volume = {2013},
issn = {1687-5893},
journal = {Advances in Human-Computer Interaction},
author = {Kristoffersson, Annica and Coradeschi, Silvia and Loutfi, Amy},
year = {2013},
doi = {10.1155/2013/902316}
}

@article{hung_facilitators_2022,
title = {Facilitators and barriers to using telepresence robots in aged care settings: {A} scoping review.},
volume = {9},
issn = {2055-6683},
journal = {Journal of rehabilitation and assistive technologies engineering},
author = {Hung, Lillian and Wong, Joey and Smith, Chelsea and Berndt, Annette and Gregorio, Mario and Horne, Neil and Jackson, Lynn and Mann, Jim and Wada, Mineko and Young, Erika},
year = {2022},
pages = {1--19},
doi = {10.1136/bmjopen-2021-051769}
}

@article{cortellessa_ai_2021,
title = {{AI} and robotics to help older adults: {Revisiting} projects in search of lessons learned},
volume = {12},
url = {https://doi.org/10.1515/pjbr-2021-0025},
doi = {10.1515/pjbr-2021-0025},
number = {1},
urldate = {2023-03-16},
journal = {Paladyn, Journal of Behavioral Robotics},
author = {Cortellessa, Gabriella and Benedictis, Riccardo De and Fracasso, Francesca and Orlandini, Andrea and Umbrico, Alessandro and Cesta, Amedeo},
year = {2021},
pages = {356--378},
file = {Cortellessa et al. - 2021 - AI and robotics to help older adults Revisiting p.pdf:/home/marc-antoine/Zotero/storage/CZEX6Q3E/Cortellessa et al. - 2021 - AI and robotics to help older adults Revisiting p.pdf:application/pdf},
}

@misc{opentera,
author = {{IntRoLab}},
title = {OpenTera: Open telerehabilitation server and micro-services },
year = {2023},
publisher = {GitHub},
journal = {GitHub repository},
url = {https://github.com/introlab/opentera}
}

@misc{pybind11,
author = {{pybind}},
title = {pybind11: Seamless operability between C++11 and Python},
year = {2023},
publisher = {GitHub},
journal = {GitHub repository},
url = {https://github.com/pybind/pybind11}
}

@inproceedings{quigley_ros_2009,
title = {{ROS}: {A}n open-source {Robot} {Operating} {System}},
volume = {3},
booktitle = {{ICRA} {Workshop} on {Open} {Source} {Software}},
author = {Quigley, Morgan and Conley, Ken and Gerkey, Brian and Faust, Josh and Foote, Tully and Leibs, Jeremy and Wheeler, Rob and Ng, Andrew},
year = {2009}
}

@misc{webrtc_ros,
author = {{RobotWebTools}},
title = {webrtc_ros: Streaming of {ROS} Image Topics using {WebRTC}},
year = {2021},
publisher = {GitHub},
journal = {GitHub repository},
url = {https://github.com/RobotWebTools/webrtc_ros}
}

@misc{gstreamer,
author = {{GStreamer}},
title = {{GStreamer} {WebRTC} demos},
year = {2022},
publisher = {GitLab},
journal = {GitLab repository},
url = {https://gitlab.freedesktop.org/gstreamer/gst-examples/-/tree/master/webrtc}
}

@misc{aiortc,
author = {{aiortc}},
title = {aiortc: WebRTC and ORTC implementation for {P}ython using asyncio},
year = {2023},
publisher = {GitHub},
journal = {GitHub repository},
url = {https://github.com/aiortc/aiortc}
}

@misc{mixedreality-webrtc,
author = {{Microsoft}},
title = {MixedReality-WebRTC: MixedReality-WebRTC is a collection of components to help mixed reality app developers integrate audio and video real-time communication into their application and improve their collaborative experience},
year = {2022},
publisher = {GitHub},
journal = {GitHub repository},
url = {https://github.com/microsoft/MixedReality-WebRTC}
}

@misc{webrtc-rs,
author = {{WebRTC.rs}},
title = {WebRTC: A pure Rust implementation of WebRTC},
year = {2023},
publisher = {GitHub},
journal = {GitHub repository},
url = {https://github.com/webrtc-rs/webrtc}
}

@article{panchea_opentera_2022,
title = {{OpenTera}: {A} microservice architecture solution for rapid prototyping of robotic solutions to {COVID}-19 challenges in care facilities},
volume = {12},
issn = {2190-7196},
number = {2},
journal = {Health \& Technology},
author = {Panchea, Adina M. and Létourneau, Dominic and Brière, Simon and Hamel, Mathieu and Maheux, Marc-Antoine and Godin, Cédric and Tousignant, Michel and Labbé, Mathieu and Ferland, François and Grondin, François and Michaud, François},
year = {2022},
pages = {583--596},
doi = {10.1007/s12553-021-00636-5}
}
108 changes: 108 additions & 0 deletions joss-paper/paper.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,108 @@
---
title: 'OpenTera WebRTC: A User-Friendly WebRTC Wrapper for JavaScript, C++ and Python'
tags:
- WebRTC
- C++
- Python
- JavaScript
authors:
- name: Marc-Antoine Maheux
orcid: 0000-0002-3983-8754
affiliation: 1
- name: Dominic Létourneau
orcid: 0000-0001-7825-7533
affiliation: 1
- name: Philippe Warren
orcid: 0009-0008-4466-0963
affiliation: 1
- name: François Michaud
orcid: 0000-0002-3639-7770
affiliation: 1
affiliations:
- name: Interdisciplinary Institute for Technological Innovation (3IT), Université de Sherbrooke, Canada
index: 1
date: 15 March 2023
bibliography: paper.bib
---

# Summary
The COVID-19 pandemic changed the landscape of telehealth and videoconferencing practices, having people of all ages and fields become users of virtual meetings and even social activities [@saltzman_loneliness_2020].
Conventional videoconferencing solutions basically establish an audio and video link between two or more participants.
However, advanced solutions like telehealth systems may require interfacing remote biometric sensors and devices (e.g., vital sign monitoring devices, pan–tilt–zoom (PTZ) cameras, and telepresence mobile robots [@kristoffersson_review_2013]), streaming of multiple video feeds, and providing specialized graphical interfaces to improve situation awareness and reduce cognitive load of the users [@panchea_opentera_2022].
Simple audio-video links also reveal insufficient to address the specific needs of older adults [@cortellessa_ai_2021], for instance in terms of easy installation and ease of use, functionalities to address loneliness [@hung_facilitators_2022] or to implement socially distanced games [@ramirez_gomez_socially_2022; @yuan_tabletop_2021].
With the objective of providing a more generic telecommunication framework, we have designed OpenTera [@opentera; @panchea_opentera_2022], a micro-service architecture for audio-video-data telecommunication services.

WebRTC^[[https://webrtc.org/](https://webrtc.org/)] is the de facto technology to develop videoconference platforms because it is secured and built for web browsers, hence improving accessibility compared to having to install a software application locally.
This makes WebRTC a good choice to develop OpenTera.
A JavaScript API is available in modern web browsers, but the developer needs to create a web server to initialize the peer-to-peer calls, which is usually named a signaling server.
The library used by most web browsers to include WebRTC support is the WebRTC Native library^[[https://webrtc.github.io/webrtc-org/native-code/](https://webrtc.github.io/webrtc-org/native-code/)].
However, adding sensors to web-based videoconference setups is limited by their compatibility with web browsers, bounded by their APIs and sandbox configuration.
Typically, web browsers support mouse, keyboard, gamepad, webcam and microphone devices. However, experimental APIs are available to use Bluetooth^[[https://developer.mozilla.org/en-US/docs/Web/API/Web_Bluetooth_API](https://developer.mozilla.org/en-US/docs/Web/API/Web_Bluetooth_API)], serial^[[https://wicg.github.io/serial/](https://wicg.github.io/serial/)] and USB^[[https://developer.mozilla.org/en-US/docs/Web/API/USB](https://developer.mozilla.org/en-US/docs/Web/API/USB)] devices, but the devices must be compatible and these APIs are not supported on all browsers.
Also, in the context of telepresence robots, a web interface built with WebRTC can control a robot, but the robot needs to run the equivalent of a web browser for the JavaScript API.
This is often done via the integration of Chromium web engine (Electron, Qt) and requires code in JavaScript, which might not be optimized.

To address these limitations, this paper presents the OpenTera WebRTC project, a multiplatform library to develop web and native applications for customizable videoconferencing, telehealth and telepresence robotics applications.
It provides bidirectional peer-to-peer multipoint communication of audio streams, multi-video feed streaming and data, and a signaling server hosted on secure, local servers, independent of tier actors and online services.

# Project Description
The OpenTera WebRTC library provides simple and similar APIs in JavaScript for web applications and in C++ and Python for native applications.
Developing native applications makes it possible to integrate a variety of devices and reduce resource usage.
Also, the library includes a secured signaling server to reduce the development time when using the library.
The C++ API uses the WebRTC Native library and is packaged to be easy to build.
Video and audio streams are provided to the library as raw images and raw audio frames to increase flexibility.
The Python API is done with pybind11 [@pybind11] to create Python bindings of the C++ API.
The Python API is easier to use, but the C++ API is faster.
To develop telepresence robots, the C++ API is integrated with ROS [@quigley_ros_2009] because this middleware is widely adopted by the research community.

GStreamer is adapted to the WebRTC Native library to provide hardware acceleration to encode and decode video streams.
So, the native APIs are well suited for embedded applications.
\autoref{tab:hardware-acceleration-compatibility} presents the hardware acceleration compatibility at the present time.
More codecs and hardware configurations can easily be added in the future.



Table: Hardware Acceleration Compatibility \label{tab:hardware-acceleration-compatibility}

| Operating System | Hardware | VP8 | VP9 | H.264 |
|------------------|------------------------------------|-----------------------|----------|-----------------------|
| Linux | Jetson TX2/Nano | Encoding and Decoding | Decoding | Encoding and Decoding |
| Linux | Jetson Xavier NX | | Decoding | Encoding and Decoding |
| Linux | Jetson AGX Xavier | | Decoding | Encoding and Decoding |
| Linux | Jetson Orin / Orin Nano / Orin NX | | Decoding | Encoding and Decoding |
| Linux | Raspberry Pi 4 | | | Encoding and Decoding |
| Linux | Intel, AMD, NVIDIA with the VA-API | Encoding and Decoding | Decoding | Encoding and Decoding |
| MacOS | Mac with the Apple Media API | | | Encoding and Decoding |


# Statement of need
WebRTC Native can be used to create native applications directly, but the library is hard to build and it is designed to use webcams and integrated peripherals.
OpenTera WebRTC native APIs add the capacity to provide any audio and video streams, making it more generic and flexible.
The integration of BoringSSL and Socket.IO code removes the need to wrap and understand other communication libraries.
Therefore, the users of OpenTera WebRTC can connect their applications to the signaling server easily.

\autoref{tab:webrtc-library-summary} presents a summary of the available WebRTC libraries to create native applications.
WebRTC ROS [@webrtc_ros] and MixedReality-WebRTC [@mixedreality-webrtc] also wrap the WebRTC Native library, but the versions used are pretty old and neither are no longer actively maintained.
GStreamer [@gstreamer] has a plugin to perform WebRTC calls. However, it is hard to use and the plugin is currently available in the gst-bad plugins, which means it is probably incomplete and misses important features to comply with the standard.
The aiortc library [@aiortc] and WebRTC.rs [@webrtc-rs] replicate the web WebRTC API in Python and Rust, respectively.
The former uses third-party party libraries for the audio and video codecs, and the latter does not yet have any codec support.
Nevertheless, WebRTC.rs supports streaming of already encoded audio and video streams.

Overall, OpenTera WebRTC provides interesting capabilities compared to the other WebRTC libraries because it offers hardware acceleration for embedded platforms, has a unified API across all languages, integrates the signaling server and wraps an up-to-date WebRTC Native version, which can be updated easily by our automated build on GitHub.


Table: WebRTC Library Summary \label{tab:webrtc-library-summary}

| Library | Web Browser API | Native API | Raw Audio Streams | Raw Video Streams | Hardware Acceleration | Signaling Server |
|---------------------|-----------------|-------------|-------------------|-------------------|-----------------------|------------------|
| WebRTC ROS | X | ROS | | X | | Integrated |
| GStreamer | | C, Python | X | X | X | Example |
| aiortc | | Python | X | X | | Example |
| MixedReality-WebRTC | | C++, C# | | X | | Example |
| WebRTC.rs | | Rust | | | | Example |
| OpenTera WebRTC | X | C++, Python | X | X | X | Integrated |


# Acknowledgements
This work was supported by the Natural Sciences and Engineering Research Council of Canada (NSERC), the Fonds de recherche du Québec – Nature et technologies (FRQNT) and the Network of Centres of Excellence of Canada on Aging Gracefully across Environments using Technology to Support Wellness, Engagement, and Long Life (AGE-WELL).

# References

0 comments on commit 314ac75

Please sign in to comment.