papers.bib

@INPROCEEDINGS{10765309,
  author={Chen, Kanyu and Wu, Erwin and Saito, Daichi and Peng, Yichen and Liao, Chen-Chieh and Kato, Akira and Koike, Hideki and Kunze, Kai},
  booktitle={2024 IEEE International Symposium on Mixed and Augmented Reality Adjunct (ISMAR-Adjunct)}, 
  title={Novel Sensing Methods for Vocal Technique Analysis: Evaluation on Electromyography and Ultrasonography}, 
  year={2024},
  volume={},
  number={},
  pages={121-125},
  abstract={Controlling vocal cord muscles is crucial for vocal performance and training, yet it is challenging to measure. This paper introduces electromyography (EMG) and ultrasonography to detect vocal muscle activity and assess pitch training skills. A pre-experiment with 16 participants analyzed muscle control discrepancies among singers of different skill levels. A subsequent user study with 12 participants evaluated EMG and ultrasonography feedback effectiveness. Findings indicate that EMG offers better temporal stability representation, while ultrasonography provides intuitive visual feedback on vocal cord activity. Both methods show potential in enhancing vocal control, offering insights for designing effective and non-invasive vocal training systems.},
  keywords={Training;Visualization;Electric potential;Ultrasonic variables measurement;Ultrasonography;Muscles;Particle measurements;Electromyography;Stability analysis;Sensors;human state;EMG measurement;ultrasonic sensing;acoustic interaction},
  doi={10.1109/ISMAR-Adjunct64951.2024.00034},
  ISSN={2771-1110},
  month={Oct},}

@INPROCEEDINGS{10494108,
  author={Matsumoto, Takashi and Wu, Erwin and Liao, Chen-Chieh and Koike, Hideki},
  booktitle={2024 IEEE Conference Virtual Reality and 3D User Interfaces (VR)}, 
  title={ARpenSki: Augmenting Ski Training with Direct and Indirect Postural Visualization}, 
  year={2024},
  volume={},
  number={},
  pages={1-9},
  abstract={Alpine skiing is a popular winter sport, and several systems have been proposed to enhance training and improve efficiency. However, many existing systems rely on simulation-based environments, which suffer from drawbacks such as a gap between real skiing and the lack of body ownership. To address these limitations, we present ARpenSki, a novel augmented reality (AR) ski training system that employs a see-through head mounted display (HMD) to deliver augmented visual training cues that may be applied on real slopes. The proposed AR system provides a transparent view of the lower half of the field of vision, where we implemented three different AR-based direct and indirect postural visualization methods. We conducted an user study to investigate the influence of different visual cues in the AR environment. Our results indicate that a simple AR visualization of the user’s spine (Figure 1.2) yields the most favorable training performance, surpassing conventional visualizations by 7% improvement in the user’s posture. Building upon these promising findings, we further tested our system on real slopes and showed the potential of a real AR skiing application.},
  keywords={Training;Visualization;Three-dimensional displays;Buildings;Resists;User interfaces;Augmented reality;Human-centered computing;Human computer interaction (HCI);Interaction paradigms;Mixed / augmented reality},
  doi={10.1109/VR58804.2024.00024},
  ISSN={2642-5254},
  month={March},}

@inproceedings{10.1145/3606038.3616151,
author = {Wu, Erwin and Matsumoto, Takashi and Liao, Chen-Chieh and Liu, Ruofan and Katsuyama, Hidetaka and Inaba, Yuki and Hakamada, Noriko and Yamamoto, Yusuke and Ishige, Yusuke and Koike, Hideki},
title = {SkiTech: An Alpine Skiing and Snowboarding Dataset of 3D Body Pose, Sole Pressure, and Electromyography},
year = {2023},
isbn = {9798400702693},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
url = {https://doi.org/10.1145/3606038.3616151},
doi = {10.1145/3606038.3616151},
abstract = {Effective analysis of skills requires high-quality, multi-modal datasets, especially in the field of artificial intelligence. However, creating such datasets for extreme sports, such as alpine skiing, can be challenging due to environmental constraints. Optical and wearable sensors may not perform optimally under diverse lighting, weather, and terrain conditions. To address these challenges, we present a comprehensive skiing/snowboarding dataset using a professional motor-based simulator. Using the realistic simulator, it is easy to obtain different types of data with a small domain gap between real-world data. Common data for skill analysis are collected, including camera images, 3D body pose, sole pressure, and leg electromyography, from athletes of different levels. Another key aspect is the comparison of cross-modal baselines, highlighting the versatility of the data across modalities. In addition, a real-world pilot test is conducted to assess the practical applicability and data robustness.},
booktitle = {Proceedings of the 6th International Workshop on Multimedia Content Analysis in Sports},
pages = {3–8},
numpages = {6},
keywords = {winter sports, sole pressure, skiing, emg, dataset, 3d body pose},
location = {Ottawa ON, Canada},
series = {MMSports '23}
}

@INPROCEEDINGS{10316487,
  author={Liu, Ruofan and Wu, Erwin and Liao, Chen-Chieh and Nishioka, Hayato and Furuya, Shinichi and Koike, Hideki},
  booktitle={2023 IEEE International Symposium on Mixed and Augmented Reality (ISMAR)}, 
  title={PianoSyncAR: Enhancing Piano Learning through Visualizing Synchronized Hand Pose Discrepancies in Augmented Reality}, 
  year={2023},
  volume={},
  number={},
  pages={859-868},
  abstract={Motor skill acquisition involves learning from spatiotemporal discrepancies between target and self-generated motions. However, in dexterous skills with numerous degrees of freedom, understanding and correcting these motor errors are challenging. This issue becomes crucial for experienced individuals who seek for mastering and sophisticating their skills, where even subtle errors need to be minimized. To enable efficient optimization of body posture in piano learning, we present PianoSyncAR, an augmented reality system that superimposes the time-varying complex hand postures of a teacher over the hand of a learner. Through a user study with 12 pianists, we demonstrate several advantages of the proposed system over conventional tablet-screen, which implicate the potential of AR training as a complementary tool for video-based skill learning in piano playing.},
  keywords={Training;Visualization;Three-dimensional displays;Motion capture;Spatiotemporal phenomena;Synchronization;Augmented reality;Human-centered computing;Human computer interaction (HCI);Interaction paradigms-Mixed / augmented reality},
  doi={10.1109/ISMAR59233.2023.00101},
  ISSN={2473-0726},
  month={Oct},}

@inproceedings{10.1145/3588028.3603679,
author = {Liao, Chen-Chieh and Kim, Jong-Hwan and Koike, Hideki and Hwang, Dong-Hyun},
title = {Content-Preserving Motion Stylization using Variational Autoencoder},
year = {2023},
isbn = {9798400701528},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
url = {https://doi.org/10.1145/3588028.3603679},
doi = {10.1145/3588028.3603679},
abstract = {This work proposes a motion style transfer network that transfers motion style between different motion categories using variational autoencoders. The proposed network effectively transfers style among various motion categories and can create stylized motion unseen in the dataset. The network contains a content-conditioned module to preserve the characteristic of the content motion, which is important for real applications. We implement the network with variational autoencoders, which enable us to control the intensity of the style and mix different styles to enrich the motion diversity.},
booktitle = {ACM SIGGRAPH 2023 Posters},
articleno = {3},
numpages = {2},
keywords = {computer graphics, machine learning, motion style transfer},
location = {Los Angeles, CA, USA},
series = {SIGGRAPH '23}
}

@inproceedings{10.1145/3577190.3614106,
author = {Sun, Yasheng and Wu, Qianyi and Zhou, Hang and Wang, Kaisiyuan and Hu, Tianshu and Liao, Chen-Chieh and Miyafuji, Shio and Liu, Ziwei and Koike, Hideki},
title = {Make Your Brief Stroke Real and Stereoscopic: 3D-Aware Simplified Sketch to Portrait Generation},
year = {2023},
isbn = {9798400700552},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
url = {https://doi.org/10.1145/3577190.3614106},
doi = {10.1145/3577190.3614106},
abstract = {Creating the photo-realistic version of people’s sketched portraits is useful to various entertainment purposes. Existing studies only generate portraits in the 2D plane with fixed views, making the results less vivid. In this paper, we present Stereoscopic Simplified Sketch-to-Portrait (SSSP), which explores the possibility of creating Stereoscopic 3D-aware portraits from simple contour sketches by involving 3D generative models. Our key insight is to design sketch-aware constraints that can fully exploit the prior knowledge of a tri-plane-based 3D-aware generative model. Specifically, our designed region-aware volume rendering strategy and global consistency constraint further enhance detail correspondences during sketch encoding. Moreover, in order to facilitate the usage of layman users, we propose a Contour-to-Sketch module with vector quantized representations, so that easily drawn contours can directly guide the generation of 3D portraits. Extensive comparisons show that our method generates high-quality results that match the sketch. Our usability study verifies that our system is preferred by users.},
booktitle = {Proceedings of the 25th International Conference on Multimodal Interaction},
pages = {388–396},
numpages = {9},
keywords = {Cross-Modal Generation, Virtual Character Creation},
location = {Paris, France},
series = {ICMI '23}
}

@inproceedings{10.1145/3544549.3585705,
author = {Liu, Ruofan and Wu, Erwin and Liao, Chen-Chieh and Nishioka, Hayato and Furuya, Shinichi and Koike, Hideki},
title = {PianoHandSync: An Alignment-based Hand Pose Discrepancy Visualization System for Piano Learning},
year = {2023},
isbn = {9781450394222},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
url = {https://doi.org/10.1145/3544549.3585705},
doi = {10.1145/3544549.3585705},
abstract = {Video-based lessons are becoming a popular way for distance piano education. However, limited by the fixed camera angle, a video is difficult to tell precise 3D hand posture, which is one of the most essential factors for learning piano. This paper presents a visualization system providing the intuitive discrepancy of hand postures in two piano performance videos. Through a motion capture system, the estimated 3D postures are visualized and discrepancies based on distinct metrics are displayed, integrated with modular functions assisting skill acquisition. A pilot study proves that the proposed visualization can be a supplementary means for only video-based lessons in terms of correcting hand postures and fingering.},
booktitle = {Extended Abstracts of the 2023 CHI Conference on Human Factors in Computing Systems},
articleno = {228},
numpages = {7},
keywords = {Motion Analysis, Piano Training, Virtual Visualization},
location = {Hamburg, Germany},
series = {CHI EA '23}
}

@misc{kobayashi2023motioncapturedatasetpractical,
      title={Motion Capture Dataset for Practical Use of AI-based Motion Editing and Stylization}, 
      author={Makito Kobayashi and Chen-Chieh Liao and Keito Inoue and Sentaro Yojima and Masafumi Takahashi},
      year={2023},
      eprint={2306.08861},
      archivePrefix={arXiv},
      primaryClass={cs.CV},
      url={https://arxiv.org/abs/2306.08861}, 
}

@inproceedings{10.1145/3582700.3582710,
author = {Liao, Chen-Chieh and Hwang, Dong-Hyun and Wu, Erwin and Koike, Hideki},
title = {AI Coach: A Motor Skill Training System using Motion Discrepancy Detection},
year = {2023},
isbn = {9781450399845},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
url = {https://doi.org/10.1145/3582700.3582710},
doi = {10.1145/3582700.3582710},
abstract = {Spatial and temporal clues found in a professional’s motion are essential for designing a training system for learning a motor skill. We investigate the potential of using neural networks to learn spatial and temporal features of advanced players in sports and to detect the fine-grained differences between motions. As a training system, we implement an AI Coach prototype application that finds the differences between two input motions and visualizes a recommendation motion for the users to correct their forms. In the user study, we investigate the effects of the proposed AI Coach and discuss the findings based on quantitative questionnaires and qualitative interviews. In the study, the proposed system can help the user better understand the difference between them and the coach. The study also reveals the necessity of coaching beginners in the early learning phases.},
booktitle = {Proceedings of the Augmented Humans International Conference 2023},
pages = {179–189},
numpages = {11},
keywords = {neural networks, golf swing, discrepancy detection, Motor skill training},
location = {Glasgow, United Kingdom},
series = {AHs '23}
}

@ARTICLE{9913343,
  author={Liao, Chen-Chieh and Hwang, Dong-Hyun and Koike, Hideki},
  journal={IEEE Access}, 
  title={AI Golf: Golf Swing Analysis Tool for Self-Training}, 
  year={2022},
  volume={10},
  number={},
  pages={106286-106295},
  abstract={In the field of the acquisition of sports skills, a common way to improve sports skills, such as golf swings, is to imitate professional players’ motions. However, it is difficult for beginners to specify the keyframes on which they should focus and which part of the body they should correct because of inconsistent timing and lack of knowledge. In this study, a golf swing analysis tool using neural networks is proposed to address this gap. The proposed system compares two motion sequences and specifies keyframes in which significant differences can be observed between the two motions. In addition, the system helps users intuitively understand the differences between themselves and professional players by using interpretable clues. The main challenge of this study is to target the fine-grained differences between users and professionals that can be used for self-training. Moreover, the significance of the proposed approach is the use of an unsupervised learning method without prior knowledge and labeled data, which will benefit future applications and research in other sports and skill training processes. In our approach, neural networks are first used to create a motion synchronizer to align motions with different phases and timing. Next, a motion discrepancy detector is implemented to find fine-grained differences between motions in latent spaces that are learned by the networks. Furthermore, we consider that learning intermediate motions may be feasible for beginners because, in this way, they can gradually change their pose to match the ideal form. Therefore, based on the synchronization and discrepancy detection results, we utilize a decoder to restore the intermediate human poses between two motions from the latent space. Finally, we suggest possible applications for analyzing and visualizing the discrepancy between the two input motions and interacting with the users. With the proposed application, users can easily understand the differences between their motions and those of various experts during self-training and learn how to improve their motions.},
  keywords={Machine learning;Computer vision;Synchronization;Neural networks;Three-dimensional displays;Detectors;Motion measurement;Sports;Computer vision;machine learning;motor skill training;golf},
  doi={10.1109/ACCESS.2022.3210261},
  ISSN={2169-3536},
  month={},}

@INPROCEEDINGS{9974362,
  author={Liao, Chen-Chieh and Kikuchi, Haruki and Hwang, Dong-Hyun and Koike, Hideki},
  booktitle={2022 IEEE International Symposium on Mixed and Augmented Reality Adjunct (ISMAR-Adjunct)}, 
  title={Virtual Club Shadow: A Real-time Projection of Golf Club Trajectory}, 
  year={2022},
  volume={},
  number={},
  pages={816-820},
  abstract={In sports like golf, the motion of a club or racket determines the outcome significantly, and it is essential to consistently move such properties with the correct forms to achieve high scores during competitions. This paper proposes a novel projection method that provides real-time visualization of golf club trajectories on the ground. The system can render the trajectory of a golf club along with conventional face-to-path angle information in real-time using an optical motion capture system and a video projector. Furthermore, the system utilizes dynamic changes of color to provide more detailed information about the pose of the golf club. This color-changing method aims to help users understand their club motion trajectories and explicitly adjust their form based on the visualization results.},
  keywords={Training;Visualization;Sports equipment;Optical feedback;Focusing;Streaming media;Real-time systems;Human-centered computing;Human computer interaction (HCI);Interactive systems and tools;Visualization;Visualization design and evaluation methods},
  doi={10.1109/ISMAR-Adjunct57072.2022.00177},
  ISSN={2771-1110},
  month={Oct},}

@inproceedings{10.1145/3532719.3543197,
author = {Wu, Erwin and Liao, Chen-Chieh and Liu, Ruofan and Koike, Hideki},
title = {Context-aware Risk Degree Prediction for Smartphone Zombies},
year = {2022},
isbn = {9781450393614},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
url = {https://doi.org/10.1145/3532719.3543197},
doi = {10.1145/3532719.3543197},
abstract = {Using smartphones while walking is becoming a social problem. Recent works try to support this issue by different warning systems. However, most only focus on detecting obstacles, without considering the risk to the user. In this paper, we propose a deep learning-based context-aware risk prediction system using a built-in camera on smartphones, aiming to notify ”smombies” by a risk-degree based algorithm. The proposed system both estimates the risk degree of a potential obstacle and the user’s status, which can also be used for distracted driving or visually impaired people.},
booktitle = {ACM SIGGRAPH 2022 Posters},
articleno = {48},
numpages = {2},
keywords = {Smombie, Risk Prediction, Deep Learning, Context-aware},
location = {Vancouver, BC, Canada},
series = {SIGGRAPH '22}
}

@inproceedings{10.1145/3532719.3543196,
author = {Liu, Ruofan and Wu, Erwin and Liao, Chen-Chieh and Nishioka, Hayato and Furuya, Shinichi and Koike, Hideki},
title = {Synchronized Hand Difference Visualization for Piano Learning},
year = {2022},
isbn = {9781450393614},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
url = {https://doi.org/10.1145/3532719.3543196},
doi = {10.1145/3532719.3543196},
abstract = {When learning a dexterous skill such as playing the piano, people commonly watch videos of a teacher. However, this conventional way has some downsides such as limited information to be retrieved and less intuitive instructions. We propose a virtual training system by visualizing differences between hands to provide intuitive feedback for skill acquisition. After synchronizing the data, two visual cues are proposed including a hand-overlay manner and a two-keyboards visualization. A pilot study confirm the superiority of the proposed methods over conventional video-viewing.},
booktitle = {ACM SIGGRAPH 2022 Posters},
articleno = {4},
numpages = {2},
keywords = {Motion Analysis, Piano Training, Virtual Visualization},
location = {Vancouver, BC, Canada},
series = {SIGGRAPH '22}
}


@inproceedings{10.1145/3476124.3488645,
author = {Liao, Chen-Chieh and Hwang, Dong-Hyun and Koike, Hideki},
title = {How Can I Swing Like Pro?: Golf Swing Analysis Tool for Self Training},
year = {2021},
isbn = {9781450386876},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
url = {https://doi.org/10.1145/3476124.3488645},
doi = {10.1145/3476124.3488645},
abstract = {In this work, we present an analysis tool to help golf beginners compare their swing motion to the swing motion of experts. The proposed application synchronizes videos with different swing phase timings using the latent features extracted by a neural network-based encoder and detects key frames where discrepant motions occur. We visualize synchronized image frames and 3D poses that help users recognize the differences and key factors that can be important for their swing skill improvement.},
booktitle = {SIGGRAPH Asia 2021 Posters},
articleno = {41},
numpages = {2},
keywords = {motor skill training, machine learning, computer vision},
location = {Tokyo, Japan},
series = {SA '21 Posters}
}


@inproceedings{10.1145/3334480.3383030,
author = {Liao, Chen-Chieh and Koike, Hideki and Nakamura, Takuto},
title = {Realtime Center of Mass Adjustment via Weight Switching Device inside a Golf Putter},
year = {2020},
isbn = {9781450368193},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
url = {https://doi.org/10.1145/3334480.3383030},
doi = {10.1145/3334480.3383030},
abstract = {In order to improve the performance in putting, we design a weight switching device that can provide various switching angles. This paper proposes a system that can change the center of mass by switching the weight to different positions around the head of a putter. The proposed system starts switching the weight at the beginning of a downswing and ends before the putter hits a ball. To verify the effectiveness of the proposed system, we conducted a user study and examined if the face to path angle of the putter changed when the weight was switched to different positions. In the user study, the participant performed putting with different switching angles. In the analysis, we focused on the differences in the face to path angle among the switching angles. The user study showed that the proposed system is effective in changing the face to path angle when switching the weight away from the putter's shaft. Based on the experimental results, the proposed system contributes to affecting the face to path angle dynamically in real-time.},
booktitle = {Extended Abstracts of the 2020 CHI Conference on Human Factors in Computing Systems},
pages = {1–8},
numpages = {8},
keywords = {golf, haptic feedback, putting, weight switching},
location = {Honolulu, HI, USA},
series = {CHI EA '20}
}