Virtual walking piano using YOLO AI
Published on HivePostify by @frugal-fun · Wed Apr 01 2026
Hi.
I'm really excited about this project. I've been working on it for a few days. Well, I say I'm working on it it's mostly been Gemini.using YOLO to create a virtual piano. I hope most people reading this have seen the film BIG with Tom Hanks.
The great thing is it doesn't require much hardware. My laptop has 8 gig of memory and it runs fairly well.
It uses YOLO to detect ankle position and if it's in a box then it plays a note.
I have experimented with a few other instruments including guitar, drums, harp and theremin but I think the piano works the best and is the most fun.
For the time being I'll have to draw the piano on the floor using either tape or chalk. Eventually I hope to get a projector.
I haven't found anyone on YouTube doing anything like this, but there must be somebody. It seems like such a obvious use of YOLO.
I switched my laptop over to Ubuntu just for this. I've been meaning to do it for a few years.
Hopefully everything you need to try this for yourself is below. Have fun!
I'll try to get some screenshots and video up in the next few days.
Frugal
The audio files: https://github.com/RyanHuynh/PianoSamples
Prerequisites: sudo apt update && sudo apt upgrade -y sudo apt install python3-pip python3-venv libgl1-mesa-glx libpulse0 -y
setting up and running a virtual environment if you need to. I did: python3 -m venv pianoenv source pianoenv/bin/activate
pip install ultralytics opencv-python pygame
The code: python import cv2 import numpy as np import pygame import threading from ultralytics import YOLO import os import json
--- 1. SETUP & CONFIG --- SAMPLEDIR = "/home/simon/PianoSamples-master/Sample 2/wav" CONFIGFILE = "pianoconfig.txt"
print("\n--- 🎹 CHROMATIC PIANO ---") octaves = int(input("How many octaves? (1, 2, or 3): ") or 2) camerapos = input("Camera position? ([F]ront or [B]ehind): ").lower() or 'f' toemode = input("Enable Toe-Offset mode? (y/n): ").lower() == 'y'
--- 2. NOTE & GEOMETRY LOGIC --- CHROMATICPATTERN = ["C", "C#", "D", "D#", "E", "F", "F#", "G", "G#", "A", "A#", "B"] notesequence = [] for octval in range(2, 2 + octaves): for note in CHROMATICPATTERN: notesequence.append(f"{note}{octval}") if camerapos == 'f': notesequence = notesequence[::-1]
def loadpts(): if os.path.exists(CONFIGFILE): try: with open(CONFIGFILE, 'r') as f: return np.array(json.load(f), dtype=np.float32) except: pass return np.array([[150, 150], [450, 150], [550, 400], [50, 400]], np.float32)
def savepts(currentpts): with open(CONFIGFILE, 'w') as f: json.dump(currentpts.tolist(), f)
pts = loadpts() selectedpoint = -1 headless = False
--- 3. AUDIO --- pygame.mixer.preinit(44100, -16, 2, 1024) pygame.init() pygame.mixer.setnumchannels(32)
def loadnote(name): filename = name.replace("#", "s") + ".wav" path = os.path.join(SAMPLEDIR, filename) return pygame.mixer.Sound(path) if os.path.exists(path) else None
pianokeys = [loadnote(n) for n in notesequence]
def playasync(sound): if sound: threading.Thread(target=sound.play, daemon=True).start()
--- 4. POLYGON CALCULATION --- def getkeypoly(idx, currentpts): totalnotes = len(notesequence) issharp = "#" in notesequence[idx] whiteslots = octaves 7 slotmap = [0, 0.6, 1, 1.6, 2, 3, 3.6, 4, 4.6, 5, 5.7, 6] lidx = idx if camerapos == 'b' else (totalnotes - 1 - idx) pos = (lidx // 12 7) + slotmap[lidx % 12] if camerapos == 'f': pos = whiteslots - pos - (0.6 if issharp else 1.0) ws, we = pos / whiteslots, (pos + (0.6 if issharp else 1.0)) / whiteslots ve = 0.45 if issharp else 1.0 tL, tR = (1-ws)currentpts[0] + wscurrentpts[1], (1-we)currentpts[0] + wecurrentpts[1] bL, bR = (1-ws)currentpts[3] + wscurrentpts[2], (1-we)currentpts[3] + wecurrentpts[2] return np.array([tL, tR, (1-ve)tR + vebR, (1-ve)tL + vebL], np.int32)
--- 5. MAIN LOOP --- model = YOLO('yolo11n-pose.pt') cap = cv2.VideoCapture(0) cv2.namedWindow("Leonessa Piano")
def mousecallback(event, x, y, flags, param): global pts, selectedpoint if headless: return if event == cv2.EVENTLBUTTONDOWN: for i in range(4): if np.linalg.norm(pts[i] - [x, y]) 0: kpts, conf = results[0].keypoints.xy[0], results[0].keypoints.conf[0] activethisframe = [None, None]
for fidx, tid in enumerate([15, 16]): if conf[tid] > 0.4: ax, ay = kpts[tid][0].item(), kpts[tid][1].item()
TOE OFFSET CALCULATION tx, ty = ax, ay if toemode: Vector from top-center to ankle topmid = (pts[0] + pts[1]) / 2 Push the detection point 20% further away from the top edge ty = ay + (ay - topmid[1]) 0.15
foundidx = None for i in range(len(notesequence)): if "#" in notesequence[i] and cv2.pointPolygonTest(keypolys[i], (tx, ty), False) >= 0: foundidx = i; break if foundidx is None: for i in range(len(notesequence)): if "#" not in notesequence[i] and cv2.pointPolygonTest(keypolys[i], (tx, ty), False) >= 0: foundidx = i; break
activethisframe[fidx] = foundidx if foundidx is not None: cv2.circle(displayframe, (int(tx), int(ty)), 10, (0, 255, 0), -1)
for i in range(2): if activethisframe[i] is not None and activethisframe[i] != currentzones[i]: playasync(pianokeys[activethisframe[i]]) currentzones[i] = activethisframe[i]
cv2.imshow("Leonessa Piano", displayframe) key = cv2.waitKey(1) & 0xFF if key == ord('q'): break elif key == ord('h'): headless = not headless finally: cap.release() cv2.destroyAllWindows()