feat(emulator): fiddling with audio, still very broken

src/emulator.py

@@ -4,15 +4,18 @@ import asyncio
 import logging
 import random
 import time
+from math import ceil
 
+import mgba.audio
 import mgba.core
 import mgba.image
 import mgba.log
 import redis
 
-from ffmpeg_manager import ffmpeg_stream
+from ffmpeg_manager import ffmpeg_audio_stream, ffmpeg_video_stream
 from redis_manager import RedisManager
 from settings import (
+    EMULATOR_FPS,
     EMULATOR_HEIGHT,
     EMULATOR_POLLING_RATE,
     EMULATOR_RAND_RATE,
@@ -58,10 +61,14 @@ async def emulator() -> None:
 
         # save frame to PNG image
         image = screen.to_pil().convert("RGB")
-        image.save(ffmpeg_stream.stdin, "PNG")
+        image.save(ffmpeg_video_stream.stdin, "PNG")
 
-        # TODO: get audio
+        n = ceil(96000 / EMULATOR_FPS)
+        audio_buffer = audio_channels.read(n)
 
+        for short in audio_buffer:
+            bidule = short.to_bytes(2, byteorder="little", signed=True)
+            ffmpeg_audio_stream.write(bidule)
         # sleep until next frame, if necessary
         sleep_t = last_frame_t - time.time() + EMULATOR_SPF
         if sleep_t > 0:
@@ -91,6 +98,7 @@ if __name__ == "__main__":
     core: mgba.core.Core = mgba.core.load_path(EMULATOR_ROM_PATH)
     screen: mgba.image.Image = mgba.image.Image(EMULATOR_WIDTH, EMULATOR_HEIGHT)
     core.set_video_buffer(screen)
+    audio_channels: mgba.audio.StereoBuffer = core.get_audio_channels()
     core.reset()
 
     # setup logging format

src/ffmpeg_manager.py

@@ -12,37 +12,100 @@ from settings import (
 )
 
 # launch ffmpeg process
-ffmpeg_stream = subprocess.Popen(
+ffmpeg_video_stream = subprocess.Popen(
     [
-        "/usr/bin/ffmpeg",
+        "/usr/bin/ffmpeg",  # ffmpeg binary location
-        "-y",
+        "-y",  # overwrite output files without asking
-        "-f",
+        "-f",  # force input file format
-        "image2pipe",
+        "image2pipe",  # allows to pipe images to ffmpeg
-        "-vcodec",
+        "-vcodec",  # set the video codec
-        "png",
+        "png",  # input images are PNGs
-        "-r",
+        "-r",  # set input frame rate
         f"{EMULATOR_FPS}",
-        "-s",
+        "-s",  # set input frame size
         f"{EMULATOR_WIDTH}x{EMULATOR_HEIGHT}",
+        "-i",  # input file url
+        "pipe:0",  # use stdin (pipe n°0) for input
+        "-f",  # force input or output file format
+        "flv",  # output an flv video
+        "-s",  # set output frame size
+        f"{FFMPEG_WIDTH}x{FFMPEG_HEIGHT}",
+        "-r",  # set output frame rate
+        f"{FFMPEG_FPS}",
+        "-b:v",  # set video output bitrate
+        FFMPEG_BITRATE,
+        "-fflags",  # set format flags
+        "nobuffer",  # reduce the latency introduced by buffering during initial input streams analysis
+        "-flags",  # set generic flags
+        "low_delay",  # force low delay
+        "-strict",  # specify how strictly to follow the standards
+        "experimental",  # allow non standardized experimental encoders...
+        RTMP_STREAM_URI,  # where to output the video
+    ],
+    stdin=subprocess.PIPE,
+    stdout=subprocess.DEVNULL,
+    stderr=subprocess.STDOUT,
+)
+
+ffmpeg_audio_stream = subprocess.Popen(
+    # [
+    #     "/usr/bin/ffmpeg",  # FFMPEG_BIN
+    #     "-i",
+    #     "-",
+    #     "-f",
+    #     "s24le",
+    #     "-acodec",
+    #     "pcm_s24le",
+    #     "-ar",
+    #     "44100",  # ouput will have 44100 Hz
+    #     "-ac",
+    #     "2",  # stereo (set to '1' for mono)
+    #     RTMP_STREAM_URI,
+    # ],
+    # [
+    #     "/usr/bin/ffmpeg",  # FFMPEG_BIN
+    #     "-f",  # audio format
+    #     "s24le",
+    #     "-acodec",  # audio codec
+    #     "pcm_s24le",
+    #     "-ar",  # audio sampling rate
+    #     "44100",
+    #     "-ac",  # number of audio canals
+    #     "2",  # stereo (set to '1' for mono)
+    #     "-i",
+    #     "pipe:",
+    #     "-y",
+    #     "-f",
+    #     "flv",
+    #     # "-tune",
+    #     # "zerolatency",
+    #     "-fflags",  # set format flags
+    #     "nobuffer",  # reduce the latency introduced by buffering during initial input streams analysis
+    #     "-flags",  # set generic flags
+    #     "low_delay",  # force low delay
+    #     "-strict",  # specify how strictly to follow the standards
+    #     "experimental",  # allow non standardized experimental things
+    #     RTMP_STREAM_URI,
+    # ],
+    [
+        "/usr/bin/ffmpeg",  # FFMPEG_BIN
+        "-f",
+        "-y",
+        "s24le",
+        "-acodec",
+        "pcm_s24le",
+        "-ar",
+        "44100",  # ouput will have 44100 Hz
+        "-ac",
+        "2",  # stereo (set to '1' for mono)
         "-i",
         "-",
         "-f",
-        "flv",
+        "bonjour",
-        "-s",
-        f"{FFMPEG_WIDTH}x{FFMPEG_HEIGHT}",
-        "-r",
-        f"{FFMPEG_FPS}",
-        "-b:v",
-        FFMPEG_BITRATE,
-        "-fflags",
-        "nobuffer",
-        "-flags",
-        "low_delay",
-        "-strict",
-        "experimental",
-        RTMP_STREAM_URI,
     ],
     stdin=subprocess.PIPE,
-    stdout=subprocess.PIPE,
+    # stdout=subprocess.STDOUT,
-    stderr=subprocess.STDOUT,
+    # stderr=subprocess.STDOUT,
 )
+
+# https://stackoverflow.com/questions/67388548/multiple-named-pipes-in-ffmpeg