Add arbitrary audio support to speakers (#982)

Speakers can now play arbitrary PCM audio, sampled at 48kHz and with a
resolution of 8 bits. Programs can build up buffers of audio locally,
play it using `speaker.playAudio`, where it is encoded to DFPWM, sent
across the network, decoded, and played on the client.

`speaker.playAudio` may return false when a chunk of audio has been
submitted but not yet sent to the client. In this case, the program
should wait for a speaker_audio_empty event and try again, repeating
until it works.

While the API is a little odd, this gives us fantastic flexibility (we
can play arbitrary streams of audio) while still being resilient in the
presence of server lag (either TPS or on the computer thread).

Some other notes:
 - There is a significant buffer on both the client and server, which
   means that sound take several seconds to finish after playing has
   started. One can force it to be stopped playing with the new
  `speaker.stop` call.

 - This also adds a `cc.audio.dfpwm` module, which allows encoding and
   decoding DFPWM1a audio files.

 - I spent so long writing the documentation for this. Who knows if it'll
   be helpful!

src/main/java/dan200/computercraft/client/ClientHooks.java

@@ -6,6 +6,7 @@
 package dan200.computercraft.client;
 
 import dan200.computercraft.ComputerCraft;
+import dan200.computercraft.client.sound.SpeakerManager;
 import dan200.computercraft.shared.peripheral.monitor.ClientMonitor;
 import net.minecraftforge.api.distmarker.Dist;
 import net.minecraftforge.client.event.ClientPlayerNetworkEvent;
@@ -22,7 +23,7 @@ public class ClientHooks
         if( event.getWorld().isClientSide() )
         {
             ClientMonitor.destroyAll();
-            SoundManager.reset();
+            SpeakerManager.reset();
         }
     }
 

src/main/java/dan200/computercraft/client/SoundManager.java

@@ -1,85 +0,0 @@
-/*
- * This file is part of ComputerCraft - http://www.computercraft.info
- * Copyright Daniel Ratcliffe, 2011-2021. Do not distribute without permission.
- * Send enquiries to dratcliffe@gmail.com
- */
-package dan200.computercraft.client;
-
-import net.minecraft.client.Minecraft;
-import net.minecraft.client.audio.ISound;
-import net.minecraft.client.audio.ITickableSound;
-import net.minecraft.client.audio.LocatableSound;
-import net.minecraft.client.audio.SoundHandler;
-import net.minecraft.util.ResourceLocation;
-import net.minecraft.util.SoundCategory;
-import net.minecraft.util.math.vector.Vector3d;
-
-import java.util.HashMap;
-import java.util.Map;
-import java.util.UUID;
-
-public class SoundManager
-{
-    private static final Map<UUID, MoveableSound> sounds = new HashMap<>();
-
-    public static void playSound( UUID source, Vector3d position, ResourceLocation event, float volume, float pitch )
-    {
-        SoundHandler soundManager = Minecraft.getInstance().getSoundManager();
-
-        MoveableSound oldSound = sounds.get( source );
-        if( oldSound != null ) soundManager.stop( oldSound );
-
-        MoveableSound newSound = new MoveableSound( event, position, volume, pitch );
-        sounds.put( source, newSound );
-        soundManager.play( newSound );
-    }
-
-    public static void stopSound( UUID source )
-    {
-        ISound sound = sounds.remove( source );
-        if( sound == null ) return;
-
-        Minecraft.getInstance().getSoundManager().stop( sound );
-    }
-
-    public static void moveSound( UUID source, Vector3d position )
-    {
-        MoveableSound sound = sounds.get( source );
-        if( sound != null ) sound.setPosition( position );
-    }
-
-    public static void reset()
-    {
-        sounds.clear();
-    }
-
-    private static class MoveableSound extends LocatableSound implements ITickableSound
-    {
-        protected MoveableSound( ResourceLocation sound, Vector3d position, float volume, float pitch )
-        {
-            super( sound, SoundCategory.RECORDS );
-            setPosition( position );
-            this.volume = volume;
-            this.pitch = pitch;
-            attenuation = ISound.AttenuationType.LINEAR;
-        }
-
-        void setPosition( Vector3d position )
-        {
-            x = (float) position.x();
-            y = (float) position.y();
-            z = (float) position.z();
-        }
-
-        @Override
-        public boolean isStopped()
-        {
-            return false;
-        }
-
-        @Override
-        public void tick()
-        {
-        }
-    }
-}

src/main/java/dan200/computercraft/client/sound/DfpwmStream.java

@@ -0,0 +1,125 @@
+/*
+ * This file is part of ComputerCraft - http://www.computercraft.info
+ * Copyright Daniel Ratcliffe, 2011-2021. Do not distribute without permission.
+ * Send enquiries to dratcliffe@gmail.com
+ */
+package dan200.computercraft.client.sound;
+
+import dan200.computercraft.shared.peripheral.speaker.SpeakerPeripheral;
+import io.netty.buffer.ByteBuf;
+import net.minecraft.client.audio.IAudioStream;
+import org.lwjgl.BufferUtils;
+
+import javax.annotation.Nonnull;
+import javax.sound.sampled.AudioFormat;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.util.ArrayDeque;
+import java.util.Queue;
+
+class DfpwmStream implements IAudioStream
+{
+    public static final int SAMPLE_RATE = SpeakerPeripheral.SAMPLE_RATE;
+
+    private static final int PREC = 10;
+    private static final int LPF_STRENGTH = 140;
+
+    private static final AudioFormat MONO_16 = new AudioFormat( SAMPLE_RATE, 16, 1, true, false );
+
+    private final Queue<ByteBuffer> buffers = new ArrayDeque<>( 2 );
+
+    private int charge = 0; // q
+    private int strength = 0; // s
+    private int lowPassCharge;
+    private boolean previousBit = false;
+
+    DfpwmStream()
+    {
+    }
+
+    void push( @Nonnull ByteBuf input )
+    {
+        int readable = input.readableBytes();
+        ByteBuffer output = ByteBuffer.allocate( readable * 16 ).order( ByteOrder.nativeOrder() );
+
+        for( int i = 0; i < readable; i++ )
+        {
+            byte inputByte = input.readByte();
+            for( int j = 0; j < 8; j++ )
+            {
+                boolean currentBit = (inputByte & 1) != 0;
+                int target = currentBit ? 127 : -128;
+
+                // q' <- q + (s * (t - q) + 128)/256
+                int nextCharge = charge + ((strength * (target - charge) + (1 << (PREC - 1))) >> PREC);
+                if( nextCharge == charge && nextCharge != target ) nextCharge += currentBit ? 1 : -1;
+
+                int z = currentBit == previousBit ? (1 << PREC) - 1 : 0;
+
+                int nextStrength = strength;
+                if( strength != z ) nextStrength += currentBit == previousBit ? 1 : -1;
+                if( nextStrength < 2 << (PREC - 8) ) nextStrength = 2 << (PREC - 8);
+
+                // Apply antijerk
+                int chargeWithAntijerk = currentBit == previousBit
+                    ? nextCharge
+                    : nextCharge + charge + 1 >> 1;
+
+                // And low pass filter: outQ <- outQ + ((expectedOutput - outQ) x 140 / 256)
+                lowPassCharge += ((chargeWithAntijerk - lowPassCharge) * LPF_STRENGTH + 0x80) >> 8;
+
+                charge = nextCharge;
+                strength = nextStrength;
+                previousBit = currentBit;
+
+                // Ideally we'd generate an 8-bit audio buffer. However, as we're piggybacking on top of another
+                // audio stream (which uses 16 bit audio), we need to keep in the same format.
+                output.putShort( (short) ((byte) (lowPassCharge & 0xFF) << 8) );
+
+                inputByte >>= 1;
+            }
+        }
+
+        output.flip();
+        synchronized( this )
+        {
+            buffers.add( output );
+        }
+    }
+
+    @Nonnull
+    @Override
+    public AudioFormat getFormat()
+    {
+        return MONO_16;
+    }
+
+    @Nonnull
+    @Override
+    public synchronized ByteBuffer read( int capacity )
+    {
+        ByteBuffer result = BufferUtils.createByteBuffer( capacity );
+        while( result.hasRemaining() )
+        {
+            ByteBuffer head = buffers.peek();
+            if( head == null ) break;
+
+            int toRead = Math.min( head.remaining(), result.remaining() );
+            result.put( head.array(), head.position(), toRead ); // TODO: In 1.17 convert this to a ByteBuffer override
+            head.position( head.position() + toRead );
+
+            if( head.hasRemaining() ) break;
+            buffers.remove();
+        }
+
+        result.flip();
+        return result;
+    }
+
+    @Override
+    public void close() throws IOException
+    {
+        buffers.clear();
+    }
+}

src/main/java/dan200/computercraft/client/sound/SpeakerInstance.java

@@ -0,0 +1,81 @@
+/*
+ * This file is part of ComputerCraft - http://www.computercraft.info
+ * Copyright Daniel Ratcliffe, 2011-2021. Do not distribute without permission.
+ * Send enquiries to dratcliffe@gmail.com
+ */
+package dan200.computercraft.client.sound;
+
+import dan200.computercraft.ComputerCraft;
+import io.netty.buffer.ByteBuf;
+import net.minecraft.client.Minecraft;
+import net.minecraft.client.audio.SoundHandler;
+import net.minecraft.util.ResourceLocation;
+import net.minecraft.util.math.vector.Vector3d;
+
+/**
+ * An instance of a speaker, which is either playing a {@link DfpwmStream} stream or a normal sound.
+ */
+public class SpeakerInstance
+{
+    public static final ResourceLocation DFPWM_STREAM = new ResourceLocation( ComputerCraft.MOD_ID, "speaker.dfpwm_fake_audio_should_not_be_played" );
+
+    private DfpwmStream currentStream;
+    private SpeakerSound sound;
+
+    SpeakerInstance()
+    {
+    }
+
+    public synchronized void pushAudio( ByteBuf buffer )
+    {
+        if( currentStream == null ) currentStream = new DfpwmStream();
+        currentStream.push( buffer );
+    }
+
+    public void playAudio( Vector3d position, float volume )
+    {
+        SoundHandler soundManager = Minecraft.getInstance().getSoundManager();
+
+        if( sound != null && sound.stream != currentStream )
+        {
+            soundManager.stop( sound );
+            sound = null;
+        }
+
+        if( sound != null && !soundManager.isActive( sound ) ) sound = null;
+
+        if( sound == null && currentStream != null )
+        {
+            sound = new SpeakerSound( DFPWM_STREAM, currentStream, position, volume, 1.0f );
+            soundManager.play( sound );
+        }
+    }
+
+    public void playSound( Vector3d position, ResourceLocation location, float volume, float pitch )
+    {
+        SoundHandler soundManager = Minecraft.getInstance().getSoundManager();
+        currentStream = null;
+
+        if( sound != null )
+        {
+            soundManager.stop( sound );
+            sound = null;
+        }
+
+        sound = new SpeakerSound( location, null, position, volume, pitch );
+        soundManager.play( sound );
+    }
+
+    void setPosition( Vector3d position )
+    {
+        if( sound != null ) sound.setPosition( position );
+    }
+
+    void stop()
+    {
+        if( sound != null ) Minecraft.getInstance().getSoundManager().stop( sound );
+
+        currentStream = null;
+        sound = null;
+    }
+}

src/main/java/dan200/computercraft/client/sound/SpeakerManager.java

@@ -0,0 +1,58 @@
+/*
+ * This file is part of ComputerCraft - http://www.computercraft.info
+ * Copyright Daniel Ratcliffe, 2011-2021. Do not distribute without permission.
+ * Send enquiries to dratcliffe@gmail.com
+ */
+package dan200.computercraft.client.sound;
+
+import net.minecraft.util.math.vector.Vector3d;
+import net.minecraftforge.api.distmarker.Dist;
+import net.minecraftforge.client.event.sound.PlayStreamingSourceEvent;
+import net.minecraftforge.eventbus.api.SubscribeEvent;
+import net.minecraftforge.fml.common.Mod;
+
+import java.util.Map;
+import java.util.UUID;
+import java.util.concurrent.ConcurrentHashMap;
+
+/**
+ * Maps speakers source IDs to a {@link SpeakerInstance}.
+ */
+@Mod.EventBusSubscriber( Dist.CLIENT )
+public class SpeakerManager
+{
+    private static final Map<UUID, SpeakerInstance> sounds = new ConcurrentHashMap<>();
+
+    @SubscribeEvent
+    public static void playStreaming( PlayStreamingSourceEvent event )
+    {
+        if( !(event.getSound() instanceof SpeakerSound) ) return;
+        SpeakerSound sound = (SpeakerSound) event.getSound();
+        if( sound.stream == null ) return;
+
+        event.getSource().attachBufferStream( sound.stream );
+        event.getSource().play();
+    }
+
+    public static SpeakerInstance getSound( UUID source )
+    {
+        return sounds.computeIfAbsent( source, x -> new SpeakerInstance() );
+    }
+
+    public static void stopSound( UUID source )
+    {
+        SpeakerInstance sound = sounds.remove( source );
+        if( sound != null ) sound.stop();
+    }
+
+    public static void moveSound( UUID source, Vector3d position )
+    {
+        SpeakerInstance sound = sounds.get( source );
+        if( sound != null ) sound.setPosition( position );
+    }
+
+    public static void reset()
+    {
+        sounds.clear();
+    }
+}

src/main/java/dan200/computercraft/client/sound/SpeakerSound.java

@@ -0,0 +1,54 @@
+/*
+ * This file is part of ComputerCraft - http://www.computercraft.info
+ * Copyright Daniel Ratcliffe, 2011-2021. Do not distribute without permission.
+ * Send enquiries to dratcliffe@gmail.com
+ */
+package dan200.computercraft.client.sound;
+
+import net.minecraft.client.audio.IAudioStream;
+import net.minecraft.client.audio.ITickableSound;
+import net.minecraft.client.audio.LocatableSound;
+import net.minecraft.util.ResourceLocation;
+import net.minecraft.util.SoundCategory;
+import net.minecraft.util.math.vector.Vector3d;
+
+import javax.annotation.Nullable;
+
+public class SpeakerSound extends LocatableSound implements ITickableSound
+{
+    DfpwmStream stream;
+
+    SpeakerSound( ResourceLocation sound, DfpwmStream stream, Vector3d position, float volume, float pitch )
+    {
+        super( sound, SoundCategory.RECORDS );
+        setPosition( position );
+        this.stream = stream;
+        this.volume = volume;
+        this.pitch = pitch;
+        attenuation = AttenuationType.LINEAR;
+    }
+
+    void setPosition( Vector3d position )
+    {
+        x = (float) position.x();
+        y = (float) position.y();
+        z = (float) position.z();
+    }
+
+    @Override
+    public boolean isStopped()
+    {
+        return false;
+    }
+
+    @Override
+    public void tick()
+    {
+    }
+
+    @Nullable
+    public IAudioStream getStream()
+    {
+        return stream;
+    }
+}