Fix some small protocol inconsistencies (#1772)

proxy/src/main/java/com/velocitypowered/proxy/connection/MinecraftConnection.java

@@ -67,7 +67,7 @@ import io.netty.util.ReferenceCountUtil;
 import java.net.InetSocketAddress;
 import java.net.SocketAddress;
 import java.security.GeneralSecurityException;
-import java.util.HashMap;
+import java.util.EnumMap;
 import java.util.Map;
 import java.util.Objects;
 import java.util.concurrent.TimeUnit;
@@ -109,7 +109,7 @@ public class MinecraftConnection extends ChannelInboundHandlerAdapter {
     this.server = server;
     this.state = StateRegistry.HANDSHAKE;
 
-    this.sessionHandlers = new HashMap<>();
+    this.sessionHandlers = new EnumMap<>(StateRegistry.class);
   }
 
   @Override

proxy/src/main/java/com/velocitypowered/proxy/protocol/netty/MinecraftDecoder.java

@@ -57,7 +57,11 @@ public class MinecraftDecoder extends ChannelInboundHandlerAdapter {
   @Override
   public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception {
     if (msg instanceof ByteBuf buf) {
+      try {
         tryDecode(ctx, buf);
+      } finally {
+        buf.release();
+      }
     } else {
       ctx.fireChannelRead(msg);
     }
@@ -65,7 +69,6 @@ public class MinecraftDecoder extends ChannelInboundHandlerAdapter {
 
   private void tryDecode(ChannelHandlerContext ctx, ByteBuf buf) throws Exception {
     if (!ctx.channel().isActive() || !buf.isReadable()) {
-      buf.release();
       return;
     }
 
@@ -75,12 +78,10 @@ public class MinecraftDecoder extends ChannelInboundHandlerAdapter {
     if (packet == null) {
       buf.readerIndex(originalReaderIndex);
       if (this.direction == ProtocolUtils.Direction.SERVERBOUND && this.state != StateRegistry.PLAY) {
-        buf.release();
         throw this.handleInvalidPacketId(packetId);
       }
-      ctx.fireChannelRead(buf);
+      ctx.fireChannelRead(buf.retain());
     } else {
-      try {
       doLengthSanityChecks(buf, packet);
 
       try {
@@ -93,9 +94,6 @@ public class MinecraftDecoder extends ChannelInboundHandlerAdapter {
         throw handleOverflow(packet, buf.readerIndex(), buf.writerIndex());
       }
       ctx.fireChannelRead(packet);
-      } finally {
-        buf.release();
-      }
     }
   }
 

proxy/src/main/java/com/velocitypowered/proxy/protocol/netty/MinecraftVarintFrameDecoder.java

@@ -94,7 +94,6 @@ public class MinecraftVarintFrameDecoder extends ByteToMessageDecoder {
     in.readerIndex(packetStart);
 
     // try to read the length of the packet
-    in.markReaderIndex();
     try {
       int length = readRawVarInt21(in);
       if (packetStart == in.readerIndex()) {
@@ -107,6 +106,7 @@ public class MinecraftVarintFrameDecoder extends ByteToMessageDecoder {
       if (length > 0) {
         if (state == StateRegistry.HANDSHAKE && direction == ProtocolUtils.Direction.SERVERBOUND) {
           if (validateServerboundHandshakePacket(in, length)) {
+            in.readerIndex(packetStart);
             return;
           }
         }
@@ -115,7 +115,7 @@ public class MinecraftVarintFrameDecoder extends ByteToMessageDecoder {
       // note that zero-length packets are ignored
       if (length > 0) {
         if (in.readableBytes() < length) {
-          in.resetReaderIndex();
+          in.readerIndex(packetStart);
         } else {
           // If enabled, rate-limit serverbound payload bytes based on frame length
           if (packetLimiter != null) {
@@ -130,7 +130,7 @@ public class MinecraftVarintFrameDecoder extends ByteToMessageDecoder {
       }
     } catch (Exception e) {
       // Reset buffer to consistent state before propagating exception to prevent memory leaks
-      in.resetReaderIndex();
+      in.readerIndex(packetStart);
       throw e;
     }
   }
@@ -140,11 +140,9 @@ public class MinecraftVarintFrameDecoder extends ByteToMessageDecoder {
         state.getProtocolRegistry(direction, ProtocolVersion.MINIMUM_VERSION);
 
     final int index = in.readerIndex();
-    try {
     final int packetId = readRawVarInt21(in);
     // Index hasn't changed, we've read nothing
     if (index == in.readerIndex()) {
-        in.resetReaderIndex();
       return true;
     }
     final int payloadLength = length - ProtocolUtils.varIntBytes(packetId);
@@ -169,11 +167,6 @@ public class MinecraftVarintFrameDecoder extends ByteToMessageDecoder {
 
     in.readerIndex(index);
     return false;
-    } catch (Exception e) {
-      // Reset buffer to consistent state before propagating exception to prevent memory leaks
-      in.readerIndex(index);
-      throw e;
-    }
   }
 
   @Override