diff --git a/.gitmodules b/.gitmodules index 5bb41c46a..d6fc37ab1 100644 --- a/.gitmodules +++ b/.gitmodules @@ -13,3 +13,6 @@ [submodule "lib/SDL3-CS"] path = lib/SDL3-CS url = https://github.com/flibitijibibo/SDL3-CS.git +[submodule "lib/dav1dfile"] + path = lib/dav1dfile + url = https://github.com/MoonsideGames/dav1dfile diff --git a/FNA.Core.csproj b/FNA.Core.csproj index e2e1f7304..3d335e2f8 100644 --- a/FNA.Core.csproj +++ b/FNA.Core.csproj @@ -331,6 +331,7 @@ + @@ -360,6 +361,7 @@ + @@ -386,5 +388,8 @@ Microsoft.Xna.Framework.Graphics.Effect.Resources.YUVToRGBAEffect.fxb + + Microsoft.Xna.Framework.Graphics.Effect.Resources.YUVToRGBAEffectR.fxb + diff --git a/FNA.NetFramework.csproj b/FNA.NetFramework.csproj index 81828c3c7..eb2522d0f 100644 --- a/FNA.NetFramework.csproj +++ b/FNA.NetFramework.csproj @@ -332,6 +332,7 @@ + @@ -361,6 +362,7 @@ + @@ -387,5 +389,8 @@ Microsoft.Xna.Framework.Graphics.Effect.Resources.YUVToRGBAEffect.fxb + + Microsoft.Xna.Framework.Graphics.Effect.Resources.YUVToRGBAEffectR.fxb + diff --git a/FNA.NetStandard.csproj b/FNA.NetStandard.csproj index 73f4cedb3..db9c3d617 100644 --- a/FNA.NetStandard.csproj +++ b/FNA.NetStandard.csproj @@ -330,6 +330,7 @@ + @@ -359,6 +360,7 @@ + @@ -385,5 +387,8 @@ Microsoft.Xna.Framework.Graphics.Effect.Resources.YUVToRGBAEffect.fxb + + Microsoft.Xna.Framework.Graphics.Effect.Resources.YUVToRGBAEffectR.fxb + diff --git a/FNA.csproj b/FNA.csproj index a736b1d8c..2258ad844 100644 --- a/FNA.csproj +++ b/FNA.csproj @@ -402,6 +402,7 @@ + @@ -431,6 +432,7 @@ + @@ -460,5 +462,8 @@ Microsoft.Xna.Framework.Graphics.Effect.Resources.YUVToRGBAEffect.fxb + + Microsoft.Xna.Framework.Graphics.Effect.Resources.YUVToRGBAEffectR.fxb + diff --git a/Makefile b/Makefile index fb602194c..d2d348875 100644 --- a/Makefile +++ b/Makefile @@ -316,6 +316,7 @@ SRC = \ src/Media/Xiph/Video.cs \ src/Media/Xiph/VideoPlayer.cs \ src/Media/Xiph/VideoPlayerTheora.cs \ + src/Media/Xiph/VideoPlayerAV1.cs \ src/NamespaceDocs.cs \ src/Plane.cs \ src/PlaneIntersectionType.cs \ @@ -342,7 +343,8 @@ SRC = \ lib/SDL2-CS/src/SDL2.cs \ lib/SDL3-CS/SDL3/SDL3.Legacy.cs \ lib/FAudio/csharp/FAudio.cs \ - lib/Theorafile/csharp/Theorafile.cs + lib/Theorafile/csharp/Theorafile.cs \ + lib/dav1dfile/csharp/dav1dfile.Legacy.cs RESDIR = src/Graphics/Effect/StockEffects/FXB RESNAME = Microsoft.Xna.Framework.Graphics.Effect.Resources @@ -353,7 +355,8 @@ RES = \ -resource:$(RESDIR)/EnvironmentMapEffect.fxb,$(RESNAME).EnvironmentMapEffect.fxb \ -resource:$(RESDIR)/SkinnedEffect.fxb,$(RESNAME).SkinnedEffect.fxb \ -resource:$(RESDIR)/SpriteEffect.fxb,$(RESNAME).SpriteEffect.fxb \ - -resource:src/Graphics/Effect/YUVToRGBA/YUVToRGBAEffect.fxb,$(RESNAME).YUVToRGBAEffect.fxb + -resource:src/Graphics/Effect/YUVToRGBA/YUVToRGBAEffect.fxb,$(RESNAME).YUVToRGBAEffect.fxb \ + -resource:src/Graphics/Effect/YUVToRGBA/YUVToRGBAEffectR.fxb,$(RESNAME).YUVToRGBAEffectR.fxb # Targets diff --git a/app.config b/app.config index 3f208db94..53d54b016 100644 --- a/app.config +++ b/app.config @@ -12,6 +12,10 @@ + + + + diff --git a/lib/dav1dfile b/lib/dav1dfile new file mode 160000 index 000000000..ed01b751f --- /dev/null +++ b/lib/dav1dfile @@ -0,0 +1 @@ +Subproject commit ed01b751f9b1b17eb0af48fb4af4282366167416 diff --git a/src/Graphics/Effect/Resources.cs b/src/Graphics/Effect/Resources.cs index ea8160e74..e70f4e84a 100644 --- a/src/Graphics/Effect/Resources.cs +++ b/src/Graphics/Effect/Resources.cs @@ -102,6 +102,18 @@ public static byte[] YUVToRGBAEffect } } + public static byte[] YUVToRGBAEffectR + { + get + { + if (yuvToRGBAEffectR == null) + { + yuvToRGBAEffectR = GetResource("YUVToRGBAEffectR"); + } + return yuvToRGBAEffectR; + } + } + #endregion #region Private Static Variables @@ -113,6 +125,7 @@ public static byte[] YUVToRGBAEffect private static byte[] skinnedEffect; private static byte[] spriteEffect; private static byte[] yuvToRGBAEffect; + private static byte[] yuvToRGBAEffectR; #endregion diff --git a/src/Graphics/Effect/YUVToRGBA/YUVToRGBAEffectR.fx b/src/Graphics/Effect/YUVToRGBA/YUVToRGBAEffectR.fx new file mode 100644 index 000000000..867f38abc --- /dev/null +++ b/src/Graphics/Effect/YUVToRGBA/YUVToRGBAEffectR.fx @@ -0,0 +1,64 @@ +/* YUV-to-RGBA Effect + * Written by Ethan "flibitijibibo" Lee + * http://www.flibitijibibo.com/ + * + * This effect is based on the YUV-to-RGBA GLSL shader found in SDL. + * Thus, it also released under the zlib license: + * http://libsdl.org/license.php + */ + +sampler samp0 : register(s0); +sampler samp1 : register(s1); +sampler samp2 : register(s2); + +float4 RescaleFactor : register(ps, c0) = float4(1, 1, 1, 1); + +void VS(inout float2 tex : TEXCOORD0, + inout float4 pos : SV_Position) +{ + pos.w = 1.0; +} + +float4 PS(float2 tex : TEXCOORD0) : SV_Target0 +{ + const float3 offset = float3(-0.0625, -0.5, -0.5); + + /* More info about colorspace conversion: + * http://www.equasys.de/colorconversion.html + * http://www.equasys.de/colorformat.html + */ +#if 1 + /* ITU-R BT.709 */ + const float3 Rcoeff = float3(1.164, 0.000, 1.793); + const float3 Gcoeff = float3(1.164, -0.213, -0.533); + const float3 Bcoeff = float3(1.164, 2.112, 0.000); +#else + /* ITU-R BT.601 */ + const float3 Rcoeff = float3(1.164, 0.000, 1.596); + const float3 Gcoeff = float3(1.164, -0.391, -0.813); + const float3 Bcoeff = float3(1.164, 2.018, 0.000); +#endif + + float3 yuv; + yuv.x = tex2D(samp0, tex).r; + yuv.y = tex2D(samp1, tex).r; + yuv.z = tex2D(samp2, tex).r; + yuv *= RescaleFactor; + yuv += offset; + + float4 rgba; + rgba.x = dot(yuv, Rcoeff); + rgba.y = dot(yuv, Gcoeff); + rgba.z = dot(yuv, Bcoeff); + rgba.w = 1.0; + return rgba; +} + +Technique T +{ + Pass P + { + VertexShader = compile vs_3_0 VS(); + PixelShader = compile ps_3_0 PS(); + } +} diff --git a/src/Graphics/Effect/YUVToRGBA/YUVToRGBAEffectR.fxb b/src/Graphics/Effect/YUVToRGBA/YUVToRGBAEffectR.fxb new file mode 100644 index 000000000..ba6b911f8 Binary files /dev/null and b/src/Graphics/Effect/YUVToRGBA/YUVToRGBAEffectR.fxb differ diff --git a/src/Media/Xiph/VideoPlayer.cs b/src/Media/Xiph/VideoPlayer.cs index 6b9f9d5ed..eb9295a1c 100644 --- a/src/Media/Xiph/VideoPlayer.cs +++ b/src/Media/Xiph/VideoPlayer.cs @@ -121,18 +121,22 @@ public float Volume internal static Dictionary codecExtensions = new Dictionary { + { "obu", "AV1" }, + { "av1", "AV1" }, { "ogv", "Theora" } }; internal static Dictionary> codecInfoReaders = new Dictionary> { + { "AV1", VideoPlayerAV1.ReadInfo }, { "Theora", VideoPlayerTheora.ReadInfo } }; internal static Dictionary> codecPlayers = new Dictionary> { + { "AV1", () => new VideoPlayerAV1() }, { "Theora", () => new VideoPlayerTheora() }, }; diff --git a/src/Media/Xiph/VideoPlayerAV1.cs b/src/Media/Xiph/VideoPlayerAV1.cs new file mode 100644 index 000000000..97f333b01 --- /dev/null +++ b/src/Media/Xiph/VideoPlayerAV1.cs @@ -0,0 +1,502 @@ +using System; +using System.Diagnostics; +using Microsoft.Xna.Framework.Graphics; +using Dav1dfile; +using SDL3; + +namespace Microsoft.Xna.Framework.Media +{ + internal unsafe class VideoPlayerAV1 : BaseYUVPlayer, IVideoPlayerCodec, IDisposable + { + #region Public Member Data: XNA VideoPlayer Implementation + + public bool IsLooped + { + get; + set; + } + + public bool IsMuted + { + get + { + return false; + } + set + { + // no-op + } + } + + public TimeSpan PlayPosition + { + get + { + return timer.Elapsed; + } + } + + public MediaState State + { + get; + private set; + } + + public Video Video + { + get; + private set; + } + + public float Volume + { + get + { + return 0.0f; + } + set + { + // no-op + } + } + + #endregion + + #region Private Member Data: XNA VideoPlayer Implementation + + // We use this to update our PlayPosition. + private Stopwatch timer; + + #endregion + + #region Private Member Data: dav1dfile + + private IntPtr context; + private Bindings.PixelLayout pixelLayout; + private int bitsPerPixel; + + private int currentFrame; + private double fps; + + #endregion + + #region Public Methods: XNA VideoPlayer Implementation + + public VideoPlayerAV1() + { + // Initialize public members. + IsLooped = false; + IsMuted = true; + State = MediaState.Stopped; + Volume = 0.0f; + + // Initialize private members. + timer = new Stopwatch(); + } + + public void Dispose() + { + if (IsDisposed) + { + return; + } + + // Stop the VideoPlayer. This gets almost everything... + Stop(); + + // Destroy the underlying GL bits. + base.Dispose(); + + if (context != IntPtr.Zero) + { + Bindings.df_close(context); + } + } + + public Texture2D GetTexture() + { + checkDisposed(); + + if (Video == null) + { + throw new InvalidOperationException(); + } + + // Be sure we can even get something... + if ( State == MediaState.Stopped || + context == IntPtr.Zero ) + { + // Screw it, give them the old one. + return videoTexture[0].RenderTarget as Texture2D; + } + + int thisFrame = (int) (timer.Elapsed.TotalMilliseconds / (1000.0 / fps)); + if (thisFrame > currentFrame) + { + // Only update the textures if we need to! + if (DecodeAndUpdateFrame(thisFrame - currentFrame) || currentFrame == -1) + { + float rescaleFactor; + if (bitsPerPixel == 12) + { + rescaleFactor = (float) (1.0 / (4096 / 65536.0)); + } + else if (bitsPerPixel == 10) + { + rescaleFactor = (float) (1.0 / (1024 / 65536.0)); + } + else + { + rescaleFactor = 1.0f; + } + + shaderProgram.Parameters["RescaleFactor"] + .SetValue(new Vector4(rescaleFactor, rescaleFactor, rescaleFactor, 1.0f)); + + // Draw the YUV textures to the framebuffer with our shader. + GL_pushState(); + currentDevice.DrawPrimitives( + PrimitiveType.TriangleStrip, + 0, + 2 + ); + GL_popState(); + } + currentFrame = thisFrame; + } + + // Check for the end... + bool ended = Bindings.df_eos(context) == 1; + if (ended) + { + // FIXME: This is part of the Duration hack! + if (Video.needsDurationHack) + { + Video.Duration = timer.Elapsed; // FIXME: Frames * FPS? -flibit + } + + // Stop and reset the timer. If we're looping, the loop will start it again. + timer.Stop(); + timer.Reset(); + + // Reset the stream no matter what happens next + Bindings.df_reset(context); + + // If looping, go back to the start. Otherwise, we'll be exiting. + if (IsLooped) + { + // Starting over! + currentFrame = -1; + + // Start! Again! + timer.Start(); + } + else + { + // We out + State = MediaState.Stopped; + } + } + + // Finally. + return videoTexture[0].RenderTarget as Texture2D; + } + + public void Play(Video video) + { + checkDisposed(); + + // We need to assign this regardless of what happens next. + Video = video; + Video.parent = this; // FIXME: Remove this when extension is replaced!!! + + // Again, no matter what happens this should be cleared! + if (context != IntPtr.Zero) + { + Bindings.df_close(context); + context = IntPtr.Zero; + } + + // FIXME: This is a part of the Duration hack! + if (Video.needsDurationHack) + { + Video.Duration = TimeSpan.MaxValue; + } + + int ok = Bindings.df_fopen(Video.handle, out context); + if (context == IntPtr.Zero || ok == 0) + { + throw new System.IO.FileNotFoundException(Video.handle); + } + + int yWidth, yHeight; + Bindings.PixelLayout layout; + try + { + byte hbd; + Bindings.df_videoinfo2(context, out yWidth, out yHeight, out layout, out hbd); + if (hbd == 2) + { + bitsPerPixel = 12; + } + else if (hbd == 1) + { + bitsPerPixel = 10; + } + else + { + bitsPerPixel = 8; + } + + if (Bindings.df_guessframerate(context, out fps) == 0) + { + // If this is not present, I hope someone manually supplied it + fps = Video.FramesPerSecond; + } + } catch { + Bindings.df_videoinfo(context, out yWidth, out yHeight, out layout); + bitsPerPixel = 8; + fps = Video.FramesPerSecond; + } + + int uvWidth, uvHeight; + + switch (layout) { + case Bindings.PixelLayout.I420: + uvWidth = yWidth / 2; + uvHeight = yHeight / 2; + break; + case Bindings.PixelLayout.I422: + uvWidth = yWidth / 2; + uvHeight = yHeight; + break; + case Bindings.PixelLayout.I444: + uvWidth = yWidth; + uvHeight = yHeight; + break; + default: + throw new NotSupportedException("Unsupported pixel layout in AV1 file"); + } + + // Sanity checks for video metadata + if (Video.Width != yWidth || Video.Height != yHeight) + { + throw new InvalidOperationException( + "XNB/OGV width/height mismatch!" + + " Width: " + Video.Width.ToString() + + " Height: " + Video.Height.ToString() + ); + } + + if (fps == 0) + { + throw new InvalidOperationException("Framerate not present in header or manually specified"); + } + + if (Math.Abs(Video.FramesPerSecond - fps) >= 1.0f) + { + throw new InvalidOperationException( + "XNB/OGV framesPerSecond mismatch!" + + " FPS: " + Video.FramesPerSecond.ToString() + ); + } + + // Check the player state before attempting anything. + if (State != MediaState.Stopped) + { + return; + } + + // Update the player state now, before initializing + State = MediaState.Playing; + + currentFrame = -1; + + // Set up the texture data + // The VideoPlayer will use the GraphicsDevice that is set now. + if (currentDevice != Video.GraphicsDevice) + { + GL_dispose(); + currentDevice = Video.GraphicsDevice; + GL_initialize(Resources.YUVToRGBAEffectR); + } + + RenderTargetBinding overlap = videoTexture[0]; + videoTexture[0] = new RenderTargetBinding( + new RenderTarget2D( + currentDevice, + yWidth, + yHeight, + false, + SurfaceFormat.Color, + DepthFormat.None, + 0, + RenderTargetUsage.PreserveContents + ) + ); + if (overlap.RenderTarget != null) + { + overlap.RenderTarget.Dispose(); + } + GL_setupTextures( + yWidth, + yHeight, + uvWidth, + uvHeight, + bitsPerPixel > 8 ? SurfaceFormat.UShortEXT : SurfaceFormat.ByteEXT + ); + + // The player can finally start now! + timer.Start(); + } + + public void Stop() + { + checkDisposed(); + + // Check the player state before attempting anything. + if (State == MediaState.Stopped) + { + return; + } + + // Update the player state. + State = MediaState.Stopped; + + // Wait for the player to end if it's still going. + timer.Stop(); + timer.Reset(); + Bindings.df_reset(context); + } + + public void Pause() + { + checkDisposed(); + + // Check the player state before attempting anything. + if (State != MediaState.Playing) + { + return; + } + + // Update the player state. + State = MediaState.Paused; + + // Pause timer + timer.Stop(); + } + + public void Resume() + { + checkDisposed(); + + // Check the player state before attempting anything. + if (State != MediaState.Paused) + { + return; + } + + // Update the player state. + State = MediaState.Playing; + + // Unpause timer + timer.Start(); + } + + #endregion + + #region Public Extensions + + // FIXME: Maybe store these to carry over to future videos? + + public void SetAudioTrackEXT(int track) + { + // not available? + } + + public void SetVideoTrackEXT(int track) + { + // not available? + } + + #endregion + + #region Private AV1 Video Stream Methods + + private bool DecodeAndUpdateFrame(int frameCount = 1) + { + IntPtr yData, uData, vData; + uint yLength, uvLength; + uint yStride, uvStride; + byte[] yScratchBuffer = null, uvScratchBuffer = null; + + int ok = Bindings.df_readvideo( + context, frameCount, + out yData, out uData, out vData, + out yLength, out uvLength, + out yStride, out uvStride + ); + + if (ok != 1) + { + return false; + } + + UploadDataToTexture(yuvTextures[0], yData, yLength, yStride, ref yScratchBuffer); + UploadDataToTexture(yuvTextures[1], uData, uvLength, uvStride, ref uvScratchBuffer); + UploadDataToTexture(yuvTextures[2], vData, uvLength, uvStride, ref uvScratchBuffer); + + return true; + } + + private void UploadDataToTexture(Texture2D texture, IntPtr data, uint length, uint stride, ref byte[] scratchBuffer) + { + int w = texture.Width, h = texture.Height, + dataH = (int)(length / stride), + availH = Math.Min(dataH, h), + eltSize = bitsPerPixel > 8 ? 2 : 1, + rowSize = eltSize * w; + + if (w == stride) + { + texture.SetDataPointerEXT(0, new Rectangle(0, 0, w, availH), data, (int)length); + return; + } + + Array.Resize(ref scratchBuffer, w * availH * eltSize); + + fixed (byte* scratch = scratchBuffer) { + for (int y = 0; y < availH; y++) + { + SDL.SDL_memcpy((IntPtr)scratch + (rowSize * y), data + (int)(stride * y), (UIntPtr)rowSize); + } + texture.SetDataPointerEXT(0, null, (IntPtr)scratch, scratchBuffer.Length); + } + } + + #endregion + + #region Video support methods + + internal static VideoPlayer.VideoInfo ReadInfo(string fileName) + { + IntPtr context; + int width, height; + Bindings.PixelLayout pixelLayout; + byte hbd; + double fps; + Bindings.df_fopen(fileName, out context); + Bindings.df_videoinfo2(context, out width, out height, out pixelLayout, out hbd); + if (Bindings.df_guessframerate(context, out fps) == 0) + { + // I hope someone will manually supply this + fps = 0; + } + Bindings.df_close(context); + + return new VideoPlayer.VideoInfo() { fps = fps, width = width, height = height }; + } + + #endregion + } +}