Added data generation functionality (liamt19#63)

bench: 4909216

.gitignore

@@ -12,3 +12,5 @@
 /Properties/launchSettings.json
 /*.exe
 /*.pdb
+/data
+/.vs/ProjectEvaluation

Lizard.csproj

@@ -25,7 +25,7 @@
     <LangVersion>Latest</LangVersion>
     <AllowUnsafeBlocks>True</AllowUnsafeBlocks>
 
-    <Configurations>Debug;Release;Development</Configurations>
+    <Configurations>Debug;Release;Datagen</Configurations>
     <EnforceCodeStyleInBuild>False</EnforceCodeStyleInBuild>
     <RunAnalyzersDuringBuild>False</RunAnalyzersDuringBuild>
     <AnalysisLevel>latest-all</AnalysisLevel>
@@ -143,9 +143,8 @@
     <Optimize>True</Optimize>
   </PropertyGroup>
 
-  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Development|x64'">
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Datagen|x64'">
     <PlatformTarget>x64</PlatformTarget>
-    <DefineConstants>$(DefineConstants);DEV</DefineConstants>
     <DebugType>embedded</DebugType>
     <Optimize>True</Optimize>
   </PropertyGroup>

Logic/Core/Position.cs

@@ -95,7 +95,7 @@ public bool CanCastle(ulong boardOcc, ulong ourOcc, CastlingStatus cr)
         /// If <paramref name="createAccumulators"/> is true, then this Position will create and incrementally update 
         /// its accumulators when making and unmaking moves.
         /// <para></para>
-        /// <paramref name="owner"/> should be set to one of the <see cref="SearchThread"/>'s within the <see cref="SearchThreadPool.SearchPool"/>
+        /// <paramref name="owner"/> should be set to one of the <see cref="SearchThread"/>'s within the <see cref="SearchThreadPool.GlobalSearchPool"/>
         /// (the <see cref="SearchThreadPool.MainThread"/> unless there are multiple threads in the pool).
         /// <br></br>
         /// If <paramref name="owner"/> is <see langword="null"/> then <paramref name="createAccumulators"/> should be false.
@@ -134,7 +134,7 @@ public Position(string fen = InitialFEN, bool createAccumulators = true, SearchT
             {
                 Debug.WriteLine($"info string Position('{fen}', {createAccumulators}, ...) has NNUE enabled and was given a nullptr for owner! " +
                                 $"Assigning this Position instance to the SearchPool's MainThread, UB and other weirdness may occur...");
-                Owner = SearchPool.MainThread;
+                Owner = GlobalSearchPool.MainThread;
             }
 
             LoadFromFEN(fen);
@@ -454,9 +454,6 @@ public void MakeNullMove()
             State->PliesFromNull = 0;
 
             SetCheckInfo();
-
-            prefetch(TranspositionTable.GetCluster(State->Hash));
-
         }
 
         /// <summary>
@@ -871,6 +868,51 @@ private void RemoveCastling(CastlingStatus cr)
         }
 
 
+
+        public void SetupForDFRC(int wIdx, int bIdx)
+        {
+            bb.Reset();
+
+            for (int sq = A2; sq <= H2; sq++)
+                bb.AddPiece(sq, White, Pawn);
+
+            for (int sq = A7; sq <= H7; sq++)
+                bb.AddPiece(sq, Black, Pawn);
+
+            IsChess960 = true;
+            int* wBackrank = stackalloc int[8];
+            int* bBackrank = stackalloc int[8];
+
+            FillWithScharnaglNumber(wIdx, wBackrank);
+            FillWithScharnaglNumber(bIdx, bBackrank);
+
+            var w = new Span<int>(wBackrank, 8);
+            var b = new Span<int>(bBackrank, 8);
+
+            for (int sq = 0; sq < 8; sq++)
+            {
+                bb.AddPiece(A1 + sq, White, wBackrank[sq]);
+                bb.AddPiece(A8 + sq, Black, bBackrank[sq]);
+
+                if (wBackrank[sq] == King)
+                    State->KingSquares[White] = A1 + sq;
+
+                if (bBackrank[sq] == King)
+                    State->KingSquares[Black] = A8 + sq;
+            }
+
+            for (int sq = 0; sq < 8; sq++)
+            {
+                //  SetCastlingStatus needs State->KingSquare which might not be set yet if we do this in the above loop
+                if (wBackrank[sq] == Rook)
+                    SetCastlingStatus(White, A1 + sq);
+
+                if (bBackrank[sq] == Rook)
+                    SetCastlingStatus(Black, A8 + sq);
+            }
+        }
+
+
         /// <summary>
         /// Returns the number of leaf nodes in the current position up to <paramref name="depth"/>.
         /// </summary>
@@ -919,7 +961,7 @@ public ulong PerftParallel(int depth, bool isRoot = false)
             opts.MaxDegreeOfParallelism = MoveListSize;
             Parallel.For(0u, size, opts, i =>
             {
-                Position threadPosition = new Position(rootFEN, false, owner: SearchPool.MainThread);
+                Position threadPosition = new Position(rootFEN, false, owner: GlobalSearchPool.MainThread);
 
                 threadPosition.MakeMove(list[i].Move);
                 ulong result = (depth >= PerftParallelMinDepth) ? threadPosition.PerftParallel(depth - 1) : threadPosition.Perft(depth - 1);
@@ -1020,6 +1062,11 @@ public bool LoadFromFEN(string fen)
                                 Log("ERROR x for i = " + i + " was '" + splits[i][x] + "' and didn't get parsed");
                             }
                         }
+
+                        if (i == 7 && popcount(bb.Pieces[King]) != 2)
+                        {
+                            Log($"FEN {fen} has {popcount(bb.Pieces[King])} kings!");
+                        }
                     }
                     //  who moves next
                     else if (i == 8)

Logic/Datagen/BulletDataFormat.cs

@@ -0,0 +1,66 @@
+
+using System.Runtime.InteropServices;
+
+namespace Lizard.Logic.Datagen
+{
+    [StructLayout(LayoutKind.Explicit)]
+    public unsafe struct BulletDataFormat : TOutputFormat
+    {
+        //  STM here is used to fix the game result, which is dependent on the STM:
+        //  If black is to move, the result is flipped around WhiteWin <-> BlackWin.
+        //  WE don't know the result when creating the entries, and STM isn't stored within them anywhere,
+        //  So manually place the STM in the last byte of padding of the entries.
+        [FieldOffset( 0)] BulletFormatEntry BFE;
+        [FieldOffset(29)] Move BestMove;
+        [FieldOffset(31)] byte STM;
+
+        public int Score
+        {
+            get => BFE.score;
+            set => BFE.score = (short)value;
+        }
+
+        public GameResult Result
+        {
+            get => (GameResult)BFE.result;
+            set => BFE.result = (byte)value;
+        }
+
+        public void SetSTM(int stm) { STM = (byte)stm; }
+        public void SetResult(GameResult gr)
+        {
+            if (STM == Black)
+            {
+                gr = (GameResult)(2 - gr);
+            }
+
+            Result = gr;
+        }
+
+        public string GetWritableTextData()
+        {
+            return "";
+        }
+
+        public byte[] GetWritableData()
+        {
+            int len = Marshal.SizeOf<BulletFormatEntry>();
+            IntPtr ptr = Marshal.AllocHGlobal(len);
+            byte[] myBuffer = new byte[len];
+
+            Marshal.StructureToPtr(BFE, ptr, false);
+            Marshal.Copy(ptr, myBuffer, 0, len);
+            Marshal.FreeHGlobal(ptr);
+
+            return myBuffer;
+        }
+
+        public void Fill(Position pos, Move bestMove, int score)
+        {
+            BFE = BulletFormatEntry.FromBitboard(ref pos.bb, pos.ToMove, (short)score, GameResult.Draw);
+            STM = (byte)pos.ToMove;
+            BestMove = bestMove;
+        }
+
+    }
+}

Logic/Datagen/BulletFormatEntry.cs

@@ -0,0 +1,113 @@
+
+using System.Buffers.Binary;
+using System.Numerics;
+using System.Runtime.InteropServices;
+
+namespace Lizard.Logic.Datagen
+{
+    [StructLayout(LayoutKind.Explicit, Size = 32)]
+    public unsafe struct BulletFormatEntry
+    {
+        public const int Size = 32;
+
+        [FieldOffset(0)] public ulong occ;
+        [FieldOffset(8)] public fixed byte pcs[16];
+        [FieldOffset(24)] public short score;
+        [FieldOffset(26)] public byte result;
+        [FieldOffset(27)] public byte ksq;
+        [FieldOffset(28)] public byte opp_ksq;
+        [FieldOffset(29)] public fixed byte _pad[3];
+
+
+
+        public void FillBitboard(ref Bitboard bb)
+        {
+            bb.Reset();
+
+            bb.Occupancy = occ;
+
+            ulong temp = occ;
+            int idx = 0;
+            while (temp != 0)
+            {
+                int sq = poplsb(&temp);
+                int piece = (pcs[idx / 2] >> (4 * (idx & 1))) & 0b1111;
+
+                bb.AddPiece(sq, piece / 8, piece % 8);
+
+                idx++;
+            }
+        }
+
+
+
+        public static BulletFormatEntry FromBitboard(ref Bitboard bb, int stm, short score, GameResult result)
+        {
+            Span<ulong> bbs = [
+                bb.Colors[White], bb.Colors[Black],
+                bb.Pieces[0], bb.Pieces[1], bb.Pieces[2],
+                bb.Pieces[3], bb.Pieces[4], bb.Pieces[5],
+            ];
+
+            if (stm == Black)
+            {
+                for (int i = 0; i < bbs.Length; i++)
+                    bbs[i] = BinaryPrimitives.ReverseEndianness(bbs[i]);
+
+                (bbs[White], bbs[Black]) = (bbs[Black], bbs[White]);
+
+                score = (short)-score;
+                result = 1 - result;
+            }
+
+            ulong occ = bbs[0] | bbs[1];
+
+            BulletFormatEntry bfe = new BulletFormatEntry
+            {
+                score = score,
+                occ = occ,
+                result =  (byte)(2 * (int)result),
+                ksq     = (byte) BitOperations.TrailingZeroCount(bbs[0] & bbs[7]),
+                opp_ksq = (byte)(BitOperations.TrailingZeroCount(bbs[1] & bbs[7]) ^ 56)
+            };
+
+            Span<byte> pieces = stackalloc byte[16];
+
+            int idx = 0;
+            ulong occ2 = occ;
+            int piece = 0;
+            while (occ2 > 0)
+            {
+                int sq = BitOperations.TrailingZeroCount(occ2);
+                ulong bit = 1UL << sq;
+                occ2 &= occ2 - 1;
+
+                byte colour = (byte)(((bit & bbs[1]) > 0 ? 1 : 0) << 3);
+                for (int i = 2; i < 8; i++)
+                    if ((bit & bbs[i]) > 0)
+                    {
+                        piece = i - 2;
+                        break;
+                    }
+
+                byte pc = (byte)(colour | (byte)piece);
+
+                bfe.pcs[idx / 2] |= (byte)(pc << (4 * (idx & 1)));
+
+                idx += 1;
+            }
+
+            return bfe;
+        }
+
+
+
+        public void WriteToBuffer(Span<byte> buff)
+        {
+            fixed (void* buffPtr = &buff[0], thisPtr = &this)
+            {
+                Unsafe.CopyBlock(buffPtr, thisPtr, BulletFormatEntry.Size);
+            }
+        }
+    }
+}

Logic/Datagen/DatagenParameters.cs

@@ -0,0 +1,23 @@
+
+namespace Lizard.Logic.Datagen
+{
+    public static class DatagenParameters
+    {
+        public const int HashSize = 8;
+
+        public const int MinOpeningPly = 8;
+        public const int MaxOpeningPly = 9;
+
+        public const int SoftNodeLimit = 5000;
+        public const int DepthLimit = 24;
+
+        public const int WritableDataLimit = 512;
+
+        public const int AdjudicateMoves = 4;
+        public const int AdjudicateScore = 3000;
+        public const int MaxFilteringScore = 6000;
+
+        public const int MaxOpeningScore = 1200;
+        public const int MaxScrambledOpeningScore = 600;
+    }
+}

Logic/Datagen/GameResult.cs

@@ -0,0 +1,10 @@
+
+namespace Lizard.Logic.Datagen
+{
+    public enum GameResult
+    {
+        WhiteWin = 2,
+        Draw = 1,
+        BlackWin = 0
+    }
+}

Logic/Datagen/OutputFormat.cs

@@ -0,0 +1,24 @@
+namespace Lizard.Logic.Datagen
+{
+    public interface TOutputFormat
+    {
+        public int Score { get; set; }
+        public GameResult Result { get; set; }
+        public void SetResult(GameResult gr);
+        public void SetSTM(int stm);
+        public string GetWritableTextData();
+        public void Fill(Position pos, Move bestMove, int score);
+
+        public static string ResultToMarlin(GameResult gr)
+        {
+            return gr switch
+            {
+                GameResult.WhiteWin => "1.0",
+                GameResult.Draw => "0.5",
+                GameResult.BlackWin => "0.0",
+            };
+        }
+    }
+
+
+}