Included more testcases and added hardware parameters on results CSV

LLLjun · May 16, 2022 · bdf3e8b · bdf3e8b
1 parent 03d5a9f
commit bdf3e8b
Show file tree

Hide file tree

Showing 5 changed files with 134 additions and 10 deletions.
diff --git a/stonne/tests/TileGenerator/MyGenerator/DenseGemmEvaluation.py b/stonne/tests/TileGenerator/MyGenerator/DenseGemmEvaluation.py
@@ -88,8 +88,8 @@ def evaluate(num_ms, dn_bw, rn_bw, M, N, K, tolerance=DEFAULT_TOLERANCE, generat
     print(f' - Speedup of the generated tile: {speedup}')
     print(f' - Pass the test (tolerance={tolerance})? {passed}')
 
-    EvaluationUtils.save_densegemm_results_csv(passed, M, N, K, generator, generatedtile, generatedtile_cycles,
-                                               min_tile, min_cycles, speedup, tolerance)
+    EvaluationUtils.save_densegemm_results_csv(passed, num_ms, dn_bw, rn_bw, M, N, K, generator, generatedtile,
+                                               generatedtile_cycles, min_tile, min_cycles, speedup, tolerance)
 
     return passed
 

diff --git a/stonne/tests/TileGenerator/MyGenerator/EvaluationUtils.py b/stonne/tests/TileGenerator/MyGenerator/EvaluationUtils.py
@@ -36,9 +36,9 @@ def check_assertions(stderr):
     return True if match else False
 
 
-def save_densegemm_results_csv(passed, M, N, K, generator, generatedtile, generatedtile_cycles, min_tile, min_cycles, speedup, tolerance):
-    fields = ['PASSED', 'M', 'N', 'K', 'GENERATOR', 'GEN-T_M', 'GEN-T_N', 'GEN-T_K', 'GEN-CYCLES', 'MIN-T_M', 'MIN-T_N', 'MIN-T_K',
-              'MIN-CYCLES', 'SPEEDUP', 'TOLERANCE']
+def save_densegemm_results_csv(passed, num_ms, dn_bw, rn_bw, M, N, K, generator, generatedtile, generatedtile_cycles, min_tile, min_cycles, speedup, tolerance):
+    fields = ['PASSED', 'NUM_MS', 'DN_BW', 'RN_BW', 'M', 'N', 'K', 'GENERATOR', 'GEN-T_M', 'GEN-T_N', 'GEN-T_K',
+              'GEN-CYCLES', 'MIN-T_M', 'MIN-T_N', 'MIN-T_K', 'MIN-CYCLES', 'SPEEDUP', 'TOLERANCE']
     with open(DENSEGEMM_CSV_FILENAME, 'a') as csvfile:
         writer = csv.DictWriter(csvfile, fieldnames=fields)
 
@@ -48,6 +48,9 @@ def save_densegemm_results_csv(passed, M, N, K, generator, generatedtile, genera
 
         # write the row with the results of the simulation to the csv file
         writer.writerow({'PASSED': passed,
+                         'NUM_MS': num_ms,
+                         'DN_BW': dn_bw,
+                         'RN_BW': rn_bw,
                          'M': M,
                          'N': N,
                          'K': K,
@@ -64,9 +67,9 @@ def save_densegemm_results_csv(passed, M, N, K, generator, generatedtile, genera
                          'TOLERANCE': tolerance})
 
 
-def save_sparsedense_results_csv(passed, M, N, K, sparsity, generator, generatedtile, generatedtile_cycles, min_tile, min_cycles, speedup, tolerance):
-    fields = ['PASSED', 'M', 'N', 'K', 'SPARSITY', 'GENERATOR', 'GEN-T_N', 'GEN-T_K', 'GEN-CYCLES', 'MIN-T_N', 'MIN-T_K', 'MIN-CYCLES',
-              'SPEEDUP', 'TOLERANCE']
+def save_sparsedense_results_csv(passed, num_ms, dn_bw, rn_bw, M, N, K, sparsity, generator, generatedtile, generatedtile_cycles, min_tile, min_cycles, speedup, tolerance):
+    fields = ['PASSED', 'NUM_MS', 'DN_BW', 'RN_BW',  'M', 'N', 'K', 'SPARSITY', 'GENERATOR', 'GEN-T_N', 'GEN-T_K',
+              'GEN-CYCLES', 'MIN-T_N', 'MIN-T_K', 'MIN-CYCLES', 'SPEEDUP', 'TOLERANCE']
     with open(SPARSEDENSE_CSV_FILENAME, 'a') as csvfile:
         writer = csv.DictWriter(csvfile, fieldnames=fields)
 
@@ -76,6 +79,9 @@ def save_sparsedense_results_csv(passed, M, N, K, sparsity, generator, generated
 
         # write the row with the results of the simulation to the csv file
         writer.writerow({'PASSED': passed,
+                         'NUM_MS': num_ms,
+                         'DN_BW': dn_bw,
+                         'RN_BW': rn_bw,
                          'M': M,
                          'N': N,
                          'K': K,

diff --git a/stonne/tests/TileGenerator/MyGenerator/SparseDenseEvaluation.py b/stonne/tests/TileGenerator/MyGenerator/SparseDenseEvaluation.py
@@ -87,7 +87,7 @@ def evaluate(num_ms, dn_bw, rn_bw, M, N, K, sparsity, tolerance=DEFAULT_TOLERANC
     print(f' - Speedup of the generated tile: {speedup}')
     print(f' - Pass the test (tolerance={tolerance})? {passed}')
 
-    EvaluationUtils.save_sparsedense_results_csv(passed, M, N, K, sparsity, generator, generatedtile,
+    EvaluationUtils.save_sparsedense_results_csv(passed, num_ms, dn_bw, rn_bw, M, N, K, sparsity, generator, generatedtile,
                                                  generatedtile_cycles, min_tile, min_cycles, speedup, tolerance)
 
     return passed

diff --git a/stonne/tests/TileGenerator/MyGenerator/Test_DenseGemm.py b/stonne/tests/TileGenerator/MyGenerator/Test_DenseGemm.py
@@ -1,6 +1,7 @@
 import os
 import unittest
 import subprocess
+import random
 try: # Try to import the local version first (usually works when executed from the command line with Python directly)
     import DenseGemmEvaluation as DenseGemm
 except ImportError: # Only works when you execute it with the '-m unittest' parameter from stonne/stonne directory
@@ -72,7 +73,39 @@ def testDenseGemmRandom3(self):
         # - Total Cycles for best tile: 5145
         self.assertTrue(DenseGemm.evaluate(num_ms=256, dn_bw=256, rn_bw=64, M=116, N=333, K=32, tolerance=PERFORMANCE_TOLERANCE, generator=GENERATOR))
 
-    # TODO: implement more tests
+    def testDenseGemmGenerateRandomly(self):
+        for i in range(100):
+            max_power = random.randint(1, 10) # 2..1024
+            num_ms = 2 ** max_power
+            dn_bw = 2 ** random.randint(0, max_power) # 1..num_ms
+            rn_bw = 2 ** random.randint(0, max_power) # 1..num_ms
+            M = random.randint(1, 1024)
+            N = random.randint(1, 1024)
+            K = random.randint(1, 1024)
+            self.assertTrue(DenseGemm.evaluate(num_ms=num_ms, dn_bw=dn_bw, rn_bw=rn_bw, M=M, N=N, K=K, tolerance=PERFORMANCE_TOLERANCE, generator=GENERATOR))
+
+    def testDenseGemmGenerateRandomlyFixHardware(self):
+        for i in range(100):
+            max_power = random.randint(1, 10) # 2..1024
+            num_ms = 2 ** max_power
+            dn_bw = num_ms
+            rn_bw = num_ms
+            M = random.randint(1, 1024)
+            N = random.randint(1, 1024)
+            K = random.randint(1, 1024)
+            self.assertTrue(DenseGemm.evaluate(num_ms=num_ms, dn_bw=dn_bw, rn_bw=rn_bw, M=M, N=N, K=K, tolerance=PERFORMANCE_TOLERANCE, generator=GENERATOR))
+
+    def testDenseGemmGenerateRandomlyFixHardwareAlwaysLess(self):
+        for i in range(100):
+            max_power = random.randint(1, 10) # 2..1024
+            num_ms = 2 ** max_power
+            dn_bw = num_ms
+            rn_bw = num_ms
+            M = random.randint(1, num_ms)
+            N = random.randint(1, num_ms)
+            K = random.randint(1, num_ms)
+            self.assertTrue(DenseGemm.evaluate(num_ms=num_ms, dn_bw=dn_bw, rn_bw=rn_bw, M=M, N=N, K=K, tolerance=PERFORMANCE_TOLERANCE, generator=GENERATOR))
+
 
 # Main method to execute all testcases of MyGenerator for DenseGEMM/FC layers
 if __name__ == "__main__":

diff --git a/stonne/tests/TileGenerator/MyGenerator/Test_SparseDense.py b/stonne/tests/TileGenerator/MyGenerator/Test_SparseDense.py
@@ -1,6 +1,7 @@
 import os
 import unittest
 import subprocess
+import random
 try: # Try to import the local version first (usually works when executed from the command line with Python directly)
     import SparseDenseEvaluation as SparseDense
 except ImportError: # Only works when you execute it with the '-m unittest' parameter from stonne/stonne directory
@@ -31,6 +32,90 @@ def testSparseDenseBasic1(self):
         for sparsity in [10, 40, 70, 90]:
             self.assertTrue(SparseDense.evaluate(num_ms=16, dn_bw=8, rn_bw=8, M=32, N=8, K=16, sparsity=sparsity, tolerance=PERFORMANCE_TOLERANCE, generator=GENERATOR))
 
+    def testSparseDenseBasic2(self):
+        for sparsity in [1, 40, 70, 99]:
+            self.assertTrue(SparseDense.evaluate(num_ms=16, dn_bw=8, rn_bw=8, M=32, N=48, K=20, sparsity=sparsity, tolerance=PERFORMANCE_TOLERANCE, generator=GENERATOR))
+
+    def testSparseDenseBasic3(self):
+        for sparsity in [1, 40, 70, 99]:
+            self.assertTrue(SparseDense.evaluate(num_ms=16, dn_bw=8, rn_bw=8, M=32, N=20, K=48, sparsity=sparsity, tolerance=PERFORMANCE_TOLERANCE, generator=GENERATOR))
+
+    def testSparseDensePowersOfTwo1(self):
+        for sparsity in range(30, 70, 2):
+            self.assertTrue(SparseDense.evaluate(num_ms=32, dn_bw=32, rn_bw=32, M=8, N=64, K=32, sparsity=sparsity, tolerance=PERFORMANCE_TOLERANCE, generator=GENERATOR))
+
+    def testSparseDensePowersOfTwo2(self):
+        for sparsity in range(1, 100, 5):
+            self.assertTrue(SparseDense.evaluate(num_ms=64, dn_bw=64, rn_bw=64, M=32, N=128, K=32, sparsity=sparsity, tolerance=PERFORMANCE_TOLERANCE, generator=GENERATOR))
+
+    def testSparseDensePowersOfTwo3(self):
+        for sparsity in range(1, 100, 5):
+            self.assertTrue(SparseDense.evaluate(num_ms=128, dn_bw=128, rn_bw=64, M=16, N=64, K=128, sparsity=sparsity, tolerance=PERFORMANCE_TOLERANCE, generator=GENERATOR))
+
+    def testSparseDensePowersOfTwo4(self):
+        for sparsity in range(1, 100, 7):
+            self.assertTrue(SparseDense.evaluate(num_ms=256, dn_bw=256, rn_bw=64, M=16, N=256, K=256, sparsity=sparsity, tolerance=PERFORMANCE_TOLERANCE, generator=GENERATOR))
+
+    def testSparseDensePowersOfTwo5(self):
+        for sparsity in range(1, 100, 7):
+            self.assertTrue(SparseDense.evaluate(num_ms=512, dn_bw=512, rn_bw=128, M=4, N=128, K=256, sparsity=sparsity, tolerance=PERFORMANCE_TOLERANCE, generator=GENERATOR))
+
+    def testSparseDensePowersOfTwo6(self):
+        for sparsity in range(0, 100, 10):
+            self.assertTrue(SparseDense.evaluate(num_ms=512, dn_bw=256, rn_bw=128, M=4, N=1024, K=64, sparsity=sparsity, tolerance=PERFORMANCE_TOLERANCE, generator=GENERATOR))
+
+    def testSparseDenseRandom1(self):
+        for sparsity in range(1, 100, 5):
+            self.assertTrue(SparseDense.evaluate(num_ms=8, dn_bw=8, rn_bw=8, M=7, N=7, K=9, sparsity=sparsity, tolerance=PERFORMANCE_TOLERANCE, generator=GENERATOR))
+
+    def testSparseDenseRandom2(self):
+        for sparsity in range(1, 100, 5):
+            self.assertTrue(SparseDense.evaluate(num_ms=16, dn_bw=16, rn_bw=16, M=5, N=17, K=9, sparsity=sparsity, tolerance=PERFORMANCE_TOLERANCE, generator=GENERATOR))
+
+    def testSparseDenseRandom3(self):
+        for sparsity in range(1, 100, 5):
+            self.assertTrue(SparseDense.evaluate(num_ms=32, dn_bw=32, rn_bw=16, M=5, N=31, K=17, sparsity=sparsity, tolerance=PERFORMANCE_TOLERANCE, generator=GENERATOR))
+
+    def testSparseDenseRandom4(self):
+        for sparsity in range(1, 100, 5):
+            self.assertTrue(SparseDense.evaluate(num_ms=64, dn_bw=32, rn_bw=16, M=3, N=70, K=91, sparsity=sparsity, tolerance=PERFORMANCE_TOLERANCE, generator=GENERATOR))
+
+    def testSparseDenseGenerateRandomly(self):
+        for i in range(100):
+            max_power = random.randint(1, 10) # 2..1024
+            num_ms = 2 ** max_power
+            dn_bw = 2 ** random.randint(0, max_power) # 1..num_ms
+            rn_bw = 2 ** random.randint(0, max_power) # 1..num_ms
+            M = random.randint(1, 1024)
+            N = random.randint(1, 1024)
+            K = random.randint(1, 1024)
+            sparsity = random.randint(0, 99)
+            self.assertTrue(SparseDense.evaluate(num_ms=num_ms, dn_bw=dn_bw, rn_bw=rn_bw, M=M, N=N, K=K, sparsity=sparsity, tolerance=PERFORMANCE_TOLERANCE, generator=GENERATOR))
+
+    def testSparseDenseGenerateRandomlyFixHardware(self):
+        for i in range(100):
+            max_power = random.randint(1, 10) # 2..1024
+            num_ms = 2 ** max_power
+            dn_bw = num_ms
+            rn_bw = num_ms
+            M = random.randint(1, 1024)
+            N = random.randint(1, 1024)
+            K = random.randint(1, 1024)
+            sparsity = random.randint(0, 99)
+            self.assertTrue(SparseDense.evaluate(num_ms=num_ms, dn_bw=dn_bw, rn_bw=rn_bw, M=M, N=N, K=K, sparsity=sparsity, tolerance=PERFORMANCE_TOLERANCE, generator=GENERATOR))
+
+    def testSparseDenseGenerateRandomlyFixHardwareAlwaysLess(self):
+        for i in range(100):
+            max_power = random.randint(1, 10) # 2..1024
+            num_ms = 2 ** max_power
+            dn_bw = num_ms
+            rn_bw = num_ms
+            M = random.randint(1, num_ms)
+            N = random.randint(1, num_ms)
+            K = random.randint(1, num_ms)
+            sparsity = random.randint(0, 99)
+            self.assertTrue(SparseDense.evaluate(num_ms=num_ms, dn_bw=dn_bw, rn_bw=rn_bw, M=M, N=N, K=K, sparsity=sparsity, tolerance=PERFORMANCE_TOLERANCE, generator=GENERATOR))
+
 
 # Main method to execute all testcases of MyGenerator for SparseDense layers
 if __name__ == "__main__":