diff --git a/sycl/test-e2e/DeviceLib/imf_bfloat16_integeral_convesions.cpp b/sycl/test-e2e/DeviceLib/imf/bfloat16_integeral_convesions.cpp
similarity index 99%
rename from sycl/test-e2e/DeviceLib/imf_bfloat16_integeral_convesions.cpp
rename to sycl/test-e2e/DeviceLib/imf/bfloat16_integeral_convesions.cpp
index 35d158744c07f..e9913aad585cd 100644
--- a/sycl/test-e2e/DeviceLib/imf_bfloat16_integeral_convesions.cpp
+++ b/sycl/test-e2e/DeviceLib/imf/bfloat16_integeral_convesions.cpp
@@ -3,15 +3,13 @@
 
 // RUN: %{build} -fno-builtin %if target-spir %{ -fsycl-device-lib-jit-link -Wno-deprecated %} -o %t2.out
 // RUN: %{run} %t2.out
-//
-// UNSUPPORTED: target-nvidia || target-amd
 
 // Windows doesn't yet have full shutdown().
 // UNSUPPORTED: ze_debug && windows
 
 #include "imf_utils.hpp"
-#include <sycl/ext/intel/math.hpp>
 #include <sycl/detail/core.hpp>
+#include <sycl/ext/intel/math.hpp>
 
 int main() {
   sycl::queue device_queue(sycl::default_selector_v);
diff --git a/sycl/test-e2e/DeviceLib/imf_double2bfloat16.cpp b/sycl/test-e2e/DeviceLib/imf/double2bfloat16.cpp
similarity index 98%
rename from sycl/test-e2e/DeviceLib/imf_double2bfloat16.cpp
rename to sycl/test-e2e/DeviceLib/imf/double2bfloat16.cpp
index 8a98c9e5cdaa0..a67a73fd5d2ec 100644
--- a/sycl/test-e2e/DeviceLib/imf_double2bfloat16.cpp
+++ b/sycl/test-e2e/DeviceLib/imf/double2bfloat16.cpp
@@ -5,8 +5,6 @@
 
 // RUN: %{build} -fno-builtin %if target-spir %{ -fsycl-device-lib-jit-link -Wno-deprecated %} -o %t1.out
 // RUN: %{run} %t1.out
-//
-// UNSUPPORTED: target-nvidia || target-amd
 
 #include "imf_utils.hpp"
 #include <sycl/ext/intel/math.hpp>
diff --git a/sycl/test-e2e/DeviceLib/imf_double2half.cpp b/sycl/test-e2e/DeviceLib/imf/double2half.cpp
similarity index 97%
rename from sycl/test-e2e/DeviceLib/imf_double2half.cpp
rename to sycl/test-e2e/DeviceLib/imf/double2half.cpp
index b65c1519594e7..908c30f9dd930 100644
--- a/sycl/test-e2e/DeviceLib/imf_double2half.cpp
+++ b/sycl/test-e2e/DeviceLib/imf/double2half.cpp
@@ -7,8 +7,6 @@
 // RUN: %{build} -fno-builtin %if target-spir %{ -fsycl-device-lib-jit-link -Wno-deprecated %} -o %t2.out
 // RUN: %{run} %t2.out
 
-// UNSUPPORTED: target-nvidia, target-amd
-
 #include "imf_utils.hpp"
 #include <sycl/ext/intel/math.hpp>
 
diff --git a/sycl/test-e2e/DeviceLib/imf_float2bfloat16.cpp b/sycl/test-e2e/DeviceLib/imf/float2bfloat16.cpp
similarity index 98%
rename from sycl/test-e2e/DeviceLib/imf_float2bfloat16.cpp
rename to sycl/test-e2e/DeviceLib/imf/float2bfloat16.cpp
index 56356294ab4d4..80f548ea21611 100644
--- a/sycl/test-e2e/DeviceLib/imf_float2bfloat16.cpp
+++ b/sycl/test-e2e/DeviceLib/imf/float2bfloat16.cpp
@@ -3,8 +3,6 @@
 
 // RUN: %{build} -fno-builtin %if target-spir %{ -fsycl-device-lib-jit-link -Wno-deprecated %} -o %t2.out
 // RUN: %{run} %t2.out
-//
-// UNSUPPORTED: target-nvidia || target-amd
 
 // All __imf_* bf16 functions are implemented via fp32 emulation, so we don't
 // need to check whether underlying device supports bf16 or not.
diff --git a/sycl/test-e2e/DeviceLib/imf_fp16_trivial_test.cpp b/sycl/test-e2e/DeviceLib/imf/fp16_trivial_test.cpp
similarity index 99%
rename from sycl/test-e2e/DeviceLib/imf_fp16_trivial_test.cpp
rename to sycl/test-e2e/DeviceLib/imf/fp16_trivial_test.cpp
index 1fa8627c4c7ef..4592af63a0668 100644
--- a/sycl/test-e2e/DeviceLib/imf_fp16_trivial_test.cpp
+++ b/sycl/test-e2e/DeviceLib/imf/fp16_trivial_test.cpp
@@ -8,8 +8,6 @@
 // RUN: %clangxx -Wno-error=unused-command-line-argument -fsycl %{sycl_target_opts} %t1.o %t2.o  -o %t.out
 // RUN: %{run} %t.out
 
-// UNSUPPORTED: target-nvidia, target-amd
-
 // Windows doesn't yet have full shutdown().
 // UNSUPPORTED: ze_debug && windows
 
@@ -417,7 +415,6 @@ void run_imf_fp16_test() {
           F3(sycl_imf::hcmadd));
     std::cout << "hcmadd passes." << std::endl;
   }
-
 }
 #endif
 
diff --git a/sycl/test-e2e/DeviceLib/imf_fp32_rounding_test.cpp b/sycl/test-e2e/DeviceLib/imf/fp32_rounding_test.cpp
similarity index 99%
rename from sycl/test-e2e/DeviceLib/imf_fp32_rounding_test.cpp
rename to sycl/test-e2e/DeviceLib/imf/fp32_rounding_test.cpp
index 461399c1c50e3..e11b05c9aa6c4 100644
--- a/sycl/test-e2e/DeviceLib/imf_fp32_rounding_test.cpp
+++ b/sycl/test-e2e/DeviceLib/imf/fp32_rounding_test.cpp
@@ -3,8 +3,6 @@
 
 // RUN: %{build} -fno-builtin %if target-spir %{ -fsycl-device-lib-jit-link -Wno-deprecated %} -o %t2.out
 // RUN: %{run} %t2.out
-//
-// UNSUPPORTED: target-nvidia || target-amd
 
 #include "imf_utils.hpp"
 #include <sycl/ext/intel/math.hpp>
diff --git a/sycl/test-e2e/DeviceLib/imf_fp32_test.cpp b/sycl/test-e2e/DeviceLib/imf/fp32_test.cpp
similarity index 99%
rename from sycl/test-e2e/DeviceLib/imf_fp32_test.cpp
rename to sycl/test-e2e/DeviceLib/imf/fp32_test.cpp
index 71c73cbdbec48..f17a8ae43f7b2 100644
--- a/sycl/test-e2e/DeviceLib/imf_fp32_test.cpp
+++ b/sycl/test-e2e/DeviceLib/imf/fp32_test.cpp
@@ -3,8 +3,6 @@
 
 // RUN: %{build} -fno-builtin %if target-spir %{ -fsycl-device-lib-jit-link -Wno-deprecated %} -o %t2.out
 // RUN: %{run} %t2.out
-//
-// UNSUPPORTED: target-nvidia || target-amd
 
 // Windows doesn't yet have full shutdown().
 // UNSUPPORTED: ze_debug && windows
diff --git a/sycl/test-e2e/DeviceLib/imf_fp64_rounding_test.cpp b/sycl/test-e2e/DeviceLib/imf/fp64_rounding_test.cpp
similarity index 99%
rename from sycl/test-e2e/DeviceLib/imf_fp64_rounding_test.cpp
rename to sycl/test-e2e/DeviceLib/imf/fp64_rounding_test.cpp
index 3b7fe2f5c677c..a07d8e3c4d4bd 100644
--- a/sycl/test-e2e/DeviceLib/imf_fp64_rounding_test.cpp
+++ b/sycl/test-e2e/DeviceLib/imf/fp64_rounding_test.cpp
@@ -4,8 +4,6 @@
 
 // RUN: %{build} -fno-builtin %if target-spir %{ -fsycl-device-lib-jit-link -Wno-deprecated %} -o %t2.out
 // RUN: %{run} %t2.out
-//
-// UNSUPPORTED: target-nvidia || target-amd
 
 // Depends on SPIR-V Backend & run-time drivers version.
 // XFAIL: spirv-backend && run-mode
diff --git a/sycl/test-e2e/DeviceLib/imf_fp64_test.cpp b/sycl/test-e2e/DeviceLib/imf/fp64_test.cpp
similarity index 99%
rename from sycl/test-e2e/DeviceLib/imf_fp64_test.cpp
rename to sycl/test-e2e/DeviceLib/imf/fp64_test.cpp
index ec94aa634e688..98723fb0f16db 100644
--- a/sycl/test-e2e/DeviceLib/imf_fp64_test.cpp
+++ b/sycl/test-e2e/DeviceLib/imf/fp64_test.cpp
@@ -4,8 +4,7 @@
 
 // RUN: %{build} -fno-builtin %if target-spir %{ -fsycl-device-lib-jit-link -Wno-deprecated %} -o %t2.out
 // RUN: %{run} %t2.out
-//
-// UNSUPPORTED: target-nvidia || target-amd
+
 #include "imf_utils.hpp"
 #include <sycl/ext/intel/math.hpp>
 
diff --git a/sycl/test-e2e/DeviceLib/imf_fp64_test2.cpp b/sycl/test-e2e/DeviceLib/imf/fp64_test2.cpp
similarity index 93%
rename from sycl/test-e2e/DeviceLib/imf_fp64_test2.cpp
rename to sycl/test-e2e/DeviceLib/imf/fp64_test2.cpp
index 7e06c74d83795..6e84dee35d7ed 100644
--- a/sycl/test-e2e/DeviceLib/imf_fp64_test2.cpp
+++ b/sycl/test-e2e/DeviceLib/imf/fp64_test2.cpp
@@ -4,8 +4,6 @@
 
 // RUN: %{build} -fno-builtin %if target-spir %{ -fsycl-device-lib-jit-link -Wno-deprecated %} -o %t2.out
 // RUN: %{run} %t2.out
-//
-// UNSUPPORTED: target-nvidia || target-amd
 
 #include "imf_utils.hpp"
 #include <sycl/ext/intel/math.hpp>
diff --git a/sycl/test-e2e/DeviceLib/imf_half_type_cast.cpp b/sycl/test-e2e/DeviceLib/imf/half_type_cast.cpp
similarity index 99%
rename from sycl/test-e2e/DeviceLib/imf_half_type_cast.cpp
rename to sycl/test-e2e/DeviceLib/imf/half_type_cast.cpp
index c9d2c721664a8..4154cfc1df162 100644
--- a/sycl/test-e2e/DeviceLib/imf_half_type_cast.cpp
+++ b/sycl/test-e2e/DeviceLib/imf/half_type_cast.cpp
@@ -7,8 +7,6 @@
 // RUN: %{build} -fno-builtin %if target-spir %{ -fsycl-device-lib-jit-link -Wno-deprecated %} -o %t2.out
 // RUN: %{run} %t2.out
 
-// UNSUPPORTED: target-nvidia, target-amd
-
 // Windows doesn't yet have full shutdown().
 // UNSUPPORTED: ze_debug && windows
 
diff --git a/sycl/test-e2e/DeviceLib/imf_utils.hpp b/sycl/test-e2e/DeviceLib/imf/imf_utils.hpp
similarity index 100%
rename from sycl/test-e2e/DeviceLib/imf_utils.hpp
rename to sycl/test-e2e/DeviceLib/imf/imf_utils.hpp
diff --git a/sycl/test-e2e/DeviceLib/imf/lit.local.cfg b/sycl/test-e2e/DeviceLib/imf/lit.local.cfg
new file mode 100644
index 0000000000000..442db188e39a9
--- /dev/null
+++ b/sycl/test-e2e/DeviceLib/imf/lit.local.cfg
@@ -0,0 +1,2 @@
+# IMF is not supported on Nvidia and AMD
+config.unsupported_features += ['target-nvidia', 'target-amd']
diff --git a/sycl/test-e2e/DeviceLib/imf_simd_emulate_test.cpp b/sycl/test-e2e/DeviceLib/imf/simd_emulate_test.cpp
similarity index 99%
rename from sycl/test-e2e/DeviceLib/imf_simd_emulate_test.cpp
rename to sycl/test-e2e/DeviceLib/imf/simd_emulate_test.cpp
index ea5e0a82def3f..c4ed55000a46d 100644
--- a/sycl/test-e2e/DeviceLib/imf_simd_emulate_test.cpp
+++ b/sycl/test-e2e/DeviceLib/imf/simd_emulate_test.cpp
@@ -3,8 +3,6 @@
 
 // RUN: %{build} -fno-builtin %if target-spir %{ -fsycl-device-lib-jit-link -Wno-deprecated %} -o %t2.out
 // RUN: %{run} %t2.out
-//
-// UNSUPPORTED: target-nvidia || target-amd
 
 // UNSUPPORTED: igc-dev
 // UNSUPPORTED-TRACKER: https://github.com/intel/llvm/issues/17008
diff --git a/sycl/test/e2e_test_requirements/no-unsupported-without-info.cpp b/sycl/test/e2e_test_requirements/no-unsupported-without-info.cpp
index 087da2de5f85c..878c26c8f26d5 100644
--- a/sycl/test/e2e_test_requirements/no-unsupported-without-info.cpp
+++ b/sycl/test/e2e_test_requirements/no-unsupported-without-info.cpp
@@ -54,7 +54,7 @@
 // tests to match the required format and in that case you should just update
 // (i.e. reduce) the number and the list below.
 //
-// NUMBER-OF-UNSUPPORTED-WITHOUT-INFO: 248
+// NUMBER-OF-UNSUPPORTED-WITHOUT-INFO: 236
 //
 // List of improperly UNSUPPORTED tests.
 // Remove the CHECK once the test has been properly UNSUPPORTED.
@@ -94,22 +94,10 @@
 // CHECK-NEXT: DeprecatedFeatures/DiscardEvents/discard_events_using_assert.cpp
 // CHECK-NEXT: DeviceLib/built-ins/printf.cpp
 // CHECK-NEXT: DeviceLib/cmath-aot.cpp
-// CHECK-NEXT: DeviceLib/imf_bfloat16_integeral_convesions.cpp
-// CHECK-NEXT: DeviceLib/imf_bfloat16_integeral_convesions.cpp
-// CHECK-NEXT: DeviceLib/imf_double2bfloat16.cpp
-// CHECK-NEXT: DeviceLib/imf_double2half.cpp
-// CHECK-NEXT: DeviceLib/imf_float2bfloat16.cpp
-// CHECK-NEXT: DeviceLib/imf_fp16_trivial_test.cpp
-// CHECK-NEXT: DeviceLib/imf_fp16_trivial_test.cpp
-// CHECK-NEXT: DeviceLib/imf_fp32_rounding_test.cpp
-// CHECK-NEXT: DeviceLib/imf_fp32_test.cpp
-// CHECK-NEXT: DeviceLib/imf_fp32_test.cpp
-// CHECK-NEXT: DeviceLib/imf_fp64_rounding_test.cpp
-// CHECK-NEXT: DeviceLib/imf_fp64_test.cpp
-// CHECK-NEXT: DeviceLib/imf_fp64_test2.cpp
-// CHECK-NEXT: DeviceLib/imf_half_type_cast.cpp
-// CHECK-NEXT: DeviceLib/imf_half_type_cast.cpp
-// CHECK-NEXT: DeviceLib/imf_simd_emulate_test.cpp
+// CHECK-NEXT: DeviceLib/imf/bfloat16_integeral_convesions.cpp
+// CHECK-NEXT: DeviceLib/imf/fp16_trivial_test.cpp
+// CHECK-NEXT: DeviceLib/imf/fp32_test.cpp
+// CHECK-NEXT: DeviceLib/imf/half_type_cast.cpp
 // CHECK-NEXT: DeviceLib/separate_compile_test.cpp
 // CHECK-NEXT: ESIMD/PerformanceTests/BitonicSortK.cpp
 // CHECK-NEXT: ESIMD/PerformanceTests/BitonicSortKv2.cpp