Move RDFT to new simd scheme

Makefile.am

@@ -22,11 +22,13 @@ SIMD_LIBS =						\
 	simd-support/libsimd_sse2_nonportable.la
 
 if HAVE_SSE2
-SSE2_LIBS = dft/simd/sse2/libdft_sse2_codelets.la
+SSE2_LIBS = dft/simd/sse2/libdft_sse2_codelets.la	\
+rdft/simd/sse2/librdft_sse2_codelets.la
 endif
 
 if HAVE_AVX256D
-AVX256D_LIBS = dft/simd/avx256d/libdft_avx256d_codelets.la
+AVX256D_LIBS = dft/simd/avx256d/libdft_avx256d_codelets.la	\
+rdft/simd/avx256d/librdft_avx256d_codelets.la
 endif
 
 if SMP

configure.ac

@@ -556,6 +556,10 @@ AC_CONFIG_FILES([
    rdft/scalar/r2cf/Makefile
    rdft/scalar/r2cb/Makefile
    rdft/scalar/r2r/Makefile
+   rdft/simd/Makefile
+   rdft/simd/common/Makefile
+   rdft/simd/sse2/Makefile
+   rdft/simd/avx256d/Makefile
 
    reodft/Makefile
 

dft/simd/Makefile.am

@@ -1,3 +1,3 @@
 SUBDIRS = common sse2 avx256d
 EXTRA_DIST = n1b.h n1f.h n2b.h n2f.h n2s.h q1b.h q1f.h t1b.h t1bu.h	\
-t1f.h t1fu.h t2b.h t2f.h t3b.h t3f.h ts.h codlist.mk
+t1f.h t1fu.h t2b.h t2f.h t3b.h t3f.h ts.h codlist.mk simd.mk

genfft/gen_hc2cdft_c.ml

@@ -195,8 +195,8 @@ let generate n =
       (twinstr_to_string "VL" (twdesc n))
   and desc = 
     Printf.sprintf
-      "static const hc2c_desc desc = {%d, \"%s\", twinstr, &GENUS, %s};\n\n"
-      n name (flops_of tree)
+      "static const hc2c_desc desc = {%d, %s, twinstr, &GENUS, %s};\n\n"
+      n (stringify name) (flops_of tree)
   and register = "X(khc2c_register)"
 
   in

kernel/ifftw.h

@@ -90,6 +90,7 @@ extern void X(extract_reim)(int sign, R *c, R **r, R **i);
 #define STRINGIZEx(x) #x
 #define STRINGIZE(x) STRINGIZEx(x)
 
+/* define HAVE_SIMD if any simd extensions are supported */
 #if defined(HAVE_SSE) || defined(HAVE_SSE2) || defined(HAVE_ALTIVEC) || \
      defined(HAVE_MIPS_PS) || defined(HAVE_AVX256D)
 #define HAVE_SIMD 1

rdft/Makefile.am

@@ -1,5 +1,5 @@
 AM_CPPFLAGS = -I$(top_srcdir)/kernel -I$(top_srcdir)/dft
-SUBDIRS = scalar #simd
+SUBDIRS = scalar simd
 
 noinst_LTLIBRARIES = librdft.la
 

rdft/codelet-rdft.h

@@ -135,7 +135,8 @@ void X(khc2c_register)(planner *p, khc2c codelet, const hc2c_desc *desc,
 
 extern const solvtab X(solvtab_rdft_r2cf);
 extern const solvtab X(solvtab_rdft_r2cb);
-extern const solvtab X(solvtab_rdft_simd);
+extern const solvtab X(solvtab_rdft_sse2);
+extern const solvtab X(solvtab_rdft_avx256d);
 
 /* real-input & output DFT-like codelets (DHT, etc.) */
 typedef struct kr2r_desc_s kr2r_desc;

rdft/conf.c

@@ -58,7 +58,12 @@ void X(rdft_conf_standard)(planner *p)
      X(solvtab_exec)(X(solvtab_rdft_r2cb), p);
      X(solvtab_exec)(X(solvtab_rdft_r2r), p);
 
-#if HAVE_SIMD
-/*     X(solvtab_exec)(X(solvtab_rdft_simd), p); */
+#if HAVE_SSE2
+     if (X(have_simd_sse2)())
+	  X(solvtab_exec)(X(solvtab_rdft_sse2), p);
+#endif
+#if HAVE_AVX256D
+     if (X(have_simd_avx256d)())
+	  X(solvtab_exec)(X(solvtab_rdft_avx256d), p);
 #endif
 }

rdft/simd/Makefile.am

@@ -1,6 +1,2 @@
-AM_CPPFLAGS = -I$(top_srcdir)/kernel -I$(top_srcdir)/rdft	\
--I$(top_srcdir)/simd
-SUBDIRS = codelets
-noinst_LTLIBRARIES = librdft_simd.la
-
-librdft_simd_la_SOURCES = hc2cbv.c hc2cfv.c hc2cbv.h hc2cfv.h
+SUBDIRS = common sse2 avx256d
+EXTRA_DIST = hc2cbv.h hc2cfv.h codlist.mk simd.mk

rdft/simd/avx256d/Makefile.am

@@ -0,0 +1,14 @@
+AM_CFLAGS = $(AVX_CFLAGS)
+noinst_LTLIBRARIES = librdft_avx256d_codelets.la
+SIMD_HEADER=simd-avx256d.h
+
+include $(top_srcdir)/rdft/simd/codlist.mk
+include $(top_srcdir)/rdft/simd/simd.mk
+
+if HAVE_AVX256D
+BUILT_SOURCES = $(EXTRA_DIST)
+endif
+
+librdft_avx256d_codelets_la_SOURCES = $(BUILT_SOURCES)
+
+

rdft/simd/codlist.mk

@@ -0,0 +1,22 @@
+# This file contains a standard list of RDFT SIMD codelets.  It is
+# included by common/Makefile to generate the C files with the actual
+# codelets in them.  It is included by {sse,sse2,...}/Makefile to
+# generate and compile stub files that include common/*.c
+
+# You can customize FFTW for special needs, e.g. to handle certain
+# sizes more efficiently, by adding new codelets to the lists of those
+# included by default.  If you change the list of codelets, any new
+# ones you added will be automatically generated when you run the
+# bootstrap script (see "Generating your own code" in the FFTW
+# manual).
+
+HC2CFDFTV = hc2cfdftv_2.c hc2cfdftv_4.c hc2cfdftv_6.c hc2cfdftv_8.c	\
+hc2cfdftv_10.c hc2cfdftv_12.c hc2cfdftv_16.c hc2cfdftv_32.c		\
+hc2cfdftv_20.c
+
+HC2CBDFTV = hc2cbdftv_2.c hc2cbdftv_4.c hc2cbdftv_6.c hc2cbdftv_8.c	\
+hc2cbdftv_10.c hc2cbdftv_12.c hc2cbdftv_16.c hc2cbdftv_32.c		\
+hc2cbdftv_20.c
+
+###########################################################################
+SIMD_CODELETS = $(HC2CFDFTV) $(HC2CBDFTV)

rdft/simd/common/Makefile.am

@@ -0,0 +1,24 @@
+# include the list of codelets
+
+include $(top_srcdir)/rdft/simd/codlist.mk
+
+ALL_CODELETS = $(SIMD_CODELETS)
+BUILT_SOURCES= $(SIMD_CODELETS) $(CODLIST)
+EXTRA_DIST = $(BUILT_SOURCES) genus.c
+INCLUDE_SIMD_HEADER="\#include SIMD_HEADER"
+XRENAME=XSIMD
+SOLVTAB_NAME = XSIMD(solvtab_rdft)
+
+# include special rules for regenerating codelets.
+include $(top_srcdir)/support/Makefile.codelets
+
+if MAINTAINER_MODE
+FLAGS_HC2C=-simd $(FLAGS_COMMON) -pipeline-latency 8 -trivial-stores -variables 32 -no-generate-bytw
+
+hc2cfdftv_%.c:  $(CODELET_DEPS) $(GEN_HC2CDFT_C)
+	($(PRELUDE_COMMANDS_RDFT); $(TWOVERS) $(GEN_HC2CDFT_C) $(FLAGS_HC2C) -n $* -dit -name hc2cfdftv_$* -include "hc2cfv.h") | $(ADD_DATE) | $(INDENT) >$@
+
+hc2cbdftv_%.c:  $(CODELET_DEPS) $(GEN_HC2CDFT_C)
+	($(PRELUDE_COMMANDS_RDFT); $(TWOVERS) $(GEN_HC2CDFT_C) $(FLAGS_HC2C) -n $* -dif -sign 1 -name hc2cbdftv_$* -include "hc2cbv.h") | $(ADD_DATE) | $(INDENT) >$@
+
+endif # MAINTAINER_MODE

rdft/simd/hc2cbv.c → rdft/simd/common/genus.c

@@ -19,14 +19,33 @@
  */
 
 #include "codelet-rdft.h"
-#include "hc2cbv.h"
+#include SIMD_HEADER
 
-#if HAVE_SIMD
-static int okp(const R *Rp, const R *Ip, const R *Rm, const R *Im, 
-	       INT rs, INT mb, INT me, INT ms, 
-	       const planner *plnr)
+#define EXTERN_CONST(t, x) extern const t x; const t x
+
+static int hc2cbv_okp(const R *Rp, const R *Ip, const R *Rm, const R *Im, 
+		      INT rs, INT mb, INT me, INT ms, 
+		      const planner *plnr)
+{
+     return (1
+	     && !NO_SIMDP(plnr)
+	     && SIMD_STRIDE_OK(rs)
+	     && SIMD_VSTRIDE_OK(ms)
+             && ((me - mb) % VL) == 0
+             && ((mb - 1) % VL) == 0 /* twiddle factors alignment */
+	     && ALIGNED(Rp)
+	     && ALIGNED(Rm)
+	     && Ip == Rp + 1
+	     && Im == Rm + 1);
+}
+
+EXTERN_CONST(hc2c_genus, XSIMD(rdft_hc2cbv_genus)) = { hc2cbv_okp, HC2R, VL };
+
+static int hc2cfv_okp(const R *Rp, const R *Ip, const R *Rm, const R *Im, 
+		      INT rs, INT mb, INT me, INT ms, 
+		      const planner *plnr)
 {
-     return (RIGHT_CPU()
+     return (1
 	     && !NO_SIMDP(plnr)
 	     && SIMD_STRIDE_OK(rs)
 	     && SIMD_VSTRIDE_OK(ms)
@@ -38,5 +57,4 @@ static int okp(const R *Rp, const R *Ip, const R *Rm, const R *Im,
 	     && Im == Rm + 1);
 }
 
-const hc2c_genus GENUS = { okp, HC2R, VL };
-#endif
+EXTERN_CONST(hc2c_genus, XSIMD(rdft_hc2cfv_genus)) = { hc2cfv_okp, R2HC, VL };

rdft/simd/hc2cbv.h

@@ -18,11 +18,11 @@
  *
  */
 
-#include "simd.h"
+#include SIMD_HEADER
 
 #define VTW VTW3
 #define TWVL TWVL3
 #define LDW(x) LDA(x, 0, 0)
 
-#define GENUS X(rdft_hc2cbv_genus)
+#define GENUS XSIMD(rdft_hc2cbv_genus)
 extern const hc2c_genus GENUS;

rdft/simd/hc2cfv.h

@@ -18,11 +18,11 @@
  *
  */
 
-#include "simd.h"
+#include SIMD_HEADER
 
 #define VTW VTW3
 #define TWVL TWVL3
 #define LDW(x) LDA(x, 0, 0)
 
-#define GENUS X(rdft_hc2cfv_genus)
+#define GENUS XSIMD(rdft_hc2cfv_genus)
 extern const hc2c_genus GENUS;

rdft/simd/simd.mk

@@ -0,0 +1,12 @@
+AM_CPPFLAGS = -I$(top_srcdir)/kernel -I$(top_srcdir)/rdft	\
+-I$(top_srcdir)/rdft/simd -I$(top_srcdir)/simd-support
+
+EXTRA_DIST = $(SIMD_CODELETS) genus.c codlist.c
+
+$(EXTRA_DIST): Makefile
+	(							\
+	echo "/* Generated automatically.  DO NOT EDIT! */";	\
+	echo "#define SIMD_HEADER \"$(SIMD_HEADER)\"";		\
+	echo "#include \"../common/"$*".c\"";			\
+	) >$@
+

rdft/simd/sse2/Makefile.am

@@ -0,0 +1,14 @@
+AM_CFLAGS = $(SSE2_CFLAGS)
+noinst_LTLIBRARIES = librdft_sse2_codelets.la
+SIMD_HEADER=simd-sse2.h
+
+include $(top_srcdir)/rdft/simd/codlist.mk
+include $(top_srcdir)/rdft/simd/simd.mk
+
+if HAVE_SSE2
+BUILT_SOURCES = $(EXTRA_DIST)
+endif
+
+librdft_sse2_codelets_la_SOURCES = $(BUILT_SOURCES)
+
+