Using vector multiply-accumulate in NEON kernels

Aprox 10% of improvememnt

src/algorithms/libs/volk_gnsssdr_module/volk_gnsssdr/kernels/volk_gnsssdr/volk_gnsssdr_16ic_resamplerxnpuppet_16ic.h

@@ -68,6 +68,7 @@ static inline void volk_gnsssdr_16ic_resamplerxnpuppet_16ic_generic(lv_16sc_t* r
 
 #endif /* LV_HAVE_GENERIC */
 
+
 #ifdef LV_HAVE_SSE2
 static inline void volk_gnsssdr_16ic_resamplerxnpuppet_16ic_a_sse2(lv_16sc_t* result, const lv_16sc_t* local_code, unsigned int num_points)
 {
@@ -94,8 +95,8 @@ static inline void volk_gnsssdr_16ic_resamplerxnpuppet_16ic_a_sse2(lv_16sc_t* re
 
 #endif
 
-#ifdef LV_HAVE_SSE2
 
+#ifdef LV_HAVE_SSE2
 static inline void volk_gnsssdr_16ic_resamplerxnpuppet_16ic_u_sse2(lv_16sc_t* result, const lv_16sc_t* local_code, unsigned int num_points)
 {
     float code_phase_step_chips = 0.1;
@@ -121,8 +122,8 @@ static inline void volk_gnsssdr_16ic_resamplerxnpuppet_16ic_u_sse2(lv_16sc_t* re
 
 #endif
 
-#ifdef LV_HAVE_NEON
 
+#ifdef LV_HAVE_NEON
 static inline void volk_gnsssdr_16ic_resamplerxnpuppet_16ic_neon(lv_16sc_t* result, const lv_16sc_t* local_code, unsigned int num_points)
 {
     float code_phase_step_chips = 0.1;

src/algorithms/libs/volk_gnsssdr_module/volk_gnsssdr/kernels/volk_gnsssdr/volk_gnsssdr_16ic_rotatorpuppet_16ic.h

@@ -42,7 +42,6 @@
 
 
 #ifdef LV_HAVE_GENERIC
-
 static inline void volk_gnsssdr_16ic_rotatorpuppet_16ic_generic(lv_16sc_t* outVector, const lv_16sc_t* inVector, unsigned int num_points)
 {
     // phases must be normalized. Phase rotator expects a complex exponential input!
@@ -59,7 +58,6 @@ static inline void volk_gnsssdr_16ic_rotatorpuppet_16ic_generic(lv_16sc_t* outVe
 
 
 #ifdef LV_HAVE_SSE3
-
 static inline void volk_gnsssdr_16ic_rotatorpuppet_16ic_a_sse3(lv_16sc_t* outVector, const lv_16sc_t* inVector, unsigned int num_points)
 {
     // phases must be normalized. Phase rotator expects a complex exponential input!
@@ -74,8 +72,8 @@ static inline void volk_gnsssdr_16ic_rotatorpuppet_16ic_a_sse3(lv_16sc_t* outVec
 
 #endif /* LV_HAVE_SSE3 */
 
-#ifdef LV_HAVE_SSE3
 
+#ifdef LV_HAVE_SSE3
 static inline void volk_gnsssdr_16ic_rotatorpuppet_16ic_u_sse3(lv_16sc_t* outVector, const lv_16sc_t* inVector, unsigned int num_points)
 {
     // phases must be normalized. Phase rotator expects a complex exponential input!
@@ -90,8 +88,8 @@ static inline void volk_gnsssdr_16ic_rotatorpuppet_16ic_u_sse3(lv_16sc_t* outVec
 
 #endif /* LV_HAVE_SSE3 */
 
-#ifdef LV_HAVE_NEON
 
+#ifdef LV_HAVE_NEON
 static inline void volk_gnsssdr_16ic_rotatorpuppet_16ic_neon(lv_16sc_t* outVector, const lv_16sc_t* inVector, unsigned int num_points)
 {
     // phases must be normalized. Phase rotator expects a complex exponential input!

src/algorithms/libs/volk_gnsssdr_module/volk_gnsssdr/kernels/volk_gnsssdr/volk_gnsssdr_16ic_x2_dot_prod_16ic.h

@@ -294,10 +294,11 @@ static inline void volk_gnsssdr_16ic_x2_dot_prod_16ic_neon(lv_16sc_t* out, const
 
 #endif /* LV_HAVE_NEON */
 
+
 #ifdef LV_HAVE_NEON
 #include <arm_neon.h>
 
-static inline void volk_gnsssdr_16ic_x2_dot_prod_16ic_neon_fma(lv_16sc_t* out, const lv_16sc_t* in_a, const lv_16sc_t* in_b, unsigned int num_points)
+static inline void volk_gnsssdr_16ic_x2_dot_prod_16ic_neon_vma(lv_16sc_t* out, const lv_16sc_t* in_a, const lv_16sc_t* in_b, unsigned int num_points)
 {
     unsigned int quarter_points = num_points / 4;
     unsigned int number;

src/algorithms/libs/volk_gnsssdr_module/volk_gnsssdr/kernels/volk_gnsssdr/volk_gnsssdr_16ic_x2_dot_prod_16ic_xn.h

@@ -354,4 +354,81 @@ static inline void volk_gnsssdr_16ic_x2_dot_prod_16ic_xn_neon(lv_16sc_t* result,
 }
 #endif /* LV_HAVE_NEON */
 
+
+#ifdef LV_HAVE_NEON
+#include <arm_neon.h>
+
+static inline void volk_gnsssdr_16ic_x2_dot_prod_16ic_xn_neon_vma(lv_16sc_t* result, const lv_16sc_t* in_common, const lv_16sc_t** in_a,  int num_a_vectors, unsigned int num_points)
+{
+    lv_16sc_t dotProduct = lv_cmake(0,0);
+
+    const unsigned int neon_iters = num_points / 4;
+
+    const lv_16sc_t** _in_a = in_a;
+    const lv_16sc_t* _in_common = in_common;
+    lv_16sc_t* _out = result;
+
+    if (neon_iters > 0)
+        {
+            __VOLK_ATTR_ALIGNED(16) lv_16sc_t dotProductVector[4];
+
+            int16x4x2_t a_val, b_val, tmp;
+
+            int16x4x2_t* accumulator;
+            accumulator = (int16x4x2_t*)malloc(num_a_vectors * sizeof(int16x4x2_t));
+
+            for(int n_vec = 0; n_vec < num_a_vectors; n_vec++)
+                {
+                    accumulator[n_vec].val[0] = vdup_n_s16(0);
+                    accumulator[n_vec].val[1] = vdup_n_s16(0);
+                }
+
+            for(unsigned int number = 0; number < neon_iters; number++)
+                {
+                    b_val = vld2_s16((int16_t*)_in_common); //load (2 byte imag, 2 byte real) x 4 into 128 bits reg
+                    __builtin_prefetch(_in_common + 8);
+                    for (int n_vec = 0; n_vec < num_a_vectors; n_vec++)
+                        {
+                            a_val = vld2_s16((int16_t*)&(_in_a[n_vec][number*4]));
+
+                            tmp.val[0] = vmul_s16(a_val.val[0], b_val.val[0]);
+                            tmp.val[1] = vmul_s16(a_val.val[1], b_val.val[0]);
+
+                            // use multiply accumulate/subtract to get result
+                            tmp.val[0] = vmls_s16(tmp.val[0], a_val.val[1], b_val.val[1]);
+                            tmp.val[1] = vmla_s16(tmp.val[1], a_val.val[0], b_val.val[1]);
+
+                            accumulator[n_vec].val[0] = vadd_s16(accumulator[n_vec].val[0], tmp.val[0]);
+                            accumulator[n_vec].val[1] = vadd_s16(accumulator[n_vec].val[1], tmp.val[1]);
+                        }
+                    _in_common += 4;
+                }
+
+            for (int n_vec = 0; n_vec < num_a_vectors; n_vec++)
+                {
+                    vst2_s16((int16_t*)dotProductVector, accumulator[n_vec]); // Store the results back into the dot product vector
+                    dotProduct = lv_cmake(0,0);
+                    for (int i = 0; i < 4; ++i)
+                        {
+                            dotProduct = lv_cmake(sat_adds16i(lv_creal(dotProduct), lv_creal(dotProductVector[i])),
+                                    sat_adds16i(lv_cimag(dotProduct), lv_cimag(dotProductVector[i])));
+                        }
+                    _out[n_vec] = dotProduct;
+                }
+            free(accumulator);
+        }
+
+    for (int n_vec = 0; n_vec < num_a_vectors; n_vec++)
+        {
+            for(unsigned int n  = neon_iters * 4; n < num_points; n++)
+                {
+                    lv_16sc_t tmp = in_common[n] * in_a[n_vec][n];
+
+                    _out[n_vec] = lv_cmake(sat_adds16i(lv_creal(_out[n_vec]), lv_creal(tmp)),
+                            sat_adds16i(lv_cimag(_out[n_vec]), lv_cimag(tmp)));
+                }
+        }
+}
+#endif /* LV_HAVE_NEON */
+
 #endif /*INCLUDED_volk_gnsssdr_16ic_xn_dot_prod_16ic_xn_H*/

src/algorithms/libs/volk_gnsssdr_module/volk_gnsssdr/kernels/volk_gnsssdr/volk_gnsssdr_16ic_x2_dotprodxnpuppet_16ic.h

@@ -109,6 +109,7 @@ static inline void volk_gnsssdr_16ic_x2_dotprodxnpuppet_16ic_u_sse2(lv_16sc_t* r
 
 #endif /* LV_HAVE_SSE2 && LV_HAVE_64 */
 
+
 #ifdef LV_HAVE_NEON
 
 static inline void volk_gnsssdr_16ic_x2_dotprodxnpuppet_16ic_neon(lv_16sc_t* result, const lv_16sc_t* local_code, const lv_16sc_t* in, unsigned int num_points)
@@ -132,6 +133,29 @@ static inline void volk_gnsssdr_16ic_x2_dotprodxnpuppet_16ic_neon(lv_16sc_t* res
 
 #endif // NEON
 
+
+#ifdef LV_HAVE_NEON
+
+static inline void volk_gnsssdr_16ic_x2_dotprodxnpuppet_16ic_neon_vma(lv_16sc_t* result, const lv_16sc_t* local_code, const lv_16sc_t* in, unsigned int num_points)
+{
+    int num_a_vectors = 3;
+    lv_16sc_t** in_a = (lv_16sc_t**)volk_gnsssdr_malloc(sizeof(lv_16sc_t*) * num_a_vectors, volk_gnsssdr_get_alignment());
+    for(unsigned int n = 0; n < num_a_vectors; n++)
+    {
+       in_a[n] = (lv_16sc_t*)volk_gnsssdr_malloc(sizeof(lv_16sc_t)*num_points, volk_gnsssdr_get_alignment());
+       memcpy((lv_16sc_t*)in_a[n], (lv_16sc_t*)in, sizeof(lv_16sc_t)*num_points);
+    }
+
+    volk_gnsssdr_16ic_x2_dot_prod_16ic_xn_neon_vma(result, local_code, (const lv_16sc_t**) in_a, num_a_vectors, num_points);
+
+    for(unsigned int n = 0; n < num_a_vectors; n++)
+    {
+        volk_gnsssdr_free(in_a[n]);
+    }
+    volk_gnsssdr_free(in_a);
+}
+
+#endif // NEON
 #endif  // INCLUDED_volk_gnsssdr_16ic_x2_dotprodxnpuppet_16ic_H
 
 

src/algorithms/libs/volk_gnsssdr_module/volk_gnsssdr/kernels/volk_gnsssdr/volk_gnsssdr_16ic_x2_rotator_dot_prod_16ic_xn.h

@@ -574,10 +574,11 @@ static inline void volk_gnsssdr_16ic_x2_rotator_dot_prod_16ic_xn_neon(lv_16sc_t*
 
 #endif /* LV_HAVE_NEON */
 
+
 #ifdef LV_HAVE_NEON
 #include <arm_neon.h>
 
-static inline void volk_gnsssdr_16ic_x2_rotator_dot_prod_16ic_xn_neon_fma(lv_16sc_t* result, const lv_16sc_t* in_common, const lv_32fc_t phase_inc, lv_32fc_t* phase, const lv_16sc_t** in_a,  int num_a_vectors, unsigned int num_points)
+static inline void volk_gnsssdr_16ic_x2_rotator_dot_prod_16ic_xn_neon_vma(lv_16sc_t* result, const lv_16sc_t* in_common, const lv_32fc_t phase_inc, lv_32fc_t* phase, const lv_16sc_t** in_a,  int num_a_vectors, unsigned int num_points)
 {
     const unsigned int neon_iters = num_points / 4;
 

src/algorithms/libs/volk_gnsssdr_module/volk_gnsssdr/kernels/volk_gnsssdr/volk_gnsssdr_16ic_x2_rotator_dotprodxnpuppet_16ic.h

@@ -69,6 +69,7 @@ static inline void volk_gnsssdr_16ic_x2_rotator_dotprodxnpuppet_16ic_generic(lv_
 
 #endif  // Generic
 
+
 #ifdef LV_HAVE_SSE3
 static inline void volk_gnsssdr_16ic_x2_rotator_dotprodxnpuppet_16ic_a_sse3(lv_16sc_t* result, const lv_16sc_t* local_code, const lv_16sc_t* in, unsigned int num_points)
 {
@@ -99,8 +100,8 @@ static inline void volk_gnsssdr_16ic_x2_rotator_dotprodxnpuppet_16ic_a_sse3(lv_1
 
 #endif // SSE3
 
-#ifdef LV_HAVE_SSE3
 
+#ifdef LV_HAVE_SSE3
 static inline void volk_gnsssdr_16ic_x2_rotator_dotprodxnpuppet_16ic_u_sse3(lv_16sc_t* result, const lv_16sc_t* local_code, const lv_16sc_t* in, unsigned int num_points)
 {
     // phases must be normalized. Phase rotator expects a complex exponential input!
@@ -130,8 +131,8 @@ static inline void volk_gnsssdr_16ic_x2_rotator_dotprodxnpuppet_16ic_u_sse3(lv_1
 
 #endif // SSE3
 
-#ifdef LV_HAVE_NEON
 
+#ifdef LV_HAVE_NEON
 static inline void volk_gnsssdr_16ic_x2_rotator_dotprodxnpuppet_16ic_neon(lv_16sc_t* result, const lv_16sc_t* local_code, const lv_16sc_t* in, unsigned int num_points)
 {
     // phases must be normalized. Phase rotator expects a complex exponential input!
@@ -161,9 +162,9 @@ static inline void volk_gnsssdr_16ic_x2_rotator_dotprodxnpuppet_16ic_neon(lv_16s
 
 #endif // NEON
 
-#ifdef LV_HAVE_NEON
 
-static inline void volk_gnsssdr_16ic_x2_rotator_dotprodxnpuppet_16ic_neon_fma(lv_16sc_t* result, const lv_16sc_t* local_code, const lv_16sc_t* in, unsigned int num_points)
+#ifdef LV_HAVE_NEON
+static inline void volk_gnsssdr_16ic_x2_rotator_dotprodxnpuppet_16ic_neon_vma(lv_16sc_t* result, const lv_16sc_t* local_code, const lv_16sc_t* in, unsigned int num_points)
 {
     // phases must be normalized. Phase rotator expects a complex exponential input!
     float rem_carrier_phase_in_rad = 0.345;
@@ -181,7 +182,7 @@ static inline void volk_gnsssdr_16ic_x2_rotator_dotprodxnpuppet_16ic_neon_fma(lv
             memcpy((lv_16sc_t*)in_a[n], (lv_16sc_t*)in, sizeof(lv_16sc_t) * num_points);
         }
 
-    volk_gnsssdr_16ic_x2_rotator_dot_prod_16ic_xn_neon_fma(result, local_code, phase_inc[0], phase, (const lv_16sc_t**) in_a, num_a_vectors, num_points);
+    volk_gnsssdr_16ic_x2_rotator_dot_prod_16ic_xn_neon_vma(result, local_code, phase_inc[0], phase, (const lv_16sc_t**) in_a, num_a_vectors, num_points);
 
     for(unsigned int n = 0; n < num_a_vectors; n++)
         {