| | |
| | | #include <immintrin.h> |
| | | #include <smmintrin.h> |
| | | |
| | | #if defined(_MSC_VER) && _MSC_VER <= 1900 |
| | | static inline __int32 _mm256_extract_epi64(__m256i a, const int index) { |
| | | return a.m256i_i64[index]; |
| | | } |
| | |
| | | return a.m256i_i32[index]; |
| | | } |
| | | |
| | | #endif |
| | | |
| | | static inline float _castu32_f32(uint32_t a) { |
| | | return *((float *)&a); |
| | | } |
| | | |
| | | static inline float _mm256_extract_float32(__m256 a, const int index) { |
| | | return _castu32_f32(_mm256_extract_epi32(_mm256_castps_si256(a), index)); |
| | | return a.m256_f32[index]; |
| | | } |
| | | |
| | | #else // Linux GCC/Clang |
| | |
| | | + _mm256_extract_epi64(val, 3); |
| | | } |
| | | |
| | | // 5x times faster than gemm()-float32 |
| | | // further optimizations: do mean-mult only for the last layer |
| | | void gemm_nn_custom_bin_mean_transposed(int M, int N, int K, float ALPHA_UNUSED, |
| | | unsigned char *A, int lda, |
| | | unsigned char *B, int ldb, |
| | |
| | | int channels_col = channels * ksize * ksize; |
| | | |
| | | // optimized version |
| | | if (height_col == height && width_col == width && stride == 1 && pad == 1) |
| | | if (height_col == height && width_col == width && stride == 1 && pad == 1 && is_fma_avx()) |
| | | { |
| | | #pragma omp parallel for |
| | | for (c = 0; c < channels_col; ++c) { |
| | |
| | | } |
| | | } |
| | | |
| | | void transpose_8x8_bits(unsigned char A[8], unsigned char B[8], int m, int n) |
| | | { |
| | | unsigned x, y, t; |
| | | |
| | | // Load the array and pack it into x and y. |
| | | |
| | | x = (A[0] << 24) | (A[m] << 16) | (A[2 * m] << 8) | A[3 * m]; |
| | | y = (A[4 * m] << 24) | (A[5 * m] << 16) | (A[6 * m] << 8) | A[7 * m]; |
| | | |
| | | t = (x ^ (x >> 7)) & 0x00AA00AA; x = x ^ t ^ (t << 7); |
| | | t = (y ^ (y >> 7)) & 0x00AA00AA; y = y ^ t ^ (t << 7); |
| | | |
| | | t = (x ^ (x >> 14)) & 0x0000CCCC; x = x ^ t ^ (t << 14); |
| | | t = (y ^ (y >> 14)) & 0x0000CCCC; y = y ^ t ^ (t << 14); |
| | | |
| | | t = (x & 0xF0F0F0F0) | ((y >> 4) & 0x0F0F0F0F); |
| | | y = ((x << 4) & 0xF0F0F0F0) | (y & 0x0F0F0F0F); |
| | | x = t; |
| | | |
| | | B[0] = x >> 24; B[n] = x >> 16; B[2 * n] = x >> 8; B[3 * n] = x; |
| | | B[4 * n] = y >> 24; B[5 * n] = y >> 16; B[6 * n] = y >> 8; B[7 * n] = y; |
| | | } |
| | | |
| | | void activate_array_cpu_custom(float *x, const int n, const ACTIVATION a) |
| | | { |
| | | int i; |
| | | int i = 0; |
| | | if (a == LINEAR) |
| | | {} |
| | | else if (a == LEAKY) |
| | | { |
| | | __m256i all256_sing1 = _mm256_set_epi32(0x80000000, 0x80000000, 0x80000000, 0x80000000, 0x80000000, 0x80000000, 0x80000000, 0x80000000); |
| | | __m256 all256_01 = _mm256_set1_ps(0.1F); |
| | | if (is_fma_avx()) { |
| | | __m256i all256_sing1 = _mm256_set_epi32(0x80000000, 0x80000000, 0x80000000, 0x80000000, 0x80000000, 0x80000000, 0x80000000, 0x80000000); |
| | | __m256 all256_01 = _mm256_set1_ps(0.1F); |
| | | |
| | | for (i = 0; i < n-8; i += 8) { |
| | | //x[i] = (x[i]>0) ? x[i] : .1*x[i]; |
| | | for (i = 0; i < n - 8; i += 8) { |
| | | //x[i] = (x[i]>0) ? x[i] : .1*x[i]; |
| | | |
| | | __m256 src256 = _mm256_loadu_ps(&x[i]); |
| | | __m256 mult256 = _mm256_mul_ps((src256), all256_01); // mult * 0.1 |
| | | __m256 src256 = _mm256_loadu_ps(&x[i]); |
| | | __m256 mult256 = _mm256_mul_ps((src256), all256_01); // mult * 0.1 |
| | | |
| | | __m256i sign256 = _mm256_and_si256(_mm256_castps_si256(src256), all256_sing1); // check sign in 8 x 32-bit floats |
| | | __m256i sign256 = _mm256_and_si256(_mm256_castps_si256(src256), all256_sing1); // check sign in 8 x 32-bit floats |
| | | |
| | | __m256 result256 = _mm256_blendv_ps(src256, mult256, _mm256_castsi256_ps(sign256)); // (sign>0) ? src : mult; |
| | | _mm256_storeu_ps(&x[i], result256); |
| | | __m256 result256 = _mm256_blendv_ps(src256, mult256, _mm256_castsi256_ps(sign256)); // (sign>0) ? src : mult; |
| | | _mm256_storeu_ps(&x[i], result256); |
| | | } |
| | | } |
| | | |
| | | for (; i < n; ++i) { |