Integrate SATD ASM for SSSE3 and SSE4.1 (#1813)

This also involves adding a new CpuFeatureLevel for SSE4.1.

Author: shssoichiro

src/asm/x86/dist.rs

@@ -45,6 +45,7 @@ declare_asm_dist_fn![
   // SSSE3
   (rav1e_sad_4x4_hbd_ssse3, u16),
   (rav1e_sad_16x16_hbd_ssse3, u16),
+  (rav1e_satd_8x8_ssse3, u8),
   // SSE2
   (rav1e_sad4x4_sse2, u8),
   (rav1e_sad4x8_sse2, u8),
@@ -57,6 +58,8 @@ declare_asm_dist_fn![
   (rav1e_sad32x32_sse2, u8),
   (rav1e_sad64x64_sse2, u8),
   (rav1e_sad128x128_sse2, u8),
+  // SSE4
+  (rav1e_satd_4x4_sse4, u8),
   // AVX
   (rav1e_sad16x4_avx2, u8),
   (rav1e_sad16x8_avx2, u8),
@@ -255,6 +258,7 @@ pub static SAD_FNS: [[Option<SadFn>; DIST_FNS_LENGTH];
 
   out[CpuFeatureLevel::SSE2 as usize] = SAD_FNS_SSE2;
   out[CpuFeatureLevel::SSSE3 as usize] = SAD_FNS_SSE2;
+  out[CpuFeatureLevel::SSE4_1 as usize] = SAD_FNS_SSE2;
   out[CpuFeatureLevel::AVX2 as usize] = SAD_FNS_AVX2;
 
   out
@@ -276,11 +280,33 @@ pub(crate) static SAD_HBD_FNS: [[Option<SadHBDFn>; DIST_FNS_LENGTH];
   let mut out = [[None; DIST_FNS_LENGTH]; CpuFeatureLevel::len()];
 
   out[CpuFeatureLevel::SSSE3 as usize] = SAD_HBD_FNS_SSSE3;
+  out[CpuFeatureLevel::SSE4_1 as usize] = SAD_HBD_FNS_SSSE3;
   out[CpuFeatureLevel::AVX2 as usize] = SAD_HBD_FNS_SSSE3;
 
   out
 };
 
+static SATD_FNS_SSSE3: [Option<SatdFn>; DIST_FNS_LENGTH] = {
+  let mut out: [Option<SatdFn>; DIST_FNS_LENGTH] = [None; DIST_FNS_LENGTH];
+
+  use BlockSize::*;
+
+  out[BLOCK_8X8 as usize] = Some(rav1e_satd_8x8_ssse3);
+
+  out
+};
+
+static SATD_FNS_SSE4: [Option<SatdFn>; DIST_FNS_LENGTH] = {
+  let mut out: [Option<SatdFn>; DIST_FNS_LENGTH] = [None; DIST_FNS_LENGTH];
+
+  use BlockSize::*;
+
+  out[BLOCK_4X4 as usize] = Some(rav1e_satd_4x4_sse4);
+  out[BLOCK_8X8 as usize] = Some(rav1e_satd_8x8_ssse3);
+
+  out
+};
+
 static SATD_FNS_AVX2: [Option<SatdFn>; DIST_FNS_LENGTH] = {
   let mut out: [Option<SatdFn>; DIST_FNS_LENGTH] = [None; DIST_FNS_LENGTH];
 
@@ -318,6 +344,8 @@ pub(crate) static SATD_FNS: [[Option<SatdFn>; DIST_FNS_LENGTH];
   CpuFeatureLevel::len()] = {
   let mut out = [[None; DIST_FNS_LENGTH]; CpuFeatureLevel::len()];
 
+  out[CpuFeatureLevel::SSSE3 as usize] = SATD_FNS_SSSE3;
+  out[CpuFeatureLevel::SSE4_1 as usize] = SATD_FNS_SSE4;
   out[CpuFeatureLevel::AVX2 as usize] = SATD_FNS_AVX2;
 
   out
@@ -416,6 +444,10 @@ mod test {
     "avx2"
   );
 
+  test_dist_fns!((8, 8), satd, 8, ssse3, "ssse3");
+
+  test_dist_fns!((4, 4), satd, 8, sse4, "sse4.1");
+
   test_dist_fns!(
     (4, 4),
     (8, 8),

src/asm/x86/mc.rs

@@ -313,6 +313,7 @@ decl_mc_fns!(
 pub(crate) static PUT_FNS: [[Option<PutFn>; 16]; CpuFeatureLevel::len()] = {
   let mut out = [[None; 16]; CpuFeatureLevel::len()];
   out[CpuFeatureLevel::SSSE3 as usize] = PUT_FNS_SSSE3;
+  out[CpuFeatureLevel::SSE4_1 as usize] = PUT_FNS_SSSE3;
   out[CpuFeatureLevel::AVX2 as usize] = PUT_FNS_AVX2;
   out
 };
@@ -372,6 +373,7 @@ decl_mct_fns!(
 pub(crate) static PREP_FNS: [[Option<PrepFn>; 16]; CpuFeatureLevel::len()] = {
   let mut out = [[None; 16]; CpuFeatureLevel::len()];
   out[CpuFeatureLevel::SSSE3 as usize] = PREP_FNS_SSSE3;
+  out[CpuFeatureLevel::SSE4_1 as usize] = PREP_FNS_SSSE3;
   out[CpuFeatureLevel::AVX2 as usize] = PREP_FNS_AVX2;
   out
 };
@@ -395,6 +397,7 @@ pub(crate) static AVG_FNS: [Option<AvgFn>; CpuFeatureLevel::len()] = {
   let mut out: [Option<AvgFn>; CpuFeatureLevel::len()] =
     [None; CpuFeatureLevel::len()];
   out[CpuFeatureLevel::SSSE3 as usize] = Some(rav1e_avg_ssse3);
+  out[CpuFeatureLevel::SSE4_1 as usize] = Some(rav1e_avg_ssse3);
   out[CpuFeatureLevel::AVX2 as usize] = Some(rav1e_avg_avx2);
   out
 };

src/cpu_features/aarch64.rs

@@ -23,9 +23,7 @@ impl CpuFeatureLevel {
 
   #[inline(always)]
   pub fn as_index(self) -> usize {
-    const LEN: usize = CpuFeatureLevel::len();
-    assert_eq!(LEN & (LEN - 1), 0);
-    self as usize & (LEN - 1)
+    self as usize
   }
 }
 

src/cpu_features/x86.rs

@@ -9,12 +9,16 @@
 
 use arg_enum_proc_macro::ArgEnum;
 use std::env;
+use std::str::FromStr;
 
 #[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, ArgEnum)]
 pub enum CpuFeatureLevel {
+  #[arg_enum(alias = "rust")]
   NATIVE,
   SSE2,
   SSSE3,
+  #[arg_enum(alias = "sse4.1")]
+  SSE4_1,
   AVX2,
 }
 
@@ -25,16 +29,16 @@ impl CpuFeatureLevel {
 
   #[inline(always)]
   pub fn as_index(self) -> usize {
-    const LEN: usize = CpuFeatureLevel::len();
-    assert_eq!(LEN & (LEN - 1), 0);
-    self as usize & (LEN - 1)
+    self as usize
   }
 }
 
 impl Default for CpuFeatureLevel {
   fn default() -> CpuFeatureLevel {
     let detected: CpuFeatureLevel = if is_x86_feature_detected!("avx2") {
       CpuFeatureLevel::AVX2
+    } else if is_x86_feature_detected!("sse4.1") {
+      CpuFeatureLevel::SSE4_1
     } else if is_x86_feature_detected!("ssse3") {
       CpuFeatureLevel::SSSE3
     } else if is_x86_feature_detected!("sse2") {
@@ -43,13 +47,7 @@ impl Default for CpuFeatureLevel {
       CpuFeatureLevel::NATIVE
     };
     let manual: CpuFeatureLevel = match env::var("RAV1E_CPU_TARGET") {
-      Ok(feature) => match feature.as_ref() {
-        "rust" => CpuFeatureLevel::NATIVE,
-        "avx2" => CpuFeatureLevel::AVX2,
-        "ssse3" => CpuFeatureLevel::SSSE3,
-        "sse2" => CpuFeatureLevel::SSE2,
-        _ => detected,
-      },
+      Ok(feature) => CpuFeatureLevel::from_str(&feature).unwrap_or(detected),
       Err(_e) => detected,
     };
     if manual > detected {