From 264a0795cecaed5531c16367d612c252f41a5233 Mon Sep 17 00:00:00 2001 From: OlivierDehaene <23298448+OlivierDehaene@users.noreply.github.com> Date: Thu, 27 Jun 2024 17:58:21 +0200 Subject: [PATCH 1/2] feat(candle): add flash gte --- backends/candle/src/lib.rs | 25 +- backends/candle/src/models/bert.rs | 1 + backends/candle/src/models/flash_gte.rs | 451 ++++++++++++++++++ backends/candle/src/models/flash_jina.rs | 1 + backends/candle/src/models/flash_jina_code.rs | 1 + backends/candle/src/models/gte.rs | 35 ++ backends/candle/src/models/jina.rs | 1 + backends/candle/src/models/jina_code.rs | 1 + backends/candle/src/models/mod.rs | 8 + 9 files changed, 520 insertions(+), 4 deletions(-) create mode 100644 backends/candle/src/models/flash_gte.rs create mode 100644 backends/candle/src/models/gte.rs diff --git a/backends/candle/src/lib.rs b/backends/candle/src/lib.rs index b9d750dc..5204536b 100644 --- a/backends/candle/src/lib.rs +++ b/backends/candle/src/lib.rs @@ -11,13 +11,13 @@ use crate::compute_cap::{ compatible_compute_cap, get_compile_compute_cap, get_runtime_compute_cap, }; use crate::models::{ - BertConfig, BertModel, DistilBertConfig, DistilBertModel, JinaBertModel, JinaCodeBertModel, - MistralConfig, Model, NomicBertModel, NomicConfig, + BertConfig, BertModel, DistilBertConfig, DistilBertModel, GTEConfig, JinaBertModel, + JinaCodeBertModel, MistralConfig, Model, NomicBertModel, NomicConfig, }; #[cfg(feature = "cuda")] use crate::models::{ - FlashBertModel, FlashDistilBertModel, FlashJinaBertModel, FlashJinaCodeBertModel, - FlashMistralModel, FlashNomicBertModel, + FlashBertModel, FlashDistilBertModel, FlashGTEModel, FlashJinaBertModel, + FlashJinaCodeBertModel, FlashMistralModel, FlashNomicBertModel, }; use anyhow::Context; use candle::{DType, Device}; @@ -57,6 +57,8 @@ enum Config { #[serde(rename(deserialize = "nomic_bert"))] NomicBert(NomicConfig), Mistral(MistralConfig), + #[serde(rename = "new")] + Gte(GTEConfig), } pub struct CandleBackend { @@ -215,6 +217,10 @@ impl CandleBackend { "Mistral is only supported on Cuda devices in fp16 with flash attention enabled" .to_string(), )), + (Config::Gte(_), Device::Cpu | Device::Metal(_)) => Err(BackendError::Start( + "GTE is only supported on Cuda devices in fp16 with flash attention enabled" + .to_string(), + )), #[cfg(feature = "cuda")] (Config::Bert(config), Device::Cuda(_)) => { if cfg!(any(feature = "flash-attn", feature = "flash-attn-v1")) @@ -333,6 +339,17 @@ impl CandleBackend { FlashMistralModel::load(vb, &config, model_type).s()?, )) } + #[cfg(feature = "cuda")] + (Config::Gte(config), Device::Cuda(_)) => { + if dtype != DType::F16 + || !cfg!(feature = "flash-attn") + || get_runtime_compute_cap().unwrap() < 80 + { + return Err(BackendError::Start("GTE is only supported on Cuda devices in fp16 with flash attention v2 enabled".to_string())); + } + tracing::info!("Starting FlashGTE model on {:?}", device); + Ok(Box::new(FlashGTEModel::load(vb, &config, model_type).s()?)) + } }; Ok(Self { diff --git a/backends/candle/src/models/bert.rs b/backends/candle/src/models/bert.rs index 5795fa27..a36a1555 100644 --- a/backends/candle/src/models/bert.rs +++ b/backends/candle/src/models/bert.rs @@ -35,6 +35,7 @@ pub enum PositionEmbeddingType { #[default] Absolute, Alibi, + Rope, } #[derive(Debug)] diff --git a/backends/candle/src/models/flash_gte.rs b/backends/candle/src/models/flash_gte.rs new file mode 100644 index 00000000..b5a6b38e --- /dev/null +++ b/backends/candle/src/models/flash_gte.rs @@ -0,0 +1,451 @@ +use crate::flash_attn::flash_attn_varlen; +use crate::layers::{HiddenAct, LayerNorm, Linear}; +use crate::models::{GTEConfig, Model, NTKScaling, PositionEmbeddingType, RopeScaling}; +use candle::{DType, Device, IndexOp, Result, Tensor}; +use candle_nn::{Embedding, Module, VarBuilder}; +use text_embeddings_backend_core::{Batch, ModelType, Pool}; + +struct GTEAttention { + qkv_linear: Linear, + o_proj: Linear, + + num_attention_heads: usize, + attention_head_size: usize, + + softmax_scale: f32, + + span: tracing::Span, +} + +impl GTEAttention { + pub fn load(vb: VarBuilder, config: >EConfig) -> Result { + let num_attention_heads = config.num_attention_heads; + let attention_head_size = config.hidden_size / config.num_attention_heads; + let hidden_size = config.hidden_size; + + let qkv_weight = vb + .pp("qkv_proj") + .get((hidden_size * 3, hidden_size), "weight")?; + let qkv_bias = vb.pp("qkv_proj").get(hidden_size * 3, "bias")?; + + let qkv_linear = Linear::new(qkv_weight, Some(qkv_bias), None); + + let o_proj_weight = vb.pp("o_proj").get((hidden_size, hidden_size), "weight")?; + let o_proj_bias = vb.pp("o_proj").get(hidden_size, "bias")?; + + let o_proj = Linear::new(o_proj_weight, Some(o_proj_bias), None); + + let softmax_scale = (1. / (attention_head_size as f64).sqrt()) as f32; + + Ok(Self { + qkv_linear, + o_proj, + num_attention_heads, + attention_head_size, + softmax_scale, + span: tracing::span!(tracing::Level::TRACE, "attention"), + }) + } + + pub fn forward( + &self, + hidden_states: &Tensor, + cu_seqlens: &Tensor, + cos: &Tensor, + sin: &Tensor, + max_s: usize, + ) -> Result { + let _enter = self.span.enter(); + + let qkv = self.qkv_linear.forward(hidden_states)?; + + // Reshape to [tokens, heads, head_size] + let mut new_qkv_shape = qkv.dims().to_vec(); + new_qkv_shape.pop(); + new_qkv_shape.push(self.num_attention_heads * 3); + new_qkv_shape.push(self.attention_head_size); + + let qkv = qkv.reshape(new_qkv_shape)?; + + // Split qkv tensor + let q = qkv.narrow(1, 0, self.num_attention_heads)?; + let k = qkv.narrow(1, self.num_attention_heads, self.num_attention_heads)?; + let v = qkv.narrow(1, self.num_attention_heads * 2, self.num_attention_heads)?; + + candle_rotary::apply_rotary_inplace(&q, &k, &cos, &sin, true)?; + + let attention = flash_attn_varlen( + &q, + &k, + &v, + None, + cu_seqlens, + cu_seqlens, + max_s, + max_s, + self.softmax_scale, + false, + None, + )?; + let attention = attention.flatten_from(candle::D::Minus2)?; + + self.o_proj.forward(&attention) + } +} + +struct GTEMLP { + up_gate_proj: Linear, + down_proj: Linear, + + act: HiddenAct, + intermediate_size: usize, + + span: tracing::Span, +} + +impl GTEMLP { + pub fn load(vb: VarBuilder, config: >EConfig) -> Result { + let intermediate_size = config.intermediate_size; + + let up_gate_proj_weight = vb + .pp("up_gate_proj") + .get((intermediate_size * 2, config.hidden_size), "weight")?; + + let up_gate_proj = Linear::new(up_gate_proj_weight, None, None); + + let down_proj_weight = vb + .pp("down_proj") + .get((config.hidden_size, intermediate_size), "weight")?; + let down_proj_bias = vb.pp("down_proj").get(config.hidden_size, "bias")?; + let down_proj = Linear::new(down_proj_weight, Some(down_proj_bias), None); + + Ok(Self { + up_gate_proj, + down_proj, + intermediate_size, + act: config.hidden_act.clone(), + span: tracing::span!(tracing::Level::TRACE, "mlp"), + }) + } + + pub fn forward(&self, hidden_states: &Tensor) -> Result { + let _enter = self.span.enter(); + + let up_gate_states = self.up_gate_proj.forward(hidden_states)?; + let up_states = up_gate_states.narrow(1, 0, self.intermediate_size)?; + let gate_states = + up_gate_states.narrow(1, self.intermediate_size, self.intermediate_size)?; + + let gate_states = match self.act { + HiddenAct::Gelu => gate_states.gelu(), + HiddenAct::Relu => gate_states.relu(), + HiddenAct::Swiglu => gate_states.silu(), + }?; + let r = self.down_proj.forward(&(gate_states * up_states)?); + r + } +} + +struct GTELayer { + attention: GTEAttention, + mlp: GTEMLP, + attention_layer_norm: LayerNorm, + mlp_layer_norm: LayerNorm, + + span: tracing::Span, +} + +impl GTELayer { + pub fn load(vb: VarBuilder, config: >EConfig) -> Result { + let attention = GTEAttention::load(vb.pp("attention"), config)?; + let mlp = GTEMLP::load(vb.pp("mlp"), config)?; + + let attention_layer_norm = + LayerNorm::load(vb.pp("attn_ln"), config.hidden_size, config.layer_norm_eps)?; + let mlp_layer_norm = + LayerNorm::load(vb.pp("mlp_ln"), config.hidden_size, config.layer_norm_eps)?; + + Ok(Self { + attention, + mlp, + attention_layer_norm, + mlp_layer_norm, + span: tracing::span!(tracing::Level::TRACE, "layer"), + }) + } + + pub fn forward( + &self, + hidden_states: &Tensor, + cu_seqlens: &Tensor, + cos: &Tensor, + sin: &Tensor, + max_s: usize, + ) -> Result { + let _enter = self.span.enter(); + let attn_output = self + .attention + .forward(&hidden_states, cu_seqlens, cos, sin, max_s)?; + let normed_attn_res_output = self + .attention_layer_norm + .forward(&attn_output, Some(hidden_states))?; + + let mlp_output = self.mlp.forward(&normed_attn_res_output)?; + let normed_mlp_res_output = self + .mlp_layer_norm + .forward(&mlp_output, Some(&normed_attn_res_output))?; + Ok(normed_mlp_res_output) + } +} + +pub struct FlashGTEModel { + word_embeddings: Embedding, + token_type_embeddings: Embedding, + layers: Vec, + embeddings_norm: LayerNorm, + cos_cache: Tensor, + sin_cache: Tensor, + pool: Pool, + pub device: Device, + + span: tracing::Span, +} + +impl FlashGTEModel { + pub fn load(vb: VarBuilder, config: >EConfig, model_type: ModelType) -> Result { + match vb.device() { + Device::Cuda(_) => {} + _ => candle::bail!("FlashGTE requires Cuda"), + } + + if vb.dtype() != DType::F16 { + candle::bail!("FlashGTE requires DType::F16") + } + + if config.logn_attention_clip1 { + candle::bail!("`logn_attention_clip1` is not supported"); + } + if config.logn_attention_scale { + candle::bail!("`logn_attention_scale` is not supported"); + } + + if config.position_embedding_type != PositionEmbeddingType::Rope { + candle::bail!("Only `PositionEmbeddingType::Rope` is supported"); + } + + let pool = match model_type { + ModelType::Classifier => { + candle::bail!("`classifier` model type is not supported for GTE") + } + ModelType::Embedding(pool) => pool, + }; + + let word_embeddings = Embedding::new( + vb.pp("embeddings.word_embeddings") + .get((config.vocab_size, config.hidden_size), "weight")?, + config.hidden_size, + ); + + let token_type_embeddings = Embedding::new( + vb.pp("embeddings.token_type_embeddings") + .get((config.type_vocab_size, config.hidden_size), "weight")?, + config.hidden_size, + ); + + let layers = (0..config.num_hidden_layers) + .map(|index| GTELayer::load(vb.pp(format!("encoder.layer.{index}")), config)) + .collect::>>()?; + + let embeddings_norm = LayerNorm::load( + vb.pp("embeddings.LayerNorm"), + config.hidden_size, + config.layer_norm_eps, + )?; + + let inv_freqs = if let Some(RopeScaling::Ntk(NTKScaling { factor })) = config.rope_scaling { + let inv_freqs = candle_rotary::inv_freqs( + layers[0].attention.attention_head_size, + config.rope_theta * factor, + vb.device(), + )?; + let s = factor.powf(2.0 / layers[0].attention.attention_head_size as f32) as f64; + inv_freqs / s + } else { + candle_rotary::inv_freqs( + layers[0].attention.attention_head_size, + config.rope_theta, + vb.device(), + ) + }?; + + let (cos_cache, sin_cache) = + candle_rotary::cos_sin(config.max_position_embeddings, &inv_freqs, vb.dtype())?; + + Ok(Self { + word_embeddings, + token_type_embeddings, + layers, + embeddings_norm, + cos_cache, + sin_cache, + pool, + device: vb.device().clone(), + span: tracing::span!(tracing::Level::TRACE, "model"), + }) + } + + pub fn forward(&self, batch: Batch) -> Result<(Option, Option)> { + let _enter = self.span.enter(); + + let batch_size = batch.cumulative_seq_lengths.len() - 1; + let shape = batch.input_ids.len(); + + // Create Cuda tensors + let input_ids = Tensor::from_vec(batch.input_ids, shape, &self.device)?; + let token_type_ids = Tensor::from_vec(batch.token_type_ids, shape, &self.device)?; + let position_ids = Tensor::from_vec(batch.position_ids, shape, &self.device)?; + let cu_seqlens = Tensor::from_vec( + batch.cumulative_seq_lengths.clone(), + batch_size + 1, + &self.device, + )?; + + let word_embeddings = self.word_embeddings.forward(&input_ids)?; + let token_type_embeddings = self.token_type_embeddings.forward(&token_type_ids)?; + let mut hidden_states = self + .embeddings_norm + .forward(&word_embeddings, Some(&token_type_embeddings))?; + + let cos = self.cos_cache.index_select(&position_ids, 0)?; + let sin = self.sin_cache.index_select(&position_ids, 0)?; + + for layer in &self.layers { + let h = layer.forward( + &hidden_states, + &cu_seqlens, + &cos, + &sin, + batch.max_length as usize, + )?; + hidden_states = h; + } + + let outputs = hidden_states; + + let has_pooling_requests = !batch.pooled_indices.is_empty(); + let has_raw_requests = !batch.raw_indices.is_empty(); + + let pooled_embeddings = if has_pooling_requests { + match self.pool { + // CLS and LastToken pooling + Pool::Cls | Pool::LastToken => { + if batch_size > 1 { + // Get token indices form cu_seqlens + let mut indices = match self.pool { + Pool::Cls => cu_seqlens.narrow(0, 0, batch_size)?, + Pool::LastToken => cu_seqlens.narrow(0, 1, batch_size)?, + _ => unreachable!(), + }; + + // If raw_indices is empty, we don't need to do anything with + // the pooled_indices + if has_raw_requests { + // We need the pooled indices to select the correct cls indices + let pooled_indices = Tensor::from_vec( + batch.pooled_indices.clone(), + batch.pooled_indices.len(), + &self.device, + )?; + + // Only select indices that requires pooling + indices = indices.index_select(&pooled_indices, 0)? + } + + // Select tokens + Some(outputs.index_select(&indices, 0)?) + } else { + Some( + match self.pool { + Pool::Cls => outputs.i(0)?, + Pool::LastToken => { + outputs.i(batch.cumulative_seq_lengths[1] as usize - 1)? + } + _ => unreachable!(), + } + .unsqueeze(0)?, + ) + } + } + // Mean pooling + Pool::Mean => { + if batch_size > 1 { + // for each request that requires pooling + let results: Result> = batch + .pooled_indices + .into_iter() + .map(|i| { + let i = i as usize; + let start = batch.cumulative_seq_lengths[i]; + let len = batch.cumulative_seq_lengths[i + 1] - start; + + // Mean + let embeddings = outputs.narrow(0, start as usize, len as usize)?; + embeddings.sum_keepdim(0)? / (len as f64) + }) + .collect(); + + // Concatenate all results + Some(Tensor::cat(&results?, 0)?) + } else { + Some((outputs.sum_keepdim(0)? / (batch.max_length as f64))?) + } + } + Pool::Splade => { + unreachable!(); + } + } + } else { + None + }; + + let raw_embeddings = if has_raw_requests { + if batch_size > 1 && has_pooling_requests { + // Create indexing vector for the embeddings + let mut final_indices: Vec = Vec::with_capacity(shape); + for i in batch.raw_indices.into_iter() { + let i = i as usize; + // Get start/end token index of this specific member of the batch + let start = batch.cumulative_seq_lengths[i]; + let end = batch.cumulative_seq_lengths[i + 1]; + + for j in start..end { + // Add indices for the tokens of this specific member of the batch + final_indices.push(j); + } + } + + let final_indices_length = final_indices.len(); + let final_indices = + Tensor::from_vec(final_indices, final_indices_length, &self.device)?; + + // Select the tokens with final indices + Some(outputs.index_select(&final_indices, 0)?) + } else { + Some(outputs) + } + } else { + None + }; + + Ok((pooled_embeddings, raw_embeddings)) + } +} + +impl Model for FlashGTEModel { + fn is_padded(&self) -> bool { + false + } + fn embed(&self, batch: Batch) -> Result<(Option, Option)> { + self.forward(batch) + } +} diff --git a/backends/candle/src/models/flash_jina.rs b/backends/candle/src/models/flash_jina.rs index c8efee18..0bab863f 100644 --- a/backends/candle/src/models/flash_jina.rs +++ b/backends/candle/src/models/flash_jina.rs @@ -242,6 +242,7 @@ impl FlashJinaBertModel { ) } PositionEmbeddingType::Absolute => None, + _ => candle::bail!("not supported"), }; match vb.device() { diff --git a/backends/candle/src/models/flash_jina_code.rs b/backends/candle/src/models/flash_jina_code.rs index 06ade2d5..b20438fe 100644 --- a/backends/candle/src/models/flash_jina_code.rs +++ b/backends/candle/src/models/flash_jina_code.rs @@ -295,6 +295,7 @@ impl FlashJinaCodeBertModel { ) } PositionEmbeddingType::Absolute => None, + _ => candle::bail!("not supported"), }; match vb.device() { diff --git a/backends/candle/src/models/gte.rs b/backends/candle/src/models/gte.rs new file mode 100644 index 00000000..e5e75638 --- /dev/null +++ b/backends/candle/src/models/gte.rs @@ -0,0 +1,35 @@ +use crate::layers::HiddenAct; +use crate::models::PositionEmbeddingType; +use serde::Deserialize; + +#[derive(Debug, Clone, PartialEq, Deserialize)] +pub struct NTKScaling { + pub factor: f32, +} + +#[derive(Debug, Clone, PartialEq, Deserialize)] +#[serde(tag = "type", rename_all = "kebab-case")] +pub enum RopeScaling { + Ntk(NTKScaling), +} + +#[derive(Debug, Clone, PartialEq, Deserialize)] +pub struct GTEConfig { + pub vocab_size: usize, + pub hidden_size: usize, + pub num_hidden_layers: usize, + pub num_attention_heads: usize, + pub intermediate_size: usize, + pub hidden_act: HiddenAct, + pub max_position_embeddings: usize, + pub type_vocab_size: usize, + pub layer_norm_type: String, + pub layer_norm_eps: f32, + pub position_embedding_type: PositionEmbeddingType, + pub rope_theta: f32, + pub rope_scaling: Option, + #[serde(default)] + pub logn_attention_scale: bool, + #[serde(default)] + pub logn_attention_clip1: bool, +} diff --git a/backends/candle/src/models/jina.rs b/backends/candle/src/models/jina.rs index 6884fcae..ecee8bfe 100644 --- a/backends/candle/src/models/jina.rs +++ b/backends/candle/src/models/jina.rs @@ -363,6 +363,7 @@ impl JinaBertModel { vb.dtype(), )?), PositionEmbeddingType::Absolute => None, + _ => candle::bail!("not supported"), }; let pool = match model_type { diff --git a/backends/candle/src/models/jina_code.rs b/backends/candle/src/models/jina_code.rs index fd004bc6..5f13fe08 100644 --- a/backends/candle/src/models/jina_code.rs +++ b/backends/candle/src/models/jina_code.rs @@ -352,6 +352,7 @@ impl JinaCodeBertModel { vb.dtype(), )?), PositionEmbeddingType::Absolute => None, + _ => candle::bail!("not supported"), }; let pool = match model_type { diff --git a/backends/candle/src/models/mod.rs b/backends/candle/src/models/mod.rs index 3e4a5785..9f616a2a 100644 --- a/backends/candle/src/models/mod.rs +++ b/backends/candle/src/models/mod.rs @@ -26,12 +26,17 @@ mod flash_nomic; #[cfg(feature = "cuda")] mod flash_distilbert; +#[cfg(feature = "cuda")] +mod flash_gte; #[cfg(feature = "cuda")] mod flash_mistral; +mod gte; pub use bert::{BertConfig, BertModel, PositionEmbeddingType}; use candle::{Result, Tensor}; pub use distilbert::{DistilBertConfig, DistilBertModel}; +#[allow(unused_imports)] +pub use gte::{GTEConfig, NTKScaling, RopeScaling}; pub use jina::JinaBertModel; pub use jina_code::JinaCodeBertModel; pub use mistral::MistralConfig; @@ -56,6 +61,9 @@ pub use flash_distilbert::FlashDistilBertModel; #[cfg(feature = "cuda")] pub use flash_mistral::FlashMistralModel; +#[cfg(feature = "cuda")] +pub use flash_gte::FlashGTEModel; + pub(crate) trait Model { fn is_padded(&self) -> bool; From 75ed54f8e67d9a941fb3f0f754a0278c752a5ee5 Mon Sep 17 00:00:00 2001 From: OlivierDehaene <23298448+OlivierDehaene@users.noreply.github.com> Date: Thu, 27 Jun 2024 18:37:43 +0200 Subject: [PATCH 2/2] add snapshots --- backends/candle/src/models/flash_gte.rs | 25 +- .../snapshots/test_flash_gte__gte_batch.snap | 2309 +++++++++++++++++ .../snapshots/test_flash_gte__gte_single.snap | 773 ++++++ backends/candle/tests/test_flash_gte.rs | 53 + 4 files changed, 3152 insertions(+), 8 deletions(-) create mode 100644 backends/candle/tests/snapshots/test_flash_gte__gte_batch.snap create mode 100644 backends/candle/tests/snapshots/test_flash_gte__gte_single.snap create mode 100644 backends/candle/tests/test_flash_gte.rs diff --git a/backends/candle/src/models/flash_gte.rs b/backends/candle/src/models/flash_gte.rs index b5a6b38e..62bf65bd 100644 --- a/backends/candle/src/models/flash_gte.rs +++ b/backends/candle/src/models/flash_gte.rs @@ -200,7 +200,7 @@ impl GTELayer { pub struct FlashGTEModel { word_embeddings: Embedding, - token_type_embeddings: Embedding, + token_type_embeddings: Option, layers: Vec, embeddings_norm: LayerNorm, cos_cache: Tensor, @@ -246,11 +246,15 @@ impl FlashGTEModel { config.hidden_size, ); - let token_type_embeddings = Embedding::new( - vb.pp("embeddings.token_type_embeddings") - .get((config.type_vocab_size, config.hidden_size), "weight")?, - config.hidden_size, - ); + let token_type_embeddings = if config.type_vocab_size > 0 { + Some(Embedding::new( + vb.pp("embeddings.token_type_embeddings") + .get((config.type_vocab_size, config.hidden_size), "weight")?, + config.hidden_size, + )) + } else { + None + }; let layers = (0..config.num_hidden_layers) .map(|index| GTELayer::load(vb.pp(format!("encoder.layer.{index}")), config)) @@ -311,10 +315,15 @@ impl FlashGTEModel { )?; let word_embeddings = self.word_embeddings.forward(&input_ids)?; - let token_type_embeddings = self.token_type_embeddings.forward(&token_type_ids)?; + let token_type_embeddings = self + .token_type_embeddings + .as_ref() + .map(|emb| emb.forward(&token_type_ids)) + .transpose()?; + let mut hidden_states = self .embeddings_norm - .forward(&word_embeddings, Some(&token_type_embeddings))?; + .forward(&word_embeddings, token_type_embeddings.as_ref())?; let cos = self.cos_cache.index_select(&position_ids, 0)?; let sin = self.sin_cache.index_select(&position_ids, 0)?; diff --git a/backends/candle/tests/snapshots/test_flash_gte__gte_batch.snap b/backends/candle/tests/snapshots/test_flash_gte__gte_batch.snap new file mode 100644 index 00000000..d8bbd66e --- /dev/null +++ b/backends/candle/tests/snapshots/test_flash_gte__gte_batch.snap @@ -0,0 +1,2309 @@ +--- +source: backends/candle/tests/test_flash_gte.rs +assertion_line: 37 +expression: embeddings_batch +--- +- - 0.4428711 + - 0.6118164 + - 0.07043457 + - -0.5917969 + - 0.88183594 + - -0.10839844 + - 0.0881958 + - -0.20483398 + - 1.9345703 + - 0.6230469 + - 0.59521484 + - 0.18737793 + - 0.030548096 + - -0.078308105 + - 0.79589844 + - -0.23474121 + - -0.8720703 + - 1.6923828 + - 0.12597656 + - -1.9267578 + - -0.34472656 + - -1.9326172 + - 0.89990234 + - -0.2006836 + - 0.024230957 + - 0.2878418 + - 0.3642578 + - 0.17956543 + - -0.51123047 + - 0.025817871 + - -0.8964844 + - 0.14147949 + - -0.39257813 + - -0.8486328 + - 0.005218506 + - 0.42358398 + - 0.23181152 + - 1.1630859 + - -1.1455078 + - 0.3388672 + - -2.1484375 + - -1.0146484 + - -0.2841797 + - 0.021499634 + - 0.16235352 + - 0.39453125 + - -0.9511719 + - 0.41064453 + - -0.15563965 + - -0.484375 + - -0.07080078 + - 0.07312012 + - 0.56103516 + - 0.028213501 + - -0.027938843 + - 0.43896484 + - 1.0136719 + - 0.7788086 + - -0.5644531 + - -1.9824219 + - -0.011169434 + - 0.50634766 + - -0.39331055 + - -0.3334961 + - -1.1679688 + - -0.07208252 + - 0.008544922 + - 0.7788086 + - -0.10021973 + - -0.30859375 + - 0.35888672 + - -0.44848633 + - -0.08679199 + - -1.0791016 + - 1.3193359 + - -1.0273438 + - 0.34228516 + - -0.45385742 + - 0.8027344 + - 0.50634766 + - 0.703125 + - -0.96533203 + - -0.042938232 + - -1.0927734 + - 0.35205078 + - -0.56689453 + - -0.5341797 + - 0.23571777 + - -2.0234375 + - -0.26342773 + - 0.1998291 + - 0.7739258 + - 0.17797852 + - 1.2255859 + - 0.9814453 + - -0.2902832 + - 1.2080078 + - 0.50097656 + - 0.2479248 + - -0.2166748 + - 0.29052734 + - -0.5004883 + - 0.26513672 + - 0.5024414 + - -0.042266846 + - -0.013465881 + - -0.6376953 + - 0.64501953 + - -1.7958984 + - 0.25512695 + - -0.20458984 + - -0.12927246 + - 0.19665527 + - 0.8076172 + - -0.96777344 + - 0.26391602 + - -0.5649414 + - -0.58740234 + - -0.15466309 + - 0.7392578 + - 0.5336914 + - 0.60253906 + - 1.1875 + - -0.37719727 + - 0.8198242 + - -0.2401123 + - 0.40527344 + - 0.17114258 + - -0.32128906 + - -0.75439453 + - 0.056488037 + - 0.16491699 + - 0.6899414 + - -0.6171875 + - -0.05316162 + - 0.43017578 + - -0.6074219 + - 0.040374756 + - -0.8618164 + - -0.7192383 + - 0.7402344 + - -0.42138672 + - 0.26831055 + - -1.5429688 + - 0.1706543 + - 1.2158203 + - -0.17077637 + - -0.13964844 + - -0.5449219 + - -0.59033203 + - -0.7783203 + - -0.18786621 + - -0.34838867 + - 0.6230469 + - -0.5878906 + - -0.90234375 + - -0.7324219 + - -0.81933594 + - 0.4819336 + - -1.1591797 + - 1.2109375 + - -1.0517578 + - 0.25219727 + - 1.2841797 + - 0.06347656 + - -0.72021484 + - -0.3671875 + - 0.07330322 + - -0.022644043 + - -0.3713379 + - -0.008430481 + - 0.3564453 + - -0.8354492 + - -0.59521484 + - -0.40405273 + - -0.027923584 + - -0.3100586 + - -1.109375 + - -0.030593872 + - 0.14086914 + - -1.2128906 + - 0.03488159 + - 0.35424805 + - -1.5683594 + - 0.17773438 + - -0.66796875 + - -1.9804688 + - -0.35327148 + - -0.2927246 + - -1.2011719 + - -0.4165039 + - 0.36645508 + - -0.049957275 + - -0.87353516 + - -0.9086914 + - 0.4790039 + - -1.4599609 + - 0.7866211 + - -0.27490234 + - 0.08477783 + - 0.2220459 + - 0.0017213821 + - -1.0927734 + - -0.006439209 + - -0.62060547 + - -0.68115234 + - 0.4572754 + - -0.61572266 + - -0.7871094 + - -1.3847656 + - 1.5566406 + - 0.2680664 + - 0.8466797 + - -0.2578125 + - -0.76416016 + - 0.6484375 + - -0.05279541 + - -0.1307373 + - -0.11853027 + - -0.6357422 + - -0.17700195 + - 0.9472656 + - -0.4958496 + - 0.015464783 + - -0.23168945 + - -1.6748047 + - -0.8911133 + - 0.86865234 + - 1.0722656 + - 0.34228516 + - 0.7182617 + - -0.5488281 + - -0.45166016 + - -0.3022461 + - 0.45336914 + - -0.28808594 + - -0.7626953 + - 0.016067505 + - 0.63183594 + - 0.46923828 + - -0.07373047 + - 1.7832031 + - -0.41796875 + - -0.89453125 + - 0.07757568 + - -0.29589844 + - -1.0673828 + - 1.3320313 + - 1.4160156 + - 0.20703125 + - -0.1060791 + - 0.1508789 + - 0.37231445 + - 0.05053711 + - -0.68896484 + - 0.7397461 + - 0.15771484 + - 0.5595703 + - -0.8198242 + - -0.17712402 + - -0.2479248 + - -0.066345215 + - 0.4194336 + - 1.0517578 + - -1.6650391 + - 0.90283203 + - 0.22937012 + - -0.5908203 + - -0.79785156 + - -0.003993988 + - -0.47314453 + - 0.19116211 + - -0.39038086 + - 0.6333008 + - -0.28686523 + - -0.8720703 + - 0.080200195 + - -1.2587891 + - 0.22790527 + - 0.55126953 + - -0.47485352 + - -0.20471191 + - 0.75390625 + - -1.0488281 + - 0.9902344 + - 1.2275391 + - 0.40966797 + - -0.36791992 + - 0.54833984 + - -0.32788086 + - 0.2208252 + - 1.1289063 + - -0.017410278 + - -0.67089844 + - 0.7236328 + - 0.20788574 + - 0.55566406 + - 0.7104492 + - -0.16589355 + - 0.57958984 + - 1.1494141 + - -0.15930176 + - -0.4963379 + - 0.016326904 + - 1.5634766 + - -0.5029297 + - 1.2011719 + - -1.0029297 + - -0.3557129 + - -0.31079102 + - -0.12670898 + - 0.81933594 + - 0.75683594 + - 1.1630859 + - 1.6669922 + - -0.8022461 + - -0.3034668 + - 0.3798828 + - -0.71484375 + - -0.7470703 + - 1.1904297 + - -0.95654297 + - 0.30297852 + - 0.16992188 + - 0.8173828 + - -0.8798828 + - 1.1376953 + - -0.4543457 + - 0.6142578 + - -0.47680664 + - 1.1181641 + - 0.01398468 + - -0.49560547 + - -0.039031982 + - 0.2878418 + - -0.71533203 + - 0.82128906 + - -0.03768921 + - 0.60546875 + - 0.7578125 + - -0.3322754 + - 0.86376953 + - -0.71435547 + - 0.7597656 + - -1.3496094 + - 0.18652344 + - -0.39990234 + - 0.31103516 + - 0.16809082 + - 1.2626953 + - -0.42919922 + - 0.2434082 + - 0.83935547 + - -1.3994141 + - 0.34472656 + - -0.68603516 + - -0.28759766 + - 0.4663086 + - 0.6816406 + - -0.48120117 + - -0.11029053 + - -0.48339844 + - 0.1706543 + - 0.29614258 + - -0.7290039 + - -0.87158203 + - 0.8457031 + - 0.57128906 + - 0.1640625 + - -0.1739502 + - -0.16394043 + - 0.8652344 + - -0.38671875 + - 0.5625 + - -0.30493164 + - 1.8251953 + - 0.041931152 + - 0.17285156 + - -1.0107422 + - -0.20788574 + - 1.1005859 + - 0.7973633 + - -1.0029297 + - 0.67041016 + - -0.96240234 + - 0.45166016 + - -0.32202148 + - -0.97753906 + - 1.7080078 + - -0.04626465 + - 0.4404297 + - 0.28833008 + - 0.32006836 + - -0.5341797 + - -0.74658203 + - -1.28125 + - -0.2076416 + - -0.57958984 + - 0.88134766 + - -0.21875 + - -0.51171875 + - 0.20214844 + - -0.9536133 + - -0.89160156 + - 0.6279297 + - -0.08929443 + - 1.2490234 + - -0.3083496 + - 0.5253906 + - -1.5771484 + - 0.86572266 + - -0.16223145 + - -0.6455078 + - -0.9008789 + - -0.68896484 + - -0.55566406 + - 0.54785156 + - -0.2052002 + - 1.0820313 + - 0.5390625 + - 0.2454834 + - -0.0053596497 + - 0.09838867 + - -0.08288574 + - 0.6616211 + - -0.06573486 + - -0.8120117 + - -0.08306885 + - 0.12176514 + - -1.4072266 + - 0.6533203 + - 0.49536133 + - 0.48339844 + - -0.6142578 + - 0.06011963 + - 0.4951172 + - -0.33935547 + - -0.85058594 + - -0.8510742 + - 0.10028076 + - 0.64501953 + - 0.62841797 + - 0.79052734 + - -0.8486328 + - -0.76708984 + - 0.38891602 + - 0.5917969 + - -0.06939697 + - -0.13366699 + - -0.8964844 + - -0.76660156 + - 1.3134766 + - 0.98046875 + - 0.37158203 + - 0.04675293 + - -1.2890625 + - 0.17102051 + - -0.16772461 + - -0.10760498 + - -0.111816406 + - 0.20251465 + - -0.65527344 + - 0.2800293 + - -1.8037109 + - 0.15270996 + - 0.25024414 + - 0.6713867 + - 0.67578125 + - -1.0322266 + - -0.4140625 + - -0.59277344 + - 0.028778076 + - -1.3515625 + - 0.12548828 + - 0.87353516 + - 0.87060547 + - -0.18518066 + - 0.7949219 + - -0.7011719 + - -0.28759766 + - 0.39013672 + - 0.43139648 + - -0.053955078 + - -0.60498047 + - 0.6381836 + - 0.20996094 + - 0.09857178 + - -0.7402344 + - 0.24731445 + - -0.41430664 + - -0.49365234 + - -0.7817383 + - -0.35766602 + - -0.53759766 + - -0.13439941 + - 0.03010559 + - 0.47070313 + - 0.16564941 + - 1.0439453 + - 0.7026367 + - 0.3251953 + - 0.021118164 + - -0.25976563 + - 1.4033203 + - 0.3383789 + - -0.0637207 + - 0.8149414 + - -0.7651367 + - -0.5332031 + - -0.19335938 + - 0.19458008 + - -0.61083984 + - 0.07684326 + - -0.6982422 + - -0.38989258 + - 0.37841797 + - 0.5966797 + - -1.9882813 + - -0.36767578 + - -2.0214844 + - 0.9682617 + - 0.46826172 + - -0.49829102 + - -0.20861816 + - -0.15332031 + - -0.13891602 + - 1.8779297 + - -0.18640137 + - 0.38134766 + - -0.1373291 + - -0.29492188 + - 0.42871094 + - -0.24523926 + - 0.16320801 + - 0.8408203 + - -0.34985352 + - 0.5307617 + - -0.14685059 + - 0.7729492 + - -0.11444092 + - -0.23522949 + - 1.2089844 + - -1.5595703 + - 0.09820557 + - -1.3476563 + - 0.6196289 + - -1.1464844 + - -0.47045898 + - -0.10656738 + - -1.3115234 + - 0.4309082 + - 0.054748535 + - 0.80078125 + - 0.43359375 + - -0.22290039 + - 0.55908203 + - -8.1484375 + - -0.7236328 + - -0.4831543 + - -0.21447754 + - 0.51904297 + - 0.5449219 + - 0.8520508 + - -0.8852539 + - 0.36865234 + - -0.1694336 + - -0.32250977 + - 0.12176514 + - 0.92578125 + - 0.38891602 + - 0.79296875 + - 0.46875 + - 0.9213867 + - 0.9970703 + - -0.32739258 + - 1.2900391 + - -0.1517334 + - 0.18640137 + - -0.6113281 + - 0.060821533 + - 0.3527832 + - -0.7915039 + - -0.3942871 + - 1.2158203 + - -0.8051758 + - 0.13598633 + - 0.21057129 + - -0.66552734 + - 1.5957031 + - 0.44262695 + - -0.24023438 + - -0.3984375 + - -1.0517578 + - -0.36499023 + - -0.5966797 + - 0.60546875 + - 1.2324219 + - 0.21520996 + - -0.18481445 + - -0.0066490173 + - -0.08239746 + - 0.7993164 + - -0.81884766 + - -1.6660156 + - -1.2861328 + - -0.8535156 + - 0.15856934 + - 0.6230469 + - 0.60595703 + - 0.5234375 + - -0.93603516 + - 0.7529297 + - 0.06738281 + - -0.002035141 + - -0.92285156 + - -0.25512695 + - -0.50878906 + - -0.7944336 + - -0.44970703 + - -0.64501953 + - 0.8569336 + - 0.12030029 + - -0.08343506 + - -0.5126953 + - 0.16540527 + - 1.6054688 + - -0.3947754 + - 0.34350586 + - -0.43066406 + - -0.04083252 + - 0.2956543 + - -0.5175781 + - 0.6567383 + - 0.57470703 + - 0.6694336 + - 0.13891602 + - 0.9790039 + - 0.16503906 + - -0.12017822 + - -0.8745117 + - 0.057861328 + - 0.24194336 + - -0.012207031 + - 0.40722656 + - -0.34936523 + - -0.1998291 + - 0.41210938 + - -0.17858887 + - -0.011474609 + - 0.10394287 + - -0.1048584 + - 0.037384033 + - 1.1806641 + - -0.68896484 + - 0.7397461 + - -0.7661133 + - -0.3540039 + - 0.6098633 + - -0.93066406 + - -0.49047852 + - -0.29418945 + - -1.4736328 + - -1.0498047 + - -1.2509766 + - -0.546875 + - -1.4882813 + - 0.4802246 + - -0.79052734 + - -0.24499512 + - -0.5517578 + - -0.08282471 + - -0.47705078 + - 0.18701172 + - 0.47827148 + - -0.21008301 + - -0.83154297 + - -0.106933594 + - 0.041656494 + - -0.09887695 + - 0.58447266 + - -0.80859375 + - -1.1503906 + - 0.19140625 + - 0.50146484 + - -0.8930664 + - -0.037750244 + - -0.4411621 + - 0.50439453 + - 1.484375 + - -0.8754883 + - -0.3227539 + - -0.98583984 + - -0.09124756 + - 0.48510742 + - -0.2775879 + - 0.80371094 + - -0.5683594 + - -0.25732422 + - 0.7504883 + - -0.37426758 + - 0.2199707 + - 0.37426758 + - -0.03375244 + - -0.40771484 + - -0.18078613 + - -0.48095703 + - -1.5078125 + - -0.6743164 + - 0.82177734 + - 0.25073242 + - -0.8261719 + - 0.33862305 + - -1.2646484 + - 0.8666992 + - -0.27392578 + - -0.34399414 + - -0.42333984 + - -0.4116211 + - 0.8027344 + - 1.1503906 + - -1.2041016 + - 0.35620117 + - -0.00016129017 + - -1.953125 + - -0.6035156 + - 0.58154297 + - 0.08703613 + - 0.48266602 + - -0.35180664 + - -0.11047363 + - 0.65966797 + - -1.4921875 + - -0.14086914 + - -0.2130127 + - 1.0244141 + - -1.4472656 + - 0.56152344 + - 1.734375 + - 0.09698486 + - -0.5415039 + - -0.1340332 + - 0.7583008 + - -0.2722168 + - 1.0488281 + - 1.0498047 + - 0.23254395 + - -0.15222168 + - 0.18005371 + - 0.097839355 + - -1.6386719 + - -0.6035156 + - -0.91748047 + - -0.93847656 + - 1.0576172 + - 0.703125 + - 0.70410156 + - 0.8198242 + - -2.25 + - -0.6357422 + - 0.35058594 + - 0.75146484 + - -0.12322998 + - -1.0859375 + - 0.50634766 + - -0.67822266 + - 0.7495117 + - -0.27685547 + - -0.13537598 +- - 0.039520264 + - 0.4255371 + - 0.19506836 + - -0.4951172 + - 0.5908203 + - -0.17102051 + - 1.3964844 + - 0.14685059 + - 1.8730469 + - 0.5810547 + - 0.7573242 + - 0.2841797 + - 0.027160645 + - -0.61328125 + - 0.24157715 + - -0.13586426 + - -0.44848633 + - 1.1748047 + - 0.5546875 + - -1.8710938 + - -0.08605957 + - -1.2617188 + - 0.7163086 + - -0.32055664 + - 0.11517334 + - 0.99658203 + - 0.09411621 + - 0.4724121 + - 0.20117188 + - 0.50439453 + - -0.07611084 + - -0.026397705 + - -0.5605469 + - -0.12670898 + - -0.7915039 + - 0.19970703 + - 0.97265625 + - 0.7597656 + - -1.6689453 + - -0.16137695 + - -1.5742188 + - -0.8232422 + - 0.3774414 + - -0.105651855 + - -0.026626587 + - 0.5175781 + - -1.21875 + - -0.67089844 + - -0.28198242 + - -0.20910645 + - 0.19665527 + - 0.64990234 + - 1.0576172 + - 0.24536133 + - 0.33862305 + - 0.76708984 + - 0.8564453 + - 0.20861816 + - -0.99853516 + - -1.0507813 + - -0.011680603 + - 0.5522461 + - -0.47558594 + - -0.8598633 + - -0.93847656 + - -0.46899414 + - 0.21325684 + - 1.4003906 + - 0.67089844 + - -0.48999023 + - -0.20861816 + - -0.53515625 + - 0.2241211 + - -1.1054688 + - 0.73339844 + - -1.2802734 + - 0.2442627 + - -0.0713501 + - 1.09375 + - 0.68408203 + - 0.94873047 + - -0.51904297 + - -0.14282227 + - -0.6435547 + - 0.32421875 + - 0.28881836 + - -0.0848999 + - -0.12792969 + - -1.9990234 + - 0.039886475 + - 0.6035156 + - -0.10223389 + - -0.095581055 + - 0.9741211 + - 0.025161743 + - -0.34033203 + - 1.1289063 + - 1.0546875 + - 0.19396973 + - -0.041290283 + - 0.9580078 + - -0.37304688 + - 0.20910645 + - 0.3161621 + - 0.44311523 + - 0.088012695 + - -1.1035156 + - 0.9350586 + - -1.3847656 + - 0.3918457 + - -0.14111328 + - -0.58251953 + - -0.43579102 + - 0.79589844 + - -0.6230469 + - 0.63427734 + - -0.6152344 + - -1.3935547 + - 0.3203125 + - 1.1542969 + - -0.022766113 + - 0.81933594 + - 0.59814453 + - -0.23718262 + - 0.8051758 + - -0.68115234 + - 0.25463867 + - -0.118896484 + - -0.15356445 + - 0.12756348 + - -0.32617188 + - -0.27734375 + - 0.95410156 + - -1.3583984 + - -0.55322266 + - -0.19372559 + - -1.0253906 + - 0.14758301 + - -1.6992188 + - -0.3317871 + - 0.5078125 + - -0.3552246 + - 0.037506104 + - -1.3037109 + - 0.016326904 + - 1.2617188 + - -0.4633789 + - 0.34521484 + - -0.8574219 + - -0.9423828 + - -0.55029297 + - 0.28564453 + - -0.1239624 + - 0.111572266 + - -0.40600586 + - -1.0878906 + - -0.5410156 + - 0.74658203 + - 0.06738281 + - -1.5810547 + - 1.1582031 + - -0.84765625 + - -0.10479736 + - 1.1591797 + - 0.08850098 + - -0.49902344 + - -0.071777344 + - 1.0751953 + - 0.6298828 + - 0.0947876 + - -0.36889648 + - 0.18457031 + - -0.5332031 + - -0.8461914 + - 0.5776367 + - -0.47265625 + - -0.53515625 + - -0.21533203 + - 0.31445313 + - 0.50878906 + - -0.80078125 + - -0.09869385 + - -0.46484375 + - 0.23132324 + - -0.1071167 + - -0.9375 + - -0.94433594 + - 0.22302246 + - -1.3183594 + - -1.2880859 + - -1.0263672 + - 0.7392578 + - -1.0771484 + - -0.06707764 + - -1.1992188 + - 0.09863281 + - -1.1689453 + - 0.703125 + - -0.51171875 + - -0.036346436 + - -0.6064453 + - 0.17321777 + - -0.26416016 + - 0.31811523 + - -1.2773438 + - 0.012527466 + - 1.4785156 + - -0.10699463 + - -0.35961914 + - -0.28881836 + - 1.0390625 + - 0.37646484 + - 0.97998047 + - -0.31567383 + - -0.72509766 + - 1.1767578 + - -0.35302734 + - -0.15820313 + - -0.08886719 + - 0.6435547 + - 0.03665161 + - 1.578125 + - -0.359375 + - -0.60791016 + - -0.048919678 + - -0.7290039 + - -0.1685791 + - 0.65283203 + - 1.0820313 + - -0.26586914 + - 0.11590576 + - -0.5361328 + - -0.81347656 + - -1.0986328 + - 1.3056641 + - -0.33984375 + - -0.23547363 + - -0.0713501 + - 0.83984375 + - 0.5654297 + - -0.74609375 + - 1.4326172 + - -0.80078125 + - -0.6508789 + - 0.42749023 + - 0.036346436 + - -0.02998352 + - 0.7915039 + - 1.0439453 + - 0.66748047 + - 0.57128906 + - 0.33276367 + - -0.31152344 + - 0.028060913 + - 0.037902832 + - 0.6381836 + - 0.06414795 + - 0.19274902 + - -0.68847656 + - 0.11767578 + - -0.4963379 + - 0.12487793 + - 0.041503906 + - 0.31567383 + - -1.3642578 + - 0.6875 + - 0.8310547 + - -0.37670898 + - -0.9199219 + - 0.039245605 + - 0.0040512085 + - -0.068603516 + - -0.011726379 + - 0.7993164 + - -0.16674805 + - -0.45703125 + - -0.71191406 + - -1.1611328 + - -0.06738281 + - 1.1181641 + - -0.44482422 + - 0.3383789 + - 0.39501953 + - -0.28930664 + - 1.0507813 + - 0.26464844 + - 0.8310547 + - -0.8935547 + - 0.15002441 + - -0.8461914 + - 0.7841797 + - 1.3574219 + - 0.05038452 + - 0.3569336 + - -0.8769531 + - -0.27441406 + - 0.28857422 + - 0.1459961 + - 0.0003156662 + - 0.043121338 + - 0.65771484 + - 0.08758545 + - -0.0033512115 + - -0.1875 + - 0.7397461 + - -0.5463867 + - 0.46826172 + - -1.1835938 + - -0.23535156 + - -0.3701172 + - -0.16711426 + - 0.5722656 + - 0.38354492 + - 1.0068359 + - 0.8955078 + - 0.17358398 + - 0.08251953 + - -0.33251953 + - -0.06994629 + - -0.6298828 + - 0.96435547 + - -0.8149414 + - 0.6376953 + - 0.38549805 + - 1.0146484 + - -1.4296875 + - 0.2109375 + - -0.46557617 + - 0.5151367 + - -0.47729492 + - 0.91748047 + - 0.058654785 + - 0.62939453 + - 0.18261719 + - 0.08685303 + - -1.2226563 + - 0.36083984 + - -0.60791016 + - 0.53808594 + - -0.042816162 + - -1.0537109 + - 1.5537109 + - -0.35009766 + - 0.9067383 + - -0.74902344 + - 0.3059082 + - -0.4177246 + - 0.23950195 + - -0.19958496 + - 0.9589844 + - -0.26416016 + - 0.17883301 + - 0.028823853 + - -0.20239258 + - 0.6230469 + - -1.2568359 + - 0.73046875 + - 1.1914063 + - 0.38842773 + - -0.49365234 + - -0.33691406 + - 0.085754395 + - 0.21142578 + - -0.096069336 + - 0.30981445 + - -1.4892578 + - 0.6958008 + - 0.16235352 + - 0.15075684 + - -0.15576172 + - -0.73828125 + - 0.97802734 + - -0.22546387 + - 0.79296875 + - -0.40625 + - 1.8994141 + - 0.053527832 + - 0.02268982 + - -0.7207031 + - -0.63134766 + - -0.07324219 + - 0.08795166 + - 0.22595215 + - 0.035247803 + - -0.83203125 + - -0.08026123 + - 0.07891846 + - -1.4433594 + - 1.2402344 + - 0.7402344 + - 0.2479248 + - -0.25732422 + - 0.29418945 + - -0.88916016 + - -0.35229492 + - -0.65771484 + - 0.048614502 + - -0.1295166 + - 0.1381836 + - -0.60253906 + - -0.5102539 + - -0.39868164 + - -1.6806641 + - -0.6455078 + - 0.24414063 + - -0.19958496 + - 0.62353516 + - -0.86035156 + - 0.10614014 + - -1.0644531 + - 0.84033203 + - -0.48266602 + - -0.30493164 + - -0.3864746 + - -0.052856445 + - -1.2080078 + - -0.43115234 + - 0.12310791 + - 0.80859375 + - 0.21838379 + - -0.027175903 + - -0.10418701 + - -0.44189453 + - -0.051757813 + - 0.6767578 + - -0.7939453 + - -0.703125 + - -0.6723633 + - -0.1743164 + - -1.3681641 + - -0.36376953 + - -0.85546875 + - 1.3095703 + - 0.22692871 + - 0.27783203 + - -0.018127441 + - -0.671875 + - -0.921875 + - -0.2841797 + - -0.45874023 + - -0.20581055 + - -0.3786621 + - 1.0615234 + - -0.8754883 + - -1.0185547 + - 0.13696289 + - 0.15734863 + - 0.57714844 + - -0.234375 + - -1.0605469 + - -0.54052734 + - 0.9375 + - 1.0097656 + - -0.8676758 + - 0.09088135 + - -0.88671875 + - 0.5229492 + - 0.08917236 + - -0.12042236 + - -0.8720703 + - 1.0849609 + - -0.46362305 + - 0.6347656 + - -1.7353516 + - -0.42382813 + - -0.017181396 + - 0.40551758 + - -0.07434082 + - -0.120910645 + - -0.10473633 + - 0.29589844 + - 0.6972656 + - -1.015625 + - 0.5371094 + - -0.51171875 + - 1.1914063 + - -0.113342285 + - 1.3095703 + - -0.8334961 + - 0.39233398 + - -0.09277344 + - 0.21032715 + - 0.24316406 + - 0.15551758 + - 0.15319824 + - -0.6533203 + - -0.3918457 + - -0.83984375 + - 0.15820313 + - -0.3544922 + - -0.19848633 + - 0.074401855 + - -0.23913574 + - -0.36791992 + - -0.5986328 + - 0.2980957 + - -0.1463623 + - -0.43823242 + - 0.9272461 + - 0.24169922 + - 0.1842041 + - -0.38793945 + - -0.54589844 + - 0.67333984 + - -0.20532227 + - 0.7084961 + - 0.19482422 + - 0.3100586 + - -0.18029785 + - 0.83691406 + - 0.09051514 + - -0.14501953 + - 0.08215332 + - 0.016418457 + - 0.2763672 + - 0.5053711 + - -0.6357422 + - -2.8535156 + - 0.09869385 + - -1.2763672 + - 0.7895508 + - 0.32861328 + - -0.58691406 + - 0.05392456 + - -0.49316406 + - 0.03439331 + - 1.3232422 + - -0.3791504 + - -0.009689331 + - 0.12915039 + - 0.39648438 + - 0.40820313 + - -0.5541992 + - 0.07409668 + - 0.16479492 + - -0.008087158 + - 0.17626953 + - -0.22290039 + - 0.93310547 + - -0.03164673 + - 0.010131836 + - 0.97998047 + - 0.16027832 + - 0.5805664 + - -0.96191406 + - 0.39916992 + - -0.23803711 + - -0.7949219 + - 0.16552734 + - -1.9003906 + - -0.5800781 + - -0.39697266 + - 0.10223389 + - 0.18554688 + - -0.7553711 + - 1.5869141 + - -10.421875 + - -0.06225586 + - 0.010032654 + - -0.56103516 + - 0.6611328 + - 0.43969727 + - 0.3322754 + - -1.2880859 + - 0.35083008 + - -0.030227661 + - -0.38964844 + - -0.19934082 + - 0.2993164 + - -0.23413086 + - 0.7788086 + - 0.2434082 + - 0.8574219 + - 0.5605469 + - -0.60253906 + - 0.74560547 + - -0.092163086 + - 0.1550293 + - 0.0060577393 + - -0.27563477 + - 0.30200195 + - -0.7753906 + - 0.16113281 + - 0.47851563 + - 0.20031738 + - -0.032348633 + - 0.6723633 + - -0.9267578 + - 1.2167969 + - 0.08666992 + - -0.0042877197 + - -0.46826172 + - 0.49487305 + - -0.037994385 + - -0.12225342 + - 0.16149902 + - 0.6699219 + - 0.11102295 + - -0.09790039 + - -0.4350586 + - -0.4189453 + - 0.36376953 + - -0.27807617 + - -1.6953125 + - -0.7861328 + - -0.042816162 + - 0.43481445 + - 0.5102539 + - 0.037719727 + - -0.6269531 + - -0.8925781 + - -0.3461914 + - 0.28271484 + - -0.3173828 + - -0.8359375 + - 0.81884766 + - -0.025634766 + - -1.0087891 + - 0.051361084 + - -0.5576172 + - 0.4494629 + - 0.15014648 + - -0.30273438 + - -0.47631836 + - 0.49731445 + - 1.28125 + - -0.33496094 + - 0.46240234 + - 0.44091797 + - 1.1992188 + - -0.6147461 + - -0.2121582 + - 0.13452148 + - 1.1835938 + - 0.27856445 + - 0.026153564 + - 1.4316406 + - -0.08312988 + - -0.25976563 + - 0.1541748 + - -0.49804688 + - -0.0099487305 + - -0.23522949 + - 0.31103516 + - 0.08532715 + - -0.3959961 + - 0.2553711 + - 0.14807129 + - -0.049591064 + - -0.50097656 + - -0.10668945 + - -0.34399414 + - 0.54541016 + - -0.94628906 + - 0.29467773 + - -0.59472656 + - 0.1361084 + - 0.06640625 + - -0.030883789 + - -0.55908203 + - -0.41186523 + - -0.95947266 + - -1.0517578 + - 0.14428711 + - -0.16186523 + - -1.5146484 + - 0.19555664 + - -0.5473633 + - -0.073913574 + - -0.24536133 + - -0.47998047 + - -1.0888672 + - -0.021987915 + - -0.5107422 + - -0.4970703 + - -0.78027344 + - -0.3671875 + - 0.46923828 + - 0.011650085 + - 0.09283447 + - -0.8222656 + - -0.7807617 + - 1.0908203 + - -0.5229492 + - -0.75927734 + - -0.47705078 + - -0.4206543 + - 0.39086914 + - 1.2792969 + - 0.5048828 + - -0.09899902 + - -0.6352539 + - -0.00077199936 + - 0.80078125 + - -0.8129883 + - 0.95166016 + - -0.70996094 + - -0.37841797 + - 0.17565918 + - 0.20361328 + - -0.037200928 + - 0.057159424 + - -0.03933716 + - 0.16137695 + - -0.14086914 + - -1.0478516 + - -1.3056641 + - -0.80126953 + - 1.4951172 + - 0.74902344 + - -0.73583984 + - -0.38842773 + - -1.3876953 + - 0.7602539 + - -0.45458984 + - 0.105163574 + - 0.92871094 + - -0.5180664 + - 0.62939453 + - 1.828125 + - 0.13671875 + - -0.5917969 + - 0.17919922 + - -0.79248047 + - -0.29663086 + - 0.5097656 + - 0.33911133 + - 0.3540039 + - -0.50341797 + - 0.1182251 + - -0.15393066 + - -1.3339844 + - -0.28442383 + - 0.59716797 + - 0.8642578 + - -1.6845703 + - 0.15039063 + - 1.9853516 + - 0.30371094 + - 0.041412354 + - 0.29492188 + - 0.30395508 + - -0.5595703 + - 0.56396484 + - 1.0253906 + - -0.6640625 + - -0.031311035 + - -0.15283203 + - -0.055267334 + - -0.4880371 + - -0.19238281 + - -0.5 + - -0.7084961 + - 0.42822266 + - 0.1541748 + - -0.014923096 + - 1.0146484 + - -1.2333984 + - -0.13317871 + - 0.6503906 + - 0.8798828 + - 0.14440918 + - -0.7792969 + - 0.2890625 + - 0.35961914 + - -0.072509766 + - -0.14611816 + - -0.041137695 +- - 0.4428711 + - 0.6118164 + - 0.07043457 + - -0.5917969 + - 0.88183594 + - -0.10839844 + - 0.0881958 + - -0.20483398 + - 1.9345703 + - 0.6230469 + - 0.59521484 + - 0.18737793 + - 0.030548096 + - -0.078308105 + - 0.79589844 + - -0.23474121 + - -0.8720703 + - 1.6923828 + - 0.12597656 + - -1.9267578 + - -0.34472656 + - -1.9326172 + - 0.89990234 + - -0.2006836 + - 0.024230957 + - 0.2878418 + - 0.3642578 + - 0.17956543 + - -0.51123047 + - 0.025817871 + - -0.8964844 + - 0.14147949 + - -0.39257813 + - -0.8486328 + - 0.005218506 + - 0.42358398 + - 0.23181152 + - 1.1630859 + - -1.1455078 + - 0.3388672 + - -2.1484375 + - -1.0146484 + - -0.2841797 + - 0.021499634 + - 0.16235352 + - 0.39453125 + - -0.9511719 + - 0.41064453 + - -0.15563965 + - -0.484375 + - -0.07080078 + - 0.07312012 + - 0.56103516 + - 0.028213501 + - -0.027938843 + - 0.43896484 + - 1.0136719 + - 0.7788086 + - -0.5644531 + - -1.9824219 + - -0.011169434 + - 0.50634766 + - -0.39331055 + - -0.3334961 + - -1.1679688 + - -0.07208252 + - 0.008544922 + - 0.7788086 + - -0.10021973 + - -0.30859375 + - 0.35888672 + - -0.44848633 + - -0.08679199 + - -1.0791016 + - 1.3193359 + - -1.0273438 + - 0.34228516 + - -0.45385742 + - 0.8027344 + - 0.50634766 + - 0.703125 + - -0.96533203 + - -0.042938232 + - -1.0927734 + - 0.35205078 + - -0.56689453 + - -0.5341797 + - 0.23571777 + - -2.0234375 + - -0.26342773 + - 0.1998291 + - 0.7739258 + - 0.17797852 + - 1.2255859 + - 0.9814453 + - -0.2902832 + - 1.2080078 + - 0.50097656 + - 0.2479248 + - -0.2166748 + - 0.29052734 + - -0.5004883 + - 0.26513672 + - 0.5024414 + - -0.042266846 + - -0.013465881 + - -0.6376953 + - 0.64501953 + - -1.7958984 + - 0.25512695 + - -0.20458984 + - -0.12927246 + - 0.19665527 + - 0.8076172 + - -0.96777344 + - 0.26391602 + - -0.5649414 + - -0.58740234 + - -0.15466309 + - 0.7392578 + - 0.5336914 + - 0.60253906 + - 1.1875 + - -0.37719727 + - 0.8198242 + - -0.2401123 + - 0.40527344 + - 0.17114258 + - -0.32128906 + - -0.75439453 + - 0.056488037 + - 0.16491699 + - 0.6899414 + - -0.6171875 + - -0.05316162 + - 0.43017578 + - -0.6074219 + - 0.040374756 + - -0.8618164 + - -0.7192383 + - 0.7402344 + - -0.42138672 + - 0.26831055 + - -1.5429688 + - 0.1706543 + - 1.2158203 + - -0.17077637 + - -0.13964844 + - -0.5449219 + - -0.59033203 + - -0.7783203 + - -0.18786621 + - -0.34838867 + - 0.6230469 + - -0.5878906 + - -0.90234375 + - -0.7324219 + - -0.81933594 + - 0.4819336 + - -1.1591797 + - 1.2109375 + - -1.0517578 + - 0.25219727 + - 1.2841797 + - 0.06347656 + - -0.72021484 + - -0.3671875 + - 0.07330322 + - -0.022644043 + - -0.3713379 + - -0.008430481 + - 0.3564453 + - -0.8354492 + - -0.59521484 + - -0.40405273 + - -0.027923584 + - -0.3100586 + - -1.109375 + - -0.030593872 + - 0.14086914 + - -1.2128906 + - 0.03488159 + - 0.35424805 + - -1.5683594 + - 0.17773438 + - -0.66796875 + - -1.9804688 + - -0.35327148 + - -0.2927246 + - -1.2011719 + - -0.4165039 + - 0.36645508 + - -0.049957275 + - -0.87353516 + - -0.9086914 + - 0.4790039 + - -1.4599609 + - 0.7866211 + - -0.27490234 + - 0.08477783 + - 0.2220459 + - 0.0017213821 + - -1.0927734 + - -0.006439209 + - -0.62060547 + - -0.68115234 + - 0.4572754 + - -0.61572266 + - -0.7871094 + - -1.3847656 + - 1.5566406 + - 0.2680664 + - 0.8466797 + - -0.2578125 + - -0.76416016 + - 0.6484375 + - -0.05279541 + - -0.1307373 + - -0.11853027 + - -0.6357422 + - -0.17700195 + - 0.9472656 + - -0.4958496 + - 0.015464783 + - -0.23168945 + - -1.6748047 + - -0.8911133 + - 0.86865234 + - 1.0722656 + - 0.34228516 + - 0.7182617 + - -0.5488281 + - -0.45166016 + - -0.3022461 + - 0.45336914 + - -0.28808594 + - -0.7626953 + - 0.016067505 + - 0.63183594 + - 0.46923828 + - -0.07373047 + - 1.7832031 + - -0.41796875 + - -0.89453125 + - 0.07757568 + - -0.29589844 + - -1.0673828 + - 1.3320313 + - 1.4160156 + - 0.20703125 + - -0.1060791 + - 0.1508789 + - 0.37231445 + - 0.05053711 + - -0.68896484 + - 0.7397461 + - 0.15771484 + - 0.5595703 + - -0.8198242 + - -0.17712402 + - -0.2479248 + - -0.066345215 + - 0.4194336 + - 1.0517578 + - -1.6650391 + - 0.90283203 + - 0.22937012 + - -0.5908203 + - -0.79785156 + - -0.003993988 + - -0.47314453 + - 0.19116211 + - -0.39038086 + - 0.6333008 + - -0.28686523 + - -0.8720703 + - 0.080200195 + - -1.2587891 + - 0.22790527 + - 0.55126953 + - -0.47485352 + - -0.20471191 + - 0.75390625 + - -1.0488281 + - 0.9902344 + - 1.2275391 + - 0.40966797 + - -0.36791992 + - 0.54833984 + - -0.32788086 + - 0.2208252 + - 1.1289063 + - -0.017410278 + - -0.67089844 + - 0.7236328 + - 0.20788574 + - 0.55566406 + - 0.7104492 + - -0.16589355 + - 0.57958984 + - 1.1494141 + - -0.15930176 + - -0.4963379 + - 0.016326904 + - 1.5634766 + - -0.5029297 + - 1.2011719 + - -1.0029297 + - -0.3557129 + - -0.31079102 + - -0.12670898 + - 0.81933594 + - 0.75683594 + - 1.1630859 + - 1.6669922 + - -0.8022461 + - -0.3034668 + - 0.3798828 + - -0.71484375 + - -0.7470703 + - 1.1904297 + - -0.95654297 + - 0.30297852 + - 0.16992188 + - 0.8173828 + - -0.8798828 + - 1.1376953 + - -0.4543457 + - 0.6142578 + - -0.47680664 + - 1.1181641 + - 0.01398468 + - -0.49560547 + - -0.039031982 + - 0.2878418 + - -0.71533203 + - 0.82128906 + - -0.03768921 + - 0.60546875 + - 0.7578125 + - -0.3322754 + - 0.86376953 + - -0.71435547 + - 0.7597656 + - -1.3496094 + - 0.18652344 + - -0.39990234 + - 0.31103516 + - 0.16809082 + - 1.2626953 + - -0.42919922 + - 0.2434082 + - 0.83935547 + - -1.3994141 + - 0.34472656 + - -0.68603516 + - -0.28759766 + - 0.4663086 + - 0.6816406 + - -0.48120117 + - -0.11029053 + - -0.48339844 + - 0.1706543 + - 0.29614258 + - -0.7290039 + - -0.87158203 + - 0.8457031 + - 0.57128906 + - 0.1640625 + - -0.1739502 + - -0.16394043 + - 0.8652344 + - -0.38671875 + - 0.5625 + - -0.30493164 + - 1.8251953 + - 0.041931152 + - 0.17285156 + - -1.0107422 + - -0.20788574 + - 1.1005859 + - 0.7973633 + - -1.0029297 + - 0.67041016 + - -0.96240234 + - 0.45166016 + - -0.32202148 + - -0.97753906 + - 1.7080078 + - -0.04626465 + - 0.4404297 + - 0.28833008 + - 0.32006836 + - -0.5341797 + - -0.74658203 + - -1.28125 + - -0.2076416 + - -0.57958984 + - 0.88134766 + - -0.21875 + - -0.51171875 + - 0.20214844 + - -0.9536133 + - -0.89160156 + - 0.6279297 + - -0.08929443 + - 1.2490234 + - -0.3083496 + - 0.5253906 + - -1.5771484 + - 0.86572266 + - -0.16223145 + - -0.6455078 + - -0.9008789 + - -0.68896484 + - -0.55566406 + - 0.54785156 + - -0.2052002 + - 1.0820313 + - 0.5390625 + - 0.2454834 + - -0.0053596497 + - 0.09838867 + - -0.08288574 + - 0.6616211 + - -0.06573486 + - -0.8120117 + - -0.08306885 + - 0.12176514 + - -1.4072266 + - 0.6533203 + - 0.49536133 + - 0.48339844 + - -0.6142578 + - 0.06011963 + - 0.4951172 + - -0.33935547 + - -0.85058594 + - -0.8510742 + - 0.10028076 + - 0.64501953 + - 0.62841797 + - 0.79052734 + - -0.8486328 + - -0.76708984 + - 0.38891602 + - 0.5917969 + - -0.06939697 + - -0.13366699 + - -0.8964844 + - -0.76660156 + - 1.3134766 + - 0.98046875 + - 0.37158203 + - 0.04675293 + - -1.2890625 + - 0.17102051 + - -0.16772461 + - -0.10760498 + - -0.111816406 + - 0.20251465 + - -0.65527344 + - 0.2800293 + - -1.8037109 + - 0.15270996 + - 0.25024414 + - 0.6713867 + - 0.67578125 + - -1.0322266 + - -0.4140625 + - -0.59277344 + - 0.028778076 + - -1.3515625 + - 0.12548828 + - 0.87353516 + - 0.87060547 + - -0.18518066 + - 0.7949219 + - -0.7011719 + - -0.28759766 + - 0.39013672 + - 0.43139648 + - -0.053955078 + - -0.60498047 + - 0.6381836 + - 0.20996094 + - 0.09857178 + - -0.7402344 + - 0.24731445 + - -0.41430664 + - -0.49365234 + - -0.7817383 + - -0.35766602 + - -0.53759766 + - -0.13439941 + - 0.03010559 + - 0.47070313 + - 0.16564941 + - 1.0439453 + - 0.7026367 + - 0.3251953 + - 0.021118164 + - -0.25976563 + - 1.4033203 + - 0.3383789 + - -0.0637207 + - 0.8149414 + - -0.7651367 + - -0.5332031 + - -0.19335938 + - 0.19458008 + - -0.61083984 + - 0.07684326 + - -0.6982422 + - -0.38989258 + - 0.37841797 + - 0.5966797 + - -1.9882813 + - -0.36767578 + - -2.0214844 + - 0.9682617 + - 0.46826172 + - -0.49829102 + - -0.20861816 + - -0.15332031 + - -0.13891602 + - 1.8779297 + - -0.18640137 + - 0.38134766 + - -0.1373291 + - -0.29492188 + - 0.42871094 + - -0.24523926 + - 0.16320801 + - 0.8408203 + - -0.34985352 + - 0.5307617 + - -0.14685059 + - 0.7729492 + - -0.11444092 + - -0.23522949 + - 1.2089844 + - -1.5595703 + - 0.09820557 + - -1.3476563 + - 0.6196289 + - -1.1464844 + - -0.47045898 + - -0.10656738 + - -1.3115234 + - 0.4309082 + - 0.054748535 + - 0.80078125 + - 0.43359375 + - -0.22290039 + - 0.55908203 + - -8.1484375 + - -0.7236328 + - -0.4831543 + - -0.21447754 + - 0.51904297 + - 0.5449219 + - 0.8520508 + - -0.8852539 + - 0.36865234 + - -0.1694336 + - -0.32250977 + - 0.12176514 + - 0.92578125 + - 0.38891602 + - 0.79296875 + - 0.46875 + - 0.9213867 + - 0.9970703 + - -0.32739258 + - 1.2900391 + - -0.1517334 + - 0.18640137 + - -0.6113281 + - 0.060821533 + - 0.3527832 + - -0.7915039 + - -0.3942871 + - 1.2158203 + - -0.8051758 + - 0.13598633 + - 0.21057129 + - -0.66552734 + - 1.5957031 + - 0.44262695 + - -0.24023438 + - -0.3984375 + - -1.0517578 + - -0.36499023 + - -0.5966797 + - 0.60546875 + - 1.2324219 + - 0.21520996 + - -0.18481445 + - -0.0066490173 + - -0.08239746 + - 0.7993164 + - -0.81884766 + - -1.6660156 + - -1.2861328 + - -0.8535156 + - 0.15856934 + - 0.6230469 + - 0.60595703 + - 0.5234375 + - -0.93603516 + - 0.7529297 + - 0.06738281 + - -0.002035141 + - -0.92285156 + - -0.25512695 + - -0.50878906 + - -0.7944336 + - -0.44970703 + - -0.64501953 + - 0.8569336 + - 0.12030029 + - -0.08343506 + - -0.5126953 + - 0.16540527 + - 1.6054688 + - -0.3947754 + - 0.34350586 + - -0.43066406 + - -0.04083252 + - 0.2956543 + - -0.5175781 + - 0.6567383 + - 0.57470703 + - 0.6694336 + - 0.13891602 + - 0.9790039 + - 0.16503906 + - -0.12017822 + - -0.8745117 + - 0.057861328 + - 0.24194336 + - -0.012207031 + - 0.40722656 + - -0.34936523 + - -0.1998291 + - 0.41210938 + - -0.17858887 + - -0.011474609 + - 0.10394287 + - -0.1048584 + - 0.037384033 + - 1.1806641 + - -0.68896484 + - 0.7397461 + - -0.7661133 + - -0.3540039 + - 0.6098633 + - -0.93066406 + - -0.49047852 + - -0.29418945 + - -1.4736328 + - -1.0498047 + - -1.2509766 + - -0.546875 + - -1.4882813 + - 0.4802246 + - -0.79052734 + - -0.24499512 + - -0.5517578 + - -0.08282471 + - -0.47705078 + - 0.18701172 + - 0.47827148 + - -0.21008301 + - -0.83154297 + - -0.106933594 + - 0.041656494 + - -0.09887695 + - 0.58447266 + - -0.80859375 + - -1.1503906 + - 0.19140625 + - 0.50146484 + - -0.8930664 + - -0.037750244 + - -0.4411621 + - 0.50439453 + - 1.484375 + - -0.8754883 + - -0.3227539 + - -0.98583984 + - -0.09124756 + - 0.48510742 + - -0.2775879 + - 0.80371094 + - -0.5683594 + - -0.25732422 + - 0.7504883 + - -0.37426758 + - 0.2199707 + - 0.37426758 + - -0.03375244 + - -0.40771484 + - -0.18078613 + - -0.48095703 + - -1.5078125 + - -0.6743164 + - 0.82177734 + - 0.25073242 + - -0.8261719 + - 0.33862305 + - -1.2646484 + - 0.8666992 + - -0.27392578 + - -0.34399414 + - -0.42333984 + - -0.4116211 + - 0.8027344 + - 1.1503906 + - -1.2041016 + - 0.35620117 + - -0.00016129017 + - -1.953125 + - -0.6035156 + - 0.58154297 + - 0.08703613 + - 0.48266602 + - -0.35180664 + - -0.11047363 + - 0.65966797 + - -1.4921875 + - -0.14086914 + - -0.2130127 + - 1.0244141 + - -1.4472656 + - 0.56152344 + - 1.734375 + - 0.09698486 + - -0.5415039 + - -0.1340332 + - 0.7583008 + - -0.2722168 + - 1.0488281 + - 1.0498047 + - 0.23254395 + - -0.15222168 + - 0.18005371 + - 0.097839355 + - -1.6386719 + - -0.6035156 + - -0.91748047 + - -0.93847656 + - 1.0576172 + - 0.703125 + - 0.70410156 + - 0.8198242 + - -2.25 + - -0.6357422 + - 0.35058594 + - 0.75146484 + - -0.12322998 + - -1.0859375 + - 0.50634766 + - -0.67822266 + - 0.7495117 + - -0.27685547 + - -0.13537598 diff --git a/backends/candle/tests/snapshots/test_flash_gte__gte_single.snap b/backends/candle/tests/snapshots/test_flash_gte__gte_single.snap new file mode 100644 index 00000000..96c169b0 --- /dev/null +++ b/backends/candle/tests/snapshots/test_flash_gte__gte_single.snap @@ -0,0 +1,773 @@ +--- +source: backends/candle/tests/test_flash_gte.rs +assertion_line: 48 +expression: embeddings_single +--- +- - 0.44262695 + - 0.6113281 + - 0.07098389 + - -0.5908203 + - 0.88134766 + - -0.10870361 + - 0.08959961 + - -0.20568848 + - 1.9335938 + - 0.62353516 + - 0.5957031 + - 0.18664551 + - 0.030349731 + - -0.07714844 + - 0.7963867 + - -0.23425293 + - -0.8720703 + - 1.6923828 + - 0.12585449 + - -1.9277344 + - -0.34448242 + - -1.9326172 + - 0.9003906 + - -0.20056152 + - 0.023971558 + - 0.28833008 + - 0.3647461 + - 0.17883301 + - -0.51123047 + - 0.02519226 + - -0.89453125 + - 0.14074707 + - -0.3918457 + - -0.84814453 + - 0.0044822693 + - 0.42407227 + - 0.23144531 + - 1.1621094 + - -1.1464844 + - 0.33862305 + - -2.1484375 + - -1.0136719 + - -0.2841797 + - 0.02142334 + - 0.16271973 + - 0.3942871 + - -0.9511719 + - 0.4099121 + - -0.15612793 + - -0.484375 + - -0.071777344 + - 0.07232666 + - 0.5595703 + - 0.028564453 + - -0.027267456 + - 0.43798828 + - 1.0136719 + - 0.7792969 + - -0.56396484 + - -1.9824219 + - -0.011451721 + - 0.50683594 + - -0.39331055 + - -0.3334961 + - -1.1679688 + - -0.072387695 + - 0.008460999 + - 0.7788086 + - -0.099609375 + - -0.30981445 + - 0.3581543 + - -0.4482422 + - -0.08532715 + - -1.0791016 + - 1.3193359 + - -1.0273438 + - 0.34350586 + - -0.45288086 + - 0.8022461 + - 0.5048828 + - 0.7026367 + - -0.96533203 + - -0.04159546 + - -1.09375 + - 0.35180664 + - -0.56689453 + - -0.5341797 + - 0.23583984 + - -2.0253906 + - -0.2619629 + - 0.19995117 + - 0.7739258 + - 0.17883301 + - 1.2255859 + - 0.98095703 + - -0.2902832 + - 1.2080078 + - 0.50097656 + - 0.24719238 + - -0.21533203 + - 0.2902832 + - -0.50097656 + - 0.26489258 + - 0.5019531 + - -0.042816162 + - -0.013328552 + - -0.63916016 + - 0.64404297 + - -1.7949219 + - 0.25439453 + - -0.2043457 + - -0.12939453 + - 0.19616699 + - 0.80908203 + - -0.9667969 + - 0.26342773 + - -0.5644531 + - -0.58691406 + - -0.1541748 + - 0.7397461 + - 0.5336914 + - 0.6020508 + - 1.1865234 + - -0.3774414 + - 0.8198242 + - -0.24084473 + - 0.40527344 + - 0.17089844 + - -0.3203125 + - -0.7548828 + - 0.056518555 + - 0.16540527 + - 0.6899414 + - -0.6166992 + - -0.052642822 + - 0.43041992 + - -0.6074219 + - 0.039886475 + - -0.8613281 + - -0.71875 + - 0.7402344 + - -0.42138672 + - 0.26953125 + - -1.5439453 + - 0.17004395 + - 1.2167969 + - -0.17102051 + - -0.13879395 + - -0.5463867 + - -0.59033203 + - -0.7783203 + - -0.1875 + - -0.34838867 + - 0.6230469 + - -0.5888672 + - -0.9038086 + - -0.7319336 + - -0.8183594 + - 0.4807129 + - -1.1611328 + - 1.2109375 + - -1.0537109 + - 0.25268555 + - 1.2851563 + - 0.06317139 + - -0.7207031 + - -0.36694336 + - 0.072265625 + - -0.022918701 + - -0.3708496 + - -0.0073547363 + - 0.35791016 + - -0.8354492 + - -0.5961914 + - -0.40429688 + - -0.027389526 + - -0.31079102 + - -1.1103516 + - -0.030670166 + - 0.14013672 + - -1.2119141 + - 0.035247803 + - 0.3527832 + - -1.5673828 + - 0.1763916 + - -0.66748047 + - -1.9804688 + - -0.35351563 + - -0.29223633 + - -1.2021484 + - -0.41601563 + - 0.36669922 + - -0.051330566 + - -0.87353516 + - -0.9086914 + - 0.4794922 + - -1.4589844 + - 0.7866211 + - -0.27441406 + - 0.084106445 + - 0.22143555 + - 0.0014467239 + - -1.0917969 + - -0.0079574585 + - -0.62060547 + - -0.6816406 + - 0.45751953 + - -0.6166992 + - -0.78759766 + - -1.3847656 + - 1.5576172 + - 0.2697754 + - 0.8461914 + - -0.25756836 + - -0.765625 + - 0.6489258 + - -0.053344727 + - -0.13195801 + - -0.119628906 + - -0.6357422 + - -0.17736816 + - 0.9477539 + - -0.49682617 + - 0.016204834 + - -0.23156738 + - -1.6738281 + - -0.890625 + - 0.8666992 + - 1.0703125 + - 0.3425293 + - 0.71972656 + - -0.5498047 + - -0.45263672 + - -0.30249023 + - 0.45336914 + - -0.28588867 + - -0.7626953 + - 0.017074585 + - 0.6308594 + - 0.47094727 + - -0.07519531 + - 1.7861328 + - -0.41723633 + - -0.8935547 + - 0.077697754 + - -0.29492188 + - -1.0673828 + - 1.3300781 + - 1.4150391 + - 0.20751953 + - -0.10571289 + - 0.15100098 + - 0.37182617 + - 0.05105591 + - -0.6875 + - 0.7392578 + - 0.15649414 + - 0.5600586 + - -0.81884766 + - -0.17700195 + - -0.24841309 + - -0.06628418 + - 0.42016602 + - 1.0527344 + - -1.6660156 + - 0.90283203 + - 0.22888184 + - -0.5917969 + - -0.7973633 + - -0.0041236877 + - -0.4741211 + - 0.19104004 + - -0.390625 + - 0.63427734 + - -0.28710938 + - -0.87158203 + - 0.08068848 + - -1.2587891 + - 0.22729492 + - 0.5517578 + - -0.47558594 + - -0.20385742 + - 0.7553711 + - -1.0498047 + - 0.99121094 + - 1.2275391 + - 0.41015625 + - -0.36791992 + - 0.54833984 + - -0.3269043 + - 0.21972656 + - 1.1298828 + - -0.01737976 + - -0.6699219 + - 0.72265625 + - 0.20800781 + - 0.5566406 + - 0.7109375 + - -0.16577148 + - 0.5805664 + - 1.1494141 + - -0.15942383 + - -0.49658203 + - 0.01625061 + - 1.5625 + - -0.50390625 + - 1.2011719 + - -1.0029297 + - -0.35473633 + - -0.3100586 + - -0.12658691 + - 0.8208008 + - 0.7578125 + - 1.1611328 + - 1.6660156 + - -0.8022461 + - -0.30273438 + - 0.38012695 + - -0.71484375 + - -0.7475586 + - 1.1914063 + - -0.9560547 + - 0.30297852 + - 0.17163086 + - 0.8183594 + - -0.87841797 + - 1.1376953 + - -0.4543457 + - 0.61376953 + - -0.47851563 + - 1.1181641 + - 0.013900757 + - -0.49560547 + - -0.03945923 + - 0.28637695 + - -0.71533203 + - 0.8203125 + - -0.037322998 + - 0.6044922 + - 0.75683594 + - -0.33276367 + - 0.8647461 + - -0.7138672 + - 0.7597656 + - -1.3496094 + - 0.18701172 + - -0.3996582 + - 0.3112793 + - 0.16809082 + - 1.2617188 + - -0.42993164 + - 0.24401855 + - 0.83935547 + - -1.3994141 + - 0.34594727 + - -0.68603516 + - -0.2878418 + - 0.46655273 + - 0.68066406 + - -0.48291016 + - -0.11004639 + - -0.48364258 + - 0.17126465 + - 0.29614258 + - -0.7290039 + - -0.8725586 + - 0.8457031 + - 0.5732422 + - 0.16186523 + - -0.17407227 + - -0.16455078 + - 0.8652344 + - -0.38598633 + - 0.56396484 + - -0.30639648 + - 1.8261719 + - 0.04208374 + - 0.1743164 + - -1.0107422 + - -0.20812988 + - 1.1015625 + - 0.79833984 + - -1.0019531 + - 0.67041016 + - -0.96191406 + - 0.45092773 + - -0.32226563 + - -0.9790039 + - 1.7089844 + - -0.04675293 + - 0.44091797 + - 0.28930664 + - 0.32104492 + - -0.5336914 + - -0.7475586 + - -1.2802734 + - -0.20861816 + - -0.57958984 + - 0.8798828 + - -0.21777344 + - -0.51220703 + - 0.20239258 + - -0.953125 + - -0.8925781 + - 0.62890625 + - -0.08935547 + - 1.2490234 + - -0.30859375 + - 0.52490234 + - -1.5771484 + - 0.8647461 + - -0.16259766 + - -0.64697266 + - -0.9003906 + - -0.68896484 + - -0.55566406 + - 0.54785156 + - -0.20458984 + - 1.0810547 + - 0.5390625 + - 0.24572754 + - -0.0054397583 + - 0.097717285 + - -0.08135986 + - 0.6616211 + - -0.06652832 + - -0.8129883 + - -0.08251953 + - 0.12164307 + - -1.4072266 + - 0.65283203 + - 0.4946289 + - 0.48364258 + - -0.6142578 + - 0.060272217 + - 0.49560547 + - -0.33911133 + - -0.85058594 + - -0.8510742 + - 0.10040283 + - 0.64453125 + - 0.6279297 + - 0.7890625 + - -0.8496094 + - -0.76708984 + - 0.38916016 + - 0.59277344 + - -0.07110596 + - -0.1340332 + - -0.8955078 + - -0.7661133 + - 1.3125 + - 0.98095703 + - 0.3708496 + - 0.046966553 + - -1.2900391 + - 0.17272949 + - -0.1673584 + - -0.1071167 + - -0.11187744 + - 0.20092773 + - -0.65478516 + - 0.27978516 + - -1.8027344 + - 0.15197754 + - 0.25 + - 0.671875 + - 0.6748047 + - -1.0322266 + - -0.41308594 + - -0.5917969 + - 0.028701782 + - -1.3515625 + - 0.12524414 + - 0.87402344 + - 0.8701172 + - -0.18334961 + - 0.79541016 + - -0.7006836 + - -0.28833008 + - 0.39086914 + - 0.43115234 + - -0.05316162 + - -0.60498047 + - 0.6386719 + - 0.21044922 + - 0.09869385 + - -0.7402344 + - 0.24731445 + - -0.4140625 + - -0.49487305 + - -0.78125 + - -0.35864258 + - -0.5366211 + - -0.13464355 + - 0.030441284 + - 0.47045898 + - 0.16552734 + - 1.0458984 + - 0.7026367 + - 0.32470703 + - 0.021697998 + - -0.26098633 + - 1.4023438 + - 0.33789063 + - -0.062469482 + - 0.81396484 + - -0.765625 + - -0.5341797 + - -0.19299316 + - 0.19433594 + - -0.6113281 + - 0.07598877 + - -0.6977539 + - -0.39038086 + - 0.3774414 + - 0.5957031 + - -1.9902344 + - -0.36889648 + - -2.0195313 + - 0.9692383 + - 0.46948242 + - -0.49975586 + - -0.20837402 + - -0.1529541 + - -0.13879395 + - 1.8769531 + - -0.18579102 + - 0.38256836 + - -0.13684082 + - -0.2956543 + - 0.42749023 + - -0.24560547 + - 0.16381836 + - 0.84228516 + - -0.35009766 + - 0.53125 + - -0.14697266 + - 0.7734375 + - -0.114990234 + - -0.23522949 + - 1.2080078 + - -1.5605469 + - 0.098083496 + - -1.3466797 + - 0.62060547 + - -1.1484375 + - -0.4699707 + - -0.106933594 + - -1.3095703 + - 0.43066406 + - 0.05444336 + - 0.80126953 + - 0.4326172 + - -0.22338867 + - 0.55908203 + - -8.1484375 + - -0.7246094 + - -0.48291016 + - -0.21386719 + - 0.51953125 + - 0.5449219 + - 0.8515625 + - -0.88378906 + - 0.36865234 + - -0.16955566 + - -0.3244629 + - 0.12188721 + - 0.92529297 + - 0.38891602 + - 0.7944336 + - 0.46850586 + - 0.921875 + - 0.9980469 + - -0.32739258 + - 1.2910156 + - -0.15161133 + - 0.18591309 + - -0.6123047 + - 0.060546875 + - 0.35327148 + - -0.7915039 + - -0.3947754 + - 1.2158203 + - -0.8046875 + - 0.13659668 + - 0.21166992 + - -0.6669922 + - 1.5957031 + - 0.44262695 + - -0.24145508 + - -0.39770508 + - -1.0517578 + - -0.36450195 + - -0.5961914 + - 0.60498047 + - 1.2314453 + - 0.21472168 + - -0.18359375 + - -0.0063056946 + - -0.08239746 + - 0.80029297 + - -0.8183594 + - -1.6660156 + - -1.2861328 + - -0.85253906 + - 0.15881348 + - 0.6225586 + - 0.60546875 + - 0.5239258 + - -0.9355469 + - 0.7529297 + - 0.06768799 + - -0.0025157928 + - -0.9238281 + - -0.25390625 + - -0.5097656 + - -0.79345703 + - -0.4501953 + - -0.6459961 + - 0.8564453 + - 0.121154785 + - -0.08404541 + - -0.51416016 + - 0.16540527 + - 1.6054688 + - -0.39404297 + - 0.34277344 + - -0.43017578 + - -0.040618896 + - 0.29614258 + - -0.5180664 + - 0.6567383 + - 0.57373047 + - 0.66845703 + - 0.14013672 + - 0.98046875 + - 0.16442871 + - -0.12042236 + - -0.8745117 + - 0.057556152 + - 0.24316406 + - -0.012084961 + - 0.40649414 + - -0.3486328 + - -0.20080566 + - 0.4116211 + - -0.17822266 + - -0.011131287 + - 0.10406494 + - -0.10601807 + - 0.038391113 + - 1.1806641 + - -0.68847656 + - 0.73779297 + - -0.7651367 + - -0.35229492 + - 0.6113281 + - -0.93066406 + - -0.49072266 + - -0.2944336 + - -1.4726563 + - -1.0498047 + - -1.2509766 + - -0.54833984 + - -1.4892578 + - 0.48120117 + - -0.7919922 + - -0.24487305 + - -0.5517578 + - -0.08355713 + - -0.4765625 + - 0.1875 + - 0.4790039 + - -0.21032715 + - -0.8310547 + - -0.10638428 + - 0.042144775 + - -0.098083496 + - 0.5839844 + - -0.80859375 + - -1.1503906 + - 0.19128418 + - 0.5019531 + - -0.8935547 + - -0.037200928 + - -0.4416504 + - 0.50390625 + - 1.4833984 + - -0.87597656 + - -0.32202148 + - -0.9848633 + - -0.09112549 + - 0.48486328 + - -0.2775879 + - 0.8051758 + - -0.56884766 + - -0.2565918 + - 0.75097656 + - -0.37402344 + - 0.21936035 + - 0.37524414 + - -0.03427124 + - -0.40771484 + - -0.18054199 + - -0.48168945 + - -1.5068359 + - -0.67333984 + - 0.82128906 + - 0.25219727 + - -0.82714844 + - 0.33935547 + - -1.2626953 + - 0.86621094 + - -0.27441406 + - -0.34350586 + - -0.42382813 + - -0.41186523 + - 0.80371094 + - 1.1513672 + - -1.2041016 + - 0.3552246 + - -0.00065660477 + - -1.953125 + - -0.60253906 + - 0.58154297 + - 0.086364746 + - 0.48217773 + - -0.35253906 + - -0.11077881 + - 0.66064453 + - -1.4921875 + - -0.14025879 + - -0.21313477 + - 1.0234375 + - -1.4482422 + - 0.56103516 + - 1.734375 + - 0.09710693 + - -0.54003906 + - -0.13305664 + - 0.75878906 + - -0.27246094 + - 1.0488281 + - 1.0507813 + - 0.23254395 + - -0.15209961 + - 0.18017578 + - 0.09790039 + - -1.6376953 + - -0.60253906 + - -0.9165039 + - -0.9379883 + - 1.0585938 + - 0.703125 + - 0.70458984 + - 0.8203125 + - -2.2519531 + - -0.63623047 + - 0.35058594 + - 0.7504883 + - -0.12408447 + - -1.0839844 + - 0.50683594 + - -0.6796875 + - 0.7495117 + - -0.27783203 + - -0.13562012 diff --git a/backends/candle/tests/test_flash_gte.rs b/backends/candle/tests/test_flash_gte.rs new file mode 100644 index 00000000..20b06b2f --- /dev/null +++ b/backends/candle/tests/test_flash_gte.rs @@ -0,0 +1,53 @@ +#![allow(dead_code, unused_imports)] +mod common; + +use crate::common::{sort_embeddings, SnapshotEmbeddings}; +use anyhow::Result; +use common::{batch, cosine_matcher, download_artifacts, load_tokenizer}; +use text_embeddings_backend_candle::CandleBackend; +use text_embeddings_backend_core::{Backend, ModelType, Pool}; + +#[test] +#[serial_test::serial] +#[cfg(all(feature = "cuda", feature = "flash-attn"))] +fn test_flash_gte() -> Result<()> { + let model_root = download_artifacts("Alibaba-NLP/gte-base-en-v1.5", None)?; + let tokenizer = load_tokenizer(&model_root)?; + + let backend = CandleBackend::new( + model_root, + "float16".to_string(), + ModelType::Embedding(Pool::Cls), + )?; + + let input_batch = batch( + vec![ + tokenizer.encode("What is Deep Learning?", true).unwrap(), + tokenizer.encode("Deep Learning is...", true).unwrap(), + tokenizer.encode("What is Deep Learning?", true).unwrap(), + ], + [0, 1, 2].to_vec(), + vec![], + ); + + let matcher = cosine_matcher(); + + let (pooled_embeddings, _) = sort_embeddings(backend.embed(input_batch)?); + let embeddings_batch = SnapshotEmbeddings::from(pooled_embeddings); + insta::assert_yaml_snapshot!("gte_batch", embeddings_batch, &matcher); + + let input_single = batch( + vec![tokenizer.encode("What is Deep Learning?", true).unwrap()], + [0].to_vec(), + vec![], + ); + + let (pooled_embeddings, _) = sort_embeddings(backend.embed(input_single)?); + let embeddings_single = SnapshotEmbeddings::from(pooled_embeddings); + + insta::assert_yaml_snapshot!("gte_single", embeddings_single, &matcher); + assert_eq!(embeddings_batch[0], embeddings_single[0]); + assert_eq!(embeddings_batch[2], embeddings_single[0]); + + Ok(()) +}