Struct Compressor 

pub struct Compressor {
    pub model: Arc<Model>,
    pub state: State,
    pub scratch: ScratchBuffers,
    pub pdf_buffer: Vec<f64>,
    pub cdf_buffer_ac: Vec<u32>,
    pub ac_freq_buffer: Vec<i64>,
    pub cdf_buffer_rans: Vec<u32>,
    pub rans_freq_buffer: Vec<i64>,
    /* private fields */
}

Main compressor/decompressor that combines RWKV7 inference with entropy coding.

The compressor maintains internal state and pre-allocated buffers to minimize allocations during the compression/decompression hot path.

Fields

model: Arc<Model>

RWKV7 model for generating probability distributions.

state: State

Model state (recurrent hidden states).

scratch: ScratchBuffers

Scratch buffers for model forward passes.

pdf_buffer: Vec<f64>

Pre-allocated PDF buffer (eliminates allocations in compression loop).

cdf_buffer_ac: Vec<u32>

Reusable AC CDF buffer (vocab_size + 1 entries).

ac_freq_buffer: Vec<i64>

Scratch frequencies for AC quantization.

cdf_buffer_rans: Vec<u32>

Reusable rANS CDF buffer (vocab_size + 1 entries).

rans_freq_buffer: Vec<i64>

Scratch frequencies for rANS quantization.

Implementations

impl Compressor

pub fn new<P: AsRef<Path>>(model_path: P) -> Result<Self>

Create a new compressor with the given model.

Arguments

• model_path - Path to RWKV7 model weights (.safetensors format)

Returns

A new Compressor ready for compression/decompression operations.

pub fn load_model<P: AsRef<Path>>(model_path: P) -> Result<Arc<Model>>

Load a model from disk and wrap it in Arc.

pub fn new_from_model(model: Arc<Model>) -> Self

Create a compressor from a preloaded model.

pub fn new_from_method(method: &str) -> Result<Self>

Create a compressor from a user method string.

pub fn reset(&mut self)

Reset the model state to initial values.

Call this between independent compression/decompression operations to ensure a clean state.

pub fn begin_online_policy_stream( &mut self, total_symbols: Option<u64>, ) -> Result<()>

Begin a policy stream with optional total symbol count.

pub fn finish_online_policy_stream(&mut self) -> Result<()>

Flush any pending TBPTT segment while preserving the current predictive state.

pub fn restart_online_policy_stream( &mut self, total_symbols: Option<u64>, ) -> Result<()>

Reset hidden state and TBPTT bookkeeping for a fresh stream.

pub fn reset_and_prime(&mut self)

Reset state and prime the first predictive distribution.

pub fn snapshot_runtime(&self) -> RuntimeSnapshot

Capture runtime state for later restoration.

pub fn restore_runtime(&mut self, snapshot: &RuntimeSnapshot)

Restore previously captured runtime state.

pub fn absorb_chain(&mut self, parts: &[&[u8]]) -> Result<()>

Absorb a sequence of byte slices as conditioning context.

pub fn cross_entropy_from_current(&mut self, data: &[u8]) -> Result<f64>

Score bytes from the current predictive state.

pub fn cross_entropy_frozen_plugin_chain( &mut self, fit_parts: &[&[u8]], data: &[u8], ) -> Result<f64>

Fit on fit_parts, then reset stream state and score data without further adaptation.

pub fn is_online(&self) -> bool

Returns true when the compressor is in online-adaptation mode.

pub fn can_adapt_online(&self) -> bool

Returns true when the current online configuration can actually adapt parameters.

pub fn tokens_processed(&self) -> u64

Number of tokens processed by the online updater.

pub fn online_method_string(&self) -> Option<&str>

Canonical method string for online mode, if enabled.

pub fn vocab_size(&self) -> usize

Get the vocabulary size (should always be 256 for byte-level).

pub fn online_apply_logits_bias(&self, logits: &[f32], pdf_out: &mut [f64])

Apply optional online bias to logits and emit normalized PDF.

pub fn logits_to_pdf(logits: &[f32], bias: Option<&[f32]>, pdf_out: &mut [f64])

Convert logits (and optional bias) to a normalized PDF.

pub fn forward_to_pdf(&mut self, token: u32, pdf_out: &mut [f64])

Forward one token and emit the resulting (optionally biased) PDF.

pub fn forward_to_internal_pdf(&mut self, token: u32)

Refresh the internal cached PDF buffer from token.

pub fn copy_current_pdf_to(&self, pdf_out: &mut [f64])

Copy the internal cached PDF into pdf_out (length must match vocab size).

pub fn online_bias_snapshot(&self) -> Option<Vec<f32>>

Snapshot current online output bias, if online mode is active.

pub fn online_bias_slice(&self) -> Option<&[f32]>

Borrow online output bias vector when online mode is active.

pub fn online_update_from_pdf(&mut self, symbol: u8, pdf: &[f64]) -> Result<()>

Apply one online update using externally supplied predictive PDF.

pub fn observe_symbol_from_pdf(&mut self, symbol: u8, pdf: &[f64]) -> Result<()>

Update online state from pdf, then advance model state with symbol.

pub fn observe_symbol_from_current_pdf(&mut self, symbol: u8) -> Result<()>

Update online state using current internal PDF, then consume symbol.

pub fn export_online<P: AsRef<Path>>(&self, model_path: P) -> Result<()>

Export model weights and JSON sidecar metadata.

pub fn compress(&mut self, data: &[u8], coder: CoderType) -> Result<Vec<u8>>

Compress data using the specified entropy coder.

Arguments

• data - Raw bytes to compress
• coder - Entropy coder to use (AC or rANS)

Returns

Compressed data including header with checksum.

pub fn compress_into<W: Write>( &mut self, data: &[u8], coder: CoderType, w: &mut W, ) -> Result<()>

Compress into an arbitrary writer.

pub fn compress_chain_into<W: Write>( &mut self, parts: &[&[u8]], coder: CoderType, w: &mut W, ) -> Result<()>

Compress a chain of byte slices into an arbitrary writer.

pub fn compress_size(&mut self, data: &[u8], coder: CoderType) -> Result<u64>

Return compressed byte size without materializing output bytes.

pub fn compress_size_chain( &mut self, parts: &[&[u8]], coder: CoderType, ) -> Result<u64>

Return compressed byte size for chained inputs.

pub fn decompress(&mut self, data: &[u8]) -> Result<Vec<u8>>

Decompress data.

Arguments

• data - Compressed data (must include header)

Returns

Original decompressed data. Returns error if checksum doesn’t match.

pub fn cross_entropy(&mut self, data: &[u8]) -> Result<f64>

Calculate cross-entropy (bits per byte) for data without compression.

This measures how well the model predicts the data, giving a theoretical lower bound on achievable compression. Useful for evaluating model quality.

Arguments

• data - Data to analyze

Returns

Average bits per byte (lower is better, 8.0 means no compression possible).

pub fn cross_entropy_conditional_chain( &mut self, prefix_parts: &[&[u8]], data: &[u8], ) -> Result<f64>

Cross entropy conditioned on chained prefix slices.

pub fn cross_entropy_conditional( &mut self, prefix: &[u8], data: &[u8], ) -> Result<f64>

Cross entropy conditioned on a single prefix slice.

pub fn joint_cross_entropy_aligned_min( &mut self, x: &[u8], y: &[u8], ) -> Result<f64>

Symmetric aligned joint cross entropy using the better ordering.

Trait Implementations

impl Clone for Compressor

fn clone(&self) -> Self

Returns a duplicate of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.