infotheory/api/
paths.rs

1//! Path-oriented convenience API surface.
2
3use rayon::prelude::*;
4
5use super::compression::{
6    NcdComputeOptions, NcdVariant, OperationParallelism, try_compress_size_backend,
7    try_ncd_bytes_backend_with_options, try_ncd_matrix_bytes_backend_with_options,
8};
9use super::metrics::{
10    d_kl_bytes, js_div_bytes, nhd_bytes, try_conditional_entropy_bytes, try_cross_entropy_bytes,
11    try_mutual_information_bytes, try_ned_bytes, try_nte_bytes, tvd_bytes,
12};
13use super::types::CompressionBackend;
14use crate::error::{InfotheoryError, InfotheoryResult};
15use crate::spec::CompiledCompressionBackend;
16use rayon::ThreadPoolBuilder;
17
18#[inline(always)]
19fn try_read_path_pair(x: &str, y: &str) -> InfotheoryResult<(Vec<u8>, Vec<u8>)> {
20    let (bx, by) = rayon::join(
21        || std::fs::read(x).map_err(InfotheoryError::from),
22        || std::fs::read(y).map_err(InfotheoryError::from),
23    );
24    Ok((bx?, by?))
25}
26
27/// Options for backend-first path compression-size operations.
28#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
29pub struct CompressionPathBatchOptions {
30    /// Operation-level parallelism policy used while processing the path batch. (external parellization, doesn't affect compression algorithm itself)
31    pub parallelism: OperationParallelism,
32}
33
34/// Read `path` and return its compressed size (bytes) using `backend`.
35pub fn try_get_compressed_size_path_backend(
36    path: &str,
37    backend: &CompiledCompressionBackend,
38) -> InfotheoryResult<u64> {
39    let data = std::fs::read(path)?;
40    try_compress_size_backend(&data, backend)
41}
42
43#[inline(always)]
44/// Read all files in `paths` in parallel and return their contents.
45pub fn try_get_bytes_from_paths(paths: &[&str]) -> InfotheoryResult<Vec<Vec<u8>>> {
46    paths
47        .par_iter()
48        .map(|path| std::fs::read(*path).map_err(InfotheoryError::from))
49        .collect()
50}
51
52/// Compute compressed sizes for `paths` using `backend`.
53pub fn try_get_compressed_sizes_from_paths_backend(
54    paths: &[&str],
55    backend: &CompiledCompressionBackend,
56) -> InfotheoryResult<Vec<u64>> {
57    try_get_compressed_sizes_from_paths_backend_with_options(
58        paths,
59        backend,
60        CompressionPathBatchOptions::default(),
61    )
62}
63
64/// Compute compressed sizes for `paths` using `backend` with explicit
65/// operation-level parallelism controls.
66pub fn try_get_compressed_sizes_from_paths_backend_with_options(
67    paths: &[&str],
68    backend: &CompiledCompressionBackend,
69    options: CompressionPathBatchOptions,
70) -> InfotheoryResult<Vec<u64>> {
71    match options.parallelism {
72        OperationParallelism::Serial => paths
73            .iter()
74            .map(|path| try_get_compressed_size_path_backend(path, backend))
75            .collect(),
76        OperationParallelism::Auto => paths
77            .par_iter()
78            .map(|path| try_get_compressed_size_path_backend(path, backend))
79            .collect(),
80        OperationParallelism::Threads(threads) => {
81            if threads <= 1 {
82                return paths
83                    .iter()
84                    .map(|path| try_get_compressed_size_path_backend(path, backend))
85                    .collect();
86            }
87            let pool = ThreadPoolBuilder::new()
88                .num_threads(threads)
89                .build()
90                .map_err(|err| {
91                    InfotheoryError::runtime(format!("failed to build rayon pool: {err}"))
92                })?;
93            pool.install(|| {
94                paths
95                    .par_iter()
96                    .map(|path| try_get_compressed_size_path_backend(path, backend))
97                    .collect()
98            })
99        }
100    }
101}
102
103/// Compute NCD for two files with an explicit compression backend.
104pub fn try_ncd_paths_backend(
105    x: &str,
106    y: &str,
107    backend: &CompressionBackend,
108    variant: NcdVariant,
109) -> InfotheoryResult<f64> {
110    let (bx, by) = rayon::join(
111        || std::fs::read(x).map_err(InfotheoryError::from),
112        || std::fs::read(y).map_err(InfotheoryError::from),
113    );
114    let compiled = backend
115        .compile()
116        .map_err(|err| InfotheoryError::invalid_backend_config(err.to_string()))?;
117    try_ncd_bytes_backend_with_options(&bx?, &by?, &compiled, variant, NcdComputeOptions::default())
118}
119
120#[inline(always)]
121/// Compute NCD for two files with an explicit compiled compression backend.
122pub fn try_ncd_paths_compiled_backend(
123    x: &str,
124    y: &str,
125    backend: &CompiledCompressionBackend,
126    variant: NcdVariant,
127) -> InfotheoryResult<f64> {
128    try_ncd_paths_compiled_backend_with_options(
129        x,
130        y,
131        backend,
132        variant,
133        NcdComputeOptions::default(),
134    )
135}
136
137/// Compute NCD for two files with an explicit compiled compression backend and
138/// explicit operation-level parallelism controls.
139pub fn try_ncd_paths_compiled_backend_with_options(
140    x: &str,
141    y: &str,
142    backend: &CompiledCompressionBackend,
143    variant: NcdVariant,
144    options: NcdComputeOptions,
145) -> InfotheoryResult<f64> {
146    let (bx, by) = rayon::join(
147        || std::fs::read(x).map_err(InfotheoryError::from),
148        || std::fs::read(y).map_err(InfotheoryError::from),
149    );
150    try_ncd_bytes_backend_with_options(&bx?, &by?, backend, variant, options)
151}
152
153/// Compute an `n x n` pairwise NCD matrix (row-major) for file paths with an explicit compiled compression backend.
154pub fn try_ncd_matrix_paths_backend(
155    paths: &[&str],
156    backend: &CompiledCompressionBackend,
157    variant: NcdVariant,
158) -> InfotheoryResult<Vec<f64>> {
159    try_ncd_matrix_paths_backend_with_options(paths, backend, variant, NcdComputeOptions::default())
160}
161
162/// Compute an `n x n` pairwise NCD matrix (row-major) for file paths with an
163/// explicit compiled compression backend and operation-level parallelism
164/// controls.
165pub fn try_ncd_matrix_paths_backend_with_options(
166    paths: &[&str],
167    backend: &CompiledCompressionBackend,
168    variant: NcdVariant,
169    options: NcdComputeOptions,
170) -> InfotheoryResult<Vec<f64>> {
171    let datas = try_get_bytes_from_paths(paths)?;
172    try_ncd_matrix_bytes_backend_with_options(&datas, backend, variant, options)
173}
174
175/// Compute normalized entropy distance (NED) for two files using the default rate backend.
176pub fn try_ned_paths(x: &str, y: &str) -> InfotheoryResult<f64> {
177    let (bx, by) = try_read_path_pair(x, y)?;
178    try_ned_bytes(&bx, &by)
179}
180
181/// Compute normalized transform effort (NTE) for two files using the default rate backend.
182pub fn try_nte_paths(x: &str, y: &str) -> InfotheoryResult<f64> {
183    let (bx, by) = try_read_path_pair(x, y)?;
184    try_nte_bytes(&bx, &by)
185}
186
187/// Compute total variation distance (TVD) between the byte distributions of two files.
188pub fn try_tvd_paths(x: &str, y: &str) -> InfotheoryResult<f64> {
189    let (bx, by) = try_read_path_pair(x, y)?;
190    Ok(tvd_bytes(&bx, &by))
191}
192
193/// Compute normalized Hellinger distance (NHD) between the byte distributions of two files.
194pub fn try_nhd_paths(x: &str, y: &str) -> InfotheoryResult<f64> {
195    let (bx, by) = try_read_path_pair(x, y)?;
196    Ok(nhd_bytes(&bx, &by))
197}
198
199/// Compute mutual information estimate for two files using the default rate backend.
200pub fn try_mutual_information_paths(x: &str, y: &str) -> InfotheoryResult<f64> {
201    let (bx, by) = try_read_path_pair(x, y)?;
202    try_mutual_information_bytes(&bx, &by)
203}
204
205/// Compute conditional entropy estimate `H(X|Y)` for two files using the default rate backend.
206pub fn try_conditional_entropy_paths(x: &str, y: &str) -> InfotheoryResult<f64> {
207    let (bx, by) = try_read_path_pair(x, y)?;
208    try_conditional_entropy_bytes(&bx, &by)
209}
210
211/// Compute cross-entropy estimate `H_train(test)` for two files using the default rate backend.
212pub fn try_cross_entropy_paths(x: &str, y: &str) -> InfotheoryResult<f64> {
213    let (bx, by) = try_read_path_pair(x, y)?;
214    try_cross_entropy_bytes(&bx, &by)
215}
216
217/// Compute KL divergence between the byte histograms of two files.
218pub fn try_kl_divergence_paths(x: &str, y: &str) -> InfotheoryResult<f64> {
219    let (bx, by) = try_read_path_pair(x, y)?;
220    Ok(d_kl_bytes(&bx, &by))
221}
222
223/// Compute Jensen-Shannon divergence between the byte histograms of two files.
224pub fn try_js_divergence_paths(x: &str, y: &str) -> InfotheoryResult<f64> {
225    let (bx, by) = try_read_path_pair(x, y)?;
226    Ok(js_div_bytes(&bx, &by))
227}
infotheory/api/paths.rs

infotheory/api/
paths.rs
