diff correctness

.github/workflows/ci.yml

@@ -205,7 +205,7 @@ jobs:
         name: crates without feature toggles
       - run: set +x; for feature in progress fs-walkdir-parallel parallel io-pipe crc32 zlib zlib-rust-backend fast-sha1 rustsha1 cache-efficiency-debug; do (cd gix-features && cargo build --features $feature --target ${{ matrix.target }}); done
         name: features of gix-features
-      - run: set +x; for name in gix-diff gix-pack; do (cd $name && cargo build --features wasm --target ${{ matrix.target }}); done
+      - run: set +x; for name in gix-pack; do (cd $name && cargo build --features wasm --target ${{ matrix.target }}); done
         name: crates with 'wasm' feature
       - run: cd gix-pack && cargo build --all-features --target ${{ matrix.target }}
         name: gix-pack with all features (including wasm)

crate-status.md

@@ -293,16 +293,29 @@ The top-level crate that acts as hub to all functionality provided by the `gix-*
 Check out the [performance discussion][gix-diff-performance] as well.
 
 * **tree**
-  * [x] changes needed to obtain _other tree_
+    * [x] changes needed to obtain _other tree_
 * **patches**    
-  * There are various ways to generate a patch from two blobs.
-  * [ ] any
+    * There are various ways to generate a patch from two blobs.
+    * [ ] text
+    * [ ] binary
 * **lines**
-  * [x] Simple line-by-line diffs powered by the `imara-diff` crate.
-* diffing, merging, working with hunks of data
-* find differences between various states, i.e. index, working tree, commit-tree
+    * [x] Simple line-by-line diffs powered by the `imara-diff` crate.
+* **generic rename tracker to find renames and copies**
+    * [x] find by exact match
+    * [x] find by similarity check
+    * [ ] heuristics to find best candidate
+    * [ ] find by basename to help detecting simple moves
+* **blob**
+    * [x] a choice of to-worktree, to-git and to-worktree-if-needed conversions
+    * [x] `textconv` filters
+    * [x] special handling of files beyond the big-file threshold.
+    * [x] detection of binary files by looking at header (first 8k bytes)
+    * [x] caching of diff-able data
+    * [x] prepare invocation of external diff program
+        - [ ] pass meta-info
+* [ ] working with hunks of data
 * [x] API documentation
-  * [ ] Examples
+    * [ ] Examples
 
 [gix-diff-performance]: https://github.com/Byron/gitoxide/discussions/74
 

gitoxide-core/src/hours/core.rs

@@ -8,7 +8,6 @@ use std::{
 
 use gix::bstr::BStr;
 use itertools::Itertools;
-use smallvec::SmallVec;
 
 use crate::hours::{
     util::{add_lines, remove_lines},
@@ -92,25 +91,16 @@ pub fn spawn_tree_delta_threads<'scope>(
                 move || -> Result<_, anyhow::Error> {
                     let mut out = Vec::new();
                     let (commits, changes, lines_count) = stats_counters;
-                    let mut attributes = line_stats
+                    let mut cache = line_stats
                         .then(|| -> anyhow::Result<_> {
-                            repo.index_or_load_from_head().map_err(Into::into).and_then(|index| {
-                                repo.attributes(
-                                    &index,
-                                    gix::worktree::stack::state::attributes::Source::IdMapping,
-                                    gix::worktree::stack::state::ignore::Source::IdMapping,
-                                    None,
-                                )
-                                .map_err(Into::into)
-                                .map(|attrs| {
-                                    let matches = attrs.selected_attribute_matches(["binary", "text"]);
-                                    (attrs, matches)
-                                })
-                            })
+                            Ok(repo.diff_resource_cache(gix::diff::blob::pipeline::Mode::ToGit, Default::default())?)
                         })
                         .transpose()?;
                     for chunk in rx {
                         for (commit_idx, parent_commit, commit) in chunk {
+                            if let Some(cache) = cache.as_mut() {
+                                cache.clear_resource_cache();
+                            }
                             commits.fetch_add(1, Ordering::Relaxed);
                             if gix::interrupt::is_triggered() {
                                 return Ok(out);
@@ -155,47 +145,34 @@ pub fn spawn_tree_delta_threads<'scope>(
                                             previous_entry_mode,
                                             id,
                                             previous_id,
-                                        } => {
-                                            match (previous_entry_mode.is_blob(), entry_mode.is_blob()) {
-                                                (false, false) => {}
-                                                (false, true) => {
-                                                    files.added += 1;
-                                                    add_lines(line_stats, &lines_count, &mut lines, id);
-                                                }
-                                                (true, false) => {
-                                                    files.removed += 1;
-                                                    remove_lines(line_stats, &lines_count, &mut lines, previous_id);
-                                                }
-                                                (true, true) => {
-                                                    files.modified += 1;
-                                                    if let Some((attrs, matches)) = attributes.as_mut() {
-                                                        let entry = attrs.at_entry(change.location, Some(false))?;
-                                                        let is_text_file = if entry.matching_attributes(matches) {
-                                                            let attrs: SmallVec<[_; 2]> =
-                                                                matches.iter_selected().collect();
-                                                            let binary = &attrs[0];
-                                                            let text = &attrs[1];
-                                                            !binary.assignment.state.is_set()
-                                                                && !text.assignment.state.is_unset()
-                                                        } else {
-                                                            // In the absence of binary or text markers, we assume it's text.
-                                                            true
-                                                        };
-
-                                                        if let Some(Ok(diff)) =
-                                                            is_text_file.then(|| change.event.diff()).flatten()
-                                                        {
-                                                            let mut nl = 0;
-                                                            let counts = diff.line_counts();
-                                                            nl += counts.insertions as usize + counts.removals as usize;
-                                                            lines.added += counts.insertions as usize;
-                                                            lines.removed += counts.removals as usize;
-                                                            lines_count.fetch_add(nl, Ordering::Relaxed);
-                                                        }
+                                        } => match (previous_entry_mode.is_blob(), entry_mode.is_blob()) {
+                                            (false, false) => {}
+                                            (false, true) => {
+                                                files.added += 1;
+                                                add_lines(line_stats, &lines_count, &mut lines, id);
+                                            }
+                                            (true, false) => {
+                                                files.removed += 1;
+                                                remove_lines(line_stats, &lines_count, &mut lines, previous_id);
+                                            }
+                                            (true, true) => {
+                                                files.modified += 1;
+                                                if let Some(cache) = cache.as_mut() {
+                                                    let mut diff = change.diff(cache).map_err(|err| {
+                                                        std::io::Error::new(std::io::ErrorKind::Other, err)
+                                                    })?;
+                                                    let mut nl = 0;
+                                                    if let Some(counts) = diff.line_counts().map_err(|err| {
+                                                        std::io::Error::new(std::io::ErrorKind::Other, err)
+                                                    })? {
+                                                        nl += counts.insertions as usize + counts.removals as usize;
+                                                        lines.added += counts.insertions as usize;
+                                                        lines.removed += counts.removals as usize;
+                                                        lines_count.fetch_add(nl, Ordering::Relaxed);
                                                     }
                                                 }
                                             }
-                                        }
+                                        },
                                     }
                                     Ok::<_, std::io::Error>(Default::default())
                                 })?;