@@ -88,11 +88,7 @@ fn chunk_fixed(text: &str, chunk_size: usize, overlap: usize) -> Vec<ChunkResult
8888
8989 let chunk_text = chunk_words. join ( " " ) ;
9090 if !chunk_text. is_empty ( ) {
91- chunks. push ( ChunkResult {
92- text : chunk_text,
93- index : chunks. len ( ) ,
94- metadata : None ,
95- } ) ;
91+ chunks. push ( ChunkResult { text : chunk_text, index : chunks. len ( ) , metadata : None } ) ;
9692 }
9793
9894 // No unseen words remain, so a trailing overlap-only chunk would be redundant.
@@ -157,7 +153,7 @@ fn chunk_recursive(text: &str, chunk_size: usize, overlap: usize) -> Vec<ChunkRe
157153 }
158154
159155 let mut split_done = false ;
160- for ( _i , sep) in separators. iter ( ) . enumerate ( ) {
156+ for sep in separators. iter ( ) {
161157 if * sep == " " {
162158 continue ;
163159 }
@@ -306,11 +302,11 @@ fn chunk_markdown(text: &str, preserve_headers: bool, overlap: usize) -> Vec<Chu
306302 }
307303 } else if list_regex. is_match ( line) {
308304 if !last_header. is_empty ( ) && !current_section. is_empty ( ) {
309- current_section. push_str ( " \n " ) ;
305+ current_section. push ( '\n' ) ;
310306 }
311307 current_section. push_str ( trimmed) ;
312308 } else if !trimmed. starts_with ( "```" ) {
313- current_section. push_str ( " " ) ;
309+ current_section. push ( ' ' ) ;
314310 current_section. push_str ( trimmed) ;
315311 }
316312
@@ -430,7 +426,7 @@ fn apply_overlap(chunks: Vec<ChunkResult>, overlap: usize) -> Vec<ChunkResult> {
430426 combined. push_str ( word) ;
431427 }
432428 if !combined. is_empty ( ) {
433- combined. push_str ( " " ) ;
429+ combined. push ( ' ' ) ;
434430 }
435431 combined. push_str ( & chunk. text ) ;
436432
@@ -523,8 +519,7 @@ mod tests {
523519 // All words from the original text should appear in some chunk.
524520 let text = "alpha beta gamma delta epsilon zeta" ;
525521 let chunks = chunk_fixed ( text, 12 , 0 ) ;
526- let combined: String =
527- chunks. iter ( ) . map ( |c| c. text . as_str ( ) ) . collect :: < Vec < _ > > ( ) . join ( " " ) ;
522+ let combined: String = chunks. iter ( ) . map ( |c| c. text . as_str ( ) ) . collect :: < Vec < _ > > ( ) . join ( " " ) ;
528523 for word in text. split_whitespace ( ) {
529524 assert ! ( combined. contains( word) , "word '{}' missing from chunked output" , word) ;
530525 }
@@ -547,8 +542,7 @@ mod tests {
547542 // A word too long to fit in chunk_size must not be split.
548543 let text = "x superlongwordthatexceedschunksize y" ;
549544 let chunks = chunk_fixed ( text, 5 , 0 ) ;
550- let combined: String =
551- chunks. iter ( ) . map ( |c| c. text . as_str ( ) ) . collect :: < Vec < _ > > ( ) . join ( " " ) ;
545+ let combined: String = chunks. iter ( ) . map ( |c| c. text . as_str ( ) ) . collect :: < Vec < _ > > ( ) . join ( " " ) ;
552546 assert ! ( combined. contains( "superlongwordthatexceedschunksize" ) ) ;
553547 }
554548
@@ -719,7 +713,11 @@ mod tests {
719713 let chunks = chunk_markdown ( text, preserve, 0 ) ;
720714 assert ! ( !chunks. is_empty( ) ) ;
721715 let combined = chunks. iter ( ) . map ( |c| c. text . as_str ( ) ) . collect :: < Vec < _ > > ( ) . join ( " " ) ;
722- assert ! ( combined. contains( "Body content" ) , "body text must be present (preserve={})" , preserve) ;
716+ assert ! (
717+ combined. contains( "Body content" ) ,
718+ "body text must be present (preserve={})" ,
719+ preserve
720+ ) ;
723721 }
724722 }
725723
@@ -859,8 +857,7 @@ mod tests {
859857
860858 #[ test]
861859 fn test_apply_overlap_single_chunk_unchanged ( ) {
862- let input =
863- vec ! [ ChunkResult { text: "only one" . to_string( ) , index: 0 , metadata: None } ] ;
860+ let input = vec ! [ ChunkResult { text: "only one" . to_string( ) , index: 0 , metadata: None } ] ;
864861 let out = apply_overlap ( input, 5 ) ;
865862 assert_eq ! ( out. len( ) , 1 ) ;
866863 assert_eq ! ( out[ 0 ] . text, "only one" ) ;
@@ -877,7 +874,7 @@ mod tests {
877874 assert_eq ! ( out. len( ) , 2 ) ;
878875 assert_eq ! ( out[ 0 ] . text, "one" ) ;
879876 // "one" is the last (and only) word from chunk[0], so it is prepended to chunk[1]
880- assert ! ( out[ 1 ] . text. starts_with( "one " ) , "got '{}'" , out[ 1 ] . text) ;
877+ assert ! ( out[ 1 ] . text. starts_with( "one " ) , "got '{}'" , out[ 1 ] . text) ;
881878 assert ! ( out[ 1 ] . text. contains( "two three" ) ) ;
882879 }
883880
@@ -914,7 +911,7 @@ mod tests {
914911 let text = "# Title\n Content.\n \n ## Subtitle\n More content." ;
915912 let strategy = ChunkingStrategy :: Markdown { preserve_headers : true , overlap : 0 } ;
916913 let chunks = chunk ( text, strategy) ;
917- assert ! ( chunks. len ( ) >= 1 ) ;
914+ assert ! ( ! chunks. is_empty ( ) ) ;
918915 let combined = chunks. iter ( ) . map ( |c| c. text . as_str ( ) ) . collect :: < Vec < _ > > ( ) . join ( " " ) ;
919916 assert ! ( combined. contains( "Title" ) , "header text must be in output" ) ;
920917 assert ! ( combined. contains( "Content" ) ) ;
@@ -928,7 +925,15 @@ mod tests {
928925 let strategy = ChunkingStrategy :: Json { overlap : 0 } ;
929926 let chunks = chunk ( text, strategy) ;
930927 assert_eq ! ( chunks. len( ) , 2 ) ;
931- assert ! ( chunks. iter( ) . any( |c| c. metadata. as_ref( ) . map( |m| m. key. as_ref( ) . unwrap( ) == "data.id" && m. value. as_ref( ) . unwrap( ) == "123" ) . unwrap_or( false ) ) ) ;
932- assert ! ( chunks. iter( ) . any( |c| c. metadata. as_ref( ) . map( |m| m. key. as_ref( ) . unwrap( ) == "data.name" && m. value. as_ref( ) . unwrap( ) == "Test" ) . unwrap_or( false ) ) ) ;
928+ assert ! ( chunks. iter( ) . any( |c| c
929+ . metadata
930+ . as_ref( )
931+ . map( |m| m. key. as_ref( ) . unwrap( ) == "data.id" && m. value. as_ref( ) . unwrap( ) == "123" )
932+ . unwrap_or( false ) ) ) ;
933+ assert ! ( chunks. iter( ) . any( |c| c
934+ . metadata
935+ . as_ref( )
936+ . map( |m| m. key. as_ref( ) . unwrap( ) == "data.name" && m. value. as_ref( ) . unwrap( ) == "Test" )
937+ . unwrap_or( false ) ) ) ;
933938 }
934939}
0 commit comments