Skip to content

Commit 921fcd1

Browse files
tisonkuntisonkun
authored
refactor: expose codec and add centralized Family (#91)
Signed-off-by: tison <wander4096@gmail.com>
1 parent 9a83070 commit 921fcd1

20 files changed

Lines changed: 314 additions & 161 deletions

File tree

‎datasketches/src/bloom/sketch.rs‎

Lines changed: 14 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -20,13 +20,11 @@ use std::hash::Hasher;
2020

2121
use crate::codec::SketchBytes;
2222
use crate::codec::SketchSlice;
23+
use crate::codec::family::Family;
2324
use crate::error::Error;
2425
use crate::hash::XxHash64;
2526

2627
// Serialization constants
27-
const PREAMBLE_LONGS_EMPTY: u8 = 3;
28-
const PREAMBLE_LONGS_STANDARD: u8 = 4;
29-
const BLOOM_FAMILY_ID: u8 = 21; // Bloom filter family ID
3028
const SERIAL_VERSION: u8 = 1;
3129
const EMPTY_FLAG_MASK: u8 = 1 << 2;
3230

@@ -353,9 +351,9 @@ impl BloomFilter {
353351
pub fn serialize(&self) -> Vec<u8> {
354352
let is_empty = self.is_empty();
355353
let preamble_longs = if is_empty {
356-
PREAMBLE_LONGS_EMPTY
354+
Family::BLOOMFILTER.min_pre_longs
357355
} else {
358-
PREAMBLE_LONGS_STANDARD
356+
Family::BLOOMFILTER.max_pre_longs
359357
};
360358

361359
let capacity = 8 * preamble_longs as usize
@@ -369,7 +367,7 @@ impl BloomFilter {
369367
// Preamble
370368
bytes.write_u8(preamble_longs); // Byte 0
371369
bytes.write_u8(SERIAL_VERSION); // Byte 1
372-
bytes.write_u8(BLOOM_FAMILY_ID); // Byte 2
370+
bytes.write_u8(Family::BLOOMFILTER.id); // Byte 2
373371
bytes.write_u8(if is_empty { EMPTY_FLAG_MASK } else { 0 }); // Byte 3: flags
374372
bytes.write_u16_le(self.num_hashes); // Bytes 4-5
375373
bytes.write_u16_le(0); // Bytes 6-7: unused
@@ -432,24 +430,22 @@ impl BloomFilter {
432430
.map_err(|_| Error::insufficient_data("flags"))?;
433431

434432
// Validate
435-
if family_id != BLOOM_FAMILY_ID {
436-
return Err(Error::invalid_family(
437-
BLOOM_FAMILY_ID,
438-
family_id,
439-
"BloomFilter",
440-
));
441-
}
433+
Family::BLOOMFILTER.validate_id(family_id)?;
442434
if serial_version != SERIAL_VERSION {
443435
return Err(Error::unsupported_serial_version(
444436
SERIAL_VERSION,
445437
serial_version,
446438
));
447439
}
448-
if preamble_longs != PREAMBLE_LONGS_EMPTY && preamble_longs != PREAMBLE_LONGS_STANDARD {
449-
return Err(Error::invalid_preamble_longs(
450-
PREAMBLE_LONGS_STANDARD,
451-
preamble_longs,
452-
));
440+
if !(Family::BLOOMFILTER.min_pre_longs..=Family::BLOOMFILTER.max_pre_longs)
441+
.contains(&preamble_longs)
442+
{
443+
return Err(Error::deserial(format!(
444+
"invalid preamble longs: expected [{}, {}], got {}",
445+
Family::BLOOMFILTER.min_pre_longs,
446+
Family::BLOOMFILTER.max_pre_longs,
447+
preamble_longs
448+
)));
453449
}
454450

455451
let is_empty = (flags & EMPTY_FLAG_MASK) != 0;

datasketches/src/codec.rs renamed to datasketches/src/codec/decode.rs

Lines changed: 23 additions & 95 deletions
Original file line numberDiff line numberDiff line change
@@ -15,226 +15,154 @@
1515
// specific language governing permissions and limitations
1616
// under the License.
1717

18-
#![allow(dead_code)]
19-
2018
use std::io;
2119
use std::io::Cursor;
2220
use std::io::Read;
2321

24-
pub(crate) struct SketchBytes {
25-
bytes: Vec<u8>,
26-
}
27-
28-
impl SketchBytes {
29-
pub fn with_capacity(capacity: usize) -> Self {
30-
Self {
31-
bytes: Vec::with_capacity(capacity),
32-
}
33-
}
34-
35-
pub fn into_bytes(self) -> Vec<u8> {
36-
self.bytes
37-
}
38-
39-
pub fn write(&mut self, buf: &[u8]) {
40-
self.bytes.extend_from_slice(buf);
41-
}
42-
43-
pub fn write_u8(&mut self, n: u8) {
44-
self.bytes.push(n);
45-
}
46-
47-
pub fn write_i8(&mut self, n: i8) {
48-
self.bytes.push(n as u8);
49-
}
50-
51-
pub fn write_u16_le(&mut self, n: u16) {
52-
self.write(&n.to_le_bytes());
53-
}
54-
55-
pub fn write_u16_be(&mut self, n: u16) {
56-
self.write(&n.to_be_bytes());
57-
}
58-
59-
pub fn write_i16_le(&mut self, n: i16) {
60-
self.write(&n.to_le_bytes());
61-
}
62-
63-
pub fn write_i16_be(&mut self, n: i16) {
64-
self.write(&n.to_be_bytes());
65-
}
66-
67-
pub fn write_u32_le(&mut self, n: u32) {
68-
self.write(&n.to_le_bytes());
69-
}
70-
71-
pub fn write_u32_be(&mut self, n: u32) {
72-
self.write(&n.to_be_bytes());
73-
}
74-
75-
pub fn write_i32_le(&mut self, n: i32) {
76-
self.write(&n.to_le_bytes());
77-
}
78-
79-
pub fn write_i32_be(&mut self, n: i32) {
80-
self.write(&n.to_be_bytes());
81-
}
82-
83-
pub fn write_u64_le(&mut self, n: u64) {
84-
self.write(&n.to_le_bytes());
85-
}
86-
87-
pub fn write_u64_be(&mut self, n: u64) {
88-
self.write(&n.to_be_bytes());
89-
}
90-
91-
pub fn write_i64_le(&mut self, n: i64) {
92-
self.write(&n.to_le_bytes());
93-
}
94-
95-
pub fn write_i64_be(&mut self, n: i64) {
96-
self.write(&n.to_be_bytes());
97-
}
98-
99-
pub fn write_f32_le(&mut self, n: f32) {
100-
self.write(&n.to_le_bytes());
101-
}
102-
103-
pub fn write_f32_be(&mut self, n: f32) {
104-
self.write(&n.to_be_bytes());
105-
}
106-
107-
pub fn write_f64_le(&mut self, n: f64) {
108-
self.write(&n.to_le_bytes());
109-
}
110-
111-
pub fn write_f64_be(&mut self, n: f64) {
112-
self.write(&n.to_be_bytes());
113-
}
114-
}
115-
116-
pub(crate) struct SketchSlice<'a> {
22+
/// A wrapper around a byte slice that provides methods for reading various types of data from it.
23+
pub struct SketchSlice<'a> {
11724
slice: Cursor<&'a [u8]>,
11825
}
11926

12027
impl SketchSlice<'_> {
28+
/// Creates a new `SketchSlice` from the given byte slice.
12129
pub fn new(slice: &[u8]) -> SketchSlice<'_> {
12230
SketchSlice {
12331
slice: Cursor::new(slice),
12432
}
12533
}
12634

35+
/// Advances the position of the slice by `n` bytes.
12736
pub fn advance(&mut self, n: u64) {
12837
let pos = self.slice.position();
12938
self.slice.set_position(pos + n);
13039
}
13140

41+
/// Reads exactly `buf.len()` bytes from the slice into `buf`.
13242
pub fn read_exact(&mut self, buf: &mut [u8]) -> io::Result<()> {
13343
self.slice.read_exact(buf)
13444
}
13545

46+
/// Reads a single byte from the slice and returns it as a `u8`.
13647
pub fn read_u8(&mut self) -> io::Result<u8> {
13748
let mut buf = [0u8; 1];
13849
self.read_exact(&mut buf)?;
13950
Ok(buf[0])
14051
}
14152

53+
/// Reads a single byte from the slice and returns it as an `i8`.
14254
pub fn read_i8(&mut self) -> io::Result<i8> {
14355
let mut buf = [0u8; 1];
14456
self.read_exact(&mut buf)?;
14557
Ok(buf[0] as i8)
14658
}
14759

60+
/// Reads a 16-bit unsigned integer from the slice in little-endian byte order.
14861
pub fn read_u16_le(&mut self) -> io::Result<u16> {
14962
let mut buf = [0u8; 2];
15063
self.read_exact(&mut buf)?;
15164
Ok(u16::from_le_bytes(buf))
15265
}
15366

67+
/// Reads a 16-bit unsigned integer from the slice in big-endian byte order.
15468
pub fn read_u16_be(&mut self) -> io::Result<u16> {
15569
let mut buf = [0u8; 2];
15670
self.read_exact(&mut buf)?;
15771
Ok(u16::from_be_bytes(buf))
15872
}
15973

74+
/// Reads a 16-bit signed integer from the slice in little-endian byte order.
16075
pub fn read_i16_le(&mut self) -> io::Result<i16> {
16176
let mut buf = [0u8; 2];
16277
self.read_exact(&mut buf)?;
16378
Ok(i16::from_le_bytes(buf))
16479
}
16580

81+
/// Reads a 16-bit signed integer from the slice in big-endian byte order.
16682
pub fn read_i16_be(&mut self) -> io::Result<i16> {
16783
let mut buf = [0u8; 2];
16884
self.read_exact(&mut buf)?;
16985
Ok(i16::from_be_bytes(buf))
17086
}
17187

88+
/// Reads a 32-bit unsigned integer from the slice in little-endian byte order.
17289
pub fn read_u32_le(&mut self) -> io::Result<u32> {
17390
let mut buf = [0u8; 4];
17491
self.read_exact(&mut buf)?;
17592
Ok(u32::from_le_bytes(buf))
17693
}
17794

95+
/// Reads a 32-bit unsigned integer from the slice in big-endian byte order.
17896
pub fn read_u32_be(&mut self) -> io::Result<u32> {
17997
let mut buf = [0u8; 4];
18098
self.read_exact(&mut buf)?;
18199
Ok(u32::from_be_bytes(buf))
182100
}
183101

102+
/// Reads a 32-bit signed integer from the slice in little-endian byte order.
184103
pub fn read_i32_le(&mut self) -> io::Result<i32> {
185104
let mut buf = [0u8; 4];
186105
self.read_exact(&mut buf)?;
187106
Ok(i32::from_le_bytes(buf))
188107
}
189108

109+
/// Reads a 32-bit signed integer from the slice in big-endian byte order.
190110
pub fn read_i32_be(&mut self) -> io::Result<i32> {
191111
let mut buf = [0u8; 4];
192112
self.read_exact(&mut buf)?;
193113
Ok(i32::from_be_bytes(buf))
194114
}
195115

116+
/// Reads a 16-bit unsigned integer from the slice in little-endian byte order.
196117
pub fn read_u64_le(&mut self) -> io::Result<u64> {
197118
let mut buf = [0u8; 8];
198119
self.read_exact(&mut buf)?;
199120
Ok(u64::from_le_bytes(buf))
200121
}
201122

123+
/// Reads a 16-bit unsigned integer from the slice in big-endian byte order.
202124
pub fn read_u64_be(&mut self) -> io::Result<u64> {
203125
let mut buf = [0u8; 8];
204126
self.read_exact(&mut buf)?;
205127
Ok(u64::from_be_bytes(buf))
206128
}
207129

130+
/// Reads a 16-bit signed integer from the slice in little-endian byte order.
208131
pub fn read_i64_le(&mut self) -> io::Result<i64> {
209132
let mut buf = [0u8; 8];
210133
self.read_exact(&mut buf)?;
211134
Ok(i64::from_le_bytes(buf))
212135
}
213136

137+
/// Reads a 16-bit signed integer from the slice in big-endian byte order.
214138
pub fn read_i64_be(&mut self) -> io::Result<i64> {
215139
let mut buf = [0u8; 8];
216140
self.read_exact(&mut buf)?;
217141
Ok(i64::from_be_bytes(buf))
218142
}
219143

144+
/// Reads a 32-bit floating-point number from the slice in little-endian byte order.
220145
pub fn read_f32_le(&mut self) -> io::Result<f32> {
221146
let mut buf = [0u8; 4];
222147
self.read_exact(&mut buf)?;
223148
Ok(f32::from_le_bytes(buf))
224149
}
225150

151+
/// Reads a 32-bit floating-point number from the slice in big-endian byte order.
226152
pub fn read_f32_be(&mut self) -> io::Result<f32> {
227153
let mut buf = [0u8; 4];
228154
self.read_exact(&mut buf)?;
229155
Ok(f32::from_be_bytes(buf))
230156
}
231157

158+
/// Reads a 64-bit floating-point number from the slice in little-endian byte order.
232159
pub fn read_f64_le(&mut self) -> io::Result<f64> {
233160
let mut buf = [0u8; 8];
234161
self.read_exact(&mut buf)?;
235162
Ok(f64::from_le_bytes(buf))
236163
}
237164

165+
/// Reads a 64-bit floating-point number from the slice in big-endian byte order.
238166
pub fn read_f64_be(&mut self) -> io::Result<f64> {
239167
let mut buf = [0u8; 8];
240168
self.read_exact(&mut buf)?;

0 commit comments

Comments
 (0)