Skip to content

Commit ee405a1

Browse files
marselestermarselester
committed
API redesign: three-layer find/decode/lookup to improve caching
1 parent bababd1 commit ee405a1

9 files changed

Lines changed: 602 additions & 469 deletions

File tree

‎README.md‎

Lines changed: 312 additions & 157 deletions
Large diffs are not rendered by default.

‎benchmarks/inspect.zig‎

Lines changed: 5 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -14,20 +14,12 @@ pub fn main() !void {
1414

1515
var db_path: []const u8 = default_db_path;
1616
var num_lookups = default_num_lookups;
17-
var fields: ?[]const []const u8 = null;
1817
if (args.len > 1) db_path = args[1];
1918
if (args.len > 2) num_lookups = try std.fmt.parseUnsigned(u64, args[2], 10);
20-
if (args.len > 3) {
21-
var items: [max_mmdb_fields][]const u8 = undefined;
22-
23-
var it = std.mem.splitScalar(u8, args[3], ',');
24-
var i: usize = 0;
25-
while (it.next()) |part| : (i += 1) {
26-
items[i] = part;
27-
}
28-
29-
fields = items[0..i];
30-
}
19+
var field_names = if (args.len > 3)
20+
try maxminddb.Fields(max_mmdb_fields).parse(args[3], ',')
21+
else
22+
maxminddb.Fields(max_mmdb_fields){};
3123

3224
std.debug.print("Benchmarking with:\n", .{});
3325
std.debug.print(" Database: {s}\n", .{db_path});
@@ -62,7 +54,7 @@ pub fn main() !void {
6254
maxminddb.any.Value,
6355
arena_allocator,
6456
ip,
65-
.{ .only = fields },
57+
.{ .only = field_names.only() },
6658
) catch |err| {
6759
std.debug.print("! Lookup error for IP {any}: {any}\n", .{ ip, err });
6860
lookup_errors += 1;

‎benchmarks/lookup.zig‎

Lines changed: 8 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -14,28 +14,22 @@ pub fn main() !void {
1414

1515
var db_path: []const u8 = default_db_path;
1616
var num_lookups = default_num_lookups;
17-
var fields: ?[]const []const u8 = null;
17+
var index_bits: u8 = 16;
1818
if (args.len > 1) db_path = args[1];
1919
if (args.len > 2) num_lookups = try std.fmt.parseUnsigned(u64, args[2], 10);
20-
if (args.len > 3) {
21-
var items: [max_mmdb_fields][]const u8 = undefined;
22-
23-
var it = std.mem.splitScalar(u8, args[3], ',');
24-
var i: usize = 0;
25-
while (it.next()) |part| : (i += 1) {
26-
items[i] = part;
27-
}
28-
29-
fields = items[0..i];
30-
}
20+
var field_names = if (args.len > 3)
21+
try maxminddb.Fields(max_mmdb_fields).parse(args[3], ',')
22+
else
23+
maxminddb.Fields(max_mmdb_fields){};
24+
if (args.len > 4) index_bits = try std.fmt.parseUnsigned(u8, args[4], 10);
3125

3226
std.debug.print("Benchmarking with:\n", .{});
3327
std.debug.print(" Database: {s}\n", .{db_path});
3428
std.debug.print(" Lookups: {d}\n", .{num_lookups});
3529
std.debug.print("Opening database...\n", .{});
3630

3731
var open_timer = try std.time.Timer.start();
38-
var db = try maxminddb.Reader.mmap(allocator, db_path, .{ .ipv4_index_first_n_bits = 16 });
32+
var db = try maxminddb.Reader.mmap(allocator, db_path, .{ .ipv4_index_first_n_bits = index_bits });
3933
defer db.close();
4034
const open_time_ms = @as(f64, @floatFromInt(open_timer.read())) /
4135
@as(f64, @floatFromInt(std.time.ns_per_ms));
@@ -62,7 +56,7 @@ pub fn main() !void {
6256
maxminddb.geolite2.City,
6357
arena_allocator,
6458
ip,
65-
.{ .only = fields },
59+
.{ .only = field_names.only() },
6660
) catch |err| {
6761
std.debug.print("! Lookup error for IP {any}: {any}\n", .{ ip, err });
6862
lookup_errors += 1;

‎benchmarks/lookup_cache.zig‎

Lines changed: 11 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -13,20 +13,12 @@ pub fn main() !void {
1313

1414
var db_path: []const u8 = default_db_path;
1515
var num_lookups = default_num_lookups;
16-
var fields: ?[]const []const u8 = null;
1716
if (args.len > 1) db_path = args[1];
1817
if (args.len > 2) num_lookups = try std.fmt.parseUnsigned(u64, args[2], 10);
19-
if (args.len > 3) {
20-
var items: [max_mmdb_fields][]const u8 = undefined;
21-
22-
var it = std.mem.splitScalar(u8, args[3], ',');
23-
var i: usize = 0;
24-
while (it.next()) |part| : (i += 1) {
25-
items[i] = part;
26-
}
27-
28-
fields = items[0..i];
29-
}
18+
var field_names = if (args.len > 3)
19+
try maxminddb.Fields(max_mmdb_fields).parse(args[3], ',')
20+
else
21+
maxminddb.Fields(max_mmdb_fields){};
3022

3123
std.debug.print("Benchmarking with:\n", .{});
3224
std.debug.print(" Database: {s}\n", .{db_path});
@@ -56,20 +48,20 @@ pub fn main() !void {
5648
std.crypto.random.bytes(&ip_bytes);
5749
const ip = std.net.Address.initIp4(ip_bytes, 0);
5850

59-
const result = db.lookupWithCache(
60-
maxminddb.geolite2.City,
61-
&cache,
62-
ip,
63-
.{ .only = fields },
64-
) catch |err| {
51+
const entry = db.find(ip, .{}) catch |err| {
6552
std.debug.print("! Lookup error for IP {any}: {any}\n", .{ ip, err });
6653
lookup_errors += 1;
6754
continue;
6855
};
69-
if (result == null) {
56+
if (entry == null) {
7057
not_found_count += 1;
7158
continue;
7259
}
60+
_ = cache.decode(&db, entry.?, .{ .only = field_names.only() }) catch |err| {
61+
std.debug.print("! Decode error for IP {any}: {any}\n", .{ ip, err });
62+
lookup_errors += 1;
63+
continue;
64+
};
7365
}
7466

7567
const elapsed_ns = timer.read();

‎benchmarks/scan.zig‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ const std = @import("std");
22
const maxminddb = @import("maxminddb");
33

44
const default_db_path: []const u8 = "GeoLite2-City.mmdb";
5+
const max_mmdb_fields = 32;
56

67
pub fn main() !void {
78
const allocator = std.heap.smp_allocator;
@@ -11,6 +12,10 @@ pub fn main() !void {
1112

1213
var db_path: []const u8 = default_db_path;
1314
if (args.len > 1) db_path = args[1];
15+
var field_names = if (args.len > 2)
16+
try maxminddb.Fields(max_mmdb_fields).parse(args[2], ',')
17+
else
18+
maxminddb.Fields(max_mmdb_fields){};
1419

1520
std.debug.print("Benchmarking with:\n", .{});
1621
std.debug.print(" Database: {s}\n", .{db_path});
@@ -34,7 +39,7 @@ pub fn main() !void {
3439
std.debug.print("Starting benchmark...\n", .{});
3540
var timer = try std.time.Timer.start();
3641

37-
var it = try db.scan(maxminddb.any.Value, allocator, network, .{});
42+
var it = try db.scan(maxminddb.any.Value, allocator, network, .{ .only = field_names.only() });
3843

3944
var n: usize = 0;
4045
while (try it.next()) |item| {

‎benchmarks/scan_cache.zig‎

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ const std = @import("std");
22
const maxminddb = @import("maxminddb");
33

44
const default_db_path: []const u8 = "GeoLite2-City.mmdb";
5+
const max_mmdb_fields = 32;
56

67
pub fn main() !void {
78
const allocator = std.heap.smp_allocator;
@@ -11,6 +12,10 @@ pub fn main() !void {
1112

1213
var db_path: []const u8 = default_db_path;
1314
if (args.len > 1) db_path = args[1];
15+
var field_names = if (args.len > 2)
16+
try maxminddb.Fields(max_mmdb_fields).parse(args[2], ',')
17+
else
18+
maxminddb.Fields(max_mmdb_fields){};
1419

1520
std.debug.print("Benchmarking with:\n", .{});
1621
std.debug.print(" Database: {s}\n", .{db_path});
@@ -37,15 +42,11 @@ pub fn main() !void {
3742
std.debug.print("Starting benchmark...\n", .{});
3843
var timer = try std.time.Timer.start();
3944

40-
var it = try db.scanWithCache(
41-
maxminddb.any.Value,
42-
&cache,
43-
network,
44-
.{},
45-
);
45+
var it = try db.entries(network, .{});
4646

4747
var n: usize = 0;
48-
while (try it.next()) |_| {
48+
while (try it.next()) |entry| {
49+
_ = try cache.decode(&db, entry, .{ .only = field_names.only() });
4950
n += 1;
5051
}
5152

‎src/filter.zig‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,12 @@ pub fn Fields(comptime capacity: usize) type {
2929
errdefer allocator.free(buf);
3030

3131
var f = try parse(buf, sep);
32+
// Don't keep the buffer if no fields were parsed, e.g., empty string.
33+
if (f.len == 0) {
34+
allocator.free(buf);
35+
return .{};
36+
}
37+
3238
f.buf = buf;
3339

3440
return f;
@@ -109,6 +115,22 @@ test "append trims whitespace-only to no-op" {
109115
try std.testing.expectEqual(null, f.only());
110116
}
111117

118+
test "parseAlloc empty string does not allocate" {
119+
var f = try Fields(4).parseAlloc(std.testing.allocator, "", ',');
120+
defer f.deinit(std.testing.allocator);
121+
122+
try std.testing.expectEqual(null, f.only());
123+
try std.testing.expectEqual(null, f.buf);
124+
}
125+
126+
test "parseAlloc whitespace-only does not allocate" {
127+
var f = try Fields(4).parseAlloc(std.testing.allocator, " , , ", ',');
128+
defer f.deinit(std.testing.allocator);
129+
130+
try std.testing.expectEqual(null, f.only());
131+
try std.testing.expectEqual(null, f.buf);
132+
}
133+
112134
test "parseAlloc frees buffer on TooManyFields" {
113135
const result = Fields(1).parseAlloc(std.testing.allocator, "a,b", ',');
114136
try std.testing.expectError(error.TooManyFields, result);

‎src/maxminddb.zig‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,9 +12,8 @@ pub const geoip2 = @import("geoip2.zig");
1212

1313
pub const Error = reader.ReadError || decoder.DecodeError;
1414
pub const Reader = reader.Reader;
15-
pub const Result = reader.Result;
16-
pub const Metadata = reader.Metadata;
1715
pub const Iterator = reader.Iterator;
16+
pub const EntryIterator = reader.EntryIterator;
1817
pub const Cache = reader.Cache;
1918
pub const Network = net.Network;
2019
pub const Map = collection.Map;

0 commit comments

Comments
 (0)