API redesign: three-layer find/decode/lookup to improve caching

Author: marselester

README.md

@@ -18,15 +18,8 @@ pub fn main() !void {
     if (args.len > 1) db_path = args[1];
     if (args.len > 2) num_lookups = try std.fmt.parseUnsigned(u64, args[2], 10);
     if (args.len > 3) {
-        var items: [max_mmdb_fields][]const u8 = undefined;
-
-        var it = std.mem.splitScalar(u8, args[3], ',');
-        var i: usize = 0;
-        while (it.next()) |part| : (i += 1) {
-            items[i] = part;
-        }
-
-        fields = items[0..i];
+        const f = try maxminddb.Fields(max_mmdb_fields).parse(args[3], ',');
+        fields = f.only();
     }
 
     std.debug.print("Benchmarking with:\n", .{});

benchmarks/inspect.zig

@@ -15,27 +15,22 @@ pub fn main() !void {
     var db_path: []const u8 = default_db_path;
     var num_lookups = default_num_lookups;
     var fields: ?[]const []const u8 = null;
+    var index_bits: u8 = 16;
     if (args.len > 1) db_path = args[1];
     if (args.len > 2) num_lookups = try std.fmt.parseUnsigned(u64, args[2], 10);
     if (args.len > 3) {
-        var items: [max_mmdb_fields][]const u8 = undefined;
-
-        var it = std.mem.splitScalar(u8, args[3], ',');
-        var i: usize = 0;
-        while (it.next()) |part| : (i += 1) {
-            items[i] = part;
-        }
-
-        fields = items[0..i];
+        const f = try maxminddb.Fields(max_mmdb_fields).parse(args[3], ',');
+        fields = f.only();
     }
+    if (args.len > 4) index_bits = try std.fmt.parseUnsigned(u8, args[4], 10);
 
     std.debug.print("Benchmarking with:\n", .{});
     std.debug.print("  Database: {s}\n", .{db_path});
     std.debug.print("  Lookups:  {d}\n", .{num_lookups});
     std.debug.print("Opening database...\n", .{});
 
     var open_timer = try std.time.Timer.start();
-    var db = try maxminddb.Reader.mmap(allocator, db_path, .{ .ipv4_index_first_n_bits = 16 });
+    var db = try maxminddb.Reader.mmap(allocator, db_path, .{ .ipv4_index_first_n_bits = index_bits });
     defer db.close();
     const open_time_ms = @as(f64, @floatFromInt(open_timer.read())) /
         @as(f64, @floatFromInt(std.time.ns_per_ms));

benchmarks/lookup.zig

@@ -17,15 +17,8 @@ pub fn main() !void {
     if (args.len > 1) db_path = args[1];
     if (args.len > 2) num_lookups = try std.fmt.parseUnsigned(u64, args[2], 10);
     if (args.len > 3) {
-        var items: [max_mmdb_fields][]const u8 = undefined;
-
-        var it = std.mem.splitScalar(u8, args[3], ',');
-        var i: usize = 0;
-        while (it.next()) |part| : (i += 1) {
-            items[i] = part;
-        }
-
-        fields = items[0..i];
+        const f = try maxminddb.Fields(max_mmdb_fields).parse(args[3], ',');
+        fields = f.only();
     }
 
     std.debug.print("Benchmarking with:\n", .{});
@@ -56,20 +49,20 @@ pub fn main() !void {
         std.crypto.random.bytes(&ip_bytes);
         const ip = std.net.Address.initIp4(ip_bytes, 0);
 
-        const result = db.lookupWithCache(
-            maxminddb.geolite2.City,
-            &cache,
-            ip,
-            .{ .only = fields },
-        ) catch |err| {
+        const entry = db.find(ip, .{}) catch |err| {
             std.debug.print("! Lookup error for IP {any}: {any}\n", .{ ip, err });
             lookup_errors += 1;
             continue;
         };
-        if (result == null) {
+        if (entry == null) {
             not_found_count += 1;
             continue;
         }
+        _ = cache.decode(&db, entry.?, .{ .only = fields }) catch |err| {
+            std.debug.print("! Decode error for IP {any}: {any}\n", .{ ip, err });
+            lookup_errors += 1;
+            continue;
+        };
     }
 
     const elapsed_ns = timer.read();

benchmarks/lookup_cache.zig

@@ -2,6 +2,7 @@ const std = @import("std");
 const maxminddb = @import("maxminddb");
 
 const default_db_path: []const u8 = "GeoLite2-City.mmdb";
+const max_mmdb_fields = 32;
 
 pub fn main() !void {
     const allocator = std.heap.smp_allocator;
@@ -10,7 +11,12 @@ pub fn main() !void {
     defer std.process.argsFree(allocator, args);
 
     var db_path: []const u8 = default_db_path;
+    var fields: ?[]const []const u8 = null;
     if (args.len > 1) db_path = args[1];
+    if (args.len > 2) {
+        const f = try maxminddb.Fields(max_mmdb_fields).parse(args[2], ',');
+        fields = f.only();
+    }
 
     std.debug.print("Benchmarking with:\n", .{});
     std.debug.print("  Database: {s}\n", .{db_path});
@@ -34,7 +40,7 @@ pub fn main() !void {
     std.debug.print("Starting benchmark...\n", .{});
     var timer = try std.time.Timer.start();
 
-    var it = try db.scan(maxminddb.any.Value, allocator, network, .{});
+    var it = try db.scan(maxminddb.any.Value, allocator, network, .{ .only = fields });
 
     var n: usize = 0;
     while (try it.next()) |item| {

benchmarks/scan.zig

@@ -2,6 +2,7 @@ const std = @import("std");
 const maxminddb = @import("maxminddb");
 
 const default_db_path: []const u8 = "GeoLite2-City.mmdb";
+const max_mmdb_fields = 32;
 
 pub fn main() !void {
     const allocator = std.heap.smp_allocator;
@@ -10,7 +11,12 @@ pub fn main() !void {
     defer std.process.argsFree(allocator, args);
 
     var db_path: []const u8 = default_db_path;
+    var fields: ?[]const []const u8 = null;
     if (args.len > 1) db_path = args[1];
+    if (args.len > 2) {
+        const f = try maxminddb.Fields(max_mmdb_fields).parse(args[2], ',');
+        fields = f.only();
+    }
 
     std.debug.print("Benchmarking with:\n", .{});
     std.debug.print("  Database: {s}\n", .{db_path});
@@ -37,15 +43,11 @@ pub fn main() !void {
     std.debug.print("Starting benchmark...\n", .{});
     var timer = try std.time.Timer.start();
 
-    var it = try db.scanWithCache(
-        maxminddb.any.Value,
-        &cache,
-        network,
-        .{},
-    );
+    var it = try db.entries(network, .{});
 
     var n: usize = 0;
-    while (try it.next()) |_| {
+    while (try it.next()) |entry| {
+        _ = try cache.decode(&db, entry, .{ .only = fields });
         n += 1;
     }
 

benchmarks/scan_cache.zig

@@ -29,6 +29,12 @@ pub fn Fields(comptime capacity: usize) type {
             errdefer allocator.free(buf);
 
             var f = try parse(buf, sep);
+            // Don't keep the buffer if no fields were parsed, e.g., empty string.
+            if (f.len == 0) {
+                allocator.free(buf);
+                return .{};
+            }
+
             f.buf = buf;
 
             return f;
@@ -109,6 +115,22 @@ test "append trims whitespace-only to no-op" {
     try std.testing.expectEqual(null, f.only());
 }
 
+test "parseAlloc empty string does not allocate" {
+    var f = try Fields(4).parseAlloc(std.testing.allocator, "", ',');
+    defer f.deinit(std.testing.allocator);
+
+    try std.testing.expectEqual(null, f.only());
+    try std.testing.expectEqual(null, f.buf);
+}
+
+test "parseAlloc whitespace-only does not allocate" {
+    var f = try Fields(4).parseAlloc(std.testing.allocator, "  ,  ,  ", ',');
+    defer f.deinit(std.testing.allocator);
+
+    try std.testing.expectEqual(null, f.only());
+    try std.testing.expectEqual(null, f.buf);
+}
+
 test "parseAlloc frees buffer on TooManyFields" {
     const result = Fields(1).parseAlloc(std.testing.allocator, "a,b", ',');
     try std.testing.expectError(error.TooManyFields, result);

src/filter.zig

@@ -12,9 +12,9 @@ pub const geoip2 = @import("geoip2.zig");
 
 pub const Error = reader.ReadError || decoder.DecodeError;
 pub const Reader = reader.Reader;
-pub const Result = reader.Result;
 pub const Metadata = reader.Metadata;
-pub const Iterator = reader.Iterator;
+pub const ResultIterator = reader.ResultIterator;
+pub const EntryIterator = reader.EntryIterator;
 pub const Cache = reader.Cache;
 pub const Network = net.Network;
 pub const Map = collection.Map;
@@ -904,7 +904,7 @@ test "decodeMetadata as any.Value" {
     var arena = std.heap.ArenaAllocator.init(allocator);
     defer arena.deinit();
 
-    const meta = try Reader.decodeMetadata(any.Value, arena.allocator(), db.src);
+    const meta = try Metadata.decodeAs(any.Value, arena.allocator(), db.src);
     try expectEqualStrings("GeoLite2-City", meta.get("database_type").?.string);
     try expectEqual(@as(u16, 6), meta.get("ip_version").?.uint16);
     try expectEqual(@as(u16, 2), meta.get("binary_format_major_version").?.uint16);