Skip to content

Commit 477b396

Browse files
tobiemhtobiemh
authored
Unify SurrealDB benchmark target and endpoints, and fix sync mode (#226)
1 parent 1d683c9 commit 477b396

7 files changed

Lines changed: 197 additions & 194 deletions

File tree

‎.github/workflows/ci.yml‎

Lines changed: 32 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -207,6 +207,19 @@ jobs:
207207
{"name":"batch_update_1000","operation":"UPDATE","batch_size":1000,"samples":100},
208208
{"name":"batch_delete_1000","operation":"DELETE","batch_size":1000,"samples":100}
209209
]
210+
CRUD_BENCH_SCANS: >-
211+
[
212+
{"name":"count_all","samples":1000,"projection":"COUNT"},
213+
{"name":"limit_id","samples":100,"projection":"ID","limit":100,"expect":100},
214+
{"name":"limit_all","samples":100,"projection":"FULL","limit":100,"expect":100},
215+
{"name":"limit_start_id","samples":100,"projection":"ID","start":5000,"limit":100,"expect":100},
216+
{"name":"limit_start_all","samples":100,"projection":"FULL","start":5000,"limit":100,"expect":100},
217+
{"name":"where_field_integer_eq","samples":100,"projection":"FULL","condition":{"sql":"number = 21","mysql":"number = 21","neo4j":"r.number = 21","mongodb":{"number":{"$eq":21}},"arangodb":"r.number == 21","surrealdb":"number = 21"},"index":{"fields":["number"]}},
218+
{"name":"where_field_integer_gte_lte","samples":100,"projection":"FULL","condition":{"sql":"number >= 18 AND number <= 21","mysql":"number >= 18 AND number <= 21","neo4j":"r.number >= 18 AND r.number <= 21","mongodb":{"number":{"$gte":18,"$lte":21}},"arangodb":"r.number >= 18 AND r.number <= 21","surrealdb":"number >= 18 AND number <= 21"},"index":{"fields":["number"]}},
219+
{"name":"where_field_fulltext_single","samples":100,"projection":"FULL","limit":1000,"condition":{"sql":"to_tsvector('english', words) @@ to_tsquery('english', 'hello')","mysql":"MATCH(words) AGAINST('hello' IN NATURAL LANGUAGE MODE)","neo4j":"hello","mongodb":{"$text":{"$search":"hello"}},"surrealdb":"words @@ 'hello'"},"index":{"fields":["words"],"index_type":"fulltext"}},
220+
{"name":"where_field_fulltext_multi_and","samples":100,"projection":"FULL","limit":1000,"condition":{"sql":"to_tsvector('english', words) @@ to_tsquery('english', 'hello & world')","mysql":"MATCH(words) AGAINST('+hello +world' IN NATURAL LANGUAGE MODE)","neo4j":"hello AND world","mongodb":{"$text":{"$search":"hello world"}},"surrealdb":"words @@ 'hello' AND words @@ 'world'"},"index":{"fields":["words"],"index_type":"fulltext"}},
221+
{"name":"where_field_fulltext_multi_or","samples":100,"projection":"FULL","limit":1000,"condition":{"sql":"to_tsvector('english', words) @@ to_tsquery('english', 'foo | bar')","mysql":"MATCH(words) AGAINST('foo bar' IN NATURAL LANGUAGE MODE)","neo4j":"foo OR bar","mongodb":{"$text":{"$search":"foo bar"}},"surrealdb":"words @@ 'foo' OR words @@ 'bar'"},"index":{"fields":["words"],"index_type":"fulltext"}}
222+
]
210223
strategy:
211224
fail-fast: false
212225
matrix:
@@ -315,20 +328,22 @@ jobs:
315328
description: SQLite
316329
# SurrealDB + Memory
317330
- name: surrealdb-memory
318-
database: surrealdb-memory
331+
database: surrealdb
332+
endpoint: -e server:memory
319333
enabled: true
320334
description: SurrealDB with in-memory storage
321335
# SurrealDB + RocksDB
322336
- name: surrealdb-rocksdb
323-
database: surrealdb-rocksdb
337+
database: surrealdb
338+
endpoint: -e server:rocksdb
324339
enabled: true
325340
description: SurrealDB with RocksDB storage
326341
# SurrealDB + SurrealKV
327342
- name: surrealdb-surrealkv
328-
database: surrealdb-surrealkv
329-
enabled: false
343+
database: surrealdb
344+
endpoint: -e server:surrealkv
345+
enabled: true
330346
description: SurrealDB with SurrealKV storage
331-
skipped: SurrealDB with SurrealKV storage benchmark awaiting fixes
332347
# SurrealDB Memory Engine
333348
- name: surrealdb-embedded-memory
334349
database: surrealdb
@@ -420,7 +435,7 @@ jobs:
420435
ulimit -u unlimited || true
421436
ulimit -l unlimited || true
422437
423-
- name: Run benchmarks (10,000 samples / 12 clients / 48 threads / key integer / random)
438+
- name: Run benchmarks (10,000 samples / 12 clients / 4 threads / key integer / random)
424439
timeout-minutes: 5
425440
if: ${{ matrix.enabled && (success() || failure()) }}
426441
run: |
@@ -431,7 +446,7 @@ jobs:
431446
docker container prune --force &>/dev/null || true
432447
docker volume prune --all --force &>/dev/null || true
433448
# Run the benchmarks
434-
${{ github.workspace }}/artifacts/crud-bench -d ${{ matrix.database }} ${{ matrix.endpoint || '' }} -s 10000 -c 12 -t 48 -k integer -r -n integer-random
449+
${{ github.workspace }}/artifacts/crud-bench -d ${{ matrix.database }} ${{ matrix.endpoint || '' }} -s 10000 -c 12 -t 4 -k integer -r -n integer-random
435450
# Kill and remove any existing crud-bench container
436451
docker container kill crud-bench &>/dev/null || true
437452
docker container rm crud-bench &>/dev/null || true
@@ -442,7 +457,7 @@ jobs:
442457
DOCKER_PRE_ARGS: ${{ matrix.DOCKER_PRE_ARGS || '' }}
443458
DOCKER_POST_ARGS: ${{ matrix.DOCKER_POST_ARGS || '' }}
444459

445-
- name: Run benchmarks (10,000 samples / 12 clients / 48 threads / key string26 / random)
460+
- name: Run benchmarks (10,000 samples / 12 clients / 4 threads / key string26 / random)
446461
timeout-minutes: 5
447462
if: ${{ matrix.enabled && (success() || failure()) }}
448463
run: |
@@ -453,7 +468,7 @@ jobs:
453468
docker container prune --force &>/dev/null || true
454469
docker volume prune --all --force &>/dev/null || true
455470
# Run the benchmarks
456-
${{ github.workspace }}/artifacts/crud-bench -d ${{ matrix.database }} ${{ matrix.endpoint || '' }} -s 10000 -c 12 -t 48 -k string26 -r -n string26-random
471+
${{ github.workspace }}/artifacts/crud-bench -d ${{ matrix.database }} ${{ matrix.endpoint || '' }} -s 10000 -c 12 -t 4 -k string26 -r -n string26-random
457472
# Kill and remove any existing crud-bench container
458473
docker container kill crud-bench &>/dev/null || true
459474
docker container rm crud-bench &>/dev/null || true
@@ -464,7 +479,7 @@ jobs:
464479
DOCKER_PRE_ARGS: ${{ matrix.DOCKER_PRE_ARGS || '' }}
465480
DOCKER_POST_ARGS: ${{ matrix.DOCKER_POST_ARGS || '' }}
466481

467-
- name: Run benchmarks (10,000 samples / 12 clients / 48 threads / key string90 / random)
482+
- name: Run benchmarks (10,000 samples / 12 clients / 4 threads / key string90 / random)
468483
timeout-minutes: 5
469484
if: ${{ matrix.enabled && (success() || failure()) }}
470485
run: |
@@ -475,7 +490,7 @@ jobs:
475490
docker container prune --force &>/dev/null || true
476491
docker volume prune --all --force &>/dev/null || true
477492
# Run the benchmarks
478-
${{ github.workspace }}/artifacts/crud-bench -d ${{ matrix.database }} ${{ matrix.endpoint || '' }} -s 10000 -c 12 -t 48 -k string90 -r -n string90-random
493+
${{ github.workspace }}/artifacts/crud-bench -d ${{ matrix.database }} ${{ matrix.endpoint || '' }} -s 10000 -c 12 -t 4 -k string90 -r -n string90-random
479494
# Kill and remove any existing crud-bench container
480495
docker container kill crud-bench &>/dev/null || true
481496
docker container rm crud-bench &>/dev/null || true
@@ -486,7 +501,7 @@ jobs:
486501
DOCKER_PRE_ARGS: ${{ matrix.DOCKER_PRE_ARGS || '' }}
487502
DOCKER_POST_ARGS: ${{ matrix.DOCKER_POST_ARGS || '' }}
488503

489-
- name: Run benchmarks (10,000 samples / 12 clients / 48 threads / key string250 / random)
504+
- name: Run benchmarks (10,000 samples / 12 clients / 4 threads / key string250 / random)
490505
timeout-minutes: 10
491506
if: ${{ matrix.enabled && (success() || failure()) }}
492507
run: |
@@ -497,7 +512,7 @@ jobs:
497512
docker container prune --force &>/dev/null || true
498513
docker volume prune --all --force &>/dev/null || true
499514
# Run the benchmarks
500-
${{ github.workspace }}/artifacts/crud-bench -d ${{ matrix.database }} ${{ matrix.endpoint || '' }} -s 10000 -c 12 -t 48 -k string250 -r -n string250-random
515+
${{ github.workspace }}/artifacts/crud-bench -d ${{ matrix.database }} ${{ matrix.endpoint || '' }} -s 10000 -c 12 -t 4 -k string250 -r -n string250-random
501516
# Kill and remove any existing crud-bench container
502517
docker container kill crud-bench &>/dev/null || true
503518
docker container rm crud-bench &>/dev/null || true
@@ -508,7 +523,7 @@ jobs:
508523
DOCKER_PRE_ARGS: ${{ matrix.DOCKER_PRE_ARGS || '' }}
509524
DOCKER_POST_ARGS: ${{ matrix.DOCKER_POST_ARGS || '' }}
510525

511-
- name: Run benchmarks (10,000 samples / 12 clients / 48 threads / key string506 / random)
526+
- name: Run benchmarks (10,000 samples / 12 clients / 4 threads / key string506 / random)
512527
timeout-minutes: 10
513528
if: ${{ matrix.enabled && (success() || failure()) }}
514529
run: |
@@ -519,7 +534,7 @@ jobs:
519534
docker container prune --force &>/dev/null || true
520535
docker volume prune --all --force &>/dev/null || true
521536
# Run the benchmarks
522-
${{ github.workspace }}/artifacts/crud-bench -d ${{ matrix.database }} ${{ matrix.endpoint || '' }} -s 10000 -c 12 -t 48 -k string506 -r -n string506-random
537+
${{ github.workspace }}/artifacts/crud-bench -d ${{ matrix.database }} ${{ matrix.endpoint || '' }} -s 10000 -c 12 -t 4 -k string506 -r -n string506-random
523538
# Kill and remove any existing crud-bench container
524539
docker container kill crud-bench &>/dev/null || true
525540
docker container rm crud-bench &>/dev/null || true
@@ -530,7 +545,7 @@ jobs:
530545
DOCKER_PRE_ARGS: ${{ matrix.DOCKER_PRE_ARGS || '' }}
531546
DOCKER_POST_ARGS: ${{ matrix.DOCKER_POST_ARGS || '' }}
532547

533-
- name: Run benchmarks (10,000 samples / 12 clients / 48 threads / key string26 / random / 0.5KiB row and object size)
548+
- name: Run benchmarks (10,000 samples / 12 clients / 4 threads / key string26 / random / 0.5KiB row and object size)
534549
timeout-minutes: 15
535550
if: ${{ matrix.enabled && (success() || failure()) }}
536551
run: |
@@ -541,7 +556,7 @@ jobs:
541556
docker container prune --force &>/dev/null || true
542557
docker volume prune --all --force &>/dev/null || true
543558
# Run the benchmarks
544-
${{ github.workspace }}/artifacts/crud-bench -d ${{ matrix.database }} ${{ matrix.endpoint || '' }} -s 10000 -c 12 -t 48 -k string26 -r -n string26-random-1k
559+
${{ github.workspace }}/artifacts/crud-bench -d ${{ matrix.database }} ${{ matrix.endpoint || '' }} -s 10000 -c 12 -t 4 -k string26 -r -n string26-random-1k
545560
# Kill and remove any existing crud-bench container
546561
docker container kill crud-bench &>/dev/null || true
547562
docker container rm crud-bench &>/dev/null || true

‎README.md‎

Lines changed: 17 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -124,7 +124,7 @@ Usage: crud-bench [OPTIONS] --database <DATABASE> --samples <SAMPLES>
124124

125125
Options:
126126
-n, --name <NAME> An optional name for the test, used as a suffix for the JSON result file name
127-
-d, --database <DATABASE> The database to benchmark [possible values: dry, map, arangodb, dragonfly, fjall, keydb, mdbx, lmdb, mongodb, mysql, neo4j, postgres, redb, redis, rocksdb, scylladb, slatedb, sqlite, surrealdb, surrealdb-memory, surrealdb-rocksdb, surrealdb-surrealkv, surrealkv, surrealmx]
127+
-d, --database <DATABASE> The database to benchmark [possible values: dry, map, arangodb, dragonfly, fjall, keydb, mdbx, lmdb, mariadb, mongodb, mysql, neo4j, postgres, redb, redis, rocksdb, scylladb, slatedb, sqlite, surrealdb, surrealds, surrealkv, surrealmx]
128128
-i, --image <IMAGE> Specify a custom Docker image
129129
-p, --privileged Whether to run Docker in privileged mode
130130
-e, --endpoint <ENDPOINT> Specify a custom endpoint to connect to
@@ -457,41 +457,23 @@ SQLite is an embedded, relational, ACID-compliant, SQL-based database.
457457
cargo run -r -- -d sqlite -s 100000 -c 12 -t 24 -r
458458
```
459459

460-
### [SurrealDB](https://surrealdb.com) (in-memory storage engine)
460+
### [SurrealDB](https://surrealdb.com)
461461

462462
```bash
463-
cargo run -r -- -d surrealdb-memory -s 100000 -c 12 -t 24 -r
463+
cargo run -r -- -d surrealdb -s 100000 -c 12 -t 24 -r
464464
```
465465

466-
### [SurrealDB](https://surrealdb.com) (RocksDB storage engine)
466+
Specify a custom endpoint using `-e` or `--endpoint` to benchmark a custom deployment:
467467

468-
```bash
469-
cargo run -r -- -d surrealdb-rocksdb -s 100000 -c 12 -t 24 -r
470-
```
471-
472-
### [SurrealDB](https://surrealdb.com) (SurrealKV storage engine)
473-
474-
```bash
475-
cargo run -r -- -d surrealdb-surrealkv -s 100000 -c 12 -t 24 -r
476-
```
477-
478-
### [SurrealDB](https://surrealdb.com) embedded (in-memory storage engine)
479-
480-
```bash
481-
cargo run -r -- -d surrealdb -e memory -s 100000 -c 12 -t 24 -r
482-
```
483-
484-
### [SurrealDB](https://surrealdb.com) embedded (RocksDB storage engine)
485-
486-
```bash
487-
cargo run -r -- -d surrealdb -e rocksdb:/tmp/db -s 100000 -c 12 -t 24 -r
488-
```
489-
490-
### [SurrealDB](https://surrealdb.com) embedded (SurrealKV storage engine)
491-
492-
```bash
493-
cargo run -r -- -d surrealdb -e surrealkv:/tmp/db -s 100000 -c 12 -t 24 -r
494-
```
468+
| Endpoint | Meaning |
469+
|----------|---------|
470+
| `server:memory` | Docker server with in-memory storage engine |
471+
| `server:rocksdb` | Docker server with RocksDB storage engine |
472+
| `server:surrealkv` | Docker server with SurrealKV storage engine |
473+
| `memory` | Embedded with in-memory storage engine. |
474+
| `rocksdb:<path>` (`rocksdb:/tmp/db`) | Embedded with RocksDB storage engine. |
475+
| `surrealkv:<path>` (`surrealkv:/tmp/db`) | Embedded with RocksDB storage engine. |
476+
| `ws://...`, `wss://...`, `http://...`, `https://...` | Remote server you manage yourself (no Docker started by crud-bench). |
495477

496478
### [SurrealKV](https://surrealkv.org)
497479

@@ -527,7 +509,7 @@ cargo run -r -- -d surrealdb -e ws://127.0.0.1:8000 -s 100000 -c 12 -t 24 -r
527509

528510
## SurrealDB Authentication
529511

530-
When benchmarking SurrealDB (including `surrealdb`, `surrealdb-memory`, `surrealdb-rocksdb`, `surrealdb-surrealkv`, and `surrealds`), you can configure authentication credentials using environment variables.
512+
When benchmarking SurrealDB (including `surrealdb`, and `surrealds`), you can configure authentication credentials using environment variables.
531513

532514
### Environment Variables
533515

@@ -536,23 +518,23 @@ When benchmarking SurrealDB (including `surrealdb`, `surrealdb-memory`, `surreal
536518

537519
These environment variables are used in two scenarios:
538520

539-
1. **Docker Container Startup**: When crud-bench automatically starts a SurrealDB Docker container, it uses these credentials to configure the database server (applies to `surrealdb-memory`, `surrealdb-rocksdb`, and `surrealdb-surrealkv`)
521+
1. **Docker Container Startup**: When crud-bench automatically starts a SurrealDB Docker container (`-d surrealdb` with no endpoint, or with `-e server:rocksdb` / `-e server:memory` / `-e server:surrealkv`), it uses these credentials to configure the database server
540522
2. **Client Authentication**: When connecting to SurrealDB instances (both embedded and networked), the benchmark client uses these credentials to authenticate
541523

542524
### Usage Examples
543525

544526
**Using default credentials (root/root):**
545527

546528
```bash
547-
cargo run -r -- -d surrealdb-rocksdb -s 100000 -c 12 -t 24 -r
529+
cargo run -r -- -d surrealdb -s 100000 -c 12 -t 24 -r
548530
```
549531

550532
**Using custom credentials via environment variables:**
551533

552534
```bash
553535
export SURREALDB_USER=admin
554536
export SURREALDB_PASS=secure_password
555-
cargo run -r -- -d surrealdb-rocksdb -s 100000 -c 12 -t 24 -r
537+
cargo run -r -- -d surrealdb -s 100000 -c 12 -t 24 -r
556538
```
557539

558540
**One-line command with custom credentials:**

‎run.sh‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -127,7 +127,7 @@ EXAMPLES:
127127
AVAILABLE DATASTORES:
128128
arangodb, dragonfly, dry, fjall, keydb, lmdb, map, mdbx, mongodb,
129129
mysql, neo4j, postgres, redb, redis, rocksdb, sqlite,
130-
surrealdb-memory, surrealdb-rocksdb, surrealdb-surrealkv,
130+
surrealdb, surrealdb-memory, surrealdb-rocksdb, surrealdb-surrealkv,
131131
surrealdb-embedded-memory, surrealdb-embedded-rocksdb,
132132
surrealdb-embedded-surrealkv, surrealkv, surrealmx
133133
@@ -316,9 +316,10 @@ redis|redis|networked|Redis|
316316
rocksdb|rocksdb|embedded|RocksDB|
317317
slatedb|slatedb|embedded|SlateDB|
318318
sqlite|sqlite|embedded|SQLite|
319-
surrealdb-memory|surrealdb-memory|networked|SurrealDB with in-memory storage|
320-
surrealdb-rocksdb|surrealdb-rocksdb|networked|SurrealDB with RocksDB storage|
321-
surrealdb-surrealkv|surrealdb-surrealkv|networked|SurrealDB with SurrealKV storage|
319+
surrealdb|surrealdb|networked|SurrealDB (server: RocksDB)|
320+
surrealdb-memory|surrealdb|networked|SurrealDB (server: memory)|-e server:memory
321+
surrealdb-rocksdb|surrealdb|networked|SurrealDB (server: RocksDB)|-e server:rocksdb
322+
surrealdb-surrealkv|surrealdb|networked|SurrealDB (server: SurrealKV)|-e server:surrealkv
322323
surrealdb-embedded-memory|surrealdb|embedded|SurrealDB embedded with in-memory storage|-e memory
323324
surrealdb-embedded-rocksdb|surrealdb|embedded|SurrealDB embedded with RocksDB storage|-e rocksdb:DATA_DIR
324325
surrealdb-embedded-surrealkv|surrealdb|embedded|SurrealDB embedded with SurrealKV storage|-e surrealkv:DATA_DIR

‎src/benchmark.rs‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,3 @@
1-
use crate::database::Database;
21
use crate::dialect::Dialect;
32
use crate::engine::{BenchmarkClient, BenchmarkEngine, ScanContext};
43
use crate::keyprovider::KeyProvider;
@@ -27,8 +26,6 @@ pub(crate) struct Benchmark {
2726
/// Whether to run containers in privileged mode
2827
pub(crate) privileged: bool,
2928
/// The container image to use
30-
pub(crate) database: Database,
31-
/// The container image to use
3229
pub(crate) image: Option<String>,
3330
/// The server endpoint to connect to
3431
pub(crate) endpoint: Option<String>,
@@ -51,7 +48,6 @@ impl Benchmark {
5148
pub(crate) fn new(args: &Args) -> Self {
5249
Self {
5350
privileged: args.privileged,
54-
database: args.database,
5551
image: args.image.to_owned(),
5652
endpoint: args.endpoint.to_owned(),
5753
clients: args.clients,

0 commit comments

Comments
 (0)