Skip to content

Commit d8f7ee7

Browse files
cheb0cheb0
committed
LID block size config
1 parent 124d460 commit d8f7ee7

12 files changed

Lines changed: 56 additions & 13 deletions

File tree

‎cmd/seq-db/seq-db.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -266,6 +266,7 @@ func startStore(
266266
SealParams: common.SealParams{
267267
IDsZstdLevel: cfg.Compression.SealedZstdCompressionLevel,
268268
LIDsZstdLevel: cfg.Compression.SealedZstdCompressionLevel,
269+
LIDBlockSize: int(cfg.Compression.LIDBlockSize),
269270
TokenListZstdLevel: cfg.Compression.SealedZstdCompressionLevel,
270271
DocsPositionsZstdLevel: cfg.Compression.SealedZstdCompressionLevel,
271272
TokenTableZstdLevel: cfg.Compression.SealedZstdCompressionLevel,
@@ -283,6 +284,7 @@ func startStore(
283284
},
284285
SkipSortDocs: !cfg.DocsSorting.Enabled,
285286
KeepMetaFile: false,
287+
LIDBlockSize: int(cfg.Compression.LIDBlockSize),
286288
},
287289
OffloadingEnabled: cfg.Offloading.Enabled,
288290
OffloadingRetention: cfg.Offloading.Retention,

‎config/config.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -200,6 +200,8 @@ type Config struct {
200200
MetasZstdCompressionLevel int `config:"metas_zstd_compression_level" default:"1"`
201201
SealedZstdCompressionLevel int `config:"sealed_zstd_compression_level" default:"3"`
202202
DocBlockZstdCompressionLevel int `config:"doc_block_zstd_compression_level" default:"3"`
203+
// LIDBlockSize sets max lids (postings) saved per LIDs block.
204+
LIDBlockSize Bytes `config:"lid_block_size" default:"64KiB"`
203205
} `config:"compression"`
204206

205207
Indexing struct {

‎config/validation.go‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,8 @@ package config
33
import (
44
"cmp"
55
"fmt"
6+
7+
"github.com/alecthomas/units"
68
)
79

810
type validateFn func() error
@@ -68,6 +70,8 @@ func (c *Config) storeValidations() []validateFn {
6870

6971
inRange("compression.sealed_zstd_compression_level", -7, 22, c.Compression.SealedZstdCompressionLevel),
7072
inRange("compression.doc_block_zstd_compression_level", -7, 22, c.Compression.DocBlockZstdCompressionLevel),
73+
greaterThan("compression.lid_block_cap", 0, c.Compression.LIDBlockSize),
74+
lessOrEqThan("compression.lid_block_cap", int(64*units.KiB), int(c.Compression.LIDBlockSize)),
7175
inRange("offloading.queue_size_percent", 0, 100, c.Offloading.QueueSizePercent),
7276

7377
greaterThan("experimental.max_regex_tokens_check", -1, c.Experimental.MaxRegexTokensCheck),
@@ -106,6 +110,18 @@ func greaterThan[T cmp.Ordered](field string, base, v T) validateFn {
106110
}
107111
}
108112

113+
func lessOrEqThan[T cmp.Ordered](field string, base, v T) validateFn {
114+
return func() error {
115+
if v > base {
116+
return fmt.Errorf(
117+
"field %q must be greater than %v",
118+
field, base,
119+
)
120+
}
121+
return nil
122+
}
123+
}
124+
109125
func inRange[T cmp.Ordered](field string, from, to, v T) validateFn {
110126
return func() error {
111127
if v < from || to < v {

‎config/validation_test.go‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,18 @@ limits:
8686
env: map[string]string{"SEQDB_OFFLOADING_QUEUE_SIZE_PERCENT": "50"},
8787
expectErr: false,
8888
},
89+
{
90+
name: "Invalid compression.lid_block_cap",
91+
cfg: baseCfg,
92+
env: map[string]string{"SEQDB_COMPRESSION_LID_BLOCK_CAP": "-1KiB"},
93+
expectErr: true,
94+
},
95+
{
96+
name: "Valid compression.lid_block_cap",
97+
cfg: baseCfg,
98+
env: map[string]string{"SEQDB_COMPRESSION_LID_BLOCK_CAP": "8KiB"},
99+
expectErr: false,
100+
},
89101
}
90102

91103
for _, tt := range tests {

‎docs/en/02-configuration.md‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -104,12 +104,13 @@ Resource allocation settings.
104104

105105
Compression level settings for various data types.
106106

107-
| Field | Type | Default | Description |
108-
|-------|------|---------|-------------|
109-
| `compression.docs_zstd_compression_level` | int | `1` | Zstandard compression level for documents |
110-
| `compression.metas_zstd_compression_level` | int | `1` | Zstandard compression level for metadata |
111-
| `compression.sealed_zstd_compression_level` | int | `3` | Zstandard compression level for sealed fractions |
112-
| `compression.doc_block_zstd_compression_level` | int | `3` | Zstandard compression level for document blocks |
107+
| Field | Type | Default | Description |
108+
|------------------------------------------------|------|---------|--------------------------------------------------|
109+
| `compression.docs_zstd_compression_level` | int | `1` | Zstandard compression level for documents |
110+
| `compression.metas_zstd_compression_level` | int | `1` | Zstandard compression level for metadata |
111+
| `compression.sealed_zstd_compression_level` | int | `3` | Zstandard compression level for sealed fractions |
112+
| `compression.doc_block_zstd_compression_level` | int | `3` | Zstandard compression level for document blocks |
113+
| `compression.lid_block_size` | int | `64KiB` | Max lids (postings) saved per LIDs block |
113114

114115
## Indexing Configuration
115116

‎docs/ru/02-configuration.md‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -104,12 +104,13 @@ id: configuration
104104

105105
Настройки уровня сжатия для различных типов данных.
106106

107-
| Параметр | Тип | Значение по умолчанию | Описание |
108-
|----------|-----|----------------------|-----------|
109-
| `compression.docs_zstd_compression_level` | int | `1` | Уровень сжатия для документов |
110-
| `compression.metas_zstd_compression_level` | int | `1` | Уровень сжатия для метаданных |
111-
| `compression.sealed_zstd_compression_level` | int | `3` | Уровень сжатия для запечатанных фракций |
112-
| `compression.doc_block_zstd_compression_level` | int | `3` | Уровень сжатия для блоков документов |
107+
| Параметр | Тип | Значение по умолчанию | Описание |
108+
|------------------------------------------------|-----|-----------------------|-----------------------------------------|
109+
| `compression.docs_zstd_compression_level` | int | `1` | Уровень сжатия для документов |
110+
| `compression.metas_zstd_compression_level` | int | `1` | Уровень сжатия для метаданных |
111+
| `compression.sealed_zstd_compression_level` | int | `3` | Уровень сжатия для запечатанных фракций |
112+
| `compression.doc_block_zstd_compression_level` | int | `3` | Уровень сжатия для блоков документов |
113+
| `compression.lid_block_size` | int | `64KiB` | Максимальное количество лидов в блоках |
113114

114115
## Конфигурация индексирования
115116

‎frac/active.go‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -114,6 +114,9 @@ func NewActive(
114114

115115
skipMaskProvider: skipMaskProvider,
116116
}
117+
if cfg.LIDBlockSize > 0 {
118+
f.info.ConstLIDBlockCap = cfg.LIDBlockSize
119+
}
117120

118121
// use of 0 as keys in maps is prohibited – it's system key, so add first element
119122
f.MIDs.Append(systemMID)

‎frac/common/seal_params.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,5 +8,6 @@ type SealParams struct {
88
TokenTableZstdLevel int
99

1010
DocBlocksZstdLevel int // DocBlocksZstdLevel is the zstd compress level of each document block.
11+
LIDBlockSize int
1112
DocBlockSize int // DocBlockSize is decompressed payload size of document block.
1213
}

‎frac/config.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ type Config struct {
55

66
SkipSortDocs bool
77
KeepMetaFile bool
8+
LIDBlockSize int
89
}
910

1011
type SearchConfig struct {

‎frac/fraction_test.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -97,6 +97,7 @@ func (s *FractionTestSuite) SetupTestCommon() {
9797
DocsPositionsZstdLevel: 1,
9898
TokenTableZstdLevel: 1,
9999
DocBlocksZstdLevel: 1,
100+
LIDBlockSize: 512,
100101
DocBlockSize: 128 * int(units.KiB),
101102
}
102103

0 commit comments

Comments
 (0)