Skip to content
Merged
Show file tree
Hide file tree
Changes from 4 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 11 additions & 7 deletions docs/en/transforms/replace.md
Original file line number Diff line number Diff line change
Expand Up @@ -4,21 +4,25 @@

## Description

Examines string value in a given field and replaces substring of the string value that matches the given string literal or regexes with the given replacement.
Examines string value in given fields and replaces substring of the string value that matches the given string literal or regexes with the given replacement.

## Options

| name | type | required | default value |
| name | type | required | default value |
|---------------|---------|----------|---------------|
| replace_field | string | yes | |
| replace_fields | array | yes | |
| pattern | string | yes | - |
| replacement | string | yes | - |
| is_regex | boolean | no | false |
| replace_first | boolean | no | false |

### replace_field [string]
### replace_fields [array]

The field you want to replace
The fields you want to replace

`replace_fields` accepts a list such as `["name"]` or `["name", "title"]`.

For backward compatibility, legacy key `replace_field = "name"` is still supported, but new configurations should use `replace_fields`.
Comment on lines +21 to +25

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If the old configuration is compatible, then IT should retain the test case for the old configuration

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hi @corgy-w

Thanks for the review :)

I still kept the legacy replace_field path in replace_transform.conf and replace_transform_multi_table.conf

Could you please review again?
If there is any other test I should add, please let me know.


### pattern [string]

Expand Down Expand Up @@ -58,7 +62,7 @@ transform {
Replace {
plugin_input = "fake"
plugin_output = "fake1"
replace_field = "name"
replace_fields = ["name"]
pattern = " "
replacement = "_"
is_regex = true
Expand Down Expand Up @@ -99,7 +103,7 @@ transform {
Replace {
plugin_input = "fake"
plugin_output = "fake1"
replace_field = "name"
replace_fields = ["name"]
pattern = ".+"
replacement = "b"
is_regex = true
Expand Down
15 changes: 9 additions & 6 deletions docs/zh/transforms/replace.md
Original file line number Diff line number Diff line change
Expand Up @@ -8,18 +8,22 @@

## 属性

| 名称 | 类型 | 是否必须 | 默认值 |
| 名称 | 类型 | 是否必须 | 默认值 |
|---------------|---------|------|-------|
| replace_field | string | yes | |
| replace_fields | array | yes | |
| pattern | string | yes | - |
| replacement | string | yes | - |
| is_regex | boolean | no | false |
| replace_first | boolean | no | false |

### replace_field [string]
### replace_fields [array]

需要替换的字段

`replace_fields` 支持列表形式,例如:`["name"]` 或 `["name", "title"]`。

为了向后兼容,旧配置 `replace_field = "name"` 仍然支持,但新配置建议统一使用 `replace_fields`。

### pattern [string]

将被替换的旧字符串
Expand Down Expand Up @@ -58,7 +62,7 @@ transform {
Replace {
plugin_input = "fake"
plugin_output = "fake1"
replace_field = "name"
replace_fields = ["name"]
pattern = " "
replacement = "_"
is_regex = true
Expand Down Expand Up @@ -99,7 +103,7 @@ transform {
Replace {
plugin_input = "fake"
plugin_output = "fake1"
replace_field = "name"
replace_fields = ["name"]
pattern = ".+"
replacement = "b"
is_regex = true
Expand All @@ -118,4 +122,3 @@ sink {
### 新版本

- 添加替换转换连接器

Original file line number Diff line number Diff line change
Expand Up @@ -29,15 +29,33 @@ public class TestReplaceIT extends TestSuiteBase {

@TestTemplate
public void testReplace(TestContainer container) throws IOException, InterruptedException {
Container.ExecResult execResult = container.executeJob("/replace_transform.conf");
// Backward compatibility old key 'replace_field' with a single string value
Container.ExecResult execResult = container.executeJob("/replace/replace_transform.conf");
Assertions.assertEquals(0, execResult.getExitCode());
}

@TestTemplate
public void testReplaceMultiTable(TestContainer container)
throws IOException, InterruptedException {
// Backward compatibility old key 'replace_field' with a single string value
Container.ExecResult execResult =
container.executeJob("/replace_transform_multi_table.conf");
container.executeJob("/replace/replace_transform_multi_table.conf");
Assertions.assertEquals(0, execResult.getExitCode());
}

@TestTemplate
public void testReplaceMultiField(TestContainer container)
throws IOException, InterruptedException {
Container.ExecResult execResult =
container.executeJob("/replace/replace_transform_multi_field.conf");
Assertions.assertEquals(0, execResult.getExitCode());
}

@TestTemplate
public void testReplaceMultiTableMultiField(TestContainer container)
throws IOException, InterruptedException {
Container.ExecResult execResult =
container.executeJob("/replace/replace_transform_multi_table_multi_field.conf");
Assertions.assertEquals(0, execResult.getExitCode());
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,109 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
######
###### This config file is a demonstration of streaming processing in seatunnel config
######

env {
job.mode = "BATCH"
}

source {
FakeSource {
plugin_output = "fake"
row.num = 100
schema = {
fields {
id = "int"
name = "string"
title = "string"
}
}
}
}

transform {
Replace {
plugin_input = "fake"
plugin_output = "fake1"
replace_fields = ["name", "title"]
pattern = ".+"
replacement = "replaced"
is_regex = true
replace_first = false
}
}

sink {
Assert {
plugin_input = "fake1"
rules =
{
row_rules = [
{
rule_type = MIN_ROW
rule_value = 100
}
],
field_rules = [
{
field_name = id
field_type = int
field_value = [
{
rule_type = NOT_NULL
}
]
},
{
field_name = name
field_type = string
field_value = [
{
rule_type = NOT_NULL
},
{
rule_type = MIN_LENGTH
rule_value = 8
},
{
rule_type = MAX_LENGTH
rule_value = 8
}
]
},
{
field_name = title
field_type = string
field_value = [
{
rule_type = NOT_NULL
},
{
rule_type = MIN_LENGTH
rule_value = 8
},
{
rule_type = MAX_LENGTH
rule_value = 8
}
]
}
]
}
}
}
Loading
Loading