Skip to content

Commit d0786b4

Browse files
claudevdmclaudevdm
committed
initial
1 parent 7264b3a commit d0786b4

26 files changed

Lines changed: 7374 additions & 21 deletions

File tree

‎.github/workflows/bqmonitor-pr.yml‎

Lines changed: 113 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,113 @@
1+
# Copyright 2026 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# https://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
name: BigQuery Anomaly Detection PR
16+
17+
on:
18+
pull_request:
19+
branches:
20+
- 'main'
21+
paths:
22+
- 'python/src/main/python/bigquery-anomaly-detection/**'
23+
- 'python/src/test/python/bigquery-anomaly-detection/**'
24+
- 'python/src/main/java/**/BigQueryAnomalyDetection*.java'
25+
- 'python/src/test/java/**/BigQueryAnomalyDetection*.java'
26+
- '.github/workflows/bqmonitor-pr.yml'
27+
workflow_dispatch:
28+
29+
concurrency:
30+
group: ${{ github.workflow }}-${{ github.ref }}
31+
cancel-in-progress: true
32+
33+
env:
34+
MAVEN_OPTS: -Dorg.slf4j.simpleLogger.log.org.apache.maven.plugins.shade=error
35+
IT_REGION: us-west2
36+
37+
permissions:
38+
actions: write
39+
checks: write
40+
contents: read
41+
pull-requests: read
42+
statuses: write
43+
44+
jobs:
45+
python_unit_tests:
46+
name: Python Unit Tests
47+
timeout-minutes: 15
48+
runs-on: ubuntu-latest
49+
steps:
50+
- name: Checkout Code
51+
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
52+
- name: Set up Python
53+
uses: actions/setup-python@v5
54+
with:
55+
python-version: '3.11'
56+
- name: Install dependencies and run tests
57+
working-directory: python
58+
run: |
59+
pip install -r src/test/python/bigquery-anomaly-detection/requirements-test.txt
60+
pip install -e src/main/python/bigquery-anomaly-detection
61+
python -m unittest discover \
62+
-s src/test/python/bigquery-anomaly-detection \
63+
-p '*_test.py' \
64+
-v
65+
java_build:
66+
name: Build
67+
timeout-minutes: 60
68+
runs-on: [self-hosted, it]
69+
steps:
70+
- name: Checkout Code
71+
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
72+
- name: Setup Environment
73+
id: setup-env
74+
uses: ./.github/actions/setup-env
75+
- name: Run Build
76+
run: |
77+
./cicd/run-build \
78+
--modules-to-build="YAML"
79+
- name: Cleanup Java Environment
80+
uses: ./.github/actions/cleanup-java-env
81+
java_integration_tests:
82+
name: Integration Tests
83+
needs: [python_unit_tests, java_build]
84+
timeout-minutes: 120
85+
runs-on: [self-hosted, it]
86+
steps:
87+
- name: Checkout Code
88+
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
89+
- name: Setup Environment
90+
id: setup-env
91+
uses: ./.github/actions/setup-env
92+
- name: Run Integration Tests
93+
run: |
94+
./cicd/run-it-tests \
95+
--modules-to-build="YAML" \
96+
--it-region="${{ env.IT_REGION }}" \
97+
--it-project="cloud-teleport-testing" \
98+
--it-artifact-bucket="cloud-teleport-testing-it-gitactions" \
99+
--it-private-connectivity="datastream-connect-2" \
100+
--test="BigQueryAnomalyDetectionIT"
101+
- name: Upload Integration Tests Report
102+
uses: actions/upload-artifact@v6
103+
if: always()
104+
with:
105+
name: surefire-integration-test-results
106+
path: |
107+
**/surefire-reports/TEST-*.xml
108+
**/surefire-reports/*.html
109+
**/surefire-reports/html/**
110+
retention-days: 1
111+
- name: Cleanup Java Environment
112+
uses: ./.github/actions/cleanup-java-env
113+
if: always()

‎plugins/core-plugin/src/main/java/com/google/cloud/teleport/plugin/DockerfileGenerator.java‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -210,6 +210,8 @@ public Builder(
210210

211211
this.parameters.put("filesToCopy", "");
212212
this.parameters.put("directoriesToCopy", "");
213+
this.parameters.put("setupFileEnv", "");
214+
this.parameters.put("setupInstall", "");
213215
this.parameters.put("commandSpec", "");
214216
}
215217

@@ -337,6 +339,25 @@ public Builder setDirectoriesToCopy(Set<String> directoriesToCopy) {
337339
return addStringParameter("directoriesToCopy", directories.toString());
338340
}
339341

342+
/**
343+
* Configures the Dockerfile to install a Python package via {@code pip install .} at build time
344+
* and sets {@code FLEX_TEMPLATE_PYTHON_SETUP_FILE} so the Beam stager packages the source code
345+
* for distribution to workers. The absolute path avoids issues with {@code os.chdir()}.
346+
*
347+
* @param setupFile the setup file name (e.g. "setup.py").
348+
* @return this {@link Builder}.
349+
*/
350+
public Builder setSetupFile(String setupFile) {
351+
Preconditions.checkArgument(!Strings.isNullOrEmpty(setupFile));
352+
String workDir =
353+
(String) this.parameters.getOrDefault("workingDirectory", DEFAULT_WORKING_DIRECTORY);
354+
addParameter(
355+
"setupFileEnv",
356+
"ENV FLEX_TEMPLATE_PYTHON_SETUP_FILE=\"" + workDir + "/" + setupFile + "\"");
357+
addParameter("setupInstall", "RUN pip install --no-cache-dir .");
358+
return this;
359+
}
360+
340361
/**
341362
* For XLANG templates, set the {@code DATAFLOW_JAVA_COMMAND_SPEC} env variable to the command
342363
* spec location on the image.

‎plugins/core-plugin/src/main/resources/Dockerfile-template-python‎

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,17 +3,27 @@ FROM ${basePythonContainerImage}
33
ARG WORKDIR=${workingDirectory}
44
RUN mkdir -p $WORKDIR
55
${filesToCopy}
6+
${directoriesToCopy}
67
WORKDIR $WORKDIR
78

8-
ENV FLEX_TEMPLATE_PYTHON_REQUIREMENTS_FILE=requirements.txt
9+
# Do NOT use ENV FLEX_TEMPLATE_PYTHON_REQUIREMENTS_FILE here. All deps are preinstalled
10+
# at build time. That env var triggers the Beam stager to re-resolve requirements on the
11+
# launcher VM, where platform tag mismatches cause pip to compile C extensions (e.g. numpy)
12+
# from source, freezing or timing out the launcher.
13+
# FLEX_TEMPLATE_PYTHON_SETUP_FILE (via setupFileEnv) IS needed to stage custom code to workers.
14+
# TODO: For templates without setup.py that need non-beam deps on workers, split requirements
15+
# into a build-only lockfile and a stager-safe extra-requirements file.
16+
ARG REQUIREMENTS_FILE=requirements.txt
917
ENV FLEX_TEMPLATE_PYTHON_PY_FILE=main.py
18+
${setupFileEnv}
1019

1120
RUN if ! [ -f requirements.txt ] ; then >&2 echo "error: no requirements.txt file found" && exit 1 ; fi
1221

1322
# Set up custom PyPi repository, if applicable
1423
${airlockConfig}
1524

16-
RUN pip install -U -r --require-hashes $FLEX_TEMPLATE_PYTHON_REQUIREMENTS_FILE
17-
RUN pip download --require-hashes --no-cache-dir --dest /tmp/dataflow-requirements-cache -r $FLEX_TEMPLATE_PYTHON_REQUIREMENTS_FILE
25+
RUN pip install -U --require-hashes -r $REQUIREMENTS_FILE
26+
${setupInstall}
27+
RUN pip download --require-hashes --no-cache-dir --dest /tmp/dataflow-requirements-cache -r $REQUIREMENTS_FILE
1828

1929
ENTRYPOINT ${entryPoint}

‎plugins/core-plugin/src/test/java/com/google/cloud/teleport/plugin/DockerfileGeneratorTest.java‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -58,8 +58,7 @@ public void testGeneratePythonDockerfileDefaults() throws IOException, TemplateE
5858
assertTrue(outputFile.exists());
5959
String fileContents = Files.asCharSource(outputFile, StandardCharsets.UTF_8).read();
6060
assertThat(fileContents).contains("FROM " + BASE_PYTHON_CONTAINER_IMAGE);
61-
assertThat(fileContents)
62-
.contains("RUN pip install -U -r --require-hashes $FLEX_TEMPLATE_PYTHON_REQUIREMENTS_FILE");
61+
assertThat(fileContents).contains("RUN pip install -U --require-hashes -r $REQUIREMENTS_FILE");
6362
assertThat(fileContents)
6463
.contains(String.format("ENTRYPOINT [\"%s\"]", PYTHON_LAUNCHER_ENTRYPOINT));
6564
}
@@ -80,8 +79,7 @@ public void testGeneratePythonDockerfile() throws IOException, TemplateException
8079
assertTrue(outputFile.exists());
8180
String fileContents = Files.asCharSource(outputFile, StandardCharsets.UTF_8).read();
8281
assertThat(fileContents).contains("FROM a python container image");
83-
assertThat(fileContents)
84-
.contains("RUN pip install -U -r --require-hashes $FLEX_TEMPLATE_PYTHON_REQUIREMENTS_FILE");
82+
assertThat(fileContents).contains("RUN pip install -U --require-hashes -r $REQUIREMENTS_FILE");
8583
assertThat(fileContents).contains("COPY main.py requirements.txt $WORKDIR/");
8684
assertThat(fileContents).contains("ENTRYPOINT [\"python/entry/point\"]");
8785
}

‎plugins/templates-maven-plugin/src/main/java/com/google/cloud/teleport/plugin/maven/TemplatesStageMojo.java‎

Lines changed: 30 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1008,7 +1008,7 @@ void prepareYamlDockerfile(TemplateDefinitions definition, String containerName)
10081008
}
10091009

10101010
List<String> entryPoint = List.of(definition.getTemplateAnnotation().entryPoint());
1011-
if (entryPoint.isEmpty()) {
1011+
if (entryPoint.isEmpty() || (entryPoint.size() == 1 && entryPoint.get(0).isEmpty())) {
10121012
entryPoint = List.of(pythonTemplateLauncherEntryPoint);
10131013
}
10141014

@@ -1053,37 +1053,53 @@ private void stageFlexPythonTemplate(
10531053
String dockerfilePath = dockerfileContainer + "/Dockerfile";
10541054
File dockerfile = new File(dockerfilePath);
10551055
if (!dockerfile.exists()) {
1056-
List<String> filesToCopy = List.of(definition.getTemplateAnnotation().filesToCopy());
1056+
List<String> allFilesToCopy = List.of(definition.getTemplateAnnotation().filesToCopy());
1057+
if (allFilesToCopy.isEmpty()) {
1058+
allFilesToCopy = List.of("main.py", "requirements.txt");
1059+
}
1060+
1061+
// Separate flat files from directories
1062+
List<String> filesToCopy = new ArrayList<>();
1063+
Set<String> directoriesToCopy = new HashSet<>();
1064+
for (String f : allFilesToCopy) {
1065+
File source = new File(dockerfileContainer + "/" + f);
1066+
if (source.isDirectory()) {
1067+
directoriesToCopy.add(f);
1068+
} else {
1069+
filesToCopy.add(f);
1070+
}
1071+
}
10571072
if (filesToCopy.isEmpty()) {
10581073
filesToCopy = List.of("main.py", "requirements.txt");
10591074
}
1075+
10601076
List<String> entryPoint = List.of(definition.getTemplateAnnotation().entryPoint());
1061-
if (entryPoint.isEmpty()) {
1077+
if (entryPoint.isEmpty() || (entryPoint.size() == 1 && entryPoint.get(0).isEmpty())) {
10621078
entryPoint = List.of(pythonTemplateLauncherEntryPoint);
10631079
}
10641080

1065-
// Copy in requirements.txt if present
1066-
File sourceRequirements = new File(outputClassesDirectory.getPath() + "/requirements.txt");
1067-
File destRequirements = new File(dockerfileContainer + "/requirements.txt");
1068-
if (sourceRequirements.exists()) {
1069-
Files.copy(
1070-
sourceRequirements.toPath(),
1071-
destRequirements.toPath(),
1072-
StandardCopyOption.REPLACE_EXISTING);
1073-
}
1074-
10751081
// Generate Dockerfile
10761082
LOG.info("Generating dockerfile " + dockerfilePath);
10771083
DockerfileGenerator.Builder dockerfileBuilder =
10781084
DockerfileGenerator.builder(
10791085
definition.getTemplateAnnotation().type(),
10801086
beamVersion,
10811087
containerName,
1082-
targetDirectory)
1088+
outputClassesDirectory)
10831089
.setBasePythonContainerImage(basePythonContainerImage)
10841090
.setFilesToCopy(filesToCopy)
10851091
.setEntryPoint(entryPoint);
10861092

1093+
if (!directoriesToCopy.isEmpty()) {
1094+
dockerfileBuilder.setDirectoriesToCopy(directoriesToCopy);
1095+
}
1096+
1097+
// Configure setup.py support if present
1098+
File setupFile = new File(dockerfileContainer + "/setup.py");
1099+
if (setupFile.exists()) {
1100+
dockerfileBuilder.setSetupFile("setup.py");
1101+
}
1102+
10871103
// Set Airlock parameters
10881104
if (internalMaven) {
10891105
dockerfileBuilder

‎pom.xml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -615,6 +615,7 @@
615615
</includes>
616616
<excludes>
617617
<exclude>**/KafkaToKafkaIT.java</exclude>
618+
<exclude>**/BigQueryAnomalyDetectionIT.java</exclude>
618619
</excludes>
619620
<excludedGroups>${direct-runner.tests}</excludedGroups>
620621
<groups>
@@ -878,6 +879,7 @@
878879
</includes>
879880
<excludes>
880881
<exclude>**/KafkaToKafkaIT.java</exclude>
882+
<exclude>**/BigQueryAnomalyDetectionIT.java</exclude>
881883
</excludes>
882884
<excludedGroups>
883885
${direct-runner.tests},

0 commit comments

Comments
 (0)