diff --git a/conductor-client-spring/build.gradle b/conductor-client-spring/build.gradle index bb8fc1847..f5d02f10e 100644 --- a/conductor-client-spring/build.gradle +++ b/conductor-client-spring/build.gradle @@ -20,6 +20,8 @@ repositories { dependencies { api project(":conductor-client") implementation 'org.springframework.boot:spring-boot-starter:3.5.13' + + testImplementation 'org.springframework.boot:spring-boot-starter-test:3.3.13' } java { diff --git a/conductor-client-spring/src/main/java/com/netflix/conductor/client/spring/ConductorClientAutoConfiguration.java b/conductor-client-spring/src/main/java/com/netflix/conductor/client/spring/ConductorClientAutoConfiguration.java index 81c2ec474..301005cf4 100644 --- a/conductor-client-spring/src/main/java/com/netflix/conductor/client/spring/ConductorClientAutoConfiguration.java +++ b/conductor-client-spring/src/main/java/com/netflix/conductor/client/spring/ConductorClientAutoConfiguration.java @@ -18,10 +18,13 @@ import java.util.Optional; import org.apache.commons.lang3.StringUtils; +import org.conductoross.conductor.client.FileClient; +import org.conductoross.conductor.client.FileClientProperties; import org.springframework.boot.autoconfigure.AutoConfiguration; import org.springframework.boot.autoconfigure.AutoConfigureOrder; import org.springframework.boot.autoconfigure.condition.ConditionalOnBean; import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; +import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.context.annotation.Bean; import org.springframework.core.Ordered; @@ -122,4 +125,18 @@ public WorkflowExecutor workflowExecutor(ConductorClient client, AnnotatedWorker public WorkflowClient workflowClient(ConductorClient client) { return new WorkflowClient(client); } + + @Bean + @ConfigurationProperties("conductor.file-client") + @ConditionalOnMissingBean + public FileClientProperties fileClientProperties() { + return new FileClientProperties(); + } + + @Bean + @ConditionalOnBean(ConductorClient.class) + @ConditionalOnMissingBean + public FileClient fileClient(ConductorClient client, FileClientProperties fileClientProperties) { + return new FileClient(client, fileClientProperties, null); + } } diff --git a/conductor-client-spring/src/main/java/io/orkes/conductor/client/spring/OrkesConductorClientAutoConfiguration.java b/conductor-client-spring/src/main/java/io/orkes/conductor/client/spring/OrkesConductorClientAutoConfiguration.java index 9eb138fc8..1f8d9992e 100644 --- a/conductor-client-spring/src/main/java/io/orkes/conductor/client/spring/OrkesConductorClientAutoConfiguration.java +++ b/conductor-client-spring/src/main/java/io/orkes/conductor/client/spring/OrkesConductorClientAutoConfiguration.java @@ -13,6 +13,7 @@ package io.orkes.conductor.client.spring; import org.apache.commons.lang3.StringUtils; +import org.conductoross.conductor.client.FileClient; import org.springframework.boot.autoconfigure.AutoConfiguration; import org.springframework.boot.autoconfigure.condition.ConditionalOnBean; import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; @@ -128,4 +129,10 @@ public SecretClient orkesSecretClient(OrkesClients clients) { return clients.getSecretClient(); } + @Bean + @ConditionalOnBean(ApiClient.class) + @ConditionalOnMissingBean + public FileClient fileClient(ApiClient client) { + return new FileClient(client); + } } diff --git a/conductor-client-spring/src/test/java/com/netflix/conductor/client/spring/ConductorClientAutoConfigurationTest.java b/conductor-client-spring/src/test/java/com/netflix/conductor/client/spring/ConductorClientAutoConfigurationTest.java new file mode 100644 index 000000000..0a1c10342 --- /dev/null +++ b/conductor-client-spring/src/test/java/com/netflix/conductor/client/spring/ConductorClientAutoConfigurationTest.java @@ -0,0 +1,52 @@ +/* + * Copyright 2026 Conductor Authors. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package com.netflix.conductor.client.spring; + +import org.conductoross.conductor.client.FileClient; +import org.conductoross.conductor.client.FileClientProperties; +import org.junit.jupiter.api.Test; +import org.springframework.boot.autoconfigure.AutoConfigurations; +import org.springframework.boot.test.context.runner.ApplicationContextRunner; + +import static org.assertj.core.api.Assertions.assertThat; + +class ConductorClientAutoConfigurationTest { + + private final ApplicationContextRunner contextRunner = new ApplicationContextRunner() + .withConfiguration(AutoConfigurations.of(ConductorClientAutoConfiguration.class)); + + @Test + void zeroConfigYieldsWorkingFileClientBean() { + contextRunner + .withPropertyValues("conductor.client.base-path=http://localhost:8080/api") + .run(context -> { + assertThat(context).hasSingleBean(FileClient.class); + FileClientProperties properties = context.getBean(FileClientProperties.class); + assertThat(properties.getLocalCacheDirectory()) + .startsWith(System.getProperty("java.io.tmpdir")); + }); + } + + @Test + void cacheDirectoryOverrideIsRespected() { + contextRunner + .withPropertyValues( + "conductor.client.base-path=http://localhost:8080/api", + "conductor.file-client.local-cache-directory=/tmp/custom-cache") + .run(context -> { + FileClientProperties properties = context.getBean(FileClientProperties.class); + assertThat(properties.getLocalCacheDirectory()).isEqualTo("/tmp/custom-cache"); + }); + } + +} diff --git a/conductor-client/src/main/java/com/netflix/conductor/client/automator/TaskRunner.java b/conductor-client/src/main/java/com/netflix/conductor/client/automator/TaskRunner.java index 7d19050f0..a4c7a20a2 100644 --- a/conductor-client/src/main/java/com/netflix/conductor/client/automator/TaskRunner.java +++ b/conductor-client/src/main/java/com/netflix/conductor/client/automator/TaskRunner.java @@ -14,6 +14,7 @@ import java.io.PrintWriter; import java.io.StringWriter; +import java.nio.file.Path; import java.util.ArrayList; import java.util.Collections; import java.util.LinkedList; @@ -33,6 +34,11 @@ import java.util.function.Function; import org.apache.commons.lang3.concurrent.BasicThreadFactory; +import org.conductoross.conductor.client.FileClient; +import org.conductoross.conductor.sdk.file.FileHandler; +import org.conductoross.conductor.sdk.file.FileUploadOptions; +import org.conductoross.conductor.sdk.file.LocalFileHandler; +import org.conductoross.conductor.sdk.file.WorkflowFileClient; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -51,6 +57,7 @@ import com.netflix.conductor.common.metadata.tasks.Task; import com.netflix.conductor.common.metadata.tasks.TaskResult; +import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Stopwatch; import com.google.common.util.concurrent.Uninterruptibles; @@ -73,6 +80,7 @@ class TaskRunner { private final EventDispatcher eventDispatcher; private final LinkedBlockingQueue tasksTobeExecuted; private final boolean enableUpdateV2; + private final FileClient fileClient; private static final int LEASE_EXTEND_RETRY_COUNT = 3; private static final double LEASE_EXTEND_DURATION_FACTOR = 0.8; private final ScheduledExecutorService leaseExtendExecutorService; @@ -87,9 +95,11 @@ class TaskRunner { int taskPollTimeout, List pollFilters, EventDispatcher eventDispatcher, - boolean useVirtualThreads) { + boolean useVirtualThreads, + FileClient fileClient) { this.worker = worker; this.taskClient = taskClient; + this.fileClient = fileClient; this.updateRetryCount = updateRetryCount; this.taskPollTimeout = taskPollTimeout; this.pollingIntervalInMillis = worker.getPollingInterval(); @@ -346,6 +356,11 @@ private void executeTask(Worker worker, Task task) { return; } + // Set FileClient on task for file storage support + if (fileClient != null) { + task.setWorkflowFileClient(new WorkflowFileClient(fileClient, task.getWorkflowInstanceId())); + } + // Calculate inbound network latency try { if(task.getExecutionMetadata().getServerSendTime() != null ){ @@ -368,6 +383,7 @@ private void executeTask(Worker worker, Task task) { worker.getIdentity()); result = worker.execute(task); stopwatch.stop(); + uploadFilesToFileStorage(result, task.getWorkflowInstanceId(), task.getTaskId()); eventDispatcher.publish(new TaskExecutionCompleted(taskType, task.getTaskId(), worker.getIdentity(), stopwatch.elapsed(TimeUnit.MILLISECONDS))); // record execution end time in task task.getExecutionMetadata().setExecutionEndTime(System.currentTimeMillis()); @@ -453,6 +469,26 @@ private void updateTaskResult(int count, Task task, TaskResult result, Worker wo } } + @VisibleForTesting + void uploadFilesToFileStorage(TaskResult result, String workflowId, String taskId) { + if (fileClient == null || result.getOutputData() == null) { + return; + } + for (var entry : result.getOutputData().entrySet()) { + if (!(entry.getValue() instanceof FileHandler fh)) { + continue; + } + if (fh.getFileHandleId() == null) { + Path path = ((LocalFileHandler) fh).getPath(); + FileUploadOptions options = new FileUploadOptions() + .setContentType(fh.getContentType()) + .setTaskId(taskId); + FileHandler uploaded = fileClient.upload(workflowId, path, options); + entry.setValue(uploaded); + } + } + } + private Optional upload(TaskResult result, String taskType) { try { return taskClient.evaluateAndUploadLargePayload(result.getOutputData(), taskType); diff --git a/conductor-client/src/main/java/com/netflix/conductor/client/automator/TaskRunnerConfigurer.java b/conductor-client/src/main/java/com/netflix/conductor/client/automator/TaskRunnerConfigurer.java index 7c174b2dc..5bb21bcf6 100644 --- a/conductor-client/src/main/java/com/netflix/conductor/client/automator/TaskRunnerConfigurer.java +++ b/conductor-client/src/main/java/com/netflix/conductor/client/automator/TaskRunnerConfigurer.java @@ -21,6 +21,7 @@ import java.util.function.Consumer; import org.apache.commons.lang3.concurrent.BasicThreadFactory; +import org.conductoross.conductor.client.FileClient; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -54,6 +55,7 @@ public class TaskRunnerConfigurer { private final List pollFilters; private final EventDispatcher eventDispatcher; private final boolean useVirtualThreads; + private final FileClient fileClient; /** * @see TaskRunnerConfigurer.Builder @@ -75,6 +77,7 @@ private TaskRunnerConfigurer(TaskRunnerConfigurer.Builder builder) { this.pollFilters = builder.pollFilters; this.eventDispatcher = builder.eventDispatcher; this.useVirtualThreads = builder.useVirtualThreads; + this.fileClient = builder.fileClient; builder.workers.forEach(this.workers::add); taskRunners = new LinkedList<>(); } @@ -173,7 +176,8 @@ private void startWorker(Worker worker) { taskPollTimeout, pollFilters, eventDispatcher, - useVirtualThreads); + useVirtualThreads, + fileClient); // startWorker(worker) is executed by several threads. // taskRunners.add(taskRunner) without synchronization could lead to a race condition and unpredictable behavior, // including potential null values being inserted or corrupted state. @@ -206,6 +210,7 @@ public static class Builder { private final List pollFilters = new LinkedList<>(); private final EventDispatcher eventDispatcher = new EventDispatcher<>(); private boolean useVirtualThreads; + private FileClient fileClient; public Builder(TaskClient taskClient, Iterable workers) { Preconditions.checkNotNull(taskClient, "TaskClient cannot be null"); @@ -352,5 +357,10 @@ public Builder withUseVirtualThreads(boolean useVirtualThreads) { this.useVirtualThreads = useVirtualThreads; return this; } + + public Builder withFileClient(FileClient fileClient) { + this.fileClient = fileClient; + return this; + } } } diff --git a/conductor-client/src/main/java/com/netflix/conductor/common/metadata/tasks/Task.java b/conductor-client/src/main/java/com/netflix/conductor/common/metadata/tasks/Task.java index 10438a435..72537366e 100644 --- a/conductor-client/src/main/java/com/netflix/conductor/common/metadata/tasks/Task.java +++ b/conductor-client/src/main/java/com/netflix/conductor/common/metadata/tasks/Task.java @@ -18,10 +18,16 @@ import java.util.Optional; import org.apache.commons.lang3.StringUtils; +import org.conductoross.conductor.sdk.file.FileHandler; +import org.conductoross.conductor.sdk.file.FileStorageException; +import org.conductoross.conductor.sdk.file.FileUploader; +import org.conductoross.conductor.sdk.file.ManagedFileHandler; +import org.conductoross.conductor.sdk.file.WorkflowFileClient; import com.netflix.conductor.common.metadata.workflow.WorkflowTask; import com.netflix.conductor.common.run.tasks.TypedTask; +import com.fasterxml.jackson.annotation.JsonIgnore; import lombok.*; @Data @@ -179,6 +185,27 @@ public boolean isRetriable() { private long firstStartTime; + @JsonIgnore + private transient WorkflowFileClient workflowFileClient; + + @JsonIgnore + public FileUploader getFileUploader() { return workflowFileClient; } + + public void setWorkflowFileClient(WorkflowFileClient workflowFileClient) { + this.workflowFileClient = workflowFileClient; + } + + public FileHandler getInputFileHandler(String key) { + Object value = getInputData().get(key); + String fileHandleId = FileHandler.extractFileHandleId(value); + if (FileHandler.isFileHandleId(fileHandleId)) { + return new ManagedFileHandler(fileHandleId, workflowFileClient); + } + throw new FileStorageException( + "Expected " + FileHandler.PREFIX + + " reference for key '" + key + "', got: " + value); + } + public void setInputData(Map inputData) { if (inputData == null) { inputData = new HashMap<>(); diff --git a/conductor-client/src/main/java/com/netflix/conductor/sdk/workflow/executor/task/AnnotatedWorker.java b/conductor-client/src/main/java/com/netflix/conductor/sdk/workflow/executor/task/AnnotatedWorker.java index 275a0ceac..7339a0ab5 100644 --- a/conductor-client/src/main/java/com/netflix/conductor/sdk/workflow/executor/task/AnnotatedWorker.java +++ b/conductor-client/src/main/java/com/netflix/conductor/sdk/workflow/executor/task/AnnotatedWorker.java @@ -12,6 +12,7 @@ */ package com.netflix.conductor.sdk.workflow.executor.task; +import java.io.IOException; import java.lang.annotation.Annotation; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; @@ -20,6 +21,11 @@ import java.lang.reflect.Type; import java.util.*; +import org.conductoross.conductor.sdk.file.FileHandler; +import org.conductoross.conductor.sdk.file.FileHandlerDeserializer; +import org.conductoross.conductor.sdk.file.FileStorageException; +import org.conductoross.conductor.sdk.file.ManagedFileHandler; + import com.netflix.conductor.client.worker.Worker; import com.netflix.conductor.common.config.ObjectMapperProvider; import com.netflix.conductor.common.metadata.tasks.Task; @@ -31,7 +37,9 @@ import com.netflix.conductor.sdk.workflow.task.OutputParam; import com.netflix.conductor.sdk.workflow.task.WorkflowInstanceIdInputParam; +import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import lombok.extern.slf4j.Slf4j; @@ -111,9 +119,9 @@ private Object[] getInvocationParameters(Task task) { Parameter[] parameters = workerMethod.getParameters(); if (parameterTypes.length == 1 && parameterTypes[0].equals(Task.class)) { - return new Object[] {task}; + return new Object[]{task}; } else if (parameterTypes.length == 1 && parameterTypes[0].equals(Map.class)) { - return new Object[] {task.getInputData()}; + return new Object[]{task.getInputData()}; } return getParameters(task, parameterTypes, parameters); @@ -124,7 +132,7 @@ private Object[] getParameters(Task task, Class[] parameterTypes, Parameter[] Object[] values = new Object[parameterTypes.length]; for (int i = 0; i < parameterTypes.length; i++) { Annotation[] paramAnnotation = parameterAnnotations[i]; - if(containsWorkflowInstanceIdInputParamAnnotation(paramAnnotation)) { + if (containsWorkflowInstanceIdInputParamAnnotation(paramAnnotation)) { validateParameterForWorkflowInstanceId(parameters[i]); values[i] = task.getWorkflowInstanceId(); } else if (paramAnnotation.length > 0) { @@ -132,7 +140,7 @@ private Object[] getParameters(Task task, Class[] parameterTypes, Parameter[] Class parameterType = parameterTypes[i]; values[i] = getInputValue(task, parameterType, type, paramAnnotation); } else { - values[i] = om.convertValue(task.getInputData(), parameterTypes[i]); + values[i] = convertWithContext(task.getInputData(), parameterTypes[i], task); } } @@ -146,7 +154,7 @@ private boolean containsWorkflowInstanceIdInputParamAnnotation(Annotation[] anno } private void validateParameterForWorkflowInstanceId(Parameter parameter) { - if(!parameter.getType().equals(String.class)) { + if (!parameter.getType().equals(String.class)) { throw new IllegalArgumentException( "Parameter " + parameter + " is annotated with " + WorkflowInstanceIdInputParam.class.getSimpleName() + " but is not of type " + String.class.getSimpleName() + "."); @@ -158,7 +166,7 @@ private Object getInputValue( InputParam ip = findInputParamAnnotation(paramAnnotation); if (ip == null) { - return om.convertValue(task.getInputData(), parameterType); + return convertWithContext(task.getInputData(), parameterType, task); } final String name = ip.value(); @@ -167,6 +175,16 @@ private Object getInputValue( return null; } + if (parameterType == FileHandler.class) { + String fileHandleId = FileHandler.extractFileHandleId(value); + if (FileHandler.isFileHandleId(fileHandleId)) { + return new ManagedFileHandler(fileHandleId, task.getWorkflowFileClient()); + } + throw new FileStorageException( + "Expected " + FileHandler.PREFIX + + " reference for param '" + name + "', got: " + value); + } + if (List.class.isAssignableFrom(parameterType)) { List list = om.convertValue(value, List.class); if (type instanceof ParameterizedType) { @@ -174,7 +192,7 @@ private Object getInputValue( Class typeOfParameter = (Class) parameterizedType.getActualTypeArguments()[0]; List parameterizedList = new ArrayList<>(); for (Object item : list) { - parameterizedList.add(om.convertValue(item, typeOfParameter)); + parameterizedList.add(convertWithContext(item, typeOfParameter, task)); } return parameterizedList; @@ -182,7 +200,20 @@ private Object getInputValue( return list; } } else { - return om.convertValue(value, parameterType); + return convertWithContext(value, parameterType, task); + } + } + + private Object convertWithContext(Object source, Class targetType, Task task) { + JsonNode tree = om.valueToTree(source); + try (JsonParser parser = tree.traverse(om)) { + return om.readerFor(targetType) + .withAttribute( + FileHandlerDeserializer.WORKFLOW_FILE_CLIENT_ATTR, + task.getWorkflowFileClient()) + .readValue(parser); + } catch (IOException e) { + throw new RuntimeException("Failed to bind task input to " + targetType.getName(), e); } } @@ -201,6 +232,15 @@ private TaskResult setValue(Object invocationResult, TaskResult result) { return result; } + if (invocationResult instanceof FileHandler fh) { + OutputParam opAnn = + workerMethod.getAnnotatedReturnType().getAnnotation(OutputParam.class); + String key = opAnn != null ? opAnn.value() : "result"; + result.getOutputData().put(key, fh); + result.setStatus(TaskResult.Status.COMPLETED); + return result; + } + OutputParam opAnnotation = workerMethod.getAnnotatedReturnType().getAnnotation(OutputParam.class); if (opAnnotation != null) { diff --git a/conductor-client/src/main/java/io/orkes/conductor/client/OrkesClients.java b/conductor-client/src/main/java/io/orkes/conductor/client/OrkesClients.java index 700f652b8..dc8d9e5e2 100644 --- a/conductor-client/src/main/java/io/orkes/conductor/client/OrkesClients.java +++ b/conductor-client/src/main/java/io/orkes/conductor/client/OrkesClients.java @@ -12,6 +12,8 @@ */ package io.orkes.conductor.client; +import org.conductoross.conductor.client.FileClient; + import com.netflix.conductor.client.http.ConductorClient; import io.orkes.conductor.client.http.*; @@ -75,4 +77,8 @@ public EnvironmentClient getEnvironmentClient() { public OrkesSchemaClient getSchemaClient() { return new OrkesSchemaClient(client); } + + public FileClient getFileClient() { + return new FileClient(client); + } } diff --git a/conductor-client/src/main/java/org/conductoross/conductor/client/FileClient.java b/conductor-client/src/main/java/org/conductoross/conductor/client/FileClient.java new file mode 100644 index 000000000..a0acfe7c6 --- /dev/null +++ b/conductor-client/src/main/java/org/conductoross/conductor/client/FileClient.java @@ -0,0 +1,370 @@ +/* + * Copyright 2026 Conductor Authors. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package org.conductoross.conductor.client; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.util.ArrayList; +import java.util.EnumMap; +import java.util.List; +import java.util.Map; + +import org.conductoross.conductor.client.model.file.FileDownloadUrlResponse; +import org.conductoross.conductor.client.model.file.FileHandle; +import org.conductoross.conductor.client.model.file.FileUploadCompleteResponse; +import org.conductoross.conductor.client.model.file.FileUploadRequest; +import org.conductoross.conductor.client.model.file.FileUploadResponse; +import org.conductoross.conductor.client.model.file.FileUploadUrlResponse; +import org.conductoross.conductor.client.model.file.MultipartCompleteRequest; +import org.conductoross.conductor.client.model.file.MultipartInitResponse; +import org.conductoross.conductor.client.model.file.StorageType; +import org.conductoross.conductor.client.storage.AzureFileStorageBackend; +import org.conductoross.conductor.client.storage.GcsFileStorageBackend; +import org.conductoross.conductor.client.storage.LocalFileStorageBackend; +import org.conductoross.conductor.client.storage.S3FileStorageBackend; +import org.conductoross.conductor.sdk.file.FileHandler; +import org.conductoross.conductor.sdk.file.FileHandlerConverter; +import org.conductoross.conductor.sdk.file.FileStorageBackend; +import org.conductoross.conductor.sdk.file.FileStorageException; +import org.conductoross.conductor.sdk.file.FileUploadOptions; +import org.conductoross.conductor.sdk.file.FileUploader; +import org.conductoross.conductor.sdk.file.ManagedFileHandler; +import org.conductoross.conductor.sdk.file.WorkflowFileClient; + +import com.netflix.conductor.client.http.ConductorClient; +import com.netflix.conductor.client.http.ConductorClientRequest; +import com.netflix.conductor.client.http.ConductorClientRequest.Method; +import com.netflix.conductor.sdk.workflow.executor.task.TaskContext; + +import com.fasterxml.jackson.core.type.TypeReference; + +/** + * Client for the Conductor file-storage REST API. + * + *

Implements {@link FileUploader} for developer-facing uploads, and exposes SDK-internal + * methods used by {@link ManagedFileHandler} for lazy metadata fetch and download. Composes a + * {@code ConductorClient} for REST calls and a {@code Map} for + * byte transfer to the underlying storage backend. + * + *

On upload, the server reports which {@link StorageType} it is configured for; the client + * looks up the matching {@link FileStorageBackend} in the map. If no backend is registered for + * the server's type the upload fails fast with a {@link FileStorageException}. + * + *

REST paths use the bare {@code fileId} (no prefix) as the path variable. JSON bodies carry + * the prefixed {@code fileHandleId} ({@code conductor://file/}); conversion is handled + * via {@link FileHandler}. + */ +public class FileClient { + + private static final TypeReference FILE_UPLOAD_RESPONSE_TYPE = new TypeReference<>() { + }; + private static final TypeReference FILE_UPLOAD_URL_RESPONSE_TYPE = new TypeReference<>() { + }; + private static final TypeReference FILE_UPLOAD_COMPLETE_RESPONSE_TYPE = new TypeReference<>() { + }; + private static final TypeReference FILE_DOWNLOAD_URL_RESPONSE_TYPE = new TypeReference<>() { + }; + private static final TypeReference FILE_HANDLE_TYPE = new TypeReference<>() { + }; + private static final TypeReference MULTIPART_INIT_RESPONSE_TYPE = new TypeReference<>() { + }; + + private final ConductorClient client; + private final FileClientProperties properties; + private final Map fileStorageBackendsByStorageType; + + /** + * Creates a client with default {@link FileClientProperties} and the full set of built-in + * backends (LOCAL, S3, AZURE_BLOB, GCS). + */ + public FileClient(ConductorClient client) { + this(client, new FileClientProperties(), createDefaultFileStorageBackends()); + } + + /** + * Full-args constructor. + * + * @param properties {@code null} to use defaults + * @param fileStorageBackendsByStorageType {@code null} to use the built-in backends + */ + public FileClient(ConductorClient client, FileClientProperties properties, Map fileStorageBackendsByStorageType) { + this.client = client; + this.properties = properties == null ? new FileClientProperties() : properties; + this.fileStorageBackendsByStorageType = fileStorageBackendsByStorageType == null ? createDefaultFileStorageBackends() : fileStorageBackendsByStorageType; + } + + // --- FileUploader (developer-facing) --- + + /** + * Uploads a local file with the given {@link FileUploadOptions}. + * + *

{@code workflowId} is required; pass {@code null} only if you are prepared to see a + * {@link FileStorageException}. When called from inside a worker and {@code options.taskId} + * is null, {@code taskId} is auto-filled from the active {@link TaskContext}. + * + * @throws FileStorageException if {@code workflowId} is null, if no backend is registered for + * the server's storage type, or if any step of the upload fails + */ + public FileHandler upload(String workflowId, Path localFile, FileUploadOptions options) { + if (workflowId == null) { + throw new FileStorageException("workflowId is required"); + } + try { + fillDefaults(options, localFile); + long fileSize = Files.size(localFile); + FileUploadRequest request = FileHandlerConverter.toFileUploadRequest(workflowId, options); + + FileUploadResponse response = createFileOnServer(request); + + FileStorageBackend storageBackend = fileStorageBackendsByStorageType.get(response.getStorageType()); + if (storageBackend == null) { + throw new FileStorageException("Server uses " + response.getStorageType() + + " but SDK only supports: " + fileStorageBackendsByStorageType.keySet()); + } + + if (options.isMultipart() && storageBackend.hasMultipartSupport()) { + uploadMultipart(response.getFileHandleId(), response.getStorageType(), localFile); + } else { + storageBackend.upload(response.getUploadUrl(), localFile); + confirmUpload(response.getFileHandleId()); + } + + return FileHandlerConverter.toManagedFileHandler(response, new WorkflowFileClient(this, workflowId), localFile, fileSize); + } catch (IOException e) { + throw new FileStorageException("Upload failed for: " + localFile, e); + } + } + + /** + * Buffers the stream to a temp file, then delegates to {@link #upload(String, Path, FileUploadOptions)}. + */ + public FileHandler upload(String workflowId, InputStream inputStream, FileUploadOptions options) { + try { + Path temp = Files.createTempFile("conductor-upload-", ".tmp"); + Files.copy(inputStream, temp, StandardCopyOption.REPLACE_EXISTING); + return upload(workflowId, temp, options); + } catch (IOException e) { + throw new FileStorageException("Failed to write InputStream to temp file", e); + } + } + + public FileHandler upload(String workflowId, Path localFile) { + return upload(workflowId, localFile, new FileUploadOptions()); + } + + public FileHandler upload(String workflowId, InputStream inputStream) { + return upload(workflowId, inputStream, new FileUploadOptions()); + } + + /** + * Populates unset options from the local file and active {@link TaskContext}. Mutates + * {@code options} in place — SDK-internal helper only. + */ + private static void fillDefaults(FileUploadOptions options, Path localFile) { + if (options.getFileName() == null) { + options.setFileName(localFile.getFileName().toString()); + } + if (options.getContentType() == null) { + options.setContentType("application/octet-stream"); + } + if (options.getTaskId() == null) { + TaskContext ctx = TaskContext.get(); + if (ctx != null) { + options.setTaskId(ctx.getTaskId()); + } + } + } + + // --- SDK-internal (public for cross-package access, not developer-facing) --- + + /** + * Downloads the content for a file to {@code destination}. Fetches a fresh presigned + * download URL from the server on each call and delegates the byte transfer to the + * {@link FileStorageBackend} registered for {@code storageType}. + */ + public void download(String workflowId, String fileHandleId, StorageType storageType, Path destination) { + FileDownloadUrlResponse urlResponse = getDownloadUrl(workflowId, fileHandleId); + try { + Files.createDirectories(destination.getParent()); + } catch (IOException e) { + throw new FileStorageException("Failed to create cache directory", e); + } + fileStorageBackendsByStorageType.get(storageType).download(urlResponse.getDownloadUrl(), destination); + } + + /** + * Fetches the {@link FileHandle} metadata via {@code GET /api/files/{fileId}}. + */ + public FileHandle getMetadata(String fileHandleId) { + ConductorClientRequest request = ConductorClientRequest.builder() + .method(Method.GET) + .path("/files/{fileId}") + .addPathParam("fileId", FileHandler.toFileId(fileHandleId)) + .build(); + return client.execute(request, FILE_HANDLE_TYPE).getData(); + } + + public int getRetryCount() { + return properties.getRetryCount(); + } + + public String getCacheDirectory() { + return properties.getLocalCacheDirectory(); + } + + /** + * Confirms that byte transfer is complete via + * {@code POST /api/files/{fileId}/upload-complete}. The server verifies the object on the + * backend, reads {@code contentHash} and actual size, and transitions the record to + * {@code UPLOADED}. + */ + void confirmUpload(String fileHandleId) { + ConductorClientRequest request = ConductorClientRequest.builder() + .method(Method.POST) + .path("/files/{fileId}/upload-complete") + .addPathParam("fileId", FileHandler.toFileId(fileHandleId)) + .build(); + client.execute(request, FILE_UPLOAD_COMPLETE_RESPONSE_TYPE); + } + + // --- Private --- + + private FileUploadResponse createFileOnServer(FileUploadRequest uploadRequest) { + ConductorClientRequest request = ConductorClientRequest.builder() + .method(Method.POST) + .path("/files") + .body(uploadRequest) + .build(); + return client.execute(request, FILE_UPLOAD_RESPONSE_TYPE).getData(); + } + + private FileDownloadUrlResponse getDownloadUrl(String workflowId, String fileHandleId) { + if (workflowId == null) { + throw new FileStorageException("WorkflowInstanceId is null"); + } + ConductorClientRequest request = ConductorClientRequest.builder() + .method(Method.GET) + .path("/files/{workflowId}/{fileId}/download-url") + .addPathParam("workflowId", workflowId) + .addPathParam("fileId", FileHandler.toFileId(fileHandleId)) + .build(); + return client.execute(request, FILE_DOWNLOAD_URL_RESPONSE_TYPE).getData(); + } + + private void uploadMultipart(String fileHandleId, StorageType storageType, Path localFile) { + try { + MultipartInitResponse init = initiateMultipartUpload(fileHandleId); + String uploadId = init.getUploadId(); + long partSize = properties.getMultipartPartSize(); + long fileSize = Files.size(localFile); + int totalParts = (int) Math.ceil((double) fileSize / partSize); + + List partETags = new ArrayList<>(); + + for (int part = 1; part <= totalParts; part++) { + long offset = (long) (part - 1) * partSize; + long length = Math.min(partSize, fileSize - offset); + + String url = getPartUploadUrl(fileHandleId, uploadId, part); + String etag = fileStorageBackendsByStorageType.get(storageType).uploadPart(url, localFile, offset, length); + partETags.add(etag); + } + + completeMultipartUpload(fileHandleId, uploadId, partETags); + } catch (IOException e) { + throw new FileStorageException("Multipart upload failed for: " + fileHandleId, e); + } + } + + private MultipartInitResponse initiateMultipartUpload(String fileHandleId) { + ConductorClientRequest request = ConductorClientRequest.builder() + .method(Method.POST) + .path("/files/{fileId}/multipart") + .addPathParam("fileId", FileHandler.toFileId(fileHandleId)) + .build(); + return client.execute(request, MULTIPART_INIT_RESPONSE_TYPE).getData(); + } + + private String getPartUploadUrl(String fileHandleId, String uploadId, int partNumber) { + ConductorClientRequest request = ConductorClientRequest.builder() + .method(Method.GET) + .path("/files/{fileId}/multipart/{uploadId}/part/{partNumber}") + .addPathParam("fileId", FileHandler.toFileId(fileHandleId)) + .addPathParam("uploadId", uploadId) + .addPathParam("partNumber", String.valueOf(partNumber)) + .build(); + return client.execute(request, FILE_UPLOAD_URL_RESPONSE_TYPE).getData().getUploadUrl(); + } + + private void completeMultipartUpload(String fileHandleId, String uploadId, + List partETags) { + MultipartCompleteRequest body = new MultipartCompleteRequest(); + body.setPartETags(partETags); + ConductorClientRequest request = ConductorClientRequest.builder() + .method(Method.POST) + .path("/files/{fileId}/multipart/{uploadId}/complete") + .addPathParam("fileId", FileHandler.toFileId(fileHandleId)) + .addPathParam("uploadId", uploadId) + .body(body) + .build(); + client.execute(request, FILE_UPLOAD_COMPLETE_RESPONSE_TYPE); + } + + private static Map createDefaultFileStorageBackends() { + return Map.ofEntries( + Map.entry(StorageType.LOCAL, new LocalFileStorageBackend()), + Map.entry(StorageType.S3, new S3FileStorageBackend()), + Map.entry(StorageType.AZURE_BLOB, new AzureFileStorageBackend()), + Map.entry(StorageType.GCS, new GcsFileStorageBackend()) + ); + } + + public static Builder builder(ConductorClient client) { + return new Builder(client); + } + + /** + * Builder for configuring a {@link FileClient}. The built-in backends (LOCAL, S3, AZURE_BLOB, + * GCS) are always included; a backend registered via {@link #addStorageBackend} overrides + * the built-in backend for the same {@link StorageType}. + */ + public static class Builder { + + private final ConductorClient client; + private FileClientProperties properties; + private final Map backends = new EnumMap<>(StorageType.class); + + private Builder(ConductorClient client) { + this.client = client; + } + + public Builder properties(FileClientProperties properties) { + this.properties = properties; + return this; + } + + public Builder addStorageBackend(FileStorageBackend backend) { + this.backends.put(backend.getStorageType(), backend); + return this; + } + + public FileClient build() { + Map resolved = new EnumMap<>(createDefaultFileStorageBackends()); + resolved.putAll(backends); + return new FileClient(client, properties, Map.copyOf(resolved)); + } + } +} diff --git a/conductor-client/src/main/java/org/conductoross/conductor/client/FileClientProperties.java b/conductor-client/src/main/java/org/conductoross/conductor/client/FileClientProperties.java new file mode 100644 index 000000000..32685a75e --- /dev/null +++ b/conductor-client/src/main/java/org/conductoross/conductor/client/FileClientProperties.java @@ -0,0 +1,64 @@ +/* + * Copyright 2026 Conductor Authors. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package org.conductoross.conductor.client; + +import java.nio.file.Path; + +/** + * Configuration for {@link FileClient}. Populated from {@code conductor.file-client.*} + * properties when using Spring auto-configuration. + */ +public class FileClientProperties { + + /** Upload and download retry count on transient failures. Default: 3. */ + private int retryCount = 3; + + /** + * Local directory under which downloaded content is cached. Default: + * {@code ${java.io.tmpdir}/conductor/files-cache}. Files are never deleted — by design + * the SDK does not clean up cached content. + */ + private String localCacheDirectory = + Path.of(getTempDirectory(), "conductor", "files-cache").toString(); + + /** Multipart upload part size in bytes. Default: 10 MiB (S3 minimum). */ + private long multipartPartSize = 10L * 1024 * 1024; + + public int getRetryCount() { + return retryCount; + } + + public void setRetryCount(int retryCount) { + this.retryCount = retryCount; + } + + public String getLocalCacheDirectory() { + return localCacheDirectory; + } + + public void setLocalCacheDirectory(String localCacheDirectory) { + this.localCacheDirectory = localCacheDirectory; + } + + public long getMultipartPartSize() { + return multipartPartSize; + } + + public void setMultipartPartSize(long multipartPartSize) { + this.multipartPartSize = multipartPartSize; + } + + private static String getTempDirectory() { + return System.getProperty("java.io.tmpdir"); + } +} diff --git a/conductor-client/src/main/java/org/conductoross/conductor/client/model/file/FileDownloadUrlResponse.java b/conductor-client/src/main/java/org/conductoross/conductor/client/model/file/FileDownloadUrlResponse.java new file mode 100644 index 000000000..9e74cefcc --- /dev/null +++ b/conductor-client/src/main/java/org/conductoross/conductor/client/model/file/FileDownloadUrlResponse.java @@ -0,0 +1,35 @@ +/* + * Copyright 2026 Conductor Authors. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package org.conductoross.conductor.client.model.file; + +/** + * Response to {@code GET /api/files/{fileId}/download-url} — a freshly issued presigned + * download URL. Requires the underlying file to be in {@link FileUploadStatus#UPLOADED}. + */ +public class FileDownloadUrlResponse { + + /** Prefixed handle: {@code conductor://file/}. */ + private String fileHandleId; + private String downloadUrl; + /** Epoch millis. */ + private long expiresAt; + + public String getFileHandleId() { return fileHandleId; } + public void setFileHandleId(String fileHandleId) { this.fileHandleId = fileHandleId; } + + public String getDownloadUrl() { return downloadUrl; } + public void setDownloadUrl(String downloadUrl) { this.downloadUrl = downloadUrl; } + + public long getExpiresAt() { return expiresAt; } + public void setExpiresAt(long expiresAt) { this.expiresAt = expiresAt; } +} diff --git a/conductor-client/src/main/java/org/conductoross/conductor/client/model/file/FileHandle.java b/conductor-client/src/main/java/org/conductoross/conductor/client/model/file/FileHandle.java new file mode 100644 index 000000000..b29dcbc91 --- /dev/null +++ b/conductor-client/src/main/java/org/conductoross/conductor/client/model/file/FileHandle.java @@ -0,0 +1,72 @@ +/* + * Copyright 2026 Conductor Authors. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package org.conductoross.conductor.client.model.file; + +/** + * Client-side mirror of the server's {@code FileHandle} — full file metadata returned by + * {@code GET /api/files/{fileId}}. Does not expose the server-internal {@code storagePath}. + */ +public class FileHandle { + + /** Prefixed handle: {@code conductor://file/}. */ + private String fileHandleId; + private String fileName; + private String contentType; + private long fileSize; + /** + * Content hash from the storage provider (S3 ETag / Azure Content-MD5 / GCS md5Hash). + * {@code null} for the local backend and before upload is confirmed. + */ + private String contentHash; + private StorageType storageType; + private FileUploadStatus uploadStatus; + private String workflowId; + private String taskId; + /** Epoch millis. */ + private long createdAt; + /** Epoch millis. */ + private long updatedAt; + + public String getFileHandleId() { return fileHandleId; } + public void setFileHandleId(String fileHandleId) { this.fileHandleId = fileHandleId; } + + public String getFileName() { return fileName; } + public void setFileName(String fileName) { this.fileName = fileName; } + + public String getContentType() { return contentType; } + public void setContentType(String contentType) { this.contentType = contentType; } + + public long getFileSize() { return fileSize; } + public void setFileSize(long fileSize) { this.fileSize = fileSize; } + + public String getContentHash() { return contentHash; } + public void setContentHash(String contentHash) { this.contentHash = contentHash; } + + public StorageType getStorageType() { return storageType; } + public void setStorageType(StorageType storageType) { this.storageType = storageType; } + + public FileUploadStatus getUploadStatus() { return uploadStatus; } + public void setUploadStatus(FileUploadStatus uploadStatus) { this.uploadStatus = uploadStatus; } + + public String getWorkflowId() { return workflowId; } + public void setWorkflowId(String workflowId) { this.workflowId = workflowId; } + + public String getTaskId() { return taskId; } + public void setTaskId(String taskId) { this.taskId = taskId; } + + public long getCreatedAt() { return createdAt; } + public void setCreatedAt(long createdAt) { this.createdAt = createdAt; } + + public long getUpdatedAt() { return updatedAt; } + public void setUpdatedAt(long updatedAt) { this.updatedAt = updatedAt; } +} diff --git a/conductor-client/src/main/java/org/conductoross/conductor/client/model/file/FileUploadCompleteResponse.java b/conductor-client/src/main/java/org/conductoross/conductor/client/model/file/FileUploadCompleteResponse.java new file mode 100644 index 000000000..16069de77 --- /dev/null +++ b/conductor-client/src/main/java/org/conductoross/conductor/client/model/file/FileUploadCompleteResponse.java @@ -0,0 +1,36 @@ +/* + * Copyright 2026 Conductor Authors. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package org.conductoross.conductor.client.model.file; + +/** + * Response to {@code POST /api/files/{fileId}/upload-complete}. Status is + * {@link FileUploadStatus#UPLOADED} on success; {@code contentHash} is the backend-reported + * hash (or {@code null} for backends that do not expose one). + */ +public class FileUploadCompleteResponse { + + /** Prefixed handle: {@code conductor://file/}. */ + private String fileHandleId; + private FileUploadStatus uploadStatus; + /** Content hash from the storage provider; {@code null} for local backend. */ + private String contentHash; + + public String getFileHandleId() { return fileHandleId; } + public void setFileHandleId(String fileHandleId) { this.fileHandleId = fileHandleId; } + + public FileUploadStatus getUploadStatus() { return uploadStatus; } + public void setUploadStatus(FileUploadStatus uploadStatus) { this.uploadStatus = uploadStatus; } + + public String getContentHash() { return contentHash; } + public void setContentHash(String contentHash) { this.contentHash = contentHash; } +} diff --git a/conductor-client/src/main/java/org/conductoross/conductor/client/model/file/FileUploadRequest.java b/conductor-client/src/main/java/org/conductoross/conductor/client/model/file/FileUploadRequest.java new file mode 100644 index 000000000..3f4bdf7f6 --- /dev/null +++ b/conductor-client/src/main/java/org/conductoross/conductor/client/model/file/FileUploadRequest.java @@ -0,0 +1,37 @@ +/* + * Copyright 2026 Conductor Authors. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package org.conductoross.conductor.client.model.file; + +/** + * Payload for {@code POST /api/files}. Describes the file the client intends to upload and + * optionally the owning {@code workflowId} / {@code taskId}. + */ +public class FileUploadRequest { + + private String fileName; + private String contentType; + private String workflowId; + private String taskId; + + public String getFileName() { return fileName; } + public void setFileName(String fileName) { this.fileName = fileName; } + + public String getContentType() { return contentType; } + public void setContentType(String contentType) { this.contentType = contentType; } + + public String getWorkflowId() { return workflowId; } + public void setWorkflowId(String workflowId) { this.workflowId = workflowId; } + + public String getTaskId() { return taskId; } + public void setTaskId(String taskId) { this.taskId = taskId; } +} diff --git a/conductor-client/src/main/java/org/conductoross/conductor/client/model/file/FileUploadResponse.java b/conductor-client/src/main/java/org/conductoross/conductor/client/model/file/FileUploadResponse.java new file mode 100644 index 000000000..5bc93a965 --- /dev/null +++ b/conductor-client/src/main/java/org/conductoross/conductor/client/model/file/FileUploadResponse.java @@ -0,0 +1,57 @@ +/* + * Copyright 2026 Conductor Authors. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package org.conductoross.conductor.client.model.file; + +/** + * Response to {@code POST /api/files}. Carries the newly assigned {@code fileHandleId} plus a + * presigned upload URL and its expiry. Status is {@link FileUploadStatus#UPLOADING} at this + * point; the client confirms completion via {@code POST /api/files/{fileId}/upload-complete}. + */ +public class FileUploadResponse { + + /** Prefixed handle: {@code conductor://file/}. */ + private String fileHandleId; + private String fileName; + private String contentType; + private StorageType storageType; + private FileUploadStatus uploadStatus; + private String uploadUrl; + /** Epoch millis. */ + private long uploadUrlExpiresAt; + /** Epoch millis. */ + private long createdAt; + + public String getFileHandleId() { return fileHandleId; } + public void setFileHandleId(String fileHandleId) { this.fileHandleId = fileHandleId; } + + public String getFileName() { return fileName; } + public void setFileName(String fileName) { this.fileName = fileName; } + + public String getContentType() { return contentType; } + public void setContentType(String contentType) { this.contentType = contentType; } + + public StorageType getStorageType() { return storageType; } + public void setStorageType(StorageType storageType) { this.storageType = storageType; } + + public FileUploadStatus getUploadStatus() { return uploadStatus; } + public void setUploadStatus(FileUploadStatus uploadStatus) { this.uploadStatus = uploadStatus; } + + public String getUploadUrl() { return uploadUrl; } + public void setUploadUrl(String uploadUrl) { this.uploadUrl = uploadUrl; } + + public long getUploadUrlExpiresAt() { return uploadUrlExpiresAt; } + public void setUploadUrlExpiresAt(long uploadUrlExpiresAt) { this.uploadUrlExpiresAt = uploadUrlExpiresAt; } + + public long getCreatedAt() { return createdAt; } + public void setCreatedAt(long createdAt) { this.createdAt = createdAt; } +} diff --git a/conductor-client/src/main/java/org/conductoross/conductor/client/model/file/FileUploadStatus.java b/conductor-client/src/main/java/org/conductoross/conductor/client/model/file/FileUploadStatus.java new file mode 100644 index 000000000..b111138e2 --- /dev/null +++ b/conductor-client/src/main/java/org/conductoross/conductor/client/model/file/FileUploadStatus.java @@ -0,0 +1,28 @@ +/* + * Copyright 2026 Conductor Authors. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package org.conductoross.conductor.client.model.file; + +/** Server-authoritative upload lifecycle state. Persisted on the server and mirrored in DTOs. */ +public enum FileUploadStatus { + /** Reserved for future use (e.g. pre-initialized records); not entered by the current flow. */ + PENDING, + /** Record created; byte transfer not yet confirmed by {@code POST /upload-complete}. */ + UPLOADING, + /** Terminal success state — content verified present on the storage backend. */ + UPLOADED, + /** + * Terminal failure — set by the background audit when an {@link #UPLOADING} record remains + * stale past the configured threshold. + */ + FAILED +} diff --git a/conductor-client/src/main/java/org/conductoross/conductor/client/model/file/FileUploadUrlResponse.java b/conductor-client/src/main/java/org/conductoross/conductor/client/model/file/FileUploadUrlResponse.java new file mode 100644 index 000000000..925b5c8f1 --- /dev/null +++ b/conductor-client/src/main/java/org/conductoross/conductor/client/model/file/FileUploadUrlResponse.java @@ -0,0 +1,35 @@ +/* + * Copyright 2026 Conductor Authors. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package org.conductoross.conductor.client.model.file; + +/** + * Response to {@code GET /api/files/{fileId}/upload-url} — a freshly issued presigned upload + * URL, used on retry when the original URL from {@code FileUploadResponse} has expired. + */ +public class FileUploadUrlResponse { + + /** Prefixed handle: {@code conductor://file/}. */ + private String fileHandleId; + private String uploadUrl; + /** Epoch millis. */ + private long expiresAt; + + public String getFileHandleId() { return fileHandleId; } + public void setFileHandleId(String fileHandleId) { this.fileHandleId = fileHandleId; } + + public String getUploadUrl() { return uploadUrl; } + public void setUploadUrl(String uploadUrl) { this.uploadUrl = uploadUrl; } + + public long getExpiresAt() { return expiresAt; } + public void setExpiresAt(long expiresAt) { this.expiresAt = expiresAt; } +} diff --git a/conductor-client/src/main/java/org/conductoross/conductor/client/model/file/MultipartCompleteRequest.java b/conductor-client/src/main/java/org/conductoross/conductor/client/model/file/MultipartCompleteRequest.java new file mode 100644 index 000000000..dab2b5828 --- /dev/null +++ b/conductor-client/src/main/java/org/conductoross/conductor/client/model/file/MultipartCompleteRequest.java @@ -0,0 +1,30 @@ +/* + * Copyright 2026 Conductor Authors. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package org.conductoross.conductor.client.model.file; + +import java.util.List; + +import org.conductoross.conductor.sdk.file.FileStorageBackend; + +/** + * Payload for {@code POST /api/files/{fileId}/multipart/{uploadId}/complete}. {@code partETags} + * is the ordered list of ETags (or backend-equivalent identifiers) returned by each + * {@link FileStorageBackend#uploadPart uploadPart} call. + */ +public class MultipartCompleteRequest { + + private List partETags; + + public List getPartETags() { return partETags; } + public void setPartETags(List partETags) { this.partETags = partETags; } +} diff --git a/conductor-client/src/main/java/org/conductoross/conductor/client/model/file/MultipartInitResponse.java b/conductor-client/src/main/java/org/conductoross/conductor/client/model/file/MultipartInitResponse.java new file mode 100644 index 000000000..e54a3dc27 --- /dev/null +++ b/conductor-client/src/main/java/org/conductoross/conductor/client/model/file/MultipartInitResponse.java @@ -0,0 +1,28 @@ +/* + * Copyright 2026 Conductor Authors. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package org.conductoross.conductor.client.model.file; + +/** Response to {@code POST /api/files/{fileId}/multipart} — initiates a multipart upload. */ +public class MultipartInitResponse { + + /** Prefixed handle: {@code conductor://file/}. */ + private String fileHandleId; + /** Backend-specific multipart identifier (S3 {@code UploadId}, GCS resumable session ID). */ + private String uploadId; + + public String getFileHandleId() { return fileHandleId; } + public void setFileHandleId(String fileHandleId) { this.fileHandleId = fileHandleId; } + + public String getUploadId() { return uploadId; } + public void setUploadId(String uploadId) { this.uploadId = uploadId; } +} diff --git a/conductor-client/src/main/java/org/conductoross/conductor/client/model/file/StorageType.java b/conductor-client/src/main/java/org/conductoross/conductor/client/model/file/StorageType.java new file mode 100644 index 000000000..824de6718 --- /dev/null +++ b/conductor-client/src/main/java/org/conductoross/conductor/client/model/file/StorageType.java @@ -0,0 +1,29 @@ +/* + * Copyright 2026 Conductor Authors. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package org.conductoross.conductor.client.model.file; + +/** + * Storage backend identifier. Shared vocabulary between server and SDK — the server stamps its + * configured type onto every file in {@code FileUploadResponse} and {@code FileHandle}; the SDK + * selects a matching {@code FileStorageBackend} for byte transfer. + */ +public enum StorageType { + /** AWS S3 (and S3-compatible services such as MinIO). */ + S3, + /** Azure Blob Storage. */ + AZURE_BLOB, + /** Google Cloud Storage. */ + GCS, + /** Server-local filesystem. Does not support multipart. */ + LOCAL +} diff --git a/conductor-client/src/main/java/org/conductoross/conductor/client/storage/AzureFileStorageBackend.java b/conductor-client/src/main/java/org/conductoross/conductor/client/storage/AzureFileStorageBackend.java new file mode 100644 index 000000000..21e987916 --- /dev/null +++ b/conductor-client/src/main/java/org/conductoross/conductor/client/storage/AzureFileStorageBackend.java @@ -0,0 +1,110 @@ +/* + * Copyright 2026 Conductor Authors. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package org.conductoross.conductor.client.storage; + +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; + +import org.conductoross.conductor.client.model.file.StorageType; +import org.conductoross.conductor.sdk.file.FileStorageBackend; +import org.conductoross.conductor.sdk.file.FileStorageException; + +import okhttp3.Request; +import okhttp3.RequestBody; +import okhttp3.Response; + +/** {@link FileStorageBackend} for Azure Blob Storage. Supports multipart via resumable block uploads. */ +public class AzureFileStorageBackend implements FileStorageBackend { + + private static final String BLOB_TYPE_HEADER = "x-ms-blob-type"; + private static final String BLOB_TYPE_VALUE = "BlockBlob"; + private static final String X_MS_CONTENT_MD_5 = "x-ms-content-md5"; + + @Override + public StorageType getStorageType() { return StorageType.AZURE_BLOB; } + + @Override + public void upload(String url, Path localFile) { + Request request = new Request.Builder() + .url(url) + .header(BLOB_TYPE_HEADER, BLOB_TYPE_VALUE) + .put(RequestBody.create(localFile.toFile(), null)) + .build(); + try (Response response = HttpBodies.CLIENT.newCall(request).execute()) { + if (!response.isSuccessful()) { + throw new FileStorageException("Azure upload failed with status: " + response.code()); + } + } catch (FileStorageException e) { + throw e; + } catch (Exception e) { + throw new FileStorageException("Azure upload failed: " + url, e); + } + } + + @Override + public void upload(String url, InputStream inputStream, long contentLength) { + Request request = new Request.Builder() + .url(url) + .header(BLOB_TYPE_HEADER, BLOB_TYPE_VALUE) + .put(HttpBodies.stream(inputStream, contentLength)) + .build(); + try (Response response = HttpBodies.CLIENT.newCall(request).execute()) { + if (!response.isSuccessful()) { + throw new FileStorageException("Azure stream upload failed with status: " + response.code()); + } + } catch (FileStorageException e) { + throw e; + } catch (Exception e) { + throw new FileStorageException("Azure stream upload failed: " + url, e); + } + } + + @Override + public void download(String url, Path destination) { + Request request = new Request.Builder().url(url).get().build(); + try (Response response = HttpBodies.CLIENT.newCall(request).execute()) { + if (!response.isSuccessful()) { + throw new FileStorageException("Azure download failed with status: " + response.code()); + } + Files.createDirectories(destination.getParent()); + try (InputStream in = response.body().byteStream()) { + Files.copy(in, destination, StandardCopyOption.REPLACE_EXISTING); + } + } catch (FileStorageException e) { + throw e; + } catch (Exception e) { + throw new FileStorageException("Azure download failed: " + url, e); + } + } + + @Override + public String uploadPart(String url, Path localFile, long offset, long length) { + Request request = new Request.Builder() + .url(url) + .header(BLOB_TYPE_HEADER, BLOB_TYPE_VALUE) + .put(HttpBodies.range(localFile, offset, length)) + .build(); + try (Response response = HttpBodies.CLIENT.newCall(request).execute()) { + if (!response.isSuccessful()) { + throw new FileStorageException("Azure part upload failed with status: " + response.code()); + } + return response.header(X_MS_CONTENT_MD_5); + } catch (FileStorageException e) { + throw e; + } catch (Exception e) { + throw new FileStorageException("Azure part upload failed: " + url, e); + } + } +} diff --git a/conductor-client/src/main/java/org/conductoross/conductor/client/storage/GcsFileStorageBackend.java b/conductor-client/src/main/java/org/conductoross/conductor/client/storage/GcsFileStorageBackend.java new file mode 100644 index 000000000..fc0c05589 --- /dev/null +++ b/conductor-client/src/main/java/org/conductoross/conductor/client/storage/GcsFileStorageBackend.java @@ -0,0 +1,96 @@ +/* + * Copyright 2026 Conductor Authors. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package org.conductoross.conductor.client.storage; + +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; + +import org.conductoross.conductor.client.model.file.StorageType; +import org.conductoross.conductor.sdk.file.FileStorageBackend; +import org.conductoross.conductor.sdk.file.FileStorageException; + +import okhttp3.Request; +import okhttp3.RequestBody; +import okhttp3.Response; + +/** {@link FileStorageBackend} for Google Cloud Storage. Supports multipart via resumable sessions. */ +public class GcsFileStorageBackend implements FileStorageBackend { + + @Override + public StorageType getStorageType() { return StorageType.GCS; } + + @Override + public void upload(String url, Path localFile) { + Request request = new Request.Builder() + .url(url) + .put(RequestBody.create(localFile.toFile(), null)) + .build(); + try (Response response = HttpBodies.CLIENT.newCall(request).execute()) { + if (!response.isSuccessful()) { + throw new FileStorageException("GCS upload failed with status: " + response.code()); + } + } catch (FileStorageException e) { + throw e; + } catch (Exception e) { + throw new FileStorageException("GCS upload failed: " + url, e); + } + } + + @Override + public void upload(String url, InputStream inputStream, long contentLength) { + Request request = new Request.Builder() + .url(url) + .put(HttpBodies.stream(inputStream, contentLength)) + .build(); + try (Response response = HttpBodies.CLIENT.newCall(request).execute()) { + if (!response.isSuccessful()) { + throw new FileStorageException("GCS upload failed with status: " + response.code()); + } + } catch (FileStorageException e) { + throw e; + } catch (Exception e) { + throw new FileStorageException("GCS stream upload failed: " + url, e); + } + } + + @Override + public void download(String url, Path destination) { + Request request = new Request.Builder().url(url).get().build(); + try (Response response = HttpBodies.CLIENT.newCall(request).execute()) { + if (!response.isSuccessful()) { + throw new FileStorageException("GCS download failed with status: " + response.code()); + } + Files.createDirectories(destination.getParent()); + try (InputStream in = response.body().byteStream()) { + Files.copy(in, destination, StandardCopyOption.REPLACE_EXISTING); + } + } catch (FileStorageException e) { + throw e; + } catch (Exception e) { + throw new FileStorageException("GCS download failed: " + url, e); + } + } + + @Override + public String uploadPart(String url, Path localFile, long offset, long length) { + // Unsupported by design + throw new UnsupportedOperationException("Not supported."); + } + + @Override + public boolean hasMultipartSupport() { + return false; + } +} diff --git a/conductor-client/src/main/java/org/conductoross/conductor/client/storage/HttpBodies.java b/conductor-client/src/main/java/org/conductoross/conductor/client/storage/HttpBodies.java new file mode 100644 index 000000000..eb00706f5 --- /dev/null +++ b/conductor-client/src/main/java/org/conductoross/conductor/client/storage/HttpBodies.java @@ -0,0 +1,69 @@ +/* + * Copyright 2026 Conductor Authors. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package org.conductoross.conductor.client.storage; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; + +import okhttp3.MediaType; +import okhttp3.OkHttpClient; +import okhttp3.RequestBody; +import okio.BufferedSink; +import okio.Okio; +import okio.Source; + +final class HttpBodies { + + static final OkHttpClient CLIENT = new OkHttpClient(); + + private HttpBodies() {} + + static RequestBody stream(InputStream inputStream, long contentLength) { + return new RequestBody() { + @Override public MediaType contentType() { return null; } + @Override public long contentLength() { return contentLength; } + @Override public void writeTo(BufferedSink sink) throws IOException { + try (Source source = Okio.source(inputStream)) { + sink.writeAll(source); + } + } + }; + } + + static RequestBody range(Path file, long offset, long length) { + return new RequestBody() { + @Override public MediaType contentType() { return null; } + @Override public long contentLength() { return length; } + @Override public void writeTo(BufferedSink sink) throws IOException { + try (InputStream in = Files.newInputStream(file)) { + long skipped = 0; + while (skipped < offset) { + long s = in.skip(offset - skipped); + if (s <= 0) break; + skipped += s; + } + byte[] buf = new byte[8192]; + long remaining = length; + while (remaining > 0) { + int n = in.read(buf, 0, (int) Math.min(buf.length, remaining)); + if (n < 0) break; + sink.write(buf, 0, n); + remaining -= n; + } + } + } + }; + } +} diff --git a/conductor-client/src/main/java/org/conductoross/conductor/client/storage/LocalFileStorageBackend.java b/conductor-client/src/main/java/org/conductoross/conductor/client/storage/LocalFileStorageBackend.java new file mode 100644 index 000000000..81c078bcb --- /dev/null +++ b/conductor-client/src/main/java/org/conductoross/conductor/client/storage/LocalFileStorageBackend.java @@ -0,0 +1,80 @@ +/* + * Copyright 2026 Conductor Authors. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package org.conductoross.conductor.client.storage; + +import java.io.IOException; +import java.io.InputStream; +import java.net.URI; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; + +import org.conductoross.conductor.client.model.file.StorageType; +import org.conductoross.conductor.sdk.file.FileStorageBackend; +import org.conductoross.conductor.sdk.file.FileStorageException; + +/** + * {@link FileStorageBackend} for the server-local filesystem (development / zero-infra mode). + * Does NOT support multipart — {@link #hasMultipartSupport()} returns {@code false}. + */ +public class LocalFileStorageBackend implements FileStorageBackend { + + @Override + public StorageType getStorageType() { + return StorageType.LOCAL; + } + + @Override + public void upload(String url, Path localFile) { + try { + Path dest = Path.of(URI.create(url)); + Files.createDirectories(dest.getParent()); + Files.copy(localFile, dest, StandardCopyOption.REPLACE_EXISTING); + } catch (IOException e) { + throw new FileStorageException("Local upload failed: " + localFile + " → " + url, e); + } + } + + @Override + public void upload(String url, InputStream inputStream, long contentLength) { + try { + Path dest = Path.of(URI.create(url)); + Files.createDirectories(dest.getParent()); + Files.copy(inputStream, dest, StandardCopyOption.REPLACE_EXISTING); + } catch (IOException e) { + throw new FileStorageException("Local upload from stream failed: " + url, e); + } + } + + @Override + public void download(String url, Path destination) { + try { + Path source = Path.of(URI.create(url)); + Files.createDirectories(destination.getParent()); + Files.copy(source, destination, StandardCopyOption.REPLACE_EXISTING); + } catch (IOException e) { + throw new FileStorageException("Local download failed: " + url, e); + } + } + + @Override + public String uploadPart(String url, Path localFile, long offset, long length) { + // Unsupported by design + throw new UnsupportedOperationException("Not supported."); + } + + @Override + public boolean hasMultipartSupport() { + return false; + } +} diff --git a/conductor-client/src/main/java/org/conductoross/conductor/client/storage/S3FileStorageBackend.java b/conductor-client/src/main/java/org/conductoross/conductor/client/storage/S3FileStorageBackend.java new file mode 100644 index 000000000..66b8602bd --- /dev/null +++ b/conductor-client/src/main/java/org/conductoross/conductor/client/storage/S3FileStorageBackend.java @@ -0,0 +1,105 @@ +/* + * Copyright 2026 Conductor Authors. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package org.conductoross.conductor.client.storage; + +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; + +import org.conductoross.conductor.client.model.file.StorageType; +import org.conductoross.conductor.sdk.file.FileStorageBackend; +import org.conductoross.conductor.sdk.file.FileStorageException; + +import okhttp3.Request; +import okhttp3.RequestBody; +import okhttp3.Response; + +/** {@link FileStorageBackend} for AWS S3 and S3-compatible services (e.g. MinIO). Supports multipart. */ +public class S3FileStorageBackend implements FileStorageBackend { + + private static final String E_TAG = "ETag"; + + @Override + public StorageType getStorageType() { return StorageType.S3; } + + @Override + public void upload(String url, Path localFile) { + Request request = new Request.Builder() + .url(url) + .put(RequestBody.create(localFile.toFile(), null)) + .build(); + try (Response response = HttpBodies.CLIENT.newCall(request).execute()) { + if (!response.isSuccessful()) { + throw new FileStorageException("S3 upload failed with status: " + response.code()); + } + } catch (FileStorageException e) { + throw e; + } catch (Exception e) { + throw new FileStorageException("S3 upload failed: " + url, e); + } + } + + @Override + public void upload(String url, InputStream inputStream, long contentLength) { + Request request = new Request.Builder() + .url(url) + .put(HttpBodies.stream(inputStream, contentLength)) + .build(); + try (Response response = HttpBodies.CLIENT.newCall(request).execute()) { + if (!response.isSuccessful()) { + throw new FileStorageException("S3 upload failed with status: " + response.code()); + } + } catch (FileStorageException e) { + throw e; + } catch (Exception e) { + throw new FileStorageException("S3 stream upload failed: " + url, e); + } + } + + @Override + public void download(String url, Path destination) { + Request request = new Request.Builder().url(url).get().build(); + try (Response response = HttpBodies.CLIENT.newCall(request).execute()) { + if (!response.isSuccessful()) { + throw new FileStorageException("S3 download failed with status: " + response.code()); + } + Files.createDirectories(destination.getParent()); + try (InputStream in = response.body().byteStream()) { + Files.copy(in, destination, StandardCopyOption.REPLACE_EXISTING); + } + } catch (FileStorageException e) { + throw e; + } catch (Exception e) { + throw new FileStorageException("S3 download failed: " + url, e); + } + } + + @Override + public String uploadPart(String url, Path localFile, long offset, long length) { + Request request = new Request.Builder() + .url(url) + .put(HttpBodies.range(localFile, offset, length)) + .build(); + try (Response response = HttpBodies.CLIENT.newCall(request).execute()) { + if (response.isSuccessful()) { + return response.header(E_TAG); + } + throw new FileStorageException("S3 part upload failed with status: " + response.code()); + } catch (FileStorageException e) { + throw e; + } catch (Exception e) { + throw new FileStorageException("S3 part upload failed: " + url, e); + } + } +} diff --git a/conductor-client/src/main/java/org/conductoross/conductor/sdk/file/FileDownloadStatus.java b/conductor-client/src/main/java/org/conductoross/conductor/sdk/file/FileDownloadStatus.java new file mode 100644 index 000000000..b998c10d6 --- /dev/null +++ b/conductor-client/src/main/java/org/conductoross/conductor/sdk/file/FileDownloadStatus.java @@ -0,0 +1,28 @@ +/* + * Copyright 2026 Conductor Authors. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package org.conductoross.conductor.sdk.file; + +/** + * Per-worker download status for a {@link ManagedFileHandler}. SDK-only — not exchanged with + * the server (the server tracks upload status, not download status). + */ +public enum FileDownloadStatus { + /** No download has been attempted yet. */ + NOT_STARTED, + /** A download is in progress under {@code ManagedFileHandler.downloadLock}. */ + DOWNLOADING, + /** Content has been cached locally and is ready for repeated reads. */ + DOWNLOADED, + /** The download exhausted its retries and failed. */ + FAILED +} diff --git a/conductor-client/src/main/java/org/conductoross/conductor/sdk/file/FileHandler.java b/conductor-client/src/main/java/org/conductoross/conductor/sdk/file/FileHandler.java new file mode 100644 index 000000000..c4b99b658 --- /dev/null +++ b/conductor-client/src/main/java/org/conductoross/conductor/sdk/file/FileHandler.java @@ -0,0 +1,132 @@ +/* + * Copyright 2026 Conductor Authors. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package org.conductoross.conductor.sdk.file; + +import java.io.InputStream; +import java.nio.file.Path; +import java.util.Map; + +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; + +/** + * Abstraction over a file passed into or out of a Conductor worker. + * + *

A {@code FileHandler} is used in two directions: + *

    + *
  • Input — a file referenced by a workflow task. The SDK resolves the incoming + * {@link #getFileHandleId() fileHandleId} string (see format below) into a + * {@link ManagedFileHandler} that downloads the content lazily on first read.
    • + *
  • Output — a local file a worker wants to publish. Create one via + * {@link #fromLocalFile(Path)}; when the worker returns, the task runner uploads the + * file and substitutes the resulting {@code fileHandleId} into the task output so + * downstream tasks can consume it.
    • + *
+ */ +@JsonSerialize(using = FileHandlerSerializer.class) +@JsonDeserialize(using = FileHandlerDeserializer.class) +public interface FileHandler { + + /** The {@code conductor://file/} prefix. */ + public static final String PREFIX = "conductor://file/"; + + /** + * Returns the Conductor-assigned identifier for this file, or {@code null} if the file + * has not been uploaded yet. + * + *

The id is an opaque string in the form + * {@code conductor://file/} — the {@code conductor://file/} prefix (see {@link #PREFIX}) + * is what lets the SDK distinguish a file reference from any other string value in task + * input/output maps and auto-convert it to a {@code FileHandler} for worker parameters. + * + *

Lifecycle: + *

    + *
  • A {@link LocalFileHandler} built via {@link #fromLocalFile(Path)} returns + * {@code null} here until the task runner uploads it. Do not read this value before + * the worker returns — read it from the uploaded handler the runner produces (e.g. + * in tests) or rely on the runner substituting it into the task output automatically.
    • + *
  • A {@link ManagedFileHandler} (produced when a worker receives a file input, or + * returned by {@code FileClient.upload(...)}) always has a non-null id.
    • + *
+ * + *

The returned value is safe to store in workflow input/output maps; any task that + * consumes it will see a {@code FileHandler} injected by the SDK. + * + * @return the {@code conductor://file/} identifier, or {@code null} for a local file + * that has not yet been uploaded + */ + String getFileHandleId(); + + InputStream getInputStream(); + + String getFileName(); + + String getContentType(); + + long getFileSize(); + + default String getFileId() { + return toFileId(getFileHandleId()); + } + + /** + * Creates a handler for a local file to be uploaded when the worker returns. + * Content type defaults to {@code application/octet-stream}. + */ + static FileHandler fromLocalFile(Path path) { + return new LocalFileHandler(path, "application/octet-stream"); + } + + /** + * Creates a handler for a local file to be uploaded when the worker returns, using the + * given content type. + */ + static FileHandler fromLocalFile(Path path, String contentType) { + return new LocalFileHandler(path, contentType); + } + + /** Wraps a bare {@code fileId} with the prefix. Returns the input unchanged if already prefixed. */ + static String toFileHandleId(String fileId) { + return fileId.startsWith(PREFIX) ? fileId : PREFIX + fileId; + } + + /** Strips the prefix from a {@code fileHandleId}. Returns the input unchanged if the prefix is absent. */ + static String toFileId(String fileHandleId) { + return fileHandleId.startsWith(PREFIX) ? fileHandleId.substring(PREFIX.length()) : fileHandleId; + } + + /** {@code true} if {@code value} is a non-null String starting with the prefix. */ + static boolean isFileHandleId(Object value) { + return value instanceof String s && s.startsWith(PREFIX); + } + + /** + * Extracts a {@code fileHandleId} from a task input value. Accepts: + *

    + *
  • a {@link String} — returned unchanged;
    • + *
  • a {@link Map} carrying a {@code fileHandleId} entry — that entry's value is returned + * (the serialized JSON form produced by {@link FileHandlerSerializer});
    • + *
+ * and returns {@code null} for anything else. The returned string is not validated against + * {@link #PREFIX}; callers should pass it to {@link #isFileHandleId} to confirm. + */ + static String extractFileHandleId(Object value) { + if (value instanceof String s) { + return s; + } + if (value instanceof Map m && m.get("fileHandleId") instanceof String s) { + return s; + } + return null; + } +} diff --git a/conductor-client/src/main/java/org/conductoross/conductor/sdk/file/FileHandlerConverter.java b/conductor-client/src/main/java/org/conductoross/conductor/sdk/file/FileHandlerConverter.java new file mode 100644 index 000000000..12a7de5a1 --- /dev/null +++ b/conductor-client/src/main/java/org/conductoross/conductor/sdk/file/FileHandlerConverter.java @@ -0,0 +1,47 @@ +/* + * Copyright 2026 Conductor Authors. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package org.conductoross.conductor.sdk.file; + +import java.nio.file.Path; + +import org.conductoross.conductor.client.model.file.FileUploadRequest; +import org.conductoross.conductor.client.model.file.FileUploadResponse; + +/** Conversion helpers between the file-storage DTOs and the internal {@link ManagedFileHandler}. */ +public class FileHandlerConverter { + + /** + * Builds a {@link ManagedFileHandler} from a completed upload response. The handler already + * has its metadata and {@code localPath} populated — no server round-trip needed on first + * access. + */ + public static ManagedFileHandler toManagedFileHandler(FileUploadResponse response, WorkflowFileClient workflowFileClient, Path localPath, long fileSize) { + ManagedFileHandler handler = new ManagedFileHandler(response.getFileHandleId(), workflowFileClient); + handler.setFileName(response.getFileName()); + handler.setContentType(response.getContentType()); + handler.setFileSize(fileSize); + handler.setStorageType(response.getStorageType()); + handler.setLocalPath(localPath); + return handler; + } + + /** Builds the {@link FileUploadRequest} payload for {@code POST /api/files}. */ + public static FileUploadRequest toFileUploadRequest(String workflowId, FileUploadOptions options) { + FileUploadRequest request = new FileUploadRequest(); + request.setFileName(options.getFileName()); + request.setContentType(options.getContentType()); + request.setWorkflowId(workflowId); + request.setTaskId(options.getTaskId()); + return request; + } +} diff --git a/conductor-client/src/main/java/org/conductoross/conductor/sdk/file/FileHandlerDeserializer.java b/conductor-client/src/main/java/org/conductoross/conductor/sdk/file/FileHandlerDeserializer.java new file mode 100644 index 000000000..7f7792842 --- /dev/null +++ b/conductor-client/src/main/java/org/conductoross/conductor/sdk/file/FileHandlerDeserializer.java @@ -0,0 +1,50 @@ +/* + * Copyright 2026 Conductor Authors. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package org.conductoross.conductor.sdk.file; + +import java.io.IOException; + +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.JsonDeserializer; + +/** + * Deserializes a {@link FileHandler} field on any POJO when the source JSON is either the raw + * {@code conductor://file/} string or a {@link FileHandlerSerializer}-shaped JSON object. + * + *

A {@link WorkflowFileClient} must be supplied via the {@code DeserializationContext} + * attribute keyed by {@link #WORKFLOW_FILE_CLIENT_ATTR}; the task runner's {@code AnnotatedWorker} + * sets this per-task before binding worker input. Without it this deserializer cannot construct + * a {@link ManagedFileHandler} and fails. + */ +public class FileHandlerDeserializer extends JsonDeserializer { + + public static final String WORKFLOW_FILE_CLIENT_ATTR = + "org.conductoross.conductor.sdk.file.workflowFileClient"; + + @Override + public FileHandler deserialize(JsonParser p, DeserializationContext ctxt) throws IOException { + Object raw = p.readValueAs(Object.class); + String fileHandleId = FileHandler.extractFileHandleId(raw); + if (!FileHandler.isFileHandleId(fileHandleId)) { + throw new FileStorageException( + "Expected " + FileHandler.PREFIX + " reference, got: " + raw); + } + Object attr = ctxt.getAttribute(WORKFLOW_FILE_CLIENT_ATTR); + if (!(attr instanceof WorkflowFileClient client)) { + throw new FileStorageException( + "FileHandler input requires a WorkflowFileClient in deserialization context"); + } + return new ManagedFileHandler(fileHandleId, client); + } +} diff --git a/conductor-client/src/main/java/org/conductoross/conductor/sdk/file/FileHandlerSerializer.java b/conductor-client/src/main/java/org/conductoross/conductor/sdk/file/FileHandlerSerializer.java new file mode 100644 index 000000000..7fd9d880b --- /dev/null +++ b/conductor-client/src/main/java/org/conductoross/conductor/sdk/file/FileHandlerSerializer.java @@ -0,0 +1,36 @@ +/* + * Copyright 2026 Conductor Authors. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package org.conductoross.conductor.sdk.file; + +import java.io.IOException; + +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.databind.JsonSerializer; +import com.fasterxml.jackson.databind.SerializerProvider; + +/** + * Serializes any {@link FileHandler} as a fixed three-field JSON object: + * {@code {fileHandleId, contentType, fileName}}. Other accessors on the interface + * (e.g. {@code getInputStream}, {@code getFileSize}) and impl-specific getters are not emitted. + */ +public class FileHandlerSerializer extends JsonSerializer { + + @Override + public void serialize(FileHandler handler, JsonGenerator gen, SerializerProvider provider) throws IOException { + gen.writeStartObject(); + gen.writeStringField("fileHandleId", handler.getFileHandleId()); + gen.writeStringField("contentType", handler.getContentType()); + gen.writeStringField("fileName", handler.getFileName()); + gen.writeEndObject(); + } +} diff --git a/conductor-client/src/main/java/org/conductoross/conductor/sdk/file/FileStorageBackend.java b/conductor-client/src/main/java/org/conductoross/conductor/sdk/file/FileStorageBackend.java new file mode 100644 index 000000000..e9b0ec089 --- /dev/null +++ b/conductor-client/src/main/java/org/conductoross/conductor/sdk/file/FileStorageBackend.java @@ -0,0 +1,71 @@ +/* + * Copyright 2026 Conductor Authors. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package org.conductoross.conductor.sdk.file; + +import java.io.InputStream; +import java.nio.file.Path; + +import org.conductoross.conductor.client.model.file.StorageType; + +/** + * SDK-side counterpart to the server's {@code FileStorage}. Handles the actual byte transfer + * between the worker and the configured storage backend (S3, Azure Blob, GCS, Local). + * + *

One implementation per backend. {@link org.conductoross.conductor.client.FileClient} + * selects the backend that matches the server-reported {@link StorageType} on each upload or + * download. + */ +public interface FileStorageBackend { + + /** Returns the {@link StorageType} this implementation handles. */ + StorageType getStorageType(); + + /** + * Uploads the full content of {@code localFile} to a server-issued URL (presigned PUT for + * S3, SAS URL for Azure, signed URL for GCS, or a direct path for local). + */ + void upload(String url, Path localFile); + + /** + * Uploads the content of {@code inputStream} to the server-issued URL. {@code contentLength} + * is required by backends that cannot accept chunked transfer. + */ + void upload(String url, InputStream inputStream, long contentLength); + + /** + * Downloads the content at the server-issued URL to {@code destination}. Creates parent + * directories as needed. + */ + void download(String url, Path destination); + + /** + * Uploads a single part of a multipart upload. + * + * @param url per-part presigned URL (S3) or resumable session URL (GCS/Azure) + * @param localFile source file + * @param offset byte offset into {@code localFile} + * @param length number of bytes to upload for this part + * @return the part's ETag (or backend-equivalent identifier), to be supplied to the + * server's complete-multipart call + */ + String uploadPart(String url, Path localFile, long offset, long length); + + /** + * Whether this backend supports multipart upload. Return {@code false} for backends that do + * not (e.g. local); {@link org.conductoross.conductor.client.FileClient} will then fall back + * to single-part upload regardless of the configured multipart threshold. + */ + default boolean hasMultipartSupport() { + return true; + } +} diff --git a/conductor-client/src/main/java/org/conductoross/conductor/sdk/file/FileStorageException.java b/conductor-client/src/main/java/org/conductoross/conductor/sdk/file/FileStorageException.java new file mode 100644 index 000000000..916488228 --- /dev/null +++ b/conductor-client/src/main/java/org/conductoross/conductor/sdk/file/FileStorageException.java @@ -0,0 +1,29 @@ +/* + * Copyright 2026 Conductor Authors. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package org.conductoross.conductor.sdk.file; + +/** + * Unchecked exception raised by the SDK for file-storage failures — upload, download, metadata + * fetch, or storage-type mismatch between the server and the registered + * {@link FileStorageBackend}s. + */ +public class FileStorageException extends RuntimeException { + + public FileStorageException(String message) { + super(message); + } + + public FileStorageException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/conductor-client/src/main/java/org/conductoross/conductor/sdk/file/FileUploadOptions.java b/conductor-client/src/main/java/org/conductoross/conductor/sdk/file/FileUploadOptions.java new file mode 100644 index 000000000..cb3ced83a --- /dev/null +++ b/conductor-client/src/main/java/org/conductoross/conductor/sdk/file/FileUploadOptions.java @@ -0,0 +1,75 @@ +/* + * Copyright 2026 Conductor Authors. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package org.conductoross.conductor.sdk.file; + +/** + * Optional metadata to attach to a file upload. {@code workflowId} is supplied as the explicit + * first argument on {@link FileUploader#upload}; this type carries everything else. + * + *

When uploading inside a worker, {@code taskId} is populated automatically from the active + * {@link com.netflix.conductor.sdk.workflow.executor.task.TaskContext} if not set here. + * Providing it explicitly overrides the auto-detected value. + * + *

Set {@link #setMultipart(boolean) multipart=true} to force multipart upload when the + * underlying backend supports it; otherwise the SDK falls back to a single-request upload. + * + * 

{@code
+ * FileUploadOptions options = new FileUploadOptions()
+ *         .setContentType("application/pdf")
+ *         .setMultipart(true);
+ * FileHandler handler = fileUploader.upload(workflowId, path, options);
+ * }
+ */ +public class FileUploadOptions { + + private String taskId; + private String fileName; + private String contentType; + private boolean multipart; + + public String getTaskId() { + return taskId; + } + + public FileUploadOptions setTaskId(String taskId) { + this.taskId = taskId; + return this; + } + + public String getFileName() { + return fileName; + } + + public FileUploadOptions setFileName(String fileName) { + this.fileName = fileName; + return this; + } + + public String getContentType() { + return contentType; + } + + public FileUploadOptions setContentType(String contentType) { + this.contentType = contentType; + return this; + } + + public boolean isMultipart() { + return multipart; + } + + public FileUploadOptions setMultipart(boolean multipart) { + this.multipart = multipart; + return this; + } +} diff --git a/conductor-client/src/main/java/org/conductoross/conductor/sdk/file/FileUploader.java b/conductor-client/src/main/java/org/conductoross/conductor/sdk/file/FileUploader.java new file mode 100644 index 000000000..775cfe981 --- /dev/null +++ b/conductor-client/src/main/java/org/conductoross/conductor/sdk/file/FileUploader.java @@ -0,0 +1,43 @@ +/* + * Copyright 2026 Conductor Authors. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package org.conductoross.conductor.sdk.file; + +import java.io.InputStream; +import java.nio.file.Path; + +/** + * Developer-facing upload API. Implemented by {@link org.conductoross.conductor.client.FileClient}. + * Use for explicit uploads — e.g. before starting a workflow, or inside a task when you want to + * control upload timing. + * + *

All overloads return a {@link FileHandler} whose {@link FileHandler#getFileHandleId()} is + * populated with the prefixed handle {@code conductor://file/}. The {@link InputStream} + * overloads stream the content to a temporary file first, then delegate to the {@link Path} + * overload. + * + *

When called from within a worker, {@code workflowId} and {@code taskId} are injected + * automatically from the active task context. Use {@link FileUploadOptions} to override them or + * to supply additional metadata when uploading outside a worker. + */ +public interface FileUploader { + + FileHandler upload(Path localFile); + + FileHandler upload(InputStream inputStream); + + /** Upload from a local file with the given options. */ + FileHandler upload(Path localFile, FileUploadOptions options); + + /** Upload from an {@link InputStream} with the given options. */ + FileHandler upload(InputStream inputStream, FileUploadOptions options); +} diff --git a/conductor-client/src/main/java/org/conductoross/conductor/sdk/file/LocalFileHandler.java b/conductor-client/src/main/java/org/conductoross/conductor/sdk/file/LocalFileHandler.java new file mode 100644 index 000000000..bead95385 --- /dev/null +++ b/conductor-client/src/main/java/org/conductoross/conductor/sdk/file/LocalFileHandler.java @@ -0,0 +1,67 @@ +/* + * Copyright 2026 Conductor Authors. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package org.conductoross.conductor.sdk.file; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; + +import org.conductoross.conductor.client.FileClient; + +/** + * {@link FileHandler} wrapping a local file that has not been uploaded. + * {@link #getFileHandleId()} returns {@code null}. Returned by + * {@link FileHandler#fromLocalFile(Path)}; consumed by + * {@link FileClient#upload(String, Path)} or by + * {@code TaskRunner}'s auto-upload interception on task completion. + */ +public class LocalFileHandler implements FileHandler { + + private final Path path; + private final String contentType; + + LocalFileHandler(Path path, String contentType) { + this.path = path; + this.contentType = contentType; + } + + @Override + public String getFileHandleId() { return null; } + + @Override + public InputStream getInputStream() { + try { + return Files.newInputStream(path); + } catch (IOException e) { + throw new FileStorageException("Failed to open local file: " + path, e); + } + } + + @Override + public String getFileName() { return path.getFileName().toString(); } + + @Override + public String getContentType() { return contentType; } + + @Override + public long getFileSize() { + try { + return Files.size(path); + } catch (IOException e) { + throw new FileStorageException("Failed to get file size: " + path, e); + } + } + + public Path getPath() { return path; } +} diff --git a/conductor-client/src/main/java/org/conductoross/conductor/sdk/file/ManagedFileHandler.java b/conductor-client/src/main/java/org/conductoross/conductor/sdk/file/ManagedFileHandler.java new file mode 100644 index 000000000..1d6d6c3d6 --- /dev/null +++ b/conductor-client/src/main/java/org/conductoross/conductor/sdk/file/ManagedFileHandler.java @@ -0,0 +1,154 @@ +/* + * Copyright 2026 Conductor Authors. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package org.conductoross.conductor.sdk.file; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.concurrent.locks.ReentrantLock; + +import org.conductoross.conductor.client.FileClient; +import org.conductoross.conductor.client.model.file.FileHandle; +import org.conductoross.conductor.client.model.file.StorageType; + +/** + * {@link FileHandler} for a file already registered on the server. Holds the prefixed + * {@code fileHandleId} and a reference to a {@link FileClient}; fetches metadata lazily on + * first accessor call, and downloads content lazily on first {@link #getInputStream()} call. + * + *

Thread-safe: concurrent {@code getInputStream()} calls serialize through a + * {@link ReentrantLock} so only one download runs per instance. Downloaded content is cached + * under {@link org.conductoross.conductor.client.FileClientProperties#getLocalCacheDirectory()} + * at {@code /_}. Two {@code ManagedFileHandler}s for the same file + * on the same worker node will therefore skip the re-download. + */ +public class ManagedFileHandler implements FileHandler { + + private final String fileHandleId; + private String fileName; + private String contentType; + private long fileSize; + private StorageType storageType; + private Path localPath; + private FileDownloadStatus downloadStatus = FileDownloadStatus.NOT_STARTED; + private final WorkflowFileClient workflowFileClient; + private final ReentrantLock downloadLock = new ReentrantLock(); + + public ManagedFileHandler(String fileHandleId, WorkflowFileClient workflowFileClient) { + this.fileHandleId = fileHandleId; + this.workflowFileClient = workflowFileClient; + } + + @Override + public String getFileHandleId() { return fileHandleId; } + + @Override + public InputStream getInputStream() { + ensureDownloaded(); + try { + return Files.newInputStream(localPath); + } catch (IOException e) { + throw new FileStorageException("Failed to open cached file: " + localPath, e); + } + } + + @Override + public String getFileName() { + ensureMetadataLoaded(); + return fileName; + } + + @Override + public String getContentType() { + ensureMetadataLoaded(); + return contentType; + } + + @Override + public long getFileSize() { + ensureMetadataLoaded(); + return fileSize; + } + + /** Fetches the {@link FileHandle} from the server on first call; a no-op thereafter. */ + private void ensureMetadataLoaded() { + if (fileName != null) { + return; + } + FileHandle metadata = workflowFileClient.getMetadata(fileHandleId); + this.fileName = metadata.getFileName(); + this.contentType = metadata.getContentType(); + this.fileSize = metadata.getFileSize(); + this.storageType = metadata.getStorageType(); + } + + /** + * Downloads content to the cache on first call. Concurrent callers block on + * {@link #downloadLock}; only the first acquires the lock and performs the download. If + * another {@code ManagedFileHandler} has already cached the file at the predictable path, + * the download is skipped. + */ + private void ensureDownloaded() { + if (downloadStatus == FileDownloadStatus.DOWNLOADED && localPath != null + && Files.exists(localPath)) { + return; + } + + downloadLock.lock(); + try { + if (downloadStatus == FileDownloadStatus.DOWNLOADED && localPath != null + && Files.exists(localPath)) { + return; + } + downloadStatus = FileDownloadStatus.DOWNLOADING; + + ensureMetadataLoaded(); + + Path destination = getCachePath(); + if (Files.exists(destination)) { + this.localPath = destination; + this.downloadStatus = FileDownloadStatus.DOWNLOADED; + return; + } + + int maxRetries = workflowFileClient.getRetryCount(); + for (int attempt = 1; attempt <= maxRetries; attempt++) { + try { + workflowFileClient.download(fileHandleId, storageType, destination); + this.localPath = destination; + this.downloadStatus = FileDownloadStatus.DOWNLOADED; + return; + } catch (Exception e) { + if (attempt == maxRetries) throw e; + } + } + } catch (Exception e) { + this.downloadStatus = FileDownloadStatus.FAILED; + throw new FileStorageException("Download failed for " + fileHandleId, e); + } finally { + downloadLock.unlock(); + } + } + + private Path getCachePath() { + String fileId = FileHandler.toFileId(fileHandleId); + return Path.of(workflowFileClient.getCacheDirectory(), fileId + "_" + fileName); + } + + void setFileName(String fileName) { this.fileName = fileName; } + void setContentType(String contentType) { this.contentType = contentType; } + void setFileSize(long fileSize) { this.fileSize = fileSize; } + void setStorageType(StorageType storageType) { this.storageType = storageType; } + void setLocalPath(Path localPath) { this.localPath = localPath; } +} diff --git a/conductor-client/src/main/java/org/conductoross/conductor/sdk/file/WorkflowFileClient.java b/conductor-client/src/main/java/org/conductoross/conductor/sdk/file/WorkflowFileClient.java new file mode 100644 index 000000000..2cae44b13 --- /dev/null +++ b/conductor-client/src/main/java/org/conductoross/conductor/sdk/file/WorkflowFileClient.java @@ -0,0 +1,67 @@ +/* + * Copyright 2026 Conductor Authors. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package org.conductoross.conductor.sdk.file; + +import java.io.InputStream; +import java.nio.file.Path; + +import org.conductoross.conductor.client.FileClient; +import org.conductoross.conductor.client.model.file.FileHandle; +import org.conductoross.conductor.client.model.file.StorageType; + +/** + * Decorator for {@link FileClient} that exposes the same file operations while keeping the + * implementation behind a simpler SDK-facing type. + */ +public class WorkflowFileClient implements FileUploader { + + private final FileClient delegate; + private final String workflowId; + + public WorkflowFileClient(FileClient delegate, String workflowId) { + this.delegate = delegate; + this.workflowId = workflowId; + } + + public FileHandler upload(Path localFile, FileUploadOptions options) { + return delegate.upload(workflowId, localFile, options); + } + + public FileHandler upload(InputStream inputStream, FileUploadOptions options) { + return delegate.upload(workflowId, inputStream, options); + } + + public FileHandler upload(Path localFile) { + return delegate.upload(workflowId, localFile); + } + + public FileHandler upload(InputStream inputStream) { + return delegate.upload(workflowId, inputStream); + } + + public void download(String fileHandleId, StorageType storageType, Path destination) { + delegate.download(workflowId, fileHandleId, storageType, destination); + } + + public FileHandle getMetadata(String fileHandleId) { + return delegate.getMetadata(fileHandleId); + } + + public int getRetryCount() { + return delegate.getRetryCount(); + } + + public String getCacheDirectory() { + return delegate.getCacheDirectory(); + } +} diff --git a/conductor-client/src/test/java/com/netflix/conductor/client/automator/TaskRunnerFileStorageTest.java b/conductor-client/src/test/java/com/netflix/conductor/client/automator/TaskRunnerFileStorageTest.java new file mode 100644 index 000000000..3b152ef8d --- /dev/null +++ b/conductor-client/src/test/java/com/netflix/conductor/client/automator/TaskRunnerFileStorageTest.java @@ -0,0 +1,171 @@ +/* + * Copyright 2026 Conductor Authors. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package com.netflix.conductor.client.automator; + +import java.io.InputStream; +import java.nio.file.Path; +import java.util.List; +import java.util.Map; +import java.util.UUID; + +import org.conductoross.conductor.client.FileClient; +import org.conductoross.conductor.sdk.file.FileHandler; +import org.conductoross.conductor.sdk.file.FileUploadOptions; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; + +import com.netflix.conductor.client.events.dispatcher.EventDispatcher; +import com.netflix.conductor.client.events.taskrunner.TaskRunnerEvent; +import com.netflix.conductor.client.http.TaskClient; +import com.netflix.conductor.client.worker.Worker; +import com.netflix.conductor.common.metadata.tasks.TaskResult; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +class TaskRunnerFileStorageTest { + + private static final String FILE_ID = "conductor://file/" + UUID.randomUUID(); + + private FileClient fileClient; + private TaskRunner runner; + + @BeforeEach + void setUp() { + fileClient = mock(FileClient.class); + Worker worker = mock(Worker.class); + when(worker.getPollingInterval()).thenReturn(100); + when(worker.getTaskDefName()).thenReturn("test_task"); + runner = newRunner(worker, fileClient); + } + + @AfterEach + void tearDown() { + if (runner != null) { + runner.shutdown(1); + } + } + + @Test + @DisplayName("uploadFilesToFileStorage uploads LocalFileHandler and replaces entry with the uploaded FileHandler") + void uploadFilesToFileStorage_uploadsFileHandlerValues() { + FileHandler uploaded = mock(FileHandler.class); + when(uploaded.getFileHandleId()).thenReturn(FILE_ID); + when(fileClient.upload(any(), any(Path.class), any(FileUploadOptions.class))) + .thenReturn(uploaded); + + TaskResult result = new TaskResult(); + FileHandler local = FileHandler.fromLocalFile(Path.of("/tmp/a.pdf"), "application/pdf"); + result.getOutputData().put("result", local); + + runner.uploadFilesToFileStorage(result, "wf-1", "task-1"); + + ArgumentCaptor opts = ArgumentCaptor.forClass(FileUploadOptions.class); + verify(fileClient, times(1)) + .upload(eq("wf-1"), eq(Path.of("/tmp/a.pdf")), opts.capture()); + assertEquals("application/pdf", opts.getValue().getContentType()); + assertEquals("task-1", opts.getValue().getTaskId()); + assertSame(uploaded, result.getOutputData().get("result")); + } + + @Test + @DisplayName("uploadFilesToFileStorage leaves an already-uploaded FileHandler in place (no upload, no flatten)") + void uploadFilesToFileStorage_keepsHandlerForAlreadyUploaded() { + PreUploadedFileHandler handler = new PreUploadedFileHandler(FILE_ID); + TaskResult result = new TaskResult(); + result.getOutputData().put("result", handler); + + runner.uploadFilesToFileStorage(result, "wf-1", "task-1"); + + verify(fileClient, never()) + .upload(any(), any(Path.class), any(FileUploadOptions.class)); + assertSame(handler, result.getOutputData().get("result")); + } + + @Test + @DisplayName("uploadFilesToFileStorage is a no-op when fileClient is null") + void uploadFilesToFileStorage_noopWhenFileClientNull() { + runner.shutdown(1); + Worker worker = mock(Worker.class); + when(worker.getPollingInterval()).thenReturn(100); + when(worker.getTaskDefName()).thenReturn("test_task"); + runner = newRunner(worker, null); + + TaskResult result = new TaskResult(); + FileHandler local = FileHandler.fromLocalFile(Path.of("/tmp/a.pdf"), "application/pdf"); + result.getOutputData().put("result", local); + + runner.uploadFilesToFileStorage(result, "wf-1", "task-1"); + + assertEquals(local, result.getOutputData().get("result")); + } + + @Test + @DisplayName("uploadFilesToFileStorage ignores non-FileHandler outputData values") + void uploadFilesToFileStorage_ignoresNonFileHandlerValues() { + FileHandler uploaded = mock(FileHandler.class); + when(uploaded.getFileHandleId()).thenReturn(FILE_ID); + when(fileClient.upload(any(), any(Path.class), any(FileUploadOptions.class))) + .thenReturn(uploaded); + + TaskResult result = new TaskResult(); + FileHandler local = FileHandler.fromLocalFile(Path.of("/tmp/a.pdf"), "application/pdf"); + result.getOutputData().put("text", "hello"); + result.getOutputData().put("count", 42); + result.getOutputData().put("file", local); + + runner.uploadFilesToFileStorage(result, "wf-1", "task-1"); + + verify(fileClient, times(1)) + .upload(eq("wf-1"), eq(Path.of("/tmp/a.pdf")), any(FileUploadOptions.class)); + assertEquals("hello", result.getOutputData().get("text")); + assertEquals(42, result.getOutputData().get("count")); + assertSame(uploaded, result.getOutputData().get("file")); + } + + @SuppressWarnings("unchecked") + private static TaskRunner newRunner(Worker worker, FileClient fileClient) { + return new TaskRunner( + worker, + mock(TaskClient.class), + 3, + Map.of(), + "test-worker-", + 1, + 100, + List.of(), + (EventDispatcher) mock(EventDispatcher.class), + false, + fileClient); + } + + static class PreUploadedFileHandler implements FileHandler { + private final String id; + PreUploadedFileHandler(String id) { this.id = id; } + @Override public String getFileHandleId() { return id; } + @Override public InputStream getInputStream() { return null; } + @Override public String getFileName() { return "pre.bin"; } + @Override public String getContentType() { return "application/octet-stream"; } + @Override public long getFileSize() { return 0L; } + } +} diff --git a/conductor-client/src/test/java/com/netflix/conductor/common/metadata/tasks/TaskGetInputFileHandlerTest.java b/conductor-client/src/test/java/com/netflix/conductor/common/metadata/tasks/TaskGetInputFileHandlerTest.java new file mode 100644 index 000000000..b09f1480d --- /dev/null +++ b/conductor-client/src/test/java/com/netflix/conductor/common/metadata/tasks/TaskGetInputFileHandlerTest.java @@ -0,0 +1,87 @@ +/* + * Copyright 2026 Conductor Authors. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package com.netflix.conductor.common.metadata.tasks; + +import java.util.Map; +import java.util.UUID; + +import org.conductoross.conductor.sdk.file.FileHandler; +import org.conductoross.conductor.sdk.file.FileStorageException; +import org.conductoross.conductor.sdk.file.WorkflowFileClient; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.Mockito.mock; + +class TaskGetInputFileHandlerTest { + + private static final String FILE_ID = "conductor://file/" + UUID.randomUUID(); + + @Test + void wrapsRawStringFileHandleId() { + Task task = newTask(Map.of("file", FILE_ID)); + + FileHandler handler = task.getInputFileHandler("file"); + + assertEquals(FILE_ID, handler.getFileHandleId()); + } + + @Test + void wrapsJsonObjectWithFileHandleId() { + Map serialized = Map.of( + "fileHandleId", FILE_ID, + "fileName", "doc.pdf", + "contentType", "application/pdf"); + Task task = newTask(Map.of("file", serialized)); + + FileHandler handler = task.getInputFileHandler("file"); + + assertEquals(FILE_ID, handler.getFileHandleId()); + } + + @Test + void throwsWhenStringLacksPrefix() { + Task task = newTask(Map.of("file", "not-a-file-id")); + + assertThrows(FileStorageException.class, () -> task.getInputFileHandler("file")); + } + + @Test + void throwsWhenJsonObjectMissingFileHandleId() { + Task task = newTask(Map.of("file", Map.of("fileName", "x.pdf"))); + + assertThrows(FileStorageException.class, () -> task.getInputFileHandler("file")); + } + + @Test + void throwsWhenJsonObjectFileHandleIdLacksPrefix() { + Task task = newTask(Map.of("file", Map.of("fileHandleId", "raw-uuid"))); + + assertThrows(FileStorageException.class, () -> task.getInputFileHandler("file")); + } + + @Test + void throwsWhenKeyAbsent() { + Task task = newTask(Map.of()); + + assertThrows(FileStorageException.class, () -> task.getInputFileHandler("file")); + } + + private static Task newTask(Map input) { + Task task = new Task(); + task.setInputData(input); + task.setWorkflowFileClient(mock(WorkflowFileClient.class)); + return task; + } +} diff --git a/conductor-client/src/test/java/com/netflix/conductor/sdk/workflow/executor/task/AnnotatedWorkerFileStorageTest.java b/conductor-client/src/test/java/com/netflix/conductor/sdk/workflow/executor/task/AnnotatedWorkerFileStorageTest.java new file mode 100644 index 000000000..2a309d0be --- /dev/null +++ b/conductor-client/src/test/java/com/netflix/conductor/sdk/workflow/executor/task/AnnotatedWorkerFileStorageTest.java @@ -0,0 +1,341 @@ +/* + * Copyright 2026 Conductor Authors. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package com.netflix.conductor.sdk.workflow.executor.task; + +import java.io.InputStream; +import java.nio.file.Path; +import java.util.Map; +import java.util.UUID; + +import org.conductoross.conductor.sdk.file.FileHandler; +import org.conductoross.conductor.sdk.file.FileStorageException; +import org.conductoross.conductor.sdk.file.WorkflowFileClient; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +import com.netflix.conductor.common.metadata.tasks.Task; +import com.netflix.conductor.common.metadata.tasks.TaskResult; +import com.netflix.conductor.sdk.workflow.task.InputParam; +import com.netflix.conductor.sdk.workflow.task.OutputParam; +import com.netflix.conductor.sdk.workflow.task.WorkerTask; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; + +class AnnotatedWorkerFileStorageTest { + + private static final String FILE_ID = "conductor://file/" + UUID.randomUUID(); + + // --- Workers --- + + static class FileReturnWorker { + @WorkerTask("file_return") + public @OutputParam("result") FileHandler doWork() { + return FileHandler.fromLocalFile(Path.of("/tmp/out.pdf"), "application/pdf"); + } + } + + static class FileReturnCustomKeyWorker { + @WorkerTask("file_return_custom_key") + public @OutputParam("doc") FileHandler doWork() { + return FileHandler.fromLocalFile(Path.of("/tmp/out.pdf"), "application/pdf"); + } + } + + static class AlreadyUploadedFileReturnWorker { + @WorkerTask("file_return_already_uploaded") + public @OutputParam("result") FileHandler doWork() { + return new PreUploadedFileHandler(FILE_ID); + } + } + + static class FileInputWorker { + @WorkerTask("file_input") + public @OutputParam("got") String doWork(@InputParam("file") FileHandler file) { + return file.getFileHandleId(); + } + } + + // POJO carrying a FileHandler field — exercises Jackson deserializer path. + // Public so Jackson's Afterburner module can generate an accessor class for it. + public static class Attachment { + public String label; + public FileHandler file; + } + + public static class AttachmentInputWorker { + @WorkerTask("attachment_input") + public @OutputParam("got") String doWork(@InputParam("att") Attachment att) { + return att.label + ":" + (att.file == null ? "null" : att.file.getFileHandleId()); + } + } + + // Non-ManagedFileHandler impl w/ a non-null handle id — simulates a worker + // returning a FileHandler that was already uploaded upstream. + static class PreUploadedFileHandler implements FileHandler { + private final String id; + PreUploadedFileHandler(String id) { this.id = id; } + @Override public String getFileHandleId() { return id; } + @Override public InputStream getInputStream() { return null; } + @Override public String getFileName() { return "pre.bin"; } + @Override public String getContentType() { return "application/octet-stream"; } + @Override public long getFileSize() { return 0L; } + } + + // --- setValue tests --- + + @Test + @DisplayName("setValue puts LocalFileHandler in outputData without uploading (TaskRunner handles upload)") + void setValue_putsFileHandlerInOutput_whenFileIdNull() throws NoSuchMethodException { + WorkflowFileClient workflowFileClient = mock(WorkflowFileClient.class); + + Task task = new Task(); + task.setStatus(Task.Status.IN_PROGRESS); + task.setWorkflowFileClient(workflowFileClient); + + FileReturnWorker bean = new FileReturnWorker(); + AnnotatedWorker worker = new AnnotatedWorker( + "file_return", bean.getClass().getMethod("doWork"), bean); + + TaskResult result = worker.execute(task); + + verify(workflowFileClient, never()).upload(any(Path.class), any()); + Object value = result.getOutputData().get("result"); + assertInstanceOf(FileHandler.class, value); + assertNull(((FileHandler) value).getFileHandleId()); + assertEquals(TaskResult.Status.COMPLETED, result.getStatus()); + } + + @Test + @DisplayName("setValue puts pre-uploaded FileHandler in outputData unchanged") + void setValue_putsFileHandlerInOutput_whenFileIdSet() throws NoSuchMethodException { + WorkflowFileClient workflowFileClient = mock(WorkflowFileClient.class); + + Task task = new Task(); + task.setStatus(Task.Status.IN_PROGRESS); + task.setWorkflowFileClient(workflowFileClient); + + AlreadyUploadedFileReturnWorker bean = new AlreadyUploadedFileReturnWorker(); + AnnotatedWorker worker = new AnnotatedWorker( + "file_return_already_uploaded", bean.getClass().getMethod("doWork"), bean); + + TaskResult result = worker.execute(task); + + verify(workflowFileClient, never()).upload(any(Path.class), any()); + Object value = result.getOutputData().get("result"); + assertInstanceOf(FileHandler.class, value); + assertEquals(FILE_ID, ((FileHandler) value).getFileHandleId()); + } + + @Test + @DisplayName("setValue honors @OutputParam value as outputData key for FileHandler return") + void setValue_usesOutputParamKey() throws NoSuchMethodException { + WorkflowFileClient workflowFileClient = mock(WorkflowFileClient.class); + + Task task = new Task(); + task.setStatus(Task.Status.IN_PROGRESS); + task.setWorkflowFileClient(workflowFileClient); + + FileReturnCustomKeyWorker bean = new FileReturnCustomKeyWorker(); + AnnotatedWorker worker = new AnnotatedWorker( + "file_return_custom_key", bean.getClass().getMethod("doWork"), bean); + + TaskResult result = worker.execute(task); + + assertInstanceOf(FileHandler.class, result.getOutputData().get("doc")); + assertNull(result.getOutputData().get("result")); + } + + // --- getInputValue tests --- + + @Test + @DisplayName("getInputValue wraps conductor://file/ id as ManagedFileHandler") + void getInputValue_wrapsFileIdAsManagedFileHandler() throws NoSuchMethodException { + WorkflowFileClient workflowFileClient = mock(WorkflowFileClient.class); + + FileInputWorker bean = new FileInputWorker(); + AnnotatedWorker worker = new AnnotatedWorker( + "file_input", bean.getClass().getMethod("doWork", FileHandler.class), bean); + + Task task = new Task(); + task.setStatus(Task.Status.IN_PROGRESS); + task.setInputData(Map.of("file", FILE_ID)); + task.setWorkflowFileClient(workflowFileClient); + + TaskResult result = worker.execute(task); + + assertEquals(FILE_ID, result.getOutputData().get("got")); + } + + @Test + @DisplayName("getInputValue accepts JSON object with fileHandleId and wraps as ManagedFileHandler") + void getInputValue_acceptsJsonObjectWithFileHandleId() throws NoSuchMethodException { + WorkflowFileClient workflowFileClient = mock(WorkflowFileClient.class); + + FileInputWorker bean = new FileInputWorker(); + AnnotatedWorker worker = new AnnotatedWorker( + "file_input", bean.getClass().getMethod("doWork", FileHandler.class), bean); + + Map serialized = Map.of( + "fileHandleId", FILE_ID, + "fileName", "doc.pdf", + "contentType", "application/pdf"); + + Task task = new Task(); + task.setStatus(Task.Status.IN_PROGRESS); + task.setInputData(Map.of("file", serialized)); + task.setWorkflowFileClient(workflowFileClient); + + TaskResult result = worker.execute(task); + + assertEquals(FILE_ID, result.getOutputData().get("got")); + } + + @Test + @DisplayName("getInputValue throws when JSON object lacks fileHandleId") + void getInputValue_throws_whenJsonObjectMissingFileHandleId() throws NoSuchMethodException { + WorkflowFileClient workflowFileClient = mock(WorkflowFileClient.class); + + FileInputWorker bean = new FileInputWorker(); + AnnotatedWorker worker = new AnnotatedWorker( + "file_input", bean.getClass().getMethod("doWork", FileHandler.class), bean); + + Task task = new Task(); + task.setStatus(Task.Status.IN_PROGRESS); + task.setInputData(Map.of("file", Map.of("fileName", "x.pdf"))); + task.setWorkflowFileClient(workflowFileClient); + + RuntimeException thrown = assertThrows(RuntimeException.class, () -> worker.execute(task)); + assertInstanceOf(FileStorageException.class, thrown.getCause()); + } + + @Test + @DisplayName("getInputValue throws FileStorageException when FileHandler param value is not a file id") + void getInputValue_throwsFileStorageException_whenValueIsNotFileId() throws NoSuchMethodException { + WorkflowFileClient workflowFileClient = mock(WorkflowFileClient.class); + + FileInputWorker bean = new FileInputWorker(); + AnnotatedWorker worker = new AnnotatedWorker( + "file_input", bean.getClass().getMethod("doWork", FileHandler.class), bean); + + Task task = new Task(); + task.setStatus(Task.Status.IN_PROGRESS); + task.setInputData(Map.of("file", "not-a-file-id")); + task.setWorkflowFileClient(workflowFileClient); + + // FileStorageException is thrown during argument resolution, caught by the + // outer catch(Exception) in execute() and rewrapped as RuntimeException. + RuntimeException thrown = assertThrows(RuntimeException.class, () -> worker.execute(task)); + assertInstanceOf(FileStorageException.class, thrown.getCause()); + } + + @Test + @DisplayName("POJO field FileHandler is bound from JSON object form") + void pojoFieldFileHandler_fromJsonObject() throws NoSuchMethodException { + WorkflowFileClient workflowFileClient = mock(WorkflowFileClient.class); + + AttachmentInputWorker bean = new AttachmentInputWorker(); + AnnotatedWorker worker = new AnnotatedWorker( + "attachment_input", bean.getClass().getMethod("doWork", Attachment.class), bean); + + Map attachment = Map.of( + "label", "invoice", + "file", Map.of( + "fileHandleId", FILE_ID, + "fileName", "inv.pdf", + "contentType", "application/pdf")); + + Task task = new Task(); + task.setStatus(Task.Status.IN_PROGRESS); + task.setInputData(Map.of("att", attachment)); + task.setWorkflowFileClient(workflowFileClient); + + TaskResult result = worker.execute(task); + + assertEquals("invoice:" + FILE_ID, result.getOutputData().get("got")); + } + + @Test + @DisplayName("POJO field FileHandler is bound from raw conductor://file/ string") + void pojoFieldFileHandler_fromRawString() throws NoSuchMethodException { + WorkflowFileClient workflowFileClient = mock(WorkflowFileClient.class); + + AttachmentInputWorker bean = new AttachmentInputWorker(); + AnnotatedWorker worker = new AnnotatedWorker( + "attachment_input", bean.getClass().getMethod("doWork", Attachment.class), bean); + + Map attachment = Map.of("label", "raw", "file", FILE_ID); + + Task task = new Task(); + task.setStatus(Task.Status.IN_PROGRESS); + task.setInputData(Map.of("att", attachment)); + task.setWorkflowFileClient(workflowFileClient); + + TaskResult result = worker.execute(task); + + assertEquals("raw:" + FILE_ID, result.getOutputData().get("got")); + } + + @Test + @DisplayName("POJO field FileHandler throws when value is not a valid reference") + void pojoFieldFileHandler_throwsOnInvalidValue() throws NoSuchMethodException { + WorkflowFileClient workflowFileClient = mock(WorkflowFileClient.class); + + AttachmentInputWorker bean = new AttachmentInputWorker(); + AnnotatedWorker worker = new AnnotatedWorker( + "attachment_input", bean.getClass().getMethod("doWork", Attachment.class), bean); + + Map attachment = Map.of("label", "bad", "file", "not-a-file-id"); + + Task task = new Task(); + task.setStatus(Task.Status.IN_PROGRESS); + task.setInputData(Map.of("att", attachment)); + task.setWorkflowFileClient(workflowFileClient); + + RuntimeException thrown = assertThrows(RuntimeException.class, () -> worker.execute(task)); + assertInstanceOf(FileStorageException.class, rootCause(thrown)); + } + + private static Throwable rootCause(Throwable t) { + Throwable cause = t; + while (cause.getCause() != null && cause.getCause() != cause) { + cause = cause.getCause(); + } + return cause; + } + + @Test + @DisplayName("getInputValue returns null FileHandler when input key is absent") + void getInputValue_returnsNull_whenValueMissing() throws NoSuchMethodException { + WorkflowFileClient workflowFileClient = mock(WorkflowFileClient.class); + + FileInputWorker bean = new FileInputWorker(); + AnnotatedWorker worker = new AnnotatedWorker( + "file_input", bean.getClass().getMethod("doWork", FileHandler.class), bean); + + Task task = new Task(); + task.setStatus(Task.Status.IN_PROGRESS); + task.setInputData(Map.of()); + task.setWorkflowFileClient(workflowFileClient); + + // Worker body dereferences a null FileHandler → NPE → FAILED + TaskResult result = worker.execute(task); + assertEquals(TaskResult.Status.FAILED, result.getStatus()); + } +} diff --git a/conductor-client/src/test/java/org/conductoross/conductor/client/FileClientPropertiesTest.java b/conductor-client/src/test/java/org/conductoross/conductor/client/FileClientPropertiesTest.java new file mode 100644 index 000000000..a5295d67b --- /dev/null +++ b/conductor-client/src/test/java/org/conductoross/conductor/client/FileClientPropertiesTest.java @@ -0,0 +1,28 @@ +/* + * Copyright 2026 Conductor Authors. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package org.conductoross.conductor.client; + +import java.nio.file.Path; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertTrue; + +class FileClientPropertiesTest { + + @Test + void defaultCacheDirectoryIsUnderJavaTmpDir() { + String expected = Path.of(System.getProperty("java.io.tmpdir")).toString(); + assertTrue(new FileClientProperties().getLocalCacheDirectory().startsWith(expected)); + } +} diff --git a/conductor-client/src/test/java/org/conductoross/conductor/client/FileClientTest.java b/conductor-client/src/test/java/org/conductoross/conductor/client/FileClientTest.java new file mode 100644 index 000000000..ca453d982 --- /dev/null +++ b/conductor-client/src/test/java/org/conductoross/conductor/client/FileClientTest.java @@ -0,0 +1,356 @@ +/* + * Copyright 2026 Conductor Authors. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package org.conductoross.conductor.client; + +import java.io.ByteArrayInputStream; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Predicate; + +import org.conductoross.conductor.client.model.file.FileDownloadUrlResponse; +import org.conductoross.conductor.client.model.file.FileHandle; +import org.conductoross.conductor.client.model.file.FileUploadCompleteResponse; +import org.conductoross.conductor.client.model.file.FileUploadResponse; +import org.conductoross.conductor.client.model.file.FileUploadStatus; +import org.conductoross.conductor.client.model.file.FileUploadUrlResponse; +import org.conductoross.conductor.client.model.file.MultipartInitResponse; +import org.conductoross.conductor.client.model.file.StorageType; +import org.conductoross.conductor.sdk.file.FileHandler; +import org.conductoross.conductor.sdk.file.FileStorageBackend; +import org.conductoross.conductor.sdk.file.FileStorageException; +import org.conductoross.conductor.sdk.file.FileUploadOptions; +import org.conductoross.conductor.sdk.file.StubFileStorageBackend; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.mockito.ArgumentCaptor; + +import com.netflix.conductor.client.http.ConductorClient; +import com.netflix.conductor.client.http.ConductorClientRequest; +import com.netflix.conductor.client.http.ConductorClientResponse; +import com.netflix.conductor.client.http.Param; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.atLeastOnce; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +class FileClientTest { + + private static final String RAW_ID = UUID.randomUUID().toString(); + private static final String HANDLE_ID = FileHandler.PREFIX + RAW_ID; + private static final String UPLOAD_URL = "stub://upload/" + RAW_ID; + private static final String DOWNLOAD_URL = "stub://download/" + RAW_ID; + + @TempDir + Path tempDir; + + private ConductorClient conductorClient; + private StubFileStorageBackend backend; + private final List stubs = new ArrayList<>(); + + @BeforeEach + void setUp() { + conductorClient = mock(ConductorClient.class); + backend = new StubFileStorageBackend(); + stubs.clear(); + // Single dispatcher — registering a matcher later just appends to the list. + when(conductorClient.execute(any(ConductorClientRequest.class), any())) + .thenAnswer(invocation -> { + ConductorClientRequest req = invocation.getArgument(0); + for (Stub stub : stubs) { + if (stub.matcher.test(req)) { + return new ConductorClientResponse<>(200, Map.of(), stub.data); + } + } + throw new AssertionError("No stubbed response for path=" + req.getPath()); + }); + } + + @Test + void uploadSinglePartByDefault() throws Exception { + Path source = writeFile("small.txt", "hello world".getBytes()); + stubCreateFile(StorageType.LOCAL); + stubConfirmUpload(); + + FileClient client = clientWithBackends(Map.of(StorageType.LOCAL, backend)); + + FileHandler handler = client.upload( + "wf-1", source, + new FileUploadOptions().setContentType("text/plain")); + + assertEquals(HANDLE_ID, handler.getFileHandleId()); + assertEquals("hello world", new String(backend.getUploaded(UPLOAD_URL))); + verifyPathCalled("/files/{fileId}/upload-complete", RAW_ID); + verifyPathNotCalled("/files/{fileId}/multipart"); + } + + @Test + void uploadUsesMultipartWhenFlagTrueAndBackendSupports() throws Exception { + Path source = writeFile("big.bin", new byte[16]); + stubCreateFile(StorageType.S3); + stubMultipartInit(); + stubMultipartComplete(); + + FileClientProperties props = new FileClientProperties(); + props.setMultipartPartSize(5); // partSize 5 + 16 bytes → 4 parts + MultipartBackend mp = new MultipartBackend(StorageType.S3); + FileClient client = clientBuilder() + .properties(props) + .addStorageBackend(mp) + .build(); + + FileHandler handler = client.upload( + "wf-1", source, + new FileUploadOptions().setMultipart(true)); + + assertEquals(HANDLE_ID, handler.getFileHandleId()); + assertEquals(4, mp.parts.get()); + verifyPathCalled("/files/{fileId}/multipart", RAW_ID); + verifyPathCalled("/files/{fileId}/multipart/{uploadId}/complete", RAW_ID); + } + + @Test + void uploadFallsBackToSinglePartWhenBackendLacksMultipart() throws Exception { + Path source = writeFile("big.bin", new byte[32]); + stubCreateFile(StorageType.LOCAL); + stubConfirmUpload(); + + StubFileStorageBackend noMultipart = new StubFileStorageBackend() { + @Override public boolean hasMultipartSupport() { return false; } + }; + FileClient client = clientBuilder() + .addStorageBackend(noMultipart) + .build(); + + client.upload("wf-1", source, new FileUploadOptions().setMultipart(true)); + + verifyPathCalled("/files/{fileId}/upload-complete", RAW_ID); + verifyPathNotCalled("/files/{fileId}/multipart"); + } + + @Test + void uploadThrowsWhenBackendMissingForServerStorageType() throws Exception { + Path source = writeFile("x.bin", new byte[] {1}); + stubCreateFile(StorageType.AZURE_BLOB); + + // Register only LOCAL; server reports AZURE_BLOB. + FileClient client = clientWithBackends(Map.of(StorageType.LOCAL, backend)); + + FileStorageException ex = assertThrows( + FileStorageException.class, () -> client.upload("wf-1", source)); + assertTrue(ex.getMessage().contains("AZURE_BLOB")); + assertTrue(ex.getMessage().contains("LOCAL")); + } + + @Test + void uploadFromInputStreamStreamsThroughTempFile() { + byte[] payload = "from-stream".getBytes(); + stubCreateFile(StorageType.LOCAL); + stubConfirmUpload(); + + FileClient client = clientWithBackends(Map.of(StorageType.LOCAL, backend)); + + FileHandler handler = client.upload( + "wf-1", new ByteArrayInputStream(payload), + new FileUploadOptions().setContentType("text/plain")); + + assertEquals(HANDLE_ID, handler.getFileHandleId()); + assertEquals("from-stream", new String(backend.getUploaded(UPLOAD_URL))); + } + + @Test + void downloadFetchesUrlAndDelegatesToBackend() throws Exception { + // Pre-seed the stub so its download() has something to serve at DOWNLOAD_URL. + Path src = writeFile("remote.bin", new byte[] {9, 8, 7}); + backend.upload(DOWNLOAD_URL, src); + stubDownloadUrl(); + + FileClient client = clientWithBackends(Map.of(StorageType.LOCAL, backend)); + + Path dest = tempDir.resolve("out/remote.bin"); + client.download("wf-1", HANDLE_ID, StorageType.LOCAL, dest); + + assertTrue(Files.exists(dest)); + verifyPathCalled("/files/{workflowId}/{fileId}/download-url", RAW_ID); + } + + @Test + void getMetadataSendsStrippedFileIdAndReturnsHandle() { + FileHandle fixture = new FileHandle(); + fixture.setFileHandleId(HANDLE_ID); + fixture.setFileName("r.pdf"); + registerStub(req -> "/files/{fileId}".equals(req.getPath()), fixture); + + FileClient client = clientWithBackends(Map.of(StorageType.LOCAL, backend)); + + FileHandle got = client.getMetadata(HANDLE_ID); + assertEquals("r.pdf", got.getFileName()); + verifyPathCalled("/files/{fileId}", RAW_ID); + } + + @Test + void confirmUploadSendsStrippedFileId() { + stubConfirmUpload(); + FileClient client = clientWithBackends(Map.of(StorageType.LOCAL, backend)); + + client.confirmUpload(HANDLE_ID); + + verifyPathCalled("/files/{fileId}/upload-complete", RAW_ID); + } + + @Test + void propertiesDefaultsAppliedWhenNull() { + FileClient client = new FileClient(conductorClient, null, null); + + assertEquals(3, client.getRetryCount()); + assertNotNull(client.getCacheDirectory()); + } + + @Test + void uploadThrowsWhenWorkflowIdNull() throws Exception { + Path source = writeFile("wf.bin", new byte[] {1}); + FileClient client = clientWithBackends(Map.of(StorageType.LOCAL, backend)); + + FileStorageException ex = assertThrows( + FileStorageException.class, () -> client.upload(null, source)); + assertTrue(ex.getMessage().contains("workflowId")); + } + + @Test + void builderCustomBackendOverridesBuiltIn() throws Exception { + Path source = writeFile("overridden.txt", "x".getBytes()); + stubCreateFile(StorageType.LOCAL); + stubConfirmUpload(); + + StubFileStorageBackend override = new StubFileStorageBackend(); + FileClient client = FileClient.builder(conductorClient) + .addStorageBackend(override) + .build(); + + client.upload("wf-1", source); + + assertNotNull(override.getUploaded(UPLOAD_URL), + "override backend should have received the upload"); + } + + // --- helpers --- + + private Path writeFile(String name, byte[] bytes) throws Exception { + return Files.write(tempDir.resolve(name), bytes); + } + + private FileClient clientWithBackends(Map backends) { + return new FileClient(conductorClient, new FileClientProperties(), backends); + } + + private FileClient.Builder clientBuilder() { + return FileClient.builder(conductorClient); + } + + private void stubCreateFile(StorageType storageType) { + FileUploadResponse response = new FileUploadResponse(); + response.setFileHandleId(HANDLE_ID); + response.setStorageType(storageType); + response.setUploadStatus(FileUploadStatus.UPLOADING); + response.setUploadUrl(UPLOAD_URL); + registerStub(req -> "/files".equals(req.getPath()), response); + } + + private void stubConfirmUpload() { + registerStub(req -> "/files/{fileId}/upload-complete".equals(req.getPath()), + new FileUploadCompleteResponse()); + } + + private void stubDownloadUrl() { + FileDownloadUrlResponse resp = new FileDownloadUrlResponse(); + resp.setFileHandleId(HANDLE_ID); + resp.setDownloadUrl(DOWNLOAD_URL); + registerStub(req -> "/files/{workflowId}/{fileId}/download-url".equals(req.getPath()), resp); + } + + private void stubMultipartInit() { + MultipartInitResponse init = new MultipartInitResponse(); + init.setFileHandleId(HANDLE_ID); + init.setUploadId("up-1"); + registerStub(req -> "/files/{fileId}/multipart".equals(req.getPath()), init); + + FileUploadUrlResponse partUrl = new FileUploadUrlResponse(); + partUrl.setFileHandleId(HANDLE_ID); + partUrl.setUploadUrl(UPLOAD_URL + "/part"); + registerStub( + req -> "/files/{fileId}/multipart/{uploadId}/part/{partNumber}".equals(req.getPath()), + partUrl); + } + + private void stubMultipartComplete() { + registerStub( + req -> "/files/{fileId}/multipart/{uploadId}/complete".equals(req.getPath()), + new FileUploadCompleteResponse()); + } + + private void registerStub(Predicate matcher, Object data) { + stubs.add(new Stub(matcher, data)); + } + + private void verifyPathCalled(String path, String expectedFileId) { + ArgumentCaptor captor = ArgumentCaptor.forClass(ConductorClientRequest.class); + verify(conductorClient, atLeastOnce()).execute(captor.capture(), any()); + boolean matched = captor.getAllValues().stream() + .anyMatch(req -> path.equals(req.getPath()) + && pathParamMatches(req.getPathParams(), "fileId", expectedFileId)); + assertTrue(matched, "expected " + path + " with fileId=" + expectedFileId); + } + + private void verifyPathNotCalled(String path) { + ArgumentCaptor captor = ArgumentCaptor.forClass(ConductorClientRequest.class); + verify(conductorClient, atLeastOnce()).execute(captor.capture(), any()); + assertTrue(captor.getAllValues().stream().noneMatch(req -> path.equals(req.getPath())), + "did not expect request to " + path); + } + + private static boolean pathParamMatches(List params, String name, String value) { + return params != null && params.stream() + .anyMatch(p -> name.equals(p.name()) && value.equals(p.value())); + } + + private record Stub(Predicate matcher, Object data) {} + + // Backend stub that claims multipart support and records parts. + private static final class MultipartBackend implements FileStorageBackend { + private final StorageType type; + final AtomicInteger parts = new AtomicInteger(); + + MultipartBackend(StorageType type) { this.type = type; } + + @Override public StorageType getStorageType() { return type; } + @Override public void upload(String url, Path localFile) { throw new AssertionError("single-part not expected"); } + @Override public void upload(String url, InputStream in, long len) { throw new AssertionError(); } + @Override public void download(String url, Path dest) { throw new AssertionError(); } + @Override public String uploadPart(String url, Path localFile, long offset, long length) { + return "etag-" + parts.incrementAndGet(); + } + @Override public boolean hasMultipartSupport() { return true; } + } +} diff --git a/conductor-client/src/test/java/org/conductoross/conductor/sdk/file/FileHandlerDeserializerTest.java b/conductor-client/src/test/java/org/conductoross/conductor/sdk/file/FileHandlerDeserializerTest.java new file mode 100644 index 000000000..0c0c691ff --- /dev/null +++ b/conductor-client/src/test/java/org/conductoross/conductor/sdk/file/FileHandlerDeserializerTest.java @@ -0,0 +1,98 @@ +/* + * Copyright 2026 Conductor Authors. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package org.conductoross.conductor.sdk.file; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import com.netflix.conductor.common.config.ObjectMapperProvider; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.ObjectReader; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.Mockito.mock; + + +class FileHandlerDeserializerTest { + + private static final String FILE_HANDLE_ID = "conductor://file/abc-123"; + + private final ObjectMapper mapper = new ObjectMapperProvider().getObjectMapper(); + private WorkflowFileClient client; + + @BeforeEach + void setUp() { + client = mock(WorkflowFileClient.class); + } + + @Test + void deserializesRawStringToManagedFileHandler() throws Exception { + FileHandler handler = readerWithClient().readValue("\"" + FILE_HANDLE_ID + "\""); + + assertInstanceOf(ManagedFileHandler.class, handler); + assertEquals(FILE_HANDLE_ID, handler.getFileHandleId()); + } + + @Test + void deserializesJsonObjectWithFileHandleIdToManagedFileHandler() throws Exception { + String json = """ + { + "fileHandleId": "%s", + "fileName": "doc.pdf", + "contentType": "application/pdf" + } + """.formatted(FILE_HANDLE_ID); + + FileHandler handler = readerWithClient().readValue(json); + + assertInstanceOf(ManagedFileHandler.class, handler); + assertEquals(FILE_HANDLE_ID, handler.getFileHandleId()); + } + + @Test + void throwsWhenStringLacksPrefix() { + assertThrows(FileStorageException.class, + () -> readerWithClient().readValue("\"raw-uuid\"")); + } + + @Test + void throwsWhenJsonObjectFileHandleIdLacksPrefix() { + String json = "{\"fileHandleId\":\"raw-uuid\"}"; + + assertThrows(FileStorageException.class, () -> readerWithClient().readValue(json)); + } + + @Test + void throwsWhenJsonObjectMissingFileHandleId() { + String json = "{\"fileName\":\"doc.pdf\"}"; + + assertThrows(FileStorageException.class, () -> readerWithClient().readValue(json)); + } + + @Test + void throwsWhenWorkflowFileClientAttributeMissing() { + // No .withAttribute(...) — context attribute is null. + ObjectReader readerWithoutClient = mapper.readerFor(FileHandler.class); + + assertThrows(FileStorageException.class, + () -> readerWithoutClient.readValue("\"" + FILE_HANDLE_ID + "\"")); + } + + private ObjectReader readerWithClient() { + return mapper.readerFor(FileHandler.class) + .withAttribute(FileHandlerDeserializer.WORKFLOW_FILE_CLIENT_ATTR, client); + } +} diff --git a/conductor-client/src/test/java/org/conductoross/conductor/sdk/file/FileHandlerSerializationTest.java b/conductor-client/src/test/java/org/conductoross/conductor/sdk/file/FileHandlerSerializationTest.java new file mode 100644 index 000000000..227fd50a9 --- /dev/null +++ b/conductor-client/src/test/java/org/conductoross/conductor/sdk/file/FileHandlerSerializationTest.java @@ -0,0 +1,114 @@ +/* + * Copyright 2026 Conductor Authors. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package org.conductoross.conductor.sdk.file; + +import java.io.InputStream; +import java.nio.file.Path; +import java.util.Map; +import java.util.Set; + +import org.junit.jupiter.api.Test; + +import com.netflix.conductor.common.config.ObjectMapperProvider; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; + +class FileHandlerSerializationTest { + + private static final ObjectMapper MAPPER = new ObjectMapperProvider().getObjectMapper(); + private static final TypeReference> MAP_TYPE = new TypeReference<>() {}; + private static final Set EXPECTED_FIELDS = Set.of("fileHandleId", "contentType", "fileName"); + + @Test + void localFileHandlerSerializesToThreeFields() { + FileHandler handler = FileHandler.fromLocalFile(Path.of("docs/note.txt"), "text/plain"); + + Map json = MAPPER.convertValue(handler, MAP_TYPE); + + assertEquals(EXPECTED_FIELDS, json.keySet()); + assertNull(json.get("fileHandleId")); // local handler not yet uploaded + assertEquals("text/plain", json.get("contentType")); + assertEquals("note.txt", json.get("fileName")); + } + + @Test + void stubManagedFileHandlerSerializesToThreeFields() { + FileHandler handler = new StubManagedFileHandler( + "conductor://file/abc-123", "report.pdf", "application/pdf"); + + Map json = MAPPER.convertValue(handler, MAP_TYPE); + + assertEquals(EXPECTED_FIELDS, json.keySet()); + assertEquals("conductor://file/abc-123", json.get("fileHandleId")); + assertEquals("application/pdf", json.get("contentType")); + assertEquals("report.pdf", json.get("fileName")); + } + + @Test + void userImplSerializesToThreeFieldsOnly() throws Exception { + FileHandler handler = new UserImpl(); + + String jsonString = MAPPER.writeValueAsString(handler); + Map json = MAPPER.readValue(jsonString, MAP_TYPE); + + assertEquals(EXPECTED_FIELDS, json.keySet()); + assertEquals("conductor://file/u-1", json.get("fileHandleId")); + assertFalse(jsonString.contains("inputStream")); + assertFalse(jsonString.contains("fileSize")); + assertFalse(jsonString.contains("extra")); + } + + @Test + void serializesInsideTaskOutputMap() { + FileHandler handler = new UserImpl(); + Map output = Map.of("file", handler, "answer", 42); + + Map json = MAPPER.convertValue(output, MAP_TYPE); + + @SuppressWarnings("unchecked") + Map fileJson = (Map) json.get("file"); + assertEquals(EXPECTED_FIELDS, fileJson.keySet()); + assertEquals(42, json.get("answer")); + } + + private static final class UserImpl implements FileHandler { + @Override public String getFileHandleId() { return "conductor://file/u-1"; } + @Override public InputStream getInputStream() { throw new UnsupportedOperationException(); } + @Override public String getFileName() { return "u.bin"; } + @Override public String getContentType() { return "application/octet-stream"; } + @Override public long getFileSize() { return 7L; } + public String getExtra() { return "should-not-leak"; } + } + + /** Minimal {@link ManagedFileHandler}-like impl for testing without a workflow client. */ + private static final class StubManagedFileHandler implements FileHandler { + private final String fileHandleId; + private final String fileName; + private final String contentType; + StubManagedFileHandler(String id, String fileName, String contentType) { + this.fileHandleId = id; + this.fileName = fileName; + this.contentType = contentType; + } + @Override public String getFileHandleId() { return fileHandleId; } + @Override public InputStream getInputStream() { throw new UnsupportedOperationException(); } + @Override public String getFileName() { return fileName; } + @Override public String getContentType() { return contentType; } + @Override public long getFileSize() { return 0L; } + } +} diff --git a/conductor-client/src/test/java/org/conductoross/conductor/sdk/file/FileHandlerTest.java b/conductor-client/src/test/java/org/conductoross/conductor/sdk/file/FileHandlerTest.java new file mode 100644 index 000000000..1fee77164 --- /dev/null +++ b/conductor-client/src/test/java/org/conductoross/conductor/sdk/file/FileHandlerTest.java @@ -0,0 +1,76 @@ +/* + * Copyright 2026 Conductor Authors. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package org.conductoross.conductor.sdk.file; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class FileHandlerTest { + + private static final String RAW_ID = "abc-123"; + private static final String HANDLE_ID = FileHandler.PREFIX + RAW_ID; + + @Test + void toFileHandleIdAddsPrefixOnce() { + assertEquals(HANDLE_ID, FileHandler.toFileHandleId(RAW_ID)); + } + + @Test + void toFileHandleIdIsIdempotent() { + String already = FileHandler.toFileHandleId(HANDLE_ID); + assertSame(HANDLE_ID, already, "must not re-prefix an already-prefixed id"); + } + + @Test + void toFileIdStripsPrefix() { + assertEquals(RAW_ID, FileHandler.toFileId(HANDLE_ID)); + } + + @Test + void toFileIdReturnsInputWhenPrefixAbsent() { + assertEquals(RAW_ID, FileHandler.toFileId(RAW_ID)); + } + + @Test + void roundTripFromRawIdIsStable() { + String handle = FileHandler.toFileHandleId(RAW_ID); + String raw = FileHandler.toFileId(handle); + assertEquals(RAW_ID, raw); + } + + @Test + void isFileHandleIdTrueForPrefixedString() { + assertTrue(FileHandler.isFileHandleId(HANDLE_ID)); + } + + @Test + void isFileHandleIdFalseForUnprefixedString() { + assertFalse(FileHandler.isFileHandleId(RAW_ID)); + } + + @Test + void isFileHandleIdFalseForNonStringValues() { + assertFalse(FileHandler.isFileHandleId(null)); + assertFalse(FileHandler.isFileHandleId(42)); + assertFalse(FileHandler.isFileHandleId(new Object())); + } + + @Test + void prefixIsTheDocumentedScheme() { + assertEquals("conductor://file/", FileHandler.PREFIX); + } +} diff --git a/conductor-client/src/test/java/org/conductoross/conductor/sdk/file/LocalFileHandlerTest.java b/conductor-client/src/test/java/org/conductoross/conductor/sdk/file/LocalFileHandlerTest.java new file mode 100644 index 000000000..69533010a --- /dev/null +++ b/conductor-client/src/test/java/org/conductoross/conductor/sdk/file/LocalFileHandlerTest.java @@ -0,0 +1,84 @@ +/* + * Copyright 2026 Conductor Authors. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package org.conductoross.conductor.sdk.file; + +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import static org.junit.jupiter.api.Assertions.*; + +class LocalFileHandlerTest { + + @TempDir + Path temp; + + @Test + void testGetFileHandleIdReturnsNull() throws Exception { + Path file = temp.resolve("test.txt"); + Files.writeString(file, "hello"); + FileHandler handler = FileHandler.fromLocalFile(file); + assertNull(handler.getFileHandleId()); + } + + @Test + void testGetInputStream() throws Exception { + Path file = temp.resolve("test.txt"); + Files.writeString(file, "hello"); + FileHandler handler = FileHandler.fromLocalFile(file); + try (InputStream in = handler.getInputStream()) { + assertEquals("hello", new String(in.readAllBytes())); + } + } + + @Test + void testGetFileName() throws Exception { + Path file = temp.resolve("report.pdf"); + Files.createFile(file); + FileHandler handler = FileHandler.fromLocalFile(file); + assertEquals("report.pdf", handler.getFileName()); + } + + @Test + void testGetContentType() throws Exception { + Path file = temp.resolve("doc.pdf"); + Files.createFile(file); + FileHandler handler = FileHandler.fromLocalFile(file, "application/pdf"); + assertEquals("application/pdf", handler.getContentType()); + } + + @Test + void testGetContentTypeDefault() throws Exception { + Path file = temp.resolve("data.bin"); + Files.createFile(file); + FileHandler handler = FileHandler.fromLocalFile(file); + assertEquals("application/octet-stream", handler.getContentType()); + } + + @Test + void testGetFileSize() throws Exception { + Path file = temp.resolve("data.bin"); + Files.write(file, new byte[]{1, 2, 3, 4, 5}); + FileHandler handler = FileHandler.fromLocalFile(file); + assertEquals(5, handler.getFileSize()); + } + + @Test + void testGetInputStreamMissingFile() { + FileHandler handler = FileHandler.fromLocalFile(Path.of("/nonexistent/file.txt")); + assertThrows(FileStorageException.class, handler::getInputStream); + } +} diff --git a/conductor-client/src/test/java/org/conductoross/conductor/sdk/file/ManagedFileHandlerTest.java b/conductor-client/src/test/java/org/conductoross/conductor/sdk/file/ManagedFileHandlerTest.java new file mode 100644 index 000000000..6ec22f5ce --- /dev/null +++ b/conductor-client/src/test/java/org/conductoross/conductor/sdk/file/ManagedFileHandlerTest.java @@ -0,0 +1,190 @@ +/* + * Copyright 2026 Conductor Authors. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package org.conductoross.conductor.sdk.file; + +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; + +import org.conductoross.conductor.client.model.file.FileHandle; +import org.conductoross.conductor.client.model.file.StorageType; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +class ManagedFileHandlerTest { + + private static final String FILE_ID = "conductor://file/abc-123"; + private static final String RAW_ID = "abc-123"; + private static final byte[] CONTENT = {1, 2, 3, 4, 5}; + + @TempDir + Path cacheDir; + + private WorkflowFileClient workflowFileClient; + + @BeforeEach + void setUp() { + workflowFileClient = mock(WorkflowFileClient.class); + when(workflowFileClient.getCacheDirectory()).thenReturn(cacheDir.toString()); + when(workflowFileClient.getRetryCount()).thenReturn(3); + } + + @Test + void metadataLazilyLoadedOnFirstAccessAndCached() { + when(workflowFileClient.getMetadata(FILE_ID)).thenReturn(metadataFixture()); + ManagedFileHandler handler = new ManagedFileHandler(FILE_ID, workflowFileClient); + + assertEquals("report.pdf", handler.getFileName()); + assertEquals("application/pdf", handler.getContentType()); + assertEquals(1234L, handler.getFileSize()); + + // three getters, only one server call + verify(workflowFileClient, times(1)).getMetadata(FILE_ID); + } + + @Test + void metadataNotFetchedIfPrePopulated() { + ManagedFileHandler handler = new ManagedFileHandler(FILE_ID, workflowFileClient); + handler.setFileName("pre.txt"); + handler.setContentType("text/plain"); + handler.setFileSize(99L); + handler.setStorageType(StorageType.LOCAL); + + assertEquals("pre.txt", handler.getFileName()); + assertEquals("text/plain", handler.getContentType()); + assertEquals(99L, handler.getFileSize()); + + verify(workflowFileClient, never()).getMetadata(anyString()); + } + + @Test + void getFileHandleIdReturnsConstructorValue() { + ManagedFileHandler handler = new ManagedFileHandler(FILE_ID, workflowFileClient); + assertEquals(FILE_ID, handler.getFileHandleId()); + } + + @Test + void getInputStreamDownloadsAndReadsContent() throws Exception { + when(workflowFileClient.getMetadata(FILE_ID)).thenReturn(metadataFixture()); + stubSuccessfulDownload(); + + ManagedFileHandler handler = new ManagedFileHandler(FILE_ID, workflowFileClient); + + try (InputStream in = handler.getInputStream()) { + assertArrayEquals(CONTENT, in.readAllBytes()); + } + + verify(workflowFileClient, times(1)).download(eq(FILE_ID), eq(StorageType.LOCAL), any(Path.class)); + } + + @Test + void downloadIsIdempotentAcrossCalls() throws Exception { + when(workflowFileClient.getMetadata(FILE_ID)).thenReturn(metadataFixture()); + stubSuccessfulDownload(); + + ManagedFileHandler handler = new ManagedFileHandler(FILE_ID, workflowFileClient); + handler.getInputStream().close(); + handler.getInputStream().close(); + handler.getInputStream().close(); + + verify(workflowFileClient, times(1)).download(anyString(), any(), any(Path.class)); + } + + @Test + void preExistingCacheFileSkipsDownload() throws Exception { + when(workflowFileClient.getMetadata(FILE_ID)).thenReturn(metadataFixture()); + // Seed the cache at the exact path ManagedFileHandler computes. + Path expected = cacheDir.resolve(RAW_ID + "_report.pdf"); + Files.write(expected, CONTENT); + + ManagedFileHandler handler = new ManagedFileHandler(FILE_ID, workflowFileClient); + try (InputStream in = handler.getInputStream()) { + assertArrayEquals(CONTENT, in.readAllBytes()); + } + + verify(workflowFileClient, never()).download(anyString(), any(), any(Path.class)); + } + + @Test + void downloadRetriesUpToConfiguredCount() throws Exception { + when(workflowFileClient.getMetadata(FILE_ID)).thenReturn(metadataFixture()); + + // Fail twice, then succeed on the 3rd attempt (retryCount = 3). + int[] calls = {0}; + doAnswer(inv -> { + calls[0]++; + if (calls[0] < 3) { + throw new FileStorageException("transient"); + } + Path dest = inv.getArgument(2); + Files.createDirectories(dest.getParent()); + Files.write(dest, CONTENT); + return null; + }).when(workflowFileClient).download(anyString(), any(), any(Path.class)); + + ManagedFileHandler handler = new ManagedFileHandler(FILE_ID, workflowFileClient); + try (InputStream in = handler.getInputStream()) { + assertArrayEquals(CONTENT, in.readAllBytes()); + } + + verify(workflowFileClient, times(3)).download(anyString(), any(), any(Path.class)); + } + + @Test + void downloadFailsAfterExhaustingRetries() { + when(workflowFileClient.getMetadata(FILE_ID)).thenReturn(metadataFixture()); + doThrow(new FileStorageException("boom")) + .when(workflowFileClient).download(anyString(), any(), any(Path.class)); + + ManagedFileHandler handler = new ManagedFileHandler(FILE_ID, workflowFileClient); + + FileStorageException ex = assertThrows( + FileStorageException.class, handler::getInputStream); + assertEquals(true, ex.getMessage().contains(FILE_ID)); + verify(workflowFileClient, times(3)).download(anyString(), any(), any(Path.class)); + } + + private FileHandle metadataFixture() { + FileHandle h = new FileHandle(); + h.setFileHandleId(FILE_ID); + h.setFileName("report.pdf"); + h.setContentType("application/pdf"); + h.setFileSize(1234L); + h.setStorageType(StorageType.LOCAL); + return h; + } + + private void stubSuccessfulDownload() { + doAnswer(inv -> { + Path dest = inv.getArgument(2); + Files.createDirectories(dest.getParent()); + Files.write(dest, CONTENT); + return null; + }).when(workflowFileClient).download(anyString(), any(), any(Path.class)); + } +} diff --git a/conductor-client/src/test/java/org/conductoross/conductor/sdk/file/StubFileStorageBackend.java b/conductor-client/src/test/java/org/conductoross/conductor/sdk/file/StubFileStorageBackend.java new file mode 100644 index 000000000..5e600d108 --- /dev/null +++ b/conductor-client/src/test/java/org/conductoross/conductor/sdk/file/StubFileStorageBackend.java @@ -0,0 +1,76 @@ +/* + * Copyright 2026 Conductor Authors. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package org.conductoross.conductor.sdk.file; + +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +import org.conductoross.conductor.client.model.file.StorageType; + +public class StubFileStorageBackend implements FileStorageBackend { + + private final Map uploads = new ConcurrentHashMap<>(); + + @Override + public StorageType getStorageType() { return StorageType.LOCAL; } + + @Override + public void upload(String url, Path localFile) { + try { + uploads.put(url, Files.readAllBytes(localFile)); + } catch (Exception e) { + throw new FileStorageException("Stub upload failed", e); + } + } + + @Override + public void upload(String url, InputStream inputStream, long contentLength) { + try { + uploads.put(url, inputStream.readAllBytes()); + } catch (Exception e) { + throw new FileStorageException("Stub stream upload failed", e); + } + } + + @Override + public void download(String url, Path destination) { + try { + byte[] data = uploads.get(url); + if (data == null) { + throw new FileStorageException("No data at: " + url); + } + Files.createDirectories(destination.getParent()); + Files.write(destination, data); + } catch (FileStorageException e) { + throw e; + } catch (Exception e) { + throw new FileStorageException("Stub download failed", e); + } + } + + @Override + public String uploadPart(String url, Path localFile, long offset, long length) { + try { + byte[] data = Files.readAllBytes(localFile); + uploads.put(url + ":" + offset, data); + return "stub-etag-" + offset; + } catch (Exception e) { + throw new FileStorageException("Stub part upload failed", e); + } + } + + public byte[] getUploaded(String url) { return uploads.get(url); } +} diff --git a/conductor-client/src/test/java/org/conductoross/conductor/sdk/file/storage/LocalFileStorageBackendTest.java b/conductor-client/src/test/java/org/conductoross/conductor/sdk/file/storage/LocalFileStorageBackendTest.java new file mode 100644 index 000000000..6b9c4f91e --- /dev/null +++ b/conductor-client/src/test/java/org/conductoross/conductor/sdk/file/storage/LocalFileStorageBackendTest.java @@ -0,0 +1,88 @@ +/* + * Copyright 2026 Conductor Authors. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package org.conductoross.conductor.sdk.file.storage; + +import java.io.ByteArrayInputStream; +import java.nio.file.FileSystemNotFoundException; +import java.nio.file.Files; +import java.nio.file.Path; + +import org.conductoross.conductor.client.model.file.StorageType; +import org.conductoross.conductor.client.storage.LocalFileStorageBackend; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import static org.junit.jupiter.api.Assertions.*; + +class LocalFileStorageBackendTest { + + private final LocalFileStorageBackend backend = new LocalFileStorageBackend(); + + @Test + void storageTypeIsLocal() { + assertEquals(StorageType.LOCAL, backend.getStorageType()); + } + + @Test + void uploadFromPathWritesToFileUri(@TempDir Path tempDir) throws Exception { + Path source = Files.write(tempDir.resolve("src.bin"), new byte[] {1, 2, 3}); + Path dest = tempDir.resolve("uploads/out.bin"); + String url = dest.toUri().toString(); + + backend.upload(url, source); + + assertTrue(Files.exists(dest)); + assertArrayEquals(new byte[] {1, 2, 3}, Files.readAllBytes(dest)); + } + + @Test + void uploadFromStreamWritesToFileUri(@TempDir Path tempDir) throws Exception { + byte[] payload = "hello".getBytes(); + Path dest = tempDir.resolve("uploads/stream.bin"); + String url = dest.toUri().toString(); + + backend.upload(url, new ByteArrayInputStream(payload), payload.length); + + assertTrue(Files.exists(dest)); + assertArrayEquals(payload, Files.readAllBytes(dest)); + } + + @Test + void downloadReadsFromFileUri(@TempDir Path tempDir) throws Exception { + Path source = Files.write(tempDir.resolve("src.bin"), new byte[] {9, 8, 7}); + Path dest = tempDir.resolve("cache/out.bin"); + String url = source.toUri().toString(); + + backend.download(url, dest); + + assertTrue(Files.exists(dest)); + assertArrayEquals(new byte[] {9, 8, 7}, Files.readAllBytes(dest)); + } + + @Test + void httpSchemeRaisesFileSystemNotFound(@TempDir Path tempDir) throws Exception { + Path source = Files.write(tempDir.resolve("src.bin"), new byte[] {0}); + String httpUrl = "https://example.com/uploads/foo"; + + assertThrows( + FileSystemNotFoundException.class, () -> backend.upload(httpUrl, source)); + } + + @Test + void relativeUrlRaisesIllegalArgument(@TempDir Path tempDir) throws Exception { + Path source = Files.write(tempDir.resolve("src.bin"), new byte[] {0}); + String relativeUrl = "uploads/foo"; + + assertThrows(IllegalArgumentException.class, () -> backend.upload(relativeUrl, source)); + } +} diff --git a/examples/file-storage/media-transcoder/.gitignore b/examples/file-storage/media-transcoder/.gitignore new file mode 100644 index 000000000..92322c4e0 --- /dev/null +++ b/examples/file-storage/media-transcoder/.gitignore @@ -0,0 +1,2 @@ +.idea/ +target/ diff --git a/examples/file-storage/media-transcoder/Dockerfile b/examples/file-storage/media-transcoder/Dockerfile new file mode 100644 index 000000000..73f302b67 --- /dev/null +++ b/examples/file-storage/media-transcoder/Dockerfile @@ -0,0 +1,12 @@ +FROM azul/zulu-openjdk-debian:21 + +WORKDIR /app +COPY target/media-transcoder-1.0.0.jar /app/app.jar +COPY src/main/resources/data /app/data + +ENV CONDUCTOR_SERVER_URL=http://conductor-server:8080/api +ENV AWS_S3_ENDPOINT=http://minio:9000 +ENV AWS_ACCESS_KEY_ID=minioadmin +ENV AWS_SECRET_ACCESS_KEY=minioadmin + +CMD ["java", "-jar", "/app/app.jar"] diff --git a/examples/file-storage/media-transcoder/devspace.yaml b/examples/file-storage/media-transcoder/devspace.yaml new file mode 100644 index 000000000..76d0cecd1 --- /dev/null +++ b/examples/file-storage/media-transcoder/devspace.yaml @@ -0,0 +1,160 @@ +version: v2beta1 +name: conductor-file-storage-e2e + +vars: + CONDUCTOR_SERVER_URL: + default: "http://localhost:8080/api" + MINIO_ENDPOINT: + default: "http://localhost:9000" + MINIO_CONSOLE: + default: "http://localhost:9001" + MINIO_USER: + default: "minioadmin" + MINIO_PASSWORD: + default: "minioadmin" + S3_BUCKET: + default: "conductor-files" + S3_REGION: + default: "us-east-1" + NAMESPACE: + default: "conductor-file-storage" + CONDUCTOR_WORKTREE: + default: "../file-storage-conductor" + RUNS: + default: "3" + +images: + media-transcoder: + image: media-transcoder + dockerfile: examples/file-storage/media-transcoder/Dockerfile + context: examples/file-storage/media-transcoder + +deployments: + minio: + kubectl: + manifests: + - ${CONDUCTOR_WORKTREE}/deploy/k8s/namespace.yaml + - ${CONDUCTOR_WORKTREE}/deploy/k8s/minio.yaml + - ${CONDUCTOR_WORKTREE}/deploy/k8s/minio-bucket-init.yaml + + conductor-server: + kubectl: + manifests: + - ${CONDUCTOR_WORKTREE}/deploy/k8s/postgres.yaml + - ${CONDUCTOR_WORKTREE}/deploy/k8s/conductor-config.yaml + - ${CONDUCTOR_WORKTREE}/deploy/k8s/conductor-server.yaml + + media-transcoder: + kubectl: + manifests: + - examples/file-storage/media-transcoder/k8s/deployment.yaml + +dev: + media-transcoder: + imageSelector: media-transcoder + sync: + - path: ./examples/file-storage/media-transcoder/src + containerPath: /app/src + ports: + - port: "5005:5005" + terminal: {} + +commands: + test-local: + description: "Deploy MinIO, start Conductor locally, run media-transcoder N times" + command: | + set -e + PASS=0; FAIL=0 + + echo "=== Deploy MinIO ===" + devspace deploy --deployments minio + kubectl -n ${NAMESPACE} wait --for=condition=ready pod -l app=minio --timeout=60s + kubectl -n ${NAMESPACE} wait --for=condition=complete job/minio-bucket-init --timeout=60s 2>/dev/null || true + kubectl -n ${NAMESPACE} port-forward svc/minio 9000:9000 9001:9001 > /dev/null 2>&1 & + PF_PID=$! + sleep 2 + echo "MinIO ready at ${MINIO_ENDPOINT}" + + echo "" + echo "=== Build + Start Conductor ===" + cd ${CONDUCTOR_WORKTREE} + ./gradlew build -x test -x spotlessCheck -x shadowJar --no-daemon -q + AWS_ACCESS_KEY_ID=${MINIO_USER} AWS_SECRET_ACCESS_KEY=${MINIO_PASSWORD} \ + nohup ./gradlew :conductor-server:bootRun \ + --args="--conductor.file-storage.enabled=true \ + --conductor.file-storage.type=s3 \ + --conductor.file-storage.s3.bucket-name=${S3_BUCKET} \ + --conductor.file-storage.s3.region=${S3_REGION} \ + --conductor.file-storage.s3.endpoint=${MINIO_ENDPOINT}" \ + --no-daemon > /tmp/conductor-server.log 2>&1 & + SERVER_PID=$! + cd - + + echo "Waiting for health..." + for i in $(seq 1 90); do + curl -sf http://localhost:8080/health > /dev/null 2>&1 && echo "Conductor UP" && break + [ "$i" -eq 90 ] && echo "ERROR: Conductor did not start" && tail -10 /tmp/conductor-server.log && kill $PF_PID $SERVER_PID 2>/dev/null && exit 1 + sleep 2 + done + + echo "" + echo "=== Run media-transcoder (${RUNS} times) ===" + for run in $(seq 1 ${RUNS}); do + echo "" + echo "--- Run $run of ${RUNS} ---" + LOG="/tmp/media-transcoder-run${run}.log" + CONDUCTOR_SERVER_URL=${CONDUCTOR_SERVER_URL} \ + AWS_S3_ENDPOINT=${MINIO_ENDPOINT} \ + AWS_ACCESS_KEY_ID=${MINIO_USER} \ + AWS_SECRET_ACCESS_KEY=${MINIO_PASSWORD} \ + mvn -f examples/file-storage/media-transcoder/pom.xml -q exec:java -Dexec.mainClass=io.conductor.example.mediatranscoder.MediaTranscoderApp > "$LOG" 2>&1 || true + + if grep -q "Workflow COMPLETED!" "$LOG"; then + echo "Run $run: PASS" + grep "Output:" "$LOG" | tail -1 + PASS=$((PASS + 1)) + else + echo "Run $run: FAIL" + grep -a "FAILED\|Exception" "$LOG" | grep -v "warning:\|already exists\|Gradle\|deprecated" | tail -3 + FAIL=$((FAIL + 1)) + fi + done + + echo "" + echo "=== Cleanup ===" + kill $SERVER_PID 2>/dev/null && echo "Stopped Conductor" || true + kill $PF_PID 2>/dev/null && echo "Stopped port-forward" || true + + echo "" + echo "===============================" + echo " Results: $PASS PASS / $FAIL FAIL (out of ${RUNS})" + echo "===============================" + [ "$FAIL" -gt 0 ] && exit 1 || exit 0 + + deploy-minio: + description: "Deploy MinIO only + port-forward" + command: | + devspace deploy --deployments minio + kubectl -n ${NAMESPACE} wait --for=condition=ready pod -l app=minio --timeout=60s + kubectl -n ${NAMESPACE} wait --for=condition=complete job/minio-bucket-init --timeout=60s 2>/dev/null || true + echo "MinIO S3: ${MINIO_ENDPOINT}" + echo "MinIO Console: ${MINIO_CONSOLE} (${MINIO_USER}/${MINIO_PASSWORD})" + kubectl -n ${NAMESPACE} port-forward svc/minio 9000:9000 9001:9001 + + teardown: + description: "Delete namespace and all resources" + command: | + echo "Deleting namespace ${NAMESPACE}..." + kubectl delete namespace ${NAMESPACE} --ignore-not-found + +profiles: + - name: local + description: "MinIO on K8s, Conductor + media-transcoder run locally" + patches: + - op: remove + path: deployments.conductor-server + - op: remove + path: deployments.media-transcoder + + - name: full-k8s + description: "Everything on K8s: MinIO + Conductor + media-transcoder" diff --git a/examples/file-storage/media-transcoder/k8s/deployment.yaml b/examples/file-storage/media-transcoder/k8s/deployment.yaml new file mode 100644 index 000000000..bf3c2c255 --- /dev/null +++ b/examples/file-storage/media-transcoder/k8s/deployment.yaml @@ -0,0 +1,32 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: media-transcoder + namespace: conductor-file-storage +spec: + replicas: 1 + selector: + matchLabels: + app: media-transcoder + template: + metadata: + labels: + app: media-transcoder + spec: + initContainers: + - name: wait-for-conductor + image: busybox + command: ["sh", "-c", "until wget -qO- http://conductor-server:8080/health; do sleep 3; done"] + containers: + - name: media-transcoder + image: media-transcoder + imagePullPolicy: IfNotPresent + env: + - name: CONDUCTOR_SERVER_URL + value: "http://conductor-server:8080/api" + - name: AWS_S3_ENDPOINT + value: "http://minio:9000" + - name: AWS_ACCESS_KEY_ID + value: "minioadmin" + - name: AWS_SECRET_ACCESS_KEY + value: "minioadmin" diff --git a/examples/file-storage/media-transcoder/pom.xml b/examples/file-storage/media-transcoder/pom.xml new file mode 100644 index 000000000..98b4d497b --- /dev/null +++ b/examples/file-storage/media-transcoder/pom.xml @@ -0,0 +1,59 @@ + + + 4.0.0 + + io.orkes.examples + media-transcoder + 1.0.0 + jar + + + 21 + 21 + UTF-8 + + + + + org.conductoross + conductor-client + 4.2.0 + + + com.fasterxml.jackson.core + jackson-databind + 2.17.1 + + + org.slf4j + slf4j-simple + 2.0.12 + + + + + + + org.apache.maven.plugins + maven-shade-plugin + 3.5.1 + + + package + shade + + + + io.conductor.example.mediatranscoder.MediaTranscoderApp + + + false + + + + + + + diff --git a/examples/file-storage/media-transcoder/src/main/java/io/conductor/example/mediatranscoder/MediaTranscoderApp.java b/examples/file-storage/media-transcoder/src/main/java/io/conductor/example/mediatranscoder/MediaTranscoderApp.java new file mode 100644 index 000000000..cafdd0408 --- /dev/null +++ b/examples/file-storage/media-transcoder/src/main/java/io/conductor/example/mediatranscoder/MediaTranscoderApp.java @@ -0,0 +1,119 @@ +/* + * Copyright 2026 Conductor Authors. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package io.conductor.example.mediatranscoder; + +import java.io.IOException; +import java.io.InputStream; +import java.util.List; +import java.util.Map; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.netflix.conductor.client.automator.TaskRunnerConfigurer; +import com.netflix.conductor.client.http.ConductorClient; +import com.netflix.conductor.client.http.MetadataClient; +import com.netflix.conductor.client.http.TaskClient; +import com.netflix.conductor.client.http.WorkflowClient; +import com.netflix.conductor.client.worker.Worker; +import com.netflix.conductor.common.metadata.workflow.StartWorkflowRequest; +import com.netflix.conductor.common.metadata.workflow.WorkflowDef; +import com.netflix.conductor.common.run.Workflow; +import com.netflix.conductor.sdk.workflow.executor.task.AnnotatedWorker; +import io.conductor.example.mediatranscoder.workers.ManifestWorker; +import io.conductor.example.mediatranscoder.workers.ThumbnailWorker; +import io.conductor.example.mediatranscoder.workers.TranscodeWorker; +import io.conductor.example.mediatranscoder.workers.UploadPrimaryVideoWorker; +import org.conductoross.conductor.client.FileClient; + +public class MediaTranscoderApp { + + public static void main(String[] args) throws Exception { + String serverUrl = System.getenv().getOrDefault( + "CONDUCTOR_SERVER_URL", "http://localhost:8080/api"); + + System.out.println("Connecting to Conductor at: " + serverUrl); + + ConductorClient client = ConductorClient.builder() + .basePath(serverUrl) + .build(); + + TaskClient taskClient = new TaskClient(client); + WorkflowClient workflowClient = new WorkflowClient(client); + MetadataClient metadataClient = new MetadataClient(client); + + FileClient fileClient = new FileClient(client); + + // 1. Register (or update) the workflow definition. + WorkflowDef workflowDef = register(metadataClient); + + // 2. Start workers. upload_primary_video runs first inside the workflow and publishes + // the primary video handle; downstream tasks consume it via + // ${upload_primary_video_ref.output.primary_video}. + // TranscodeWorker is an @WorkerTask-annotated bean — wrap it as an AnnotatedWorker so + // the TaskRunner treats it like any Worker. Its input is bound to a TranscodeInput POJO + // by the SDK's FileHandlerDeserializer, demonstrating FileHandler as a POJO field. + TranscodeWorker transcodeBean = new TranscodeWorker(); + AnnotatedWorker transcodeWorker = new AnnotatedWorker( + "transcode_video", + TranscodeWorker.class.getMethod("transcode", TranscodeWorker.TranscodeInput.class), + transcodeBean); + + List workers = List.of( + new UploadPrimaryVideoWorker(), + transcodeWorker, + new ThumbnailWorker(), + new ManifestWorker()); + + TaskRunnerConfigurer configurer = new TaskRunnerConfigurer.Builder(taskClient, workers) + .withFileClient(fileClient) + .withThreadCount(4) + .build(); + configurer.init(); + System.out.println("Workers started: upload_primary_video, transcode_video, extract_thumbnail, create_manifest"); + + // 3. Start workflow — no inputs; upload_primary_video publishes primaryVideo. + StartWorkflowRequest request = new StartWorkflowRequest(); + request.setName(workflowDef.getName()); + request.setVersion(workflowDef.getVersion()); + request.setInput(Map.of()); + + String workflowId = workflowClient.startWorkflow(request); + System.out.println("Workflow started: " + workflowId); + + // 5. Poll for completion + System.out.println("Waiting for workflow to complete..."); + for (int i = 0; i < 30; i++) { + Thread.sleep(2000); + Workflow workflow = workflowClient.getWorkflow(workflowId, true); + System.out.println(" Status: " + workflow.getStatus()); + if (workflow.getStatus().isTerminal()) { + System.out.println("Workflow " + workflow.getStatus() + "!"); + System.out.println("Output: " + workflow.getOutput()); + configurer.shutdown(); + System.exit(workflow.getStatus().isSuccessful() ? 0 : 1); + } + } + + System.err.println("Workflow did not complete in 60s"); + configurer.shutdown(); + System.exit(1); + } + + public static WorkflowDef register(MetadataClient metadataClient) throws IOException { + try (InputStream is = MediaTranscoderApp.class.getResourceAsStream("/workflow/media_transcode.json")) { + WorkflowDef def = new ObjectMapper().readValue(is, WorkflowDef.class); + metadataClient.updateWorkflowDefs(List.of(def)); + System.out.println("Registered workflow: " + def.getName() + " v" + def.getVersion()); + return def; + } + } +} diff --git a/examples/file-storage/media-transcoder/src/main/java/io/conductor/example/mediatranscoder/workers/ManifestWorker.java b/examples/file-storage/media-transcoder/src/main/java/io/conductor/example/mediatranscoder/workers/ManifestWorker.java new file mode 100644 index 000000000..f22794e49 --- /dev/null +++ b/examples/file-storage/media-transcoder/src/main/java/io/conductor/example/mediatranscoder/workers/ManifestWorker.java @@ -0,0 +1,80 @@ +/* + * Copyright 2026 Conductor Authors. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package io.conductor.example.mediatranscoder.workers; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; + +import com.netflix.conductor.client.worker.Worker; +import com.netflix.conductor.common.metadata.tasks.Task; +import com.netflix.conductor.common.metadata.tasks.TaskResult; +import org.conductoross.conductor.sdk.file.FileHandler; +import org.conductoross.conductor.sdk.file.FileUploadOptions; + +public class ManifestWorker implements Worker { + + public static final java.lang.String STRING = "{\"video\":\"%%s\",\"videoName\":\"%%s\",\"videoSize\":%%d,\"thumbnail\":\"%%s\",\"thumbName\":\"%%s\",\"thumbSize\":%%d}"; + + @Override + public String getTaskDefName() { + return "create_manifest"; + } + + @Override + public TaskResult execute(Task task) { + TaskResult result = new TaskResult(task); + FileHandler video = task.getInputFileHandler("transcoded_video"); + FileHandler thumb = task.getInputFileHandler("thumbnail"); + + try { + System.out.println("[create_manifest] Inputs:"); + describe(video); + describe(thumb); + + String manifest = String.format( + STRING.formatted(), + video.getFileHandleId(), video.getFileName(), video.getFileSize(), + thumb.getFileHandleId(), thumb.getFileName(), thumb.getFileSize()); + + Path manifestFile = Files.createTempFile("manifest-", ".json"); + Files.writeString(manifestFile, manifest); + + FileUploadOptions options = new FileUploadOptions().setContentType("application/json").setTaskId(task.getTaskId()); + FileHandler uploaded = task.getFileUploader().upload(manifestFile, options); + result.getOutputData().put("output_file", uploaded.getFileHandleId()); + result.setStatus(TaskResult.Status.COMPLETED); + + System.out.println("[create_manifest] Uploaded: " + uploaded.getFileHandleId()); + System.out.println("[create_manifest] Content: " + manifest); + } catch (Exception e) { + result.setStatus(TaskResult.Status.FAILED); + result.setReasonForIncompletion(e.getMessage()); + e.printStackTrace(); + } + return result; + } + + private static void describe(FileHandler fileHandler) throws IOException { + String content = new String(fileHandler.getInputStream().readAllBytes(), StandardCharsets.UTF_8); + System.out.println("================="); + System.out.printf("fileHandleId=%s fileName=%s contentType=%s size=%d bytes%n content=%s%n", + fileHandler.getFileHandleId(), + fileHandler.getFileName(), + fileHandler.getContentType(), + fileHandler.getFileSize(), + content); + System.out.println("================="); + } +} diff --git a/examples/file-storage/media-transcoder/src/main/java/io/conductor/example/mediatranscoder/workers/ThumbnailWorker.java b/examples/file-storage/media-transcoder/src/main/java/io/conductor/example/mediatranscoder/workers/ThumbnailWorker.java new file mode 100644 index 000000000..67a96d587 --- /dev/null +++ b/examples/file-storage/media-transcoder/src/main/java/io/conductor/example/mediatranscoder/workers/ThumbnailWorker.java @@ -0,0 +1,53 @@ +/* + * Copyright 2026 Conductor Authors. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package io.conductor.example.mediatranscoder.workers; + +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; + +import com.netflix.conductor.client.worker.Worker; +import com.netflix.conductor.common.metadata.tasks.Task; +import com.netflix.conductor.common.metadata.tasks.TaskResult; +import org.conductoross.conductor.sdk.file.FileHandler; +import org.conductoross.conductor.sdk.file.FileUploadOptions; +import org.conductoross.conductor.sdk.file.FileUploader; + +public class ThumbnailWorker implements Worker { + + @Override + public String getTaskDefName() { return "extract_thumbnail"; } + + @Override + public TaskResult execute(Task task) { + TaskResult result = new TaskResult(task); + FileHandler primary = task.getInputFileHandler("primary_video"); + + try (InputStream in = primary.getInputStream()) { + Path thumbnail = Files.createTempFile("thumb-", ".png"); + Files.write(thumbnail, "PNG_THUMBNAIL_DATA".getBytes()); + + FileUploadOptions options = new FileUploadOptions().setContentType("image/png").setTaskId(task.getTaskId()); + FileHandler uploaded = task.getFileUploader().upload(thumbnail, options); + result.getOutputData().put("output_file", uploaded.getFileHandleId()); + result.setStatus(TaskResult.Status.COMPLETED); + + System.out.println("[extract_thumbnail] Uploaded: " + uploaded.getFileHandleId()); + } catch (Exception e) { + result.setStatus(TaskResult.Status.FAILED); + result.setReasonForIncompletion(e.getMessage()); + e.printStackTrace(); + } + return result; + } +} diff --git a/examples/file-storage/media-transcoder/src/main/java/io/conductor/example/mediatranscoder/workers/TranscodeWorker.java b/examples/file-storage/media-transcoder/src/main/java/io/conductor/example/mediatranscoder/workers/TranscodeWorker.java new file mode 100644 index 000000000..fe7861461 --- /dev/null +++ b/examples/file-storage/media-transcoder/src/main/java/io/conductor/example/mediatranscoder/workers/TranscodeWorker.java @@ -0,0 +1,42 @@ +/* + * Copyright 2026 Conductor Authors. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package io.conductor.example.mediatranscoder.workers; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; + +import com.netflix.conductor.sdk.workflow.task.OutputParam; +import com.netflix.conductor.sdk.workflow.task.WorkerTask; +import org.conductoross.conductor.sdk.file.FileHandler; + +public class TranscodeWorker { + + public static class TranscodeInput { + public FileHandler primary_video; + public String resolution; + } + + @WorkerTask("transcode_video") + public @OutputParam("output_file") FileHandler transcode(TranscodeInput input) throws IOException { + Path transcoded = Files.createTempFile("transcoded-" + input.resolution + "-", ".mp4"); + try (InputStream in = input.primary_video.getInputStream()) { + Files.write(transcoded, ("TRANSCODED:" + input.resolution + "\n").getBytes()); + Files.write(transcoded, in.readAllBytes(), StandardOpenOption.APPEND); + } + System.out.println("[transcode_video] Transcoded to " + transcoded); + return FileHandler.fromLocalFile(transcoded, "video/mp4"); + } +} diff --git a/examples/file-storage/media-transcoder/src/main/java/io/conductor/example/mediatranscoder/workers/UploadPrimaryVideoWorker.java b/examples/file-storage/media-transcoder/src/main/java/io/conductor/example/mediatranscoder/workers/UploadPrimaryVideoWorker.java new file mode 100644 index 000000000..f93c3060f --- /dev/null +++ b/examples/file-storage/media-transcoder/src/main/java/io/conductor/example/mediatranscoder/workers/UploadPrimaryVideoWorker.java @@ -0,0 +1,57 @@ +/* + * Copyright 2026 Conductor Authors. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package io.conductor.example.mediatranscoder.workers; + +import java.io.InputStream; + +import com.netflix.conductor.client.worker.Worker; +import com.netflix.conductor.common.metadata.tasks.Task; +import com.netflix.conductor.common.metadata.tasks.TaskResult; +import org.conductoross.conductor.sdk.file.FileHandler; +import org.conductoross.conductor.sdk.file.FileUploadOptions; + +public class UploadPrimaryVideoWorker implements Worker { + + private static final String RESOURCE_PATH = "/data/primary_video.mov"; + + @Override + public String getTaskDefName() { + return "upload_primary_video"; + } + + @Override + public TaskResult execute(Task task) { + TaskResult result = new TaskResult(task); + + try (InputStream in = UploadPrimaryVideoWorker.class.getResourceAsStream(RESOURCE_PATH)) { + if (in == null) { + throw new IllegalStateException("Primary video not found on classpath at " + RESOURCE_PATH); + } + + FileUploadOptions options = new FileUploadOptions() + .setFileName("primary_video.mov") + .setContentType("video/quicktime"); + FileHandler uploaded = task.getFileUploader().upload(in, options); + + result.getOutputData().put("primary_video", uploaded); + result.setStatus(TaskResult.Status.COMPLETED); + + System.out.println("[upload_primary_video] Uploaded: " + uploaded.getFileHandleId()); + } catch (Exception e) { + result.setStatus(TaskResult.Status.FAILED); + result.setReasonForIncompletion(e.getMessage()); + e.printStackTrace(); + } + return result; + } +} diff --git a/examples/file-storage/media-transcoder/src/main/resources/data/primary_video.mov b/examples/file-storage/media-transcoder/src/main/resources/data/primary_video.mov new file mode 100644 index 000000000..cbdd5bac5 --- /dev/null +++ b/examples/file-storage/media-transcoder/src/main/resources/data/primary_video.mov @@ -0,0 +1 @@ +FAKE_MASTER_VIDEO_DATA_FOR_TESTING diff --git a/examples/file-storage/media-transcoder/src/main/resources/workflow/media_transcode.json b/examples/file-storage/media-transcoder/src/main/resources/workflow/media_transcode.json new file mode 100644 index 000000000..7061b5aa9 --- /dev/null +++ b/examples/file-storage/media-transcoder/src/main/resources/workflow/media_transcode.json @@ -0,0 +1,62 @@ +{ + "name": "media_transcode", + "version": 1, + "tasks": [ + { + "name": "upload_primary_video", + "taskReferenceName": "upload_primary_video_ref", + "type": "SIMPLE", + "inputParameters": {} + }, + { + "name": "fork_processing", + "taskReferenceName": "fork_ref", + "type": "FORK_JOIN", + "forkTasks": [ + [ + { + "name": "transcode_video", + "taskReferenceName": "transcode_ref", + "type": "SIMPLE", + "inputParameters": { + "primary_video": "${upload_primary_video_ref.output.primary_video}", + "resolution": "720p" + } + } + ], + [ + { + "name": "extract_thumbnail", + "taskReferenceName": "thumbnail_ref", + "type": "SIMPLE", + "inputParameters": { + "primary_video": "${upload_primary_video_ref.output.primary_video}" + } + } + ] + ] + }, + { + "name": "join_processing", + "taskReferenceName": "join_ref", + "type": "JOIN", + "joinOn": ["transcode_ref", "thumbnail_ref"] + }, + { + "name": "create_manifest", + "taskReferenceName": "manifest_ref", + "type": "SIMPLE", + "inputParameters": { + "transcoded_video": "${transcode_ref.output.output_file}", + "thumbnail": "${thumbnail_ref.output.output_file}" + } + } + ], + "inputParameters": [], + "outputParameters": { + "primary_video": "${upload_primary_video_ref.output.primary_video}", + "manifest": "${manifest_ref.output.output_file}", + "transcoded_video": "${transcode_ref.output.output_file}", + "thumbnail": "${thumbnail_ref.output.output_file}" + } +} diff --git a/tests/src/test/java/io/orkes/conductor/client/filestorage/FileStorageIntegrationTest.java b/tests/src/test/java/io/orkes/conductor/client/filestorage/FileStorageIntegrationTest.java new file mode 100644 index 000000000..e64386a38 --- /dev/null +++ b/tests/src/test/java/io/orkes/conductor/client/filestorage/FileStorageIntegrationTest.java @@ -0,0 +1,210 @@ +/* + * Copyright 2026 Conductor Authors. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package io.orkes.conductor.client.filestorage; + +import java.io.ByteArrayInputStream; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Duration; +import java.util.Map; +import java.util.Set; + +import org.conductoross.conductor.client.FileClient; +import org.conductoross.conductor.sdk.file.FileHandler; +import org.conductoross.conductor.sdk.file.FileUploadOptions; +import org.conductoross.conductor.sdk.file.FileUploader; +import org.conductoross.conductor.sdk.file.ManagedFileHandler; +import org.conductoross.conductor.sdk.file.WorkflowFileClient; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; + +import com.netflix.conductor.common.config.ObjectMapperProvider; +import com.netflix.conductor.common.metadata.workflow.StartWorkflowRequest; +import com.netflix.conductor.common.metadata.workflow.WorkflowDef; +import com.netflix.conductor.sdk.workflow.def.ConductorWorkflow; +import com.netflix.conductor.sdk.workflow.def.tasks.Wait; +import com.netflix.conductor.sdk.workflow.executor.WorkflowExecutor; + +import io.orkes.conductor.client.http.OrkesMetadataClient; +import io.orkes.conductor.client.http.OrkesWorkflowClient; +import io.orkes.conductor.client.util.ClientTestUtil; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +@Disabled +public class FileStorageIntegrationTest { + + private static final String WAIT_WF_NAME = "file_storage_integration_wait_wf"; + + private static final ObjectMapper MAPPER = new ObjectMapperProvider().getObjectMapper(); + private static final TypeReference> MAP_TYPE = new TypeReference<>() {}; + + @TempDir + static Path tempDir; + + private static FileClient fileClient; + private static OrkesWorkflowClient workflowClient; + private static OrkesMetadataClient metadataClient; + private static String workflowId; + + @BeforeAll + static void setUp() { + fileClient = ClientTestUtil.getOrkesClients().getFileClient(); + workflowClient = ClientTestUtil.getOrkesClients().getWorkflowClient(); + metadataClient = ClientTestUtil.getOrkesClients().getMetadataClient(); + + WorkflowExecutor executor = new WorkflowExecutor(ClientTestUtil.getClient(), 10); + ConductorWorkflow wf = new ConductorWorkflow<>(executor); + wf.setName(WAIT_WF_NAME); + wf.setVersion(1); + wf.add(new Wait("wait_task", Duration.ofSeconds(300))); + wf.setTimeoutPolicy(WorkflowDef.TimeoutPolicy.TIME_OUT_WF); + wf.setTimeoutSeconds(600); + wf.registerWorkflow(true, true); + + StartWorkflowRequest req = new StartWorkflowRequest(); + req.setName(WAIT_WF_NAME); + req.setVersion(1); + req.setInput(Map.of()); + workflowId = workflowClient.startWorkflow(req); + assertNotNull(workflowId); + } + + @AfterAll + static void cleanUp() { + try { + workflowClient.terminateWorkflow(workflowId, "integration test cleanup"); + } catch (Exception ignored) { + } + try { + metadataClient.unregisterWorkflowDef(WAIT_WF_NAME, 1); + } catch (Exception ignored) { + } + } + + @Test + @DisplayName("upload(Path) returns prefixed handle and preserves filename + content type") + void uploadFromPath() throws Exception { + Path file = writeFile("hello.txt", "hello world"); + + FileHandler handler = fileClient.upload(workflowId, file, + new FileUploadOptions().setContentType("text/plain")); + + assertNotNull(handler.getFileHandleId()); + assertTrue(handler.getFileHandleId().startsWith(FileHandler.PREFIX)); + assertEquals("hello.txt", handler.getFileName()); + assertEquals("text/plain", handler.getContentType()); + } + + @Test + @DisplayName("upload(InputStream) buffers to temp file and uploads") + void uploadFromInputStream() { + byte[] payload = "stream payload".getBytes(StandardCharsets.UTF_8); + + FileHandler handler = fileClient.upload(workflowId, new ByteArrayInputStream(payload), + new FileUploadOptions().setFileName("stream.txt").setContentType("text/plain")); + + assertNotNull(handler.getFileHandleId()); + assertTrue(handler.getFileHandleId().startsWith(FileHandler.PREFIX)); + } + + @Test + @DisplayName("a fresh ManagedFileHandler downloads the bytes uploaded under the same handle id") + void downloadRoundTripsContent() throws Exception { + Path file = writeFile("data.bin", "round-trip content 12345"); + + FileHandler uploaded = fileClient.upload(workflowId, file, + new FileUploadOptions().setContentType("application/octet-stream")); + + ManagedFileHandler downloaded = new ManagedFileHandler( + uploaded.getFileHandleId(), new WorkflowFileClient(fileClient, workflowId)); + + try (InputStream in = downloaded.getInputStream()) { + assertEquals("round-trip content 12345", new String(in.readAllBytes(), StandardCharsets.UTF_8)); + } + } + + @Test + @DisplayName("multiple uploads produce distinct handle ids") + void multipleUploadsProduceDistinctHandleIds() throws Exception { + FileHandler h1 = fileClient.upload(workflowId, writeFile("doc1.pdf", "pdf 1"), + new FileUploadOptions().setContentType("application/pdf")); + FileHandler h2 = fileClient.upload(workflowId, writeFile("doc2.pdf", "pdf 2"), + new FileUploadOptions().setContentType("application/pdf")); + + assertNotEquals(h1.getFileHandleId(), h2.getFileHandleId()); + } + + @Test + @DisplayName("FileUploader interface (WorkflowFileClient) yields the same handle shape as FileClient") + void uploadViaFileUploaderInterface() throws Exception { + FileUploader uploader = new WorkflowFileClient(fileClient, workflowId); + + FileHandler handler = uploader.upload(writeFile("via-interface.txt", "interface test"), + new FileUploadOptions().setContentType("text/plain")); + + assertNotNull(handler.getFileHandleId()); + assertTrue(handler.getFileHandleId().startsWith(FileHandler.PREFIX)); + } + + @Test + @DisplayName("multipart=false (default) uploads via single-request path") + void singleRequestUploadIsDefault() throws Exception { + FileHandler handler = fileClient.upload(workflowId, writeFile("default.txt", "single"), + new FileUploadOptions().setContentType("text/plain")); + + assertNotNull(handler.getFileHandleId()); + } + + @Test + @DisplayName("multipart=true uploads succeed (multipart path on capable backends; falls back otherwise)") + void multipartFlagTrueDoesNotFail() throws Exception { + FileHandler handler = fileClient.upload(workflowId, writeFile("mp.bin", "multipart payload"), + new FileUploadOptions().setContentType("application/octet-stream").setMultipart(true)); + + assertNotNull(handler.getFileHandleId()); + assertTrue(handler.getFileHandleId().startsWith(FileHandler.PREFIX)); + } + + @Test + @DisplayName("FileHandler serializes to the fixed three-field shape") + void fileHandlerSerializesToThreeFields() throws Exception { + FileHandler uploaded = fileClient.upload(workflowId, writeFile("ser.txt", "json shape"), + new FileUploadOptions().setContentType("text/plain")); + + Map json = MAPPER.convertValue(uploaded, MAP_TYPE); + + assertEquals(Set.of("fileHandleId", "contentType", "fileName"), json.keySet()); + assertEquals(uploaded.getFileHandleId(), json.get("fileHandleId")); + assertEquals("text/plain", json.get("contentType")); + assertEquals("ser.txt", json.get("fileName")); + } + + private static Path writeFile(String name, String content) throws Exception { + Path file = tempDir.resolve(name); + Files.writeString(file, content); + return file; + } +}