diff --git a/.docker/selfhost/schema.json b/.docker/selfhost/schema.json index e959273733150..82dde71d92133 100644 --- a/.docker/selfhost/schema.json +++ b/.docker/selfhost/schema.json @@ -353,7 +353,7 @@ "properties": { "endpoint": { "type": "string", - "description": "The S3 compatible endpoint. Example: \"https://s3.us-east-1.amazonaws.com\" or \"https://.r2.cloudflarestorage.com\"." + "description": "The S3 compatible endpoint (used by aws-s3 provider). Optional; if omitted, endpoint is derived from region." }, "region": { "type": "string", @@ -420,10 +420,6 @@ "type": "object", "description": "The config for the S3 compatible storage provider.", "properties": { - "endpoint": { - "type": "string", - "description": "The S3 compatible endpoint. Example: \"https://s3.us-east-1.amazonaws.com\" or \"https://.r2.cloudflarestorage.com\"." - }, "region": { "type": "string", "description": "The region for the storage provider. Example: \"us-east-1\" or \"auto\" for R2." @@ -490,6 +486,13 @@ "description": "The presigned key for the cloudflare r2 storage provider." } } + }, + "jurisdiction": { + "type": "string", + "enum": [ + "eu" + ], + "description": "Optional jurisdiction for the cloudflare r2 endpoint. Set to \"eu\" for EU buckets." } } } @@ -548,7 +551,7 @@ "properties": { "endpoint": { "type": "string", - "description": "The S3 compatible endpoint. Example: \"https://s3.us-east-1.amazonaws.com\" or \"https://.r2.cloudflarestorage.com\"." + "description": "The S3 compatible endpoint (used by aws-s3 provider). Optional; if omitted, endpoint is derived from region." }, "region": { "type": "string", @@ -615,10 +618,6 @@ "type": "object", "description": "The config for the S3 compatible storage provider.", "properties": { - "endpoint": { - "type": "string", - "description": "The S3 compatible endpoint. Example: \"https://s3.us-east-1.amazonaws.com\" or \"https://.r2.cloudflarestorage.com\"." - }, "region": { "type": "string", "description": "The region for the storage provider. Example: \"us-east-1\" or \"auto\" for R2." @@ -685,6 +684,13 @@ "description": "The presigned key for the cloudflare r2 storage provider." } } + }, + "jurisdiction": { + "type": "string", + "enum": [ + "eu" + ], + "description": "Optional jurisdiction for the cloudflare r2 endpoint. Set to \"eu\" for EU buckets." } } } @@ -1192,7 +1198,7 @@ "properties": { "endpoint": { "type": "string", - "description": "The S3 compatible endpoint. Example: \"https://s3.us-east-1.amazonaws.com\" or \"https://.r2.cloudflarestorage.com\"." + "description": "The S3 compatible endpoint (used by aws-s3 provider). Optional; if omitted, endpoint is derived from region." }, "region": { "type": "string", @@ -1259,10 +1265,6 @@ "type": "object", "description": "The config for the S3 compatible storage provider.", "properties": { - "endpoint": { - "type": "string", - "description": "The S3 compatible endpoint. Example: \"https://s3.us-east-1.amazonaws.com\" or \"https://.r2.cloudflarestorage.com\"." - }, "region": { "type": "string", "description": "The region for the storage provider. Example: \"us-east-1\" or \"auto\" for R2." @@ -1329,6 +1331,13 @@ "description": "The presigned key for the cloudflare r2 storage provider." } } + }, + "jurisdiction": { + "type": "string", + "enum": [ + "eu" + ], + "description": "Optional jurisdiction for the cloudflare r2 endpoint. Set to \"eu\" for EU buckets." } } } diff --git a/packages/backend/server/src/base/storage/__tests__/r2.spec.ts b/packages/backend/server/src/base/storage/__tests__/r2.spec.ts new file mode 100644 index 0000000000000..36b2dbe3740a9 --- /dev/null +++ b/packages/backend/server/src/base/storage/__tests__/r2.spec.ts @@ -0,0 +1,85 @@ +import test from 'ava'; + +import { R2StorageProvider } from '../providers/r2'; + +function endpointOf(provider: R2StorageProvider) { + return provider.endpointUrl; +} + +test('R2 provider should use account endpoint by default', t => { + const provider = new R2StorageProvider( + { + accountId: 'test-account', + region: 'auto', + credentials: { + accessKeyId: 'test', + secretAccessKey: 'test', + }, + }, + 'test-bucket' + ); + + t.is( + endpointOf(provider), + 'https://test-account.r2.cloudflarestorage.com/test-bucket' + ); +}); + +test('R2 provider should append jurisdiction suffix for EU buckets', t => { + const provider = new R2StorageProvider( + { + accountId: 'test-account', + jurisdiction: 'eu', + region: 'auto', + credentials: { + accessKeyId: 'test', + secretAccessKey: 'test', + }, + }, + 'test-bucket' + ); + + t.is( + endpointOf(provider), + 'https://test-account.eu.r2.cloudflarestorage.com/test-bucket' + ); +}); + +test('R2 provider should throw when accountId is missing', t => { + t.throws( + () => + new R2StorageProvider( + { + region: 'auto', + credentials: { + accessKeyId: 'test', + secretAccessKey: 'test', + }, + } as any, + 'test-bucket' + ) + ); +}); + +test( + 'R2 provider should use default endpoint when jurisdiction is explicitly undefined', + t => { + const provider = new R2StorageProvider( + { + accountId: 'test-account', + jurisdiction: undefined, + region: 'auto', + credentials: { + accessKeyId: 'test', + secretAccessKey: 'test', + }, + }, + 'test-bucket' + ); + + t.is( + endpointOf(provider), + 'https://test-account.r2.cloudflarestorage.com/test-bucket' + ); + } +); diff --git a/packages/backend/server/src/base/storage/providers/index.ts b/packages/backend/server/src/base/storage/providers/index.ts index c8480e8b54446..3ee821eb57106 100644 --- a/packages/backend/server/src/base/storage/providers/index.ts +++ b/packages/backend/server/src/base/storage/providers/index.ts @@ -3,7 +3,11 @@ import { Type } from '@nestjs/common'; import { JSONSchema } from '../../config'; import { FsStorageConfig, FsStorageProvider } from './fs'; import { StorageProvider } from './provider'; -import { R2StorageConfig, R2StorageProvider } from './r2'; +import { + R2_JURISDICTIONS, + R2StorageConfig, + R2StorageProvider, +} from './r2'; import { S3StorageConfig, S3StorageProvider } from './s3'; export type StorageProviderName = 'fs' | 'aws-s3' | 'cloudflare-r2'; @@ -38,7 +42,7 @@ const S3ConfigSchema: JSONSchema = { endpoint: { type: 'string', description: - 'The S3 compatible endpoint. Example: "https://s3.us-east-1.amazonaws.com" or "https://.r2.cloudflarestorage.com".', + 'The S3 compatible endpoint (used by aws-s3 provider). Optional; if omitted, endpoint is derived from region.', }, region: { type: 'string', @@ -89,6 +93,17 @@ const S3ConfigSchema: JSONSchema = { }, }; +const S3ConfigPropertiesWithoutEndpoint = Object.fromEntries( + Object.entries( + ( + S3ConfigSchema as { + type: 'object'; + properties?: Record; + } + ).properties ?? {} + ).filter(([key]) => key !== 'endpoint') +) as Record; + export const StorageJSONSchema: JSONSchema = { oneOf: [ { @@ -137,12 +152,18 @@ export const StorageJSONSchema: JSONSchema = { config: { ...S3ConfigSchema, properties: { - ...S3ConfigSchema.properties, + ...S3ConfigPropertiesWithoutEndpoint, accountId: { type: 'string' as const, description: 'The account id for the cloudflare r2 storage provider.', }, + jurisdiction: { + type: 'string' as const, + enum: [...R2_JURISDICTIONS], + description: + 'Optional jurisdiction for the cloudflare r2 endpoint. Set to "eu" for EU buckets.', + }, usePresignedURL: { type: 'object' as const, description: diff --git a/packages/backend/server/src/base/storage/providers/r2.ts b/packages/backend/server/src/base/storage/providers/r2.ts index 71d511e46c85f..520c31598f217 100644 --- a/packages/backend/server/src/base/storage/providers/r2.ts +++ b/packages/backend/server/src/base/storage/providers/r2.ts @@ -15,8 +15,15 @@ import { SIGNED_URL_EXPIRED, } from './utils'; -export interface R2StorageConfig extends S3StorageConfig { +export const R2_JURISDICTIONS = ['eu'] as const; +type R2Jurisdiction = (typeof R2_JURISDICTIONS)[number]; + +export interface R2StorageConfig extends Omit< + S3StorageConfig, + 'endpoint' | 'forcePathStyle' +> { accountId: string; + jurisdiction?: R2Jurisdiction; usePresignedURL?: { enabled: boolean; urlPrefix?: string; @@ -33,11 +40,16 @@ export class R2StorageProvider extends S3StorageProvider { bucket: string ) { assert(config.accountId, 'accountId is required for R2 storage provider'); + const account = config.jurisdiction + ? `${config.accountId}.${config.jurisdiction}` + : config.accountId; + const endpoint = `https://${account}.r2.cloudflarestorage.com`; + super( { ...config, forcePathStyle: true, - endpoint: `https://${config.accountId}.r2.cloudflarestorage.com`, + endpoint, }, bucket ); diff --git a/packages/backend/server/src/base/storage/providers/s3.ts b/packages/backend/server/src/base/storage/providers/s3.ts index 0759dbc5d1b2c..1e9937822306a 100644 --- a/packages/backend/server/src/base/storage/providers/s3.ts +++ b/packages/backend/server/src/base/storage/providers/s3.ts @@ -47,10 +47,46 @@ function resolveEndpoint(config: S3StorageConfig) { return `https://s3.${config.region}.amazonaws.com`; } +function joinPath(basePath: string, suffix: string) { + const trimmedBase = basePath.endsWith('/') ? basePath.slice(0, -1) : basePath; + const trimmedSuffix = suffix.startsWith('/') ? suffix.slice(1) : suffix; + if (!trimmedBase) { + return `/${trimmedSuffix}`; + } + if (!trimmedSuffix) { + return trimmedBase; + } + return `${trimmedBase}/${trimmedSuffix}`; +} + +function composeEndpointUrl(config: S3CompatConfig) { + const url = new URL(config.endpoint); + if (config.forcePathStyle) { + const firstSegment = url.pathname.split('/').find(Boolean); + if (firstSegment !== config.bucket) { + url.pathname = joinPath(url.pathname, config.bucket); + } + return url.toString(); + } + + const firstSegment = url.pathname.split('/').find(Boolean); + const hostHasBucket = url.hostname.startsWith(`${config.bucket}.`); + const pathHasBucket = firstSegment === config.bucket; + if (!hostHasBucket && !pathHasBucket) { + url.hostname = `${config.bucket}.${url.hostname}`; + } + return url.toString(); +} + export class S3StorageProvider implements StorageProvider { protected logger: Logger; protected client: S3CompatClient; private readonly usePresignedURL: boolean; + private readonly endpoint: string; + + get endpointUrl() { + return this.endpoint; + } constructor( config: S3StorageConfig, @@ -69,6 +105,7 @@ export class S3StorageProvider implements StorageProvider { }, }; + this.endpoint = composeEndpointUrl(compatConfig); this.client = createS3CompatClient(compatConfig, credentials); this.usePresignedURL = usePresignedURL?.enabled ?? false; this.logger = new Logger(`${S3StorageProvider.name}:${bucket}`);