feat: data migration from mongo/fs to the PostgreSQL sink

package.json

@@ -54,6 +54,7 @@
   "scripts": {
     "cli": "tsx ./packages/git-proxy-cli/index.ts",
     "cli:js": "node ./packages/git-proxy-cli/dist/index.js",
+    "migrate:postgres": "tsx scripts/migrate-to-postgres.ts",
     "client": "vite --config vite.config.ts",
     "clientinstall": "npm install --prefix client",
     "server": "cross-env ALLOWED_ORIGINS=* tsx index.ts",

scripts/migrate-to-postgres.ts

@@ -0,0 +1,83 @@
+#!/usr/bin/env tsx
+
+/**
+ * Copyright 2026 GitProxy Contributors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import yargs from 'yargs';
+import { hideBin } from 'yargs/helpers';
+
+import { getDatabase } from '../src/config';
+import * as postgres from '../src/db/postgres';
+import { migrate, MigrationSource } from '../src/db/postgres/migrate';
+import { createFileSource } from '../src/db/postgres/migrateFileSource';
+import { createMongoSource } from '../src/db/postgres/migrateMongoSource';
+
+const argv = yargs(hideBin(process.argv))
+  .usage('Usage: $0 --from <mongo|fs> [options]')
+  .option('from', {
+    choices: ['mongo', 'fs'] as const,
+    demandOption: true,
+    describe: 'Source backend to migrate from',
+  })
+  .option('mongoUrl', {
+    type: 'string',
+    describe: 'MongoDB connection string (required when --from mongo)',
+  })
+  .option('dataDir', {
+    type: 'string',
+    describe: 'NeDB data directory (defaults to ./.data/db) when --from fs',
+  })
+  .strict()
+  .parseSync();
+
+const buildSource = async (): Promise<MigrationSource> => {
+  if (argv.from === 'mongo') {
+    if (!argv.mongoUrl) {
+      throw new Error('--mongoUrl is required when --from mongo');
+    }
+    return createMongoSource(argv.mongoUrl);
+  }
+  return createFileSource(argv.dataDir);
+};
+
+const main = async (): Promise<void> => {
+  // The destination is the active sink, so it must be postgres. Reading the
+  // source is independent (its own driver), so the two never clash.
+  const db = getDatabase();
+  if (db.type !== 'postgres') {
+    throw new Error(
+      `The active sink is "${db.type}", but this migration writes to postgres. ` +
+        'Enable the postgres sink (with its connectionString or ' +
+        'GIT_PROXY_POSTGRES_CONNECTION_STRING) before running this.',
+    );
+  }
+
+  const source = await buildSource();
+  try {
+    const summary = await migrate(source, postgres, { log: (message) => console.log(message) });
+    console.log('Migration complete:');
+    console.log(`  users:  ${summary.users.imported} imported, ${summary.users.skipped} skipped`);
+    console.log(`  repos:  ${summary.repos.imported} imported, ${summary.repos.skipped} skipped`);
+    console.log(`  pushes: ${summary.pushes.imported} imported`);
+  } finally {
+    await source.close();
+  }
+};
+
+main().catch((err) => {
+  console.error(`Migration failed: ${err instanceof Error ? err.message : err}`);
+  process.exit(1);
+});

src/db/postgres/migrate.ts

@@ -0,0 +1,115 @@
+/**
+ * Copyright 2026 GitProxy Contributors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { Action } from '../../proxy/actions';
+import { Repo, User } from '../types';
+
+/**
+ * A read-only view over a backend (mongo or fs) that data is migrated *from*.
+ * Implementations own their own connection and must be closed by the caller.
+ */
+export interface MigrationSource {
+  getUsers(): Promise<User[]>;
+  getRepos(): Promise<Repo[]>;
+  getPushes(): Promise<Action[]>;
+  close(): Promise<void>;
+}
+
+/**
+ * The subset of the Postgres adapter used to write migrated records. The
+ * adapter module satisfies this shape directly, so the CLI can pass it as-is.
+ */
+export interface MigrationDestination {
+  findUser(username: string): Promise<User | null>;
+  findUserByEmail(email: string): Promise<User | null>;
+  createUser(user: User): Promise<void>;
+  getRepoByUrl(url: string): Promise<Repo | null>;
+  createRepo(repo: Repo): Promise<Repo>;
+  writeAudit(action: Action): Promise<void>;
+}
+
+export interface MigrationSummary {
+  users: { imported: number; skipped: number };
+  repos: { imported: number; skipped: number };
+  pushes: { imported: number };
+}
+
+export interface MigrateOptions {
+  /** Receives human-readable progress lines. Defaults to a no-op. */
+  log?: (message: string) => void;
+}
+
+/**
+ * Copy users, repos and pushes from `source` into the Postgres `destination`.
+ *
+ * Idempotent and re-runnable: users and repos that already exist (matched by
+ * username/email and URL respectively) are skipped, and pushes are upserted by
+ * their stable string id. Record `_id`s are intentionally NOT carried over —
+ * Postgres assigns fresh UUIDs; push ids (TEXT) are preserved by the upsert.
+ */
+export const migrate = async (
+  source: MigrationSource,
+  destination: MigrationDestination,
+  options: MigrateOptions = {},
+): Promise<MigrationSummary> => {
+  const log = options.log ?? (() => undefined);
+  const summary: MigrationSummary = {
+    users: { imported: 0, skipped: 0 },
+    repos: { imported: 0, skipped: 0 },
+    pushes: { imported: 0 },
+  };
+
+  const users = await source.getUsers();
+  log(`Migrating ${users.length} user(s)...`);
+  for (const user of users) {
+    const existing =
+      (await destination.findUser(user.username)) ||
+      (user.email ? await destination.findUserByEmail(user.email) : null);
+    if (existing) {
+      summary.users.skipped++;
+      continue;
+    }
+    // Legacy documents can lack optional fields the writers dereference:
+    // users synced from AD may have no email (the mail attribute is
+    // optional) or gitAccount. Default them like the mongo upsert path does.
+    await destination.createUser({
+      ...user,
+      email: user.email ?? '',
+      gitAccount: user.gitAccount ?? '',
+    });
+    summary.users.imported++;
+  }
+
+  const repos = await source.getRepos();
+  log(`Migrating ${repos.length} repo(s)...`);
+  for (const repo of repos) {
+    if (await destination.getRepoByUrl(repo.url)) {
+      summary.repos.skipped++;
+      continue;
+    }
+    await destination.createRepo(repo);
+    summary.repos.imported++;
+  }
+
+  const pushes = await source.getPushes();
+  log(`Migrating ${pushes.length} push(es)...`);
+  for (const push of pushes) {
+    await destination.writeAudit(push);
+    summary.pushes.imported++;
+  }
+
+  return summary;
+};

src/db/postgres/migrateFileSource.ts

@@ -0,0 +1,54 @@
+/**
+ * Copyright 2026 GitProxy Contributors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import path from 'path';
+
+import Datastore from '@seald-io/nedb';
+
+import { Action } from '../../proxy/actions';
+import { toClass } from '../helper';
+import { Repo, User } from '../types';
+import { MigrationSource } from './migrate';
+
+// Where the `fs` sink keeps its NeDB datastores.
+const DEFAULT_DATA_DIR = './.data/db';
+
+/**
+ * Build a read-only {@link MigrationSource} backed by the NeDB datastores the
+ * `fs` sink writes. `dataDir` defaults to the location the sink uses
+ * (`./.data/db`). Record `_id`s are ignored by the Postgres writers, which
+ * assign fresh UUIDs.
+ */
+export const createFileSource = (dataDir: string = DEFAULT_DATA_DIR): MigrationSource => {
+  const load = (file: string): Datastore =>
+    new Datastore({ filename: path.join(dataDir, file), autoload: true });
+
+  const users = load('users.db');
+  const repos = load('repos.db');
+  const pushes = load('pushes.db');
+
+  const readAll = async <T>(store: Datastore, proto: object): Promise<T[]> => {
+    const docs = await store.findAsync<Record<string, unknown>>({});
+    return docs.map((doc) => toClass(doc, proto) as T);
+  };
+
+  return {
+    getUsers: () => readAll<User>(users, User.prototype),
+    getRepos: () => readAll<Repo>(repos, Repo.prototype),
+    getPushes: () => readAll<Action>(pushes, Action.prototype),
+    close: () => Promise.resolve(),
+  };
+};

src/db/postgres/migrateMongoSource.ts

@@ -0,0 +1,52 @@
+/**
+ * Copyright 2026 GitProxy Contributors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { MongoClient, MongoClientOptions } from 'mongodb';
+
+import { Action } from '../../proxy/actions';
+import { toClass } from '../helper';
+import { Repo, User } from '../types';
+import { MigrationSource } from './migrate';
+
+/**
+ * Build a read-only {@link MigrationSource} backed by a MongoDB instance.
+ *
+ * The connection is explicit (not the configured sink) so the importer can read
+ * the legacy backend while the active sink points at the Postgres destination.
+ * The caller owns the lifecycle and must `close()` the source when done.
+ */
+export const createMongoSource = async (
+  connectionString: string,
+  options: MongoClientOptions = {},
+): Promise<MigrationSource> => {
+  const client = new MongoClient(connectionString, options);
+  await client.connect();
+  const db = client.db();
+
+  const readAll = async <T>(collection: string, proto: object): Promise<T[]> => {
+    const docs = await db.collection(collection).find().toArray();
+    // toClass drops mongo class identity; the ObjectId `_id` it carries through
+    // is ignored by the Postgres writers, which assign fresh UUIDs.
+    return docs.map((doc) => toClass(doc, proto) as T);
+  };
+
+  return {
+    getUsers: () => readAll<User>('users', User.prototype),
+    getRepos: () => readAll<Repo>('repos', Repo.prototype),
+    getPushes: () => readAll<Action>('pushes', Action.prototype),
+    close: () => client.close(),
+  };
+};