feat: token writer service + refactor (#24)

PR replaces ssh+script based token writing with a token writer service
which runs on Maxwell to write the token files on request, along with
that the codebase is restructured into two packages (zwop and the token
writer services).

Additionally:

- redid some of the state management/config steps, temporarily(?) making
them worse with *plans* to improve on later
- tidied up build/install process, removed unused scripts, etc...
- updated container definition to build both packages and export them as
stages which can be targeted
  - updated compose files to use this
  - updated github workflows to build and publish images
- added the one line config change 🫠 to set the mymdc scope
so that damnit credentials can be used

Closes #16

Author: RobertRosca

.github/workflows/build-image.yml

@@ -6,6 +6,9 @@ on:
       - main
   release:
     types: [published]
+  pull_request:
+    branches:
+      - main
 
 env:
   REGISTRY: ghcr.io
@@ -21,6 +24,15 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - uses: astral-sh/setup-uv@v5
+
+      - name: Resolve version
+        id: version
+        working-directory: ./packages/zwop
+        run: echo "app_version=$(uvx hatch version)" >> "$GITHUB_OUTPUT"
 
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
@@ -32,25 +44,52 @@ jobs:
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: Extract metadata (tags, labels) for Docker
-        id: meta
+      - name: Extract metadata (tags, labels) for zwop
+        id: meta_zwop
         uses: docker/metadata-action@v5
         with:
           images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
           tags: |
             type=ref,event=branch
-            type=semver,pattern={{version}}
-            type=semver,pattern={{major}}.{{minor}}
-            type=semver,pattern={{major}}
+            type=ref,event=pr
+            type=raw,value=${{ steps.version.outputs.app_version }}
+            type=raw,value=staging,enable=${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }}
+            type=raw,value=stable,enable=${{ github.event_name == 'release' }}
+
+      - name: Build zwop image
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          target: zwop
+          push: true # ${{ github.event_name != 'pull_request' }}
+          tags: ${{ steps.meta_zwop.outputs.tags }}
+          labels: ${{ steps.meta_zwop.outputs.labels }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+          build-args: |
+            APP_VERSION=${{ steps.version.outputs.app_version }}
+
+      - name: Extract metadata (tags, labels) for zwop-tws
+        id: meta_zwop_tws
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-tws
+          tags: |
+            type=ref,event=branch
+            type=ref,event=pr
+            type=raw,value=${{ steps.version.outputs.app_version }}
             type=raw,value=staging,enable=${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }}
             type=raw,value=stable,enable=${{ github.event_name == 'release' }}
 
-      - name: Build and push Docker image
+      - name: Build zwop-tws image
         uses: docker/build-push-action@v6
         with:
           context: .
-          push: true
-          tags: ${{ steps.meta.outputs.tags }}
-          labels: ${{ steps.meta.outputs.labels }}
+          target: zwop-tws
+          push: true # ${{ github.event_name != 'pull_request' }}
+          tags: ${{ steps.meta_zwop_tws.outputs.tags }}
+          labels: ${{ steps.meta_zwop_tws.outputs.labels }}
           cache-from: type=gha
-          cache-to: type=gha,mode=max
+          cache-to: type=gha,mode=max
+          build-args: |
+            APP_VERSION=${{ steps.version.outputs.app_version }}

.gitignore

@@ -1,14 +1,13 @@
-id_ed25519
-id_ed25519.pub
 migration
 *.zuliprc
 .env
 .task
 clients.json
 zuliprc.json
 node_modules
-src/zulip_write_only_proxy/_version.py
-src/zulip_write_only_proxy/frontend/static
+certs
+packages/_version.py
+packages/zwop/src/zwop/frontend/static
 zuliprc
 
 # Byte-compiled / optimized / DLL files

Dockerfile

@@ -1,61 +1,71 @@
 # syntax=docker/dockerfile:1
+FROM python:3.13-slim AS base
+COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/
 
-## Frontend
-FROM node:21-slim AS frontend
-ENV PNPM_HOME="/pnpm"
-ENV PATH="$PNPM_HOME:$PATH"
-RUN corepack enable
+ARG APP_VERSION=0.0.0.dev0
 
 WORKDIR /app
 
-COPY --link ./package.json ./pnpm-lock.yaml ./tailwind.config.js ./
-
-RUN --mount=type=cache,id=pnpm,target=/pnpm/store pnpm install --frozen-lockfile
-
-COPY ./src/zulip_write_only_proxy/frontend/templates/ \
-  ./src/zulip_write_only_proxy/frontend/templates/
-
-RUN pnpm build
-
-ADD --link https://unpkg.com/htmx.org@1.9.10/dist/htmx.js \
-  https://unpkg.com/htmx.org@1.9.10/dist/htmx.min.js \
-  ./src/zulip_write_only_proxy/frontend/static/
+ENV \
+  PYTHONOPTIMIZE=2 \
+  SETUPTOOLS_SCM_PRETEND_VERSION=${APP_VERSION} \
+  UV_CACHE_DIR=/opt/uv-cache/ \
+  UV_COMPILE_BYTECODE=1 \
+  UV_LINK_MODE=copy
 
+RUN apt update && \
+  apt install -y wget --no-install-recommends && \
+  rm -rf /var/lib/apt/lists/*
 
-## Server
-FROM python:3.14-slim
-COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/
 
-WORKDIR /app
+FROM base AS dev
 
 ENV \
-  PYTHONOPTIMIZE=2 \
-  UV_COMPILE_BYTECODE=1 \
-  UV_LINK_MODE=copy \
-  UV_CACHE_DIR=/opt/uv-cache/
+  PNPM_HOME="/pnpm" \
+  PATH="/pnpm:$PATH"
 
-RUN apt update && apt install -y openssh-client wget git && rm -rf /var/lib/apt/lists/*
+RUN apt update && \
+  apt install -y node-corepack --no-install-recommends && \
+  corepack enable && \
+  rm -rf /var/lib/apt/lists/*
 
-COPY --link ./pyproject.toml ./uv.lock /app/
+COPY --link ./pyproject.toml ./uv.lock ./.python-version /app/
 
 RUN --mount=type=cache,target=/opt/uv-cache/ \
-  uv sync --locked --no-install-project
+  uv sync --frozen --no-install-workspace
 
 COPY --link README.md ./
-COPY --link src ./src
+COPY --link packages ./packages
+
+RUN --mount=type=cache,id=pnpm,target=/pnpm/store cd packages/zwop \
+  pnpm install --frozen-lockfile
 
 RUN --mount=type=cache,target=/opt/uv-cache/ \
-    --mount=type=bind,source=.git,target=/app/.git \
   uv sync --locked
 
-COPY --link --from=frontend /app/src/zulip_write_only_proxy/frontend \
-  /app/src/zulip_write_only_proxy/frontend
 
-EXPOSE 8000
+FROM dev AS build
+
+RUN --mount=type=cache,target=/opt/uv-cache/ \
+    --mount=type=cache,id=pnpm,target=/pnpm/store \
+  uv build --all-packages
+
+
+FROM base AS zwop-tws
+RUN --mount=type=bind,from=build,source=/app/dist,target=/dist \
+    --mount=type=cache,target=/opt/uv-cache/ \
+  uv pip install --system /dist/zwop_tws-*.whl
+
+CMD ["zwop-tws"]
+
+
+FROM base AS zwop
+RUN --mount=type=bind,from=build,source=/app/dist,target=/dist \
+    --mount=type=cache,target=/opt/uv-cache/ \
+  uv pip install --system /dist/zwop-*.whl /dist/zwop_tws-*.whl
 
-ENV ZWOP_ADDRESS=http://0.0.0.0:8000
+CMD ["zwop"]
 
-CMD ["uv", "run", "-m", "zulip_write_only_proxy.main"]
 
-HEALTHCHECK --start-interval=1s --start-period=30s --interval=60s \
-  CMD wget http://0.0.0.0:8000/api/health || exit 1
+# Use dev as base so that default target is dev
+FROM dev

README.md

@@ -136,7 +136,7 @@ Here is an example configuration, placed in `.vscode/launch.json`:
             "request": "launch",
             "module": "uvicorn",
             "args": [
-                "zulip_write_only_proxy.main:app",
+                "zwop.main:app",
                 "--port",
                 "8000",
             ],

Taskfile.yml

@@ -1,26 +1,20 @@
-
 name: zwop-dev
 
 services:
   server:
     environment:
       - "ZWOP_TOKEN_WRITER__ZWOP_URL=http://localhost:8000/${COMPOSE_PROJECT_NAME}"
-    develop:
-      watch:
-        - action: sync
-          path: ./src/zulip_write_only_proxy
-          target: /app/src/zulip_write_only_proxy
-          ignore:
-            - __pycache__/
-            - "*.pyc"
-            - ".pytest_cache/"
-        - action: sync+restart
-          path: ./pyproject.toml
-          target: /app/pyproject.toml
-        - action: sync+restart
-          path: ./src/zulip_write_only_proxy/main.py
-          target: /app/src/zulip_write_only_proxy/main.py
-    restart: never
+      - "ZWOP_TOKEN_WRITER__URL=https://tws:8443/"
+      - "ZWOP_ADDRESS=http://0.0.0.0:8000"
+
+  tws:
+    build:
+      context: .
+      target: zwop-tws
+    volumes:
+      - ./certs:/app/certs
+    env_file:
+      - .env
 
   traefik:
     image: "traefik:latest"

compose.override.yml

@@ -6,4 +6,3 @@ include:
 services:
   server:
     image: ghcr.io/european-xfel/zulip-write-only-proxy:stable
-    pull_policy: "every_10m"

compose.prod.yml

@@ -9,9 +9,8 @@ networks:
 
 services:
   server:
-    build: .
     environment:
       - "ZWOP_TOKEN_WRITER__ZWOP_URL=https://exfldadev01.desy.de/${COMPOSE_PROJECT_NAME}"
     networks:
       - proxy
-    restart: unless-stopped
+    restart: unless-stopped

compose.staging.yml

@@ -1,14 +1,18 @@
 services:
   server:
-    build: .
+    build:
+      context: .
+      target: zwop
     volumes:
-      - ./config:/config
+      - ./config:/app/config
+      - ./certs:/app/certs
     env_file:
       - .env
     environment:
       - "ZWOP_PROXY_ROOT=/${COMPOSE_PROJECT_NAME}"
       - "ZWOP_PROXY_HEADERS=true"
       - "ZWOP_FORWARDED_ALLOW_IPS=*"
+      - "ZWOP_ADDRESS=http://0.0.0.0:8000"
     labels:
       - "traefik.enable=true"
       - "traefik.http.middlewares.${COMPOSE_PROJECT_NAME}-stripprefix.stripprefix.prefixes=/${COMPOSE_PROJECT_NAME}"