Skip to content

NOTES.md

Latest commit

 

History

History
1146 lines (927 loc) · 34.4 KB

NOTES.md

File metadata and controls

1146 lines (927 loc) · 34.4 KB
Raw

Notes

Commands

mkdir -p ~/Documents/kiro-glone && rsync -a --delete --exclude={'.git','.DS_Store','NOTES.md'} ~/Documents/GitHub/glone/ ~/Documents/kiro-glone
kiro ~/Documents/kiro-glone

Snippets

from pathlib import Path
from gaveta.json import write_json
write_json(repos, Path("repos.json"))

.github/dependabot.yml file

# More info: https://docs.github.com/en/code-security/supply-chain-security/configuration-options-for-dependency-updates
version: 2
updates:
  # pip, pipenv, pip-compile, or poetry
  # More info: https://docs.github.com/en/code-security/supply-chain-security/about-dependabot-version-updates
  - package-ecosystem: "pip"
    directory: "/"
    schedule:
      interval: "weekly"
    # Disable version updates for pip dependencies
    open-pull-requests-limit: 0

.github/workflows/release.yml file

# Source: https://cjolowicz.github.io/posts/hypermodern-python-06-ci-cd/

name: Release
on:
  release:
    types: [published]
jobs:
  release:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v2
      - uses: actions/setup-python@v1
        with:
          python-version: "3.6"
          architecture: x64
      - run: pip install poetry==1.1.11
      - run: poetry build
      - run: poetry publish --username=__token__ --password=${{ secrets.PYPI_TOKEN }}

.editorconfig file

# More info: https://editorconfig.org/
# List of properties: https://github.com/editorconfig/editorconfig/wiki/EditorConfig-Properties
root = true

[*]
indent_style = space
indent_size = 4
insert_final_newline = true
trim_trailing_whitespace = true
end_of_line = lf
charset = utf-8

[*.{js,json,y{a,}ml,html}]
indent_style = space
indent_size = 2

[Makefile]
indent_style = tab

[*.{md,rst}]
trim_trailing_whitespace = false
indent_style = space
indent_size = 2

Makefile file

.PHONY: all check type isort black lint bandit

CMD:=poetry run
PYMODULE:=glone

all: check type isort black lint bandit

check:
	poetry check

type:
	$(CMD) mypy $(PYMODULE)

isort:
	$(CMD) isort $(PYMODULE)

black:
	$(CMD) black $(PYMODULE)

lint:
	$(CMD) flake8 $(PYMODULE)

bandit:
	$(CMD) bandit -r $(PYMODULE)

pyproject.toml file (Poetry)

[tool.poetry]
name = "glone"
version = "0.1.0"
description = "A Python CLI to backup all your GitHub repositories."
authors = ["João Palmeiro <joaommpalmeiro@gmail.com>"]
license = "MIT"
readme = "README.md"
homepage = "https://github.com/joaopalmeiro/glone"
repository = "https://github.com/joaopalmeiro/glone"
keywords = ["glone", "cli", "backup", "repo", "repository", "github", "github-api"]
classifiers = [
    "Development Status :: 3 - Alpha",
    "Environment :: Console",
    "Intended Audience :: Developers",
    "Intended Audience :: Information Technology",
    "Intended Audience :: Science/Research",
    "Natural Language :: English",
    "Operating System :: OS Independent",
    "Programming Language :: Python :: Implementation :: CPython",
    "Topic :: Terminals",
    "Topic :: Utilities"
]

[tool.poetry.urls]
"Bug Tracker" = "https://github.com/joaopalmeiro/glone/issues"
"Twitter" = "https://twitter.com/joaompalmeiro"

[tool.poetry.dependencies]
python = "^3.6"
importlib-metadata = {version = "^1.0", python = "<3.8"}
click = "^7.1.2"
ghapi = "^0.1.19"
fastcore = "^1.3.27"
humanize = "^3.13.1"

[tool.poetry.dev-dependencies]
mypy = "^0.812"
isort = "^5.7.0"
black = "^20.8b1"
flake8 = "^3.8.4"
bandit = "^1.7.0"
flake8-bugbear = "^21.3.1"
flake8-comprehensions = "^3.3.1"
pep8-naming = "^0.11.1"
flake8-builtins = "^1.5.3"

[build-system]
requires = ["poetry-core>=1.0.0"]
build-backend = "poetry.core.masonry.api"

[tool.poetry.scripts]
glone = "glone.cli:main"

pyproject.toml file (Hatch)

[build-system]
requires = ["hatchling==1.27.0"]
build-backend = "hatchling.build"

[project]
name = "glone"
dynamic = ["version"]
description = "A CLI to back up all your GitHub repositories."
readme = "README.md"
requires-python = ">=3.10"
license = "MIT"
authors = [{ name = "João Palmeiro", email = "joaopalmeiro@proton.me" }]
classifiers = [
  "Development Status :: 4 - Beta",
  "Environment :: Console",
  "Intended Audience :: Developers",
  "Intended Audience :: Information Technology",
  "Intended Audience :: Science/Research",
  "License :: OSI Approved :: MIT License",
  "Natural Language :: English",
  "Operating System :: OS Independent",
  "Programming Language :: Python",
  "Programming Language :: Python :: 3",
  "Programming Language :: Python :: 3 :: Only",
  "Programming Language :: Python :: 3.10",
  "Programming Language :: Python :: 3.11",
  "Programming Language :: Python :: 3.12",
  "Programming Language :: Python :: 3.13",
  "Programming Language :: Python :: Implementation :: CPython",
  "Topic :: Utilities",
]
dependencies = [
  "click>=8.1.0",
  "httpx>=0.28.0",
  "platformdirs>=4.0.0",
  "pydantic>=2.11.0",
  "trio>=0.29.0",
  "gaveta==0.6.0",
]

[project.urls]
Issues = "https://github.com/joaopalmeiro/glone/issues"
Source = "https://github.com/joaopalmeiro/glone"

[project.scripts]
glone = "glone.cli:main"

[tool.hatch.version]
path = "src/glone/__init__.py"

[tool.hatch.envs.default]
detached = false
dependencies = ["ruff==0.6.9", "mypy==1.12.0"]

[tool.hatch.envs.default.scripts]
lint = ["- ruff check .", "mypy"]
format = ["- ruff check --fix .", "ruff format ."]

[tool.hatch.build]
packages = ["src/glone"]

README.md file

# glone

[![security: bandit](https://img.shields.io/badge/security-bandit-yellow.svg)](https://github.com/PyCQA/bandit)
[![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black)

A Python CLI to backup all your GitHub repositories.

## Development

- `poetry install`
- `poetry shell`

## Tech Stack

- [Click](https://click.palletsprojects.com/) (for the interface)

### Packaging and Development

- [Poetry](https://python-poetry.org/)
- [Mypy](http://mypy-lang.org/)
- [isort](https://pycqa.github.io/isort/)
- [Black](https://github.com/psf/black)
- [Flake8](https://flake8.pycqa.org/)
  - [flake8-bugbear](https://github.com/PyCQA/flake8-bugbear)
  - [flake8-comprehensions](https://github.com/adamchainz/flake8-comprehensions)
  - [pep8-naming](https://github.com/PyCQA/pep8-naming)
  - [flake8-builtins](https://github.com/gforcada/flake8-builtins)
- [Bandit](https://bandit.readthedocs.io/)

This CLI was created with [Cookiecutter](https://github.com/audreyr/cookiecutter) and the [`joaopalmeiro/cookiecutter-templates/python-cli`](https://github.com/joaopalmeiro/cookiecutter-templates) project template.

## Notes

- [Backup script](https://github.com/joaopalmeiro/scriptkit-playground/blob/main/google-zx/backup-gh.mjs).
- [ghapi documentation](https://ghapi.fast.ai/) ([API](https://ghapi.fast.ai/fullapi.html)).
- Commands:
  - `glone joaopalmeiro`.
  - `glone joaopalmeiro -o ~/Downloads`.
  - `glone joaopalmeiro -o ~/Downloads -f tar`.
  - `glone --help`.
  - `glone --version`.
- [GitPython](https://github.com/gitpython-developers/GitPython) package.
- [GitHub CLI](https://cli.github.com/) (a.k.a. `gh`).
- [humanize](https://github.com/jmoiron/humanize/) package. `poetry add humanize`.
# glone

[![Hatch project](https://img.shields.io/badge/%F0%9F%A5%9A-Hatch-4051b5.svg)](https://github.com/pypa/hatch)
[![Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff)
[![Pydantic v2](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/pydantic/pydantic/main/docs/badge/v2.json)](https://pydantic.dev)

A CLI to back up all your [GitHub](https://github.com/) repositories.

- [Source code](https://github.com/joaopalmeiro/glone)
- [PyPI package](https://pypi.org/project/glone/)
- [Snyk Advisor](https://snyk.io/advisor/python/glone)

## Usage

### Via [pipx](https://github.com/pypa/pipx)

```bash
pipx run glone --help
```

```bash
pipx run glone
```

## Development

Install [pyenv](https://github.com/pyenv/pyenv) (if necessary).

```bash
pyenv install && pyenv versions
```

```bash
pip install hatch==1.14.0 && hatch --version
```

```bash
hatch config set dirs.env.virtual .hatch
```

```bash
hatch config show
```

```bash
hatch env create
```

```bash
hatch status
```

```bash
hatch env show
```

```bash
hatch dep show table
```

```bash
hatch run pip list
```

```bash
hatch run glone --help
```

```bash
hatch run glone
```

```bash
hatch run lint
```

```bash
hatch run format
```

## Deployment

```bash
hatch version micro
```

```bash
hatch version minor
```

```bash
hatch version major
```

```bash
hatch build --clean
```

```bash
echo "v$(hatch version)" | pbcopy
```

- Commit and push changes.
- Create a tag on [GitHub Desktop](https://github.blog/2020-05-12-create-and-push-tags-in-the-latest-github-desktop-2-5-release/).
- Check [GitHub](https://github.com/joaopalmeiro/glone/tags).

```bash
hatch publish
```

- Check [PyPI](https://pypi.org/project/glone/).

glone/__init__.py file

import sys

if sys.version_info[:2] >= (3, 8):
    import importlib.metadata as importlib_metadata
else:
    import importlib_metadata

__version__ = importlib_metadata.version(__name__)
async def get_single_archive(
    client: httpx.AsyncClient,
    repo: Repo,
    output_folder: Path,
    limiter: trio.CapacityLimiter,
) -> None:
    async with limiter:
        archive_url = generate_archive_endpoint(repo)
        filename = f"{repo.name}-{repo.default_branch}.{ARCHIVE_FORMAT}"
        output_archive = output_folder / filename

        r = await client.get(archive_url)

        async with await trio.open_file(output_archive, mode="wb") as f:
            # await f.write(r.content)
            # or
            async for chunk in r.aiter_bytes():
                await f.write(chunk)

        click.echo(f"{output_archive} ✓")

glone/cli.py file

from datetime import datetime
from pathlib import Path
from typing import Generator

import click
from fastcore.net import urlsend
from ghapi.all import GH_HOST, GhApi, paged
from humanize import naturalsize

from . import __version__
from .constants import (
    ARCHIVE_FILE_FORMATS,
    DEFAULT_ENV_VARIABLE,
    OUTPUT_FOLDER_PREFIX,
    OUTPUT_FOLDER_SEP,
)
from .utils import get_folder_file_count, get_folder_size


@click.command()
@click.argument("username", type=str)
@click.option(
    "-o",
    "--output",
    type=click.Path(
        exists=True, file_okay=False, dir_okay=True, writable=True, resolve_path=True
    ),
    default=".",
    metavar="PATH",
    show_default=True,
)
@click.option(
    "-f",
    "--file-format",
    type=click.Choice(ARCHIVE_FILE_FORMATS, case_sensitive=False),
    default=ARCHIVE_FILE_FORMATS[0],
    show_default=True,
)
@click.option(
    "-t",
    "--token",
    type=str,
    metavar="VALUE",
    envvar=DEFAULT_ENV_VARIABLE,
    show_envvar=True,
)
@click.version_option(version=__version__)
def main(username: str, output: str, file_format: str, token: str) -> None:
    """A Python CLI to backup all your GitHub repositories."""
    # More info:
    # - https://ghapi.fast.ai/core.html#GhApi
    # - https://ghapi.fast.ai/core.html#Operations
    # (if don't pass the token parameter, then your GITHUB_TOKEN environment variable
    # will be used, if available)
    # - https://ghapi.fast.ai/page.html#paged
    api = GhApi(owner=username, token=token)
    # click.echo(api.headers)
    # click.echo(api.func_dict)

    # More info:
    # - https://ghapi.fast.ai/fullapi.html#repos
    # - https://docs.github.com/en/rest/reference/repos#list-public-repositories
    # (it is not for a specific user)
    # - https://docs.github.com/en/rest/reference/repos#list-repositories-for-a-user
    # (public repositories)
    # - https://docs.github.com/en/rest/reference/repos#list-repositories-for-the-authenticated-user
    # (all repositories)

    # repos = api.repos.list_for_user(username=username, type="all", sort="pushed")
    repos: Generator = paged(
        api.repos.list_for_authenticated_user,
        visibility="all",
        affiliation="owner",
        sort="full_name",
    )

    # More info:
    # - https://stackoverflow.com/a/50110841
    # - https://docs.python.org/3.6/library/pathlib.html#pathlib.Path.mkdir
    # - https://stackoverflow.com/a/32490661
    # - https://docs.python.org/3.6/library/pathlib.html#pathlib.Path.open
    timestamp: str = datetime.today().strftime(f"%Y%m%d{OUTPUT_FOLDER_SEP}%H%M%S")
    output_folder = (
        Path(output) / f"{OUTPUT_FOLDER_PREFIX}{OUTPUT_FOLDER_SEP}{timestamp}"
    )
    output_folder.mkdir(parents=False, exist_ok=False)

    click.echo(f"Output folder: {output_folder}")

    # More info:
    # - https://docs.github.com/en/rest/reference/repos#download-a-repository-archive-zip
    # - https://docs.github.com/en/rest/reference/repos#download-a-repository-archive-tar
    # - https://github.com/fastai/ghapi/issues/22
    # - https://github.com/fastai/fastcore/pull/308
    # - https://github.com/fastai/fastcore/blob/1.3.27/fastcore/net.py#L203
    # - https://stackoverflow.com/a/67964008
    # (ref="" for the master/main branch)
    # Note: It is not working. Use an alternative. See error message for debugging.
    # It would work if the execution was via this if branch, for example:
    # https://github.com/fastai/fastcore/blob/1.3.27/fastcore/net.py#L209
    # api.repos.download_zipball_archive(repo="glone", ref="")
    # api.repos.download_zipball_archive(repo="glone", ref="", archive_format="zip")

    # Workaround:
    # - https://fastcore.fast.ai/net.html#urlsend
    # - https://docs.github.com/en/rest/reference/actions#download-an-artifact
    # - https://docs.python.org/3.6/library/functions.html#open
    # - https://stackoverflow.com/a/6633693
    # - https://click.palletsprojects.com/en/7.x/options/?highlight=choice#choice-options
    # zip_url = (
    #     f"{GH_HOST}/repos/{username}/" + "{repo}/" + f"{file_format}ball" + "/{ref}"
    # )
    # route = {"repo": "glone", "ref": "", "archive_format": file_format}
    # or
    # route = {"repo": "glone", "ref": "", "archive_format": "zip"}

    # click.echo(zip_url)
    # click.echo(route)

    # res, headers = urlsend(
    #     zip_url, "GET", headers=api.headers, route=route, return_headers=True
    # )
    # click.echo(headers)

    # _, _, output_filename = headers["content-disposition"].partition("filename=")
    # click.echo(output_filename)
    # with open(output_folder / output_filename, "wb") as fh:
    #     fh.write(res)

    zip_url = (
        f"{GH_HOST}/repos/{username}/" + "{repo}/" + f"{file_format}ball" + "/{ref}"
    )
    for page in repos:
        # click.echo(len(page))
        for repo in page:
            click.echo(f"Repo: {repo.name}")

            route = {"repo": repo.name, "ref": "", "archive_format": file_format}
            res, headers = urlsend(
                zip_url, "GET", headers=api.headers, route=route, return_headers=True
            )

            _, _, output_filename = headers["content-disposition"].partition(
                "filename="
            )
            output_file_path = output_folder / output_filename

            with open(output_file_path, "wb") as fh:
                fh.write(res)

            click.echo(f"Archive file: {output_file_path}")
            # break
        # break

    click.echo(f"Number of archive files/repos: {get_folder_file_count(output_folder)}")
    # Compare with:
    # du -ch <OUTPUT_FOLDER>/*
    # du -sh <OUTPUT_FOLDER>
    size = get_folder_size(output_folder)
    click.echo(
        "Output folder size (approximate): "
        f"{naturalsize(size, binary=False, gnu=False)}"
    )

    click.echo("Done!")

glone/constants.py file

from typing import Tuple

DEFAULT_ENV_VARIABLE: str = "GITHUB_TOKEN"

# More info: https://en.wikipedia.org/wiki/ZIP_(file_format)
ARCHIVE_FILE_FORMATS: Tuple[str, str] = ("zip", "tar")

OUTPUT_FOLDER_PREFIX: str = "github"
OUTPUT_FOLDER_SEP: str = "-"

glone/utils.py file

from pathlib import Path


def get_folder_size(path: Path) -> int:
    # More info:
    # - https://docs.python.org/3.6/library/pathlib.html
    # - https://docs.python.org/3.6/library/pathlib.html#pathlib.Path.rglob
    # - https://docs.python.org/3.6/library/os.html#os.stat_result
    # - https://geekflare.com/check-file-folder-size-in-python/
    total_size = sum([f.stat().st_size for f in path.rglob("*")])
    return total_size


def get_folder_file_count(path: Path) -> int:
    # More info:
    # - https://www.kite.com/python/answers/how-to-count-the-number-of-files-in-a-directory-in-python
    # - https://docs.python.org/3.6/library/pathlib.html#pathlib.Path.iterdir
    # - https://docs.python.org/3.6/library/pathlib.html#pathlib.Path.is_file
    total_count = len([p for p in path.iterdir() if p.is_file()])
    return total_count

glone/cli/models.py file

class Repos(RootModel[list[Repo]]):
    root: list[Repo]

    def __iter__(self):
        return iter(self.root)

    def __len__(self) -> int:
        return len(self.root)

src/glone/cli/__init__.py file

from pathlib import Path

import click
import httpx
import trio
from gaveta.files import ensure_dir
from gaveta.time import ISOFormat, now_iso

from glone import __version__
from glone.cli.constants import (
    ARCHIVE_FORMAT,
    BASE_OUTPUT_FOLDER,
    BASE_URL,
    DEFAULT_ENV_VARIABLE,
    REPOS_URL,
)
from glone.cli.models import Repo, Repos


def save_repos(repos: list[Repo], output_folder: Path) -> None:
    output_repos = output_folder / "repos.json"

    with output_repos.open(mode="w", encoding="utf-8") as f:
        f.write(Repos.dump_json(repos, indent=2).decode("utf-8"))
        f.write("\n")

    click.echo(f"{output_repos} ✓")


async def get_repos(headers: httpx.Headers) -> list[Repo]:
    repos = []
    next_url = httpx.URL(
        REPOS_URL, params={"visibility": "all", "affiliation": "owner"}
    )

    async with httpx.AsyncClient(headers=headers) as client:
        while True:
            r = await client.get(next_url)

            data = r.json()
            repos.extend(data)

            try:
                next_url = httpx.URL(r.links["next"]["url"])
            except KeyError:
                break

    return Repos.validate_python(repos)


def generate_archive_endpoint(repo: Repo) -> str:
    return f"/repos/{repo.full_name}/{ARCHIVE_FORMAT}ball/{repo.default_branch}"


async def get_single_archive(
    client: httpx.AsyncClient,
    repo: Repo,
    output_folder: Path,
    limiter: trio.CapacityLimiter,
) -> None:
    async with limiter:
        archive_url = generate_archive_endpoint(repo)
        filename = f"{repo.name}-{repo.default_branch}.{ARCHIVE_FORMAT}"
        output_archive = output_folder / filename

        r = await client.get(archive_url)

        async with await trio.open_file(output_archive, mode="wb") as f:
            async for chunk in r.aiter_bytes():
                await f.write(chunk)

        click.echo(f"{output_archive} ✓")


async def get_archives(
    headers: httpx.Headers, repos: list[Repo], output_folder: Path
) -> None:
    limiter = trio.CapacityLimiter(20)

    async with (
        httpx.AsyncClient(
            base_url=BASE_URL, headers=headers, follow_redirects=True
        ) as client,
        trio.open_nursery() as nursery,
    ):
        for repo in repos:
            nursery.start_soon(get_single_archive, client, repo, output_folder, limiter)


@click.command()
@click.option(
    "-t",
    "--token",
    type=str,
    metavar="VALUE",
    envvar=DEFAULT_ENV_VARIABLE,
    show_envvar=True,
)
@click.version_option(version=__version__)
def main(token: str) -> None:
    """A CLI to back up all your GitHub repositories."""
    output_folder = BASE_OUTPUT_FOLDER / now_iso(ISOFormat.BASIC)
    ensure_dir(output_folder)

    headers = httpx.Headers(
        {
            "Accept": "application/vnd.github+json",
            "Authorization": f"Bearer {token}",
            "X-GitHub-Api-Version": "2022-11-28",
        }
    )

    repos = trio.run(get_repos, headers)
    save_repos(repos, output_folder)

    trio.run(get_archives, headers, repos, output_folder)

    click.echo(f"Number of repos: {len(repos)}")
    click.echo(f"Output folder: {output_folder}")
    click.echo("Done!")

.gitignore file

# .ruff_cache/: https://docs.astral.sh/ruff/settings/#cache-dir
# dist/: https://hatch.pypa.io/latest/config/build/#output-directory
# .mypy_cache/: https://github.com/python/mypy/blob/v1.12.0/docs/source/config_file.rst#incremental-mode

__pycache__/
.hatch/
.mypy_cache/
.ruff_cache/
dist/

.DS_Store

repos.json
from pathlib import Path

import click
import httpx
import trio
from gaveta.files import ensure_dir
from gaveta.time import ISOFormat, now_iso

from glone import __version__
from glone.cli.constants import (
    ARCHIVE_FORMAT,
    BASE_OUTPUT_FOLDER,
    BASE_URL,
    DEFAULT_ENV_VARIABLE,
    REPOS_URL,
)
from glone.cli.models import Repo, Repos

# ── Tuning knobs ──────────────────────────────────────────────────────────────
MAX_CONCURRENT_DOWNLOADS = 20
CHUNK_SIZE = 1024 * 1024  # 1 MB — fewer async write() calls through trio's thread pool
# ─────────────────────────────────────────────────────────────────────────────


def save_repos(repos: list[Repo], output_folder: Path) -> None:
    output_repos = output_folder / "repos.json"
    with output_repos.open(mode="w", encoding="utf-8") as f:
        f.write(Repos.dump_json(repos, indent=2).decode("utf-8"))
        f.write("\n")
    click.echo(f"{output_repos} ✓")


def generate_archive_endpoint(repo: Repo) -> str:
    return f"/repos/{repo.full_name}/{ARCHIVE_FORMAT}ball/{repo.default_branch}"


async def download_archive(
    client: httpx.AsyncClient,
    repo: Repo,
    output_folder: Path,
    limiter: trio.CapacityLimiter,
) -> None:
    async with limiter:
        url = generate_archive_endpoint(repo)
        filename = f"{repo.name}-{repo.default_branch}.{ARCHIVE_FORMAT}"
        output_archive = output_folder / filename

        # stream="GET": bytes go straight to disk as they arrive instead of
        # buffering the entire archive in RAM before the first write.
        try:
            async with client.stream("GET", url) as r:
                r.raise_for_status()
                async with await trio.open_file(output_archive, mode="wb") as f:
                    async for chunk in r.aiter_bytes(chunk_size=CHUNK_SIZE):
                        await f.write(chunk)
            click.echo(f"{output_archive} ✓")
        except Exception as exc:
            # Catch per-task: one failed download must not cancel the nursery
            # and abort all other in-flight downloads.
            click.echo(f"{output_archive} ✗ ({exc})", err=True)



async def run(headers: httpx.Headers, output_folder: Path) -> list[Repo]:
    all_repos: list[Repo] = []
    limiter = trio.CapacityLimiter(MAX_CONCURRENT_DOWNLOADS)

    async with (
        # One shared client for both repo listing and archive downloads:
        #   • connection pool is reused — no repeated TLS handshakes
        #   • http2=True multiplexes concurrent API calls over fewer TCP connections
        #     (install httpx[http2] to enable: `pip install httpx[http2]`)
        httpx.AsyncClient(
            base_url=BASE_URL,
            headers=headers,
            follow_redirects=True,
            http2=True,
            limits=httpx.Limits(
                max_connections=MAX_CONCURRENT_DOWNLOADS + 5,
                max_keepalive_connections=MAX_CONCURRENT_DOWNLOADS,
                keepalive_expiry=30,
            ),
        ) as client,
        trio.open_nursery() as nursery,
    ):
        # Pipeline: kick off archive downloads for each page of repos immediately,
        # so downloads are already running while pagination is still in progress.
        # Original code waited for ALL repos before starting ANY download.
        next_url: httpx.URL | None = httpx.URL(
            REPOS_URL, params={"visibility": "all", "affiliation": "owner", "per_page": 100}
        )
        while next_url is not None:
            r = await client.get(next_url)
            r.raise_for_status()
            page = Repos.validate_python(r.json())
            all_repos.extend(page)
            for repo in page:
                nursery.start_soon(download_archive, client, repo, output_folder, limiter)
            next_url = httpx.URL(r.links["next"]["url"]) if "next" in r.links else None

        # Nursery exit waits for all in-flight downloads to finish.

    return all_repos


@click.command()
@click.option(
    "-t",
    "--token",
    type=str,
    metavar="VALUE",
    envvar=DEFAULT_ENV_VARIABLE,
    show_envvar=True,
)
@click.version_option(version=__version__)
def main(token: str) -> None:
    """A CLI to back up all your GitHub repositories."""
    output_folder = BASE_OUTPUT_FOLDER / now_iso(ISOFormat.BASIC)
    ensure_dir(output_folder)
    headers = httpx.Headers(
        {
            "Accept": "application/vnd.github+json",
            "Authorization": f"Bearer {token}",
            "X-GitHub-Api-Version": "2022-11-28",
        }
    )
    # Single trio.run() keeps repo listing and all downloads in one event loop.
    repos = trio.run(run, headers, output_folder)
    save_repos(repos, output_folder)
    click.echo(f"Number of repos: {len(repos)}")
    click.echo(f"Output folder: {output_folder}")
    click.echo("Done!")
from pathlib import Path

import click
import httpx2
import humanize
import trio
from gaveta.files import ensure_dir
from gaveta.time import ISOFormat, now_iso

from glone import __version__
from glone.cli.constants import (
    ARCHIVE_FORMAT,
    BASE_OUTPUT_FOLDER,
    BASE_URL,
    DEFAULT_ENV_VARIABLE,
    USER_AGENT,
)
from glone.cli.models import Repo, Repos


def folder_size(folder: Path) -> str:
    total = sum(f.stat().st_size for f in folder.iterdir())
    return humanize.naturalsize(total)


def save_repos(repos: list[Repo], output_folder: Path) -> None:
    output_repos = output_folder / "repos.json"

    with output_repos.open(mode="w", encoding="utf-8") as f:
        f.write(Repos.dump_json(repos, indent=2).decode("utf-8"))
        f.write("\n")

    click.echo(f"{output_repos} ✓")


async def fetch_repos(client: httpx2.AsyncClient) -> list[Repo]:
    repos = []
    next_url = "/user/repos"
    params = {"visibility": "all", "affiliation": "owner"}

    while next_url is not None:
        r = await client.get(next_url, params=params)
        params = None  # only needed on the first request
        repos.extend(r.json())

        next_link = r.links.get("next", {}).get("url")
        if next_link:
            parsed = httpx2.URL(next_link)
            next_url = str(parsed.copy_with(scheme="", host="", port=None))
        else:
            next_url = None

    return Repos.validate_python(repos)


def archive_path(repo: Repo, output_folder: Path) -> tuple[str, Path]:
    endpoint = f"/repos/{repo.full_name}/{ARCHIVE_FORMAT}ball/{repo.default_branch}"
    filename = f"{repo.name}-{repo.default_branch}.{ARCHIVE_FORMAT}"
    return endpoint, output_folder / filename


async def download_archive(
    client: httpx2.AsyncClient,
    endpoint: str,
    output_archive: Path,
    limiter: trio.CapacityLimiter,
) -> None:
    async with limiter:
        async with await trio.open_file(output_archive, mode="wb") as f:
            async with client.stream("GET", endpoint) as r:
                async for chunk in r.aiter_bytes():
                    await f.write(chunk)

        click.echo(f"{output_archive} ✓")


async def run(headers: httpx2.Headers, output_folder: Path) -> list[Repo]:
    """Single async entrypoint: fetch repo list then download all archives concurrently."""
    async with httpx2.AsyncClient(
        base_url=BASE_URL, headers=headers, follow_redirects=True
    ) as client:
        repos = await fetch_repos(client)
        save_repos(repos, output_folder)

        limiter = trio.CapacityLimiter(20)
        targets = [archive_path(repo, output_folder) for repo in repos]

        async with trio.open_nursery() as nursery:
            for endpoint, output_archive in targets:
                nursery.start_soon(download_archive, client, endpoint, output_archive, limiter)

    return repos


@click.command()
@click.option(
    "-t",
    "--token",
    type=str,
    metavar="VALUE",
    envvar=DEFAULT_ENV_VARIABLE,
    show_envvar=True,
)
@click.version_option(version=__version__)
def main(token: str) -> None:
    """A CLI to back up all your GitHub repositories."""
    output_folder = BASE_OUTPUT_FOLDER / now_iso(ISOFormat.BASIC)
    ensure_dir(output_folder)

    headers = httpx2.Headers(
        {
            "Accept": "application/vnd.github+json",
            "Authorization": f"Bearer {token}",
            "X-GitHub-Api-Version": "2026-03-10",
            "User-Agent": USER_AGENT,
        }
    )

    repos = trio.run(run, headers, output_folder)

    click.echo(f"Number of repos: {len(repos)}")
    click.echo(f"Output folder: {output_folder}")
    click.echo(f"Output folder size: {folder_size(output_folder)}")
    click.echo("Done!")
async def get_single_archive(
    client: httpx2.AsyncClient,
    repo: Repo,
    output_folder: Path,
    limiter: trio.CapacityLimiter,
) -> None:
    archive_url = generate_archive_endpoint(repo)
    filename = f"{repo.name}-{repo.default_branch}.{ARCHIVE_FORMAT}"
    output_archive = output_folder / filename

    async with limiter, await trio.open_file(output_archive, mode="wb") as f, client.stream("GET", archive_url) as r:
        async for chunk in r.aiter_bytes():
            await f.write(chunk)

    click.echo(f"{output_archive} ✓")