diff --git a/.basedpyright/baseline.json b/.basedpyright/baseline.json index f51f7ce13..0616936cd 100644 --- a/.basedpyright/baseline.json +++ b/.basedpyright/baseline.json @@ -8,75 +8,9 @@ "endColumn": 45, "lineCount": 1 } - }, - { - "code": "reportArgumentType", - "range": { - "startColumn": 41, - "endColumn": 48, - "lineCount": 1 - } - }, - { - "code": "reportArgumentType", - "range": { - "startColumn": 32, - "endColumn": 39, - "lineCount": 1 - } - }, - { - "code": "reportCallIssue", - "range": { - "startColumn": 25, - "endColumn": 46, - "lineCount": 1 - } - }, - { - "code": "reportArgumentType", - "range": { - "startColumn": 30, - "endColumn": 39, - "lineCount": 1 - } - } - ], - "./git/db.py": [ - { - "code": "reportIncompatibleMethodOverride", - "range": { - "startColumn": 8, - "endColumn": 12, - "lineCount": 1 - } - }, - { - "code": "reportIncompatibleMethodOverride", - "range": { - "startColumn": 8, - "endColumn": 14, - "lineCount": 1 - } } ], "./git/index/base.py": [ - { - "code": "reportArgumentType", - "range": { - "startColumn": 30, - "endColumn": 36, - "lineCount": 1 - } - }, - { - "code": "reportArgumentType", - "range": { - "startColumn": 28, - "endColumn": 34, - "lineCount": 1 - } - }, { "code": "reportArgumentType", "range": { @@ -186,14 +120,6 @@ "endColumn": 25, "lineCount": 1 } - }, - { - "code": "reportArgumentType", - "range": { - "startColumn": 52, - "endColumn": 84, - "lineCount": 1 - } } ], "./git/objects/tag.py": [ @@ -274,32 +200,6 @@ } } ], - "./git/refs/log.py": [ - { - "code": "reportArgumentType", - "range": { - "startColumn": 30, - "endColumn": 34, - "lineCount": 1 - } - }, - { - "code": "reportAttributeAccessIssue", - "range": { - "startColumn": 17, - "endColumn": 22, - "lineCount": 1 - } - }, - { - "code": "reportArgumentType", - "range": { - "startColumn": 28, - "endColumn": 30, - "lineCount": 1 - } - } - ], "./git/refs/reference.py": [ { "code": "reportIncompatibleVariableOverride", @@ -310,16 +210,6 @@ } } ], - "./git/refs/symbolic.py": [ - { - "code": "reportAttributeAccessIssue", - "range": { - "startColumn": 15, - "endColumn": 20, - "lineCount": 1 - } - } - ], "./git/refs/tag.py": [ { "code": "reportIncompatibleMethodOverride", @@ -339,22 +229,6 @@ } ], "./git/remote.py": [ - { - "code": "reportAttributeAccessIssue", - "range": { - "startColumn": 26, - "endColumn": 38, - "lineCount": 1 - } - }, - { - "code": "reportAttributeAccessIssue", - "range": { - "startColumn": 26, - "endColumn": 38, - "lineCount": 1 - } - }, { "code": "reportAttributeAccessIssue", "range": { @@ -373,14 +247,6 @@ } ], "./git/repo/base.py": [ - { - "code": "reportReturnType", - "range": { - "startColumn": 15, - "endColumn": 28, - "lineCount": 1 - } - }, { "code": "reportTypedDictNotRequiredAccess", "range": { @@ -446,16 +312,6 @@ } } ], - "./git/repo/fun.py": [ - { - "code": "reportReturnType", - "range": { - "startColumn": 11, - "endColumn": 20, - "lineCount": 1 - } - } - ], "./test/deprecation/test_basic.py": [ { "code": "reportUnusedExpression", diff --git a/.github/workflows/backend-benchmark.yml b/.github/workflows/backend-benchmark.yml new file mode 100644 index 000000000..dccf01cf5 --- /dev/null +++ b/.github/workflows/backend-benchmark.yml @@ -0,0 +1,78 @@ +name: Backend benchmark + +on: + push: + branches: [main] + pull_request: + workflow_dispatch: + +permissions: + contents: read + +jobs: + compare-backends: + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@v7 + with: + fetch-depth: 0 + persist-credentials: false + - uses: actions/setup-python@v7 + with: + python-version: "3.12" + - name: Ensure Git 2.52 or newer + run: | + if ! dpkg --compare-versions "$(git version | awk '{print $3}')" ge 2.52; then + sudo add-apt-repository --yes ppa:git-core/ppa + sudo apt-get update + sudo apt-get install --yes git + fi + - name: Install separate backends with the same interpreter + run: | + python -m venv .bench-cli + python -m venv .bench-gix + .bench-cli/bin/python -m pip install . -r test/performance/benchmark-requirements.txt + .bench-gix/bin/python -m pip install '.[gix]' -r test/performance/benchmark-requirements.txt + - name: Prepare an existing repository outside the timed operations + env: + BENCH_REPO: ${{ runner.temp }}/benchmark-repo + GIT_CONFIG_NOSYSTEM: "1" + GIT_CONFIG_GLOBAL: /dev/null + run: | + git init --object-format=sha1 --ref-format=files "$BENCH_REPO" + git -C "$BENCH_REPO" fetch --no-tags "$GITHUB_WORKSPACE" 6ba2c0a2f9ee7feffd7e079621c4845820180c9a + git -C "$BENCH_REPO" checkout -b benchmark FETCH_HEAD + git -C "$BENCH_REPO" config core.excludesFile /dev/null + git -C "$BENCH_REPO" config core.attributesFile /dev/null + git -C "$BENCH_REPO" config core.fsmonitor false + git -C "$BENCH_REPO" config core.untrackedCache false + git -C "$BENCH_REPO" config gc.auto 0 + git -C "$BENCH_REPO" config maintenance.auto false + mkdir "$BENCH_REPO/.benchmark-cache" + touch "$BENCH_REPO/.benchmark-cache/probe" "$BENCH_REPO/benchmark-untracked.txt" + echo '.benchmark-cache/' >> "$BENCH_REPO/.git/info/exclude" + mkdir benchmark-results + - name: Measure GixPython operations, opening and discovery first + run: | + .bench-gix/bin/python -m test.performance.bench_repository \ + --repo "$RUNNER_TEMP/benchmark-repo" --expect-backend gix -o benchmark-results/gix.json + - name: Measure CLI operations, opening and discovery + run: | + .bench-cli/bin/python -m test.performance.bench_repository \ + --repo "$RUNNER_TEMP/benchmark-repo" --expect-backend cli -o benchmark-results/cli.json + - name: Check parity and publish comparison + run: | + .bench-gix/bin/python -m test.performance.compare_backends \ + benchmark-results/cli.json benchmark-results/gix.json \ + --cli-budget test/performance/cli-budget.json > benchmark-results/comparison.md + cat benchmark-results/comparison.md >> "$GITHUB_STEP_SUMMARY" + .bench-gix/bin/python -m pyperf compare_to \ + benchmark-results/cli.json benchmark-results/gix.json --table --table-format md \ + >> "$GITHUB_STEP_SUMMARY" + - name: Retain raw timings and comparison + if: always() + uses: actions/upload-artifact@v7 + with: + name: backend-benchmark + path: benchmark-results/ diff --git a/.github/workflows/cygwin-test.yml b/.github/workflows/cygwin-test.yml index 4f7032f31..ce9f7078a 100644 --- a/.github/workflows/cygwin-test.yml +++ b/.github/workflows/cygwin-test.yml @@ -45,7 +45,9 @@ jobs: - name: Install Cygwin uses: cygwin/cygwin-install-action@v6 with: - packages: curl git python39 python-setuptools-wheel + packages: >- + curl gcc-core git libcurl-devel libexpat-devel libiconv-devel + libssl-devel make python39 python-setuptools-wheel zlib-devel add-to-path: false # No need to change $PATH outside the Cygwin environment. - name: Arrange for verbose output @@ -53,6 +55,15 @@ jobs: # Arrange for verbose output but without shell environment setup details. echo 'set -x' >~/.bash_profile + - name: Build minimum supported Cygwin Git + run: | + # Cygwin's packaged Git is older than our minimum supported version. + git clone --depth 1 --branch v2.52.0 -- https://github.com/git/git.git /tmp/git-source + # Use Cygwin's normal prefix: GitPython checks for uname next to git. + make -C /tmp/git-source -j2 prefix=/usr NO_GETTEXT=YesPlease NO_TCLTK=YesPlease NO_PERL=YesPlease install + hash -r + test "$(git version)" = 'git version 2.52.0' + - name: Special configuration for Cygwin git run: | git config --global --add safe.directory "$(pwd)" @@ -87,6 +98,9 @@ jobs: run: | pip install ./smmap ./gitdb '.[test]' + - name: Verify Cygwin Git detection + run: python -c 'from git import Git; assert Git.is_cygwin()' + - name: Show POSIX file ownership # Cygwin's `ls -ld` reports the NTFS Owner SID via Cygwin's SID-to-uid # mapping (well-known SIDs by their RID, machine-local accounts by diff --git a/.github/workflows/downstream.yml b/.github/workflows/downstream.yml new file mode 100644 index 000000000..d874cbacd --- /dev/null +++ b/.github/workflows/downstream.yml @@ -0,0 +1,38 @@ +name: Downstream compatibility + +on: + push: + branches: [main] + pull_request: + workflow_dispatch: + +permissions: + contents: read + +jobs: + test: + runs-on: ubuntu-latest + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + project: [langchain, mlflow, bandit, swebench, datahub] + backend: [gix, cli] + steps: + - uses: actions/checkout@v7 + with: + persist-credentials: false + - name: Ensure Git 2.52 or newer + run: | + if ! dpkg --compare-versions "$(git version | awk '{print $3}')" ge 2.52; then + sudo add-apt-repository --yes ppa:git-core/ppa + sudo apt-get update + sudo apt-get install --yes git + fi + - name: Install uv + uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0 + with: + version: '0.12.13' + enable-cache: false + - name: Test latest release against this checkout + run: uv run test/downstream/run.py "${{ matrix.project }}" --backend "${{ matrix.backend }}" diff --git a/.github/workflows/git-cli.yml b/.github/workflows/git-cli.yml new file mode 100644 index 000000000..5f94293dd --- /dev/null +++ b/.github/workflows/git-cli.yml @@ -0,0 +1,42 @@ +name: Minimum Git CLI + +on: + push: + branches: [main] + pull_request: + workflow_dispatch: + +permissions: + contents: read + +jobs: + git-2-52: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - uses: actions/checkout@v7 + with: + repository: git/git + ref: v2.52.0 + path: .git-source + - uses: actions/setup-python@v7 + with: + python-version: '3.12' + - name: Build minimum supported Git + working-directory: .git-source + run: | + make -j2 prefix="$RUNNER_TEMP/git" NO_GETTEXT=YesPlease NO_TCLTK=YesPlease NO_PERL=YesPlease NO_CURL=YesPlease install + echo "$RUNNER_TEMP/git/bin" >> "$GITHUB_PATH" + - name: Install GitPython and test dependencies + run: python -m pip install ./smmap ./gitdb '.[test]' + - name: Verify native format and safety contracts + env: + GIT_CONFIG_NOSYSTEM: '1' + GIT_CONFIG_GLOBAL: /dev/null + GIT_AUTHOR_NAME: GitPython Tests + GIT_AUTHOR_EMAIL: tests@example.invalid + GIT_COMMITTER_NAME: GitPython Tests + GIT_COMMITTER_EMAIL: tests@example.invalid + run: | + git version + python -m pytest --no-cov --tb=short test/test_cli_objects_index.py test/test_cli_safety.py test/test_reflog.py test/test_config.py test/test_remote_cli.py test/test_submodule.py::test_submodule_cli_lifecycle diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 3fbbe2ec6..7c9604950 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -14,10 +14,12 @@ permissions: jobs: test: + name: test (${{ matrix.os-type }}, ${{ matrix.python-version }}${{ matrix.backend == 'gix' && ', gix' || '' }}) strategy: matrix: os-type: [ubuntu, macos, windows] python-version: ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13", "3.14", "3.14t", "3.15", "3.15t"] + backend: [cli] exclude: - os-type: macos python-version: "3.14t" @@ -32,6 +34,21 @@ jobs: experimental: true - python-version: "3.15t" experimental: true + - os-type: ubuntu + python-version: "3.12" + backend: gix + build-docs: false + experimental: false + - os-type: macos + python-version: "3.12" + backend: gix + build-docs: false + experimental: false + - os-type: windows + python-version: "3.12" + backend: gix + build-docs: false + experimental: false fail-fast: false @@ -51,6 +68,34 @@ jobs: with: python-version: ${{ matrix.python-version }} allow-prereleases: ${{ matrix.experimental }} + cache: ${{ matrix.backend == 'gix' && 'pip' || '' }} + cache-dependency-path: | + pyproject.toml + requirements.txt + test-requirements.txt + gix-requirements.txt + + - name: Prepare Rust for GixPython source builds + if: matrix.backend == 'gix' + id: rust + run: | + # GixPython 0.1.0 publishes macOS wheels only and needs Rust >= 1.89. + rustup toolchain install stable --profile minimal + rustup default stable + rustc --version + echo "version=$(rustc --version)" >> "$GITHUB_OUTPUT" + + - name: Cache GixPython Cargo dependencies and build output + if: matrix.backend == 'gix' + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + ${{ runner.temp }}/gix-target + key: gix-${{ runner.os }}-${{ runner.arch }}-py${{ matrix.python-version }}-${{ steps.rust.outputs.version }}-${{ hashFiles('gix-requirements.txt') }} + restore-keys: | + gix-${{ runner.os }}-${{ runner.arch }}-py${{ matrix.python-version }}-${{ steps.rust.outputs.version }}- - name: Set up WSL (Windows) if: matrix.os-type == 'windows' @@ -58,6 +103,19 @@ jobs: with: wsl-version: 1 + - name: Install current Git (Linux) + if: matrix.os-type == 'ubuntu' + run: | + sudo add-apt-repository --yes ppa:git-core/ppa + sudo apt-get update + sudo apt-get install --yes git + + - name: Install current Git (macOS) + if: matrix.os-type == 'macos' + run: | + brew install git + echo "$(brew --prefix git)/bin" >> "$GITHUB_PATH" + - name: Prepare this repo for tests run: | ./init-tests-after-clone.sh @@ -76,7 +134,13 @@ jobs: - name: Install project and test dependencies run: | - pip install ./smmap ./gitdb '.[test]' + pip install ./smmap ./gitdb ".[${EXTRAS}]" + python -c 'import os; from git import _backend; print("Backend:", _backend.name); assert _backend.name == os.environ["EXPECTED_BACKEND"]' + env: + EXTRAS: ${{ matrix.backend == 'gix' && 'test,gix' || 'test' }} + EXPECTED_BACKEND: ${{ matrix.backend }} + # Keep Cargo output when pip removes its temporary source directory. + CARGO_TARGET_DIR: ${{ runner.temp }}/gix-target - name: Show POSIX file ownership # Linux and macOS only. On Windows, Git Bash's `ls -ld` reports a @@ -159,9 +223,22 @@ jobs: - name: Test with pytest run: | - pytest --color=yes -p no:sugar --instafail -vv + if [[ "$RUNNER_OS" == "Windows" && ! -v GIT_PYTHON_GIT_EXECUTABLE ]]; then + # Absolute paths let Windows share minimum-version probes. + git_executable="$(python -c 'import os, shutil; print(os.path.abspath(shutil.which("git")))')" + export GIT_PYTHON_GIT_EXECUTABLE="$git_executable" + fi + pytest --color=yes -p no:sugar --instafail -vv --durations=30 \ + --junitxml=test-results/pytest.xml --backend-report=test-results/backend.json continue-on-error: false + - name: Retain test results and backend operation counts + if: always() + uses: actions/upload-artifact@v7 + with: + name: tests-${{ matrix.os-type }}-${{ matrix.python-version }}-${{ matrix.backend }} + path: test-results/ + - name: Documentation if: matrix.build-docs run: | diff --git a/AGENTS.md b/AGENTS.md index 6b2963cf4..3e2100bae 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -4,6 +4,46 @@ Before starting work, read and follow [CONTRIBUTING.md](CONTRIBUTING.md), including the [Prevent agent impersonation](CONTRIBUTING.md#prevent-agent-impersonation) section governing identification when communicating through a person's account. +# Gix backend implementations + +Gix backend operations must use Gix through the supported GixPython (`gix`) +APIs. Python/stdlib implementations, direct filesystem inference, and custom +native implementations outside Gix are not substitutes for a missing Gix +operation. Do not implement Git behavior independently to avoid CLI calls. + +"Native" in code, documentation, reports and benchmarks means execution by +Gitoxide through GixPython's `gix` API. It never means Python-implemented Git +behavior. Every eliminated CLI call must be backed by identified Gix calls. + +Python glue may validate inputs, adapt Gix results to GitPython's return types +and formatting, manage the retained `gix.Repository`, and select a CLI fallback. +It must not invent a successful result or repair missing Git semantics with a +separate implementation. Retaining and explicitly recreating a Gix repository +through the shared accessor is allowed. + +Adapting native locations may append the fixed metadata leaf names `modules` +and `COMMIT_EDITMSG` to `Repository.git_dir()`, whose location Gix resolves. +Input paths may be canonicalized before reopening through Gix; return metadata +from that native handle. These adaptations must match Git in regression tests +and do not authorize a Python implementation of the general `--git-path` rules. + +If Gix lacks a capability, differs from the required behavior, or its use is +unclear, keep the existing Git CLI path and document the gap in +`doc/gix-backend.md`. State the expected behavior, available evidence, and what +GixPython or Gitoxide needs to expose or fix upstream. Tests, coverage reports +and benchmark ceilings must reflect that fallback; a lower CLI count alone +does not demonstrate a Gix implementation. + +Track every observed difference from the equivalent Git operation as a Git +compatibility bug, even if the Gix behavior is intentional. Gix must provide +matching behavior, at least through a mode as strict as Git. Record Git's +expected behavior, Gix's actual behavior, versions, reproduction or regression, +and the affected adapter/fallback in the ledger. Keep these bugs open when a +Gix-based adapter workaround restores parity; close them only after verifying +the upstream fix or compatible mode. Distinguish demonstrated mismatches from +missing APIs and unverified behavior. Do not repair a compatibility bug by +implementing Git semantics in Python. + # Commit messages Follow Conventional Commits for every commit. Every commit must have a diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 76f276323..8e248cc0a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -31,6 +31,19 @@ Attributing AI assistance in commit metadata, for example with a `Co-authored-by trailer, is welcome but not required. Code is reviewed the same way regardless of its origin. +## Temporary test directories + +Use `test.cleanup.TemporaryDirectory` for isolated temporary directories and +`test.cleanup.cleanup_directory` when disposing of directories created by test +fixtures. Close repository handles before removing their files. Cleanup is +best-effort: it logs filesystem errors, removes whatever it can, and leaves +locked files behind without failing or skipping a test. The shared writable +repository decorators still keep failed tests' directories for debugging. + +Deletions and renames that exercise library behavior or prepare a fixture for +reuse must remain strict. If a cached fixture cannot be cleaned up, rebuild it +at a fresh location before handing it to another test. + ## Fuzzing Test Specific Documentation For details related to contributing to the fuzzing test suite and OSS-Fuzz integration, please diff --git a/MANIFEST.in b/MANIFEST.in index eac2a1514..55cd86673 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -5,6 +5,7 @@ include LICENSE include README.md include VERSION include requirements.txt +include gix-requirements.txt include test-requirements.txt include git/py.typed diff --git a/README.md b/README.md index 029b4fa00..294b2cfc7 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,10 @@ probably the skills to scratch that itch of mine: implement `git` in a way that If you like the idea and want to learn more, please head over to [gitoxide](https://github.com/Byron/gitoxide), an implementation of 'git' in [Rust](https://www.rust-lang.org). -*(Please note that `gitoxide` is not currently available for use in Python, and that Rust is required.)* +The experimental `GitPython[gix]` extra uses the local GixPython bindings for +supported operations, with automatic Git CLI fallback. See the +[local installation, test commands, and conversion ledger](doc/gix-backend.md). +GixPython is not yet available on PyPI. ## GitPython @@ -40,10 +43,10 @@ The project is open to contributions of all kinds, as well as new maintainers. ### REQUIREMENTS GitPython needs the `git` executable to be installed on the system and available in your -`PATH` for most operations. If it is not in your `PATH`, you can help GitPython find it +`PATH` for repository operations. If it is not in your `PATH`, you can help GitPython find it by setting the `GIT_PYTHON_GIT_EXECUTABLE=` environment variable. -- Git (1.7.x or newer) +- Git (2.52 or newer) - Python >= 3.8 The list of dependencies are listed in [`./requirements.txt`](https://github.com/gitpython-developers/GitPython/blob/main/requirements.txt) and [`./test-requirements.txt`](https://github.com/gitpython-developers/GitPython/blob/main/test-requirements.txt). diff --git a/doc/gix-backend.md b/doc/gix-backend.md new file mode 100644 index 000000000..94e4352ac --- /dev/null +++ b/doc/gix-backend.md @@ -0,0 +1,987 @@ +# GixPython backend + +Install the `gix` extra to make GitPython use GixPython for supported operations: +`GitPython[gix]`, or `.[gix]` from this checkout. GixPython is the distribution +name; `gix` is its import name. The extra uses the official PyPI release +`GixPython==0.1.0` on CPython 3.11 or newer. On older Python versions, the +dependency is omitted and GitPython uses the CLI backend. The ordinary +GitPython installation still supports Python 3.8 or newer. + +To install from this checkout with an existing interpreter: + +```sh +uv venv --python .venv/bin/python .tox/gix +uv pip install --python .tox/gix/bin/python --editable '.[test,gix]' +``` + +`tox -e gix` also resolves the published release from the package index. + +GixPython 0.1.0 publishes macOS wheels. On Windows and Linux, installation +builds the released source distribution and requires Rust 1.89 or newer and +a platform C/C++ toolchain. The full-suite Gix CI jobs prepare a stable Rust +toolchain and cache pip's built wheels, Cargo dependencies and compilation +output. + +Python extras add dependencies; they do not leave a runtime feature bit. +GitPython selects this backend when `gix` can be imported. Installing GixPython +separately therefore has the same effect. Use separate virtual environments to +compare the CLI and native installations. There is no backend environment +variable, constructor option, or runtime switch. + +Git 2.52 or newer remains required. Unsupported operations and call options use +the existing CLI implementation. Public `repo.git.()` calls retain +their command-line behavior. Native dispatch applies to library-managed calls. + +Here and in coverage reports, **native means Gitoxide execution through +GixPython's `gix` API**. It never means Python-implemented Git behavior. Python +glue validates inputs, adapts Gix results and manages repository handles. For +example, object reads call `Repository.find_object()` and reference edits call +`Repository.edit_references_as()`. The two fixed metadata paths use +`Repository.git_dir()` plus the requested basename; that adaptation does not +provide general Git-path resolution. A `native` counter must represent a +Gix-backed operation, and backend timings include its Python glue and any CLI +fallbacks. + +## Install and test without package indexes + +The prepared environments in this checkout are `.venv` (CLI) and `.tox/gix` +(GixPython). Run either installation against a disposable repository fixture: + +```sh +.venv/bin/python test/run-local.py --no-cov -q +.tox/gix/bin/python test/run-local.py --no-cov -q \ + --backend-report=.cache/gix-coverage.json +``` + +On Windows, use Git Bash, as CI does, and the environments' `Scripts` +directories. To retain CI's coverage and pytest options: + +```sh +.venv/Scripts/python.exe test/run-local.py --color=yes -p no:sugar --instafail -vv --durations=30 \ + --junitxml=.cache/windows-cli.xml --backend-report=.cache/windows-cli.json +.tox/gix/Scripts/python.exe test/run-local.py --color=yes -p no:sugar --instafail -vv --durations=30 \ + --junitxml=.cache/windows-gix.xml --backend-report=.cache/windows-gix.json +``` + +The Windows runner resolves `git.exe` from `PATH` to an absolute path so +GitPython can share minimum-version checks. An explicit +`GIT_PYTHON_GIT_EXECUTABLE` override is preserved and also used for the runner's +direct Git commands. Git Bash normally selects `mingw64/bin/git.exe`; the +`cmd/git.exe` and `bin/git.exe` launchers add overhead to local measurements. +The private config retains CI's `core.autocrlf=true` on Windows and appends +the test aliases directly, matching CI without introducing config includes. + +The runner uses local version tags, creates an isolated Git configuration, +prepares the historical test fixture inside a temporary shared clone, gives +pytest a separate temporary root for each run, and disables package-index +access. Cleanup of these isolated directories is best-effort, so a locked +leftover cannot change pytest's exit status. Tests use local repositories, +including the tutorial example. The suite needs loopback sockets for its Git daemon and +permission to inspect its own child processes. It does not need a remote Git +server. Missing local tags or packages are errors, not invitations to download. + +The ignored `.cache/gix-wheels` directory contains the published GixPython wheel and +the build/test dependencies available on this machine. To recreate an +environment with an already-installed CPython interpreter: + +```sh +uv --cache-dir .cache/uv venv --offline --python .venv/bin/python .tox/gix +uv --cache-dir .cache/uv pip install --offline --no-index \ + --find-links .cache/gix-wheels --python .tox/gix/bin/python \ + --editable '.[test,gix]' +``` + +For another machine, populate that directory from PyPI or its package cache +before running the offline installation tests. Select wheels compatible with +the interpreter and platform; ordinary and free-threaded CPython use different +wheels. On platforms without a published wheel, first build one from the +released source distribution and retain it in the wheelhouse. + +## Conversion coverage + +The pytest header names the selected backend. Native runs print operation +counts and fallback reasons, and `--backend-report=PATH` saves the same records +as JSON. A record looks like: + +```json +{"method": "IndexFile.write", "outcome": "CLI: split index preservation (GIX-13)", "count": 2} +``` + +Search the report for `not converted` to find remaining command adapters, and +for `GIX-` to find upstream limitations. A high-level method can use several +native and CLI operations; counts describe operations, not entire user calls. +The `git.backend` DEBUG logger reports these decisions outside pytest. +`git._backend.statistics()` returns a snapshot of the process-local counters. +Zero observed calls means untested in that run, not unsupported. + +Successful `Git.execute` launches also appear as `CLI process` records. This +includes raw `repo.git` calls and failed Git commands that started a process, +but excludes failed process creation, direct subprocesses in test helpers, +other Python processes and Git's own child processes. A persistent `cat-file` +launch counts once; subsequent requests through it do not. These counts apply +to both installations and are separate from native/fallback decisions. + +Pytest prints session launch totals split into setup, call, teardown and +collection/session work. `--backend-report` keeps its existing record-list +format and adds `pytest.*` phase records. The `Git.execute` record is +process-lifetime cumulative (including import-time probes); `pytest.session_total` +is the current session's total. Optional ceilings fail the test session on an +increase while allowing reductions: + +```sh +.tox/gix/bin/python test/run-local.py --no-cov -q \ + --backend-report=.cache/gix-processes.json \ + --max-cli-processes=TOTAL --max-cli-test-processes=CALLS +``` + +Replace `TOTAL`/`CALLS` with baselines for the same selected tests and platform. +Setup is deliberately counted separately because fixture improvements and +test additions can change it independently of native backend capability. +These are pytest phases: `unittest.TestCase.setUp()`/`tearDown()` run inside +the call phase, so that column can still include their fixture work. Counts +cover the current Python process; they are not aggregated across xdist workers. +For a stable CI assertion, the pinned repository benchmark uses per-measurement +ceilings in `test/performance/cli-budget.json`: the warm Gix journey launches +one CLI process, opening launches one, and nested discovery launches three. +Reduce these ceilings as conversions land. The other operation rows have zero +ceilings except the one-process patch diff. + +Before the Gix-only audit, this instrumentation on CPython 3.12.14/macOS arm64 and official +GixPython 0.1.0 recorded a full suite without coverage passing 1,674 tests and 38 +subtests (79 skipped, one xfailed) in 408.81 seconds. It recorded **30,898 +`Git.execute` launches**: 6,523 in pytest setup and 24,375 in call phases, +with zero in teardown or collection/session phases. The process-lifetime +counter is 30,899 because the import-time Git probe precedes pytest session +accounting. The same run recorded 21,409 fallback decisions. These totals +include explicit CLI tests and fixture commands, including non-Git commands +passed directly to `Git.execute`; they are a local baseline, not a count of +only fallback calls. The expanded warm benchmark instead isolates a user journey: +43 CLI launches versus one Gix launch, opening nine versus one, and nested +discovery eleven versus three after restoring the reference-format fallback +and sharing successful version checks. +The historical full-suite counts above predate this correction. A constant +`files` answer was not a Gix query; `GIX-1` records the remaining native +repository-format validation gap. + +| GitPython entry point / managed command | Gix implementation | Remaining CLI cases | +| --- | --- | --- | +| `Git.get_object_header`, ODB `info` | Object resolution and native header lookup | Custom storage, unsupported revision grammar | +| `Git.stream_object_data`, ODB `stream` | Object bytes in an independent stream | Objects larger than 8 MiB; GIX-2 | +| Commit/tree serialization readback | Existing ODB stream; no separate `cat-file` process | The same storage and large-object fallbacks as ODB reads | +| `Repo` opening/discovery | Native storage, worktree, object format and empty-tree metadata | Repository-format validation, unsupported layouts/environment and native validation gaps; GIX-1/14/19 | +| Standalone discovery helpers | Native storage directories and gitfiles reopened at their canonical native Git directory | Rejected candidates and the same layout/environment guards as `Repo` opening; GIX-14 | +| `Repo.rev_parse`, `rev_parse` metadata queries | Revision lookup, common directory, object format, bare flag, worktree root, fixed `modules`/`COMMIT_EDITMSG` locations | Repository-format validation, other metadata paths, symlinked metadata leaves, message searches and describe forms; GIX-1/14/17/22 | +| Revision path/mode metadata | Native revision specification, including tree paths and index stages | Custom/sparse indexes, unsupported revision grammar or path normalization | +| Tree enumeration, `ls_tree` | Native tree entries with modes, names, and object IDs | Other command options | +| Reference object reads | One exact native lookup for direct, symbolic and absent references | Partial names and native decoding errors | +| Reference-name validation | Native full-name validation without opening a repository | Standalone names outside the native grammar; GIX-20 | +| Managed `symbolic_ref` | Nonrecursive target lookup | Mutation, command-level missing-reference diagnostics | +| Reference enumeration, `for_each_ref` | Sorted reference names and literal prefixes, preserving symbolic aliases | Dangling symbolic refs, root refs, glob patterns, other formats/options; GIX-25 | +| Managed `config --get KEY` | Merged repository config snapshot | Files/streams, enumeration, mutation; GIX-12 | +| Submodule enumeration and cached fields | Dedicated native `.gitmodules` parser over the existing worktree/blob source, with raw path/URL/branch values | Other/duplicate sections, ambiguous brackets, missing/implicit fields and parser errors; GIX-12 | +| Commit hooks | No bound native lookup/execution API | All hooks, including absent hooks; GIX-23 | +| Worktree inventory, `worktree list` | Main and linked worktree metadata, including bare main repositories and locks | Prunable entries; GIX-14 | +| `Repo.merge_base`, `Repo.is_ancestor` | Native graph queries for two revisions | Octopus/fork-point and other options | +| `Commit.count` | Reachable commit count, skip/limit/first-parent | Path filters and other revision options | +| `Commit.iter_items`, `Repo.iter_commits` | First-parent walks and a single tip | General history ordering; GIX-8 | +| Index entry reads, `ls_files` | Stage/mode/OID/path and exposed index flags | Custom and sparse indexes; GIX-3/4 | +| `IndexFile.version`, `update_index` query | Native index version | Actual `update-index` mutations | +| `IndexFile.write` and index persistence | Edit a private native index, publish through the existing lock | Custom, sparse, split, non-v2, unmerged, overlapping, or null-ID entries; GIX-3/4/13 | +| ODB `store`, managed `hash_object` | Native object hashing/writing with byte-fidelity preflight | Large/nonseekable input and changed serialization; GIX-2/5 | +| `IndexFile.write_tree` | Native tree editor with child-kind and null-ID validation | Missing non-gitlink children, invalid entries; GIX-7/24 | +| Tree serialization, `mktree` | Native tree editor with child-kind and null-ID validation | Missing children, unsupported or invalid entries; GIX-7/24 | +| Fresh index preparation, `read_tree` | Empty index or index from one tree | Existing index, merges, non-v2 selection; GIX-13 | +| `Commit.create_from_tree`, `commit_tree` | Explicit author/committer and unsigned UTF-8 commit | Identity cleanup, object-kind validation, other encodings, signature formats, signing/options; GIX-6/21/24 | +| Reference log reads, `reflog` | Existence and GitPython's reflog format | Orphan logs, arbitrary formats, writes; GIX-10 | +| Committer identity, `var` | Native process/config identity from `Repository.committer()` | Per-command identity overrides, other variables and normalization/identity errors; GIX-21 | +| Selected `set_object` / reference deletion, `update_ref` | Native ref edits with GitPython's existing validation | Strict create/CAS, branch-target validation, symbolic aliases other than HEAD, symbolic detachment, active-branch HEAD log, unchanged targets and log-message cleanup; GIX-9/10/21/24 | +| Tree/commit `Diffable.diff` | Raw change records and unambiguous exact renames | Patches, paths, index/worktree/root diff, inexact/ambiguous renames; GIX-11 | +| `Commit.stats` | Native Myers line counts, binary counts, first-parent tree comparison | Diff attributes, other algorithms, gitlinks, quoted filenames, large blobs; GIX-2/18 | +| `Repo.is_dirty` | Index/tree/worktree status, configured submodule checks and pathspecs | Custom/sparse indexes and non-root command directories | +| `Repo.untracked_files` | Native directory walk | Extra status options and non-root command directories | +| `Repo.ignored` | Native excludes with tracked-file suppression | Symlink/submodule traversal and unsupported path normalization | + +Repository-backed native operations fall back for reftable (GIX-1) and repositories +with `extensions.compatObjectFormat` (GIX-19). Process-returning calls, timeouts, +custom storage/environment, global Git options, and unhandled command options +also retain their CLI contracts. A broken installed extension is reported as +an import error; only an absent top-level `gix` selects CLI mode. + +## Performance work + +Use the [measurements after the Gix-only audit](#measurements-after-the-gix-only-audit) +for the corrected implementation. Older snapshots retain their original +workloads and include shortcuts that were subsequently removed. + +### Shared minimum-version checks + +Both installations reuse successful minimum-Git-version checks across fresh +`Git` wrappers with the same resolved executable, executable metadata, working +directory, effective environment and `Git.refresh()` generation. Two extra +checks in the fixed warm probe now launch zero processes instead of two; +two equivalent cold wrappers launch one probe. Public `version_info` caching +remains per instance, and old versions still fail before repository creation. + +This is shared CLI housekeeping: it caches an actual Git answer and records +no native success. It is not a Gix implementation of a missing capability. +The pinned warm Gix opening/discovery ceilings fall from 2/4 to 1/3 here; +the reference-format query and rejected-candidate diagnostics remain real CLI +calls. Cold contexts can still require a version probe. + +Global Git options and ambiguous Windows executable search retain per-instance +probes; explicit absolute executables can share checks on Windows. The shared +cache is bounded to 128 contexts. Call `Git.refresh()` if a launcher's reported +version changes through external state without changing its executable or +environment metadata. + +### Existing-repository benchmark + +[`test/performance/README.md`](../test/performance/README.md) describes the +`pyperf` harness and fixed GitPython 3.1.45 fixture. It measures eleven public-API +operations and their complete journey on one already-open `git.Repo`, plus +separate direct opening and discovery from `git/objects`. Fresh high-level +wrappers preserve the cost of actual operations; imports, fixture preparation +and parity preflight are outside timing. Native repository refresh remains +inside operation timing, so these measurements include the cost of keeping +retained state current. + +Submodule inventory, revision path/mode lookup and raw commit/tree readback +extend the already-open journey, each with a zero-CLI ceiling. The readback +measurement reads existing bytes, including the signed fixture commit, without +rewriting objects. Historical tables below retain their original workloads. + +At `244e418da6cc43de129cbe2908d11be4c5ad457a`, using official GixPython +0.1.0 and the same existing CPython 3.12.14/macOS arm64 interpreter for both +installations, with Git 2.54.0 (Apple Git-157), six worker processes with three values each produced the +following means and standard deviations. GixPython ran first, then CLI; +all eleven result digests matched. The fixture has one branch, one untracked +file and an ignored directory, with ambient Git configuration disabled. + +| Measurement | CLI (ms) | GixPython (ms) | CLI / Gix | +| --- | ---: | ---: | ---: | +| Already-open journey | 230.90 ± 6.87 | 260.14 ± 13.37 | 0.89× | +| Reference inventory | 22.77 ± 1.15 | 1.17 ± 0.26 | 19.52× | +| 25 first-parent commits with metadata | 9.24 ± 0.46 | 73.61 ± 7.24 | 0.13× | +| Root tree and README blob | 16.02 ± 1.07 | 2.40 ± 0.46 | 6.66× | +| Index entries | 8.43 ± 0.57 | 2.00 ± 0.38 | 4.21× | +| Commit count and ancestry queries | 82.28 ± 2.27 | 36.19 ± 0.84 | 2.27× | +| Latest diff and commit statistics | 29.98 ± 1.66 | 60.98 ± 2.45 | 0.49× | +| Patch diff (includes CLI fallback) | 17.87 ± 1.28 | 10.86 ± 1.01 | 1.64× | +| Dirty/untracked/ignored paths | 68.56 ± 3.18 | 64.74 ± 2.00 | 1.06× | +| Direct opening and close | 95.08 ± 4.64 | 35.84 ± 2.02 | 2.65× | +| Nested discovery and close | 112.58 ± 5.38 | 49.20 ± 4.16 | 2.29× | + +Ratios above one favor GixPython. This workload's complete journey is about +13% slower with GixPython: metadata reads and commit statistics offset gains +elsewhere. These are warm-cache measurements on one machine, with `pyperf` +stability warnings for several samples; they do not establish a general +speedup. Full-suite times below also include setup and coverage. + +The separate `Backend benchmark` CI job runs both installations on one runner, +publishes every measurement and `pyperf` significance reporting, and retains +raw JSON and comparison artifacts. Results include revisions and per-operation +native/fallback decisions. Adding a `MEASUREMENTS` entry extends the journey, +individual timings and parity checks together. Opening/discovery remain +separate lifecycle measurements. CI fails on execution/parity errors; timing +ratios are observational on shared runners. + +### Historical retained-handle and opening measurements + +These results predate the Gix-only audit. The measured implementation inferred +reference format, repaired linked-worktree metadata and rejected some discovery +candidates in Python. Its zero-CLI counts and associated opening speedups do +not establish Gix coverage and are superseded by the corrected implementation. +Keep the raw observations below only as a historical record. + +At `c49bbec0afb806df66a33fa9c5d76e21ac8f0314`, the same pinned fixture, +official GixPython 0.1.0 and existing CPython 3.12.14/macOS arm64 interpreter +were measured sequentially, Gix first and CLI second, after tests finished. +Each backend used three worker processes, three values per worker, four loops +per value and one warmup. All eleven result digests matched and the checked-in +CLI ceilings passed. Raw local results are +`/private/tmp/gitpython-retained-final-{gix,cli}.json`; this table is a journal +snapshot, while CI continues to publish fresh measurements. + +| Measurement | CLI mean ± stdev (ms) | GixPython mean ± stdev (ms) | CLI / Gix | Native / fallback | CLI launches: CLI / Gix | +| --- | ---: | ---: | ---: | ---: | ---: | +| journey | 209.44 ± 17.65 | 263.25 ± 5.52 | 0.80× | 55 / 2 | 23 / 1 | +| inventory | 19.76 ± 0.78 | 0.89 ± 0.03 | 22.30× | 3 / 0 | 3 / 0 | +| history_25 | 7.49 ± 0.17 | 79.02 ± 9.67 | 0.09× | 26 / 0 | 1 / 0 | +| browse_tree_and_blob | 13.55 ± 0.39 | 2.06 ± 0.41 | 6.59× | 5 / 0 | 2 / 0 | +| read_index | 6.96 ± 0.08 | 2.59 ± 0.47 | 2.69× | 1 / 0 | 1 / 0 | +| revision_graph | 63.27 ± 2.57 | 32.72 ± 0.22 | 1.93× | 9 / 0 | 6 / 0 | +| diff_and_stats | 20.78 ± 0.37 | 65.33 ± 1.25 | 0.32× | 5 / 0 | 3 / 0 | +| patch_diff | 14.06 ± 0.39 | 8.61 ± 0.08 | 1.63× | 3 / 2 | 2 / 1 | +| worktree_status | 56.61 ± 1.43 | 70.68 ± 2.18 | 0.80× | 3 / 0 | 5 / 0 | +| open_repository | 69.28 ± 0.59 | 1.00 ± 0.07 | 69.20× | 1 / 0 | 11 / 0 | +| discover_repository | 81.68 ± 0.68 | 0.90 ± 0.19 | 90.52× | 1 / 0 | 13 / 0 | + +The pre-audit implementation launched no CLI processes for these opening and +discovery probes. It measured about 69× and 91× faster lifecycle operations, +while the complete already-open journey was about 26% slower with Gix. These +lifecycle ratios include the shortcuts removed by the audit. Retaining a Python +handle alone does not resolve the existing history/statistics costs. Several +rows have `pyperf` sample-size or variability warnings, so timings describe +this machine and workload rather than a general speed guarantee. + +The full Gix suite without coverage passed **1,678 tests and 38 subtests**, +with 79 skips and one expected failure, in **337.51 seconds (5m 37.51s)**. +One expected path-expansion deprecation warning was reported. It recorded +**22,319 `Git.execute` launches**: 3,833 setup, 18,486 call, zero teardown or +collection/session. The process-lifetime count is 22,320 including the import +probe. Against the preceding 408.81-second/30,898-launch baseline, this local +run took 17.4% less time and launched 27.8% fewer CLI processes; the suite has +four additional regressions, and the timing comparison is observational. + +Affected tests ran with Gix before CLI: 304/264 repository, command, backend +and safety tests; 136 command-guard tests per backend; 31 positional tests per +backend; 13 process-count/budget tests per backend. Repository suites also +passed 14 subtests, with three skips. Ruff lint/format, mypy and basedpyright +passed. Both code changes are separate Tix commits: `02e2cc44` retains native +handles and the original `c49bbec0` attempted zero-launch opening/discovery. +The rewritten opening commit restores real format queries and native metadata +fallbacks; the current coverage table and ceilings reflect those requirements. + +### Measurements after the Gix-only audit + +At `5ef5c62f2e2dafe3e7ddb0deed6998fd7092d360`, the adapters used Gix for +supported operations and retained CLI fallback for missing native capabilities. +The audit removed Python reference-format inference, metadata-path construction, +committer resolution, reflog cleanup, discovery repairs and absent-hook success. +Its findings were recorded in GIX-1/10/14/21/22/23; the capability follow-up below +refines those requirements. Retained native repository state and conversions +that actually use Gix APIs remain in place. +The full-suite and timed benchmark figures in this section describe that +snapshot, before the subsequent metadata and discovery changes. + +The full Gix suite without coverage passed **1,710 tests and 38 subtests**, +with 79 skips, one expected failure and one existing path-deprecation warning. +It took **342.36 pytest seconds / 342.80 seconds including the runner +(5m 42.80s)** on existing CPython 3.12.14/macOS arm64, official GixPython 0.1.0 +and Git 2.54.0. This includes CLI-inventory and Trace2 overhead. + +The corrected run recorded **22,694 `Git.execute` launches**. The earlier +12,394-launch result included the removed Python shortcuts and is superseded +as a coverage claim. The corrected run also contains 19 additional regression +cases; these are observed totals for their respective revisions. + +| Launch source | Before audit (`d2e788a1`) | Corrected (`5ef5c62f`) | +| --- | ---: | ---: | +| Managed operations with Gix enabled | 8,836 | 16,871 | +| Managed operations with Gix explicitly disabled | 309 | 339 | +| Minimum-version probes | 2,483 | 4,698 | +| Raw `Git.execute` calls | 766 | 786 | +| **Total pytest launches** | **12,394** | **22,694** | + +The corrected total splits into 4,464 setup and 18,230 call-phase launches, +with zero in teardown or collection/session. Call phases include `unittest` +fixture work. The Python inventory reconciles exactly with the pytest counter; +the process-lifetime backend count is 22,695 including its import probe. +Trace2 additionally identifies 3,685 Git child processes and 37 processes +outside the captured wrapper. Its total overlaps the Python inventory, so the +two totals must not be added together. + +The benchmark used the same pinned GitPython 3.1.45 repository and interpreter +for both backends, sequentially after tests, Gix first and CLI second. Each +used three worker processes, three values per worker, four loops per value +and one warmup. All **14 result digests matched**, and every corrected CLI +ceiling passed. The eleven operations form one journey on an already-open +`Repo`; opening and discovery remain separate lifecycle measurements. + +| Measurement | CLI mean ± stdev (ms) | GixPython mean ± stdev (ms) | CLI / Gix | Native / fallback | CLI launches: CLI / Gix | +| --- | ---: | ---: | ---: | ---: | ---: | +| journey | 314.35 ± 2.33 | 264.94 ± 3.30 | 1.19× | 79 / 2 | 43 / 1 | +| inventory | 19.50 ± 0.31 | 0.99 ± 0.08 | 19.68× | 3 / 0 | 3 / 0 | +| history_25 | 7.59 ± 0.12 | 73.32 ± 2.17 | 0.10× | 26 / 0 | 1 / 0 | +| browse_tree_and_blob | 13.45 ± 0.21 | 1.78 ± 0.22 | 7.55× | 5 / 0 | 2 / 0 | +| read_index | 7.00 ± 0.11 | 2.09 ± 0.27 | 3.35× | 1 / 0 | 1 / 0 | +| submodule_inventory | 72.10 ± 0.71 | 3.21 ± 0.20 | 22.44× | 13 / 0 | 11 / 0 | +| revision_paths | 55.27 ± 0.42 | 1.41 ± 0.12 | 39.12× | 6 / 0 | 8 / 0 | +| object_readback | 6.60 ± 0.16 | 1.22 ± 0.10 | 5.43× | 5 / 0 | 1 / 0 | +| revision_graph | 62.26 ± 1.15 | 33.25 ± 0.36 | 1.87× | 9 / 0 | 6 / 0 | +| diff_and_stats | 21.43 ± 1.28 | 67.83 ± 7.58 | 0.32× | 5 / 0 | 3 / 0 | +| patch_diff | 13.96 ± 0.12 | 8.97 ± 0.23 | 1.56× | 3 / 2 | 2 / 1 | +| worktree_status | 36.59 ± 1.01 | 70.55 ± 4.64 | 0.52× | 3 / 0 | 5 / 0 | +| open_repository | 58.47 ± 1.12 | 7.71 ± 0.25 | 7.58× | 1 / 1 | 9 / 1 | +| discover_repository | 70.72 ± 0.68 | 20.49 ± 0.38 | 3.45× | 1 / 5 | 11 / 3 | + +The expanded warm journey is **1.19× faster** with Gix in this sample, with +43 CLI launches reduced to the single patch-diff launch. Opening is **7.58× +faster** and retains one reference-format query; discovery is **3.45× faster** +and adds two CLI queries for rejected native candidates. Its five fallback +decisions include both discovery and command-adapter decisions, while only +three processes are launched. Cold contexts can also need a version probe. +The previous zero-launch opening/discovery results included Python substitutes +and are historical only. + +History, diff/statistics and worktree-status measurements remain slower with +Gix despite making no CLI launches. They need separate profiling. Several +rows have `pyperf` sample-size or variability warnings; these measurements +describe this machine and workload rather than a general speed guarantee. + +Every affected selection passed with Gix before CLI. The final full suite above +used Gix; CLI validation used the affected selections. Repository-wide Ruff +lint/format, mypy and basedpyright passed. The benchmark comparison independently +checked result parity, fixture/source identity and all process ceilings. +Generic configuration and standalone reference-name gaps remain documented +under GIX-12/20. `Repo._get_gix_repository(recreate=True)` still explicitly +recreates retained native state; its snapshot limitations are documented below. + +Raw benchmarks, comparison tables and logs from that snapshot are under +`.cache/gix-only-audit/benchmark-*` and `.cache/gix-only-audit/pyperf-comparison.md`. +The joined inventory, per-call CSV/JSONL, grouped commands, passing full-suite +log and runner timing are in `.cache/gix-only-audit/full-inventory-5ef5c62/`. +The older `.cache/gix-conversions/benchmark-{gix,cli}.json` and +`final-inventory-d2e788a/` remain historical evidence. CI continues to publish +fresh measurements and enforce the checked-in operation-specific ceilings. + +### Native capability follow-up + +The follow-up used the same official GixPython 0.1.0 and existing CPython +3.12.14, with runtime and test code at +`ee947c98540396d5644f3a7fea7c807633f4b5b8`. Each implementation change was +amended into its corresponding Tix commit and checked with Gix before CLI. + +- Reference storage can already be identified through native config. Direct + SHA-1 and SHA-256 regressions confirm that reftable `HEAD` access raises + `gix.Error` with an unsupported-storage cause; the binding does not expose a + separate `Unsupported` exception. Strict native opening and `HEAD` access + still accept unknown repository extensions that Git rejects, so the CLI + format query remains necessary for validation (GIX-1). +- Native `git_dir()` supplies the locations for `modules` and `COMMIT_EDITMSG`, + including linked worktrees. The adapter now adds only those fixed leaf names, + after reopening noncanonical Git paths through Gix. Each ordinary lookup + drops from one CLI launch to zero. Other metadata names and symlinked leaves + retain Git's path resolution (GIX-22). +- Gix already exposes the worktree of a bare main repository. Combining its + configured `is_bare()` flag with `workdir()` fixes opening and inventory + in the adapter; the original classification difference remains tracked as + an upstream compatibility bug. Gitfiles are reopened at their + canonical native Git directory with the same strict options; supported + symlinked-target helpers now need zero CLI launches instead of one. This + also prevents an arbitrary gitfile location from becoming the worktree. + Dangling `commondir` handling and typed failure information remain gaps; + rejected candidates keep Git diagnostics (GIX-14). +- Native snapshot overrides resolve committer name, email and date. A cheap + independent repository clone is needed to isolate per-command overrides: + Python copying is unsupported and the internal binding clone shares state. + Shared snapshots and process environment remain untouched (GIX-21). +- `LogChange` already configures each ref edit's message and log policy. A + direct native regression preserves whitespace where Git normalizes it, so + message cleanup still needs a native helper or policy (GIX-10). + +Final focused validation passed **120 Gix tests and 14 subtests** (one skip) +in 22.93 seconds, followed by **34 CLI tests and 14 subtests** (two skips, +including the native-only module) in 6.14 seconds. This covered the complete +native-backend, launch-counter and benchmark-comparison modules plus the +affected shared discovery, gitfile, worktree and repository-construction tests. +Repository-wide Ruff lint/format, mypy and basedpyright passed. + +An untimed benchmark preflight, Gix first and CLI second, matched all **14 +result digests** and passed the existing launch ceilings. The warm journey +remains CLI/Gix **43/1** launches, opening **9/1**, and nested discovery +**11/3**; the new fixed-path and gitfile-helper reductions are asserted by +their focused tests. No full-suite rerun or fresh timed benchmark is claimed +for this follow-up. Logs, backend reports and the count comparison are under +`.cache/gix-capability-followup/` (`15-final-*` and `16-benchmark-*`). + +### Windows CI process overhead + +The Python 3.12 jobs in [PR #2274's Python package run](https://github.com/gitpython-developers/GitPython/actions/runs/37628358853) +spent 57m 25s in pytest on Windows and 5m 2s on Ubuntu. Their process reports +counted 85,777 and 82,683 `Git.execute` launches respectively. The Windows +submodule test modules accounted for approximately 36 minutes, based on the +timestamps of their test results; the delay was spread across many operations. + +A local Windows profile of `test_file_handle_leaks` and +`test_update_no_fetch_is_recursive[root-no-fetch]`, with CI's coverage and +pytest options, counted 2,447 launches. About 65 of 77 seconds were spent in +Git command execution, including 44 seconds in repository construction; +forced garbage collection accounted for about three seconds. Combining +`--show-ref-format`, `--show-object-format`, `--is-bare-repository` and +`--git-common-dir` in one `rev-parse` invocation reduced the same selection to +2,009 launches: three saved for each of 146 repository opens. Git still +computes all four values. The common-directory path is last and split only +after the three scalar fields, preserving paths containing newlines. + +Three paired measurements of 20 ordinary repository opens on Windows gave +median times of 300.74 ms before and 221.18 ms after, with matching metadata +and 11 versus eight launches per open. These use CPython 3.12.13 and Git +2.55.0.windows.3; timing is observational, while the launch reduction is +checked by the regression tests. + +A 50-call `git version` probe on the same host measured median times of +17.49 ms per call on Windows and 1.27 ms in Ubuntu WSL, which uses CPython +3.14.4 and Git 2.53.0. Even trivial Git commands carry appreciable startup +cost on Windows. + +This optimizes the CLI backend's construction path. Supported Gix discovery +still uses its native metadata and the separate Git format-validation query; +the Gix launch ceilings and open compatibility bugs remain unchanged. +CI now includes `--durations=30` so subsequent slow tests are visible directly. +Local Windows reproduction must retain CI's `core.autocrlf=true` setting and +place pytest's temporary directories outside a Git checkout. Isolating all +Git configuration without restoring that setting changes the newline test; +placing `--basetemp` under this checkout lets empty-directory Git probes +discover the parent repository instead. + +The full local baseline at `105114db` recorded 86,032 launches in 3,990.28s; +the two submodule modules accounted for 2,421.59s. It had 1,652 passing tests +and the two harness failures described above, both of which passed unchanged +after correcting the setup. The patched run exited successfully with 1,662 +passed, 60 skipped, nine expected failures, two unexpected passes and 40 +passing subtests. It recorded 77,732 launches in 3,715.16s. These runs +overlapped, so their wall times are not a controlled comparison. The paired +repository-open benchmark above measures the affected operation separately. +Logs, profiles, JUnit results and backend reports are retained locally under +`.cache/ci-performance/`. + +### Submodule fixture process overhead on Windows + +Windows CI and the local runner now resolve `git.exe` from `PATH` to an +absolute `GIT_PYTHON_GIT_EXECUTABLE` when no override was supplied. This enables +the existing minimum-version cache without changing the library's executable +lookup or cache invalidation. Submodule fixtures also pass `skip_hooks=True` +when committing their test history; the library default and dedicated hook +tests continue to run hooks. + +A serial Windows comparison on CPython 3.12.13, Git 2.55.0.windows.5 and +GixPython 0.1.0 selected the Windows destination-name rejection tests, +`test_update_no_fetch_restores_deinitialized_submodule`, and +`test_update_no_fetch_is_recursive`. Both runs used the direct Git executable +on `PATH`, CI's flat alias config and `core.autocrlf=true`, with profiling +enabled and coverage disabled. All 37 cases passed in both runs. + +| Test invocation and fixtures | Pytest time | Git launches | Version probes | Hook launches | +| --- | ---: | ---: | ---: | ---: | +| `GIT_PYTHON_GIT_EXECUTABLE=git`, fixture hooks enabled | 87.10s | 1,437 | 403 | 177 | +| Absolute executable, fixture hooks skipped | 77.20s | 1,128 | 271 | 0 | + +The 309 eliminated launches comprise 132 redundant version probes and three +hook checks for each of 59 fixture commits. This sample was about 11% faster; +it is not a full-suite speedup estimate. Git Bash CI already selects +`mingw64/bin/git.exe` and appends the alias config directly, so correcting the +local runner's launcher/config differences is not an additional CI gain. +Logs, profiles, JUnit results and operation reports are retained locally under +`.cache/test-cleanup/windows-submodule-{baseline,optimized}*`. + +### Test-suite setup measurements + +The fixture optimizations below are implemented as separate commits, each +validated with GixPython before CLI. +The original local measurements use official GixPython 0.1.0 and existing +CPython 3.12.14 on macOS arm64 at `89c609cf92335e75652d7725682e215a9ea080e5`. + +The full suite with coverage took 757.85 seconds wall-clock (757.04 seconds +reported by pytest): 1,649 tests and 38 subtests passed, 79 skipped, and one +expected failure. Operation reporting counted 114,800 native operations and +39,002 CLI fallback decisions. Those decisions are not a complete subprocess +count: raw `repo.git` calls and subprocesses started by Git are not all counted. + +### Retain native repository state (implemented) + +Each `git.Repo` owns a native `gix.Repository`. Managed operations access it through +`Repo._get_gix_repository()`, which reuses the instance by default. +`Repo._get_gix_repository(recreate=True)` replaces it explicitly; this method is +the control point for a future configurable reuse policy. Managed operations reuse it; +`close()` releases it, and a later operation can reopen it. Pickling excludes +native resources and restores the command's weak owner reference. Gix provides +thread-safe handle access and automatic index/ODB snapshot refresh. A per-Repo +lock serializes handle refresh, without serializing read operations. + +Retaining native state is an intentional deviation from Git's fresh process per +command. Configuration/environment and observed CLI changes trigger best-effort +refresh, but this is not a promise that every external filesystem change is +immediately visible in every native cache. Explicit recreation or closing and +reopening the `Repo` provides fresh native state. Configuration queries currently +use a separate fresh handle because the retained execution handle overrides +hooks, fsmonitor and automatic maintenance; those overrides must not leak into +configuration results. The accessor itself removes no CLI calls: ten supported +metadata queries launch zero processes before and after this refactor. + +Configuration and storage metadata changes, environment changes and raw CLI +launches cause `reload()` before reuse. Configuration queries use a separate +fresh handle so synthetic safety settings never appear as user settings. +Configurations with includes conservatively reload on each operation because +the binding does not expose all included source paths. Storage overrides, +reftable and compatibility object formats retain their existing CLI guards. + +An earlier 1,000-call benchmark without coverage measured an object read using +one native handle at 0.046 ms per call, compared with 0.635 ms through the +fresh-open stream adapter. Opening and validating a native repository alone +took 0.418 ms per call. These small repeated-read measurements do not estimate +whole-suite speedup. + +### Reuse prepared test fixtures (implemented) + +A 19-case submodule rejection sample without coverage took 15.62 seconds: +1,258 `Git.execute()` calls accounted for 11.86 seconds, while 1,995 native +repository-open/validation attempts accounted for 0.84 seconds. Most work was +fixture setup. This sample supports prioritizing repeated setup subprocesses; +it is not a profile of the entire suite. + +The original collection found 1,729 parameterized cases from 703 distinct source bodies. +Source inspection identified the following conservative set of 422 cases whose +assertions do not require changing repository state after preparation. The +remaining cases have not been exhaustively classified; these are reuse +candidates, not proof that a shared fixture is safe in every execution order. + +| Original cases | Repeated setup | Implemented change | +| --- | --- | --- | +| 166 | `TExc` (157) and `TestActor` (9) inherit repository-building `TestBase`. | Use the existing `TestCase` base without repository setup. | +| 182 | Three submodule rejection bodies repeatedly build `movable_submodule`, then check snapshots for no mutation. | Prepare logical-name baselines once and copy the parent per case, retaining fresh wrappers and independent writable files. | +| 51 | Six submodule rejection bodies prepare nested metadata, separate metadata, intermediate/leaf symlinks, or retained metadata before checking rejection. | Cache ten prepared layouts and restore complete copies at their original paths, preserving absolute Git links and symlinks. Cleanup is best-effort; a locked active copy invalidates the layout so the next case rebuilds at a fresh path. | +| 15 | Eight revision-query bodies rebuild the same four-commit graph, refs, index, and reflogs through `rev_parse_repo`. | Prepare the graph once and copy it for every consumer, including mutating cases; recreate repository, branch and commit wrappers. | +| 8 | Tree lookup bodies clone and check out `0.3.2.1` through `with_rw_repo`. | Read the historical tree directly through the existing class repository, removing clones and checkouts. | + +The `movable_submodule` baseline also serves mutating cases: all writable +refs, index, config, objects, worktree and module metadata are filesystem copies, +while the local clone source remains shared and immutable. The original 322 +consumers no longer repeat repository initialization and submodule cloning. +The original 68 `local_submodule` cases copy both source and parent from one +prepared two-commit layout because these tests also mutate the source. The +fixture relocates all source URLs and records the private URL in parent history +for `RootModule` comparisons. No mutable Git metadata is hard-linked. + +Historical dependency sources are now lazy session fixtures. Originally all +25 `TestBase` classes reconstructed both sources (50 builds), although only four +classes called the URL helpers. The suite now prepares each needed source once, +with consuming tests retaining independent writable clones. + +Four additional checks exercise isolation: edits and refs in movable copies, +restoration after deliberate mutation with an absolute symlink, private source +commits, and revision-graph changes. Existing security no-side-effect snapshots +remain in place. Native repository lifetime and refresh limitations are +documented above. + +Per-change affected tests on existing CPython 3.12.14/macOS arm64 with official +GixPython 0.1.0, without coverage (wall-clock seconds including runner setup): + +| Change | Passed cases | GixPython first | CLI second | +| --- | --- | --- | --- | +| Repository-free actor/exception tests | 166 | 0.47 s | 0.49 s | +| Lazy historical sources, all consumers | 206, plus 14 subtests; 6 skipped, 1 xfailed | 90.89 s | 160.82 s | +| Movable baseline, top-level submodule tests | 333 | 88.63 s | 178.75 s | +| Prepared rejection variants | 51 | 12.34 s | 26.23 s | +| Layout restoration check | 1 | 1.33 s | 1.62 s | +| Prepared revision graph | 23 | 4.39 s | 7.57 s | +| Historical tree lookups, whole tree module | 22 | 1.66 s | 3.86 s | +| Prepared no-fetch source and parent | 69 | 56.27 s | 117.52 s | + +These selections overlap, and their timings are validation records rather than +isolated before/after benchmarks. The full-suite measurements below provide the +broader comparison. Only the existing interpreter was used. + +At `c8ee26c7da373e28cc7ede17ef2eaadd763fcdfc`, the full GixPython suite +with coverage passed in **404.61 seconds wall-clock (6m45s)**, with pytest +reporting 404.12 seconds: 1,653 passed, 79 skipped, one expected failure, +and 38 subtests passed. Coverage remains 90%. Compared with the original +757.85-second run, this saved 353.24 seconds (46.6%, about 1.87 times faster). +This is one local before/after run per revision, not a statistical benchmark. + +The same operation counters now record 102,581 native operations and 21,402 +CLI fallback decisions, down from 114,800 and 39,002 respectively. Raw Git +calls and child processes remain outside those counters. + +The subsequent full CLI run with coverage passed in **755.78 seconds +wall-clock (12m36s)**, with pytest reporting 755.30 seconds: 1,617 passed, +80 skipped, one expected failure, and 38 subtests passed. CLI coverage is 82%; +backend-specific tests account for different collection and coverage. This CLI +run validates the optimized suite; there is no matching pre-change CLI full-run +measurement here. Repository-wide Ruff lint/format, mypy and pyright passed. + +## GixPython / Gitoxide follow-up ledger + +These are local findings, not filed issues. **Every observed divergence from +the equivalent Git operation is a compatibility bug**, including intentional +differences. Gix must offer matching behavior, at least through an API or mode +as strict as Git. This covers acceptance/rejection, validation, returned data, +ordering, paths and mutation side effects. `strict_config(True)` and +`bail_if_untrusted(True)` alone do not provide that guarantee (GIX-1/14). + +The status distinguishes reproduced bugs from missing APIs and unverified +coverage. Low-level primitives can remain available, but a compatible API or +mode is still required. An adapter workaround, including reopening through +Gix, does not close an upstream compatibility bug. Retain its expected/actual +behavior, version and reproduction until the upstream fix or compatible mode +has been verified. Python must not implement the missing Git behavior. + +Unless stated otherwise, the evidence uses official GixPython 0.1.0, its pinned +Gitoxide revision `f819565c2c4c56619c4888acef6cf3b8144cbccb`, and Git 2.54.0 on +macOS arm64 with existing CPython 3.12.14. The repository and commands below +identify the historical fixture where needed. + +| ID | Status | Finding and evidence | Required Gix behavior/API | +| --- | --- | --- | --- | +| GIX-1 | Bug; missing API | Native config exposes `extensions.refStorage`; reading `HEAD` in SHA-1/SHA-256 reftable repositories raises `gix.Error` with an unsupported-storage cause. However, strict native opening plus a successful `HEAD` read accepts unknown version-1 extensions that Git rejects. It also accepts `extensions.refStorage=files` with repository version 0, which Git rejects. | Git-compatible repository-format/extension validation, typed native errors, and eventual reftable support. A dedicated format getter is not needed just to identify storage. Keep the CLI format query because it also validates the repository, with regressions for both the unsupported-HEAD signal and unknown extensions. | +| GIX-2 | Missing API | Object lookup buffers full contents. `write_blob_stream` also reads its input into memory. | Bounded-memory object readers/writers usable by Python. The adapter currently caps native blob transfers at 8 MiB. | +| GIX-3 | Missing API | Index bindings open the repository's index but cannot load an arbitrary existing index file. `set_path()` only chooses its write destination. | An index loader accepting a path, including relative-path semantics. | +| GIX-4 | Missing API | Sparse index expansion is not exposed. | Expansion preserving skip-worktree and other metadata before editing. | +| GIX-5 | Unverified | Structured-object writers may decode/re-encode supplied bytes. This is a compatibility guard, not a demonstrated corruption bug. | A raw-object writer, or a guaranteed byte-preserving contract. The current adapter checks the round trip in an in-memory ODB first. | +| GIX-6 | Missing API | Native commit construction accepts a string message. GitPython also supports arbitrary message bytes and other declared encodings. | Byte-message commit construction with explicit encoding/signature semantics. | +| GIX-7 | Missing API | Native tree editing requires existing non-gitlink child objects. Git's `mktree --missing` / `write-tree --missing-ok` permit missing children. | Missing-object tree construction with equivalent validation. | +| GIX-8 | Bug | General revision iteration differs from Git's default ordering. The same fixture enumerated 5,439 commits in both engines, with the first ordering difference at position 178. Its 1,923-commit first-parent chain matched. | A Git-compatible walk order; counts and first-parent walks already work. See the reproduction notes below. | +| GIX-9 | Bug | `PreviousValue.MustNotExist` accepts an existing ref already at the requested target. Native tests explicitly permit this; Git's strict create/CAS rejects it. | A strict native expectation, enforced under the same ref lock. A Python existence precheck would race. | +| GIX-10 | Bug; missing API | `LogChange` preserves ` keep\t spaces ` where Git stores `keep spaces`. Updating the checked-out branch directly or through an alias omits the HEAD log entry Git appends. Updating that branch to its unchanged target also omits Git's HEAD log entry. Detaching symbolic HEAD records a null old OID instead of Git's resolved previous commit. `RefLog.Only` cannot express an independent arbitrary old OID, and reference-bound readers cannot access orphan logs. | Fix message normalization, secondary HEAD logging and old-OID selection in Gix, at least in a Git-strict transaction mode; expose raw reflog append and orphan-log access. `LogChange` already configures each edit's message, mode and force-create flag; `core.logAllRefUpdates` controls creation, not cleanup. Keep the existing `_update_ref` and `_reflog` fallbacks before mutation; do not normalize messages in Python. | +| GIX-11 | Unverified | Inexact and ambiguous rename pairing/scores have not been established as Git-compatible. This is a conservative guard, not a claimed native bug. | Verified pairing, scoring, tie-breaking and option parity. Exact unambiguous renames are native. | +| GIX-12 | Missing API | Generic config bindings lack standalone parsing, ordered section/key enumeration, multivars, unset/remove-section and source-scoped queries. Dedicated `.gitmodules` parsing is available and now used for supported submodule reads. | Those operations for `GitConfigParser` and remote/branch configuration. Native merged getters cannot replace repository-only config readers. | +| GIX-13 | Bug; missing API | Native index writing normalizes v4 to v2, offers no version setter, and expands split indexes. | Version and split-index preservation. Tests now assert that a split index remains split after an update. | +| GIX-14 | Bug; missing API | For a linked worktree of a bare main repository, `is_bare()` returns true while `git rev-parse --is-bare-repository` returns false; `workdir()` does expose the worktree. Initial gitfile opening can retain a noncanonical target and use an arbitrary gitfile's location as the worktree, unlike Git. Discovery accepts undecodable HEADs until explicit `head()` decoding and ignores dangling `commondir` symlinks that Git rejects. Bindings flatten failure kinds into `gix.Error`. Windows also exposes adapter path-formatting differences and a worker panic on an undecodable `commondir`; see the Windows reproduction below. | Provide Git-compatible classification, canonical gitfile locations and layout validation, plus typed errors instead of panics. Format native locations compatibly with Git's command output. The adapter combines Gix metadata, reopens through Gix and forces HEAD decoding, retaining CLI for remaining validation/diagnostic gaps. Those workarounds do not close these compatibility bugs. A Git-strict mode must cover these cases; strict config and trust settings alone do not. | +| GIX-15 | Bug | Native blame disagrees with Git even with Myers and rewrite tracking selected. In the fixture's `README.md`, lines 150 and 158 are attributed to the opposite commits. Incremental order also differs. | Attribution and incremental-output parity before replacing `Repo.blame` / `blame_incremental`. | +| GIX-16 | Bug; missing API | Native archive streaming takes a tree rather than a commit. Its TAR omits Git's global PAX commit comment, leaves `export-subst` placeholders literal, and writes ordinary modes as `0644` where Git uses `0664`. | Commit-aware export substitution, metadata and permission parity. Both engines respected `export-ignore` in the probe. | +| GIX-17 | Bug | Revision parsing accepts abbreviated IDs with `-dirty`, prefers the OID suffix over an exact describe-shaped tag, and treats escaped braces in message searches differently. | Git-compatible revision grammar and regex semantics. Existing `test_rev_parse.py` cases reproduce all three. | +| GIX-18 | Bug; missing API | The tree-diff resource cache reads attributes from the index; Git also consults worktree attributes. Native binary classification with textconv configured can also differ even in `to_git` mode. | Independent attribute-source control and `--no-textconv` parity. Statistics currently fall back for any effective `diff` attribute. | +| GIX-19 | Unverified | GixPython opens repositories with `extensions.compatObjectFormat`, but translation and object-write compatibility have not been validated. The installed Git reports `compatibility hash algorithm support requires Rust` when asked to translate an ID. This is an unvalidated capability guard, not evidence of missing native object mappings. | Verify compatibility-format reads, writes and OID translation against a Git build that supports them. All operations use CLI in the meantime. | +| GIX-20 | Bug; missing API | `Target.Symbolic(name)` validates full names but rejects standalone names accepted by `git check-ref-format --allow-onelevel`. Four of 430 audited calls differed: `refs` twice, `hellothere`, and `valid_one_level_refname`. Further probes include digits, punctuation and Unicode (`1`, `A1`, `HEAD_1`, `A-B`, `A.B`, `Ä`). | Expose Git-compatible partial/one-level name validation, or correct the native restriction where appropriate. This is a binding/behavior shortcoming to revisit upstream; standalone names rejected natively retain CLI validation. | +| GIX-21 | Bug; missing API | `Repository.committer()` preserves a process identity of ` Name. ` / ` email@example.invalid ` while `git var GIT_COMMITTER_IDENT` returns `Name.` / `email@example.invalid`. Native snapshot overrides for name, email and date work, but modifying the retained snapshot would leak per-command identity across threads. GixPython 0.1.0 has no general independent repository clone: Python copying fails and internal `RepoHandle::clone()` shares its `Arc` state. | Expose Git-compatible identity cleanup and a cheap independent native clone with isolated config, including per-command removal semantics. Reopening isolates state but repeats setup; `with_object_memory()` changes object-write behavior and is not a general clone. Keep `_checked_signature` validation and CLI fallback for cleanup/overrides. Never clean identity fields in Python or mutate shared snapshots/process environment to handle a command. | +| GIX-22 | Missing API | The native Git directory is sufficient for the two fixed metadata leaves used here: `modules` and `COMMIT_EDITMSG`. Git's `path.c` applies no special relocation to either, including in linked worktrees. The adapter appends only these names to the canonical native Git directory. | No new binding is needed for these ordinary paths. Other `--git-path` names and symlinked metadata leaves retain CLI resolution; a general resolver must handle config/environment overrides, common/private storage and canonical targets. `Repository.modules_path()` means `.gitmodules`, not the `modules` storage directory. | +| GIX-23 | Missing API | No hook lookup or execution API is bound in GixPython 0.1.0. The removed absence fast path read config through Gix but used Python `os.stat()` to declare success. | Bind native hook lookup with configured/default path, linked-worktree, missing/nonexecutable-hook and diagnostic semantics, plus execution where needed. Until then all managed hook calls use Git, including `--ignore-missing` no-ops. A Python filesystem check is not a Gix implementation. | +| GIX-24 | Bug | In both object formats, `new_commit_as()` accepts a blob as the tree or a parent where `git commit-tree` rejects it. Tree-editor `upsert()`/`write()` accepts object IDs whose actual kinds disagree with blob/tree/gitlink modes; `git mktree --missing` rejects all three tested mismatches. `edit_references_as()` accepts a blob target under `refs/heads/`, rejected by `git update-ref`. | Provide checked commit/tree construction and reference edits, at least in Git-strict mode. Match Git's kind and direct-branch target validation before writing and under the required ref locks, including its distinct handling of symbolic aliases. Existing `_commit_tree`, `_write_tree` and `_update_ref` guards query Gix headers and select CLI on invalid inputs; they do not replace Gix writes with Python. | +| GIX-25 | Bug; missing API | After `git symbolic-ref refs/heads/dangling refs/heads/missing`, `Repository.references().all()` enumerates the dangling name, while `git for-each-ref --format=%(refname)` omits it. Valid symbolic aliases must remain present. Both SHA-1 and SHA-256 probes reproduce this difference. | Expose Git-compatible enumeration with the same dangling-reference and diagnostic behavior, retaining the raw iterator for callers that need it. `_for_each_ref` forces Gix to resolve symbolic targets and falls back to Git on failure; keep that fallback until a compatible Gix API/mode is verified. | + +Windows reproduction for GIX-14 uses CPython 3.12.13, Git +2.55.0.windows.3 and the released GixPython 0.1.0 source distribution with +the same pinned Gitoxide revision above. Both `105114db` and the CLI metadata +batching change (`f6779dbb`) reproduce the following existing failures: + +- `test_native_command_queries_match_cli` and + `test_native_worktree_inventory_includes_bare_main` return backslashes in + the adapter's native path text where Git returns forward slashes, in both + object formats. The locations agree; the command-output formatting does not. +- The `commondir=b"\xff"` subtest of + `test_repo_discovery_rejects_invalid_metadata` raises + `RuntimeError: native worker panicked` instead of the CLI backend's + `InvalidGitRepositoryError`. The binding must reject this input through its + normal native error contract so discovery can take the existing fallback. + +The adapters now format locations returned by `Repository.common_dir()`, +`workdir()` and `git_dir()` with the existing Windows-only path separator +conversion, including `Repo.common_dir` and gitfile worktree paths. Gix still +resolves every location. Ordinary, bare and linked repository metadata and +worktree-list queries compare exactly against Git without additional CLI +calls. POSIX filenames containing literal backslashes are unaffected. + +Discovery catches only the binding's exact +`RuntimeError("native worker panicked")` and selects Git's existing validation +path. The malformed metadata regression then raises `InvalidGitRepositoryError`. +Unrelated runtime +errors still propagate. Both normal native write errors and worker panics +after a mutation begins are tested to ensure they never trigger a CLI retry. +GIX-14 remains open: these adapter changes do not fix the upstream panic or +the other discovery compatibility bugs. + +The tests now pass gitfile operands in Git's Windows-compatible spelling and +compare symlink resolution against the current platform's Git output. Status +and ignore parity runs with a portable filename everywhere; only the extra +newline-filename variant is skipped on Windows. Blob-reference updates also +compare against Git: a direct branch rejects the blob, while a symbolic alias +outside `refs/heads/` accepts it on Git 2.55.0.windows.3, in both object +formats. That alias behavior is not evidence of a Gix mismatch and does not +invalidate GIX-24's separate direct-branch comparison. + +For GIX-8, at fixture commit `44e0a8ec55c42559dfcdf5117710b26261a7c937`, +compare `git rev-list HEAD` with +`repo.rev_walk([repo.head_id()]).sorting("newest_first").all()`. +The first different IDs were Git's `5a53ae6d68e318a85be78fb5fcee4d3aa9dfbb48` +and gix's `ee854dcb62220adeae4feb59bd10185e7ac02957`. +Native walk builders return new values: retain the result of +`first_parent_only()` and other builder methods. + +For GIX-15, compare `git blame --line-porcelain HEAD -- README.md` with +`repo.blame_file("README.md", repo.head_id(), +gix.BlameOptions(diff_algorithm="myers", rewrites=gix.Rewrites()))` at that same +fixture commit. The differing commits are +`2ddd5e5ef89da7f1e3b3a7d081fbc7f5c46ac11c` and +`3aacb3717ad78ec40e5b168a7ce8109aee6f156e`. `VERSION` and +`git/objects/commit.py` matched in the comparison. + +For GIX-18, set `file diff=forced` in `.gitattributes`, +`diff.forced.binary=true`, and `diff.forced.textconv=false`. Change a NUL-containing +line in `file`: the native `to_git` cache reports one insertion and one removal, +while Git with `--no-textconv --numstat` reports `-\t-\tfile` (binary). + +For GIX-24, use a repository containing an empty tree, a commit and a blob. +Pass the blob ID as the tree or a parent to `new_commit_as()` and compare +`git commit-tree BLOB` or `git commit-tree TREE -p BLOB`. Insert a commit ID +under the `blob` or `tree` kind with `Tree.edit().upsert()`, or a blob ID under +`commit`, then `write()`; Git's corresponding `mktree --missing` entry rejects +each mismatched kind. Finally, submit a `RefEdit.update_with_log()` for +`refs/heads/bad` with `Target.Object(blob_id)` and `PreviousValue.Any`; +`git update-ref refs/heads/bad BLOB` rejects that branch target. + +The GIX-10 transaction probes start with HEAD pointing to `refs/heads/main` +at a first commit. Apply `RefEdit.update_with_log()` with `PreviousValue.Any`, +`LogChange.force_create_reflog=True` and a fixed message, comparing +`git update-ref --create-reflog -m MESSAGE`. Test advancing the branch, writing +its current target again, updating through a symbolic alias, and detaching HEAD +with `with_deref(False)` / `--no-deref`. Compare both the branch and HEAD logs, +including the previous OID, as recorded in the ledger. + +The latest direct comparisons covered 12 cases in each of SHA-1 and SHA-256: +six object/branch type mismatches, identity cleanup, dangling enumeration and +four reflog transactions. Each case called Gix first and Git second. The local +probe and captured results are in +`.cache/gix-compatibility-ledger/{probe-contracts.py,contracts.json}`. +Existing [backend regressions](../test/test_gix_backend.py) protect the adapter +fallbacks, including `test_tree_writes_validate_child_object_kinds`, +`test_branch_blob_updates_match_git`, +`test_commit_identity_cleanup_matches_git` and +`test_reference_enumeration_preserves_aliases_and_skips_dangling_refs`. +This ledger update changes no runtime code or CLI budgets; the earlier +runtime validation and timings remain attributed to their measured snapshots. + +### Remaining configuration reads (GIX-12) + +The October 7 inventory identified 951 remote-enumeration reads, 674 actor +lookups, 262 remote-property reads, 202 tracking-branch lookups and 111 fetch +refspec checks. These are historical call-site counts, not additive savings. +The dedicated `.gitmodules` parser now covers supported submodule reads; +the other consumers still need generic config binding additions. + +Local probes with official GixPython 0.1.0 and the existing CPython 3.12.14 +confirmed these contracts against both GitPython backends: + +| Consumer | GitPython behavior to preserve | Why the existing native API is insufficient | +| --- | --- | --- | +| Remote enumeration | Repository-local declaration order: `z-last`, `a-first` | `remote_names()` returned the merged, sorted `a-first`, `global-only`, `z-last`. | +| Remote URL/properties | The cached raw `url` property returned the last value, `../second`; `urls` yielded both `../first` and `../second`. | Native `Remote.url(Fetch)` selected only `../first`. Generic multivalue enumeration and raw snapshot semantics are still needed. | +| Actor lookup | A supplied file/stream reader returned `Reader Actor`, independent of the repository's `Repository Actor`. | Repository identity lookup cannot honor arbitrary source lists, reader snapshots, or GitPython's identity fallback rules. | +| Tracking branch | `branch.main.remote=z-last` and `merge=refs/heads/topic` produced `refs/remotes/z-last/topic`. | Native tracking lookup applied the configured fetch mapping and returned `refs/foreign/topic`. Replacing this API requires the original scalar config values. | +| Fetch refspec presence | The cached local reader tests whether a value exists before invoking fetch. | Native remote creation/refspec access applies URL/refspec parsing and merged configuration; it does not expose this raw, source-scoped presence check. | +| General config readers | Explicit files/streams, their order, included files, duplicate values and cached reader state | `ConfigFile` exposes only `boolean`, `integer`, `string`, `set_raw_value` and `to_bstring`. `OpenOptions.isolated()` also disables includes: a local included value read as `yes` through GitPython was absent natively. | + +The needed additions are standalone generic parsing, ordered section/key and +multivalue access, source selection, and mutation/removal primitives. The +submodule adapter's conservative section guards can be removed once those +bindings exist. No generic configuration parser is implemented in Python. +The fixed general-config probe remains **2 CLI launches before and after** +this investigation; this item does not claim a process reduction. + +## Other remaining operations + +| Operation family | Why it still uses Git | +| --- | --- | +| `Repo.init`, clone | The different bundled initialization template set remains a Git compatibility bug; a compatible mode must reproduce Git's effective templates. Git's template/reinitialization options are not exposed. Clone also needs GitPython's environment, local-copy, progress and cleanup contracts. | +| Fetch, push, pull, remote pruning | Fetch outcomes do not expose the per-ref old IDs/status/notes needed for `FetchInfo`. Push/pull porcelain is absent. Network mutations cannot be retried after a partial native attempt. | +| Checkout, reset, index checkout/merge, move/remove | No bound equivalent of checkout-index/unpack-trees preserving these worktree/index semantics. Native tree/commit merge is a different operation. | +| Branch/tag porcelain and symbolic-ref mutation | Strict creation, reflogs, config movement, signing, message cleanup and safety contracts need more than a raw ref edit. | +| Object enumeration and alternate-directory listing | No binding for complete ODB object iteration or alternate-store enumeration; `cat-file --batch-all-objects` / `count-objects` remain. | +| Name-rev, trailer parsing, cherry | No equivalent bound operation. Native describe is not name-rev. | +| Config-backed remote/submodule maintenance | Missing config enumeration/multivar/removal operations (GIX-12), plus filesystem and worktree lifecycle requirements. | +| Hook execution | Every explicit hook request, including an absent hook, retains Git-managed lookup, execution and diagnostics until GIX-23 is addressed. Native operations retain GitPython's default hook/fsmonitor/maintenance restrictions. | + +## Maintaining the adapters + +`git/_backend.py` contains the capability checks. Managed command adapters return +the same bytes/text as the existing parser expects; direct adapters return the +existing GitPython object types. Keep operand validation before dispatch. + +Gix APIs must provide Git behavior. Python glue may validate inputs, format +native results and select fallback; it must not implement missing Git semantics +or fabricate successful answers to lower CLI counts. Document missing or +unclear capabilities here and retain the CLI path until Gix provides them. +Name the actual Gix calls behind each conversion. Record every observed Git/Gix +mismatch as a compatibility bug with expected/actual behavior, version and a +reproduction or regression. A Gix-based workaround does not resolve that bug; +require an upstream fix or a verified mode as strict as Git before closing it. + +Decide whether to fall back before mutating anything. Read/preparation failures +can use Git to preserve its public diagnostics. Once `_write()` begins, +`gix.Error` failures become `GitCommandError`; other exceptions propagate. +Neither may trigger a second CLI mutation. +Access native repository state through `Repo._get_gix_repository()` for bound +commands. Reuse is the default, with explicit recreation available; keep the +snapshot deviation and refresh behavior above documented. Close native iterators +when partially consumed. + +Add each conversion with a no-subprocess check and a Git parity check where +practical, in its own commit. Update this ledger when a limitation changes; +the runtime report's reason should point to the corresponding entry. + +## CI coverage + +The [Python package workflow](../.github/workflows/pythonpackage.yml) adds one +Python 3.12 Gix job each on Ubuntu, macOS and Windows, preserving all 28 CLI +combinations and their existing check names. It installs `.[test,gix]`, verifies +the selected backend and runs the full suite with the same coverage and pytest +options. Every job retains JUnit results +and `--backend-report` operation counts under its `tests-OS-PYTHON-BACKEND` +artifact, and prints the 30 slowest test durations. The extra Gix jobs do not +duplicate the documentation build. Their check names include `gix`, such as +`test (ubuntu, 3.12, gix)`. + +On platforms without a matching wheel, pip builds the official source release. +`CARGO_TARGET_DIR` keeps compilation output outside pip's temporary source +directory. The Cargo cache distinguishes OS, architecture, Python and Rust +versions and `gix-requirements.txt`; a release change can reuse dependencies +from the same platform and toolchain. + +## Windows validation + +Local Windows checks use CPython 3.12.13, Git 2.55.0.windows.3 and GixPython +0.1.0 built from the released source distribution. The installed extension +matches the cached Windows ABI3 wheel byte-for-byte; that wheel's SHA-256 is +`1a0addd5f569e2ff7fb2b45c38be1093f6cabbead176c779533b983b86307dc2`. + +The full baseline at `f6779dbb` reproduced all 17 failures described above: +16 ordinary failures and the subtest with malformed `commondir`. The full +patched suite passed with **1,760 tests and 40 subtests**, 58 skipped, nine +expected failures and three unexpected passes in 2,242.66 seconds. The three +unexpected passes and the path-deprecation warning also appeared in the +baseline. Ruff, mypy, basedpyright and the pinned pre-commit checks passed. + +The baseline recorded 21,598 CLI launches in 2,279.10 seconds; the patched +run recorded 21,826 CLI launches and 86,936 native operations. Totals include the additional +regressions and tests that previously stopped at failing assertions. No CLI +ceilings were lowered. Both runs used isolated snapshots and CI's coverage, +Git configuration and pytest options. They overlapped, so the elapsed times +are not a controlled speed comparison. Logs, JUnit results and operation +reports are retained locally in `.cache/gix-windows/`. + +The CLI repository/discovery regressions passed with 138 tests, 14 subtests, +nine skips and one expected failure. The changed native path, discovery, +reference and mutation-error cases also passed under Ubuntu WSL: 43 tests +and 14 subtests, using CPython 3.14.4, Git 2.53.0 and a separate released-source +GixPython build. This includes the POSIX newline-filename cases. + +## Published release validation + +The official Apple Silicon ABI3 wheel for GixPython 0.1.0 has SHA-256 +`6b68758cb54a90c8eb893ac7815e74ccd9b48b197aaecec951366a6326168f52`. +It was downloaded from PyPI and verified against the published digest. +At the backend-selection commit, its SHA-1/SHA-256 smoke checks and fresh +extra-installation check passed (3 tests) on the existing CPython 3.12.14/macOS +environment. The dependency commit and all 31 descendants passed the fast QA +profile: Ruff lint/format, pre-commit, mypy, basedpyright, universal dependency +resolution, and fatal lint checks for the bundled dependencies. The full +published-wheel suite subsequently passed with coverage: 1,649 tests and +38 subtests passed, 79 skipped, and one expected failure, in 757.85 seconds +wall-clock on the same interpreter/platform. The performance notes above +record the tested revision and future optimization candidates. + +## Historical local artifact validation + +The original integration was validated with GixPython 0.1.0 from local checkout +`c8a9fabc03c326ad7b0afdf03d0bda4e62fdff52`, with Gitoxide revision +`f819565c2c4c56619c4888acef6cf3b8144cbccb` and its existing vendored fixes. +The ordinary Apple Silicon ABI3 wheel in `pygix/dist` has SHA-256 +`62a03fe63e0e043c89f8271c10540cb00d31a15cf81fc1f45d3e2feb82a44e9f`. +The installed extension was compared byte-for-byte with that wheel. + +Validation uses CPython 3.12.14 on macOS. The focused backend tests cover both +SHA-1 and SHA-256. Platform-specific skipped tests are not claims of validation +on other operating systems or free-threaded Python. The full suite and operation +report can be reproduced with the commands above. + +| Check | Result | +| --- | --- | +| Full CLI installation | 1,360 passed, 80 skipped, 1 expected failure; 38 subtests passed. | +| Full GixPython installation | 1,396 passed, 79 skipped, 1 expected failure; 38 subtests passed. | +| Expanded native parity/fallback checks | 36 passed, including object-kind validation, identity cleanup, symbolic aliases/dangling refs, and compatibility-format guards. | +| Ruff lint/format, mypy, basedpyright | Passed. | +| Offline package build | Built an sdist and rebuilt its wheel; the archive includes `gix-requirements.txt` and the wheel declares the `gix` extra. | + +The wheel directory also supported a fresh offline `.[test,gix]` installation. +The full installation test checks that its new environment selects the same +backend as the environment running the tests. Tox itself was not installed on +this machine; the underlying interpreter-based command was used directly. diff --git a/doc/source/changes.rst b/doc/source/changes.rst index 8447228db..64ddddc3b 100644 --- a/doc/source/changes.rst +++ b/doc/source/changes.rst @@ -2,6 +2,103 @@ Changelog ========= +Unreleased: Git CLI migration +============================= + +GitPython now delegates repository discovery, references, configuration, object +storage, tree construction, index operations, and revision parsing to Git. The +default backend supports SHA-1 and SHA-256 repositories with either files or +reftable reference storage. Repository object IDs must no longer be assumed to +contain 20 binary bytes or 40 hexadecimal characters. +Legacy class-level ``NULL_BIN_SHA`` and ``NULL_HEX_SHA`` constants retain their +SHA-1 values; do not use their width to interpret repository object IDs. + +Git executable and safety +------------------------- + +* Git **2.52 or newer** is required for library-managed operations. Older versions + raise ``UnsupportedOperation`` before repository mutation; there is no Python + implementation fallback. ``GIT_PYTHON_GIT_EXECUTABLE`` still selects Git. +* Every managed command uses argument sequences without a shell. Existing unsafe + option/protocol checks remain, and operands and stdin records receive additional + validation. NUL-delimited ``cat-file`` requests prevent newline injection. + Previously accepted option-like revision/ref arguments can now be rejected. +* New plumbing calls suppress implicit hooks, filesystem monitors, automatic + maintenance, and lazy network fetches. Index staging continues to store raw + content rather than introduce clean filters. Diffs disable external diff and + text conversion programs; blame disables text conversion unless unsafe options + are explicitly enabled. Explicit commit hooks use ``git hook run`` and honor + ``skip_hooks``; Git combines their output, so ``HookExecutionError`` may carry + former stdout text in stderr. Trailer additions reject executable trailer + configuration instead of invoking it. +* Existing working-tree conversions retain Git's behavior: status and working-tree + diffs can run configured clean filters; checkout, clone, and archive can run + smudge filters. Archive defaults to Git's built-in formats and internal gzip; + custom format commands require ``allow_unsafe_options=True``. Tag creation + suppresses configured signing by default; signing, verification, and editor + options require the same opt-in. +* The raw ``repo.git`` interface remains available for direct Git commands. + Existing explicit unsafe-option and unsafe-protocol opt-ins remain independent. + They do not permit argument or stdin-record injection, or reordering command + options past the library's safety flags. +* Known command outcomes retain established GitPython/configparser exceptions + where distinguishable. Other failures raise ``GitCommandError`` with Git's + exit status and diagnostic output; exact error text may differ. + +API changes +----------- + +* ``GitCmdObjectDB`` no longer inherits ``LooseObjectDB``. Its object reads, + writes, existence checks, and enumeration use Git, including packed objects. + Precompressed input streams and custom object-output writers are unsupported. + The deprecated ``GitDB`` remains explicitly selectable with its existing + warning and limitations; ``gitdb`` remains a dependency for shared types. +* ``Repo.object_format`` and ``Repo.ref_format`` report Git's storage formats. + ``Repo.alternates`` is read-only and reports effective absolute alternate + directories, including environment and transitive alternates. Direct editing + of the alternates file through this property is removed. +* Index entries retain their mode, object ID, path, and stage. Raw stat fields, + arbitrary cache flags/extensions/checksums, ``IndexFileSHA1Writer``, and the + standalone binary index read/write/merge helpers are removed. Use + ``IndexFile.from_tree()``, ``IndexFile.new()``, ``write()`` and ``write_tree()``. + Git preserves untouched metadata when an existing index is edited. Temporary + indexes isolate tree/merge operations from the real index and working tree. + Git's platform-specific index filename restrictions apply. Unsupported entries + raise ``ValueError`` before the original index changes, including names with + colons or control characters on Windows and backslashes with Cygwin Git. + Tree objects can still contain names that the working tree or index cannot + represent. + ``version`` is read-only. ``from_tree()`` accepts ``trivial``, ``aggressive``, + and ``verbose`` options; arbitrary ``read-tree`` keyword forwarding is removed. +* Standalone binary tree parsers, serializers, and multi-tree traversal helpers + are removed. Use ``Tree`` traversal/cache operations and the index interface. + Tree/commit serialization adapters that remain write their objects to the + repository to obtain Git-produced bytes. + Commit creation preserves message bytes through plain stdin. Injecting or + reusing arbitrary ``gpgsig`` headers is unsupported; modified signed commits + become unsigned. +* ``RefLog`` is associated with a reference, not a filesystem path. Keep using + ``ref.log()``, ``ref.log_entry()`` and ``ref.log_append()``. ``RefLogEntry`` now + contains ``newhexsha``, ``actor``, ``time`` and ``message``; ``oldhexsha`` and + the old tuple layout are removed. Reads follow Git's commit-reflog view, which + omits entries targeting non-commit or unavailable objects. Old IDs are never + inferred from adjacent entries. Raw reflog file/stream read, write, and rewrite + APIs are removed. Reference updates follow Git's reflog creation/update rules, + including updates when ``logmsg`` is omitted. +* ``GitConfigParser`` uses Git's configuration syntax and canonical key spelling, + and no longer subclasses ``configparser.RawConfigParser``. + Common getters, typed values, duplicate values, file/stream inputs and mutation + methods remain. Invalid Git syntax previously accepted by Python is rejected. + Empty-section creation and valueless writes are removed; valueless reads remain. + Relative includes from streams are rejected by Git. Locks cover each native + mutation rather than the writer object's lifetime. +* ``Submodule.rename()`` is removed. Moving a submodule preserves its logical name, + matching Git. Normal removal retains Git's recoverable submodule metadata. + Re-adding with ``no_checkout=True`` can reuse that metadata without changing refs; + incompatible URLs, branches, and clone-only options are rejected before mutation. + Fetch/pull results are derived from command output rather than ``FETCH_HEAD`` + file parsing. Revision strings follow Git's native revision grammar. + 3.2.1 ===== diff --git a/doc/source/intro.rst b/doc/source/intro.rst index dec2ec0fa..abf2d517a 100644 --- a/doc/source/intro.rst +++ b/doc/source/intro.rst @@ -6,7 +6,7 @@ Overview / Install GitPython is a python library used to interact with git repositories, high-level like git-porcelain, or low-level like git-plumbing. -It provides abstractions of git objects for easy access of repository data, and additionally allows you to access the git repository more directly using either a pure python implementation, or the faster, but more resource intensive git command implementation. +It provides Python objects for repository data and delegates Git operations to the Git executable. The default backend supports both SHA-1 and SHA-256 object IDs and both files and reftable reference storage. The legacy pure-Python GitDB backend remains available but is deprecated. The object database implementation is optimized for handling large quantities of objects and large datasets, which is achieved by using low-level structures and data streaming. @@ -14,10 +14,8 @@ Requirements ============ * `Python`_ >= 3.8 -* `Git`_ 1.7.0 or newer - It should also work with older versions, but it may be that some operations - involving remotes will not work as expected. -* `GitDB`_ - a pure python git database implementation +* `Git`_ 2.52 or newer +* `GitDB`_ - shared data types and the deprecated legacy object database * `typing_extensions`_ >= 3.7.3.4 (if python < 3.10) .. _Python: https://www.python.org diff --git a/doc/source/tutorial.rst b/doc/source/tutorial.rst index a1ac1eace..34f6692b5 100644 --- a/doc/source/tutorial.rst +++ b/doc/source/tutorial.rst @@ -122,7 +122,7 @@ You can traverse down to :class:`git objects ` through :start-after: # [12-test_init_repo_object] :end-before: # ![12-test_init_repo_object] -The :class:`index ` is also called stage in git-speak. It is used to prepare new commits, and can be used to keep results of merge operations. Our index implementation allows to stream date into the index, which is useful for bare repositories that do not have a working tree. +The :class:`index ` is also called stage in git-speak. It is used to prepare new commits, and can be used to keep results of merge operations. Our index implementation allows to stream data into the index, which is useful for bare repositories that do not have a working tree. .. literalinclude:: ../../test/test_docs.py :language: python @@ -166,7 +166,7 @@ A :class:`symbolic reference ` can point to :start-after: # [3-test_references_and_objects] :end-before: # ![3-test_references_and_objects] -Access the :class:`reflog ` easily. +Access the :class:`reflog ` through a reference. Entries expose the new object ID, reflog actor, timestamp, and message in oldest-first order. They follow Git's commit-reflog view; exact old IDs and raw reflog-file access are unavailable. .. literalinclude:: ../../test/test_docs.py :language: python @@ -202,7 +202,7 @@ Change the :class:`symbolic reference ` to Understanding Objects ********************* -An Object is anything storable in git's object database. Objects contain information about their type, their uncompressed size as well as the actual data. Each object is uniquely identified by a binary SHA1 hash, being 20 bytes in size, or 40 bytes in hexadecimal notation. +An Object is anything storable in git's object database. Objects contain information about their type, their uncompressed size as well as the actual data. Each object is identified by an object ID in the repository's hash format, reported by ``repo.object_format``. SHA-1 IDs contain 20 binary bytes (40 hexadecimal characters); SHA-256 IDs contain 32 binary bytes (64 hexadecimal characters). Treat IDs returned by Git as opaque values instead of assuming a fixed width. Git only knows 4 distinct object types being :class:`Blobs `, :class:`Trees `, :class:`Commits ` and :class:`Tags `. @@ -341,7 +341,7 @@ As trees allow direct access to their intermediate child entries only, use the t The Index Object **************** -The git index is the stage containing changes to be written with the next commit or where merges finally have to take place. You may freely access and manipulate this information using the :class:`IndexFile ` object. +The git index is the stage containing changes to be written with the next commit or where merges finally have to take place. You may access and manipulate semantic entries (mode, object ID, path, and stage) using the :class:`IndexFile ` object. Git reads and writes the underlying index; raw stat fields and binary index extensions are not exposed. Modify the index with ease .. literalinclude:: ../../test/test_docs.py @@ -377,13 +377,13 @@ You can easily access configuration information for a remote by accessing option :start-after: # [26-test_references_and_objects] :end-before: # ![26-test_references_and_objects] -You can also specify per-call custom environments using a new context manager on the Git command, e.g. for using a specific SSH key. The following example works with `git` starting at *v2.3*:: +You can also specify per-call custom environments using a context manager on the Git command, e.g. for using a specific SSH key:: ssh_cmd = 'ssh -i id_deployment_key' with repo.git.custom_environment(GIT_SSH_COMMAND=ssh_cmd): repo.remotes.origin.fetch() -This one sets a custom script to be executed in place of `ssh`, and can be used in `git` prior to *v2.3*:: +Alternatively, set a custom script to be executed in place of `ssh`:: ssh_executable = os.path.join(rw_dir, 'my_ssh_executable.sh') with repo.git.custom_environment(GIT_SSH=ssh_executable): diff --git a/fuzzing/README.md b/fuzzing/README.md index 286f529eb..0d557d658 100644 --- a/fuzzing/README.md +++ b/fuzzing/README.md @@ -46,6 +46,10 @@ capabilities, jump into the "Getting Started" section below. Before contributing to fuzzing efforts, ensure Python and Docker are installed on your machine. Docker is required for running fuzzers in containers provided by OSS-Fuzz and for safely executing test files directly. [Install Docker](https://docs.docker.com/get-docker/) following the official guide if you do not already have it. +The fuzz targets require **Git 2.52 or newer**, including the Git executable bundled into OSS-Fuzz artifacts. +The local development image builds pinned Git 2.52.0. The OSS-Fuzz bootstrap and build scripts reject older selected +Git executables before preparing artifacts; update the external OSS-Fuzz image when its installed Git is too old. + ### Understanding Existing Fuzz Targets Review the `fuzz-targets/` directory to familiarize yourself with how existing tests are implemented. See @@ -67,6 +71,8 @@ Contains Python files for each fuzz test. **Things to Know**: - Each fuzz test targets a specific part of GitPython's functionality. +- Repository and object targets exercise Git-backed workflows. Removed Python index/tree binary parsers are no longer + fuzz targets; malformed configuration and object IDs can now be rejected by Git or by Python input validation. - Test files adhere to the naming convention: `fuzz_.py`, where `` indicates the functionality targeted by the test. - Any functionality that involves performing operations on input data is a possible candidate for fuzz testing, but diff --git a/fuzzing/fuzz-targets/fuzz_blob.py b/fuzzing/fuzz-targets/fuzz_blob.py index ce888e85f..afbdf7780 100644 --- a/fuzzing/fuzz-targets/fuzz_blob.py +++ b/fuzzing/fuzz-targets/fuzz_blob.py @@ -16,17 +16,16 @@ def TestOneInput(data): with tempfile.TemporaryDirectory() as temp_dir: repo = git.Repo.init(path=temp_dir) - binsha = fdp.ConsumeBytes(20) + binsha = fdp.ConsumeBytes(repo._oid_size) mode = fdp.ConsumeInt(fdp.ConsumeIntInRange(0, fdp.remaining_bytes())) path = fdp.ConsumeUnicodeNoSurrogates(fdp.remaining_bytes()) try: blob = git.Blob(repo, binsha, mode, path) - except AssertionError as e: - if "Require 20 byte binary sha, got" in str(e): + except ValueError: + if len(binsha) != repo._oid_size: return -1 - else: - raise e + raise _ = blob.mime_type diff --git a/fuzzing/fuzz-targets/fuzz_config.py b/fuzzing/fuzz-targets/fuzz_config.py index 4eddc32ff..87ef6e846 100644 --- a/fuzzing/fuzz-targets/fuzz_config.py +++ b/fuzzing/fuzz-targets/fuzz_config.py @@ -39,6 +39,10 @@ def TestOneInput(data): git_config.read() except (MissingSectionHeaderError, ParsingError, UnicodeDecodeError): return -1 # Reject inputs raising expected exceptions + except git.GitCommandError as e: + if e.status == 128: + return -1 # Git rejects invalid includes and other configuration input. + raise # Preserve crashes and unexpected command failures. except ValueError as e: if "embedded null byte" in str(e): # The `os.path.expanduser` function, which does not accept strings diff --git a/fuzzing/fuzz-targets/fuzz_diff.py b/fuzzing/fuzz-targets/fuzz_diff.py index d4bd68b57..3e15a3b7b 100644 --- a/fuzzing/fuzz-targets/fuzz_diff.py +++ b/fuzzing/fuzz-targets/fuzz_diff.py @@ -40,8 +40,8 @@ def TestOneInput(data): repo, a_rawpath=fdp.ConsumeBytes(fdp.ConsumeIntInRange(0, fdp.remaining_bytes())), b_rawpath=fdp.ConsumeBytes(fdp.ConsumeIntInRange(0, fdp.remaining_bytes())), - a_blob_id=fdp.ConsumeBytes(20), - b_blob_id=fdp.ConsumeBytes(20), + a_blob_id=fdp.ConsumeBytes(repo._oid_size * 2), + b_blob_id=fdp.ConsumeBytes(repo._oid_size * 2), a_mode=fdp.ConsumeBytes(fdp.ConsumeIntInRange(0, fdp.remaining_bytes())), b_mode=fdp.ConsumeBytes(fdp.ConsumeIntInRange(0, fdp.remaining_bytes())), new_file=fdp.ConsumeBool(), @@ -55,11 +55,10 @@ def TestOneInput(data): ) except BinasciiError: return -1 - except AssertionError as e: - if "Require 20 byte binary sha, got" in str(e): + except ValueError as e: + if "Object ID does not match the repository object format" in str(e): return -1 - else: - raise e + raise _ = diff.__str__() _ = diff.a_path diff --git a/fuzzing/fuzz-targets/fuzz_repo.py b/fuzzing/fuzz-targets/fuzz_repo.py index 7bd82c120..9045b7fee 100644 --- a/fuzzing/fuzz-targets/fuzz_repo.py +++ b/fuzzing/fuzz-targets/fuzz_repo.py @@ -1,5 +1,4 @@ import atheris -import io import sys import os import tempfile @@ -15,9 +14,7 @@ def TestOneInput(data): fdp = atheris.FuzzedDataProvider(data) - with tempfile.TemporaryDirectory() as temp_dir: - repo = git.Repo.init(path=temp_dir) - + with tempfile.TemporaryDirectory() as temp_dir, git.Repo.init(path=temp_dir) as repo: # Generate a minimal set of files based on fuzz data to minimize I/O operations. file_paths = [os.path.join(temp_dir, f"File{i}") for i in range(min(3, fdp.ConsumeIntInRange(1, 3)))] for file_path in file_paths: @@ -27,15 +24,14 @@ def TestOneInput(data): # fuzzer coverage plateaus. f.write(fdp.ConsumeBytes(fdp.ConsumeIntInRange(1, 512))) - repo.index.add(file_paths) - repo.index.commit(fdp.ConsumeUnicodeNoSurrogates(fdp.ConsumeIntInRange(1, 80))) - - fuzz_tree = git.Tree(repo, git.Tree.NULL_BIN_SHA, 0, "") - - try: - fuzz_tree._deserialize(io.BytesIO(data)) - except IndexError: + message = fdp.ConsumeUnicodeNoSurrogates(fdp.ConsumeIntInRange(1, 80)) + if "\0" in message: return -1 + repo.index.add(file_paths) + actor = git.Actor("Fuzzing", "fuzzing@example.invalid") + commit = repo.index.commit(message, author=actor, committer=actor, skip_hooks=True) + for blob in commit.tree.blobs: + blob.data_stream.read() def main(): diff --git a/fuzzing/local-dev-helpers/Dockerfile b/fuzzing/local-dev-helpers/Dockerfile index 426de05dd..a947ecfeb 100644 --- a/fuzzing/local-dev-helpers/Dockerfile +++ b/fuzzing/local-dev-helpers/Dockerfile @@ -11,7 +11,10 @@ COPY . . # Update package managers, install necessary packages, and cleanup unnecessary files in a single RUN to keep the image smaller. RUN apt-get update && \ - apt-get install -y git clang && \ + apt-get install -y git clang build-essential libssl-dev zlib1g-dev libexpat1-dev libcurl4-openssl-dev && \ + git clone --depth 1 --branch v2.52.0 -- https://github.com/git/git.git /tmp/git-source && \ + make -C /tmp/git-source -j2 prefix=/usr/local NO_GETTEXT=YesPlease NO_TCLTK=YesPlease NO_PERL=YesPlease install && \ + rm -rf /tmp/git-source && \ python -m pip install --upgrade pip && \ python -m pip install atheris && \ python -m pip install -e . && \ diff --git a/fuzzing/oss-fuzz-scripts/build.sh b/fuzzing/oss-fuzz-scripts/build.sh index c156e872d..4b3356319 100644 --- a/fuzzing/oss-fuzz-scripts/build.sh +++ b/fuzzing/oss-fuzz-scripts/build.sh @@ -5,6 +5,18 @@ set -euo pipefail +git_binary="$(command -v git)" || { + printf 'GitPython fuzzing requires Git 2.52 or newer; git was not found on PATH.\n' >&2 + exit 1 +} +git_version="$("$git_binary" --version)" +if [[ ! "$git_version" =~ ^git\ version\ ([0-9]+)\.([0-9]+)(\.|[[:space:]]|$) ]] || + ((10#${BASH_REMATCH[1]} < 2 || (10#${BASH_REMATCH[1]} == 2 && 10#${BASH_REMATCH[2]} < 52))); then + printf 'GitPython fuzzing requires Git 2.52 or newer; %s reports %s. Update the container Git installation.\n' \ + "$git_binary" "$git_version" >&2 + exit 1 +fi + python3 -m pip install . find "$SRC" -maxdepth 1 \ @@ -15,5 +27,5 @@ find "$SRC" -maxdepth 1 \ # Build fuzzers in $OUT. find "$SRC/gitpython/fuzzing" -name 'fuzz_*.py' -print0 | while IFS= read -r -d '' fuzz_harness; do - compile_python_fuzzer "$fuzz_harness" --add-binary="$(command -v git):." --add-data="$SRC/explicit-exceptions-list.txt:." + compile_python_fuzzer "$fuzz_harness" --add-binary="$git_binary:." --add-data="$SRC/explicit-exceptions-list.txt:." done diff --git a/fuzzing/oss-fuzz-scripts/container-environment-bootstrap.sh b/fuzzing/oss-fuzz-scripts/container-environment-bootstrap.sh index 924a3cbf3..9d878d7c1 100755 --- a/fuzzing/oss-fuzz-scripts/container-environment-bootstrap.sh +++ b/fuzzing/oss-fuzz-scripts/container-environment-bootstrap.sh @@ -16,6 +16,15 @@ for cmd in python3 git wget zip; do } done +git_binary="$(command -v git)" +git_version="$("$git_binary" --version)" +if [[ ! "$git_version" =~ ^git\ version\ ([0-9]+)\.([0-9]+)(\.|[[:space:]]|$) ]] || + ((10#${BASH_REMATCH[1]} < 2 || (10#${BASH_REMATCH[1]} == 2 && 10#${BASH_REMATCH[2]} < 52))); then + printf 'GitPython fuzzing requires Git 2.52 or newer; %s reports %s. Update the container Git installation.\n' \ + "$git_binary" "$git_version" >&2 + exit 1 +fi + ############# # Functions # ############# @@ -85,7 +94,7 @@ prepare_dictionaries_for_fuzz_targets() { ######################## # Seed corpora and dictionaries are hosted in a separate repository to avoid additional bloat in this repo. # We clone into the $WORK directory because OSS-Fuzz cleans it up after building the image, keeping the image small. -git clone --depth 1 https://github.com/gitpython-developers/qa-assets.git "$WORK/qa-assets" +"$git_binary" clone --depth 1 -- https://github.com/gitpython-developers/qa-assets.git "$WORK/qa-assets" create_seed_corpora_zips "$WORK/qa-assets/gitpython/corpora" @@ -97,7 +106,7 @@ pushd "$SRC/gitpython/" # This file can then be used by fuzz harnesses to check exception tracebacks and filter out explicitly raised or otherwise # anticipated exceptions to reduce false positive test failures. -git grep -n --recurse-submodules -e '\braise\b' -e '\bassert\b' -- '*.py' -- ':!setup.py' -- ':!test/**' -- ':!fuzzing/**' > "$SRC/explicit-exceptions-list.txt" +"$git_binary" grep -n --recurse-submodules -e '\braise\b' -e '\bassert\b' -- '*.py' -- ':!setup.py' -- ':!test/**' -- ':!fuzzing/**' > "$SRC/explicit-exceptions-list.txt" popd diff --git a/git/__init__.py b/git/__init__.py index ecc6cd94e..8a8e71311 100644 --- a/git/__init__.py +++ b/git/__init__.py @@ -95,7 +95,7 @@ import warnings -from gitdb.util import to_hex_sha +from git.util import to_hex_sha from git.exc import ( AmbiguousObjectName, diff --git a/git/_backend.py b/git/_backend.py new file mode 100644 index 000000000..97098470f --- /dev/null +++ b/git/_backend.py @@ -0,0 +1,1304 @@ +"""Optional native implementation of library-managed Git operations. + +Installing ``GitPython[gix]`` makes the ``gix`` module available. Unsupported +operations return ``NotImplemented`` before mutation and use the existing CLI +implementation. Native mutation failures are never retried through the CLI. +""" + +from collections import Counter +from glob import glob +from importlib import import_module +import io +from itertools import islice +import logging +import os +import re +import stat +from threading import Lock +from typing import Any, Callable, Dict, Iterator, List, Optional, Sequence, Tuple, cast + +from git.compat import defenc, safe_decode +from git.exc import GitCommandError + +try: + gix: Any = import_module("gix") +except ModuleNotFoundError as exc: + if exc.name != "gix": + raise + gix = None + +name = "gix" if gix is not None else "cli" +_counts: Counter = Counter() +_lock = Lock() +_logger = logging.getLogger("git.backend") + + +def record(method: str, outcome: str) -> None: + with _lock: + _counts[method, outcome] += 1 + _logger.debug("%s: %s", method, outcome) + + +def statistics() -> Dict[Tuple[str, str], int]: + """Return counts by operation and native/fallback reason for this process.""" + with _lock: + return dict(_counts) + + +class _Unsupported(Exception): + """A capability decision made before a native mutation.""" + + +def _fallback(method: str, reason: str) -> Any: + record(method, "CLI: " + reason) + return NotImplemented + + +def discover_repository(path: str, environment: Dict[str, Any]) -> Any: + """Open one discovery candidate; the caller controls parent traversal.""" + if gix is None: + return NotImplemented + effective = {**os.environ, **environment} + if any( + effective.get(key) + for key in ("GIT_COMMON_DIR", "GIT_OBJECT_DIRECTORY", "GIT_ALTERNATE_OBJECT_DIRECTORIES", "GIT_NAMESPACE") + ) or any(value != os.environ.get(key) for key, value in environment.items() if key != "GIT_WORK_TREE"): + return _fallback("Repo.open", "storage or command environment") + options = gix.OpenOptions().open_path_as_is(True).bail_if_untrusted(True).strict_config(True) + options = options.config_overrides( + ["core.fsmonitor=false", "gc.auto=0", "maintenance.auto=false", "core.hooksPath=" + os.devnull] + ) + try: + repo = gix.open_opts(path, options) + if os.path.isfile(path): + # Reopening also makes Gix derive the worktree from its Git directory, + # rather than retaining an arbitrary gitfile's location as a worktree. + repo = gix.open_opts(os.path.realpath(repo.git_dir()), options) + snapshot = repo.config_snapshot() + if snapshot.string("extensions.refStorage") == b"reftable": + return _fallback("Repo.open", "reftable (GIX-1)") + if snapshot.string("extensions.compatObjectFormat") is not None: + return _fallback("Repo.open", "compatibility object format (GIX-19)") + # Discovery accepts undecodable HEADs; force the native reference decoder. + repo.head() + if not repo.is_bare() and repo.workdir() is None and not effective.get("GIT_WORK_TREE"): + return _fallback("Repo.open", "missing native worktree metadata (GIX-14)") + commondir = os.path.join(repo.git_dir(), "commondir") + if os.path.lexists(commondir): + # Gix ignores invalid commondir files on common repositories. A common + # directory must have been resolved and contain actual shared storage. + if os.path.realpath(repo.common_dir()) == os.path.realpath(repo.git_dir()) or not all( + os.path.isdir(os.path.join(repo.common_dir(), entry)) for entry in ("objects", "refs") + ): + return _fallback("Repo.open", "common-directory validation (GIX-14)") + record("Repo.open", "native") + return repo + except gix.Error as exc: + # Retain Git's rejection/diagnostics for layouts Gix cannot open yet. + _logger.debug("native discovery: %s", exc) + return _fallback("Repo.open", "native discovery diagnostics") + except RuntimeError as exc: + if str(exc) != "native worker panicked": + raise + # Undecodable Windows commondir paths panic in GixPython 0.1.0. + # Discovery is read-only; keep Git's validation and error contract. + _logger.debug("native discovery: %s", exc) + return _fallback("Repo.open", "native discovery worker panic (GIX-14)") + + +def _repository(command: Any, env: Dict[str, Any], *, query_config: bool = False) -> Any: + owner = command._repo() if command._repo is not None else None + if owner is not None: + return owner._get_gix_repository(command=command, env=env, query_config=query_config) + return _open_repository(command, env, query_config=query_config) + + +def _open_repository(command: Any, env: Dict[str, Any], *, query_config: bool = False) -> Any: + if command._git_options or command._persistent_git_options: + raise _Unsupported("global command options") + overrides = {**command.environment(), **env} + effective = {**os.environ, **overrides} + path = overrides.get("GIT_DIR") + if not path: + raise _Unsupported("repository not bound") + if not os.path.isabs(path): + path = os.path.join(command.working_dir or os.getcwd(), path) + for key in ("GIT_COMMON_DIR", "GIT_OBJECT_DIRECTORY", "GIT_ALTERNATE_OBJECT_DIRECTORIES", "GIT_NAMESPACE"): + if effective.get(key): + raise _Unsupported("storage environment") + supported = {"GIT_DIR", "GIT_WORK_TREE", "GIT_INDEX_FILE"} + supported.update( + "GIT_%s_%s" % (role, field) for role in ("AUTHOR", "COMMITTER") for field in ("NAME", "EMAIL", "DATE") + ) + if any(key not in supported and value != os.environ.get(key) for key, value in overrides.items()): + raise _Unsupported("command environment") + options = gix.OpenOptions().open_path_as_is(True).bail_if_untrusted(True).strict_config(True) + if not query_config: + options = options.config_overrides( + ["core.fsmonitor=false", "gc.auto=0", "maintenance.auto=false", "core.hooksPath=" + os.devnull] + ) + owner = command._repo() if command._repo is not None else None + state = None + repo = None + if owner is not None and not query_config: + # Gix refreshes index/ODB snapshots itself; configuration is loaded at open. + paths = [ + path, + os.fspath(owner.common_dir), + os.path.join(owner.common_dir, "config"), + os.path.join(path, "config.worktree"), + ] + paths += [ + effective.get("GIT_CONFIG_SYSTEM", "/etc/gitconfig"), + effective.get("GIT_CONFIG_GLOBAL", os.path.expanduser("~/.gitconfig")), + os.path.join(effective.get("XDG_CONFIG_HOME", os.path.expanduser("~/.config")), "git", "config"), + os.path.join(owner.common_dir, "objects", "info", "alternates"), + os.path.join(path, "commondir"), + os.path.join(path, "gitdir"), + ] + stamps: List[Any] = [] + for filename in paths: + try: + info = os.stat(filename) + stamps.append((info.st_dev, info.st_ino, info.st_size, info.st_mtime_ns, info.st_ctime_ns)) + except FileNotFoundError: + stamps.append(None) + state = (path, tuple(sorted(effective.items())), tuple(stamps)) + repo = owner._gix_repository + if repo is not None and os.path.realpath(repo.git_dir()) != os.path.realpath(path): + repo = None + if repo is not None and state != owner._gix_state: + repo.reload() + if repo is None: + repo = gix.open_opts(path, options) + snapshot = repo.config_snapshot() + if snapshot.string("extensions.refStorage") == b"reftable": + raise _Unsupported("reftable (GIX-1)") + if snapshot.string("extensions.compatObjectFormat") is not None: + raise _Unsupported("compatibility object format (GIX-19)") + if effective.get("GIT_WORK_TREE") and (owner is None or query_config or state != owner._gix_state): + workdir = effective["GIT_WORK_TREE"] + if not os.path.isabs(workdir): + workdir = os.path.join(command.working_dir or os.getcwd(), workdir) + repo.set_workdir(workdir) + if owner is not None and not query_config: + owner._gix_repository = repo + # ponytail: includes can load arbitrary files; reopen until Gix exposes their source paths. + if state != owner._gix_state: + included = re.search(rb"\[include(?:if)?[\s\]]", repo.config_snapshot().plumbing().to_bstring(), re.I) + owner._gix_state = None if included else state + return repo + + +def _canonical_repository(repo: Any) -> Any: + """Have Gix resolve repository metadata again from a canonical input path.""" + path = os.fspath(repo.git_dir()) + canonical = os.path.realpath(path) + return gix.open_opts(canonical, repo.open_options()) if path != canonical else repo + + +def _check_revision_grammar(ref: str) -> None: + if ref.startswith(":/") or "^{/" in ref or "-dirty" in ref or re.search(r"-\d+-g[0-9a-fA-F]+", ref): + raise _Unsupported("revision grammar differences (GIX-17)") + + +def _oid(repo: Any, ref: Any, env: Optional[Dict[str, Any]] = None) -> Any: + ref = safe_decode(ref) if isinstance(ref, bytes) else str(ref) + if re.fullmatch(r"[a-fA-F0-9]{40}|[a-fA-F0-9]{64}", ref): + return gix.ObjectId(ref) + _check_revision_grammar(ref) + if ref.startswith(":"): + _index(repo, {"env": env or {}}) + if not hasattr(repo, "rev_parse_single"): + raise _Unsupported("revision feature disabled") + return repo.rev_parse_single(ref) + + +def object_data(command: Any, ref: bytes, *, stream: bool = False) -> Any: + """Use native object reads without changing the public cat-file interface.""" + if gix is None: + return NotImplemented + method = "stream_object_data" if stream else "get_object_header" + try: + repo = _repository(command, {}) + oid = _oid(repo, ref, command.environment()) + header = repo.try_find_header(oid) + if header is None: + raise ValueError("SHA %s could not be resolved" % oid) + result: Tuple[Any, ...] = (str(oid), header.kind(), header.size()) + if stream: + # ponytail: native lookup buffers the object; use CLI above 8 MiB until GIX-2 provides streaming. + if header.size() > 8 * 1024 * 1024: + raise _Unsupported("large object streaming (GIX-2)") + result += (io.BytesIO(repo.find_object(oid).data),) + except _Unsupported as exc: + return _fallback(method, str(exc)) + except gix.Error as exc: + _logger.debug("%s native read: %s", method, exc) + return _fallback(method, "native read diagnostics") + record(method, "native") + return result + + +def revision_info(command: Any, ref: str) -> Any: + """Resolve an object and its tree/index path metadata in one native parse.""" + if gix is None: + return NotImplemented + try: + _check_revision_grammar(ref) + repo = _repository(command, {}) + if ref.startswith(":"): + _index(repo, {"env": command.environment()}) + spec = repo.rev_parse(ref) + oid = spec.single() + if oid is None: + raise _Unsupported("multiple revisions") + metadata = spec.path_and_mode() + except _Unsupported as exc: + return _fallback("Repo.rev_parse", str(exc)) + except gix.Error as exc: + _logger.debug("native revision metadata: %s", exc) + return _fallback("Repo.rev_parse", "native read diagnostics") + record("Repo.rev_parse", "native") + return str(oid), metadata + + +def _rev_parse(repo: Any, args: List[str], kwargs: Dict[str, Any]) -> bytes: + from git.util import to_native_path_linux + + if args[:2] == ["--path-format=absolute", "--git-path"]: + if len(args) != 3 or args[2] not in ("modules", "COMMIT_EDITMSG"): + raise _Unsupported("Git metadata path resolution (GIX-22)") + # Git keeps these fixed leaves in the private Git directory, including + # linked worktrees. Other names have their own config/environment rules. + path = os.path.join(_canonical_repository(repo).git_dir(), args[2]) + if os.path.islink(path): + raise _Unsupported("symlinked metadata path (GIX-22)") + return os.fsencode(to_native_path_linux(path)) + b"\n" + if args == ["--path-format=absolute", "--git-common-dir"]: + return os.fsencode(to_native_path_linux(os.path.abspath(repo.common_dir()))) + b"\n" + if args == ["--is-bare-repository"]: + return b"true\n" if repo.is_bare() and repo.workdir() is None else b"false\n" + if args == ["--show-object-format"]: + return str(repo.object_hash()).encode("ascii") + b"\n" + if args == ["--show-ref-format"]: + # Config identifies reftable and HEAD reports it as unsupported, but + # neither validates unknown repository extensions as Git's query does. + raise _Unsupported("reference storage format query (GIX-1)") + if args == ["--show-toplevel"] and repo.workdir() is not None: + return os.fsencode(to_native_path_linux(os.path.abspath(repo.workdir()))) + b"\n" + if args[:1] != ["--verify"] or args[-2:-1] != ["--end-of-options"]: + raise _Unsupported("discovery or revision options") + if args[:-2] not in (["--verify"], ["--verify", "--quiet"]): + raise _Unsupported("revision options") + return str(_oid(repo, args[-1], kwargs.get("env"))).encode("ascii") + b"\n" + + +def _ls_tree(repo: Any, args: List[str], kwargs: Dict[str, Any]) -> bytes: + if len(args) != 3 or args[:2] != ["-z", "--full-tree"]: + raise _Unsupported("tree options") + with repo.find_tree(_oid(repo, args[2])).iter() as entries: + return b"".join( + b"%06o %s %s\t%s\0" + % ( + entry.mode(), + b"tree" if entry.kind() == "tree" else b"commit" if entry.kind() == "commit" else b"blob", + str(entry.id()).encode("ascii"), + entry.filename(), + ) + for entry in entries + ) + + +def check_ref_name(name: str) -> Any: + """Validate a full reference name without opening a repository.""" + if gix is None: + return NotImplemented + try: + gix.Target.Symbolic(name) + except gix.Error as exc: + if "/" not in name: + return _fallback("Reference.validate", "standalone reference names (GIX-20)") + record("Reference.validate", "native") + raise ValueError("Invalid reference %r" % name) from exc + record("Reference.validate", "native") + return None + + +def reference_info(command: Any, path: str) -> Any: + """Read an exact reference target, including an absent reference, in one lookup.""" + if gix is None: + return NotImplemented + try: + if path != "HEAD" and not path.startswith("refs/"): + raise _Unsupported("partial reference names") + repo = _repository(command, {}) + reference = repo.try_find_reference(path) + result = None + if reference is not None: + if reference.name() != os.fsencode(path): + raise _Unsupported("partial reference lookup") + target = reference.target() + name = target.try_name() + result = (None, safe_decode(name)) if name is not None else (str(target.try_id()), None) + except _Unsupported as exc: + return _fallback("Reference.read", str(exc)) + except gix.Error as exc: + _logger.debug("native reference read: %s", exc) + return _fallback("Reference.read", "native read diagnostics") + record("Reference.read", "native") + return result + + +def _symbolic_ref(repo: Any, args: List[str], kwargs: Dict[str, Any]) -> bytes: + if len(args) != 4 or args[:3] != ["--quiet", "--no-recurse", "--"]: + raise _Unsupported("reference mutation or options") + reference = repo.try_find_reference(args[-1]) + if reference is None: + raise _Unsupported("missing reference diagnostics") + target = reference.target().try_name() + if target is None: + raise GitCommandError(["git", "symbolic-ref"], 1) + return target + b"\n" + + +def _for_each_ref(repo: Any, args: List[str], kwargs: Dict[str, Any]) -> bytes: + if not args or args[0] != "--format=%(refname)": + raise _Unsupported("reference format/options") + split = args.index("--") if "--" in args else len(args) + if args[1:split] or len(args[split + 1 :]) > 1: + raise _Unsupported("root refs or reference options") + prefix = os.fsencode(args[-1]) if len(args) > split + 1 else b"" + if any(char in prefix for char in b"*?["): + raise _Unsupported("reference glob patterns") + with repo.references().all() as refs: + names = [] + for ref in refs: + names.append(ref.name()) + if ref.target().try_name() is not None: + # Git omits dangling symbolic refs; defer their diagnostics to it. + ref.follow_to_object() + return b"".join( + name + b"\n" + for name in sorted(names) + if not prefix or name == prefix or name.startswith(prefix.rstrip(b"/") + b"/") + ) + + +def submodule_config(source: Any) -> Any: + """Read the fields used by submodule objects from their selected config source.""" + if gix is None: + return NotImplemented + try: + if isinstance(source, io.BytesIO): + data = source.getvalue() + else: + with open(source, "rb") as stream: + data = stream.read() + # ponytail: no bound section enumeration; conservatively reject other '[' + # occurrences and duplicate sections until GIX-12 can preserve their order. + if b"\0" in data or re.search(rb"\[(?!submodule[ \t])", data, re.I): + raise _Unsupported("non-submodule configuration sections (GIX-12)") + modules = gix.ModulesFile.from_bytes(data) + with modules.names() as cursor: + names = list(cursor) + if data.count(b"[") != len(names): + raise _Unsupported("duplicate or ambiguous configuration sections (GIX-12)") + config = modules.config() + result = {} + for raw_name in names: + name = raw_name.decode(defenc) + values = {} + for option in ("path", "url", "branch"): + key = "submodule." + name + "." + option + value = config.string(key) + if value is None: + if option != "branch" or config.boolean(key) is not None: + raise _Unsupported("missing or implicit submodule fields") + else: + values[option] = value.decode(defenc) + result[name] = values + except _Unsupported as exc: + return _fallback("Submodule.config", str(exc)) + except (gix.Error, OSError, UnicodeError) as exc: + _logger.debug("native submodule configuration: %s", exc) + return _fallback("Submodule.config", "native parse diagnostics") + record("Submodule.config", "native") + return result + + +def _config(repo: Any, args: List[str], kwargs: Dict[str, Any]) -> bytes: + if len(args) == 2 and args[0] == "--get": + snapshot = repo.config_snapshot() + value = snapshot.string(args[1]) + if value is None: + if snapshot.boolean(args[1]) is None: + raise GitCommandError(["git", "config"], 1) + value = b"" # An implicit boolean is an empty value in --get output. + return value + b"\n" + raise _Unsupported("config file parsing, enumeration, or mutation (GIX-12)") + + +def _worktree(repo: Any, args: List[str], kwargs: Dict[str, Any]) -> bytes: + from git.util import to_native_path_linux + + if args != ["list", "--porcelain", "-z"] or not hasattr(repo, "worktrees"): + raise _Unsupported("worktree operation or feature disabled") + proxies = repo.worktrees() + if any(proxy.is_prunable() for proxy in proxies): + raise _Unsupported("prunable worktree diagnostics") + main = _canonical_repository(repo.main_repo()) + + def describe(local: Any) -> List[bytes]: + workdir = local.workdir() + fields = [b"worktree " + os.fsencode(to_native_path_linux(os.fspath(workdir or local.git_dir())))] + # Gix's is_bare() reflects configuration, even for a linked worktree. + if local.is_bare() and workdir is None: + fields.append(b"bare") + else: + head = local.head() + oid = head.id() + fields.append( + b"HEAD " + (str(oid).encode("ascii") if oid is not None else b"0" * local.object_hash().len_in_hex()) + ) + fields.append(b"detached" if head.is_detached() else b"branch " + head.referent_name()) + return fields + + records = [b"\0".join(describe(main)) + b"\0\0"] + for proxy in sorted(proxies, key=lambda proxy: os.fsencode(proxy.base())): + fields = describe(proxy.into_repo()) + if proxy.is_locked(): + reason = proxy.lock_reason() + fields.append(b"locked" + (b" " + reason if reason else b"")) + records.append(b"\0".join(fields) + b"\0\0") + return b"".join(records) + + +def _worktree_root(command: Any, repo: Any) -> str: + root = repo.workdir() + if ( + root is None + or repo.prefix() is not None + or os.path.realpath(command.working_dir or os.getcwd()) != os.path.realpath(root) + ): + raise _Unsupported("worktree or command working directory") + return os.fspath(root) + + +def is_dirty(command: Any, index: bool, working_tree: bool, untracked: bool, submodules: bool, path: Any) -> Any: + if gix is None: + return NotImplemented + method = "Repo.is_dirty" + try: + repo = _repository(command, {}) + _worktree_root(command, repo) + if not hasattr(repo, "status"): + raise _Unsupported("status feature disabled") + patterns = [os.fspath(path)] if path else [] + if any("\0" in pattern for pattern in patterns): + raise _Unsupported("invalid pathspec") + status = ( + repo.status() + .index(_index(repo, {"env": command.environment()})) + .untracked_files("files" if untracked else "none") + .tree_index_track_renames(None) + .index_worktree_rewrites(None) + .index_worktree_submodules("configured" if submodules else "all", check_dirty=submodules) + ) + dirty = False + with status.into_iter(patterns) as entries: + for item in entries: + if item.summary() is None: + continue + if item.kind == "TreeIndex": + if index and (submodules or item.details["entry_mode"] != 0o160000): + dirty = True + break + elif item.details["kind"] == "DirectoryContents": + if untracked and item.details["entry"]["status"] == "Untracked": + dirty = True + break + elif working_tree: + dirty = True + break + except _Unsupported as exc: + return _fallback(method, str(exc)) + except gix.Error as exc: + _logger.debug("is_dirty native read: %s", exc) + return _fallback(method, "native read diagnostics") + record(method, "native") + return dirty + + +def untracked_files(command: Any, args: Tuple[Any, ...], options: Dict[str, Any]) -> Any: + if gix is None: + return NotImplemented + method = "Repo.untracked_files" + try: + if options.keys() - {"ignore_submodules"}: + raise _Unsupported("status options") + repo = _repository(command, {}) + _worktree_root(command, repo) + if not hasattr(repo, "dirwalk_iter"): + raise _Unsupported("dirwalk feature disabled") + index = _index(repo, {"env": command.environment()}) + patterns = command._unpack_args([arg for arg in args if arg is not None]) + if any("\0" in path for path in patterns): + raise _Unsupported("invalid pathspec") + walk_options = repo.dirwalk_options().emit_untracked("matching").emit_tracked(False) + result = [] + with repo.dirwalk_iter(index, patterns, walk_options) as entries: + for item in entries: + entry = item.entry + if entry.status == "Untracked": + path = entry.rela_path + if entry.disk_kind in ("Directory", "Repository"): + path += b"/" + result.append(path) + result.sort() + except _Unsupported as exc: + return _fallback(method, str(exc)) + except gix.Error as exc: + _logger.debug("untracked_files native read: %s", exc) + return _fallback(method, "native read diagnostics") + record(method, "native") + return [safe_decode(path) for path in result] + + +def ignored(command: Any, paths: Sequence[Any]) -> Any: + if gix is None: + return NotImplemented + method = "Repo.ignored" + try: + repo = _repository(command, {}) + root = _worktree_root(command, repo) + if not hasattr(repo, "excludes"): + raise _Unsupported("excludes feature disabled") + index = _index(repo, {"env": command.environment()}) + excludes = repo.excludes(index) + with index.entries() as entries: + tracked = {entry.path() for entry in entries} + result = [] + for path in paths: + path = os.fspath(path) + relative = os.path.relpath(path, root) if os.path.isabs(path) else path + relative = relative.replace(os.sep, "/") + if any(part in (".", "..", ".git") for part in relative.split("/")) or relative.startswith(":"): + raise _Unsupported("ignore path normalization") + # check-ignore rejects traversal through symlinks and tracked gitlinks. + parent = relative.rstrip("/") + while "/" in parent: + parent = parent.rsplit("/", 1)[0] + if os.path.islink(os.path.join(root, parent)) or os.fsencode(parent) in tracked: + raise _Unsupported("ignore path traverses symlink or submodule") + if os.fsencode(relative.rstrip("/")) in tracked: + continue + mode: Optional[int] + try: + mode = os.lstat(os.path.join(root, relative)).st_mode + except FileNotFoundError: + mode = stat.S_IFDIR if relative.endswith("/") else None + else: + mode = 0o40000 if stat.S_ISDIR(mode) else 0o120000 if stat.S_ISLNK(mode) else 0o100644 + if excludes.at_entry(os.fsencode(relative), mode).is_excluded(): + result.append(path) + except _Unsupported as exc: + return _fallback(method, str(exc)) + except gix.Error as exc: + _logger.debug("ignored native read: %s", exc) + return _fallback(method, "native read diagnostics") + record(method, "native") + return result + + +def _merge_base(repo: Any, args: List[str], kwargs: Dict[str, Any]) -> bytes: + if not hasattr(repo, "merge_base"): + raise _Unsupported("revision feature disabled") + ancestor = args[:1] == ["--is-ancestor"] + if ancestor: + args = args[1:] + if len(args) != 3 or args[0] != "--": + raise _Unsupported("merge-base options") + one, two = (_oid(repo, value) for value in args[1:]) + if kwargs.get("all"): + ids = repo.merge_bases_many(one, [two]) + else: + base = repo.merge_base(one, two) + ids = [base] if base is not None else [] + if not ids or (ancestor and ids[0] != one): + raise GitCommandError(["git", "merge-base"], 1) + return b"" if ancestor else b"".join(str(oid).encode("ascii") + b"\n" for oid in ids) + + +def _walk(repo: Any, rev: str, options: Dict[str, Any]) -> Iterator[str]: + if not hasattr(repo, "rev_walk"): + raise _Unsupported("revision feature disabled") + if options.keys() - {"max_count", "skip", "first_parent"}: + raise _Unsupported("history options or ordering (GIX-8)") + if options.get("first_parent") not in (None, True, False): + raise _Unsupported("history options") + start, count = options.get("skip", 0), options.get("max_count") + if type(start) is not int or start < 0 or (count is not None and (type(count) is not int or count < 0)): + raise _Unsupported("history limits") + tip = repo.find_object(_oid(repo, rev)).peel_to_commit().id + platform = repo.rev_walk([tip]) + if options.get("first_parent"): + platform = platform.first_parent_only() + cursor = platform.all() + + def iterate() -> Iterator[str]: + try: + with cursor: + for item in islice(cursor, start, None if count is None else start + count): + yield str(item.id) + except gix.Error as exc: + raise GitCommandError(["gix", "rev-list", rev], 128, str(exc)) from exc + + return iterate() + + +def history(command: Any, rev: str, paths: Any, options: Dict[str, Any], *, count: bool = False) -> Any: + """Count reachable commits, or lazily walk the first-parent chain.""" + if gix is None: + return NotImplemented + method = "Commit.count" if count else "Commit.iter_items" + try: + if paths: + raise _Unsupported("history path filtering") + if ( + not count + and not options.get("first_parent") + and not (options.get("max_count") == 1 and not options.get("skip")) + ): + raise _Unsupported("Git history ordering (GIX-8)") + iterator = _walk(_repository(command, {}), rev, options) + result = sum(1 for _ in iterator) if count else iterator + except _Unsupported as exc: + return _fallback(method, str(exc)) + except gix.Error as exc: + _logger.debug("%s native preparation: %s", method, exc) + return _fallback(method, "native preparation diagnostics") + record(method, "native") + return result + + +def _date(signature: Any) -> bytes: + offset = signature.time.offset + hours, minutes = divmod(abs(offset) // 60, 60) + return b"%d %s%02d%02d" % (signature.time.seconds, b"-" if offset < 0 else b"+", hours, minutes) + + +def _reflog(repo: Any, args: List[str], kwargs: Dict[str, Any]) -> bytes: + if len(args) == 3 and args[:2] == ["exists", "--"]: + reference = repo.try_find_reference(args[-1]) + if reference is None: + raise _Unsupported("orphan reflog lookup") + if not reference.log_exists(): + raise GitCommandError(["git", "reflog", "exists"], 1) + return b"" + if ( + len(args) != 10 + or args[:8] + != [ + "show", + "--format=%H%x00%gn%x00%ge%x00%gD%x00%gs", + "--date=raw", + "-z", + "--no-abbrev", + "--no-decorate", + "--no-notes", + "--no-color", + ] + or args[-1] != "--" + ): + raise _Unsupported("reflog writing or format") + reference = repo.try_find_reference(args[-2]) + if reference is None: + raise _Unsupported("orphan reflog lookup") + cursor = reference.log_iter().rev() + if cursor is None: + return b"" + output = [] + with cursor: + for line in cursor: + if not repo.has_object(line.new_oid) or repo.find_header(line.new_oid).kind() != "commit": + continue + signature = line.signature + output.append( + b"\0".join( + [ + str(line.new_oid).encode("ascii"), + signature.name, + signature.email, + os.fsencode(args[-2]) + b"@{" + _date(signature) + b"}", + line.message, + ] + ) + + b"\0" + ) + return b"".join(output) + + +def _index(repo: Any, kwargs: Dict[str, Any]) -> Any: + if not hasattr(repo, "index_or_empty"): + raise _Unsupported("index feature disabled") + path = kwargs.get("env", {}).get("GIT_INDEX_FILE", os.environ.get("GIT_INDEX_FILE")) + if path and not os.path.isabs(path): + raise _Unsupported("relative index path") + if path and os.path.abspath(path) != os.path.abspath(repo.index_path()): + raise _Unsupported("custom index path (GIX-3)") + index = repo.index_or_empty() + if index.is_sparse(): + raise _Unsupported("sparse index (GIX-4)") + return index + + +def _ls_files(repo: Any, args: List[str], kwargs: Dict[str, Any]) -> bytes: + if args != ["--stage", "-v", "-z", "--full-name"]: + raise _Unsupported("index listing options") + with _index(repo, kwargs).entries() as entries: + output = [] + for entry in entries: + flag = b"S" if entry.flags & (1 << 30) else b"M" if entry.stage() else b"H" + if entry.flags & (1 << 15): + flag = flag.lower() + output.append( + b"%s %06o %s %d\t%s\0" % (flag, entry.mode, str(entry.id).encode("ascii"), entry.stage(), entry.path()) + ) + return b"".join(output) + + +def _update_index(repo: Any, args: List[str], kwargs: Dict[str, Any]) -> bytes: + if args != ["--show-index-version"]: + raise _Unsupported("index mutation options") + return str(_index(repo, kwargs).version()).encode("ascii") + b"\n" + + +def _index_from_tree(repo: Any, tree: Any) -> Any: + if not hasattr(repo, "index_from_tree"): + raise _Unsupported("index feature disabled") + if ( + repo.config_snapshot().integer("index.version") not in (None, 2) + or os.environ.get("GIT_INDEX_VERSION", "2") != "2" + ): + raise _Unsupported("new index version selection (GIX-13)") + return repo.index_from_tree(tree) + + +def materialize_index(command: Any, source: Any, destination: str, desired: Dict[Any, Any], dirty: Any) -> Any: + """Write a private index, retaining native metadata for unchanged entries.""" + if gix is None: + return NotImplemented + method = "IndexFile.write" + try: + if any(stage for _path, stage in desired): + raise _Unsupported("unmerged index editing") + if any(not entry.binsha.strip(b"\0") for entry in desired.values()): + raise _Unsupported("null index object IDs") + names = {os.fsencode(path) for path, _stage in desired} + for path in names: + while b"/" in path: + path = path.rsplit(b"/", 1)[0] + if path in names: + raise _Unsupported("overlapping index paths") + repo = _repository(command, {}) + if os.path.exists(source): + if repo.config_snapshot().boolean("core.splitIndex") or glob( + os.path.join(os.path.dirname(os.fspath(source)), "sharedindex.*") + ): + raise _Unsupported("split index preservation (GIX-13)") + index = _index(repo, {"env": {"GIT_INDEX_FILE": os.fspath(source)}}) + if index.version() != 2: + raise _Unsupported("index version preservation (GIX-13)") + else: + index = _index_from_tree(repo, repo.empty_tree()) + # Keep path-independent flags that GitPython exposes; changed paths get fresh stat data. + mask = (3 << 12) | (1 << 15) | (1 << 30) + changed = {os.fsencode(path) for path in dirty} + with index.entries() as entries: + current = list(entries) + for entry in current: + wanted = desired.get((safe_decode(entry.path()), entry.stage())) + if wanted is None or (entry.mode, str(entry.id), entry.flags & mask) != ( + wanted.mode, + wanted.hexsha, + wanted.flags & mask, + ): + changed.add(entry.path()) + changed.update(names - {entry.path() for entry in current}) + # ponytail: per-entry removals shift a vector; use a bulk binding for very large edits. + for position, entry in reversed(list(enumerate(current))): + if entry.path() in changed: + index.remove_entry_at_index(position) + for (path, _stage), entry in desired.items(): + if os.fsencode(path) in changed: + index.dangerously_push_entry( + gix.IndexStat(), gix.ObjectId(entry.hexsha), entry.flags & mask, entry.mode, os.fsencode(path) + ) + index.sort_entries() + index.verify_entries() + index.set_path(destination) + _write("index", index.write) + except _Unsupported as exc: + return _fallback(method, str(exc)) + except gix.Error as exc: + _logger.debug("materialize_index native preparation: %s", exc) + return _fallback(method, "native preparation diagnostics") + record(method, "native") + return None + + +def _write(method: str, function: Callable[[], Any]) -> Any: + """Once a write starts, an error must not cause a second attempt through Git.""" + try: + return function() + except gix.Error as exc: + raise GitCommandError(["gix", method], 128, str(exc)) from exc + + +def _input(kwargs: Dict[str, Any]) -> bytes: + stream = kwargs.get("istream") + if stream is None or not hasattr(stream, "seekable") or not stream.seekable(): + raise _Unsupported("nonseekable input") + start = stream.tell() + try: + stream.seek(0, 2) + if stream.tell() - start > 8 * 1024 * 1024: + raise _Unsupported("large object streaming (GIX-2)") + stream.seek(start) + data = stream.read() + finally: + stream.seek(start) + if not isinstance(data, bytes): + raise _Unsupported("nonbinary input") + return data + + +def _hash_object(repo: Any, args: List[str], kwargs: Dict[str, Any]) -> bytes: + if len(args) not in (3, 4) or args[0] != "-t" or args[-1] != "--stdin": + raise _Unsupported("object hashing options") + write = args[2:-1] == ["-w"] + if args[2:-1] not in ([], ["-w"]): + raise _Unsupported("object hashing options") + kind = args[1] + if kind not in ("blob", "tree", "commit", "tag"): + raise _Unsupported("object kind") + data = _input(kwargs) + # Validate and check the encoded bytes in memory before touching object storage. + memory = repo.with_object_memory() + oid = memory.write_object(kind, data) + if memory.find_object(oid).data != data: + raise _Unsupported("object serialization changes bytes (GIX-5)") + if write: + oid = _write("hash_object", lambda: repo.write_object(kind, data)) + kwargs["istream"].seek(len(data), 1) + return str(oid).encode("ascii") + b"\n" + + +_ENTRY_KINDS = {0o100644: "blob", 0o100755: "exe", 0o120000: "link", 0o160000: "commit", 0o40000: "tree"} + + +def _write_tree(repo: Any, entries: Sequence[Tuple[bytes, int, str]]) -> str: + names = {name for name, _mode, _oid in entries} + if len(names) != len(entries): + raise _Unsupported("duplicate or overlapping tree paths") + for name in names: + while b"/" in name: + name = name.rsplit(b"/", 1)[0] + if name in names: + raise _Unsupported("duplicate or overlapping tree paths") + for _path, mode, oid in entries: + object_id = gix.ObjectId(oid) + if object_id.is_null(): + raise _Unsupported("null tree object IDs") + header = repo.try_find_header(object_id) + if header is None: + if mode != 0o160000: + raise _Unsupported("missing tree children (GIX-7)") + elif header.kind() != ("tree" if mode == 0o40000 else "commit" if mode == 0o160000 else "blob"): + raise _Unsupported("tree child object kind") + with repo.empty_tree().edit() as editor: + for path, mode, oid in entries: + editor.upsert(path, _ENTRY_KINDS[mode], gix.ObjectId(oid)) + return str(_write("write_tree", editor.write)) + + +def write_tree(command: Any, entries: Sequence[Tuple[bytes, int, str]]) -> Any: + if gix is None: + return NotImplemented + try: + result = _write_tree(_repository(command, {}), entries) + except _Unsupported as exc: + return _fallback("IndexFile.write_tree", str(exc)) + except gix.Error as exc: + _logger.debug("write_tree native preparation: %s", exc) + return _fallback("IndexFile.write_tree", "native preparation diagnostics") + record("IndexFile.write_tree", "native") + return result + + +def tree_diff(repository: Any, left: Any, right: Any, paths: Any, patch: bool, options: Dict[str, Any]) -> Any: + """Build GitPython's raw tree diff directly from native change records.""" + if gix is None: + return NotImplemented + from git.diff import Diff, DiffIndex, Lit_change_type + + method = "Diffable.diff" + try: + if not hasattr(left, "hexsha") or not (hasattr(right, "hexsha") or isinstance(right, str)): + raise _Unsupported("index/worktree/root diff") + if patch or paths: + raise _Unsupported("patch formatting or path filtering") + if options.keys() - {"R", "no_renames"} or any(type(value) is not bool for value in options.values()): + raise _Unsupported("diff options") + if "no_renames" in options and not options["no_renames"]: + raise _Unsupported("configured rename detection") + repo = _repository(repository.git, {}) + if not hasattr(repo, "diff_tree_to_tree"): + raise _Unsupported("tree-diff feature disabled") + before = repo.find_object(_oid(repo, left.hexsha)).peel_to_tree() + after = repo.find_object(_oid(repo, getattr(right, "hexsha", right))).peel_to_tree() + if options.get("R"): + before, after = after, before + diff_options = gix.DiffOptions().track_path().track_rewrites(None) + changes = repo.diff_tree_to_tree(before, after, diff_options) + if not options.get("no_renames"): + added = [str(c.id()) for c in changes if c.kind == "Addition" and c.entry_mode() != 0o40000] + deleted = [str(c.id()) for c in changes if c.kind == "Deletion" and c.entry_mode() != 0o40000] + if added and deleted: + if (set(added) - set(deleted) and set(deleted) - set(added)) or ( + len(set(added)) != len(added) or len(set(deleted)) != len(deleted) + ): + raise _Unsupported("inexact or ambiguous rename detection (GIX-11)") + # Exact rewrites need no blob filters, textconv, or external diff drivers. + diff_options = diff_options.track_rewrites(gix.Rewrites(percentage=None)) + changes = repo.diff_tree_to_tree(before, after, diff_options) + result: DiffIndex[Diff] = DiffIndex() + for change in changes: + details = change.details + kind, path, mode, oid = change.kind, change.location(), change.entry_mode(), str(change.id()) + previous_mode = details.get("previous_entry_mode", details.get("source_entry_mode", mode)) + previous_oid = str(details.get("previous_id", details.get("source_id", change.id()))) + if mode == 0o40000 or previous_mode == 0o40000: + if mode != previous_mode: + raise _Unsupported("directory type change") + continue + new_file, deleted_file, renamed = kind == "Addition", kind == "Deletion", kind == "Rewrite" + change_type = ( + "A" + if new_file + else "D" + if deleted_file + else "R" + if renamed + else ("T" if mode & 0o170000 != previous_mode & 0o170000 else "M") + ) + source = change.source_location() if renamed else path + result.append( + Diff( + repository, + source, + path, + None if new_file else previous_oid, + None if deleted_file else oid, + "%06o" % (0 if new_file else previous_mode), + "%06o" % (0 if deleted_file else mode), + new_file, + deleted_file, + False, + source if renamed else None, + path if renamed else None, + "", + cast(Lit_change_type, change_type), + 100 if renamed else None, + ) + ) + result.sort(key=lambda diff: diff.b_rawpath or b"") + except _Unsupported as exc: + return _fallback(method, str(exc)) + except gix.Error as exc: + _logger.debug("tree_diff native read: %s", exc) + return _fallback(method, "native read diagnostics") + record(method, "native") + return result + + +def commit_stats(commit: Any) -> Any: + """Count changed lines natively, retaining Git's statistics representation.""" + if gix is None: + return NotImplemented + from git.util import Stats + + method = "Commit.stats" + try: + repo = _repository(commit.repo.git, {}) + if not hasattr(repo, "diff_tree_to_tree") or not hasattr(repo, "attributes_only"): + raise _Unsupported("diff or attributes feature disabled") + if repo.config_snapshot().string("diff.algorithm") not in (None, b"myers", b"default"): + raise _Unsupported("statistics diff algorithm") + native = repo.find_commit(gix.ObjectId(commit.hexsha)) + with native.parent_ids() as parents: + parent = next(parents, None) + before = repo.empty_tree() if parent is None else repo.find_commit(parent).tree() + changes = repo.diff_tree_to_tree(before, native.tree(), gix.DiffOptions().track_path().track_rewrites(None)) + index = _index(repo, {"env": commit.repo.git.environment()}) + # The blob cache reads index attributes, whereas Git also reads worktree attributes. + attributes = [repo.attributes_only(index, source) for source in ("id_mapping", "worktree_then_id_mapping")] + cache = repo.diff_resource_cache_for_tree_diff() + lines = [] + for change in sorted(changes, key=lambda change: change.location()): + mode = change.entry_mode() + previous_mode = change.details.get("previous_entry_mode", mode) + if mode == 0o40000 or previous_mode == 0o40000: + if mode != previous_mode: + raise _Unsupported("statistics directory type change") + continue + if mode == 0o160000 or previous_mode == 0o160000: + raise _Unsupported("statistics gitlinks") + path = change.location() + if any(byte < 32 or byte >= 127 or byte in b'"\\' for byte in path): + raise _Unsupported("statistics filename quoting") + for stack in attributes: + outcome = stack.selected_attribute_matches(["diff"]) + stack.at_entry(path, mode).matching_attributes(outcome) + with outcome.iter_selected() as matches: + if any(match.state != "unspecified" for match in matches): + raise _Unsupported("statistics diff attributes (GIX-18)") + for oid in (change.id(), change.details.get("previous_id")): + if oid is not None and repo.find_header(oid).size() > 8 * 1024 * 1024: + raise _Unsupported("large object streaming (GIX-2)") + counts = change.diff(cache).line_counts() + cache.clear_resource_cache() + kind = {"Addition": "A", "Deletion": "D", "Modification": "M"}[change.kind] + if mode & 0o170000 != previous_mode & 0o170000: + kind = "T" + lines.append( + "%s\t%d\t%d\t%s\n" + % (kind, counts.insertions if counts else 0, counts.removals if counts else 0, safe_decode(path)) + ) + result = Stats._list_from_string(commit.repo, "".join(lines)) + except _Unsupported as exc: + return _fallback(method, str(exc)) + except gix.Error as exc: + _logger.debug("commit_stats native read: %s", exc) + return _fallback(method, "native read diagnostics") + record(method, "native") + return result + + +def _mktree(repo: Any, args: List[str], kwargs: Dict[str, Any]) -> bytes: + if args != ["-z", "--missing"]: + raise _Unsupported("tree writing options") + data = _input(kwargs) + entries = [] + for record in data.split(b"\0"): + if not record: + continue + metadata, path = record.split(b"\t", 1) + mode_bytes, kind, oid_bytes = metadata.split() + mode = int(mode_bytes, 8) + if mode not in _ENTRY_KINDS or b"/" in path or path in (b"", b".", b"..", b".git"): + raise _Unsupported("tree entry") + expected = b"tree" if mode == 0o40000 else b"commit" if mode == 0o160000 else b"blob" + if kind != expected: + raise _Unsupported("tree entry kind") + entries.append((path, mode, oid_bytes.decode("ascii"))) + oid = _write_tree(repo, entries) + kwargs["istream"].seek(len(data), 1) + return oid.encode("ascii") + b"\n" + + +def _read_tree(repo: Any, args: List[str], kwargs: Dict[str, Any]) -> bytes: + if len(args) != 1 or (args[0] != "--empty" and args[0].startswith("-")): + raise _Unsupported("index merge options") + path = kwargs.get("env", {}).get("GIT_INDEX_FILE") + if not path or os.path.exists(path): + raise _Unsupported("existing index metadata") + tree = repo.empty_tree() if args == ["--empty"] else repo.find_object(_oid(repo, args[0])).peel_to_tree() + index = _index_from_tree(repo, tree) + index.set_path(path) + _write("read_tree", index.write) + return b"" + + +def _checked_signature(signature: Any) -> Any: + # Git strips "crud" at the edges and angle brackets/newlines within identities. + crud = bytes(range(33)) + b".,:;<>\"'\\" + for value in (signature.name, signature.email): + if not value or value != value.strip(crud) or any(char in value for char in b"<>\r\n\0"): + raise _Unsupported("identity normalization") + return signature + + +def _signature(env: Dict[str, Any], role: str) -> Any: + prefix = "GIT_" + role + "_" + name, email, date = (env.get(prefix + field) for field in ("NAME", "EMAIL", "DATE")) + match = re.fullmatch(r"(-?\d+) ([+-])(\d\d)(\d\d)", date or "") + if not name or not email or match is None: + raise _Unsupported("identity resolution") + seconds, sign, hours, minutes = match.groups() + offset = (int(hours) * 60 + int(minutes)) * 60 * (-1 if sign == "-" else 1) + return _checked_signature(gix.Signature(name, email, int(seconds), offset)) + + +def _committer(repo: Any, env: Dict[str, Any]) -> Any: + for field in ("NAME", "EMAIL", "DATE"): + key = "GIT_COMMITTER_" + field + if key in env and env[key] != os.environ.get(key): + raise _Unsupported("per-command committer identity (GIX-21)") + if os.environ.get(key) == "": + raise _Unsupported("empty identity") + signature = repo.committer() + if signature is None: + raise _Unsupported("identity resolution") + return _checked_signature(signature) + + +def _var(repo: Any, args: List[str], kwargs: Dict[str, Any]) -> bytes: + if args != ["GIT_COMMITTER_IDENT"]: + raise _Unsupported("Git variable") + signature = _committer(repo, kwargs.get("env", {})) + return signature.name + b" <" + signature.email + b"> " + _date(signature) + b"\n" + + +def _update_ref(repo: Any, args: List[str], kwargs: Dict[str, Any]) -> bytes: + if "--" not in args: + raise _Unsupported("reference transactions (GIX-9)") + split = args.index("--") + options, operands = args[:split], args[split + 1 :] + deref, force_log, delete, message = True, False, False, "" + cursor = iter(options) + for option in cursor: + if option == "--no-deref": + deref = False + elif option == "--create-reflog": + force_log = True + elif option == "-d": + delete = True + elif option == "-m": + message = next(cursor, "") + else: + raise _Unsupported("reference options") + # LogChange sets each edit's message, but its writer does not normalize it. + if re.search(r"[\t\r\n\v\f]|^ | $| {2}", message): + raise _Unsupported("reflog message cleanup (GIX-10)") + if delete: + if len(operands) != 1 or deref: + raise _Unsupported("reference deletion options") + edit = gix.RefEdit.delete(operands[0], gix.PreviousValue.Any).with_deref(False) + _write("update_ref", lambda: repo.edit_references_as([edit])) + return b"" + if len(operands) != 2: + raise _Unsupported("strict reference creation / compare-and-swap (GIX-9)") + name, target = operands + oid = _oid(repo, target) + kind = repo.find_header(oid).kind() + if (name == "HEAD" or name.startswith("refs/heads/")) and kind != "commit": + raise _Unsupported("branch target validation") + reference = repo.try_find_reference(name) + if reference is not None: + if deref and name != "HEAD" and reference.target().try_name() is not None: + raise _Unsupported("symbolic reference update (GIX-10)") + if not deref and reference.target().try_name() is not None: + raise _Unsupported("detaching symbolic ref reflog (GIX-10)") + if reference.follow_to_object() == oid: + raise _Unsupported("unchanged reference reflog (GIX-10)") + head = repo.try_find_reference("HEAD") + for _ in range(5): + if head is None or head.target().try_name() is None: + break + if head.target().try_name() == os.fsencode(name): + raise _Unsupported("active branch HEAD reflog (GIX-10)") + head = head.follow() + log = gix.LogChange() + log.force_create_reflog = force_log + log.message = message + edit = gix.RefEdit.update_with_log(name, gix.Target.Object(oid), gix.PreviousValue.Any, log).with_deref(deref) + signature = _committer(repo, kwargs.get("env", {})) + _write("update_ref", lambda: repo.edit_references_as([edit], signature)) + return b"" + + +def _commit_tree(repo: Any, args: List[str], kwargs: Dict[str, Any]) -> bytes: + if len(args) < 2 or args[0] != "--no-gpg-sign" or args[2::2] != ["-p"] * len(args[2::2]): + raise _Unsupported("commit options") + if len(args) % 2 or any(setting.lower() != "i18n.commitencoding=utf-8" for setting in kwargs.get("_config", ())): + raise _Unsupported("commit encoding (GIX-6)") + data = _input(kwargs) + try: + message = data.decode("utf-8") + except UnicodeDecodeError: + raise _Unsupported("commit message bytes (GIX-6)") from None + parents = args[3::2] + if len(set(parents)) != len(parents): + raise _Unsupported("duplicate parents") + author = _signature(kwargs.get("env", {}), "AUTHOR") + committer = _signature(kwargs.get("env", {}), "COMMITTER") + tree = _oid(repo, args[1]) + # Native commit creation accepts literal IDs, whereas commit-tree verifies kinds. + if repo.find_header(tree).kind() != "tree" or any( + repo.find_header(_oid(repo, p)).kind() != "commit" for p in parents + ): + raise _Unsupported("commit object kinds") + commit = _write("commit_tree", lambda: repo.new_commit_as(committer, author, message, tree, parents)) + kwargs["istream"].seek(len(data), 1) + return str(commit.id).encode("ascii") + b"\n" + + +_HANDLERS: Dict[str, Callable[[Any, List[str], Dict[str, Any]], bytes]] = { + "rev_parse": _rev_parse, + "ls_tree": _ls_tree, + "symbolic_ref": _symbolic_ref, + "for_each_ref": _for_each_ref, + "merge_base": _merge_base, + "ls_files": _ls_files, + "update_index": _update_index, + "hash_object": _hash_object, + "mktree": _mktree, + "read_tree": _read_tree, + "commit_tree": _commit_tree, + "reflog": _reflog, + "var": _var, + "update_ref": _update_ref, + "config": _config, + "worktree": _worktree, +} + + +def dispatch( + command: Any, method: str, args: Tuple[Any, ...], kwargs: Dict[str, Any], config: Sequence[str] = () +) -> Any: + if gix is None: + return NotImplemented + handler = _HANDLERS.get(method) + if handler is None: + return _fallback(method, "not converted") + allowed = {"env", "stdout_as_string", "strip_newline_in_stdout", "with_extended_output"} + if method == "merge_base": + allowed.add("all") + if method in ("hash_object", "mktree", "commit_tree"): + allowed.add("istream") + if kwargs.keys() - allowed: + return _fallback(method, "command/process options") + if config and method != "commit_tree": + return _fallback(method, "configuration overrides") + if config: + kwargs = dict(kwargs, _config=config) + kwargs = dict(kwargs, env={**command.environment(), **(kwargs.get("env") or {})}) + try: + args_list = command._unpack_args([arg for arg in args if arg is not None]) + repo = _repository(command, kwargs.get("env", {}), query_config=method == "config") + output = handler(repo, args_list, kwargs) + except _Unsupported as exc: + return _fallback(method, str(exc)) + except GitCommandError: + record(method, "native") + raise + except gix.Error as exc: + _logger.debug("%s native read: %s", method, exc) + return _fallback(method, "native read diagnostics") + record(method, "native") + if kwargs.get("strip_newline_in_stdout", True) and output.endswith(b"\n"): + output = output[:-1] + result = safe_decode(output) if kwargs.get("stdout_as_string", True) else output + return (0, result, "") if kwargs.get("with_extended_output") else result diff --git a/git/cmd.py b/git/cmd.py index cb88e7c24..eab47da5c 100644 --- a/git/cmd.py +++ b/git/cmd.py @@ -13,6 +13,7 @@ import logging import os import re +import shutil import signal import subprocess from subprocess import DEVNULL, PIPE, Popen @@ -22,12 +23,14 @@ import warnings from git.compat import defenc, force_bytes, safe_decode +from git import _backend from git.exc import ( CommandError, GitCommandError, GitCommandNotFound, UnsafeOptionError, UnsafeProtocolError, + UnsupportedOperation, ) from git.util import ( cygpath, @@ -636,6 +639,7 @@ class Git(metaclass=_GitMeta): "_git_options", "_persistent_git_options", "_environment", + "_repo", ) _excluded_ = ( @@ -643,8 +647,12 @@ class Git(metaclass=_GitMeta): "cat_file_header", "_version_info", "_version_info_token", + "_repo", ) + _version_info: Optional[Tuple[int, ...]] + _version_info_token: object + # Match Git's leading transport selector, including an empty helper name. # Git also selects the command-executing ext helper for an ext:// URL. re_unsafe_protocol = re.compile(r"([A-Za-z0-9][A-Za-z0-9+.-]*|)::|ext://") @@ -737,6 +745,7 @@ def __setstate__(self, d: Dict[str, Any]) -> None: """ _refresh_token = object() # Since None would match an initial _version_info_token. + _version_check_cache: Dict[Tuple[Any, ...], Tuple[int, ...]] = {} @classmethod def refresh(cls, path: Union[None, PathLike] = None) -> bool: @@ -1079,6 +1088,138 @@ def _option_candidates(cls, args: Sequence[Any] = (), kwargs: Optional[Mapping[s ) return options + @staticmethod + def _check_operand(value: Any, label: str = "operand") -> str: + """Validate a single name/revision, before Git can interpret it as an option. + + Paths and free-form payloads need their own validation and framing instead. + """ + value = safe_decode(value) if isinstance(value, bytes) else str(value) + if value.startswith("-") or any(char in value for char in "\0\r\n"): + raise UnsafeOptionError(f"Invalid {label}: {value!r}") + return value + + def _require_version(self) -> None: + key = None + if self._version_info_token is not self._refresh_token: + key = self._version_check_key() + if key is not None: + version = self._version_check_cache.get(key) + if version is not None: + self._version_info = version + self._version_info_token = key[0] + version = self.version_info + if version < (2, 52): + raise UnsupportedOperation("GitPython requires Git 2.52 or newer for repository operations") + if key is not None and key not in self._version_check_cache: + # ponytail: clear at 128 contexts; use LRU if diverse environments churn. + if len(self._version_check_cache) >= 128: + self._version_check_cache.clear() + self._version_check_cache[key] = version + + def _version_check_key(self) -> Optional[Tuple[Any, ...]]: + """Identify the executable and context of a minimum-version probe.""" + executable = self.GIT_PYTHON_GIT_EXECUTABLE + if not executable or self._git_options or self._persistent_git_options: + return None + environment = {**os.environ, "LANGUAGE": "C", "LC_ALL": "C", **self._environment} + try: + cwd = os.path.realpath(self._working_dir or os.getcwd()) + if not os.access(cwd, os.X_OK): + return None + resolved: Optional[str] + if os.path.isabs(executable): + resolved = executable + elif sys.platform == "win32": + # CreateProcess searches in the parent context, unlike execvpe. + return None + elif os.path.dirname(executable): + resolved = os.path.join(cwd, executable) + else: + path = environment.get("PATH") + path = os.defpath if path is None else path + resolved = shutil.which( + executable, path=os.pathsep.join(os.path.join(cwd, p) for p in path.split(os.pathsep)) + ) + if resolved is None: + return None + info = os.stat(resolved) + except OSError: + return None + return ( + self._refresh_token, + type(self), + executable, + resolved, + cwd, + tuple(sorted(environment.items())), + (info.st_dev, info.st_ino, info.st_size, info.st_mtime_ns, info.st_ctime_ns), + ) + + def _call_process_safe( + self, + method: str, + *args: Any, + _allow_hooks: bool = False, + _allow_network: bool = False, + _config: Sequence[str] = (), + **kwargs: Any, + ) -> Any: + """Run library-owned plumbing without introducing executable configuration. + + Callers validate their operands and any forwarded options using the existing + command-specific guards. This does not restrict the public raw Git interface. + """ + self._check_operand(method, "command") + if kwargs.get("shell"): + raise UnsafeOptionError("Library operations cannot run through a shell") + if _allow_hooks and method != "hook": + raise UnsafeOptionError("Only explicit hook operations may enable hooks") + insertion = kwargs.get("insert_kwargs_after") + if insertion is not None and not ( + method == "remote" and args and args[0] == insertion and insertion in ("add", "set-url", "update") + ): + raise UnsafeOptionError("Library command options cannot be reordered past safety flags") + for setting in _config: + key, separator, value = setting.partition("=") + if not separator or ( + key.lower() not in ("i18n.commitencoding", "diff.mnemonicprefix", "fetch.output", "core.abbrev") + and not (key == "protocol.file.allow" and value in ("always", "never", "user")) + and not (key in ("tar.tgz.command", "tar.tar.gz.command") and value == "git archive gzip") + ): + raise UnsafeOptionError(f"Unsupported internal Git configuration: {key!r}") + for arg in self._unpack_args([arg for arg in args if arg is not None]) + self.transform_kwargs( + **{key: value for key, value in kwargs.items() if key not in execute_kwargs} + ): + if "\0" in arg: + raise UnsafeOptionError("Git arguments cannot contain NUL bytes") + options = ["--no-pager", "--no-optional-locks"] + # These settings would become visible as user configuration in `config`. + if method != "config": + config = ["core.fsmonitor=false", "gc.auto=0", "maintenance.auto=false"] + if not _allow_hooks: + config.append(f"core.hooksPath={os.devnull}") + for setting in [*config, *_config]: + options.extend(("-c", setting)) + elif _config: + raise ValueError("Configuration queries must not include synthetic settings") + native = _backend.dispatch(self, method, args, kwargs, _config) + if native is not NotImplemented: + return native + self._require_version() + env = dict(kwargs.pop("env", {}) or {}) + env.update(LC_ALL="C", LANGUAGE="C") + if not _allow_network: + env.update(GIT_NO_LAZY_FETCH="1", GIT_TERMINAL_PROMPT="0") + return self._call_process( + method, + *args, + _safe_git_options=options, + shell=False, + env=env, + **{key: value for key, value in kwargs.items() if key != "shell"}, + ) + AutoInterrupt: TypeAlias = _AutoInterrupt CatFileContentStream: TypeAlias = _CatFileContentStream @@ -1098,11 +1239,12 @@ def __init__(self, working_dir: Union[None, PathLike] = None) -> None: self._persistent_git_options: List[str] = [] # Extra environment variables to pass to git commands - self._environment: Dict[str, str] = {} + self._environment: Dict[str, Optional[str]] = {} + self._repo: Any = None # Weak reference; the Repo owns native resources. # Cached version slots - self._version_info: Union[Tuple[int, ...], None] = None - self._version_info_token: object = None + self._version_info = None + self._version_info_token = None # Cached command slots self.cat_file_header: Union[None, TBD] = None @@ -1182,7 +1324,7 @@ def version_info(self) -> Tuple[int, ...]: return self._version_info # Run "git version" and parse it. - process_version = self._call_process("version") + process_version = cast(str, self._call_process("version", shell=False)) version_string = process_version.split(" ")[2] version_fields = version_string.split(".")[:4] leading_numeric_fields = itertools.takewhile(str.isdigit, version_fields) @@ -1466,16 +1608,18 @@ def execute( # Start the process. inline_env = env - env = os.environ.copy() + environment: Dict[str, Optional[str]] = dict(os.environ) # Attempt to force all output to plain ASCII English, which is what some parsing # code may expect. # According to https://askubuntu.com/a/311796, we are setting LANGUAGE as well # just to be sure. - env["LANGUAGE"] = "C" - env["LC_ALL"] = "C" - env.update(self._environment) + environment["LANGUAGE"] = "C" + environment["LC_ALL"] = "C" + environment.update(self._environment) if inline_env is not None: - env.update(inline_env) + environment.update(inline_env) + # Internal or per-call None overrides remove inherited variables. + env = {key: value for key, value in environment.items() if value is not None} if sys.platform == "win32": if kill_after_timeout is not None: @@ -1521,6 +1665,10 @@ def execute( except cmd_not_found_exception as err: raise GitCommandNotFound(redacted_command, err) from err else: + _backend.record("Git.execute", "CLI process") + owner = self._repo() if self._repo is not None else None + if owner is not None: + owner._gix_state = None # Replace with a typeguard for Popen[bytes]? proc.stdout = cast(BinaryIO, proc.stdout) proc.stderr = cast(BinaryIO, proc.stderr) @@ -1675,7 +1823,7 @@ def as_text(stdout_value: Union[bytes, str, None]) -> str: else: return stdout_value - def environment(self) -> Dict[str, str]: + def environment(self) -> Dict[str, Optional[str]]: return self._environment def update_environment(self, **kwargs: Any) -> Dict[str, Union[str, None]]: @@ -1759,7 +1907,7 @@ def _unpack_args(cls, arg_list: Sequence[Any]) -> List[str]: for arg in arg_list: outlist.extend(cls._unpack_args(arg)) else: - outlist.append(str(arg_list)) + outlist.append(os.fsdecode(arg_list) if isinstance(arg_list, os.PathLike) else str(arg_list)) return outlist @@ -1844,6 +1992,7 @@ def _call_process( """ # Handle optional arguments prior to calling transform_kwargs. # Otherwise these'll end up in args, which is bad. + safe_git_options = kwargs.pop("_safe_git_options", ()) exec_kwargs = {k: v for k, v in kwargs.items() if k in execute_kwargs} opts_kwargs = {k: v for k, v in kwargs.items() if k not in execute_kwargs} @@ -1877,12 +2026,14 @@ def _call_process( call.extend(self._git_options) self._git_options = () + call.extend(safe_git_options) + call.append(dashify(method)) call.extend(args_list) return self.execute(call, **exec_kwargs) - def _parse_object_header(self, header_line: str) -> Tuple[str, str, int]: + def _parse_object_header(self, header_line: Union[str, bytes]) -> Tuple[str, str, int]: """ :param header_line: A line of the form:: @@ -1895,6 +2046,8 @@ def _parse_object_header(self, header_line: str) -> Tuple[str, str, int]: :raise ValueError: If the header contains indication for an error due to incorrect input sha. """ + if isinstance(header_line, bytes): + header_line = header_line.decode("ascii", "replace") tokens = header_line.split() if len(tokens) != 3: if not tokens: @@ -1909,42 +2062,56 @@ def _parse_object_header(self, header_line: str) -> Tuple[str, str, int]: # END handle actual return value # END error handling - if len(tokens[0]) != 40: + if ( + not re.fullmatch(r"[0-9a-fA-F]+", tokens[0]) + or len(tokens[0]) % 2 + or tokens[1] not in ("blob", "tree", "commit", "tag") + or not tokens[2].isdigit() + ): raise ValueError("Failed to parse header: %r" % header_line) return (tokens[0], tokens[1], int(tokens[2])) def _prepare_ref(self, ref: object) -> bytes: - # Required for command to separate refs on stdin, as bytes. + # `cat-file -Z` separates both requests and responses with NUL, so paths + # containing newlines cannot inject requests or desynchronize the process. if isinstance(ref, bytes): - # Assume 40 bytes hexsha - bin-to-ascii for some reason returns bytes, not text. - refstr: str = ref.decode("ascii") + refstr: str = ref.decode(defenc, "surrogateescape") elif not isinstance(ref, str): refstr = str(ref) # Could be ref-object. else: refstr = ref - if not refstr.endswith("\n"): - refstr += "\n" - return refstr.encode(defenc) + if "\0" in refstr or refstr.startswith("-"): + raise UnsafeOptionError("Object queries cannot contain NUL or start with '-'") + return refstr.encode(defenc, "surrogateescape") + b"\0" def _get_persistent_cmd(self, attr_name: str, cmd_name: str, *args: Any, **kwargs: Any) -> "Git.AutoInterrupt": cur_val = getattr(self, attr_name) if cur_val is not None: return cur_val - options = {"istream": PIPE, "as_process": True} + options: Dict[str, Any] = {"istream": PIPE, "as_process": True} options.update(kwargs) - cmd = self._call_process(cmd_name, *args, **options) + cmd = self._call_process_safe(cmd_name, *args, **options) setattr(self, attr_name, cmd) cmd = cast("Git.AutoInterrupt", cmd) return cmd - def __get_object_header(self, cmd: "Git.AutoInterrupt", ref: Union[str, bytes]) -> Tuple[str, str, int]: + def __get_object_header(self, cmd: "Git.AutoInterrupt", request: bytes) -> Tuple[str, str, int]: if cmd.stdin and cmd.stdout: - cmd.stdin.write(self._prepare_ref(ref)) + cmd.stdin.write(request) cmd.stdin.flush() - return self._parse_object_header(cmd.stdout.readline()) + header = bytearray() + while True: + char = cmd.stdout.read(1) + if not char: + cmd.wait() + raise ValueError("Git closed the object stream before its response") + if char == b"\0": + break + header.extend(char) + return self._parse_object_header(bytes(header)) else: raise ValueError("cmd stdin was empty") @@ -1959,8 +2126,12 @@ def get_object_header(self, ref: Union[str, bytes]) -> Tuple[str, str, int]: :return: (hexsha, type_string, size_as_int) """ - cmd = self._get_persistent_cmd("cat_file_header", "cat_file", batch_check=True) - return self.__get_object_header(cmd, ref) + request = self._prepare_ref(ref) + native = _backend.object_data(self, request[:-1]) + if native is not NotImplemented: + return native + cmd = self._get_persistent_cmd("cat_file_header", "cat_file", batch_check=True, Z=True) + return self.__get_object_header(cmd, request) def get_object_data(self, ref: Union[str, bytes]) -> Tuple[str, str, int, bytes]: """Similar to :meth:`get_object_header`, but returns object data as well. @@ -1986,8 +2157,12 @@ def stream_object_data(self, ref: Union[str, bytes]) -> Tuple[str, str, int, "Gi This method is not threadsafe. You need one independent :class:`Git` instance per thread to be safe! """ - cmd = self._get_persistent_cmd("cat_file_all", "cat_file", batch=True) - hexsha, typename, size = self.__get_object_header(cmd, ref) + request = self._prepare_ref(ref) + native = _backend.object_data(self, request[:-1], stream=True) + if native is not NotImplemented: + return native + cmd = self._get_persistent_cmd("cat_file_all", "cat_file", batch=True, Z=True) + hexsha, typename, size = self.__get_object_header(cmd, request) cmd_stdout = cmd.stdout if cmd.stdout is not None else io.BytesIO() return (hexsha, typename, size, self.CatFileContentStream(size, cmd_stdout)) diff --git a/git/config.py b/git/config.py index a3437d525..ed89498b4 100644 --- a/git/config.py +++ b/git/config.py @@ -3,133 +3,31 @@ # This module is part of GitPython and is released under the # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ -"""Parser for reading and writing configuration files.""" +"""Git configuration access through ``git config``.""" __all__ = ["GitConfigParser", "SectionConstraint"] -import abc import configparser as cp -import fnmatch -import inspect -import logging import os import os.path as osp import re import sys -from functools import wraps -from io import BufferedReader, IOBase - -# typing------------------------------------------------------- -from typing import ( - IO, - TYPE_CHECKING, - Any, - Callable, - Dict, - Generic, - List, - OrderedDict, - Sequence, - Tuple, - TypeVar, - Union, - cast, -) +import tempfile +from contextlib import contextmanager +from typing import Any, Dict, Generic, Iterator, List, OrderedDict, Sequence, Tuple, TypeVar, Union, TYPE_CHECKING from git.compat import defenc, force_text +from git.exc import GitCommandError from git.types import _T, ConfigLevels_Tup, Lit_config_levels, PathLike, assert_never -from git.util import LockFile if TYPE_CHECKING: from io import BytesIO - from git.repo.base import Repo T_ConfigParser = TypeVar("T_ConfigParser", bound="GitConfigParser") T_OMD_value = TypeVar("T_OMD_value", str, bytes, int, float, bool, None) - OrderedDict_OMD = OrderedDict[str, List[T_OMD_value]] - -# ------------------------------------------------------------- - -_logger = logging.getLogger(__name__) - CONFIG_LEVELS: ConfigLevels_Tup = ("system", "user", "global", "repository") -"""The configuration level of a configuration file.""" - -CONDITIONAL_INCLUDE_REGEXP = re.compile(r"(?<=includeif )\"(gitdir|gitdir/i|onbranch|hasconfig:remote\.\*\.url):(.+)\"") -"""Section pattern to detect conditional includes. - -See: https://git-scm.com/docs/git-config#_conditional_includes -""" - -UNSAFE_CONFIG_CHARS_RE = re.compile(r"[\r\n\x00]") -"""Characters that cannot be safely written in config names or values.""" - -VALID_CONFIG_OPTION_NAME_RE = re.compile(r"^[A-Za-z0-9_.-]+$") -"""Pattern for option names that can be written without changing config syntax.""" - - -class MetaParserBuilder(abc.ABCMeta): # noqa: B024 - """Utility class wrapping base-class methods into decorators that assure read-only - properties.""" - - def __new__(cls, name: str, bases: Tuple, clsdict: Dict[str, Any]) -> "MetaParserBuilder": - """Equip all base-class methods with a needs_values decorator, and all non-const - methods with a :func:`set_dirty_and_flush_changes` decorator in addition to - that. - """ - kmm = "_mutating_methods_" - if kmm in clsdict: - mutating_methods = clsdict[kmm] - for base in bases: - methods = (t for t in inspect.getmembers(base, inspect.isroutine) if not t[0].startswith("_")) - for method_name, method in methods: - if method_name in clsdict: - continue - method_with_values = needs_values(method) - if method_name in mutating_methods: - method_with_values = set_dirty_and_flush_changes(method_with_values) - # END mutating methods handling - - clsdict[method_name] = method_with_values - # END for each name/method pair - # END for each base - # END if mutating methods configuration is set - - new_type = super().__new__(cls, name, bases, clsdict) - return new_type - - -def needs_values(func: Callable[..., _T]) -> Callable[..., _T]: - """Return a method for ensuring we read values (on demand) before we try to access - them.""" - - @wraps(func) - def assure_data_present(self: "GitConfigParser", *args: Any, **kwargs: Any) -> _T: - self.read() - return func(self, *args, **kwargs) - - # END wrapper method - return assure_data_present - - -def set_dirty_and_flush_changes(non_const_func: Callable[..., _T]) -> Callable[..., _T]: - """Return a method that checks whether given non constant function may be called. - - If so, the instance will be set dirty. Additionally, we flush the changes right to - disk. - """ - - def flush_changes(self: "GitConfigParser", *args: Any, **kwargs: Any) -> _T: - rval = non_const_func(self, *args, **kwargs) - self._dirty = True - self.write() - return rval - - # END wrapper method - flush_changes.__name__ = non_const_func.__name__ - return flush_changes class SectionConstraint(Generic[T_ConfigParser]): @@ -146,6 +44,10 @@ class SectionConstraint(Generic[T_ConfigParser]): _valid_attrs_ = ( "get_value", + "get_values", + "add_value", + "items", + "items_all", "set_value", "get", "set", @@ -291,60 +193,19 @@ def get_config_path(config_level: Lit_config_levels) -> str: ) -class GitConfigParser(cp.RawConfigParser, metaclass=MetaParserBuilder): - """Implements specifics required to read git style configuration files. +class GitConfigParser: + """Read and modify Git configuration using Git's parser and file locking. - This variation behaves much like the :manpage:`git-config(1)` command, such that the - configuration will be read on demand based on the filepath given during - initialization. - - The changes will automatically be written once the instance goes out of scope, but - can be triggered manually as well. - - The configuration file will be locked if you intend to change values preventing - other instances to write concurrently. - - :note: - Section and option names are case-insensitive; quoted subsection names are - case-sensitive. Names retain their first spelling when enumerated or written. - Case variants are merged, preserving all values in the order they are read. + File paths, byte streams, and lists of sources are accepted for reading. Writers + operate on one source, updating it immediately. Git locks each mutation; a writer + does not reserve a lifetime lock. Git canonicalizes enumerated section/option + names, while preserving subsection case and duplicate values. - :note: - If used as a context manager, this will release the locked file. - - :note: - Options without a value are stored as ``None`` and written without ``=``. - :meth:`get_value` and :meth:`get_values` return an empty string for them, - while :meth:`getboolean` returns ``True``. An explicit empty value is - stored as an empty string and reads as ``False`` with :meth:`getboolean`. + Empty sections, raw configuration parsing/serialization, and writing valueless + options are not supported. Existing valueless options remain readable. """ - # { Configuration - t_lock = LockFile - """The lock type determines the type of lock to use in new configuration readers. - - They must be compatible to the :class:`~git.util.LockFile` interface. - A suitable alternative would be the :class:`~git.util.BlockingLockFile`. - """ - - re_comment = re.compile(r"^\s*[#;]") - # } END configuration - - optvalueonly_source = r"\s*(?P