diff --git a/.github/workflows/alpine-test.yml b/.github/workflows/alpine-test.yml index 2c1eed391..b7de7482e 100644 --- a/.github/workflows/alpine-test.yml +++ b/.github/workflows/alpine-test.yml @@ -2,8 +2,11 @@ name: test-alpine on: [push, pull_request, workflow_dispatch] +permissions: + contents: read + jobs: - build: + test: runs-on: ubuntu-latest container: @@ -16,14 +19,14 @@ jobs: steps: - name: Prepare Alpine Linux run: | - apk add sudo git git-daemon python3 py3-pip + apk add sudo git git-daemon python3 py3-pip py3-virtualenv echo 'Defaults env_keep += "CI GITHUB_* RUNNER_*"' >/etc/sudoers.d/ci_env addgroup -g 127 docker - adduser -D -u 1001 runner + adduser -D -u 1001 runner # TODO: Check if this still works on GHA as intended. adduser runner docker shell: sh -exo pipefail {0} # Run this as root, not the "runner" user. - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: fetch-depth: 0 @@ -44,23 +47,23 @@ jobs: # and cause subsequent tests to fail cat test/fixtures/.gitconfig >> ~/.gitconfig - - name: Set up virtualenv + - name: Set up virtual environment run: | python -m venv .venv - . .venv/bin/activate - printf '%s=%s\n' 'PATH' "$PATH" 'VIRTUAL_ENV' "$VIRTUAL_ENV" >>"$GITHUB_ENV" - name: Update PyPA packages run: | - # Get the latest pip, wheel, and prior to Python 3.12, setuptools. - python -m pip install -U pip $(pip freeze --all | grep -ow ^setuptools) wheel + . .venv/bin/activate + python -m pip install -U pip 'setuptools; python_version<"3.12"' wheel - name: Install project and test dependencies run: | - pip install ".[test]" + . .venv/bin/activate + pip install '.[test]' - name: Show version and platform information run: | + . .venv/bin/activate uname -a command -v git python git version @@ -69,4 +72,5 @@ jobs: - name: Test with pytest run: | + . .venv/bin/activate pytest --color=yes -p no:sugar --instafail -vv diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index ae5241898..e243416a8 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -19,38 +19,48 @@ on: jobs: analyze: - name: Analyze + name: Analyze (${{ matrix.language }}) # Runner size impacts CodeQL analysis time. To learn more, please see: # - https://gh.io/recommended-hardware-resources-for-running-codeql # - https://gh.io/supported-runners-and-hardware-resources - # - https://gh.io/using-larger-runners - # Consider using larger runners for possible analysis time improvements. + # - https://gh.io/using-larger-runners (GitHub.com only) + # Consider using larger runners or machines with greater resources for possible analysis time improvements. runs-on: ${{ (matrix.language == 'swift' && 'macos-latest') || 'ubuntu-latest' }} - timeout-minutes: ${{ (matrix.language == 'swift' && 120) || 360 }} permissions: - actions: read - contents: read security-events: write strategy: fail-fast: false matrix: - language: [ 'python' ] - # CodeQL supports [ 'c-cpp', 'csharp', 'go', 'java-kotlin', 'javascript-typescript', 'python', 'ruby', 'swift' ] - # Use only 'java-kotlin' to analyze code written in Java, Kotlin or both - # Use only 'javascript-typescript' to analyze code written in JavaScript, TypeScript or both - # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support - + include: + - language: actions + build-mode: none + - language: python + build-mode: none + # CodeQL supports the following values keywords for 'language': 'actions', 'c-cpp', 'csharp', 'go', 'java-kotlin', 'javascript-typescript', 'python', 'ruby', 'swift' + # Use `c-cpp` to analyze code written in C, C++ or both + # Use 'java-kotlin' to analyze code written in Java, Kotlin or both + # Use 'javascript-typescript' to analyze code written in JavaScript, TypeScript or both + # To learn more about changing the languages that are analyzed or customizing the build mode for your analysis, + # see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/customizing-your-advanced-setup-for-code-scanning. + # If you are analyzing a compiled language, you can modify the 'build-mode' for that language to customize how + # your codebase is analyzed, see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/codeql-code-scanning-for-compiled-languages steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@v6 + + # Add any setup steps before running the `github/codeql-action/init` action. + # This includes steps like installing compilers or runtimes (`actions/setup-node` + # or others). This is typically only required for manual builds. + # - name: Setup runtime (example) + # uses: actions/setup-example@v1 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@v3 + uses: github/codeql-action/init@v4 with: languages: ${{ matrix.language }} - setup-python-dependencies: false + build-mode: ${{ matrix.build-mode }} # If you wish to specify custom queries, you can do so here or in a config file. # By default, queries listed here will override any specified in a config file. # Prefix the list here with "+" to use these queries and those in the config file. @@ -58,23 +68,23 @@ jobs: # For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs # queries: security-extended,security-and-quality - - # Autobuild attempts to build any compiled languages (C/C++, C#, Go, Java, or Swift). - # If this step fails, then you should remove it and run the build manually (see below) - - name: Autobuild - uses: github/codeql-action/autobuild@v3 - + # If the analyze step fails for one of the languages you are analyzing with + # "We were unable to automatically build your code", modify the matrix above + # to set the build mode to "manual" for that language. Then modify this step + # to build your code. # ℹī¸ Command-line programs to run using the OS shell. # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun - - # If the Autobuild fails above, remove it and uncomment the following three lines. - # modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance. - - # - run: | - # echo "Run, Build Application using script" - # ./location_of_script_within_repo/buildscript.sh + - if: matrix.build-mode == 'manual' + shell: bash + run: | + echo 'If you are using a "manual" build mode for one or more of the' \ + 'languages you are analyzing, replace this with the commands to build' \ + 'your code, for example:' + echo ' make bootstrap' + echo ' make release' + exit 1 - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v3 + uses: github/codeql-action/analyze@v4 with: category: "/language:${{matrix.language}}" diff --git a/.github/workflows/cygwin-test.yml b/.github/workflows/cygwin-test.yml index bde4ea659..327e1f10c 100644 --- a/.github/workflows/cygwin-test.yml +++ b/.github/workflows/cygwin-test.yml @@ -2,35 +2,47 @@ name: test-cygwin on: [push, pull_request, workflow_dispatch] -jobs: - build: - runs-on: windows-latest +permissions: + contents: read +jobs: + test: strategy: + matrix: + selection: [fast, perf] + include: + - selection: fast + additional-pytest-args: --ignore=test/performance + - selection: perf + additional-pytest-args: test/performance + fail-fast: false + runs-on: windows-latest + env: CHERE_INVOKING: "1" CYGWIN_NOWINPATH: "1" defaults: run: - shell: C:\tools\cygwin\bin\bash.exe --login --norc -eo pipefail -o igncr "{0}" + shell: C:\cygwin\bin\bash.exe --login --norc -eo pipefail -o igncr "{0}" steps: - name: Force LF line endings run: | git config --global core.autocrlf false # Affects the non-Cygwin git. - shell: bash # Use Git Bash instead of Cygwin Bash for this step. + shell: pwsh # Do this outside Cygwin, to affect actions/checkout. - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: fetch-depth: 0 - - name: Set up Cygwin - uses: egor-tensin/setup-cygwin@v4 + - name: Install Cygwin + uses: cygwin/cygwin-install-action@v6 with: - packages: python39=3.9.16-1 python39-pip python39-virtualenv git + packages: git python39 python-pip-wheel python-setuptools-wheel python-wheel-wheel + add-to-path: false # No need to change $PATH outside the Cygwin environment. - name: Arrange for verbose output run: | @@ -55,19 +67,18 @@ jobs: # and cause subsequent tests to fail cat test/fixtures/.gitconfig >> ~/.gitconfig - - name: Ensure the "pip" command is available + - name: Set up virtual environment run: | - # This is used unless, and before, an updated pip is installed. - ln -s pip3 /usr/bin/pip + python3.9 -m venv .venv + echo 'BASH_ENV=.venv/bin/activate' >>"$GITHUB_ENV" - name: Update PyPA packages run: | - # Get the latest pip, wheel, and prior to Python 3.12, setuptools. - python -m pip install -U pip $(pip freeze --all | grep -ow ^setuptools) wheel + python -m pip install -U pip 'setuptools; python_version<"3.12"' wheel - name: Install project and test dependencies run: | - pip install ".[test]" + pip install '.[test]' - name: Show version and platform information run: | @@ -77,6 +88,6 @@ jobs: python --version python -c 'import os, sys; print(f"sys.platform={sys.platform!r}, os.name={os.name!r}")' - - name: Test with pytest + - name: Test with pytest (${{ matrix.additional-pytest-args }}) run: | - pytest --color=yes -p no:sugar --instafail -vv + pytest --color=yes -p no:sugar --instafail -vv ${{ matrix.additional-pytest-args }} diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index a0e81a993..956b38963 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -2,14 +2,17 @@ name: Lint on: [push, pull_request, workflow_dispatch] +permissions: + contents: read + jobs: lint: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - - uses: actions/setup-python@v5 + - uses: actions/setup-python@v6 with: python-version: "3.x" diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 747db62f0..ac764d9a7 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -9,39 +9,51 @@ permissions: contents: read jobs: - build: + test: strategy: - fail-fast: false matrix: - os: ["ubuntu-22.04", "macos-latest", "windows-latest"] - python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"] + os-type: [ubuntu, macos, windows] + python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12", "3.13", "3.13t"] exclude: - - os: "macos-latest" - python-version: "3.7" + - os-type: macos + python-version: "3.7" # Not available for the ARM-based macOS runners. + - os-type: macos + python-version: "3.13t" + - os-type: windows + python-version: "3.13" # FIXME: Fix and enable Python 3.13 on Windows (#1955). + - os-type: windows + python-version: "3.13t" include: + - os-ver: latest + - os-type: ubuntu + python-version: "3.7" + os-ver: "22.04" - experimental: false - runs-on: ${{ matrix.os }} + fail-fast: false + + runs-on: ${{ matrix.os-type }}-${{ matrix.os-ver }} defaults: run: shell: bash --noprofile --norc -exo pipefail {0} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: fetch-depth: 0 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} allow-prereleases: ${{ matrix.experimental }} - name: Set up WSL (Windows) - if: startsWith(matrix.os, 'windows') - uses: Vampire/setup-wsl@v3.1.1 + if: matrix.os-type == 'windows' + uses: Vampire/setup-wsl@v6.0.0 with: + wsl-version: 1 distribution: Alpine additional-packages: bash @@ -59,12 +71,11 @@ jobs: - name: Update PyPA packages run: | - # Get the latest pip, wheel, and prior to Python 3.12, setuptools. - python -m pip install -U pip $(pip freeze --all | grep -ow ^setuptools) wheel + python -m pip install -U pip 'setuptools; python_version<"3.12"' wheel - name: Install project and test dependencies run: | - pip install ".[test]" + pip install '.[test]' - name: Show version and platform information run: | @@ -76,7 +87,7 @@ jobs: # For debugging hook tests on native Windows systems that may have WSL. - name: Show bash.exe candidates (Windows) - if: startsWith(matrix.os, 'windows') + if: matrix.os-type == 'windows' run: | set +e bash.exe -c 'printenv WSL_DISTRO_NAME; uname -a' @@ -84,14 +95,13 @@ jobs: continue-on-error: true - name: Check types with mypy + if: matrix.python-version != '3.7' && matrix.python-version != '3.8' run: | - mypy --python-version=${{ matrix.python-version }} + mypy --python-version="${PYTHON_VERSION%t}" # Version only, with no "t" for free-threaded. env: MYPY_FORCE_COLOR: "1" TERM: "xterm-256color" # For color: https://github.com/python/mypy/issues/13817 - # With new versions of mypy new issues might arise. This is a problem if there is - # nobody able to fix them, so we have to ignore errors until that changes. - continue-on-error: true + PYTHON_VERSION: ${{ matrix.python-version }} - name: Test with pytest run: | @@ -101,5 +111,5 @@ jobs: - name: Documentation if: matrix.python-version != '3.7' run: | - pip install ".[doc]" + pip install '.[doc]' make -C doc html diff --git a/.gitignore b/.gitignore index d85569405..eab294a65 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,8 @@ __pycache__/ # Transient editor files *.swp *~ +\#*# +.#*# # Editor configuration nbproject diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 424cc5f37..737b56d45 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,15 +1,15 @@ repos: - repo: https://github.com/codespell-project/codespell - rev: v2.3.0 + rev: v2.4.1 hooks: - id: codespell additional_dependencies: [tomli] exclude: ^test/fixtures/ - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.6.0 + rev: v0.11.12 hooks: - - id: ruff + - id: ruff-check args: ["--fix"] exclude: ^git/ext/ - id: ruff-format @@ -23,7 +23,7 @@ repos: exclude: ^test/fixtures/polyglot$|^git/ext/ - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.6.0 + rev: v5.0.0 hooks: - id: end-of-file-fixer exclude: ^test/fixtures/|COPYING|LICENSE @@ -33,6 +33,6 @@ repos: - id: check-merge-conflict - repo: https://github.com/abravalheri/validate-pyproject - rev: v0.19 + rev: v0.24.1 hooks: - id: validate-pyproject diff --git a/AUTHORS b/AUTHORS index 45b14c961..b57113edd 100644 --- a/AUTHORS +++ b/AUTHORS @@ -55,5 +55,6 @@ Contributors are: -Eliah Kagan -Ethan Lin -Jonas Scharpf +-Gordon Marx Portions derived from other open source works and are clearly marked. diff --git a/SECURITY.md b/SECURITY.md index d39425b70..0aea34845 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -11,4 +11,6 @@ Only the latest version of GitPython can receive security updates. If a vulnerab ## Reporting a Vulnerability -Please report private portions of a vulnerability to . Doing so helps to receive updates and collaborate on the matter, without disclosing it publicliy right away. +Please report private portions of a vulnerability to . Doing so helps to receive updates and collaborate on the matter, without disclosing it publicly right away. + +Vulnerabilities in GitPython's dependencies [gitdb](https://github.com/gitpython-developers/gitdb/blob/master/SECURITY.md) or [smmap](https://github.com/gitpython-developers/smmap/blob/master/SECURITY.md), which primarily exist to support GitPython, can be reported here as well, at that same link. The affected package (`GitPython`, `gitdb`, or `smmap`) can be included in the report, if known. diff --git a/VERSION b/VERSION index e6af1c454..3c91929a4 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.1.44 +3.1.45 diff --git a/doc/Makefile b/doc/Makefile index ddeadbd7e..7e0d325fe 100644 --- a/doc/Makefile +++ b/doc/Makefile @@ -3,7 +3,7 @@ # You can set these variables from the command line. BUILDDIR = build -SPHINXOPTS = -W +SPHINXOPTS = SPHINXBUILD = sphinx-build PAPER = diff --git a/doc/source/changes.rst b/doc/source/changes.rst index 00a3c660e..151059ed2 100644 --- a/doc/source/changes.rst +++ b/doc/source/changes.rst @@ -2,6 +2,12 @@ Changelog ========= +3.1.45 +====== + +See the following for all changes. +https://github.com/gitpython-developers/GitPython/releases/tag/3.1.45 + 3.1.44 ====== diff --git a/fuzzing/fuzz-targets/fuzz_submodule.py b/fuzzing/fuzz-targets/fuzz_submodule.py index d22b0aa5b..afa653d0d 100644 --- a/fuzzing/fuzz-targets/fuzz_submodule.py +++ b/fuzzing/fuzz-targets/fuzz_submodule.py @@ -9,11 +9,17 @@ get_max_filename_length, ) -# Setup the git environment +# Setup the Git environment setup_git_environment() from git import Repo, GitCommandError, InvalidGitRepositoryError +def sanitize_input(input_str, max_length=255): + """Sanitize and truncate inputs to avoid invalid Git operations.""" + sanitized = "".join(ch for ch in input_str if ch.isalnum() or ch in ("-", "_", ".")) + return sanitized[:max_length] + + def TestOneInput(data): fdp = atheris.FuzzedDataProvider(data) @@ -24,12 +30,23 @@ def TestOneInput(data): try: with tempfile.TemporaryDirectory() as submodule_temp_dir: sub_repo = Repo.init(submodule_temp_dir, bare=fdp.ConsumeBool()) - sub_repo.index.commit(fdp.ConsumeUnicodeNoSurrogates(fdp.ConsumeIntInRange(1, 512))) + commit_message = sanitize_input(fdp.ConsumeUnicodeNoSurrogates(fdp.ConsumeIntInRange(1, 512))) + sub_repo.index.commit(commit_message) - submodule_name = fdp.ConsumeUnicodeNoSurrogates( - fdp.ConsumeIntInRange(1, max(1, get_max_filename_length(repo.working_tree_dir))) + submodule_name = sanitize_input( + fdp.ConsumeUnicodeNoSurrogates( + fdp.ConsumeIntInRange(1, get_max_filename_length(repo.working_tree_dir)) + ) ) - submodule_path = os.path.join(repo.working_tree_dir, submodule_name) + + submodule_path = os.path.relpath( + os.path.join(repo.working_tree_dir, submodule_name), + start=repo.working_tree_dir, + ) + + # Ensure submodule_path is valid + if not submodule_name or submodule_name.startswith("/") or ".." in submodule_name: + return -1 # Reject invalid input so they are not added to the corpus submodule = repo.create_submodule(submodule_name, submodule_path, url=sub_repo.git_dir) repo.index.commit("Added submodule") @@ -39,25 +56,38 @@ def TestOneInput(data): value_length = fdp.ConsumeIntInRange(1, max(1, fdp.remaining_bytes())) writer.set_value( - fdp.ConsumeUnicodeNoSurrogates(key_length), fdp.ConsumeUnicodeNoSurrogates(value_length) + sanitize_input(fdp.ConsumeUnicodeNoSurrogates(key_length)), + sanitize_input(fdp.ConsumeUnicodeNoSurrogates(value_length)), ) writer.release() - submodule.update(init=fdp.ConsumeBool(), dry_run=fdp.ConsumeBool(), force=fdp.ConsumeBool()) + submodule.update( + init=fdp.ConsumeBool(), + dry_run=fdp.ConsumeBool(), + force=fdp.ConsumeBool(), + ) + submodule_repo = submodule.module() - new_file_name = fdp.ConsumeUnicodeNoSurrogates( - fdp.ConsumeIntInRange(1, max(1, get_max_filename_length(submodule_repo.working_tree_dir))) + new_file_name = sanitize_input( + fdp.ConsumeUnicodeNoSurrogates( + fdp.ConsumeIntInRange(1, get_max_filename_length(submodule_repo.working_tree_dir)) + ) ) new_file_path = os.path.join(submodule_repo.working_tree_dir, new_file_name) with open(new_file_path, "wb") as new_file: new_file.write(fdp.ConsumeBytes(fdp.ConsumeIntInRange(1, 512))) + submodule_repo.index.add([new_file_path]) submodule_repo.index.commit("Added new file to submodule") repo.submodule_update(recursive=fdp.ConsumeBool()) - submodule_repo.head.reset(commit="HEAD~1", working_tree=fdp.ConsumeBool(), head=fdp.ConsumeBool()) - # Use fdp.PickValueInList to ensure at least one of 'module' or 'configuration' is True + submodule_repo.head.reset( + commit="HEAD~1", + working_tree=fdp.ConsumeBool(), + head=fdp.ConsumeBool(), + ) + module_option_value, configuration_option_value = fdp.PickValueInList( [(True, False), (False, True), (True, True)] ) @@ -82,12 +112,7 @@ def TestOneInput(data): ): return -1 except Exception as e: - if isinstance(e, ValueError) and "embedded null byte" in str(e): - return -1 - elif isinstance(e, OSError) and "File name too long" in str(e): - return -1 - else: - return handle_exception(e) + return handle_exception(e) def main(): diff --git a/git/cmd.py b/git/cmd.py index 2048a43fa..15d7820df 100644 --- a/git/cmd.py +++ b/git/cmd.py @@ -60,6 +60,11 @@ overload, ) +if sys.version_info >= (3, 10): + from typing import TypeAlias +else: + from typing_extensions import TypeAlias + from git.types import Literal, PathLike, TBD if TYPE_CHECKING: @@ -207,7 +212,7 @@ def pump_stream( ) if stderr_handler: error_str: Union[str, bytes] = ( - "error: process killed because it timed out." f" kill_after_timeout={kill_after_timeout} seconds" + f"error: process killed because it timed out. kill_after_timeout={kill_after_timeout} seconds" ) if not decode_streams and isinstance(p_stderr, BinaryIO): # Assume stderr_handler needs binary input. @@ -268,12 +273,12 @@ def _safer_popen_windows( if shell: # The original may be immutable, or the caller may reuse it. Mutate a copy. env = {} if env is None else dict(env) - env["NoDefaultCurrentDirectoryInExePath"] = "1" # The "1" can be an value. + env["NoDefaultCurrentDirectoryInExePath"] = "1" # The "1" can be any value. # When not using a shell, the current process does the search in a # CreateProcessW API call, so the variable must be set in our environment. With # a shell, that's unnecessary if https://github.com/python/cpython/issues/101283 - # is patched. In Python versions where it is unpatched, and in the rare case the + # is patched. In Python versions where it is unpatched, in the rare case the # ComSpec environment variable is unset, the search for the shell itself is # unsafe. Setting NoDefaultCurrentDirectoryInExePath in all cases, as done here, # is simpler and protects against that. (As above, the "1" can be any value.) @@ -308,6 +313,230 @@ def dict_to_slots_and__excluded_are_none(self: object, d: Mapping[str, Any], exc ## -- End Utilities -- @} + +class _AutoInterrupt: + """Process wrapper that terminates the wrapped process on finalization. + + This kills/interrupts the stored process instance once this instance goes out of + scope. It is used to prevent processes piling up in case iterators stop reading. + + All attributes are wired through to the contained process object. + + The wait method is overridden to perform automatic status code checking and possibly + raise. + """ + + __slots__ = ("proc", "args", "status") + + # If this is non-zero it will override any status code during _terminate, used + # to prevent race conditions in testing. + _status_code_if_terminate: int = 0 + + def __init__(self, proc: Union[None, subprocess.Popen], args: Any) -> None: + self.proc = proc + self.args = args + self.status: Union[int, None] = None + + def _terminate(self) -> None: + """Terminate the underlying process.""" + if self.proc is None: + return + + proc = self.proc + self.proc = None + if proc.stdin: + proc.stdin.close() + if proc.stdout: + proc.stdout.close() + if proc.stderr: + proc.stderr.close() + # Did the process finish already so we have a return code? + try: + if proc.poll() is not None: + self.status = self._status_code_if_terminate or proc.poll() + return + except OSError as ex: + _logger.info("Ignored error after process had died: %r", ex) + + # It can be that nothing really exists anymore... + if os is None or getattr(os, "kill", None) is None: + return + + # Try to kill it. + try: + proc.terminate() + status = proc.wait() # Ensure the process goes away. + + self.status = self._status_code_if_terminate or status + except OSError as ex: + _logger.info("Ignored error after process had died: %r", ex) + # END exception handling + + def __del__(self) -> None: + self._terminate() + + def __getattr__(self, attr: str) -> Any: + return getattr(self.proc, attr) + + # TODO: Bad choice to mimic `proc.wait()` but with different args. + def wait(self, stderr: Union[None, str, bytes] = b"") -> int: + """Wait for the process and return its status code. + + :param stderr: + Previously read value of stderr, in case stderr is already closed. + + :warn: + May deadlock if output or error pipes are used and not handled separately. + + :raise git.exc.GitCommandError: + If the return status is not 0. + """ + if stderr is None: + stderr_b = b"" + stderr_b = force_bytes(data=stderr, encoding="utf-8") + status: Union[int, None] + if self.proc is not None: + status = self.proc.wait() + p_stderr = self.proc.stderr + else: # Assume the underlying proc was killed earlier or never existed. + status = self.status + p_stderr = None + + def read_all_from_possibly_closed_stream(stream: Union[IO[bytes], None]) -> bytes: + if stream: + try: + return stderr_b + force_bytes(stream.read()) + except (OSError, ValueError): + return stderr_b or b"" + else: + return stderr_b or b"" + + # END status handling + + if status != 0: + errstr = read_all_from_possibly_closed_stream(p_stderr) + _logger.debug("AutoInterrupt wait stderr: %r" % (errstr,)) + raise GitCommandError(remove_password_if_present(self.args), status, errstr) + return status + + +_AutoInterrupt.__name__ = "AutoInterrupt" +_AutoInterrupt.__qualname__ = "Git.AutoInterrupt" + + +class _CatFileContentStream: + """Object representing a sized read-only stream returning the contents of + an object. + + This behaves like a stream, but counts the data read and simulates an empty stream + once our sized content region is empty. + + If not all data are read to the end of the object's lifetime, we read the rest to + ensure the underlying stream continues to work. + """ + + __slots__ = ("_stream", "_nbr", "_size") + + def __init__(self, size: int, stream: IO[bytes]) -> None: + self._stream = stream + self._size = size + self._nbr = 0 # Number of bytes read. + + # Special case: If the object is empty, has null bytes, get the final + # newline right away. + if size == 0: + stream.read(1) + # END handle empty streams + + def read(self, size: int = -1) -> bytes: + bytes_left = self._size - self._nbr + if bytes_left == 0: + return b"" + if size > -1: + # Ensure we don't try to read past our limit. + size = min(bytes_left, size) + else: + # They try to read all, make sure it's not more than what remains. + size = bytes_left + # END check early depletion + data = self._stream.read(size) + self._nbr += len(data) + + # Check for depletion, read our final byte to make the stream usable by + # others. + if self._size - self._nbr == 0: + self._stream.read(1) # final newline + # END finish reading + return data + + def readline(self, size: int = -1) -> bytes: + if self._nbr == self._size: + return b"" + + # Clamp size to lowest allowed value. + bytes_left = self._size - self._nbr + if size > -1: + size = min(bytes_left, size) + else: + size = bytes_left + # END handle size + + data = self._stream.readline(size) + self._nbr += len(data) + + # Handle final byte. + if self._size - self._nbr == 0: + self._stream.read(1) + # END finish reading + + return data + + def readlines(self, size: int = -1) -> List[bytes]: + if self._nbr == self._size: + return [] + + # Leave all additional logic to our readline method, we just check the size. + out = [] + nbr = 0 + while True: + line = self.readline() + if not line: + break + out.append(line) + if size > -1: + nbr += len(line) + if nbr > size: + break + # END handle size constraint + # END readline loop + return out + + # skipcq: PYL-E0301 + def __iter__(self) -> "Git.CatFileContentStream": + return self + + def __next__(self) -> bytes: + line = self.readline() + if not line: + raise StopIteration + + return line + + next = __next__ + + def __del__(self) -> None: + bytes_left = self._size - self._nbr + if bytes_left: + # Read and discard - seeking is impossible within a stream. + # This includes any terminating newline. + self._stream.read(bytes_left + 1) + # END handle incomplete read + + +_CatFileContentStream.__name__ = "CatFileContentStream" +_CatFileContentStream.__qualname__ = "Git.CatFileContentStream" + + _USE_SHELL_DEFAULT_MESSAGE = ( "Git.USE_SHELL is deprecated, because only its default value of False is safe. " "It will be removed in a future release." @@ -321,7 +550,7 @@ def dict_to_slots_and__excluded_are_none(self: object, d: Mapping[str, Any], exc ) -def _warn_use_shell(extra_danger: bool) -> None: +def _warn_use_shell(*, extra_danger: bool) -> None: warnings.warn( _USE_SHELL_DANGER_MESSAGE if extra_danger else _USE_SHELL_DEFAULT_MESSAGE, DeprecationWarning, @@ -337,12 +566,12 @@ class _GitMeta(type): def __getattribute(cls, name: str) -> Any: if name == "USE_SHELL": - _warn_use_shell(False) + _warn_use_shell(extra_danger=False) return super().__getattribute__(name) def __setattr(cls, name: str, value: Any) -> Any: if name == "USE_SHELL": - _warn_use_shell(value) + _warn_use_shell(extra_danger=value) super().__setattr__(name, value) if not TYPE_CHECKING: @@ -728,221 +957,9 @@ def check_unsafe_options(cls, options: List[str], unsafe_options: List[str]) -> f"{unsafe_option} is not allowed, use `allow_unsafe_options=True` to allow it." ) - class AutoInterrupt: - """Process wrapper that terminates the wrapped process on finalization. - - This kills/interrupts the stored process instance once this instance goes out of - scope. It is used to prevent processes piling up in case iterators stop reading. - - All attributes are wired through to the contained process object. - - The wait method is overridden to perform automatic status code checking and - possibly raise. - """ - - __slots__ = ("proc", "args", "status") - - # If this is non-zero it will override any status code during _terminate, used - # to prevent race conditions in testing. - _status_code_if_terminate: int = 0 - - def __init__(self, proc: Union[None, subprocess.Popen], args: Any) -> None: - self.proc = proc - self.args = args - self.status: Union[int, None] = None - - def _terminate(self) -> None: - """Terminate the underlying process.""" - if self.proc is None: - return - - proc = self.proc - self.proc = None - if proc.stdin: - proc.stdin.close() - if proc.stdout: - proc.stdout.close() - if proc.stderr: - proc.stderr.close() - # Did the process finish already so we have a return code? - try: - if proc.poll() is not None: - self.status = self._status_code_if_terminate or proc.poll() - return - except OSError as ex: - _logger.info("Ignored error after process had died: %r", ex) - - # It can be that nothing really exists anymore... - if os is None or getattr(os, "kill", None) is None: - return - - # Try to kill it. - try: - proc.terminate() - status = proc.wait() # Ensure the process goes away. - - self.status = self._status_code_if_terminate or status - except OSError as ex: - _logger.info("Ignored error after process had died: %r", ex) - # END exception handling - - def __del__(self) -> None: - self._terminate() - - def __getattr__(self, attr: str) -> Any: - return getattr(self.proc, attr) - - # TODO: Bad choice to mimic `proc.wait()` but with different args. - def wait(self, stderr: Union[None, str, bytes] = b"") -> int: - """Wait for the process and return its status code. - - :param stderr: - Previously read value of stderr, in case stderr is already closed. - - :warn: - May deadlock if output or error pipes are used and not handled - separately. - - :raise git.exc.GitCommandError: - If the return status is not 0. - """ - if stderr is None: - stderr_b = b"" - stderr_b = force_bytes(data=stderr, encoding="utf-8") - status: Union[int, None] - if self.proc is not None: - status = self.proc.wait() - p_stderr = self.proc.stderr - else: # Assume the underlying proc was killed earlier or never existed. - status = self.status - p_stderr = None - - def read_all_from_possibly_closed_stream(stream: Union[IO[bytes], None]) -> bytes: - if stream: - try: - return stderr_b + force_bytes(stream.read()) - except (OSError, ValueError): - return stderr_b or b"" - else: - return stderr_b or b"" - - # END status handling - - if status != 0: - errstr = read_all_from_possibly_closed_stream(p_stderr) - _logger.debug("AutoInterrupt wait stderr: %r" % (errstr,)) - raise GitCommandError(remove_password_if_present(self.args), status, errstr) - return status - - # END auto interrupt - - class CatFileContentStream: - """Object representing a sized read-only stream returning the contents of - an object. - - This behaves like a stream, but counts the data read and simulates an empty - stream once our sized content region is empty. - - If not all data are read to the end of the object's lifetime, we read the - rest to ensure the underlying stream continues to work. - """ - - __slots__ = ("_stream", "_nbr", "_size") - - def __init__(self, size: int, stream: IO[bytes]) -> None: - self._stream = stream - self._size = size - self._nbr = 0 # Number of bytes read. - - # Special case: If the object is empty, has null bytes, get the final - # newline right away. - if size == 0: - stream.read(1) - # END handle empty streams - - def read(self, size: int = -1) -> bytes: - bytes_left = self._size - self._nbr - if bytes_left == 0: - return b"" - if size > -1: - # Ensure we don't try to read past our limit. - size = min(bytes_left, size) - else: - # They try to read all, make sure it's not more than what remains. - size = bytes_left - # END check early depletion - data = self._stream.read(size) - self._nbr += len(data) - - # Check for depletion, read our final byte to make the stream usable by - # others. - if self._size - self._nbr == 0: - self._stream.read(1) # final newline - # END finish reading - return data - - def readline(self, size: int = -1) -> bytes: - if self._nbr == self._size: - return b"" - - # Clamp size to lowest allowed value. - bytes_left = self._size - self._nbr - if size > -1: - size = min(bytes_left, size) - else: - size = bytes_left - # END handle size - - data = self._stream.readline(size) - self._nbr += len(data) - - # Handle final byte. - if self._size - self._nbr == 0: - self._stream.read(1) - # END finish reading - - return data - - def readlines(self, size: int = -1) -> List[bytes]: - if self._nbr == self._size: - return [] - - # Leave all additional logic to our readline method, we just check the size. - out = [] - nbr = 0 - while True: - line = self.readline() - if not line: - break - out.append(line) - if size > -1: - nbr += len(line) - if nbr > size: - break - # END handle size constraint - # END readline loop - return out - - # skipcq: PYL-E0301 - def __iter__(self) -> "Git.CatFileContentStream": - return self - - def __next__(self) -> bytes: - line = self.readline() - if not line: - raise StopIteration - - return line - - next = __next__ + AutoInterrupt: TypeAlias = _AutoInterrupt - def __del__(self) -> None: - bytes_left = self._size - self._nbr - if bytes_left: - # Read and discard - seeking is impossible within a stream. - # This includes any terminating newline. - self._stream.read(bytes_left + 1) - # END handle incomplete read + CatFileContentStream: TypeAlias = _CatFileContentStream def __init__(self, working_dir: Union[None, PathLike] = None) -> None: """Initialize this instance with: @@ -971,7 +988,7 @@ def __init__(self, working_dir: Union[None, PathLike] = None) -> None: def __getattribute__(self, name: str) -> Any: if name == "USE_SHELL": - _warn_use_shell(False) + _warn_use_shell(extra_danger=False) return super().__getattribute__(name) def __getattr__(self, name: str) -> Any: @@ -1319,7 +1336,7 @@ def communicate() -> Tuple[AnyStr, AnyStr]: out, err = proc.communicate() watchdog.cancel() if kill_check.is_set(): - err = 'Timeout: the command "%s" did not complete in %d ' "secs." % ( + err = 'Timeout: the command "%s" did not complete in %d secs.' % ( " ".join(redacted_command), timeout, ) diff --git a/git/config.py b/git/config.py index de3508360..769929441 100644 --- a/git/config.py +++ b/git/config.py @@ -66,7 +66,7 @@ CONFIG_LEVELS: ConfigLevels_Tup = ("system", "user", "global", "repository") """The configuration level of a configuration file.""" -CONDITIONAL_INCLUDE_REGEXP = re.compile(r"(?<=includeIf )\"(gitdir|gitdir/i|onbranch):(.+)\"") +CONDITIONAL_INCLUDE_REGEXP = re.compile(r"(?<=includeIf )\"(gitdir|gitdir/i|onbranch|hasconfig:remote\.\*\.url):(.+)\"") """Section pattern to detect conditional includes. See: https://git-scm.com/docs/git-config#_conditional_includes @@ -87,15 +87,15 @@ def __new__(cls, name: str, bases: Tuple, clsdict: Dict[str, Any]) -> "MetaParse mutating_methods = clsdict[kmm] for base in bases: methods = (t for t in inspect.getmembers(base, inspect.isroutine) if not t[0].startswith("_")) - for name, method in methods: - if name in clsdict: + for method_name, method in methods: + if method_name in clsdict: continue method_with_values = needs_values(method) - if name in mutating_methods: + if method_name in mutating_methods: method_with_values = set_dirty_and_flush_changes(method_with_values) # END mutating methods handling - clsdict[name] = method_with_values + clsdict[method_name] = method_with_values # END for each name/method pair # END for each base # END if mutating methods configuration is set @@ -496,19 +496,26 @@ def string_decode(v: str) -> str: if mo: # We might just have handled the last line, which could contain a quotation we want to remove. optname, vi, optval = mo.group("option", "vi", "value") + optname = self.optionxform(optname.rstrip()) + if vi in ("=", ":") and ";" in optval and not optval.strip().startswith('"'): pos = optval.find(";") if pos != -1 and optval[pos - 1].isspace(): optval = optval[:pos] optval = optval.strip() - if optval == '""': - optval = "" - # END handle empty string - optname = self.optionxform(optname.rstrip()) - if len(optval) > 1 and optval[0] == '"' and optval[-1] != '"': + + if len(optval) < 2 or optval[0] != '"': + # Does not open quoting. + pass + elif optval[-1] != '"': + # Opens quoting and does not close: appears to start multi-line quoting. is_multi_line = True optval = string_decode(optval[1:]) - # END handle multi-line + elif optval.find("\\", 1, -1) == -1 and optval.find('"', 1, -1) == -1: + # Opens and closes quoting. Single line, and all we need is quote removal. + optval = optval[1:-1] + # TODO: Handle other quoted content, especially well-formed backslash escapes. + # Preserves multiple values for duplicate optnames. cursect.add(optname, optval) else: @@ -567,11 +574,11 @@ def _included_paths(self) -> List[Tuple[str, str]]: if keyword.endswith("/i"): value = re.sub( r"[a-zA-Z]", - lambda m: "[{}{}]".format(m.group().lower(), m.group().upper()), + lambda m: f"[{m.group().lower()!r}{m.group().upper()!r}]", value, ) if self._repo.git_dir: - if fnmatch.fnmatchcase(str(self._repo.git_dir), value): + if fnmatch.fnmatchcase(os.fspath(self._repo.git_dir), value): paths += self.items(section) elif keyword == "onbranch": @@ -583,7 +590,11 @@ def _included_paths(self) -> List[Tuple[str, str]]: if fnmatch.fnmatchcase(branch_name, value): paths += self.items(section) - + elif keyword == "hasconfig:remote.*.url": + for remote in self._repo.remotes: + if fnmatch.fnmatchcase(remote.url, value): + paths += self.items(section) + break return paths def read(self) -> None: # type: ignore[override] @@ -622,8 +633,6 @@ def read(self) -> None: # type: ignore[override] file_path = cast(IO[bytes], file_path) self._read(file_path, file_path.name) else: - # Assume a path if it is not a file-object. - file_path = cast(PathLike, file_path) try: with open(file_path, "rb") as fp: file_ok = True @@ -757,7 +766,7 @@ def _assure_writable(self, method_name: str) -> None: if self.read_only: raise IOError("Cannot execute non-constant method %s.%s" % (self, method_name)) - def add_section(self, section: str) -> None: + def add_section(self, section: "cp._SectionName") -> None: """Assures added options will stay in order.""" return super().add_section(section) diff --git a/git/diff.py b/git/diff.py index 9c6ae59e0..23cb5675e 100644 --- a/git/diff.py +++ b/git/diff.py @@ -23,13 +23,14 @@ List, Match, Optional, + Sequence, Tuple, TYPE_CHECKING, TypeVar, Union, cast, ) -from git.types import Literal, PathLike +from git.types import PathLike, Literal if TYPE_CHECKING: from subprocess import Popen @@ -289,7 +290,7 @@ class DiffIndex(List[T_Diff]): The class improves the diff handling convenience. """ - change_type = ("A", "C", "D", "R", "M", "T") + change_type: Sequence[Literal["A", "C", "D", "R", "M", "T"]] = ("A", "C", "D", "R", "M", "T") # noqa: F821 """Change type invariant identifying possible ways a blob can have changed: * ``A`` = Added diff --git a/git/ext/gitdb b/git/ext/gitdb index 775cfe829..4c63ee663 160000 --- a/git/ext/gitdb +++ b/git/ext/gitdb @@ -1 +1 @@ -Subproject commit 775cfe8299ea5474f605935469359a9d1cdb49dc +Subproject commit 4c63ee6636a6a3370f58b05d0bd19fec2f16dd5a diff --git a/git/index/base.py b/git/index/base.py index 39cc9143c..93de7933c 100644 --- a/git/index/base.py +++ b/git/index/base.py @@ -407,7 +407,7 @@ def raise_exc(e: Exception) -> NoReturn: r = str(self.repo.working_tree_dir) rs = r + os.sep for path in paths: - abs_path = str(path) + abs_path = os.fspath(path) if not osp.isabs(abs_path): abs_path = osp.join(r, path) # END make absolute path @@ -508,7 +508,7 @@ def iter_blobs( :param predicate: Function(t) returning ``True`` if tuple(stage, Blob) should be yielded by - the iterator. A default filter, the `~git.index.typ.BlobFilter`, allows you + the iterator. A default filter, the :class:`~git.index.typ.BlobFilter`, allows you to yield blobs only if they match a given list of paths. """ for entry in self.entries.values(): @@ -530,7 +530,10 @@ def unmerged_blobs(self) -> Dict[PathLike, List[Tuple[StageType, Blob]]]: stage. That is, a file removed on the 'other' branch whose entries are at stage 3 will not have a stage 3 entry. """ - is_unmerged_blob = lambda t: t[0] != 0 + + def is_unmerged_blob(t: Tuple[StageType, Blob]) -> bool: + return t[0] != 0 + path_map: Dict[PathLike, List[Tuple[StageType, Blob]]] = {} for stage, blob in self.iter_blobs(is_unmerged_blob): path_map.setdefault(blob.path, []).append((stage, blob)) @@ -653,9 +656,12 @@ def _to_relative_path(self, path: PathLike) -> PathLike: return path if self.repo.bare: raise InvalidGitRepositoryError("require non-bare repository") - if not osp.normpath(str(path)).startswith(str(self.repo.working_tree_dir)): + if not osp.normpath(path).startswith(str(self.repo.working_tree_dir)): raise ValueError("Absolute path %r is not in git repository at %r" % (path, self.repo.working_tree_dir)) - return os.path.relpath(path, self.repo.working_tree_dir) + result = os.path.relpath(path, self.repo.working_tree_dir) + if os.fspath(path).endswith(os.sep) and not result.endswith(os.sep): + result += os.sep + return result def _preprocess_add_items( self, items: Union[PathLike, Sequence[Union[PathLike, Blob, BaseIndexEntry, "Submodule"]]] @@ -687,12 +693,17 @@ def _store_path(self, filepath: PathLike, fprogress: Callable) -> BaseIndexEntry This must be ensured in the calling code. """ st = os.lstat(filepath) # Handles non-symlinks as well. + if S_ISLNK(st.st_mode): # In PY3, readlink is a string, but we need bytes. # In PY2, it was just OS encoded bytes, we assumed UTF-8. - open_stream: Callable[[], BinaryIO] = lambda: BytesIO(force_bytes(os.readlink(filepath), encoding=defenc)) + def open_stream() -> BinaryIO: + return BytesIO(force_bytes(os.readlink(filepath), encoding=defenc)) else: - open_stream = lambda: open(filepath, "rb") + + def open_stream() -> BinaryIO: + return open(filepath, "rb") + with open_stream() as stream: fprogress(filepath, False, filepath) istream = self.repo.odb.store(IStream(Blob.type, st.st_size, stream)) @@ -767,7 +778,7 @@ def add( - path string Strings denote a relative or absolute path into the repository pointing - to an existing file, e.g., ``CHANGES``, `lib/myfile.ext``, + to an existing file, e.g., ``CHANGES``, ``lib/myfile.ext``, ``/home/gitrepo/lib/myfile.ext``. Absolute paths must start with working tree directory of this index's @@ -786,7 +797,7 @@ def add( They are added at stage 0. - - :class:~`git.objects.blob.Blob` or + - :class:`~git.objects.blob.Blob` or :class:`~git.objects.submodule.base.Submodule` object Blobs are added as they are assuming a valid mode is set. @@ -812,7 +823,7 @@ def add( - :class:`~git.index.typ.BaseIndexEntry` or type - Handling equals the one of :class:~`git.objects.blob.Blob` objects, but + Handling equals the one of :class:`~git.objects.blob.Blob` objects, but the stage may be explicitly set. Please note that Index Entries require binary sha's. @@ -995,7 +1006,7 @@ def remove( The path string may include globs, such as ``*.c``. - - :class:~`git.objects.blob.Blob` object + - :class:`~git.objects.blob.Blob` object Only the path portion is used in this case. @@ -1025,7 +1036,7 @@ def remove( args.append("--") # Preprocess paths. - paths = self._items_to_rela_paths(items) + paths = list(map(os.fspath, self._items_to_rela_paths(items))) # type: ignore[arg-type] removed_paths = self.repo.git.rm(args, paths, **kwargs).splitlines() # Process output to gain proper paths. @@ -1333,8 +1344,11 @@ def handle_stderr(proc: "Popen[bytes]", iter_checked_out_files: Iterable[PathLik kwargs["as_process"] = True kwargs["istream"] = subprocess.PIPE proc = self.repo.git.checkout_index(args, **kwargs) + # FIXME: Reading from GIL! - make_exc = lambda: GitCommandError(("git-checkout-index",) + tuple(args), 128, proc.stderr.read()) + def make_exc() -> GitCommandError: + return GitCommandError(("git-checkout-index", *args), 128, proc.stderr.read()) + checked_out_files: List[PathLike] = [] for path in paths: @@ -1345,11 +1359,11 @@ def handle_stderr(proc: "Popen[bytes]", iter_checked_out_files: Iterable[PathLik try: self.entries[(co_path, 0)] except KeyError: - folder = str(co_path) + folder = co_path if not folder.endswith("/"): folder += "/" for entry in self.entries.values(): - if str(entry.path).startswith(folder): + if os.fspath(entry.path).startswith(folder): p = entry.path self._write_path_to_stdin(proc, p, p, make_exc, fprogress, read_from_stdout=False) checked_out_files.append(p) diff --git a/git/index/fun.py b/git/index/fun.py index 59cce6ae6..629c19b1e 100644 --- a/git/index/fun.py +++ b/git/index/fun.py @@ -36,7 +36,7 @@ ) from git.util import IndexFileSHA1Writer, finalize_process -from .typ import BaseIndexEntry, IndexEntry, CE_NAMEMASK, CE_STAGESHIFT +from .typ import CE_EXTENDED, BaseIndexEntry, IndexEntry, CE_NAMEMASK, CE_STAGESHIFT from .util import pack, unpack # typing ----------------------------------------------------------------------------- @@ -87,7 +87,7 @@ def run_commit_hook(name: str, index: "IndexFile", *args: str) -> None: return env = os.environ.copy() - env["GIT_INDEX_FILE"] = safe_decode(str(index.path)) + env["GIT_INDEX_FILE"] = safe_decode(os.fspath(index.path)) env["GIT_EDITOR"] = ":" cmd = [hp] try: @@ -158,7 +158,7 @@ def write_cache( write = stream_sha.write # Header - version = 2 + version = 3 if any(entry.extended_flags for entry in entries) else 2 write(b"DIRC") write(pack(">LL", version, len(entries))) @@ -172,6 +172,8 @@ def write_cache( plen = len(path) & CE_NAMEMASK # Path length assert plen == len(path), "Path %s too long to fit into index" % entry.path flags = plen | (entry.flags & CE_NAMEMASK_INV) # Clear possible previous values. + if entry.extended_flags: + flags |= CE_EXTENDED write( pack( ">LLLLLL20sH", @@ -185,6 +187,8 @@ def write_cache( flags, ) ) + if entry.extended_flags: + write(pack(">H", entry.extended_flags)) write(path) real_size = (tell() - beginoffset + 8) & ~7 write(b"\0" * ((beginoffset + real_size) - tell())) @@ -206,8 +210,7 @@ def read_header(stream: IO[bytes]) -> Tuple[int, int]: unpacked = cast(Tuple[int, int], unpack(">LL", stream.read(4 * 2))) version, num_entries = unpacked - # TODO: Handle version 3: extended data, see read-cache.c. - assert version in (1, 2) + assert version in (1, 2, 3), "Unsupported git index version %i, only 1, 2, and 3 are supported" % version return version, num_entries @@ -260,12 +263,15 @@ def read_cache( ctime = unpack(">8s", read(8))[0] mtime = unpack(">8s", read(8))[0] (dev, ino, mode, uid, gid, size, sha, flags) = unpack(">LLLLLL20sH", read(20 + 4 * 6 + 2)) + extended_flags = 0 + if flags & CE_EXTENDED: + extended_flags = unpack(">H", read(2))[0] path_size = flags & CE_NAMEMASK path = read(path_size).decode(defenc) real_size = (tell() - beginoffset + 8) & ~7 read((beginoffset + real_size) - tell()) - entry = IndexEntry((mode, sha, flags, path, ctime, mtime, dev, ino, uid, gid, size)) + entry = IndexEntry((mode, sha, flags, path, ctime, mtime, dev, ino, uid, gid, size, extended_flags)) # entry_key would be the method to use, but we save the effort. entries[(path, entry.stage)] = entry count += 1 diff --git a/git/index/typ.py b/git/index/typ.py index 974252528..927633a9f 100644 --- a/git/index/typ.py +++ b/git/index/typ.py @@ -32,6 +32,9 @@ CE_VALID = 0x8000 CE_STAGESHIFT = 12 +CE_EXT_SKIP_WORKTREE = 0x4000 +CE_EXT_INTENT_TO_ADD = 0x2000 + # } END invariants @@ -87,6 +90,8 @@ class BaseIndexEntryHelper(NamedTuple): uid: int = 0 gid: int = 0 size: int = 0 + # version 3 extended flags, only when (flags & CE_EXTENDED) is set + extended_flags: int = 0 class BaseIndexEntry(BaseIndexEntryHelper): @@ -102,7 +107,7 @@ def __new__( cls, inp_tuple: Union[ Tuple[int, bytes, int, PathLike], - Tuple[int, bytes, int, PathLike, bytes, bytes, int, int, int, int, int], + Tuple[int, bytes, int, PathLike, bytes, bytes, int, int, int, int, int, int], ], ) -> "BaseIndexEntry": """Override ``__new__`` to allow construction from a tuple for backwards @@ -134,6 +139,14 @@ def stage(self) -> int: """ return (self.flags & CE_STAGEMASK) >> CE_STAGESHIFT + @property + def skip_worktree(self) -> bool: + return (self.extended_flags & CE_EXT_SKIP_WORKTREE) > 0 + + @property + def intent_to_add(self) -> bool: + return (self.extended_flags & CE_EXT_INTENT_TO_ADD) > 0 + @classmethod def from_blob(cls, blob: Blob, stage: int = 0) -> "BaseIndexEntry": """:return: Fully equipped BaseIndexEntry at the given stage""" @@ -179,7 +192,7 @@ def from_base(cls, base: "BaseIndexEntry") -> "IndexEntry": Instance of type :class:`BaseIndexEntry`. """ time = pack(">LL", 0, 0) - return IndexEntry((base.mode, base.binsha, base.flags, base.path, time, time, 0, 0, 0, 0, 0)) + return IndexEntry((base.mode, base.binsha, base.flags, base.path, time, time, 0, 0, 0, 0, 0)) # type: ignore[arg-type] @classmethod def from_blob(cls, blob: Blob, stage: int = 0) -> "IndexEntry": @@ -198,5 +211,5 @@ def from_blob(cls, blob: Blob, stage: int = 0) -> "IndexEntry": 0, 0, blob.size, - ) + ) # type: ignore[arg-type] ) diff --git a/git/index/util.py b/git/index/util.py index e59cb609f..982a5afb7 100644 --- a/git/index/util.py +++ b/git/index/util.py @@ -15,7 +15,7 @@ # typing ---------------------------------------------------------------------- -from typing import Any, Callable, TYPE_CHECKING, Optional, Type +from typing import Any, Callable, TYPE_CHECKING, Optional, Type, cast from git.types import Literal, PathLike, _T @@ -106,7 +106,7 @@ def git_working_dir(func: Callable[..., _T]) -> Callable[..., _T]: @wraps(func) def set_git_working_dir(self: "IndexFile", *args: Any, **kwargs: Any) -> _T: cur_wd = os.getcwd() - os.chdir(str(self.repo.working_tree_dir)) + os.chdir(cast(PathLike, self.repo.working_tree_dir)) try: return func(self, *args, **kwargs) finally: diff --git a/git/objects/base.py b/git/objects/base.py index eeaebc09b..faf600c6b 100644 --- a/git/objects/base.py +++ b/git/objects/base.py @@ -122,7 +122,7 @@ def new(cls, repo: "Repo", id: Union[str, "Reference"]) -> AnyGitObject: :return: New :class:`Object` instance of a type appropriate to the object type behind `id`. The id of the newly created object will be a binsha even though the - input id may have been a `~git.refs.reference.Reference` or rev-spec. + input id may have been a :class:`~git.refs.reference.Reference` or rev-spec. :param id: :class:`~git.refs.reference.Reference`, rev-spec, or hexsha. @@ -218,7 +218,7 @@ class IndexObject(Object): """Base for all objects that can be part of the index file. The classes representing git object types that can be part of the index file are - :class:`~git.objects.tree.Tree and :class:`~git.objects.blob.Blob`. In addition, + :class:`~git.objects.tree.Tree` and :class:`~git.objects.blob.Blob`. In addition, :class:`~git.objects.submodule.base.Submodule`, which is not really a git object type but can be part of an index file, is also a subclass. """ diff --git a/git/objects/blob.py b/git/objects/blob.py index 58de59642..f7d49c9cc 100644 --- a/git/objects/blob.py +++ b/git/objects/blob.py @@ -6,6 +6,7 @@ __all__ = ["Blob"] from mimetypes import guess_type +import os import sys if sys.version_info >= (3, 8): @@ -44,5 +45,5 @@ def mime_type(self) -> str: """ guesses = None if self.path: - guesses = guess_type(str(self.path)) + guesses = guess_type(os.fspath(self.path)) return guesses and guesses[0] or self.DEFAULT_MIME_TYPE diff --git a/git/objects/commit.py b/git/objects/commit.py index 0ceb46609..8c51254a2 100644 --- a/git/objects/commit.py +++ b/git/objects/commit.py @@ -289,7 +289,7 @@ def name_rev(self) -> str: """ :return: String describing the commits hex sha based on the closest - `~git.refs.reference.Reference`. + :class:`~git.refs.reference.Reference`. :note: Mostly useful for UI purposes. @@ -349,7 +349,7 @@ def iter_items( return cls._iter_from_process_or_stream(repo, proc) def iter_parents(self, paths: Union[PathLike, Sequence[PathLike]] = "", **kwargs: Any) -> Iterator["Commit"]: - R"""Iterate _all_ parents of this commit. + R"""Iterate *all* parents of this commit. :param paths: Optional path or list of paths limiting the :class:`Commit`\s to those that @@ -900,7 +900,7 @@ def co_authors(self) -> List[Actor]: if self.message: results = re.findall( r"^Co-authored-by: (.*) <(.*?)>$", - self.message, + str(self.message), re.MULTILINE, ) for author in results: diff --git a/git/objects/submodule/base.py b/git/objects/submodule/base.py index fa60bcdaf..d183672db 100644 --- a/git/objects/submodule/base.py +++ b/git/objects/submodule/base.py @@ -11,6 +11,7 @@ import stat import sys import uuid +import urllib import git from git.cmd import Git @@ -65,7 +66,7 @@ if TYPE_CHECKING: from git.index import IndexFile from git.objects.commit import Commit - from git.refs import Head + from git.refs import Head, RemoteReference from git.repo import Repo # ----------------------------------------------------------------------------- @@ -351,7 +352,12 @@ def _clone_repo( module_abspath_dir = osp.dirname(module_abspath) if not osp.isdir(module_abspath_dir): os.makedirs(module_abspath_dir) - module_checkout_path = osp.join(str(repo.working_tree_dir), path) + module_checkout_path = osp.join(repo.working_tree_dir, path) # type: ignore[arg-type] + + if url.startswith("../"): + remote_name = cast("RemoteReference", repo.active_branch.tracking_branch()).remote_name + repo_remote_url = repo.remote(remote_name).url + url = os.path.join(repo_remote_url, url) clone = git.Repo.clone_from( url, @@ -535,7 +541,7 @@ def add( if sm.exists(): # Reretrieve submodule from tree. try: - sm = repo.head.commit.tree[str(path)] + sm = repo.head.commit.tree[os.fspath(path)] sm._name = name return sm except KeyError: @@ -794,9 +800,13 @@ def update( + "Cloning url '%s' to '%s' in submodule %r" % (self.url, checkout_module_abspath, self.name), ) if not dry_run: + if self.url.startswith("."): + url = urllib.parse.urljoin(self.repo.remotes.origin.url + "/", self.url) + else: + url = self.url mrepo = self._clone_repo( self.repo, - self.url, + url, self.path, self.name, n=True, diff --git a/git/objects/tree.py b/git/objects/tree.py index 09184a781..a3d611c80 100644 --- a/git/objects/tree.py +++ b/git/objects/tree.py @@ -5,6 +5,7 @@ __all__ = ["TreeModifier", "Tree"] +import os import sys import git.diff as git_diff @@ -50,7 +51,9 @@ # -------------------------------------------------------- -cmp: Callable[[str, str], int] = lambda a, b: (a > b) - (a < b) + +def cmp(a: str, b: str) -> int: + return (a > b) - (a < b) class TreeModifier: @@ -228,7 +231,7 @@ def _iter_convert_to_object(self, iterable: Iterable[TreeCacheTup]) -> Iterator[ raise TypeError("Unknown mode %o found in tree data for path '%s'" % (mode, path)) from e # END for each item - def join(self, file: str) -> IndexObjUnion: + def join(self, file: PathLike) -> IndexObjUnion: """Find the named object in this tree's contents. :return: @@ -239,6 +242,7 @@ def join(self, file: str) -> IndexObjUnion: If the given file or tree does not exist in this tree. """ msg = "Blob or Tree named %r not found" + file = os.fspath(file) if "/" in file: tree = self item = self @@ -267,7 +271,7 @@ def join(self, file: str) -> IndexObjUnion: raise KeyError(msg % file) # END handle long paths - def __truediv__(self, file: str) -> IndexObjUnion: + def __truediv__(self, file: PathLike) -> IndexObjUnion: """The ``/`` operator is another syntax for joining. See :meth:`join` for details. diff --git a/git/refs/head.py b/git/refs/head.py index 683634451..3c43993e7 100644 --- a/git/refs/head.py +++ b/git/refs/head.py @@ -22,7 +22,6 @@ from git.types import Commit_ish, PathLike if TYPE_CHECKING: - from git.objects import Commit from git.refs import RemoteReference from git.repo import Repo @@ -44,9 +43,6 @@ class HEAD(SymbolicReference): __slots__ = () - # TODO: This can be removed once SymbolicReference.commit has static type hints. - commit: "Commit" - def __init__(self, repo: "Repo", path: PathLike = _HEAD_NAME) -> None: if path != self._HEAD_NAME: raise ValueError("HEAD instance must point to %r, got %r" % (self._HEAD_NAME, path)) @@ -149,7 +145,7 @@ class Head(Reference): k_config_remote_ref = "merge" # Branch to merge from remote. @classmethod - def delete(cls, repo: "Repo", *heads: "Union[Head, str]", force: bool = False, **kwargs: Any) -> None: + def delete(cls, repo: "Repo", *heads: "Union[Head, str]", force: bool = False, **kwargs: Any) -> None: # type: ignore[override] """Delete the given heads. :param force: diff --git a/git/refs/log.py b/git/refs/log.py index 17e3a94b3..4751cff99 100644 --- a/git/refs/log.py +++ b/git/refs/log.py @@ -126,7 +126,7 @@ def from_line(cls, line: bytes) -> "RefLogEntry": elif len(fields) == 2: info, msg = fields else: - raise ValueError("Line must have up to two TAB-separated fields." " Got %s" % repr(line_str)) + raise ValueError("Line must have up to two TAB-separated fields. Got %s" % repr(line_str)) # END handle first split oldhexsha = info[:40] @@ -145,7 +145,7 @@ def from_line(cls, line: bytes) -> "RefLogEntry": actor = Actor._from_string(info[82 : email_end + 1]) time, tz_offset = parse_date(info[email_end + 2 :]) # skipcq: PYL-W0621 - return RefLogEntry((oldhexsha, newhexsha, actor, (time, tz_offset), msg)) + return RefLogEntry((oldhexsha, newhexsha, actor, (time, tz_offset), msg)) # type: ignore [arg-type] class RefLog(List[RefLogEntry], Serializable): diff --git a/git/refs/reference.py b/git/refs/reference.py index e5d473779..0c4327225 100644 --- a/git/refs/reference.py +++ b/git/refs/reference.py @@ -3,6 +3,7 @@ __all__ = ["Reference"] +import os from git.util import IterableObj, LazyMixin from .symbolic import SymbolicReference, T_References @@ -65,7 +66,7 @@ def __init__(self, repo: "Repo", path: PathLike, check_path: bool = True) -> Non If ``False``, you can provide any path. Otherwise the path must start with the default path prefix of this type. """ - if check_path and not str(path).startswith(self._common_path_default + "/"): + if check_path and not os.fspath(path).startswith(self._common_path_default + "/"): raise ValueError(f"Cannot instantiate {self.__class__.__name__!r} from path {path}") self.path: str # SymbolicReference converts to string at the moment. super().__init__(repo, path) diff --git a/git/refs/symbolic.py b/git/refs/symbolic.py index 510850b2e..99af4f57c 100644 --- a/git/refs/symbolic.py +++ b/git/refs/symbolic.py @@ -4,6 +4,7 @@ __all__ = ["SymbolicReference"] import os +from pathlib import Path from gitdb.exc import BadName, BadObject @@ -39,8 +40,8 @@ if TYPE_CHECKING: from git.config import GitConfigParser from git.objects.commit import Actor - from git.refs import Head, TagReference, RemoteReference, Reference from git.refs.log import RefLogEntry + from git.refs.reference import Reference from git.repo import Repo @@ -76,10 +77,10 @@ class SymbolicReference: def __init__(self, repo: "Repo", path: PathLike, check_path: bool = False) -> None: self.repo = repo - self.path = path + self.path: PathLike = path def __str__(self) -> str: - return str(self.path) + return os.fspath(self.path) def __repr__(self) -> str: return '' % (self.__class__.__name__, self.path) @@ -103,7 +104,7 @@ def name(self) -> str: In case of symbolic references, the shortest assumable name is the path itself. """ - return str(self.path) + return os.fspath(self.path) @property def abspath(self) -> PathLike: @@ -178,7 +179,7 @@ def _check_ref_name_valid(ref_path: PathLike) -> None: """ previous: Union[str, None] = None one_before_previous: Union[str, None] = None - for c in str(ref_path): + for c in os.fspath(ref_path): if c in " ~^:?*[\\": raise ValueError( f"Invalid reference '{ref_path}': references cannot contain spaces, tildes (~), carets (^)," @@ -212,7 +213,7 @@ def _check_ref_name_valid(ref_path: PathLike) -> None: raise ValueError(f"Invalid reference '{ref_path}': references cannot end with a forward slash (/)") elif previous == "@" and one_before_previous is None: raise ValueError(f"Invalid reference '{ref_path}': references cannot be '@'") - elif any(component.endswith(".lock") for component in str(ref_path).split("/")): + elif any(component.endswith(".lock") for component in Path(ref_path).parts): raise ValueError( f"Invalid reference '{ref_path}': references cannot have slash-separated components that end with" " '.lock'" @@ -235,7 +236,7 @@ def _get_ref_info_helper( tokens: Union[None, List[str], Tuple[str, str]] = None repodir = _git_dir(repo, ref_path) try: - with open(os.path.join(repodir, str(ref_path)), "rt", encoding="UTF-8") as fp: + with open(os.path.join(repodir, ref_path), "rt", encoding="UTF-8") as fp: # type: ignore[arg-type] value = fp.read().rstrip() # Don't only split on spaces, but on whitespace, which allows to parse lines like: # 60b64ef992065e2600bfef6187a97f92398a9144 branch 'master' of git-server:/path/to/repo @@ -387,19 +388,25 @@ def set_object( # set the commit on our reference return self._get_reference().set_object(object, logmsg) - commit = property( - _get_commit, - set_commit, # type: ignore[arg-type] - doc="Query or set commits directly", - ) + @property + def commit(self) -> "Commit": + """Query or set commits directly""" + return self._get_commit() + + @commit.setter + def commit(self, commit: Union[Commit, "SymbolicReference", str]) -> "SymbolicReference": + return self.set_commit(commit) + + @property + def object(self) -> AnyGitObject: + """Return the object our ref currently refers to""" + return self._get_object() - object = property( - _get_object, - set_object, # type: ignore[arg-type] - doc="Return the object our ref currently refers to", - ) + @object.setter + def object(self, object: Union[AnyGitObject, "SymbolicReference", str]) -> "SymbolicReference": + return self.set_object(object) - def _get_reference(self) -> "SymbolicReference": + def _get_reference(self) -> "Reference": """ :return: :class:`~git.refs.reference.Reference` object we point to @@ -411,7 +418,7 @@ def _get_reference(self) -> "SymbolicReference": sha, target_ref_path = self._get_ref_info(self.repo, self.path) if target_ref_path is None: raise TypeError("%s is a detached symbolic reference as it points to %r" % (self, sha)) - return self.from_path(self.repo, target_ref_path) + return cast("Reference", self.from_path(self.repo, target_ref_path)) def set_reference( self, @@ -496,12 +503,14 @@ def set_reference( return self # Aliased reference - reference: Union["Head", "TagReference", "RemoteReference", "Reference"] - reference = property( # type: ignore[assignment] - _get_reference, - set_reference, # type: ignore[arg-type] - doc="Returns the Reference we point to", - ) + @property + def reference(self) -> "Reference": + return self._get_reference() + + @reference.setter + def reference(self, ref: Union[AnyGitObject, "SymbolicReference", str]) -> "SymbolicReference": + return self.set_reference(ref) + ref = reference def is_valid(self) -> bool: @@ -606,7 +615,7 @@ def to_full_path(cls, path: Union[PathLike, "SymbolicReference"]) -> PathLike: full_ref_path = path if not cls._common_path_default: return full_ref_path - if not str(path).startswith(cls._common_path_default + "/"): + if not os.fspath(path).startswith(cls._common_path_default + "/"): full_ref_path = "%s/%s" % (cls._common_path_default, path) return full_ref_path @@ -698,7 +707,7 @@ def _create( if not force and os.path.isfile(abs_ref_path): target_data = str(target) if isinstance(target, SymbolicReference): - target_data = str(target.path) + target_data = os.fspath(target.path) if not resolve: target_data = "ref: " + target_data with open(abs_ref_path, "rb") as fd: @@ -834,7 +843,7 @@ def _iter_items( # Read packed refs. for _sha, rela_path in cls._iter_packed_refs(repo): - if rela_path.startswith(str(common_path)): + if rela_path.startswith(os.fspath(common_path)): rela_paths.add(rela_path) # END relative path matches common path # END packed refs reading @@ -908,8 +917,7 @@ def from_path(cls: Type[T_References], repo: "Repo", path: PathLike) -> T_Refere SymbolicReference, ): try: - instance: T_References - instance = ref_type(repo, path) + instance = cast(T_References, ref_type(repo, path)) if instance.__class__ is SymbolicReference and instance.is_detached: raise ValueError("SymbolicRef was detached, we drop it") else: @@ -923,4 +931,4 @@ def from_path(cls: Type[T_References], repo: "Repo", path: PathLike) -> T_Refere def is_remote(self) -> bool: """:return: True if this symbolic reference points to a remote branch""" - return str(self.path).startswith(self._remote_common_path_default + "/") + return os.fspath(self.path).startswith(self._remote_common_path_default + "/") diff --git a/git/refs/tag.py b/git/refs/tag.py index 1e38663ae..4525b09cb 100644 --- a/git/refs/tag.py +++ b/git/refs/tag.py @@ -45,8 +45,8 @@ class TagReference(Reference): _common_default = "tags" _common_path_default = Reference._common_path_default + "/" + _common_default - @property - def commit(self) -> "Commit": # type: ignore[override] # LazyMixin has unrelated commit method + @property # type: ignore[misc] + def commit(self) -> "Commit": # LazyMixin has unrelated commit method """:return: Commit object the tag ref points to :raise ValueError: @@ -80,8 +80,8 @@ def tag(self) -> Union["TagObject", None]: return None # Make object read-only. It should be reasonably hard to adjust an existing tag. - @property - def object(self) -> AnyGitObject: # type: ignore[override] + @property # type: ignore[misc] + def object(self) -> AnyGitObject: return Reference._get_object(self) @classmethod diff --git a/git/repo/base.py b/git/repo/base.py index db89cdf41..1f543cc57 100644 --- a/git/repo/base.py +++ b/git/repo/base.py @@ -126,6 +126,7 @@ class Repo: working_dir: PathLike """The working directory of the git command.""" + # stored as string for easier processing, but annotated as path for clearer intention _working_tree_dir: Optional[PathLike] = None git_dir: PathLike @@ -215,15 +216,13 @@ def __init__( epath = path or os.getenv("GIT_DIR") if not epath: epath = os.getcwd() + epath = os.fspath(epath) if Git.is_cygwin(): # Given how the tests are written, this seems more likely to catch Cygwin # git used from Windows than Windows git used from Cygwin. Therefore # changing to Cygwin-style paths is the relevant operation. - epath = cygpath(str(epath)) + epath = cygpath(epath) - epath = epath or path or os.getcwd() - if not isinstance(epath, str): - epath = str(epath) if expand_vars and re.search(self.re_envvars, epath): warnings.warn( "The use of environment variables in paths is deprecated" @@ -354,21 +353,19 @@ def __ne__(self, rhs: object) -> bool: def __hash__(self) -> int: return hash(self.git_dir) - # Description property - def _get_description(self) -> str: + @property + def description(self) -> str: + """The project's description""" filename = osp.join(self.git_dir, "description") with open(filename, "rb") as fp: return fp.read().rstrip().decode(defenc) - def _set_description(self, descr: str) -> None: + @description.setter + def description(self, descr: str) -> None: filename = osp.join(self.git_dir, "description") with open(filename, "wb") as fp: fp.write((descr + "\n").encode(defenc)) - description = property(_get_description, _set_description, doc="the project's description") - del _get_description - del _set_description - @property def working_tree_dir(self) -> Optional[PathLike]: """ @@ -514,7 +511,7 @@ def create_submodule(self, *args: Any, **kwargs: Any) -> Submodule: def iter_submodules(self, *args: Any, **kwargs: Any) -> Iterator[Submodule]: """An iterator yielding Submodule instances. - See the `~git.objects.util.Traversable` interface for a description of `args` + See the :class:`~git.objects.util.Traversable` interface for a description of `args` and `kwargs`. :return: @@ -522,7 +519,7 @@ def iter_submodules(self, *args: Any, **kwargs: Any) -> Iterator[Submodule]: """ return RootModule(self).traverse(*args, **kwargs) - def submodule_update(self, *args: Any, **kwargs: Any) -> Iterator[Submodule]: + def submodule_update(self, *args: Any, **kwargs: Any) -> RootModule: """Update the submodules, keeping the repository consistent as it will take the previous state into consideration. @@ -686,11 +683,7 @@ def _config_reader( git_dir: Optional[PathLike] = None, ) -> GitConfigParser: if config_level is None: - files = [ - self._get_config_path(cast(Lit_config_levels, f), git_dir) - for f in self.config_level - if cast(Lit_config_levels, f) - ] + files = [self._get_config_path(f, git_dir) for f in self.config_level if f] else: files = [self._get_config_path(config_level, git_dir)] return GitConfigParser(files, read_only=True, repo=self) @@ -885,13 +878,14 @@ def _set_daemon_export(self, value: object) -> None: elif not value and fileexists: os.unlink(filename) - daemon_export = property( - _get_daemon_export, - _set_daemon_export, - doc="If True, git-daemon may export this repository", - ) - del _get_daemon_export - del _set_daemon_export + @property + def daemon_export(self) -> bool: + """If True, git-daemon may export this repository""" + return self._get_daemon_export() + + @daemon_export.setter + def daemon_export(self, value: object) -> None: + self._set_daemon_export(value) def _get_alternates(self) -> List[str]: """The list of alternates for this repo from which objects can be retrieved. @@ -929,11 +923,14 @@ def _set_alternates(self, alts: List[str]) -> None: with open(alternates_path, "wb") as f: f.write("\n".join(alts).encode(defenc)) - alternates = property( - _get_alternates, - _set_alternates, - doc="Retrieve a list of alternates paths or set a list paths to be used as alternates", - ) + @property + def alternates(self) -> List[str]: + """Retrieve a list of alternates paths or set a list paths to be used as alternates""" + return self._get_alternates() + + @alternates.setter + def alternates(self, alts: List[str]) -> None: + self._set_alternates(alts) def is_dirty( self, @@ -959,7 +956,7 @@ def is_dirty( if not submodules: default_args.append("--ignore-submodules") if path: - default_args.extend(["--", str(path)]) + default_args.extend(["--", os.fspath(path)]) if index: # diff index against HEAD. if osp.isfile(self.index.path) and len(self.git.diff("--cached", *default_args)): @@ -1359,9 +1356,9 @@ def _clone( ) -> "Repo": odbt = kwargs.pop("odbt", odb_default_type) - # When pathlib.Path or other class-based path is passed - if not isinstance(path, str): - path = str(path) + # url may be a path and this has no effect if it is a string + url = os.fspath(url) + path = os.fspath(path) ## A bug win cygwin's Git, when `--bare` or `--separate-git-dir` # it prepends the cwd or(?) the `url` into the `path, so:: @@ -1378,7 +1375,7 @@ def _clone( multi = shlex.split(" ".join(multi_options)) if not allow_unsafe_protocols: - Git.check_unsafe_protocols(str(url)) + Git.check_unsafe_protocols(url) if not allow_unsafe_options: Git.check_unsafe_options(options=list(kwargs.keys()), unsafe_options=cls.unsafe_git_clone_options) if not allow_unsafe_options and multi_options: @@ -1387,7 +1384,7 @@ def _clone( proc = git.clone( multi, "--", - Git.polish_url(str(url)), + Git.polish_url(url), clone_path, with_extended_output=True, as_process=True, @@ -1482,7 +1479,7 @@ def clone( self.common_dir, path, type(self.odb), - progress, + progress, # type: ignore[arg-type] multi_options, allow_unsafe_protocols=allow_unsafe_protocols, allow_unsafe_options=allow_unsafe_options, @@ -1543,7 +1540,7 @@ def clone_from( url, to_path, GitCmdObjectDB, - progress, + progress, # type: ignore[arg-type] multi_options, allow_unsafe_protocols=allow_unsafe_protocols, allow_unsafe_options=allow_unsafe_options, diff --git a/git/repo/fun.py b/git/repo/fun.py index 182cf82ed..3f00e60ea 100644 --- a/git/repo/fun.py +++ b/git/repo/fun.py @@ -286,7 +286,7 @@ def rev_parse(repo: "Repo", rev: str) -> AnyGitObject: # END handle refname else: if ref is not None: - obj = cast("Commit", ref.commit) + obj = ref.commit # END handle ref # END initialize obj on first token @@ -301,7 +301,13 @@ def rev_parse(repo: "Repo", rev: str) -> AnyGitObject: # Handle type. if output_type == "commit": - pass # Default. + obj = cast("TagObject", obj) + if obj and obj.type == "tag": + obj = deref_tag(obj) + else: + # Cannot do anything for non-tags. + pass + # END handle tag elif output_type == "tree": try: obj = cast(AnyGitObject, obj) @@ -399,7 +405,7 @@ def rev_parse(repo: "Repo", rev: str) -> AnyGitObject: # END end handle tag except (IndexError, AttributeError) as e: raise BadName( - f"Invalid revision spec '{rev}' - not enough " f"parent commits to reach '{token}{int(num)}'" + f"Invalid revision spec '{rev}' - not enough parent commits to reach '{token}{int(num)}'" ) from e # END exception handling # END parse loop diff --git a/git/types.py b/git/types.py index cce184530..100fff43f 100644 --- a/git/types.py +++ b/git/types.py @@ -13,7 +13,6 @@ Sequence as Sequence, Tuple, TYPE_CHECKING, - Type, TypeVar, Union, ) @@ -130,7 +129,8 @@ https://git-scm.com/docs/gitglossary#def_object_type """ -Lit_commit_ish: Type[Literal["commit", "tag"]] +if TYPE_CHECKING: + Lit_commit_ish = Literal["commit", "tag"] """Deprecated. Type of literal strings identifying typically-commitish git object types. Prior to a bugfix, this type had been defined more broadly. Any usage is in practice diff --git a/git/util.py b/git/util.py index 9e8ac821d..c3ffdd62b 100644 --- a/git/util.py +++ b/git/util.py @@ -36,7 +36,7 @@ import logging import os import os.path as osp -import pathlib +from pathlib import Path import platform import re import shutil @@ -272,9 +272,9 @@ def stream_copy(source: BinaryIO, destination: BinaryIO, chunk_size: int = 512 * def join_path(a: PathLike, *p: PathLike) -> PathLike: R"""Join path tokens together similar to osp.join, but always use ``/`` instead of possibly ``\`` on Windows.""" - path = str(a) + path = os.fspath(a) for b in p: - b = str(b) + b = os.fspath(b) if not b: continue if b.startswith("/"): @@ -290,18 +290,18 @@ def join_path(a: PathLike, *p: PathLike) -> PathLike: if sys.platform == "win32": def to_native_path_windows(path: PathLike) -> PathLike: - path = str(path) + path = os.fspath(path) return path.replace("/", "\\") def to_native_path_linux(path: PathLike) -> str: - path = str(path) + path = os.fspath(path) return path.replace("\\", "/") to_native_path = to_native_path_windows else: # No need for any work on Linux. def to_native_path_linux(path: PathLike) -> str: - return str(path) + return os.fspath(path) to_native_path = to_native_path_linux @@ -372,7 +372,7 @@ def is_exec(fpath: str) -> bool: progs = [] if not path: path = os.environ["PATH"] - for folder in str(path).split(os.pathsep): + for folder in os.fspath(path).split(os.pathsep): folder = folder.strip('"') if folder: exe_path = osp.join(folder, program) @@ -397,7 +397,7 @@ def _cygexpath(drive: Optional[str], path: str) -> str: p = cygpath(p) elif drive: p = "/proc/cygdrive/%s/%s" % (drive.lower(), p) - p_str = str(p) # ensure it is a str and not AnyPath + p_str = os.fspath(p) # ensure it is a str and not AnyPath return p_str.replace("\\", "/") @@ -418,7 +418,7 @@ def _cygexpath(drive: Optional[str], path: str) -> str: def cygpath(path: str) -> str: """Use :meth:`git.cmd.Git.polish_url` instead, that works on any environment.""" - path = str(path) # Ensure is str and not AnyPath. + path = os.fspath(path) # Ensure is str and not AnyPath. # Fix to use Paths when 3.5 dropped. Or to be just str if only for URLs? if not path.startswith(("/cygdrive", "//", "/proc/cygdrive")): for regex, parser, recurse in _cygpath_parsers: @@ -438,7 +438,7 @@ def cygpath(path: str) -> str: def decygpath(path: PathLike) -> str: - path = str(path) + path = os.fspath(path) m = _decygpath_regex.match(path) if m: drive, rest_path = m.groups() @@ -464,6 +464,12 @@ def _is_cygwin_git(git_executable: str) -> bool: # Just a name given, not a real path. uname_cmd = osp.join(git_dir, "uname") + + if not (Path(uname_cmd).is_file() and os.access(uname_cmd, os.X_OK)): + _logger.debug(f"Failed checking if running in CYGWIN: {uname_cmd} is not an executable") + _is_cygwin_cache[git_executable] = is_cygwin + return is_cygwin + process = subprocess.Popen([uname_cmd], stdout=subprocess.PIPE, universal_newlines=True) uname_out, _ = process.communicate() # retcode = process.poll() @@ -484,7 +490,9 @@ def is_cygwin_git(git_executable: PathLike) -> bool: ... def is_cygwin_git(git_executable: Union[None, PathLike]) -> bool: - if sys.platform == "win32": # TODO: See if we can use `sys.platform != "cygwin"`. + # TODO: when py3.7 support is dropped, use the new interpolation f"{variable=}" + _logger.debug(f"sys.platform={sys.platform!r}, git_executable={git_executable!r}") + if sys.platform != "cygwin": return False elif git_executable is None: return False @@ -515,7 +523,7 @@ def expand_path(p: PathLike, expand_vars: bool = ...) -> str: def expand_path(p: Union[None, PathLike], expand_vars: bool = True) -> Optional[PathLike]: - if isinstance(p, pathlib.Path): + if isinstance(p, Path): return p.resolve() try: p = osp.expanduser(p) # type: ignore[arg-type] @@ -1135,7 +1143,7 @@ def _obtain_lock(self) -> None: # END endless loop -class IterableList(List[T_IterableObj]): +class IterableList(List[T_IterableObj]): # type: ignore[type-var] """List of iterable objects allowing to query an object by id or by named index:: heads = repo.heads @@ -1200,24 +1208,20 @@ def __getattr__(self, attr: str) -> T_IterableObj: return list.__getattribute__(self, attr) def __getitem__(self, index: Union[SupportsIndex, int, slice, str]) -> T_IterableObj: # type: ignore[override] - assert isinstance(index, (int, str, slice)), "Index of IterableList should be an int or str" - if isinstance(index, int): return list.__getitem__(self, index) elif isinstance(index, slice): raise ValueError("Index should be an int or str") else: try: - return getattr(self, index) + return getattr(self, cast(str, index)) except AttributeError as e: - raise IndexError("No item found with id %r" % (self._prefix + index)) from e + raise IndexError(f"No item found with id {self._prefix}{index}") from e # END handle getattr def __delitem__(self, index: Union[SupportsIndex, int, slice, str]) -> None: - assert isinstance(index, (int, str)), "Index of IterableList should be an int or str" - delindex = cast(int, index) - if not isinstance(index, int): + if isinstance(index, str): delindex = -1 name = self._prefix + index for i, item in enumerate(self): diff --git a/pyproject.toml b/pyproject.toml index 090972eed..149f2dc92 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,7 +19,6 @@ testpaths = "test" # Space separated list of paths from root e.g test tests doc # filterwarnings ignore::WarningType # ignores those warnings [tool.mypy] -python_version = "3.8" files = ["git/", "test/deprecation/"] disallow_untyped_defs = true no_implicit_optional = true @@ -60,16 +59,14 @@ lint.select = [ # "UP", # See: https://docs.astral.sh/ruff/rules/#pyupgrade-up ] lint.extend-select = [ - # "A", # See: https://pypi.org/project/flake8-builtins - "B", # See: https://pypi.org/project/flake8-bugbear - "C4", # See: https://pypi.org/project/flake8-comprehensions - "TCH004", # See: https://docs.astral.sh/ruff/rules/runtime-import-in-type-checking-block/ + # "A", # See: https://pypi.org/project/flake8-builtins + "B", # See: https://pypi.org/project/flake8-bugbear + "C4", # See: https://pypi.org/project/flake8-comprehensions + "TC004", # See: https://docs.astral.sh/ruff/rules/runtime-import-in-type-checking-block/ ] lint.ignore = [ - "E203", # Whitespace before ':' - "E731", # Do not assign a `lambda` expression, use a `def` + # If it becomes necessary to ignore any rules, list them here. ] -lint.ignore-init-module-imports = true lint.unfixable = [ "F401", # Module imported but unused ] diff --git a/requirements-dev.txt b/requirements-dev.txt index f626644af..066b192b8 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,8 +1,8 @@ --r requirements.txt --r test-requirements.txt - -# For additional local testing/linting - to be added elsewhere eventually. -ruff -shellcheck -pytest-icdiff -# pytest-profiling +-r requirements.txt +-r test-requirements.txt + +# For additional local testing/linting - to be added elsewhere eventually. +ruff >=0.8 +shellcheck +pytest-icdiff +# pytest-profiling diff --git a/requirements.txt b/requirements.txt index 7159416a9..61d8403b0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,2 @@ gitdb>=4.0.1,<5 -typing-extensions>=3.7.4.3;python_version<"3.8" +typing-extensions>=3.10.0.2;python_version<"3.10" diff --git a/setup.py b/setup.py index f28fedb85..a7b1eab00 100755 --- a/setup.py +++ b/setup.py @@ -95,7 +95,6 @@ def _stamp_version(filename: str) -> None: # "Development Status :: 7 - Inactive", "Environment :: Console", "Intended Audience :: Developers", - "License :: OSI Approved :: BSD License", "Operating System :: OS Independent", "Operating System :: POSIX", "Operating System :: Microsoft :: Windows", diff --git a/test-requirements.txt b/test-requirements.txt index 75e9e81fa..e6e01c683 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,7 +1,7 @@ coverage[toml] ddt >= 1.1.1, != 1.4.3 mock ; python_version < "3.8" -mypy +mypy==1.18.2 ; python_version >= "3.9" # pin mypy version to avoid new errors pre-commit pytest >= 7.3.1 pytest-cov diff --git a/test/deprecation/test_types.py b/test/deprecation/test_types.py index f97375a85..d3c6af645 100644 --- a/test/deprecation/test_types.py +++ b/test/deprecation/test_types.py @@ -36,7 +36,7 @@ def test_can_access_lit_commit_ish_but_it_is_not_usable() -> None: assert 'Literal["commit", "tag"]' in message, "Has new definition." assert "GitObjectTypeString" in message, "Has new type name for old definition." - _: Lit_commit_ish = "commit" # type: ignore[valid-type] + _: Lit_commit_ish = "commit" # It should be as documented (even though deliberately unusable in static checks). assert Lit_commit_ish == Literal["commit", "tag"] diff --git a/test/fixtures/git_config_with_empty_quotes b/test/fixtures/git_config_with_empty_quotes new file mode 100644 index 000000000..f11fe4248 --- /dev/null +++ b/test/fixtures/git_config_with_empty_quotes @@ -0,0 +1,2 @@ +[core] + filemode = "" diff --git a/test/fixtures/git_config_with_extra_whitespace b/test/fixtures/git_config_with_extra_whitespace new file mode 100644 index 000000000..0f727cb5d --- /dev/null +++ b/test/fixtures/git_config_with_extra_whitespace @@ -0,0 +1,2 @@ +[init] + defaultBranch = trunk diff --git a/test/fixtures/git_config_with_quotes b/test/fixtures/git_config_with_quotes new file mode 100644 index 000000000..40e6710d9 --- /dev/null +++ b/test/fixtures/git_config_with_quotes @@ -0,0 +1,3 @@ +[user] + name = "Cody Veal" + email = "cveal05@gmail.com" diff --git a/test/fixtures/git_config_with_quotes_escapes b/test/fixtures/git_config_with_quotes_escapes new file mode 100644 index 000000000..33332c221 --- /dev/null +++ b/test/fixtures/git_config_with_quotes_escapes @@ -0,0 +1,9 @@ +[custom] + hasnewline = "first\nsecond" + hasbackslash = "foo\\bar" + hasquote = "ab\"cd" + hastrailingbackslash = "word\\" + hasunrecognized = "p\qrs" + hasunescapedquotes = "ab"cd"e" + ordinary = "hello world" + unquoted = good evening diff --git a/test/fixtures/git_config_with_quotes_whitespace_inside b/test/fixtures/git_config_with_quotes_whitespace_inside new file mode 100644 index 000000000..c6014cc61 --- /dev/null +++ b/test/fixtures/git_config_with_quotes_whitespace_inside @@ -0,0 +1,2 @@ +[core] + commentString = "# " diff --git a/test/fixtures/git_config_with_quotes_whitespace_outside b/test/fixtures/git_config_with_quotes_whitespace_outside new file mode 100644 index 000000000..4b1615a51 --- /dev/null +++ b/test/fixtures/git_config_with_quotes_whitespace_outside @@ -0,0 +1,2 @@ +[init] + defaultBranch = "trunk" diff --git a/test/fixtures/index_extended_flags b/test/fixtures/index_extended_flags new file mode 100644 index 000000000..f03713b68 Binary files /dev/null and b/test/fixtures/index_extended_flags differ diff --git a/test/lib/helper.py b/test/lib/helper.py index 5d91447ea..6a8b714e6 100644 --- a/test/lib/helper.py +++ b/test/lib/helper.py @@ -10,6 +10,7 @@ "with_rw_directory", "with_rw_repo", "with_rw_and_rw_remote_repo", + "PathLikeMock", "TestBase", "VirtualEnvironment", "TestCase", @@ -20,6 +21,7 @@ ] import contextlib +from dataclasses import dataclass from functools import wraps import gc import io @@ -49,6 +51,15 @@ _logger = logging.getLogger(__name__) + +@dataclass +class PathLikeMock: + path: str + + def __fspath__(self) -> str: + return self.path + + # { Routines @@ -149,7 +160,7 @@ def repo_creator(self): os.chdir(rw_repo.working_dir) try: return func(self, rw_repo) - except: # noqa: E722 B001 + except: # noqa: E722 _logger.info("Keeping repo after failure: %s", repo_dir) repo_dir = None raise @@ -309,7 +320,7 @@ def remote_repo_creator(self): with cwd(rw_repo.working_dir): try: return func(self, rw_repo, rw_daemon_repo) - except: # noqa: E722 B001 + except: # noqa: E722 _logger.info( "Keeping repos after failure: \n rw_repo_dir: %s \n rw_daemon_repo_dir: %s", rw_repo_dir, @@ -415,9 +426,15 @@ def __init__(self, env_dir, *, with_pip): if with_pip: # The upgrade_deps parameter to venv.create is 3.9+ only, so do it this way. - command = [self.python, "-m", "pip", "install", "--upgrade", "pip"] - if sys.version_info < (3, 12): - command.append("setuptools") + command = [ + self.python, + "-m", + "pip", + "install", + "--upgrade", + "pip", + 'setuptools; python_version<"3.12"', + ] subprocess.check_output(command) @property diff --git a/test/test_clone.py b/test/test_clone.py index 126ef0063..143a3b51f 100644 --- a/test/test_clone.py +++ b/test/test_clone.py @@ -1,12 +1,23 @@ # This module is part of GitPython and is released under the # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ +import os +import os.path as osp +import pathlib +import sys +import tempfile +from unittest import skip + +from git import GitCommandError, Repo +from git.exc import UnsafeOptionError, UnsafeProtocolError + +from test.lib import TestBase, with_rw_directory, with_rw_repo, PathLikeMock + from pathlib import Path import re import git - -from test.lib import TestBase, with_rw_directory +import pytest class TestClone(TestBase): @@ -29,3 +40,287 @@ def test_checkout_in_non_empty_dir(self, rw_dir): ) else: self.fail("GitCommandError not raised") + + @with_rw_directory + def test_clone_from_pathlib(self, rw_dir): + original_repo = Repo.init(osp.join(rw_dir, "repo")) + + Repo.clone_from(pathlib.Path(original_repo.git_dir), pathlib.Path(rw_dir) / "clone_pathlib") + + @with_rw_directory + def test_clone_from_pathlike(self, rw_dir): + original_repo = Repo.init(osp.join(rw_dir, "repo")) + Repo.clone_from(PathLikeMock(original_repo.git_dir), PathLikeMock(os.path.join(rw_dir, "clone_pathlike"))) + + @with_rw_directory + def test_clone_from_pathlib_withConfig(self, rw_dir): + original_repo = Repo.init(osp.join(rw_dir, "repo")) + + cloned = Repo.clone_from( + original_repo.git_dir, + pathlib.Path(rw_dir) / "clone_pathlib_withConfig", + multi_options=[ + "--recurse-submodules=repo", + "--config core.filemode=false", + "--config submodule.repo.update=checkout", + "--config filter.lfs.clean='git-lfs clean -- %f'", + ], + allow_unsafe_options=True, + ) + + self.assertEqual(cloned.config_reader().get_value("submodule", "active"), "repo") + self.assertEqual(cloned.config_reader().get_value("core", "filemode"), False) + self.assertEqual(cloned.config_reader().get_value('submodule "repo"', "update"), "checkout") + self.assertEqual( + cloned.config_reader().get_value('filter "lfs"', "clean"), + "git-lfs clean -- %f", + ) + + def test_clone_from_with_path_contains_unicode(self): + with tempfile.TemporaryDirectory() as tmpdir: + unicode_dir_name = "\u0394" + path_with_unicode = os.path.join(tmpdir, unicode_dir_name) + os.makedirs(path_with_unicode) + + try: + Repo.clone_from( + url=self._small_repo_url(), + to_path=path_with_unicode, + ) + except UnicodeEncodeError: + self.fail("Raised UnicodeEncodeError") + + @with_rw_directory + @skip( + """The referenced repository was removed, and one needs to set up a new + password controlled repo under the org's control.""" + ) + def test_leaking_password_in_clone_logs(self, rw_dir): + password = "fakepassword1234" + try: + Repo.clone_from( + url="https://fakeuser:{}@fakerepo.example.com/testrepo".format(password), + to_path=rw_dir, + ) + except GitCommandError as err: + assert password not in str(err), "The error message '%s' should not contain the password" % err + # Working example from a blank private project. + Repo.clone_from( + url="https://gitlab+deploy-token-392045:mLWhVus7bjLsy8xj8q2V@gitlab.com/mercierm/test_git_python", + to_path=rw_dir, + ) + + @with_rw_repo("HEAD") + def test_clone_unsafe_options(self, rw_repo): + with tempfile.TemporaryDirectory() as tdir: + tmp_dir = pathlib.Path(tdir) + tmp_file = tmp_dir / "pwn" + unsafe_options = [ + f"--upload-pack='touch {tmp_file}'", + f"-u 'touch {tmp_file}'", + "--config=protocol.ext.allow=always", + "-c protocol.ext.allow=always", + ] + for unsafe_option in unsafe_options: + with self.assertRaises(UnsafeOptionError): + rw_repo.clone(tmp_dir, multi_options=[unsafe_option]) + assert not tmp_file.exists() + + unsafe_options = [ + {"upload-pack": f"touch {tmp_file}"}, + {"u": f"touch {tmp_file}"}, + {"config": "protocol.ext.allow=always"}, + {"c": "protocol.ext.allow=always"}, + ] + for unsafe_option in unsafe_options: + with self.assertRaises(UnsafeOptionError): + rw_repo.clone(tmp_dir, **unsafe_option) + assert not tmp_file.exists() + + @pytest.mark.xfail( + sys.platform == "win32", + reason=( + "File not created. A separate Windows command may be needed. This and the " + "currently passing test test_clone_unsafe_options must be adjusted in the " + "same way. Until then, test_clone_unsafe_options is unreliable on Windows." + ), + raises=AssertionError, + ) + @with_rw_repo("HEAD") + def test_clone_unsafe_options_allowed(self, rw_repo): + with tempfile.TemporaryDirectory() as tdir: + tmp_dir = pathlib.Path(tdir) + tmp_file = tmp_dir / "pwn" + unsafe_options = [ + f"--upload-pack='touch {tmp_file}'", + f"-u 'touch {tmp_file}'", + ] + for i, unsafe_option in enumerate(unsafe_options): + destination = tmp_dir / str(i) + assert not tmp_file.exists() + # The options will be allowed, but the command will fail. + with self.assertRaises(GitCommandError): + rw_repo.clone(destination, multi_options=[unsafe_option], allow_unsafe_options=True) + assert tmp_file.exists() + tmp_file.unlink() + + unsafe_options = [ + "--config=protocol.ext.allow=always", + "-c protocol.ext.allow=always", + ] + for i, unsafe_option in enumerate(unsafe_options): + destination = tmp_dir / str(i) + assert not destination.exists() + rw_repo.clone(destination, multi_options=[unsafe_option], allow_unsafe_options=True) + assert destination.exists() + + @with_rw_repo("HEAD") + def test_clone_safe_options(self, rw_repo): + with tempfile.TemporaryDirectory() as tdir: + tmp_dir = pathlib.Path(tdir) + options = [ + "--depth=1", + "--single-branch", + "-q", + ] + for option in options: + destination = tmp_dir / option + assert not destination.exists() + rw_repo.clone(destination, multi_options=[option]) + assert destination.exists() + + @with_rw_repo("HEAD") + def test_clone_from_unsafe_options(self, rw_repo): + with tempfile.TemporaryDirectory() as tdir: + tmp_dir = pathlib.Path(tdir) + tmp_file = tmp_dir / "pwn" + unsafe_options = [ + f"--upload-pack='touch {tmp_file}'", + f"-u 'touch {tmp_file}'", + "--config=protocol.ext.allow=always", + "-c protocol.ext.allow=always", + ] + for unsafe_option in unsafe_options: + with self.assertRaises(UnsafeOptionError): + Repo.clone_from(rw_repo.working_dir, tmp_dir, multi_options=[unsafe_option]) + assert not tmp_file.exists() + + unsafe_options = [ + {"upload-pack": f"touch {tmp_file}"}, + {"u": f"touch {tmp_file}"}, + {"config": "protocol.ext.allow=always"}, + {"c": "protocol.ext.allow=always"}, + ] + for unsafe_option in unsafe_options: + with self.assertRaises(UnsafeOptionError): + Repo.clone_from(rw_repo.working_dir, tmp_dir, **unsafe_option) + assert not tmp_file.exists() + + @pytest.mark.xfail( + sys.platform == "win32", + reason=( + "File not created. A separate Windows command may be needed. This and the " + "currently passing test test_clone_from_unsafe_options must be adjusted in the " + "same way. Until then, test_clone_from_unsafe_options is unreliable on Windows." + ), + raises=AssertionError, + ) + @with_rw_repo("HEAD") + def test_clone_from_unsafe_options_allowed(self, rw_repo): + with tempfile.TemporaryDirectory() as tdir: + tmp_dir = pathlib.Path(tdir) + tmp_file = tmp_dir / "pwn" + unsafe_options = [ + f"--upload-pack='touch {tmp_file}'", + f"-u 'touch {tmp_file}'", + ] + for i, unsafe_option in enumerate(unsafe_options): + destination = tmp_dir / str(i) + assert not tmp_file.exists() + # The options will be allowed, but the command will fail. + with self.assertRaises(GitCommandError): + Repo.clone_from( + rw_repo.working_dir, destination, multi_options=[unsafe_option], allow_unsafe_options=True + ) + assert tmp_file.exists() + tmp_file.unlink() + + unsafe_options = [ + "--config=protocol.ext.allow=always", + "-c protocol.ext.allow=always", + ] + for i, unsafe_option in enumerate(unsafe_options): + destination = tmp_dir / str(i) + assert not destination.exists() + Repo.clone_from( + rw_repo.working_dir, destination, multi_options=[unsafe_option], allow_unsafe_options=True + ) + assert destination.exists() + + @with_rw_repo("HEAD") + def test_clone_from_safe_options(self, rw_repo): + with tempfile.TemporaryDirectory() as tdir: + tmp_dir = pathlib.Path(tdir) + options = [ + "--depth=1", + "--single-branch", + "-q", + ] + for option in options: + destination = tmp_dir / option + assert not destination.exists() + Repo.clone_from(rw_repo.common_dir, destination, multi_options=[option]) + assert destination.exists() + + def test_clone_from_unsafe_protocol(self): + with tempfile.TemporaryDirectory() as tdir: + tmp_dir = pathlib.Path(tdir) + tmp_file = tmp_dir / "pwn" + urls = [ + f"ext::sh -c touch% {tmp_file}", + "fd::17/foo", + ] + for url in urls: + with self.assertRaises(UnsafeProtocolError): + Repo.clone_from(url, tmp_dir / "repo") + assert not tmp_file.exists() + + def test_clone_from_unsafe_protocol_allowed(self): + with tempfile.TemporaryDirectory() as tdir: + tmp_dir = pathlib.Path(tdir) + tmp_file = tmp_dir / "pwn" + urls = [ + f"ext::sh -c touch% {tmp_file}", + "fd::/foo", + ] + for url in urls: + # The URL will be allowed into the command, but the command will + # fail since we don't have that protocol enabled in the Git config file. + with self.assertRaises(GitCommandError): + Repo.clone_from(url, tmp_dir / "repo", allow_unsafe_protocols=True) + assert not tmp_file.exists() + + def test_clone_from_unsafe_protocol_allowed_and_enabled(self): + with tempfile.TemporaryDirectory() as tdir: + tmp_dir = pathlib.Path(tdir) + tmp_file = tmp_dir / "pwn" + urls = [ + f"ext::sh -c touch% {tmp_file}", + ] + allow_ext = [ + "--config=protocol.ext.allow=always", + ] + for url in urls: + # The URL will be allowed into the command, and the protocol is enabled, + # but the command will fail since it can't read from the remote repo. + assert not tmp_file.exists() + with self.assertRaises(GitCommandError): + Repo.clone_from( + url, + tmp_dir / "repo", + multi_options=allow_ext, + allow_unsafe_protocols=True, + allow_unsafe_options=True, + ) + assert tmp_file.exists() + tmp_file.unlink() diff --git a/test/test_config.py b/test/test_config.py index 92997422d..56ac0f304 100644 --- a/test/test_config.py +++ b/test/test_config.py @@ -373,6 +373,41 @@ def test_conditional_includes_from_branch_name_error(self, rw_dir): assert not config._has_includes() assert config._included_paths() == [] + @with_rw_directory + def test_conditional_includes_remote_url(self, rw_dir): + # Initiate mocked repository. + repo = mock.Mock() + repo.remotes = [mock.Mock(url="https://github.com/foo/repo")] + + # Initiate config files. + path1 = osp.join(rw_dir, "config1") + path2 = osp.join(rw_dir, "config2") + template = '[includeIf "hasconfig:remote.*.url:{}"]\n path={}\n' + + # Ensure that config with hasconfig and full url is correct. + with open(path1, "w") as stream: + stream.write(template.format("https://github.com/foo/repo", path2)) + + with GitConfigParser(path1, repo=repo) as config: + assert config._has_includes() + assert config._included_paths() == [("path", path2)] + + # Ensure that config with hasconfig and incorrect url is incorrect. + with open(path1, "w") as stream: + stream.write(template.format("incorrect", path2)) + + with GitConfigParser(path1, repo=repo) as config: + assert not config._has_includes() + assert config._included_paths() == [] + + # Ensure that config with hasconfig and url using glob pattern is correct. + with open(path1, "w") as stream: + stream.write(template.format("**/**github.com*/**", path2)) + + with GitConfigParser(path1, repo=repo) as config: + assert config._has_includes() + assert config._included_paths() == [("path", path2)] + def test_rename(self): file_obj = self._to_memcache(fixture_path("git_config")) with GitConfigParser(file_obj, read_only=False, merge_includes=False) as cw: @@ -391,13 +426,17 @@ def test_complex_aliases(self): with GitConfigParser(file_obj, read_only=False) as w_config: self.assertEqual( w_config.get("alias", "rbi"), - '"!g() { git rebase -i origin/${1:-master} ; } ; g"', + "!g() { git rebase -i origin/${1:-master} ; } ; g", ) self.assertEqual( file_obj.getvalue(), self._to_memcache(fixture_path(".gitconfig")).getvalue(), ) + def test_config_with_extra_whitespace(self): + cr = GitConfigParser(fixture_path("git_config_with_extra_whitespace"), read_only=True) + self.assertEqual(cr.get("init", "defaultBranch"), "trunk") + def test_empty_config_value(self): cr = GitConfigParser(fixture_path("git_config_with_empty_value"), read_only=True) @@ -406,6 +445,44 @@ def test_empty_config_value(self): with self.assertRaises(cp.NoOptionError): cr.get_value("color", "ui") + def test_config_with_quotes(self): + cr = GitConfigParser(fixture_path("git_config_with_quotes"), read_only=True) + + self.assertEqual(cr.get("user", "name"), "Cody Veal") + self.assertEqual(cr.get("user", "email"), "cveal05@gmail.com") + + def test_config_with_empty_quotes(self): + cr = GitConfigParser(fixture_path("git_config_with_empty_quotes"), read_only=True) + self.assertEqual(cr.get("core", "filemode"), "", "quotes can form a literal empty string as value") + + def test_config_with_quotes_with_literal_whitespace(self): + cr = GitConfigParser(fixture_path("git_config_with_quotes_whitespace_inside"), read_only=True) + self.assertEqual(cr.get("core", "commentString"), "# ") + + def test_config_with_quotes_with_whitespace_outside_value(self): + cr = GitConfigParser(fixture_path("git_config_with_quotes_whitespace_outside"), read_only=True) + self.assertEqual(cr.get("init", "defaultBranch"), "trunk") + + def test_config_with_quotes_containing_escapes(self): + """For now just suppress quote removal. But it would be good to interpret most of these.""" + cr = GitConfigParser(fixture_path("git_config_with_quotes_escapes"), read_only=True) + + # These can eventually be supported by substituting the represented character. + self.assertEqual(cr.get("custom", "hasnewline"), R'"first\nsecond"') + self.assertEqual(cr.get("custom", "hasbackslash"), R'"foo\\bar"') + self.assertEqual(cr.get("custom", "hasquote"), R'"ab\"cd"') + self.assertEqual(cr.get("custom", "hastrailingbackslash"), R'"word\\"') + self.assertEqual(cr.get("custom", "hasunrecognized"), R'"p\qrs"') + + # It is less obvious whether and what to eventually do with this. + self.assertEqual(cr.get("custom", "hasunescapedquotes"), '"ab"cd"e"') + + # Cases where quote removal is clearly safe should happen even after those. + self.assertEqual(cr.get("custom", "ordinary"), "hello world") + + # Cases without quotes should still parse correctly even after those, too. + self.assertEqual(cr.get("custom", "unquoted"), "good evening") + def test_get_values_works_without_requiring_any_other_calls_first(self): file_obj = self._to_memcache(fixture_path("git_config_multiple")) cr = GitConfigParser(file_obj, read_only=True) diff --git a/test/test_fun.py b/test/test_fun.py index b8593b400..a456b8aab 100644 --- a/test/test_fun.py +++ b/test/test_fun.py @@ -243,6 +243,7 @@ def test_tree_traversal(self): B_old = self.rorepo.tree("1f66cfbbce58b4b552b041707a12d437cc5f400a") # old base tree # Two very different trees. + entries = traverse_trees_recursive(odb, [B_old.binsha, H.binsha], "") self._assert_tree_entries(entries, 2) @@ -251,7 +252,10 @@ def test_tree_traversal(self): self._assert_tree_entries(oentries, 2) # Single tree. - is_no_tree = lambda i, d: i.type != "tree" + + def is_no_tree(i, _d): + return i.type != "tree" + entries = traverse_trees_recursive(odb, [B.binsha], "") assert len(entries) == len(list(B.traverse(predicate=is_no_tree))) self._assert_tree_entries(entries, 1) diff --git a/test/test_git.py b/test/test_git.py index 94e68ecf0..4a54d0d9b 100644 --- a/test/test_git.py +++ b/test/test_git.py @@ -747,7 +747,7 @@ def test_environment(self, rw_dir): path = osp.join(rw_dir, "failing-script.sh") with open(path, "wt") as stream: - stream.write("#!/usr/bin/env sh\n" "echo FOO\n") + stream.write("#!/usr/bin/env sh\necho FOO\n") os.chmod(path, 0o777) rw_repo = Repo.init(osp.join(rw_dir, "repo")) @@ -762,17 +762,18 @@ def test_environment(self, rw_dir): def test_handle_process_output(self): from git.cmd import handle_process_output, safer_popen - line_count = 5002 - count = [None, 0, 0] + expected_line_count = 5002 + actual_lines = [None, [], []] - def counter_stdout(line): - count[1] += 1 + def stdout_handler(line): + actual_lines[1].append(line) - def counter_stderr(line): - count[2] += 1 + def stderr_handler(line): + actual_lines[2].append(line) cmdline = [ sys.executable, + "-S", # Keep any `CoverageWarning` messages out of the subprocess stderr. fixture_path("cat_file.py"), str(fixture_path("issue-301_stderr")), ] @@ -784,10 +785,10 @@ def counter_stderr(line): shell=False, ) - handle_process_output(proc, counter_stdout, counter_stderr, finalize_process) + handle_process_output(proc, stdout_handler, stderr_handler, finalize_process) - self.assertEqual(count[1], line_count) - self.assertEqual(count[2], line_count) + self.assertEqual(len(actual_lines[1]), expected_line_count, repr(actual_lines[1])) + self.assertEqual(len(actual_lines[2]), expected_line_count, repr(actual_lines[2])) def test_execute_kwargs_set_agrees_with_method(self): parameter_names = inspect.signature(cmd.Git.execute).parameters.keys() diff --git a/test/test_index.py b/test/test_index.py index c586a0b5a..33490f907 100644 --- a/test/test_index.py +++ b/test/test_index.py @@ -16,6 +16,7 @@ import subprocess import sys import tempfile +from unittest import mock from gitdb.base import IStream @@ -36,14 +37,7 @@ from git.objects import Blob from git.util import Actor, cwd, hex_to_bin, rmtree -from test.lib import ( - TestBase, - VirtualEnvironment, - fixture, - fixture_path, - with_rw_directory, - with_rw_repo, -) +from test.lib import TestBase, VirtualEnvironment, fixture, fixture_path, with_rw_directory, with_rw_repo, PathLikeMock HOOKS_SHEBANG = "#!/usr/bin/env sh\n" @@ -329,7 +323,10 @@ def test_index_file_from_tree(self, rw_repo): assert len([e for e in three_way_index.entries.values() if e.stage != 0]) # ITERATE BLOBS - merge_required = lambda t: t[0] != 0 + + def merge_required(t): + return t[0] != 0 + merge_blobs = list(three_way_index.iter_blobs(merge_required)) assert merge_blobs assert merge_blobs[0][0] in (1, 2, 3) @@ -583,11 +580,15 @@ def mixed_iterator(): yield entry.path elif type_id == 1: # path (PathLike) yield Path(entry.path) - elif type_id == 2: # blob + elif type_id == 2: # path mock (PathLike) + yield PathLikeMock(entry.path) + elif type_id == 3: # path mock in a blob + yield Blob(rw_repo, entry.binsha, entry.mode, entry.path) + elif type_id == 4: # blob yield Blob(rw_repo, entry.binsha, entry.mode, entry.path) - elif type_id == 3: # BaseIndexEntry + elif type_id == 5: # BaseIndexEntry yield BaseIndexEntry(entry[:4]) - elif type_id == 4: # IndexEntry + elif type_id == 6: # IndexEntry yield entry else: raise AssertionError("Invalid Type") @@ -1015,6 +1016,27 @@ class Mocked: rel = index._to_relative_path(path) self.assertEqual(rel, os.path.relpath(path, root)) + def test__to_relative_path_absolute_trailing_slash(self): + repo_root = os.path.join(osp.abspath(os.sep), "directory1", "repo_root") + + class Mocked: + bare = False + git_dir = repo_root + working_tree_dir = repo_root + + repo = Mocked() + path = os.path.join(repo_root, f"directory2{os.sep}") + index = IndexFile(repo) + + expected_path = f"directory2{os.sep}" + actual_path = index._to_relative_path(path) + self.assertEqual(expected_path, actual_path) + + with mock.patch("git.index.base.os.path") as ospath_mock: + ospath_mock.relpath.return_value = f"directory2{os.sep}" + actual_path = index._to_relative_path(path) + self.assertEqual(expected_path, actual_path) + @pytest.mark.xfail( type(_win_bash_status) is WinBashStatus.Absent, reason="Can't run a hook on Windows without bash.exe.", @@ -1173,7 +1195,7 @@ def test_commit_msg_hook_fail(self, rw_repo): raise AssertionError("Should have caught a HookExecutionError") @with_rw_repo("HEAD") - def test_index_add_pathlike(self, rw_repo): + def test_index_add_pathlib(self, rw_repo): git_dir = Path(rw_repo.git_dir) file = git_dir / "file.txt" @@ -1181,6 +1203,15 @@ def test_index_add_pathlike(self, rw_repo): rw_repo.index.add(file) + @with_rw_repo("HEAD") + def test_index_add_pathlike(self, rw_repo): + git_dir = Path(rw_repo.git_dir) + + file = git_dir / "file.txt" + file.touch() + + rw_repo.index.add(PathLikeMock(str(file))) + @with_rw_repo("HEAD") def test_index_add_non_normalized_path(self, rw_repo): git_dir = Path(rw_repo.git_dir) @@ -1193,6 +1224,48 @@ def test_index_add_non_normalized_path(self, rw_repo): rw_repo.index.add(non_normalized_path) + def test_index_file_v3(self): + index = IndexFile(self.rorepo, fixture_path("index_extended_flags")) + assert index.entries + assert index.version == 3 + assert len(index.entries) == 4 + assert index.entries[("init.t", 0)].skip_worktree + + # Write the data - it must match the original. + with tempfile.NamedTemporaryFile() as tmpfile: + index.write(tmpfile.name) + assert Path(tmpfile.name).read_bytes() == Path(fixture_path("index_extended_flags")).read_bytes() + + @with_rw_directory + def test_index_file_v3_with_git_command(self, tmp_dir): + tmp_dir = Path(tmp_dir) + with cwd(tmp_dir): + git = Git(tmp_dir) + git.init() + + file = tmp_dir / "file.txt" + file.write_text("hello") + git.add("--intent-to-add", "file.txt") # intent-to-add sets extended flag + + repo = Repo(tmp_dir) + index = repo.index + + assert len(index.entries) == 1 + assert index.version == 3 + entry = list(index.entries.values())[0] + assert entry.path == "file.txt" + assert entry.intent_to_add + + file2 = tmp_dir / "file2.txt" + file2.write_text("world") + index.add(["file2.txt"]) + index.write() + + status_str = git.status(porcelain=True) + status_lines = status_str.splitlines() + assert " A file.txt" in status_lines + assert "A file2.txt" in status_lines + class TestIndexUtils: @pytest.mark.parametrize("file_path_type", [str, Path]) diff --git a/test/test_installation.py b/test/test_installation.py index ae6472e98..7c82bd403 100644 --- a/test/test_installation.py +++ b/test/test_installation.py @@ -2,6 +2,7 @@ # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ import ast +import functools import os import subprocess @@ -11,50 +12,22 @@ class TestInstallation(TestBase): @with_rw_directory def test_installation(self, rw_dir): - venv = self._set_up_venv(rw_dir) + venv, run = self._set_up_venv(rw_dir) - result = subprocess.run( - [venv.pip, "install", "."], - stdout=subprocess.PIPE, - cwd=venv.sources, - ) - self.assertEqual( - 0, - result.returncode, - msg=result.stderr or result.stdout or "Can't install project", - ) + result = run([venv.pip, "install", "."]) + self._check_result(result, "Can't install project") - result = subprocess.run( - [venv.python, "-c", "import git"], - stdout=subprocess.PIPE, - cwd=venv.sources, - ) - self.assertEqual( - 0, - result.returncode, - msg=result.stderr or result.stdout or "Self-test failed", - ) + result = run([venv.python, "-c", "import git"]) + self._check_result(result, "Self-test failed") - result = subprocess.run( - [venv.python, "-c", "import gitdb; import smmap"], - stdout=subprocess.PIPE, - cwd=venv.sources, - ) - self.assertEqual( - 0, - result.returncode, - msg=result.stderr or result.stdout or "Dependencies not installed", - ) + result = run([venv.python, "-c", "import gitdb; import smmap"]) + self._check_result(result, "Dependencies not installed") # Even IF gitdb or any other dependency is supplied during development by # inserting its location into PYTHONPATH or otherwise patched into sys.path, # make sure it is not wrongly inserted as the *first* entry. - result = subprocess.run( - [venv.python, "-c", "import sys; import git; print(sys.path)"], - stdout=subprocess.PIPE, - cwd=venv.sources, - ) - syspath = result.stdout.decode("utf-8").splitlines()[0] + result = run([venv.python, "-c", "import sys; import git; print(sys.path)"]) + syspath = result.stdout.splitlines()[0] syspath = ast.literal_eval(syspath) self.assertEqual( "", @@ -64,10 +37,37 @@ def test_installation(self, rw_dir): @staticmethod def _set_up_venv(rw_dir): + # Initialize the virtual environment. venv = VirtualEnvironment(rw_dir, with_pip=True) + + # Make its src directory a symlink to our own top-level source tree. os.symlink( os.path.dirname(os.path.dirname(__file__)), venv.sources, target_is_directory=True, ) - return venv + + # Create a convenience function to run commands in it. + run = functools.partial( + subprocess.run, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + cwd=venv.sources, + env={**os.environ, "PYTHONWARNINGS": "error"}, + ) + + return venv, run + + def _check_result(self, result, failure_summary): + self.assertEqual( + 0, + result.returncode, + msg=self._prepare_failure_message(result, failure_summary), + ) + + @staticmethod + def _prepare_failure_message(result, failure_summary): + stdout = result.stdout.rstrip() + stderr = result.stderr.rstrip() + return f"{failure_summary}\n\nstdout:\n{stdout}\n\nstderr:\n{stderr}" diff --git a/test/test_quick_doc.py b/test/test_quick_doc.py index 4ef75f4aa..98658e02f 100644 --- a/test/test_quick_doc.py +++ b/test/test_quick_doc.py @@ -173,7 +173,7 @@ def test_cloned_repo_object(self, local_dir): # [15-test_cloned_repo_object] def print_files_from_git(root, level=0): for entry in root: - print(f'{"-" * 4 * level}| {entry.path}, {entry.type}') + print(f"{'-' * 4 * level}| {entry.path}, {entry.type}") if entry.type == "tree": print_files_from_git(entry, level + 1) diff --git a/test/test_refs.py b/test/test_refs.py index 08096e69e..329515807 100644 --- a/test/test_refs.py +++ b/test/test_refs.py @@ -25,7 +25,7 @@ import git.refs as refs from git.util import Actor -from test.lib import TestBase, with_rw_repo +from test.lib import TestBase, with_rw_repo, PathLikeMock class TestRefs(TestBase): @@ -43,6 +43,25 @@ def test_from_path(self): self.assertRaises(ValueError, TagReference, self.rorepo, "refs/invalid/tag") # Works without path check. TagReference(self.rorepo, "refs/invalid/tag", check_path=False) + # Check remoteness + assert Reference(self.rorepo, "refs/remotes/origin").is_remote() + + def test_from_pathlike(self): + # Should be able to create any reference directly. + for ref_type in (Reference, Head, TagReference, RemoteReference): + for name in ("rela_name", "path/rela_name"): + full_path = ref_type.to_full_path(PathLikeMock(name)) + instance = ref_type.from_path(self.rorepo, PathLikeMock(full_path)) + assert isinstance(instance, ref_type) + # END for each name + # END for each type + + # Invalid path. + self.assertRaises(ValueError, TagReference, self.rorepo, "refs/invalid/tag") + # Works without path check. + TagReference(self.rorepo, PathLikeMock("refs/invalid/tag"), check_path=False) + # Check remoteness + assert Reference(self.rorepo, PathLikeMock("refs/remotes/origin")).is_remote() def test_tag_base(self): tag_object_refs = [] diff --git a/test/test_remote.py b/test/test_remote.py index 5ddb41bc0..b1d686f05 100644 --- a/test/test_remote.py +++ b/test/test_remote.py @@ -44,7 +44,7 @@ class TestRemoteProgress(RemoteProgress): __slots__ = ("_seen_lines", "_stages_per_op", "_num_progress_messages") - def __init__(self): + def __init__(self) -> None: super().__init__() self._seen_lines = [] self._stages_per_op = {} @@ -103,6 +103,9 @@ def assert_received_message(self): assert self._num_progress_messages +TestRemoteProgress.__test__ = False # type: ignore + + class TestRemote(TestBase): def tearDown(self): gc.collect() diff --git a/test/test_repo.py b/test/test_repo.py index e38da5bb6..2a92c2523 100644 --- a/test/test_repo.py +++ b/test/test_repo.py @@ -14,7 +14,8 @@ import pickle import sys import tempfile -from unittest import mock, skip +from unittest import mock +from pathlib import Path import pytest @@ -36,11 +37,11 @@ Submodule, Tree, ) -from git.exc import BadObject, UnsafeOptionError, UnsafeProtocolError +from git.exc import BadObject from git.repo.fun import touch from git.util import bin_to_hex, cwd, cygpath, join_path_native, rmfile, rmtree -from test.lib import TestBase, fixture, with_rw_directory, with_rw_repo +from test.lib import TestBase, fixture, with_rw_directory, with_rw_repo, PathLikeMock def iter_flatten(lol): @@ -105,6 +106,11 @@ def test_repo_creation_pathlib(self, rw_repo): r_from_gitdir = Repo(pathlib.Path(rw_repo.git_dir)) self.assertEqual(r_from_gitdir.git_dir, rw_repo.git_dir) + @with_rw_repo("0.3.2.1") + def test_repo_creation_pathlike(self, rw_repo): + r_from_gitdir = Repo(PathLikeMock(rw_repo.git_dir)) + self.assertEqual(r_from_gitdir.git_dir, rw_repo.git_dir) + def test_description(self): txt = "Test repository" self.rorepo.description = txt @@ -214,285 +220,6 @@ def test_date_format(self, rw_dir): # @-timestamp is the format used by git commit hooks. repo.index.commit("Commit messages", commit_date="@1400000000 +0000") - @with_rw_directory - def test_clone_from_pathlib(self, rw_dir): - original_repo = Repo.init(osp.join(rw_dir, "repo")) - - Repo.clone_from(original_repo.git_dir, pathlib.Path(rw_dir) / "clone_pathlib") - - @with_rw_directory - def test_clone_from_pathlib_withConfig(self, rw_dir): - original_repo = Repo.init(osp.join(rw_dir, "repo")) - - cloned = Repo.clone_from( - original_repo.git_dir, - pathlib.Path(rw_dir) / "clone_pathlib_withConfig", - multi_options=[ - "--recurse-submodules=repo", - "--config core.filemode=false", - "--config submodule.repo.update=checkout", - "--config filter.lfs.clean='git-lfs clean -- %f'", - ], - allow_unsafe_options=True, - ) - - self.assertEqual(cloned.config_reader().get_value("submodule", "active"), "repo") - self.assertEqual(cloned.config_reader().get_value("core", "filemode"), False) - self.assertEqual(cloned.config_reader().get_value('submodule "repo"', "update"), "checkout") - self.assertEqual( - cloned.config_reader().get_value('filter "lfs"', "clean"), - "git-lfs clean -- %f", - ) - - def test_clone_from_with_path_contains_unicode(self): - with tempfile.TemporaryDirectory() as tmpdir: - unicode_dir_name = "\u0394" - path_with_unicode = os.path.join(tmpdir, unicode_dir_name) - os.makedirs(path_with_unicode) - - try: - Repo.clone_from( - url=self._small_repo_url(), - to_path=path_with_unicode, - ) - except UnicodeEncodeError: - self.fail("Raised UnicodeEncodeError") - - @with_rw_directory - @skip( - """The referenced repository was removed, and one needs to set up a new - password controlled repo under the org's control.""" - ) - def test_leaking_password_in_clone_logs(self, rw_dir): - password = "fakepassword1234" - try: - Repo.clone_from( - url="https://fakeuser:{}@fakerepo.example.com/testrepo".format(password), - to_path=rw_dir, - ) - except GitCommandError as err: - assert password not in str(err), "The error message '%s' should not contain the password" % err - # Working example from a blank private project. - Repo.clone_from( - url="https://gitlab+deploy-token-392045:mLWhVus7bjLsy8xj8q2V@gitlab.com/mercierm/test_git_python", - to_path=rw_dir, - ) - - @with_rw_repo("HEAD") - def test_clone_unsafe_options(self, rw_repo): - with tempfile.TemporaryDirectory() as tdir: - tmp_dir = pathlib.Path(tdir) - tmp_file = tmp_dir / "pwn" - unsafe_options = [ - f"--upload-pack='touch {tmp_file}'", - f"-u 'touch {tmp_file}'", - "--config=protocol.ext.allow=always", - "-c protocol.ext.allow=always", - ] - for unsafe_option in unsafe_options: - with self.assertRaises(UnsafeOptionError): - rw_repo.clone(tmp_dir, multi_options=[unsafe_option]) - assert not tmp_file.exists() - - unsafe_options = [ - {"upload-pack": f"touch {tmp_file}"}, - {"u": f"touch {tmp_file}"}, - {"config": "protocol.ext.allow=always"}, - {"c": "protocol.ext.allow=always"}, - ] - for unsafe_option in unsafe_options: - with self.assertRaises(UnsafeOptionError): - rw_repo.clone(tmp_dir, **unsafe_option) - assert not tmp_file.exists() - - @pytest.mark.xfail( - sys.platform == "win32", - reason=( - "File not created. A separate Windows command may be needed. This and the " - "currently passing test test_clone_unsafe_options must be adjusted in the " - "same way. Until then, test_clone_unsafe_options is unreliable on Windows." - ), - raises=AssertionError, - ) - @with_rw_repo("HEAD") - def test_clone_unsafe_options_allowed(self, rw_repo): - with tempfile.TemporaryDirectory() as tdir: - tmp_dir = pathlib.Path(tdir) - tmp_file = tmp_dir / "pwn" - unsafe_options = [ - f"--upload-pack='touch {tmp_file}'", - f"-u 'touch {tmp_file}'", - ] - for i, unsafe_option in enumerate(unsafe_options): - destination = tmp_dir / str(i) - assert not tmp_file.exists() - # The options will be allowed, but the command will fail. - with self.assertRaises(GitCommandError): - rw_repo.clone(destination, multi_options=[unsafe_option], allow_unsafe_options=True) - assert tmp_file.exists() - tmp_file.unlink() - - unsafe_options = [ - "--config=protocol.ext.allow=always", - "-c protocol.ext.allow=always", - ] - for i, unsafe_option in enumerate(unsafe_options): - destination = tmp_dir / str(i) - assert not destination.exists() - rw_repo.clone(destination, multi_options=[unsafe_option], allow_unsafe_options=True) - assert destination.exists() - - @with_rw_repo("HEAD") - def test_clone_safe_options(self, rw_repo): - with tempfile.TemporaryDirectory() as tdir: - tmp_dir = pathlib.Path(tdir) - options = [ - "--depth=1", - "--single-branch", - "-q", - ] - for option in options: - destination = tmp_dir / option - assert not destination.exists() - rw_repo.clone(destination, multi_options=[option]) - assert destination.exists() - - @with_rw_repo("HEAD") - def test_clone_from_unsafe_options(self, rw_repo): - with tempfile.TemporaryDirectory() as tdir: - tmp_dir = pathlib.Path(tdir) - tmp_file = tmp_dir / "pwn" - unsafe_options = [ - f"--upload-pack='touch {tmp_file}'", - f"-u 'touch {tmp_file}'", - "--config=protocol.ext.allow=always", - "-c protocol.ext.allow=always", - ] - for unsafe_option in unsafe_options: - with self.assertRaises(UnsafeOptionError): - Repo.clone_from(rw_repo.working_dir, tmp_dir, multi_options=[unsafe_option]) - assert not tmp_file.exists() - - unsafe_options = [ - {"upload-pack": f"touch {tmp_file}"}, - {"u": f"touch {tmp_file}"}, - {"config": "protocol.ext.allow=always"}, - {"c": "protocol.ext.allow=always"}, - ] - for unsafe_option in unsafe_options: - with self.assertRaises(UnsafeOptionError): - Repo.clone_from(rw_repo.working_dir, tmp_dir, **unsafe_option) - assert not tmp_file.exists() - - @pytest.mark.xfail( - sys.platform == "win32", - reason=( - "File not created. A separate Windows command may be needed. This and the " - "currently passing test test_clone_from_unsafe_options must be adjusted in the " - "same way. Until then, test_clone_from_unsafe_options is unreliable on Windows." - ), - raises=AssertionError, - ) - @with_rw_repo("HEAD") - def test_clone_from_unsafe_options_allowed(self, rw_repo): - with tempfile.TemporaryDirectory() as tdir: - tmp_dir = pathlib.Path(tdir) - tmp_file = tmp_dir / "pwn" - unsafe_options = [ - f"--upload-pack='touch {tmp_file}'", - f"-u 'touch {tmp_file}'", - ] - for i, unsafe_option in enumerate(unsafe_options): - destination = tmp_dir / str(i) - assert not tmp_file.exists() - # The options will be allowed, but the command will fail. - with self.assertRaises(GitCommandError): - Repo.clone_from( - rw_repo.working_dir, destination, multi_options=[unsafe_option], allow_unsafe_options=True - ) - assert tmp_file.exists() - tmp_file.unlink() - - unsafe_options = [ - "--config=protocol.ext.allow=always", - "-c protocol.ext.allow=always", - ] - for i, unsafe_option in enumerate(unsafe_options): - destination = tmp_dir / str(i) - assert not destination.exists() - Repo.clone_from( - rw_repo.working_dir, destination, multi_options=[unsafe_option], allow_unsafe_options=True - ) - assert destination.exists() - - @with_rw_repo("HEAD") - def test_clone_from_safe_options(self, rw_repo): - with tempfile.TemporaryDirectory() as tdir: - tmp_dir = pathlib.Path(tdir) - options = [ - "--depth=1", - "--single-branch", - "-q", - ] - for option in options: - destination = tmp_dir / option - assert not destination.exists() - Repo.clone_from(rw_repo.common_dir, destination, multi_options=[option]) - assert destination.exists() - - def test_clone_from_unsafe_protocol(self): - with tempfile.TemporaryDirectory() as tdir: - tmp_dir = pathlib.Path(tdir) - tmp_file = tmp_dir / "pwn" - urls = [ - f"ext::sh -c touch% {tmp_file}", - "fd::17/foo", - ] - for url in urls: - with self.assertRaises(UnsafeProtocolError): - Repo.clone_from(url, tmp_dir / "repo") - assert not tmp_file.exists() - - def test_clone_from_unsafe_protocol_allowed(self): - with tempfile.TemporaryDirectory() as tdir: - tmp_dir = pathlib.Path(tdir) - tmp_file = tmp_dir / "pwn" - urls = [ - f"ext::sh -c touch% {tmp_file}", - "fd::/foo", - ] - for url in urls: - # The URL will be allowed into the command, but the command will - # fail since we don't have that protocol enabled in the Git config file. - with self.assertRaises(GitCommandError): - Repo.clone_from(url, tmp_dir / "repo", allow_unsafe_protocols=True) - assert not tmp_file.exists() - - def test_clone_from_unsafe_protocol_allowed_and_enabled(self): - with tempfile.TemporaryDirectory() as tdir: - tmp_dir = pathlib.Path(tdir) - tmp_file = tmp_dir / "pwn" - urls = [ - f"ext::sh -c touch% {tmp_file}", - ] - allow_ext = [ - "--config=protocol.ext.allow=always", - ] - for url in urls: - # The URL will be allowed into the command, and the protocol is enabled, - # but the command will fail since it can't read from the remote repo. - assert not tmp_file.exists() - with self.assertRaises(GitCommandError): - Repo.clone_from( - url, - tmp_dir / "repo", - multi_options=allow_ext, - allow_unsafe_protocols=True, - allow_unsafe_options=True, - ) - assert tmp_file.exists() - tmp_file.unlink() - @with_rw_repo("HEAD") def test_max_chunk_size(self, repo): class TestOutputStream(TestBase): @@ -643,6 +370,15 @@ def test_is_dirty_with_path(self, rwrepo): assert rwrepo.is_dirty(path="doc") is False assert rwrepo.is_dirty(untracked_files=True, path="doc") is True + @with_rw_repo("HEAD") + def test_is_dirty_with_pathlib_and_pathlike(self, rwrepo): + with open(osp.join(rwrepo.working_dir, "git", "util.py"), "at") as f: + f.write("junk") + assert rwrepo.is_dirty(path=Path("git")) is True + assert rwrepo.is_dirty(path=PathLikeMock("git")) is True + assert rwrepo.is_dirty(path=Path("doc")) is False + assert rwrepo.is_dirty(path=PathLikeMock("doc")) is False + def test_head(self): self.assertEqual(self.rorepo.head.reference.object, self.rorepo.active_branch.object) @@ -1064,9 +800,9 @@ def test_rev_parse(self): # TODO: Dereference tag into a blob 0.1.7^{blob} - quite a special one. # Needs a tag which points to a blob. - # ref^0 returns commit being pointed to, same with ref~0, and ^{} + # ref^0 returns commit being pointed to, same with ref~0, ^{}, and ^{commit} tag = rev_parse("0.1.4") - for token in ("~0", "^0", "^{}"): + for token in ("~0", "^0", "^{}", "^{commit}"): self.assertEqual(tag.object, rev_parse("0.1.4%s" % token)) # END handle multiple tokens diff --git a/test/test_submodule.py b/test/test_submodule.py index d88f9dab0..2bf0940c9 100644 --- a/test/test_submodule.py +++ b/test/test_submodule.py @@ -28,7 +28,7 @@ from git.repo.fun import find_submodule_git_dir, touch from git.util import HIDE_WINDOWS_KNOWN_ERRORS, join_path_native, to_native_path_linux -from test.lib import TestBase, with_rw_directory, with_rw_repo +from test.lib import TestBase, with_rw_directory, with_rw_repo, PathLikeMock @contextlib.contextmanager @@ -58,6 +58,7 @@ def update(self, op, cur_count, max_count, message=""): print(op, cur_count, max_count, message) +TestRootProgress.__test__ = False prog = TestRootProgress() @@ -174,6 +175,10 @@ def _do_base_tests(self, rwrepo): sma = Submodule.add(rwrepo, sm.name, sm.path) assert sma.path == sm.path + # Adding existing as pathlike + sma = Submodule.add(rwrepo, sm.name, PathLikeMock(sm.path)) + assert sma.path == sm.path + # No url and no module at path fails. self.assertRaises(ValueError, Submodule.add, rwrepo, "newsubm", "pathtorepo", url=None) @@ -753,6 +758,22 @@ def test_add_empty_repo(self, rwdir): ) # END for each checkout mode + @with_rw_directory + @_patch_git_config("protocol.file.allow", "always") + def test_update_submodule_with_relative_path(self, rwdir): + repo_path = osp.join(rwdir, "parent") + repo = git.Repo.init(repo_path) + module_repo_path = osp.join(rwdir, "module") + module_repo = git.Repo.init(module_repo_path) + module_repo.git.commit(m="test", allow_empty=True) + repo.git.submodule("add", "../module", "module") + repo.index.commit("add submodule") + + cloned_repo_path = osp.join(rwdir, "cloned_repo") + cloned_repo = git.Repo.clone_from(repo_path, cloned_repo_path) + + cloned_repo.submodule_update(init=True, recursive=True) + @with_rw_directory @_patch_git_config("protocol.file.allow", "always") def test_list_only_valid_submodules(self, rwdir): @@ -916,7 +937,7 @@ def assert_exists(sm, value=True): csm.repo.index.commit("Have to commit submodule change for algorithm to pick it up") assert csm.url == "bar" - self.assertRaises( + self.assertRaises( # noqa: B017 Exception, rsm.update, recursive=True, @@ -1334,3 +1355,23 @@ def test_submodule_update_unsafe_options_allowed(self, rw_repo): for unsafe_option in unsafe_options: with self.assertRaises(GitCommandError): submodule.update(clone_multi_options=[unsafe_option], allow_unsafe_options=True) + + @with_rw_directory + @_patch_git_config("protocol.file.allow", "always") + def test_submodule_update_relative_url(self, rwdir): + parent_path = osp.join(rwdir, "parent") + parent_repo = git.Repo.init(parent_path) + submodule_path = osp.join(rwdir, "module") + submodule_repo = git.Repo.init(submodule_path) + submodule_repo.git.commit(m="initial commit", allow_empty=True) + + parent_repo.git.submodule("add", "../module", "module") + parent_repo.index.commit("add submodule with relative URL") + + cloned_path = osp.join(rwdir, "cloned_repo") + cloned_repo = git.Repo.clone_from(parent_path, cloned_path) + + cloned_repo.submodule_update(init=True, recursive=True) + + has_module = any(sm.name == "module" for sm in cloned_repo.submodules) + assert has_module, "Relative submodule was not updated properly" diff --git a/test/test_tree.py b/test/test_tree.py index 73158113d..629fd4d32 100644 --- a/test/test_tree.py +++ b/test/test_tree.py @@ -8,10 +8,14 @@ from pathlib import Path import subprocess +import pytest + from git.objects import Blob, Tree +from git.repo import Repo from git.util import cwd from test.lib import TestBase, with_rw_directory +from .lib.helper import PathLikeMock, with_rw_repo class TestTree(TestBase): @@ -126,12 +130,18 @@ def test_traverse(self): assert len(list(root)) == len(list(root.traverse(depth=1))) # Only choose trees. - trees_only = lambda i, d: i.type == "tree" + + def trees_only(i, _d): + return i.type == "tree" + trees = list(root.traverse(predicate=trees_only)) assert len(trees) == len([i for i in root.traverse() if trees_only(i, 0)]) # Test prune. - lib_folder = lambda t, d: t.path == "lib" + + def lib_folder(t, _d): + return t.path == "lib" + pruned_trees = list(root.traverse(predicate=trees_only, prune=lib_folder)) assert len(pruned_trees) < len(trees) @@ -155,3 +165,57 @@ def test_traverse(self): assert root[item.path] == item == root / item.path # END for each item assert found_slash + + @with_rw_repo("0.3.2.1") + def test_repo_lookup_string_path(self, rw_repo): + repo = Repo(rw_repo.git_dir) + blob = repo.tree() / ".gitignore" + assert isinstance(blob, Blob) + assert blob.hexsha == "787b3d442a113b78e343deb585ab5531eb7187fa" + + @with_rw_repo("0.3.2.1") + def test_repo_lookup_pathlike_path(self, rw_repo): + repo = Repo(rw_repo.git_dir) + blob = repo.tree() / PathLikeMock(".gitignore") + assert isinstance(blob, Blob) + assert blob.hexsha == "787b3d442a113b78e343deb585ab5531eb7187fa" + + @with_rw_repo("0.3.2.1") + def test_repo_lookup_invalid_string_path(self, rw_repo): + repo = Repo(rw_repo.git_dir) + with pytest.raises(KeyError): + repo.tree() / "doesnotexist" + + @with_rw_repo("0.3.2.1") + def test_repo_lookup_invalid_pathlike_path(self, rw_repo): + repo = Repo(rw_repo.git_dir) + with pytest.raises(KeyError): + repo.tree() / PathLikeMock("doesnotexist") + + @with_rw_repo("0.3.2.1") + def test_repo_lookup_nested_string_path(self, rw_repo): + repo = Repo(rw_repo.git_dir) + blob = repo.tree() / "git/__init__.py" + assert isinstance(blob, Blob) + assert blob.hexsha == "d87dcbdbb65d2782e14eea27e7f833a209c052f3" + + @with_rw_repo("0.3.2.1") + def test_repo_lookup_nested_pathlike_path(self, rw_repo): + repo = Repo(rw_repo.git_dir) + blob = repo.tree() / PathLikeMock("git/__init__.py") + assert isinstance(blob, Blob) + assert blob.hexsha == "d87dcbdbb65d2782e14eea27e7f833a209c052f3" + + @with_rw_repo("0.3.2.1") + def test_repo_lookup_folder_string_path(self, rw_repo): + repo = Repo(rw_repo.git_dir) + tree = repo.tree() / "git" + assert isinstance(tree, Tree) + assert tree.hexsha == "ec8ae429156d65afde4bbb3455570193b56f0977" + + @with_rw_repo("0.3.2.1") + def test_repo_lookup_folder_pathlike_path(self, rw_repo): + repo = Repo(rw_repo.git_dir) + tree = repo.tree() / PathLikeMock("git") + assert isinstance(tree, Tree) + assert tree.hexsha == "ec8ae429156d65afde4bbb3455570193b56f0977" diff --git a/test/test_util.py b/test/test_util.py index dad2f3dcd..000830f41 100644 --- a/test/test_util.py +++ b/test/test_util.py @@ -34,6 +34,7 @@ LockFile, cygpath, decygpath, + is_cygwin_git, get_user_id, remove_password_if_present, rmtree, @@ -349,6 +350,24 @@ def test_decygpath(self, wpath, cpath): assert wcpath == wpath.replace("/", "\\"), cpath +class TestIsCygwinGit: + """Tests for :func:`is_cygwin_git`""" + + def test_on_path_executable(self): + # Currently we assume tests run on Cygwin use Cygwin git. See #533 and #1455 for background. + if sys.platform == "cygwin": + assert is_cygwin_git("git") + else: + assert not is_cygwin_git("git") + + def test_none_executable(self): + assert not is_cygwin_git(None) + + def test_with_missing_uname(self): + """Test for handling when `uname` isn't in the same directory as `git`""" + assert not is_cygwin_git("/bogus_path/git") + + class _Member: """A member of an IterableList."""