diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 16d5f11bc..7f803ee43 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -6,11 +6,6 @@ updates: schedule: interval: weekly -- package-ecosystem: gitsubmodule - directory: "/" - schedule: - interval: weekly - - package-ecosystem: pre-commit directory: "/" schedule: diff --git a/.github/workflows/alpine-test.yml b/.github/workflows/alpine-test.yml index a5b63f423..b10336a15 100644 --- a/.github/workflows/alpine-test.yml +++ b/.github/workflows/alpine-test.yml @@ -63,17 +63,13 @@ jobs: - name: Install project and test dependencies run: | . .venv/bin/activate - pip install '.[test]' + pip install ./smmap ./gitdb '.[test]' - name: Show POSIX file ownership run: | ls -ld -- \ "$(pwd)" \ "$(pwd)/.git" \ - "$(pwd)/git/ext/gitdb" \ - "$(pwd)/git/ext/gitdb/.git" \ - "$(pwd)/git/ext/gitdb/gitdb/ext/smmap" \ - "$(pwd)/git/ext/gitdb/gitdb/ext/smmap/.git" \ "${HOME:?HOME is not set}/.gitconfig" \ 2>&1 || true diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 1ad1a612c..eeb68cd0c 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -62,6 +62,10 @@ jobs: with: languages: ${{ matrix.language }} build-mode: ${{ matrix.build-mode }} + config: | + paths-ignore: + - gitdb/gitdb/test/** + - smmap/smmap/test/** # If you wish to specify custom queries, you can do so here or in a config file. # By default, queries listed here will override any specified in a config file. # Prefix the list here with "+" to use these queries and those in the config file. diff --git a/.github/workflows/cygwin-test.yml b/.github/workflows/cygwin-test.yml index 6f9f347a9..b8150e93d 100644 --- a/.github/workflows/cygwin-test.yml +++ b/.github/workflows/cygwin-test.yml @@ -57,8 +57,6 @@ jobs: run: | git config --global --add safe.directory "$(pwd)" git config --global --add safe.directory "$(pwd)/.git" - git config --global --add safe.directory "$(pwd)/git/ext/gitdb" - git config --global --add safe.directory "$(pwd)/git/ext/gitdb/gitdb/ext/smmap" git config --global core.autocrlf false - name: Prepare this repo for tests @@ -84,7 +82,7 @@ jobs: - name: Install project and test dependencies run: | - pip install '.[test]' + pip install ./smmap ./gitdb '.[test]' - name: Show POSIX file ownership # Cygwin's `ls -ld` reports the NTFS Owner SID via Cygwin's SID-to-uid @@ -96,12 +94,6 @@ jobs: ls -ld -- \ "$(pwd)" \ "$(pwd)/.git" \ - "$(pwd)/git/ext/gitdb" \ - "$(pwd)/git/ext/gitdb/.git" \ - "$(pwd)/.git/modules/gitdb" \ - "$(pwd)/git/ext/gitdb/gitdb/ext/smmap" \ - "$(pwd)/git/ext/gitdb/gitdb/ext/smmap/.git" \ - "$(pwd)/.git/modules/gitdb/modules/smmap" \ "${HOME:?HOME is not set}/.gitconfig" \ 2>&1 || true @@ -114,12 +106,6 @@ jobs: $paths = @( "$pwd", "$pwd\.git", - "$pwd\git\ext\gitdb", - "$pwd\git\ext\gitdb\.git", - "$pwd\.git\modules\gitdb", - "$pwd\git\ext\gitdb\gitdb\ext\smmap", - "$pwd\git\ext\gitdb\gitdb\ext\smmap\.git", - "$pwd\.git\modules\gitdb\modules\smmap", "$env:USERPROFILE\.gitconfig" ) foreach ($p in $paths) { diff --git a/.github/workflows/dependencies.yml b/.github/workflows/dependencies.yml new file mode 100644 index 000000000..4803e733b --- /dev/null +++ b/.github/workflows/dependencies.yml @@ -0,0 +1,42 @@ +name: Dependency packages + +on: + push: + branches: [main] + pull_request: + workflow_dispatch: + +permissions: + contents: read + +jobs: + test: + runs-on: ${{ matrix.python-version == '3.7' && 'ubuntu-22.04' || 'ubuntu-latest' }} + strategy: + fail-fast: false + matrix: + project: [smmap, gitdb] + python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12", "3.13", "3.13t"] + + steps: + - uses: actions/checkout@v7 + with: + fetch-depth: 0 + - uses: actions/setup-python@v7 + with: + python-version: ${{ matrix.python-version }} + allow-prereleases: true + - name: Install test dependencies + run: | + python -m pip install --upgrade pip + python -m pip install pytest flake8 ./smmap + if test "${{ matrix.project }}" = gitdb; then python -m pip install ./gitdb; fi + - name: Lint + run: flake8 "${{ matrix.project }}/${{ matrix.project }}" --count --select=E9,F63,F7,F82 --show-source --statistics + - name: Test + run: | + if test "${{ matrix.project }}" = gitdb; then + GITDB_TEST_GIT_REPO_BASE="$(git rev-parse --git-common-dir)" pytest -o addopts= -v gitdb/gitdb/test + else + pytest -o addopts= -v smmap/smmap/test + fi diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index fe3e26236..a95f3ded6 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -89,7 +89,7 @@ jobs: - name: Install project and test dependencies run: | - pip install '.[test]' + pip install ./smmap ./gitdb '.[test]' - name: Show POSIX file ownership # Linux and macOS only. On Windows, Git Bash's `ls -ld` reports a @@ -101,10 +101,6 @@ jobs: ls -ld -- \ "$(pwd)" \ "$(pwd)/.git" \ - "$(pwd)/git/ext/gitdb" \ - "$(pwd)/git/ext/gitdb/.git" \ - "$(pwd)/git/ext/gitdb/gitdb/ext/smmap" \ - "$(pwd)/git/ext/gitdb/gitdb/ext/smmap/.git" \ "${HOME:?HOME is not set}/.gitconfig" \ 2>&1 || true @@ -118,10 +114,6 @@ jobs: $paths = @( "$pwd", "$pwd\.git", - "$pwd\git\ext\gitdb", - "$pwd\git\ext\gitdb\.git", - "$pwd\git\ext\gitdb\gitdb\ext\smmap", - "$pwd\git\ext\gitdb\gitdb\ext\smmap\.git", "$env:USERPROFILE\.gitconfig" ) foreach ($p in $paths) { diff --git a/.gitmodules b/.gitmodules deleted file mode 100644 index 251eeeec4..000000000 --- a/.gitmodules +++ /dev/null @@ -1,3 +0,0 @@ -[submodule "gitdb"] - url = https://github.com/gitpython-developers/gitdb.git - path = git/ext/gitdb diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a13e85d40..0ec820905 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,3 +1,5 @@ +exclude: ^(?:gitdb|smmap)/ + repos: - repo: https://github.com/codespell-project/codespell rev: v2.4.2 diff --git a/README.md b/README.md index 412d38205..c5a069bfe 100644 --- a/README.md +++ b/README.md @@ -101,15 +101,13 @@ In the less common case that you do not want to install test dependencies, `pip #### With editable *dependencies* (not preferred, and rarely needed) -In rare cases, you may want to work on GitPython and one or both of its [gitdb](https://github.com/gitpython-developers/gitdb) and [smmap](https://github.com/gitpython-developers/smmap) dependencies at the same time, with changes in your local working copy of gitdb or smmap immediately reflected in the behavior of your local working copy of GitPython. This can be done by making editable installations of those dependencies in the same virtual environment where you install GitPython. - -If you want to do that *and* you want the versions in GitPython's git submodules to be used, then pass `-e git/ext/gitdb` and/or `-e git/ext/gitdb/gitdb/ext/smmap` to `pip install`. This can be done in any order, and in separate `pip install` commands or the same one, so long as `-e` appears before *each* path. For example, you can install GitPython, gitdb, and smmap editably in the currently active virtual environment this way: +GitPython, [gitdb](gitdb), and [smmap](smmap) live in this repository as independently packaged projects. To work on all three at once, install each one editably in the same virtual environment: ```sh -pip install -e ".[test]" -e git/ext/gitdb -e git/ext/gitdb/gitdb/ext/smmap +pip install -e smmap -e gitdb -e ".[test]" ``` -The submodules must have been cloned for that to work, but that will already be the case if you have run `./init-tests-after-clone.sh`. You can use `pip list` to check which packages are installed editably and which are installed normally. +You can use `pip list` to check which packages are installed editably and which are installed normally. To reiterate, this approach should only rarely be used. For most development it is preferable to allow the gitdb and smmap dependencices to be retrieved automatically from PyPI in their latest stable packaged versions. diff --git a/doc/source/intro.rst b/doc/source/intro.rst index d053bd117..e1075b1c9 100644 --- a/doc/source/intro.rst +++ b/doc/source/intro.rst @@ -97,11 +97,6 @@ and cloned using:: $ git clone https://github.com/gitpython-developers/GitPython git-python -Initialize all submodules to obtain the required dependencies with:: - - $ cd git-python - $ git submodule update --init --recursive - Finally verify the installation by running unit tests:: $ python -m unittest diff --git a/git/ext/gitdb b/git/ext/gitdb deleted file mode 160000 index 335c0f661..000000000 --- a/git/ext/gitdb +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 335c0f66173eecdc7b2597c2b6c3d1fde795df30 diff --git a/git/objects/fun.py b/git/objects/fun.py index fe57da13a..ad5fbd59b 100644 --- a/git/objects/fun.py +++ b/git/objects/fun.py @@ -115,11 +115,11 @@ def tree_entries_from_data(data: bytes) -> List[EntryTup]: # Default encoding for strings in git is UTF-8. # Only use the respective unicode object if the byte stream was encoded. name_bytes = data[ns:i] - name = safe_decode(name_bytes) + name = safe_decode(bytes(name_bytes)) # Byte is NULL, get next 20. i += 1 - sha = data[i : i + 20] + sha = bytes(data[i : i + 20]) i = i + 20 out.append((sha, mode, name)) # END for each byte in data stream diff --git a/gitdb/.coveragerc b/gitdb/.coveragerc new file mode 100644 index 000000000..71b6ef701 --- /dev/null +++ b/gitdb/.coveragerc @@ -0,0 +1,7 @@ +[run] +source = gitdb + +; to make nosetests happy +[report] +include = */gitdb/* +omit = */gitdb/ext/* diff --git a/gitdb/.gitignore b/gitdb/.gitignore new file mode 100644 index 000000000..8b7da9277 --- /dev/null +++ b/gitdb/.gitignore @@ -0,0 +1,10 @@ +MANIFEST +.coverage +build/ +dist/ +*.pyc +*.o +*.so +.noseids +*.sublime-workspace +*.egg-info diff --git a/gitdb/AUTHORS b/gitdb/AUTHORS new file mode 100644 index 000000000..6c7e9b997 --- /dev/null +++ b/gitdb/AUTHORS @@ -0,0 +1,4 @@ +Creator: Sebastian Thiel + +Contributors: + - Ram Rachum (@cool-RR) diff --git a/gitdb/LICENSE b/gitdb/LICENSE new file mode 100644 index 000000000..0d6fe8bdb --- /dev/null +++ b/gitdb/LICENSE @@ -0,0 +1,42 @@ +Copyright (C) 2010, 2011 Sebastian Thiel and contributors +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + +* Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. + +* Neither the name of the GitDB project nor the names of +its contributors may be used to endorse or promote products derived +from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED +TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +Additional Licenses +------------------- +The files at +gitdb/test/fixtures/packs/pack-11fdfa9e156ab73caae3b6da867192221f2089c2.idx +and +gitdb/test/fixtures/packs/pack-11fdfa9e156ab73caae3b6da867192221f2089c2.pack +are licensed under GNU GPL as part of the git source repository, +see http://en.wikipedia.org/wiki/Git_%28software%29 for more information. + +They are not required for the actual operation, which is why they are not found +in the distribution package. diff --git a/gitdb/MANIFEST.in b/gitdb/MANIFEST.in new file mode 100644 index 000000000..b939b5ded --- /dev/null +++ b/gitdb/MANIFEST.in @@ -0,0 +1,18 @@ +include VERSION +include LICENSE +include CHANGES +include AUTHORS +include README +include MANIFEST.in + +include gitdb/_fun.c +include gitdb/_delta_apply.c +include gitdb/_delta_apply.h + +graft gitdb/test + +global-exclude .git* +global-exclude *.pyc +global-exclude *.so +global-exclude *.dll +global-exclude *.o diff --git a/gitdb/Makefile b/gitdb/Makefile new file mode 100644 index 000000000..ad0912ffe --- /dev/null +++ b/gitdb/Makefile @@ -0,0 +1,7 @@ +.PHONY: all clean + +all: + @grep -Ee '^[a-z].*:' Makefile | cut -d: -f1 | grep -vF all + +clean: + rm -rf build/ dist/ .eggs/ .tox/ diff --git a/gitdb/README.rst b/gitdb/README.rst new file mode 100644 index 000000000..45344a078 --- /dev/null +++ b/gitdb/README.rst @@ -0,0 +1,69 @@ +GitDB +===== + +GitDB allows you to access bare git repositories for reading and writing. It aims at allowing full access to loose objects as well as packs with performance and scalability in mind. It operates exclusively on streams, allowing to handle large objects with a small memory footprint. + +Installation +============ + +.. image:: https://img.shields.io/pypi/v/gitdb.svg + :target: https://pypi.python.org/pypi/gitdb/ + :alt: Latest Version +.. image:: https://img.shields.io/pypi/pyversions/gitdb.svg + :target: https://pypi.python.org/pypi/gitdb/ + :alt: Supported Python versions +.. image:: https://readthedocs.org/projects/gitdb/badge/?version=latest + :target: https://readthedocs.org/projects/gitdb/?badge=latest + :alt: Documentation Status + +From `PyPI `_:: + + pip install gitdb + +SPEEDUPS +======== + +If you want to go up to 20% faster, you can install gitdb-speedups with:: + + pip install gitdb-speedups + +However, please note that gitdb-speedups is not currently maintained. + +REQUIREMENTS +============ + +* smmap - declared as a dependency, automatically installed +* pytest - for running the tests + +SOURCE +====== + +The source is available in the GitPython repository on GitHub: + +https://github.com/gitpython-developers/GitPython/tree/main/gitdb + +Run the tests with:: + + pytest + +DEVELOPMENT +=========== + +.. image:: https://github.com/gitpython-developers/GitPython/actions/workflows/dependencies.yml/badge.svg + :target: https://github.com/gitpython-developers/GitPython/actions/workflows/dependencies.yml + +The library is considered mature, and not under active development. Its primary (known) use is in GitPython. + +INFRASTRUCTURE +============== + +* Discussions + * https://github.com/gitpython-developers/GitPython/discussions + +* Issue Tracker + * https://github.com/gitpython-developers/GitPython/issues + +LICENSE +======= + +New BSD License diff --git a/gitdb/SECURITY.md b/gitdb/SECURITY.md new file mode 100644 index 000000000..95389ff6e --- /dev/null +++ b/gitdb/SECURITY.md @@ -0,0 +1,3 @@ +# Security Policy + +See [GitPython](https://github.com/gitpython-developers/GitPython/blob/main/SECURITY.md). Vulnerabilities found in `gitdb` can be reported there. diff --git a/gitdb/doc/.gitignore b/gitdb/doc/.gitignore new file mode 100644 index 000000000..567609b12 --- /dev/null +++ b/gitdb/doc/.gitignore @@ -0,0 +1 @@ +build/ diff --git a/gitdb/doc/Makefile b/gitdb/doc/Makefile new file mode 100644 index 000000000..b10926ae2 --- /dev/null +++ b/gitdb/doc/Makefile @@ -0,0 +1,89 @@ +# Makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = sphinx-build +PAPER = +BUILDDIR = build + +# Internal variables. +PAPEROPT_a4 = -D latex_paper_size=a4 +PAPEROPT_letter = -D latex_paper_size=letter +ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source + +.PHONY: help clean html dirhtml pickle json htmlhelp qthelp latex changes linkcheck doctest + +help: + @echo "Please use \`make ' where is one of" + @echo " html to make standalone HTML files" + @echo " dirhtml to make HTML files named index.html in directories" + @echo " pickle to make pickle files" + @echo " json to make JSON files" + @echo " htmlhelp to make HTML files and a HTML help project" + @echo " qthelp to make HTML files and a qthelp project" + @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" + @echo " changes to make an overview of all changed/added/deprecated items" + @echo " linkcheck to check all external links for integrity" + @echo " doctest to run all doctests embedded in the documentation (if enabled)" + +clean: + -rm -rf $(BUILDDIR)/* + +html: + $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." + +dirhtml: + $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." + +pickle: + $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle + @echo + @echo "Build finished; now you can process the pickle files." + +json: + $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json + @echo + @echo "Build finished; now you can process the JSON files." + +htmlhelp: + $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp + @echo + @echo "Build finished; now you can run HTML Help Workshop with the" \ + ".hhp project file in $(BUILDDIR)/htmlhelp." + +qthelp: + $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp + @echo + @echo "Build finished; now you can run "qcollectiongenerator" with the" \ + ".qhcp project file in $(BUILDDIR)/qthelp, like this:" + @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/GitDB.qhcp" + @echo "To view the help file:" + @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/GitDB.qhc" + +latex: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo + @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." + @echo "Run \`make all-pdf' or \`make all-ps' in that directory to" \ + "run these through (pdf)latex." + +changes: + $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes + @echo + @echo "The overview file is in $(BUILDDIR)/changes." + +linkcheck: + $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck + @echo + @echo "Link check complete; look for any errors in the above output " \ + "or in $(BUILDDIR)/linkcheck/output.txt." + +doctest: + $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest + @echo "Testing of doctests in the sources finished, look at the " \ + "results in $(BUILDDIR)/doctest/output.txt." diff --git a/gitdb/doc/source/algorithm.rst b/gitdb/doc/source/algorithm.rst new file mode 100644 index 000000000..2e01b3fbe --- /dev/null +++ b/gitdb/doc/source/algorithm.rst @@ -0,0 +1,97 @@ +######################## +Discussion of Algorithms +######################## + +************ +Introduction +************ +As you know, the pure-python object database support for GitPython is provided by the GitDB project. It is meant to be my backup plan to ensure that the DataVault (http://www.youtube.com/user/ByronBates99?feature=mhum#p/c/2A5C6EF5BDA8DB5C ) can handle reading huge files, especially those which were consolidated into packs. A nearly fully packed state is anticipated for the data-vaults repository, and reading these packs efficiently is an essential task. + +This document informs you about my findings in the struggle to improve the way packs are read to reduce memory usage required to handle huge files. It will try to conclude where future development could go to assure big delta-packed files can be read without the need of 8GB+ RAM. + +GitDB's main feature is the use of streams, hence the amount of memory used to read a database object is minimized, at the cost of some additional processing overhead to keep track of the stream state. This works great for legacy objects, which are essentially a zip-compressed byte-stream. + +Streaming data from delta-packed objects is far more difficult though, and only technically possible within certain limits, and at relatively high processing costs. My first observation was that there doesn't appear to be 'the one and only' algorithm which is generally superior. They all have their pros and cons, but fortunately this allows the implementation to choose the one most suited based on the amount of delta streams, as well as the size of the base, which allows an early and cheap estimate of the target size of the final data. + +********************************** +Traditional Delta-Apply-Algorithms +********************************** + +The brute-force CGit delta-apply algorithm +========================================== +CGit employs a simple and relatively brute-force algorithm, which resolves all delta streams recursively. When the recursion reaches the base level of the deltas, it will be decompressed into a buffer, then the first delta gets decompressed into a second buffer. From that, the target size of the delta can be extracted, to allocated a third buffer to hold the result of the operation, which consists of reading the delta stream byte-wise, to apply the operations in order, as described by single-byte opcodes. During recursion, each target buffer of the preceding delta-apply operation is used as base buffer for the next delta-apply operation, until the last delta was applied, leaving the final target buffer as result. + +About Delta-Opcodes +------------------- +There are only two kinds of opcodes, 'add-bytes' and 'copy-from-base'. One 'add-bytes' opcode can encode up to 7 bit of additional bytes to be copied from the delta stream into the target buffer. +A 'copy-from-base' opcode encodes a 32 bit offset into the base buffer, as well as the amount of bytes to be copied, which are up to 2^24 bytes. We may conclude that delta-bases may not be larger than 2^32+2^24 bytes in the current, extensible, implementation. +When generating the delta, git prefers copy operations over add operations, as they are much more efficient. Usually, the most recent, or biggest version of a file is used as base, whereas older and smaller versions of the file are expressed by copying only portions of the newest file. As it is not efficiently possible to represent all changes that way, add-bytes operations fill the gap where needed. All this explains why git can only add 128 bytes with one opcode, as it tries to minimize their use. +This implies that recent file history can usually be extracted faster than old history, which may involve many more deltas. + +Performance considerations +-------------------------- +The performance bottleneck of this algorithm appear to be the throughput of your RAM, as both opcodes will just trigger memcpy operations from one memory location to another, times the amount of deltas to apply. This in fact is very fast, even for big files above 100 MB. +Memory allocation could become an issue as you need the base buffer, the target buffer as well as the decompressed delta stream in memory at the same time. The continuous allocation and deallocation of possibly big buffers may support memory fragmentation. Whether it really kicks in, especially on 64 bit machines, is unproven though. +Nonetheless, the cgit implementation is currently the fastest one. + +The brute-force GitDB algorithm +=============================== +Despite of working essentially the same way as the CGit brute-force algorithm, GitDB minimizes the amount of allocations to 2 + num of deltas. The amount memory allocated peaks whiles the deltas are applied, as the base and target buffer, as well as the decompressed stream, are held in memory. +To achieve this, GitDB retrieves all delta-streams in advance, and peaks into their header information to determine the maximum size of the target buffer, just by reading 512 bytes of the compressed stream. If there is more than one delta to apply, the base buffer is set large enough to hold the biggest target buffer required by the delta streams. +Now it is possible to iterate all deltas, oldest first, newest last, and apply them using the buffers. At the end of each iteration, the buffers are swapped. + +Performance Results +------------------- +The performance test is performed on an aggressively packed repository with the history of cgit. 5000 sha's are extracted and read one after another. The delta-chains have a length of 0 to about 35. The pure-python implementation can stream the data of all objects (totaling 62,2 MiB) with an average rate of 8.1 MiB/s, which equals about 654 streams/s. +There are two bottlenecks: The major is the collection of the delta streams, which involves plenty of pack-lookup. This lookup is expensive in python, and is overly expensive. Its not overly critical though, as it only limits the amount of streams per second, not the actual data rate when applying the deltas. +Applying the deltas happens to be the second bottleneck, if the files to be processed get bigger. The more opcodes have to be processed, the more python slow function calls will dominate the result. As an example, it takes nearly 8 seconds to unpack a 125 MB file, where cgit only takes 2.4 s. + +To eliminate a few performance concerns, some key routines were rewritten in C. This changes the numbers of this particular test significantly, but not drastically, as the major bottleneck (delta collection) is still in place. Another performance reduction is due to the fact that plenty of other code related to the deltas is still pure-python. +Now all 5000 objects can be read at a rate of 11.1 MiB /s, or 892 streams/s. Fortunately, unpacking a big object is now done in 2.5s, which is just a tad slower than cgit, but with possibly less memory fragmentation. + + +************************************** +Paving the way towards delta streaming +************************************** + +GitDB's reverse delta aggregation algorithm +=========================================== +The idea of this algorithm is to merge all delta streams into one, which can then be applied in just one go. + +In the current implementation, delta streams are parsed into DeltaChunks (->**DC**). Each DC represents one copy-from-base operation, or one or multiple consecutive add-bytes operations. DeltaChunks know about their target offset in the target buffer, and their size. Their target offsets are consecutive, i.e. one chunk ends where the next one begins, regarding their logical extend in the target buffer. +Add-bytes DCs additional store their data to apply, copy-from-base DCs store the offset into the base buffer from which to copy bytes. + +During processing, one starts with the latest (i.e. topmost) delta stream (->**TDS**), and iterates through its ancestor delta streams (->ADS) to merge them into the growing toplevel delta stream.. + +The merging works by following a set of rules: + * Merge into the top-level delta from the youngest ancestor delta to the oldest one + * When merging one ADS, iterate from the first to the last chunk in TDS, then: + + * skip all add-bytes DCs. If bytes are added, these will always overwrite any operation coming from any ADS at the same offset. + * copy-from-base DCs will copy a slice of the respective portion of the ADS ( as defined by their base offset ) and use it to replace the original chunk. This acts as a 'virtual' copy-from-base operation. + + * Finish the merge once all ADS have been handled, or once the TDS only consists of add-byte DCs. The remaining copy-from-base DCs will copy from the original base buffer accordingly. + +Applying the TDS is as straightforward as applying any other DS. The base buffer is required to be kept in memory. In the current implementation, a full-size target buffer is allocated to hold the result of applying the chunk information. Here it is already possible to stream the result, which is feasible only if the memory of the base buffer + the memory of the TDS are smaller than a full size target buffer. Streaming will always make sense if the peak resulting from having the base, target and TDS buffers in memory together is unaffordable. + +The memory consumption during the TDS processing is only the condensed delta-bytes, for each ADS an additional index is required which costs 8 byte per DC. When applying the TDS, one requires an allocated base buffer too.The target buffer can be allocated, but may be a writer as well. + +Performance Results +------------------- +The benchmarking context was the same as for the brute-force GitDB algorithm. This implementation is far more complex than the said brute-force implementation, which clearly reflects in the numbers. It's pure-python throughput is at only 1.1 MiB/s, which equals 89 streams/s. +The biggest performance bottleneck is the slicing of the parsed delta streams, where the program spends most of its time due to hundred thousands of calls. + +To get a more usable version of the algorithm, it was implemented in C, such that python must do no more than two calls to get all the work done. The first prepares the TDS, the second applies it, writing it into a target buffer. +The throughput reaches 15.2 MiB/s, which equals 1221 streams/s, which makes it nearly 14 times faster than the pure python version, and amazingly even 1.35 times faster than the brute-force C implementation. As a comparison, cgit is able to stream about 20 MiB when controlling it through a pipe. GitDBs performance may still improve once pack access is reimplemented in C as well. + +A 125 MB file took 2.5 seconds to unpack for instance, which is only 20% slower than the c implementation of the brute-force algorithm. + + +Future work +=========== + +Another very promising option is that streaming of delta data is indeed possible. Depending on the configuration of the copy-from-base operations, different optimizations could be applied to reduce the amount of memory required for the final processed delta stream. Some configurations may even allow it to stream data from the base buffer, instead of pre-loading it for random access. + +The ability to stream files at reduced memory costs would only be feasible for big files, and would have to be paid with extra pre-processing time. + +A very first and simple implementation could avoid memory peaks by streaming the TDS in conjunction with a base buffer, instead of writing everything into a fully allocated target buffer. diff --git a/gitdb/doc/source/api.rst b/gitdb/doc/source/api.rst new file mode 100644 index 000000000..9dea95764 --- /dev/null +++ b/gitdb/doc/source/api.rst @@ -0,0 +1,113 @@ +.. _api-label: + +############# +API Reference +############# + +**************** +Database.Base +**************** + +.. automodule:: gitdb.db.base + :members: + :undoc-members: + +**************** +Database.Git +**************** + +.. automodule:: gitdb.db.git + :members: + :undoc-members: + +**************** +Database.Loose +**************** + +.. automodule:: gitdb.db.loose + :members: + :undoc-members: + +**************** +Database.Memory +**************** + +.. automodule:: gitdb.db.mem + :members: + :undoc-members: + +**************** +Database.Pack +**************** + +.. automodule:: gitdb.db.pack + :members: + :undoc-members: + +****************** +Database.Reference +****************** + +.. automodule:: gitdb.db.ref + :members: + :undoc-members: + +************ +Base +************ + +.. automodule:: gitdb.base + :members: + :undoc-members: + +************ +Functions +************ + +.. automodule:: gitdb.fun + :members: + :undoc-members: + +************ +Pack +************ + +.. automodule:: gitdb.pack + :members: + :undoc-members: + +************ +Streams +************ + +.. automodule:: gitdb.stream + :members: + :undoc-members: + +************ +Types +************ + +.. automodule:: gitdb.typ + :members: + :undoc-members: + + +************ +Utilities +************ + +.. automodule:: gitdb.util + :members: + :undoc-members: + + + + + + + + + + + diff --git a/gitdb/doc/source/changes.rst b/gitdb/doc/source/changes.rst new file mode 100644 index 000000000..b4340e4dc --- /dev/null +++ b/gitdb/doc/source/changes.rst @@ -0,0 +1,141 @@ +######### +Changelog +######### + +****** +4.0.12 +****** + +- various improvements - please see the release on GitHub for details. + +****** +4.0.11 +****** + +- various improvements - please see the release on GitHub for details. + +****** +4.0.10 +****** + +- improvements to the way external packages are imported. + +***** +4.0.9 +***** + +- re-release of 4.0.8 to get a valid signature. + +***** +4.0.8 +***** + +* drop support for python 3.4 and 3.5 due to EOL +* Updated upper bound for smmap requirement in setup.py + (`#69 `_) + +***** +4.0.7 +***** + +* Updated upper bound for smmap requirement in setup.py + (`#69 `_) + +***** +4.0.6 +***** + +* Bumped upper bound for smmap requirement + (`#67 `_, + `#68 `_) + +***** +4.0.5 +***** + +* Re-release of 4.0.4, with known signature + +***** +4.0.4 +***** + +* Support for PyOxidizer + +***** +4.0.2 +***** + +* Updated to release as Pure Python Wheel rather than Universal Wheel + (`#62 `_) + +***** +4.0.1 +***** + +* Switched back to the gitdb package name on PyPI and fixed the gitdb2 mirror package + (`#59 `_) +* Switched back to require smmap package and fixed version requirement to >= 3.0.1, < 4 + (`#59 `_) +* Updated smmap submodule + +*********** +3.0.3.post1 +*********** + +* Fixed changelogs for v3.0.2 and v3.0.3 + +***** +3.0.3 +***** + +* Changed ``force_bytes`` to use UTF-8 encoding by default + (`#49 `_) +* Restricted smmap2 version requirement to < 3 +* Updated requirements.txt + +***** +3.0.2 +***** + +* Removed Python 2 compatibility shims + (`#56 `_) + +***** +0.6.1 +***** + +* Fixed possibly critical error, see https://github.com/gitpython-developers/GitPython/issues/220 + + - However, it only seems to occur on high-entropy data and didn't reoccour after the fix + +***** +0.6.0 +***** + +* Added support got python 3.X +* Removed all `async` dependencies and all `*_async` versions of methods with it. + +***** +0.5.4 +***** +* Adjusted implementation to use the SlidingMemoryManager by default in python 2.6 for efficiency reasons. In Python 2.4, the StaticMemoryManager will be used instead. + +***** +0.5.3 +***** +* Added support for smmap. SmartMMap allows resources to be managed and controlled. This brings the implementation closer to the way git handles memory maps, such that unused cached memory maps will automatically be freed once a resource limit is hit. The memory limit on 32 bit systems remains though as a sliding mmap implementation is not used for performance reasons. + +***** +0.5.2 +***** +* Improved performance of the c implementation, which now uses reverse-delta-aggregation to make a memory bound operation CPU bound. + +***** +0.5.1 +***** +* Restored most basic python 2.4 compatibility, such that gitdb can be imported within python 2.4, pack access cannot work though. This at least allows Super-Projects to provide their own workarounds, or use everything but pack support. + +***** +0.5.0 +***** +Initial Release diff --git a/gitdb/doc/source/conf.py b/gitdb/doc/source/conf.py new file mode 100644 index 000000000..b387f6094 --- /dev/null +++ b/gitdb/doc/source/conf.py @@ -0,0 +1,193 @@ +# +# GitDB documentation build configuration file, created by +# sphinx-quickstart on Wed Jun 30 00:01:32 2010. +# +# This file is execfile()d with the current directory set to its containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +import sys +import os + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +sys.path.append(os.path.abspath('../../')) + +# -- General configuration ----------------------------------------------------- + +# Add any Sphinx extension module names here, as strings. They can be extensions +# coming with Sphinx (named 'sphinx.ext.*') or your custom ones. +extensions = ['sphinx.ext.autodoc'] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['.templates'] + +# The suffix of source filenames. +source_suffix = '.rst' + +# The encoding of source files. +#source_encoding = 'utf-8' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = 'GitDB' +copyright = '2011, Sebastian Thiel' + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# +# The short X.Y version. +version = '0.5' +# The full version, including alpha/beta/rc tags. +release = '0.5.3' + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +#language = None + +# There are two options for replacing |today|: either, you set today to some +# non-false value, then it is used: +#today = '' +# Else, today_fmt is used as the format for a strftime call. +#today_fmt = '%B %d, %Y' + +# List of documents that shouldn't be included in the build. +#unused_docs = [] + +# List of directories, relative to source directory, that shouldn't be searched +# for source files. +exclude_trees = [] + +# The reST default role (used for this markup: `text`) to use for all documents. +#default_role = None + +# If true, '()' will be appended to :func: etc. cross-reference text. +#add_function_parentheses = True + +# If true, the current module name will be prepended to all description +# unit titles (such as .. function::). +#add_module_names = True + +# If true, sectionauthor and moduleauthor directives will be shown in the +# output. They are ignored by default. +#show_authors = False + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +# A list of ignored prefixes for module index sorting. +#modindex_common_prefix = [] + + +# -- Options for HTML output --------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. Major themes that come with +# Sphinx are currently 'default' and 'sphinxdoc'. +html_theme = 'default' + +html_theme_options = { + "stickysidebar": "true" +} + +# Add any paths that contain custom themes here, relative to this directory. +#html_theme_path = [] + +# The name for this set of Sphinx documents. If None, it defaults to +# " v documentation". +#html_title = None + +# A shorter title for the navigation bar. Default is the same as html_title. +#html_short_title = None + +# The name of an image file (relative to this directory) to place at the top +# of the sidebar. +#html_logo = None + +# The name of an image file (within the static path) to use as favicon of the +# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 +# pixels large. +#html_favicon = None + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +#html_static_path = ['.static'] + +# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, +# using the given strftime format. +#html_last_updated_fmt = '%b %d, %Y' + +# If true, SmartyPants will be used to convert quotes and dashes to +# typographically correct entities. +#html_use_smartypants = True + +# Custom sidebar templates, maps document names to template names. +#html_sidebars = {} + +# Additional templates that should be rendered to pages, maps page names to +# template names. +#html_additional_pages = {} + +# If false, no module index is generated. +#html_use_modindex = True + +# If false, no index is generated. +#html_use_index = True + +# If true, the index is split into individual pages for each letter. +#html_split_index = False + +# If true, links to the reST sources are added to the pages. +#html_show_sourcelink = True + +# If true, an OpenSearch description file will be output, and all pages will +# contain a tag referring to it. The value of this option must be the +# base URL from which the finished HTML is served. +#html_use_opensearch = '' + +# If nonempty, this is the file name suffix for HTML files (e.g. ".xhtml"). +#html_file_suffix = '' + +# Output file base name for HTML help builder. +htmlhelp_basename = 'GitDBdoc' + + +# -- Options for LaTeX output -------------------------------------------------- + +# The paper size ('letter' or 'a4'). +#latex_paper_size = 'letter' + +# The font size ('10pt', '11pt' or '12pt'). +#latex_font_size = '10pt' + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, author, documentclass [howto/manual]). +latex_documents = [ + ('index', 'GitDB.tex', 'GitDB Documentation', + 'Sebastian Thiel', 'manual'), +] + +# The name of an image file (relative to this directory) to place at the top of +# the title page. +#latex_logo = None + +# For "manual" documents, if this is true, then toplevel headings are parts, +# not chapters. +#latex_use_parts = False + +# Additional stuff for the LaTeX preamble. +#latex_preamble = '' + +# Documents to append as an appendix to all manuals. +#latex_appendices = [] + +# If false, no module index is generated. +#latex_use_modindex = True diff --git a/gitdb/doc/source/index.rst b/gitdb/doc/source/index.rst new file mode 100644 index 000000000..5223e6bd9 --- /dev/null +++ b/gitdb/doc/source/index.rst @@ -0,0 +1,26 @@ +.. GitDB documentation master file, created by + sphinx-quickstart on Wed Jun 30 00:01:32 2010. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +Welcome to GitDB's documentation! +================================= + +Contents: + +.. toctree:: + :maxdepth: 2 + + intro + tutorial + api + algorithm + changes + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` + diff --git a/gitdb/doc/source/intro.rst b/gitdb/doc/source/intro.rst new file mode 100644 index 000000000..434138616 --- /dev/null +++ b/gitdb/doc/source/intro.rst @@ -0,0 +1,38 @@ +######## +Overview +######## + +The *GitDB* project implements interfaces to allow read and write access to git repositories. In its core lies the *db* package, which contains all database types necessary to read a complete git repository. These are the ``LooseObjectDB``, the ``PackedDB`` and the ``ReferenceDB`` which are combined into the ``GitDB`` to combine every aspect of the git database. + +For this to work, GitDB implements pack reading, as well as loose object reading and writing. Data is always encapsulated in streams, which allows huge files to be handled as well as small ones, usually only chunks of the stream are kept in memory for processing, never the whole stream at once. + +Interfaces are used to describe the API, making it easy to provide alternate implementations. + +================ +Installing GitDB +================ +Its easiest to install gitdb using the *pip* program:: + + $ pip install gitdb + +As the command will install gitdb in your respective python distribution, you will most likely need root permissions to authorize the required changes. + +If you have downloaded the source archive, the package can be installed by running the ``setup.py`` script:: + + $ python setup.py install + +=============== +Getting Started +=============== +It is advised to have a look at the :ref:`Usage Guide ` for a brief introduction on the different database implementations. + +================= +Source Repository +================= +The latest source can be cloned using git from github: + + * https://github.com/gitpython-developers/gitdb + +License Information +=================== +*GitDB* is licensed under the New BSD License. diff --git a/gitdb/doc/source/tutorial.rst b/gitdb/doc/source/tutorial.rst new file mode 100644 index 000000000..55a737f6b --- /dev/null +++ b/gitdb/doc/source/tutorial.rst @@ -0,0 +1,114 @@ +.. _tutorial-label: + +########### +Usage Guide +########### +This text briefly introduces you to the basic design decisions and accompanying types. + +****** +Design +****** +The *GitDB* project models a standard git object database and implements it in pure python. This means that data, being classified by one of four types, can can be stored in the database and will in future be referred to by the generated SHA1 key, which is a 20 byte string within python. + +*GitDB* implements *RW* access to loose objects, as well as *RO* access to packed objects. Compound Databases allow to combine multiple object databases into one. + +All data is read and written using streams, which effectively prevents more than a chunk of the data being kept in memory at once mostly [#]_. + +******* +Streams +******* +In order to assure the object database can handle objects of any size, a stream interface is used for data retrieval as well as to fill data into the database. + +Basic Stream Types +================== +There are two fundamentally different types of streams, **IStream**\ s and **OStream**\ s. IStreams are mutable and are used to provide data streams to the database to create new objects. + +OStreams are immutable and are used to read data from the database. The base of this type, **OInfo**, contains only type and size information of the queried object, but no stream, which is slightly faster to retrieve depending on the database. + +OStreams are tuples, IStreams are lists. Both, OInfo and OStream, have the same member ordering which allows quick conversion from one type to another. + +**************************** +Data Query and Data Addition +**************************** +Databases support query and/or addition of objects using simple interfaces. They are called **ObjectDBR** for read-only access, and **ObjectDBW** for write access to create new objects. + +Both have two sets of methods, one of which allows interacting with single objects, the other one allowing to handle a stream of objects simultaneously and asynchronously. + +Acquiring information about an object from a database is easy if you have a SHA1 to refer to the object:: + + + ldb = LooseObjectDB(fixture_path("../../../.git/objects")) + + for sha1 in ldb.sha_iter(): + oinfo = ldb.info(sha1) + ostream = ldb.stream(sha1) + assert oinfo[:3] == ostream[:3] + + assert len(ostream.read()) == ostream.size + # END for each sha in database + +To store information, you prepare an *IStream* object with the required information. The provided stream will be read and converted into an object, and the respective 20 byte SHA1 identifier is stored in the IStream object:: + + data = "my data" + istream = IStream("blob", len(data), StringIO(data)) + + # the object does not yet have a sha + assert istream.binsha is None + ldb.store(istream) + # now the sha is set + assert len(istream.binsha) == 20 + assert ldb.has_object(istream.binsha) + +********************** +Asynchronous Operation +********************** +For each read or write method that allows a single-object to be handled, an *_async* version exists which reads items to be processed from a channel, and writes the operation's result into an output channel that is read by the caller or by other async methods, to support chaining. + +Using asynchronous operations is easy, but chaining multiple operations together to form a complex one would require you to read the docs of the *async* package. At the current time, due to the *GIL*, the *GitDB* can only achieve true concurrency during zlib compression and decompression if big objects, if the respective c modules where compiled in *async*. + +Asynchronous operations are scheduled by a *ThreadPool* which resides in the *gitdb.util* module:: + + from gitdb.util import pool + + # set the pool to use two threads + pool.set_size(2) + + # synchronize the mode of operation + pool.set_size(0) + + +Use async methods with readers, which supply items to be processed. The result is given through readers as well:: + + from async import IteratorReader + + # Create a reader from an iterator + reader = IteratorReader(ldb.sha_iter()) + + # get reader for object streams + info_reader = ldb.stream_async(reader) + + # read one + info = info_reader.read(1)[0] + + # read all the rest until depletion + ostreams = info_reader.read() + + + +********* +Databases +********* +A database implements different interfaces, one if which will always be the *ObjectDBR* interface to support reading of object information and streams. + +The *Loose Object Database* as well as the *Packed Object Database* are *File Databases*, hence they operate on a directory which contains files they can read. + +File databases implementing the *ObjectDBW* interface can also be forced to write their output into the specified stream, using the ``set_ostream`` method. This effectively allows you to redirect its output to anywhere you like. + +*Compound Databases* are not implementing their own access type, but instead combine multiple database implementations into one. Examples for this database type are the *Reference Database*, which reads object locations from a file, and the *GitDB* which combines loose, packed and referenced objects into one database interface. + +For more information about the individual database types, please see the :ref:`API Reference `, and the unittests for the respective types. + + +---- + +.. [#] When reading streams from packs, all deltas are currently applied and the result written into a memory map before the first byte is returned. Future versions of the delta-apply algorithm might improve on this. diff --git a/gitdb/gitdb.pro b/gitdb/gitdb.pro new file mode 100644 index 000000000..a682615c4 --- /dev/null +++ b/gitdb/gitdb.pro @@ -0,0 +1,48 @@ + +OTHER_FILES += \ + setup.py \ + README.rst \ + MANIFEST \ + Makefile \ + LICENSE \ + AUTHORS \ + gitdb/util.py \ + gitdb/typ.py \ + gitdb/stream.py \ + gitdb/pack.py \ + gitdb/__init__.py \ + gitdb/fun.py \ + gitdb/exc.py \ + gitdb/base.py \ + doc/source/tutorial.rst \ + doc/source/intro.rst \ + doc/source/index.rst \ + doc/source/conf.py \ + doc/source/changes.rst \ + doc/source/api.rst \ + doc/source/algorithm.rst \ + gitdb/db/ref.py \ + gitdb/db/pack.py \ + gitdb/db/mem.py \ + gitdb/db/loose.py \ + gitdb/db/__init__.py \ + gitdb/db/git.py \ + gitdb/db/base.py \ + gitdb/test/test_util.py \ + gitdb/test/test_stream.py \ + gitdb/test/test_pack.py \ + gitdb/test/test_example.py \ + gitdb/test/test_base.py \ + gitdb/test/lib.py \ + gitdb/test/__init__.py \ + gitdb/test/performance/test_stream.py \ + gitdb/test/performance/test_pack_streaming.py \ + gitdb/test/performance/test_pack.py \ + gitdb/test/performance/lib.py + +HEADERS += \ + gitdb/_delta_apply.h + +SOURCES += \ + gitdb/_fun.c \ + gitdb/_delta_apply.c diff --git a/gitdb/gitdb.pro.user b/gitdb/gitdb.pro.user new file mode 100644 index 000000000..3ca1e2182 --- /dev/null +++ b/gitdb/gitdb.pro.user @@ -0,0 +1,266 @@ + + + + ProjectExplorer.Project.ActiveTarget + 0 + + + ProjectExplorer.Project.EditorSettings + + Default + + + + ProjectExplorer.Project.Target.0 + + Desktop + + Qt4ProjectManager.Target.DesktopTarget + 0 + 0 + 1 + + + 0 + Build + + ProjectExplorer.BuildSteps.Build + + + + Make + + Qt4ProjectManager.MakeStep + true + + clean + + + + 1 + Clean + + ProjectExplorer.BuildSteps.Clean + + 2 + false + + Qt in PATH Release + + Qt4ProjectManager.Qt4BuildConfiguration + 0 + /home/byron/projects/git-python/git/ext/gitdb/gitdbpro-build-desktop + 3 + 0 + false + + + + + qmake + + QtProjectManager.QMakeBuildStep + + + + Make + + Qt4ProjectManager.MakeStep + false + + + + 2 + Build + + ProjectExplorer.BuildSteps.Build + + + + Make + + Qt4ProjectManager.MakeStep + true + + clean + + + + 1 + Clean + + ProjectExplorer.BuildSteps.Clean + + 2 + false + + Qt in PATH Debug + + Qt4ProjectManager.Qt4BuildConfiguration + 2 + /home/byron/projects/git-python/git/ext/gitdb/gitdbpro-build-desktop + 3 + 0 + true + + + + + qmake + + QtProjectManager.QMakeBuildStep + + + + Make + + Qt4ProjectManager.MakeStep + false + + + + 2 + Build + + ProjectExplorer.BuildSteps.Build + + + + Make + + Qt4ProjectManager.MakeStep + true + + clean + + + + 1 + Clean + + ProjectExplorer.BuildSteps.Clean + + 2 + false + + Qt 4.6.2 OpenSource Release + + Qt4ProjectManager.Qt4BuildConfiguration + 0 + /home/byron/projects/git-python/git/ext/gitdb/gitdbpro-build-desktop + 2 + 0 + true + + + + + qmake + + QtProjectManager.QMakeBuildStep + + + + Make + + Qt4ProjectManager.MakeStep + false + + + + 2 + Build + + ProjectExplorer.BuildSteps.Build + + + + Make + + Qt4ProjectManager.MakeStep + true + + clean + + + + 1 + Clean + + ProjectExplorer.BuildSteps.Clean + + 2 + false + + Qt 4.6.2 OpenSource Debug + + Qt4ProjectManager.Qt4BuildConfiguration + 2 + /home/byron/projects/git-python/git/ext/gitdb/gitdbpro-build-desktop + 2 + 0 + true + + 4 + + + 0 + Deploy + + ProjectExplorer.BuildSteps.Deploy + + 1 + No deployment + + ProjectExplorer.DefaultDeployConfiguration + + 1 + + gitdb + + Qt4ProjectManager.Qt4RunConfiguration + 2 + + gitdb.pro + false + false + + false + + 3768 + true + false + + + + /usr/bin/pytest + gitdb/test/test_pack.py + + 2 + python + false + + $BUILDDIR + Run python + test-pack + ProjectExplorer.CustomExecutableRunConfiguration + 3768 + true + false + + 2 + + + + ProjectExplorer.Project.TargetCount + 1 + + + ProjectExplorer.Project.Updater.EnvironmentId + {d38778e3-6b24-4419-8fc3-c8cc320f55e0} + + + ProjectExplorer.Project.Updater.FileVersion + 8 + + diff --git a/gitdb/gitdb/__init__.py b/gitdb/gitdb/__init__.py new file mode 100644 index 000000000..04920428d --- /dev/null +++ b/gitdb/gitdb/__init__.py @@ -0,0 +1,16 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: https://opensource.org/license/bsd-3-clause/ +"""Initialize the object database module""" + +__author__ = "Sebastian Thiel" +__contact__ = "byronimo@gmail.com" +__homepage__ = "https://github.com/gitpython-developers/GitPython/tree/main/gitdb" +version_info = (4, 0, 12) +__version__ = '.'.join(str(i) for i in version_info) + +# default imports +from gitdb.base import * +from gitdb.db import * +from gitdb.stream import * diff --git a/gitdb/gitdb/base.py b/gitdb/gitdb/base.py new file mode 100644 index 000000000..9a23a4f71 --- /dev/null +++ b/gitdb/gitdb/base.py @@ -0,0 +1,315 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: https://opensource.org/license/bsd-3-clause/ +"""Module with basic data structures - they are designed to be lightweight and fast""" +from gitdb.util import bin_to_hex + +from gitdb.fun import ( + type_id_to_type_map, + type_to_type_id_map +) + +__all__ = ('OInfo', 'OPackInfo', 'ODeltaPackInfo', + 'OStream', 'OPackStream', 'ODeltaPackStream', + 'IStream', 'InvalidOInfo', 'InvalidOStream') + +#{ ODB Bases + + +class OInfo(tuple): + + """Carries information about an object in an ODB, providing information + about the binary sha of the object, the type_string as well as the uncompressed size + in bytes. + + It can be accessed using tuple notation and using attribute access notation:: + + assert dbi[0] == dbi.binsha + assert dbi[1] == dbi.type + assert dbi[2] == dbi.size + + The type is designed to be as lightweight as possible.""" + __slots__ = tuple() + + def __new__(cls, sha, type, size): + return tuple.__new__(cls, (sha, type, size)) + + def __init__(self, *args): + tuple.__init__(self) + + #{ Interface + @property + def binsha(self): + """:return: our sha as binary, 20 bytes""" + return self[0] + + @property + def hexsha(self): + """:return: our sha, hex encoded, 40 bytes""" + return bin_to_hex(self[0]) + + @property + def type(self): + return self[1] + + @property + def type_id(self): + return type_to_type_id_map[self[1]] + + @property + def size(self): + return self[2] + #} END interface + + +class OPackInfo(tuple): + + """As OInfo, but provides a type_id property to retrieve the numerical type id, and + does not include a sha. + + Additionally, the pack_offset is the absolute offset into the packfile at which + all object information is located. The data_offset property points to the absolute + location in the pack at which that actual data stream can be found.""" + __slots__ = tuple() + + def __new__(cls, packoffset, type, size): + return tuple.__new__(cls, (packoffset, type, size)) + + def __init__(self, *args): + tuple.__init__(self) + + #{ Interface + + @property + def pack_offset(self): + return self[0] + + @property + def type(self): + return type_id_to_type_map[self[1]] + + @property + def type_id(self): + return self[1] + + @property + def size(self): + return self[2] + + #} END interface + + +class ODeltaPackInfo(OPackInfo): + + """Adds delta specific information, + Either the 20 byte sha which points to some object in the database, + or the negative offset from the pack_offset, so that pack_offset - delta_info yields + the pack offset of the base object""" + __slots__ = tuple() + + def __new__(cls, packoffset, type, size, delta_info): + return tuple.__new__(cls, (packoffset, type, size, delta_info)) + + #{ Interface + @property + def delta_info(self): + return self[3] + #} END interface + + +class OStream(OInfo): + + """Base for object streams retrieved from the database, providing additional + information about the stream. + Generally, ODB streams are read-only as objects are immutable""" + __slots__ = tuple() + + def __new__(cls, sha, type, size, stream, *args, **kwargs): + """Helps with the initialization of subclasses""" + return tuple.__new__(cls, (sha, type, size, stream)) + + def __init__(self, *args, **kwargs): + tuple.__init__(self) + + #{ Stream Reader Interface + + def read(self, size=-1): + return self[3].read(size) + + @property + def stream(self): + return self[3] + + #} END stream reader interface + + +class ODeltaStream(OStream): + + """Uses size info of its stream, delaying reads""" + + def __new__(cls, sha, type, size, stream, *args, **kwargs): + """Helps with the initialization of subclasses""" + return tuple.__new__(cls, (sha, type, size, stream)) + + #{ Stream Reader Interface + + @property + def size(self): + return self[3].size + + #} END stream reader interface + + +class OPackStream(OPackInfo): + + """Next to pack object information, a stream outputting an undeltified base object + is provided""" + __slots__ = tuple() + + def __new__(cls, packoffset, type, size, stream, *args): + """Helps with the initialization of subclasses""" + return tuple.__new__(cls, (packoffset, type, size, stream)) + + #{ Stream Reader Interface + def read(self, size=-1): + return self[3].read(size) + + @property + def stream(self): + return self[3] + #} END stream reader interface + + +class ODeltaPackStream(ODeltaPackInfo): + + """Provides a stream outputting the uncompressed offset delta information""" + __slots__ = tuple() + + def __new__(cls, packoffset, type, size, delta_info, stream): + return tuple.__new__(cls, (packoffset, type, size, delta_info, stream)) + + #{ Stream Reader Interface + def read(self, size=-1): + return self[4].read(size) + + @property + def stream(self): + return self[4] + #} END stream reader interface + + +class IStream(list): + + """Represents an input content stream to be fed into the ODB. It is mutable to allow + the ODB to record information about the operations outcome right in this instance. + + It provides interfaces for the OStream and a StreamReader to allow the instance + to blend in without prior conversion. + + The only method your content stream must support is 'read'""" + __slots__ = tuple() + + def __new__(cls, type, size, stream, sha=None): + return list.__new__(cls, (sha, type, size, stream, None)) + + def __init__(self, type, size, stream, sha=None): + list.__init__(self, (sha, type, size, stream, None)) + + #{ Interface + @property + def hexsha(self): + """:return: our sha, hex encoded, 40 bytes""" + return bin_to_hex(self[0]) + + def _error(self): + """:return: the error that occurred when processing the stream, or None""" + return self[4] + + def _set_error(self, exc): + """Set this input stream to the given exc, may be None to reset the error""" + self[4] = exc + + error = property(_error, _set_error) + + #} END interface + + #{ Stream Reader Interface + + def read(self, size=-1): + """Implements a simple stream reader interface, passing the read call on + to our internal stream""" + return self[3].read(size) + + #} END stream reader interface + + #{ interface + + def _set_binsha(self, binsha): + self[0] = binsha + + def _binsha(self): + return self[0] + + binsha = property(_binsha, _set_binsha) + + def _type(self): + return self[1] + + def _set_type(self, type): + self[1] = type + + type = property(_type, _set_type) + + def _size(self): + return self[2] + + def _set_size(self, size): + self[2] = size + + size = property(_size, _set_size) + + def _stream(self): + return self[3] + + def _set_stream(self, stream): + self[3] = stream + + stream = property(_stream, _set_stream) + + #} END odb info interface + + +class InvalidOInfo(tuple): + + """Carries information about a sha identifying an object which is invalid in + the queried database. The exception attribute provides more information about + the cause of the issue""" + __slots__ = tuple() + + def __new__(cls, sha, exc): + return tuple.__new__(cls, (sha, exc)) + + def __init__(self, sha, exc): + tuple.__init__(self, (sha, exc)) + + @property + def binsha(self): + return self[0] + + @property + def hexsha(self): + return bin_to_hex(self[0]) + + @property + def error(self): + """:return: exception instance explaining the failure""" + return self[1] + + +class InvalidOStream(InvalidOInfo): + + """Carries information about an invalid ODB stream""" + __slots__ = tuple() + +#} END ODB Bases diff --git a/gitdb/gitdb/const.py b/gitdb/gitdb/const.py new file mode 100644 index 000000000..6391d796f --- /dev/null +++ b/gitdb/gitdb/const.py @@ -0,0 +1,4 @@ +BYTE_SPACE = b' ' +NULL_BYTE = b'\0' +NULL_HEX_SHA = "0" * 40 +NULL_BIN_SHA = NULL_BYTE * 20 diff --git a/gitdb/gitdb/db/__init__.py b/gitdb/gitdb/db/__init__.py new file mode 100644 index 000000000..20fd2280d --- /dev/null +++ b/gitdb/gitdb/db/__init__.py @@ -0,0 +1,11 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: https://opensource.org/license/bsd-3-clause/ + +from gitdb.db.base import * +from gitdb.db.loose import * +from gitdb.db.mem import * +from gitdb.db.pack import * +from gitdb.db.git import * +from gitdb.db.ref import * diff --git a/gitdb/gitdb/db/base.py b/gitdb/gitdb/db/base.py new file mode 100644 index 000000000..7312fe00f --- /dev/null +++ b/gitdb/gitdb/db/base.py @@ -0,0 +1,278 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: https://opensource.org/license/bsd-3-clause/ +"""Contains implementations of database retrieveing objects""" +from gitdb.util import ( + join, + LazyMixin, + hex_to_bin +) + +from gitdb.utils.encoding import force_text +from gitdb.exc import ( + BadObject, + AmbiguousObjectName +) + +from itertools import chain +from functools import reduce + + +__all__ = ('ObjectDBR', 'ObjectDBW', 'FileDBBase', 'CompoundDB', 'CachingDB') + + +class ObjectDBR: + + """Defines an interface for object database lookup. + Objects are identified either by their 20 byte bin sha""" + + def __contains__(self, sha): + return self.has_obj + + #{ Query Interface + def has_object(self, sha): + """ + Whether the object identified by the given 20 bytes + binary sha is contained in the database + + :return: True if the object identified by the given 20 bytes + binary sha is contained in the database""" + raise NotImplementedError("To be implemented in subclass") + + def info(self, sha): + """ :return: OInfo instance + :param sha: bytes binary sha + :raise BadObject:""" + raise NotImplementedError("To be implemented in subclass") + + def stream(self, sha): + """:return: OStream instance + :param sha: 20 bytes binary sha + :raise BadObject:""" + raise NotImplementedError("To be implemented in subclass") + + def size(self): + """:return: amount of objects in this database""" + raise NotImplementedError() + + def sha_iter(self): + """Return iterator yielding 20 byte shas for all objects in this data base""" + raise NotImplementedError() + + #} END query interface + + +class ObjectDBW: + + """Defines an interface to create objects in the database""" + + def __init__(self, *args, **kwargs): + self._ostream = None + + #{ Edit Interface + def set_ostream(self, stream): + """ + Adjusts the stream to which all data should be sent when storing new objects + + :param stream: if not None, the stream to use, if None the default stream + will be used. + :return: previously installed stream, or None if there was no override + :raise TypeError: if the stream doesn't have the supported functionality""" + cstream = self._ostream + self._ostream = stream + return cstream + + def ostream(self): + """ + Return the output stream + + :return: overridden output stream this instance will write to, or None + if it will write to the default stream""" + return self._ostream + + def store(self, istream): + """ + Create a new object in the database + :return: the input istream object with its sha set to its corresponding value + + :param istream: IStream compatible instance. If its sha is already set + to a value, the object will just be stored in the our database format, + in which case the input stream is expected to be in object format ( header + contents ). + :raise IOError: if data could not be written""" + raise NotImplementedError("To be implemented in subclass") + + #} END edit interface + + +class FileDBBase: + + """Provides basic facilities to retrieve files of interest, including + caching facilities to help mapping hexsha's to objects""" + + def __init__(self, root_path): + """Initialize this instance to look for its files at the given root path + All subsequent operations will be relative to this path + :raise InvalidDBRoot: + **Note:** The base will not perform any accessablity checking as the base + might not yet be accessible, but become accessible before the first + access.""" + super().__init__() + self._root_path = root_path + + #{ Interface + def root_path(self): + """:return: path at which this db operates""" + return self._root_path + + def db_path(self, rela_path): + """ + :return: the given relative path relative to our database root, allowing + to pontentially access datafiles""" + return join(self._root_path, force_text(rela_path)) + #} END interface + + +class CachingDB: + + """A database which uses caches to speed-up access""" + + #{ Interface + def update_cache(self, force=False): + """ + Call this method if the underlying data changed to trigger an update + of the internal caching structures. + + :param force: if True, the update must be performed. Otherwise the implementation + may decide not to perform an update if it thinks nothing has changed. + :return: True if an update was performed as something change indeed""" + + # END interface + + +def _databases_recursive(database, output): + """Fill output list with database from db, in order. Deals with Loose, Packed + and compound databases.""" + if isinstance(database, CompoundDB): + dbs = database.databases() + output.extend(db for db in dbs if not isinstance(db, CompoundDB)) + for cdb in (db for db in dbs if isinstance(db, CompoundDB)): + _databases_recursive(cdb, output) + else: + output.append(database) + # END handle database type + + +class CompoundDB(ObjectDBR, LazyMixin, CachingDB): + + """A database which delegates calls to sub-databases. + + Databases are stored in the lazy-loaded _dbs attribute. + Define _set_cache_ to update it with your databases""" + + def _set_cache_(self, attr): + if attr == '_dbs': + self._dbs = list() + elif attr == '_db_cache': + self._db_cache = dict() + else: + super()._set_cache_(attr) + + def _db_query(self, sha): + """:return: database containing the given 20 byte sha + :raise BadObject:""" + # most databases use binary representations, prevent converting + # it every time a database is being queried + try: + return self._db_cache[sha] + except KeyError: + pass + # END first level cache + + for db in self._dbs: + if db.has_object(sha): + self._db_cache[sha] = db + return db + # END for each database + raise BadObject(sha) + + #{ ObjectDBR interface + + def has_object(self, sha): + try: + self._db_query(sha) + return True + except BadObject: + return False + # END handle exceptions + + def info(self, sha): + return self._db_query(sha).info(sha) + + def stream(self, sha): + return self._db_query(sha).stream(sha) + + def size(self): + """:return: total size of all contained databases""" + return reduce(lambda x, y: x + y, (db.size() for db in self._dbs), 0) + + def sha_iter(self): + return chain(*(db.sha_iter() for db in self._dbs)) + + #} END object DBR Interface + + #{ Interface + + def databases(self): + """:return: tuple of database instances we use for lookups""" + return tuple(self._dbs) + + def update_cache(self, force=False): + # something might have changed, clear everything + self._db_cache.clear() + stat = False + for db in self._dbs: + if isinstance(db, CachingDB): + stat |= db.update_cache(force) + # END if is caching db + # END for each database to update + return stat + + def partial_to_complete_sha_hex(self, partial_hexsha): + """ + :return: 20 byte binary sha1 from the given less-than-40 byte hexsha (bytes or str) + :param partial_hexsha: hexsha with less than 40 byte + :raise AmbiguousObjectName: """ + databases = list() + _databases_recursive(self, databases) + partial_hexsha = force_text(partial_hexsha) + len_partial_hexsha = len(partial_hexsha) + if len_partial_hexsha % 2 != 0: + partial_binsha = hex_to_bin(partial_hexsha + "0") + else: + partial_binsha = hex_to_bin(partial_hexsha) + # END assure successful binary conversion + + candidate = None + for db in databases: + full_bin_sha = None + try: + if hasattr(db, 'partial_to_complete_sha_hex'): + full_bin_sha = db.partial_to_complete_sha_hex(partial_hexsha) + else: + full_bin_sha = db.partial_to_complete_sha(partial_binsha, len_partial_hexsha) + # END handle database type + except BadObject: + continue + # END ignore bad objects + if full_bin_sha: + if candidate and candidate != full_bin_sha: + raise AmbiguousObjectName(partial_hexsha) + candidate = full_bin_sha + # END handle candidate + # END for each db + if not candidate: + raise BadObject(partial_binsha) + return candidate + + #} END interface diff --git a/gitdb/gitdb/db/git.py b/gitdb/gitdb/db/git.py new file mode 100644 index 000000000..a1ed14280 --- /dev/null +++ b/gitdb/gitdb/db/git.py @@ -0,0 +1,85 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: https://opensource.org/license/bsd-3-clause/ +from gitdb.db.base import ( + CompoundDB, + ObjectDBW, + FileDBBase +) + +from gitdb.db.loose import LooseObjectDB +from gitdb.db.pack import PackedDB +from gitdb.db.ref import ReferenceDB + +from gitdb.exc import InvalidDBRoot + +import os + +__all__ = ('GitDB', ) + + +class GitDB(FileDBBase, ObjectDBW, CompoundDB): + + """A git-style object database, which contains all objects in the 'objects' + subdirectory + + ``IMPORTANT``: The usage of this implementation is highly discouraged as it fails to release file-handles. + This can be a problem with long-running processes and/or big repositories. + """ + # Configuration + PackDBCls = PackedDB + LooseDBCls = LooseObjectDB + ReferenceDBCls = ReferenceDB + + # Directories + packs_dir = 'pack' + loose_dir = '' + alternates_dir = os.path.join('info', 'alternates') + + def __init__(self, root_path): + """Initialize ourselves on a git objects directory""" + super().__init__(root_path) + + def _set_cache_(self, attr): + if attr == '_dbs' or attr == '_loose_db': + self._dbs = list() + loose_db = None + for subpath, dbcls in ((self.packs_dir, self.PackDBCls), + (self.loose_dir, self.LooseDBCls), + (self.alternates_dir, self.ReferenceDBCls)): + path = self.db_path(subpath) + if os.path.exists(path): + self._dbs.append(dbcls(path)) + if dbcls is self.LooseDBCls: + loose_db = self._dbs[-1] + # END remember loose db + # END check path exists + # END for each db type + + # should have at least one subdb + if not self._dbs: + raise InvalidDBRoot(self.root_path()) + # END handle error + + # we the first one should have the store method + assert loose_db is not None and hasattr(loose_db, 'store'), "First database needs store functionality" + + # finally set the value + self._loose_db = loose_db + else: + super()._set_cache_(attr) + # END handle attrs + + #{ ObjectDBW interface + + def store(self, istream): + return self._loose_db.store(istream) + + def ostream(self): + return self._loose_db.ostream() + + def set_ostream(self, ostream): + return self._loose_db.set_ostream(ostream) + + #} END objectdbw interface diff --git a/gitdb/gitdb/db/loose.py b/gitdb/gitdb/db/loose.py new file mode 100644 index 000000000..e6765cdeb --- /dev/null +++ b/gitdb/gitdb/db/loose.py @@ -0,0 +1,268 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: https://opensource.org/license/bsd-3-clause/ +from contextlib import suppress + +from gitdb.db.base import ( + FileDBBase, + ObjectDBR, + ObjectDBW +) + +from gitdb.exc import ( + BadObject, + AmbiguousObjectName +) + +from gitdb.stream import ( + DecompressMemMapReader, + FDCompressedSha1Writer, + FDStream, + Sha1Writer +) + +from gitdb.base import ( + OStream, + OInfo +) + +from gitdb.util import ( + file_contents_ro_filepath, + ENOENT, + hex_to_bin, + bin_to_hex, + exists, + chmod, + isfile, + remove, + rename, + dirname, + basename, + join +) + +from gitdb.fun import ( + chunk_size, + loose_object_header_info, + write_object, + stream_copy +) + +from gitdb.utils.encoding import force_bytes + +import tempfile +import os +import sys +import time + + +__all__ = ('LooseObjectDB', ) + + +class LooseObjectDB(FileDBBase, ObjectDBR, ObjectDBW): + + """A database which operates on loose object files""" + + # CONFIGURATION + # chunks in which data will be copied between streams + stream_chunk_size = chunk_size + + # On windows we need to keep it writable, otherwise it cannot be removed + # either + new_objects_mode = int("444", 8) + if os.name == 'nt': + new_objects_mode = int("644", 8) + + def __init__(self, root_path): + super().__init__(root_path) + self._hexsha_to_file = dict() + # Additional Flags - might be set to 0 after the first failure + # Depending on the root, this might work for some mounts, for others not, which + # is why it is per instance + self._fd_open_flags = getattr(os, 'O_NOATIME', 0) + + #{ Interface + def object_path(self, hexsha): + """ + :return: path at which the object with the given hexsha would be stored, + relative to the database root""" + return join(hexsha[:2], hexsha[2:]) + + def readable_db_object_path(self, hexsha): + """ + :return: readable object path to the object identified by hexsha + :raise BadObject: If the object file does not exist""" + with suppress(KeyError): + return self._hexsha_to_file[hexsha] + # END ignore cache misses + + # try filesystem + path = self.db_path(self.object_path(hexsha)) + if exists(path): + self._hexsha_to_file[hexsha] = path + return path + # END handle cache + raise BadObject(hexsha) + + def partial_to_complete_sha_hex(self, partial_hexsha): + """:return: 20 byte binary sha1 string which matches the given name uniquely + :param name: hexadecimal partial name (bytes or ascii string) + :raise AmbiguousObjectName: + :raise BadObject: """ + candidate = None + for binsha in self.sha_iter(): + if bin_to_hex(binsha).startswith(force_bytes(partial_hexsha)): + # it can't ever find the same object twice + if candidate is not None: + raise AmbiguousObjectName(partial_hexsha) + candidate = binsha + # END for each object + if candidate is None: + raise BadObject(partial_hexsha) + return candidate + + #} END interface + + def _map_loose_object(self, sha): + """ + :return: memory map of that file to allow random read access + :raise BadObject: if object could not be located""" + db_path = self.db_path(self.object_path(bin_to_hex(sha))) + try: + return file_contents_ro_filepath(db_path, flags=self._fd_open_flags) + except OSError as e: + if e.errno != ENOENT: + # try again without noatime + try: + return file_contents_ro_filepath(db_path) + except OSError as new_e: + raise BadObject(sha) from new_e + # didn't work because of our flag, don't try it again + self._fd_open_flags = 0 + else: + raise BadObject(sha) from e + # END handle error + # END exception handling + + def set_ostream(self, stream): + """:raise TypeError: if the stream does not support the Sha1Writer interface""" + if stream is not None and not isinstance(stream, Sha1Writer): + raise TypeError("Output stream musst support the %s interface" % Sha1Writer.__name__) + return super().set_ostream(stream) + + def info(self, sha): + m = self._map_loose_object(sha) + try: + typ, size = loose_object_header_info(m) + return OInfo(sha, typ, size) + finally: + if hasattr(m, 'close'): + m.close() + # END assure release of system resources + + def stream(self, sha): + m = self._map_loose_object(sha) + type, size, stream = DecompressMemMapReader.new(m, close_on_deletion=True) + return OStream(sha, type, size, stream) + + def has_object(self, sha): + try: + self.readable_db_object_path(bin_to_hex(sha)) + return True + except BadObject: + return False + # END check existence + + def store(self, istream): + """note: The sha we produce will be hex by nature""" + tmp_path = None + writer = self.ostream() + if writer is None: + # open a tmp file to write the data to + fd, tmp_path = tempfile.mkstemp(prefix='obj', dir=self._root_path) + + if istream.binsha is None: + writer = FDCompressedSha1Writer(fd) + else: + writer = FDStream(fd) + # END handle direct stream copies + # END handle custom writer + + try: + try: + if istream.binsha is not None: + # copy as much as possible, the actual uncompressed item size might + # be smaller than the compressed version + stream_copy(istream.read, writer.write, sys.maxsize, self.stream_chunk_size) + else: + # write object with header, we have to make a new one + write_object(istream.type, istream.size, istream.read, writer.write, + chunk_size=self.stream_chunk_size) + # END handle direct stream copies + finally: + if tmp_path: + writer.close() + # END assure target stream is closed + except: + if tmp_path: + remove(tmp_path) + raise + # END assure tmpfile removal on error + + hexsha = None + if istream.binsha: + hexsha = istream.hexsha + else: + hexsha = writer.sha(as_hex=True) + # END handle sha + + if tmp_path: + obj_path = self.db_path(self.object_path(hexsha)) + obj_dir = dirname(obj_path) + os.makedirs(obj_dir, exist_ok=True) + # END handle destination directory + # rename onto existing doesn't work on NTFS + if isfile(obj_path): + remove(tmp_path) + else: + rename(tmp_path, obj_path) + # end rename only if needed + + # Ensure rename is actually done and file is stable + # Retry up to 14 times - quadratic wait & retry in ms. + # The total maximum wait time is 1000ms, which should be vastly enough for the + # OS to return and commit the file to disk. + for backoff_ms in [1, 4, 9, 16, 25, 36, 49, 64, 81, 100, 121, 144, 169, 181]: + with suppress(PermissionError): + # make sure its readable for all ! It started out as rw-- tmp file + # but needs to be rwrr + chmod(obj_path, self.new_objects_mode) + break + time.sleep(backoff_ms / 1000.0) + else: + raise PermissionError( + "Impossible to apply `chmod` to file {}".format(obj_path) + ) + + # END handle dry_run + + istream.binsha = hex_to_bin(hexsha) + return istream + + def sha_iter(self): + # find all files which look like an object, extract sha from there + for root, dirs, files in os.walk(self.root_path()): + root_base = basename(root) + if len(root_base) != 2: + continue + + for f in files: + if len(f) != 38: + continue + yield hex_to_bin(root_base + f) + # END for each file + # END for each walk iteration + + def size(self): + return len(tuple(self.sha_iter())) diff --git a/gitdb/gitdb/db/mem.py b/gitdb/gitdb/db/mem.py new file mode 100644 index 000000000..d4772fdb5 --- /dev/null +++ b/gitdb/gitdb/db/mem.py @@ -0,0 +1,110 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: https://opensource.org/license/bsd-3-clause/ +"""Contains the MemoryDatabase implementation""" +from gitdb.db.loose import LooseObjectDB +from gitdb.db.base import ( + ObjectDBR, + ObjectDBW +) + +from gitdb.base import ( + OStream, + IStream, +) + +from gitdb.exc import ( + BadObject, + UnsupportedOperation +) + +from gitdb.stream import ( + ZippedStoreShaWriter, + DecompressMemMapReader, +) + +from io import BytesIO + +__all__ = ("MemoryDB", ) + + +class MemoryDB(ObjectDBR, ObjectDBW): + + """A memory database stores everything to memory, providing fast IO and object + retrieval. It should be used to buffer results and obtain SHAs before writing + it to the actual physical storage, as it allows to query whether object already + exists in the target storage before introducing actual IO""" + + def __init__(self): + super().__init__() + self._db = LooseObjectDB("path/doesnt/matter") + + # maps 20 byte shas to their OStream objects + self._cache = dict() + + def set_ostream(self, stream): + raise UnsupportedOperation("MemoryDB's always stream into memory") + + def store(self, istream): + zstream = ZippedStoreShaWriter() + self._db.set_ostream(zstream) + + istream = self._db.store(istream) + zstream.close() # close to flush + zstream.seek(0) + + # don't provide a size, the stream is written in object format, hence the + # header needs decompression + decomp_stream = DecompressMemMapReader(zstream.getvalue(), close_on_deletion=False) + self._cache[istream.binsha] = OStream(istream.binsha, istream.type, istream.size, decomp_stream) + + return istream + + def has_object(self, sha): + return sha in self._cache + + def info(self, sha): + # we always return streams, which are infos as well + return self.stream(sha) + + def stream(self, sha): + try: + ostream = self._cache[sha] + # rewind stream for the next one to read + ostream.stream.seek(0) + return ostream + except KeyError as e: + raise BadObject(sha) from e + # END exception handling + + def size(self): + return len(self._cache) + + def sha_iter(self): + return self._cache.keys() + + #{ Interface + def stream_copy(self, sha_iter, odb): + """Copy the streams as identified by sha's yielded by sha_iter into the given odb + The streams will be copied directly + **Note:** the object will only be written if it did not exist in the target db + + :return: amount of streams actually copied into odb. If smaller than the amount + of input shas, one or more objects did already exist in odb""" + count = 0 + for sha in sha_iter: + if odb.has_object(sha): + continue + # END check object existence + + ostream = self.stream(sha) + # compressed data including header + sio = BytesIO(ostream.stream.data()) + istream = IStream(ostream.type, ostream.size, sio, sha) + + odb.store(istream) + count += 1 + # END for each sha + return count + #} END interface diff --git a/gitdb/gitdb/db/pack.py b/gitdb/gitdb/db/pack.py new file mode 100644 index 000000000..274ea5990 --- /dev/null +++ b/gitdb/gitdb/db/pack.py @@ -0,0 +1,206 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: https://opensource.org/license/bsd-3-clause/ +"""Module containing a database to deal with packs""" +from gitdb.db.base import ( + FileDBBase, + ObjectDBR, + CachingDB +) + +from gitdb.util import LazyMixin + +from gitdb.exc import ( + BadObject, + UnsupportedOperation, + AmbiguousObjectName +) + +from gitdb.pack import PackEntity + +from functools import reduce + +import os +import glob + +__all__ = ('PackedDB', ) + +#{ Utilities + + +class PackedDB(FileDBBase, ObjectDBR, CachingDB, LazyMixin): + + """A database operating on a set of object packs""" + + # sort the priority list every N queries + # Higher values are better, performance tests don't show this has + # any effect, but it should have one + _sort_interval = 500 + + def __init__(self, root_path): + super().__init__(root_path) + # list of lists with three items: + # * hits - number of times the pack was hit with a request + # * entity - Pack entity instance + # * sha_to_index - PackIndexFile.sha_to_index method for direct cache query + # self._entities = list() # lazy loaded list + self._hit_count = 0 # amount of hits + self._st_mtime = 0 # last modification data of our root path + + def _set_cache_(self, attr): + if attr == '_entities': + self._entities = list() + self.update_cache(force=True) + # END handle entities initialization + + def _sort_entities(self): + self._entities.sort(key=lambda l: l[0], reverse=True) + + def _pack_info(self, sha): + """:return: tuple(entity, index) for an item at the given sha + :param sha: 20 or 40 byte sha + :raise BadObject: + **Note:** This method is not thread-safe, but may be hit in multi-threaded + operation. The worst thing that can happen though is a counter that + was not incremented, or the list being in wrong order. So we safe + the time for locking here, lets see how that goes""" + # presort ? + if self._hit_count % self._sort_interval == 0: + self._sort_entities() + # END update sorting + + for item in self._entities: + index = item[2](sha) + if index is not None: + item[0] += 1 # one hit for you + self._hit_count += 1 # general hit count + return (item[1], index) + # END index found in pack + # END for each item + + # no hit, see whether we have to update packs + # NOTE: considering packs don't change very often, we safe this call + # and leave it to the super-caller to trigger that + raise BadObject(sha) + + #{ Object DB Read + + def has_object(self, sha): + try: + self._pack_info(sha) + return True + except BadObject: + return False + # END exception handling + + def info(self, sha): + entity, index = self._pack_info(sha) + return entity.info_at_index(index) + + def stream(self, sha): + entity, index = self._pack_info(sha) + return entity.stream_at_index(index) + + def sha_iter(self): + for entity in self.entities(): + index = entity.index() + sha_by_index = index.sha + for index in range(index.size()): + yield sha_by_index(index) + # END for each index + # END for each entity + + def size(self): + sizes = [item[1].index().size() for item in self._entities] + return reduce(lambda x, y: x + y, sizes, 0) + + #} END object db read + + #{ object db write + + def store(self, istream): + """Storing individual objects is not feasible as a pack is designed to + hold multiple objects. Writing or rewriting packs for single objects is + inefficient""" + raise UnsupportedOperation() + + #} END object db write + + #{ Interface + + def update_cache(self, force=False): + """ + Update our cache with the actually existing packs on disk. Add new ones, + and remove deleted ones. We keep the unchanged ones + + :param force: If True, the cache will be updated even though the directory + does not appear to have changed according to its modification timestamp. + :return: True if the packs have been updated so there is new information, + False if there was no change to the pack database""" + stat = os.stat(self.root_path()) + if not force and stat.st_mtime <= self._st_mtime: + return False + # END abort early on no change + self._st_mtime = stat.st_mtime + + # packs are supposed to be prefixed with pack- by git-convention + # get all pack files, figure out what changed + pack_files = set(glob.glob(os.path.join(self.root_path(), "pack-*.pack"))) + our_pack_files = {item[1].pack().path() for item in self._entities} + + # new packs + for pack_file in (pack_files - our_pack_files): + # init the hit-counter/priority with the size, a good measure for hit- + # probability. Its implemented so that only 12 bytes will be read + entity = PackEntity(pack_file) + self._entities.append([entity.pack().size(), entity, entity.index().sha_to_index]) + # END for each new packfile + + # removed packs + for pack_file in (our_pack_files - pack_files): + del_index = -1 + for i, item in enumerate(self._entities): + if item[1].pack().path() == pack_file: + del_index = i + break + # END found index + # END for each entity + assert del_index != -1 + del(self._entities[del_index]) + # END for each removed pack + + # reinitialize prioritiess + self._sort_entities() + return True + + def entities(self): + """:return: list of pack entities operated upon by this database""" + return [item[1] for item in self._entities] + + def partial_to_complete_sha(self, partial_binsha, canonical_length): + """:return: 20 byte sha as inferred by the given partial binary sha + :param partial_binsha: binary sha with less than 20 bytes + :param canonical_length: length of the corresponding canonical representation. + It is required as binary sha's cannot display whether the original hex sha + had an odd or even number of characters + :raise AmbiguousObjectName: + :raise BadObject: """ + candidate = None + for item in self._entities: + item_index = item[1].index().partial_sha_to_index(partial_binsha, canonical_length) + if item_index is not None: + sha = item[1].index().sha(item_index) + if candidate and candidate != sha: + raise AmbiguousObjectName(partial_binsha) + candidate = sha + # END handle full sha could be found + # END for each entity + + if candidate: + return candidate + + # still not found ? + raise BadObject(partial_binsha) + + #} END interface diff --git a/gitdb/gitdb/db/ref.py b/gitdb/gitdb/db/ref.py new file mode 100644 index 000000000..5536db0f2 --- /dev/null +++ b/gitdb/gitdb/db/ref.py @@ -0,0 +1,81 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: https://opensource.org/license/bsd-3-clause/ +from gitdb.db.base import ( + CompoundDB, +) + +__all__ = ('ReferenceDB', ) + + +class ReferenceDB(CompoundDB): + + """A database consisting of database referred to in a file""" + + # Configuration + # Specifies the object database to use for the paths found in the alternates + # file. If None, it defaults to the GitDB + ObjectDBCls = None + + def __init__(self, ref_file): + super().__init__() + self._ref_file = ref_file + + def _set_cache_(self, attr): + if attr == '_dbs': + self._dbs = list() + self._update_dbs_from_ref_file() + else: + super()._set_cache_(attr) + # END handle attrs + + def _update_dbs_from_ref_file(self): + dbcls = self.ObjectDBCls + if dbcls is None: + # late import + from gitdb.db.git import GitDB + dbcls = GitDB + # END get db type + + # try to get as many as possible, don't fail if some are unavailable + ref_paths = list() + try: + with open(self._ref_file, 'r', encoding="utf-8") as f: + ref_paths = [l.strip() for l in f] + except OSError: + pass + # END handle alternates + + ref_paths_set = set(ref_paths) + cur_ref_paths_set = {db.root_path() for db in self._dbs} + + # remove existing + for path in (cur_ref_paths_set - ref_paths_set): + for i, db in enumerate(self._dbs[:]): + if db.root_path() == path: + del(self._dbs[i]) + continue + # END del matching db + # END for each path to remove + + # add new + # sort them to maintain order + added_paths = sorted(ref_paths_set - cur_ref_paths_set, key=lambda p: ref_paths.index(p)) + for path in added_paths: + try: + db = dbcls(path) + # force an update to verify path + if isinstance(db, CompoundDB): + db.databases() + # END verification + self._dbs.append(db) + except Exception: + # ignore invalid paths or issues + pass + # END for each path to add + + def update_cache(self, force=False): + # re-read alternates and update databases + self._update_dbs_from_ref_file() + return super().update_cache(force) diff --git a/gitdb/gitdb/exc.py b/gitdb/gitdb/exc.py new file mode 100644 index 000000000..752dafdbb --- /dev/null +++ b/gitdb/gitdb/exc.py @@ -0,0 +1,57 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: https://opensource.org/license/bsd-3-clause/ +"""Module with common exceptions""" +from gitdb.util import to_hex_sha + +__all__ = [ + 'AmbiguousObjectName', + 'BadName', + 'BadObject', + 'BadObjectType', + 'InvalidDBRoot', + 'ODBError', + 'ParseError', + 'UnsupportedOperation', + 'to_hex_sha', +] + +class ODBError(Exception): + """All errors thrown by the object database""" + + +class InvalidDBRoot(ODBError): + """Thrown if an object database cannot be initialized at the given path""" + + +class BadObject(ODBError): + """The object with the given SHA does not exist. Instantiate with the + failed sha""" + + def __str__(self): + return "BadObject: %s" % to_hex_sha(self.args[0]) + + +class BadName(ODBError): + """A name provided to rev_parse wasn't understood""" + + def __str__(self): + return "Ref '%s' did not resolve to an object" % self.args[0] + + +class ParseError(ODBError): + """Thrown if the parsing of a file failed due to an invalid format""" + + +class AmbiguousObjectName(ODBError): + """Thrown if a possibly shortened name does not uniquely represent a single object + in the database""" + + +class BadObjectType(ODBError): + """The object had an unsupported type""" + + +class UnsupportedOperation(ODBError): + """Thrown if the given operation cannot be supported by the object database""" diff --git a/gitdb/gitdb/fun.py b/gitdb/gitdb/fun.py new file mode 100644 index 000000000..a272e5caa --- /dev/null +++ b/gitdb/gitdb/fun.py @@ -0,0 +1,704 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: https://opensource.org/license/bsd-3-clause/ +"""Contains basic c-functions which usually contain performance critical code +Keeping this code separate from the beginning makes it easier to out-source +it into c later, if required""" + +import zlib +from gitdb.util import byte_ord +decompressobj = zlib.decompressobj + +import mmap +from itertools import islice +from functools import reduce + +from gitdb.const import NULL_BYTE, BYTE_SPACE +from gitdb.utils.encoding import force_text +from gitdb.typ import ( + str_blob_type, + str_commit_type, + str_tree_type, + str_tag_type, +) + +from io import StringIO + +# INVARIANTS +OFS_DELTA = 6 +REF_DELTA = 7 +delta_types = (OFS_DELTA, REF_DELTA) + +type_id_to_type_map = { + 0: b'', # EXT 1 + 1: str_commit_type, + 2: str_tree_type, + 3: str_blob_type, + 4: str_tag_type, + 5: b'', # EXT 2 + OFS_DELTA: "OFS_DELTA", # OFFSET DELTA + REF_DELTA: "REF_DELTA" # REFERENCE DELTA +} + +type_to_type_id_map = { + str_commit_type: 1, + str_tree_type: 2, + str_blob_type: 3, + str_tag_type: 4, + "OFS_DELTA": OFS_DELTA, + "REF_DELTA": REF_DELTA, +} + +# used when dealing with larger streams +chunk_size = 1000 * mmap.PAGESIZE + +__all__ = ('is_loose_object', 'loose_object_header_info', 'msb_size', 'pack_object_header_info', + 'write_object', 'loose_object_header', 'stream_copy', 'apply_delta_data', + 'is_equal_canonical_sha', 'connect_deltas', 'DeltaChunkList', 'create_pack_object_header') + + +#{ Structures + +def _set_delta_rbound(d, size): + """Truncate the given delta to the given size + :param size: size relative to our target offset, may not be 0, must be smaller or equal + to our size + :return: d""" + d.ts = size + + # NOTE: data is truncated automatically when applying the delta + # MUST NOT DO THIS HERE + return d + + +def _move_delta_lbound(d, bytes): + """Move the delta by the given amount of bytes, reducing its size so that its + right bound stays static + :param bytes: amount of bytes to move, must be smaller than delta size + :return: d""" + if bytes == 0: + return + + d.to += bytes + d.so += bytes + d.ts -= bytes + if d.data is not None: + d.data = d.data[bytes:] + # END handle data + + return d + + +def delta_duplicate(src): + return DeltaChunk(src.to, src.ts, src.so, src.data) + + +def delta_chunk_apply(dc, bbuf, write): + """Apply own data to the target buffer + :param bbuf: buffer providing source bytes for copy operations + :param write: write method to call with data to write""" + if dc.data is None: + # COPY DATA FROM SOURCE + write(bbuf[dc.so:dc.so + dc.ts]) + else: + # APPEND DATA + # what's faster: if + 4 function calls or just a write with a slice ? + # Considering data can be larger than 127 bytes now, it should be worth it + if dc.ts < len(dc.data): + write(dc.data[:dc.ts]) + else: + write(dc.data) + # END handle truncation + # END handle chunk mode + + +class DeltaChunk: + + """Represents a piece of a delta, it can either add new data, or copy existing + one from a source buffer""" + __slots__ = ( + 'to', # start offset in the target buffer in bytes + 'ts', # size of this chunk in the target buffer in bytes + 'so', # start offset in the source buffer in bytes or None + 'data', # chunk of bytes to be added to the target buffer, + # DeltaChunkList to use as base, or None + ) + + def __init__(self, to, ts, so, data): + self.to = to + self.ts = ts + self.so = so + self.data = data + + def __repr__(self): + return "DeltaChunk(%i, %i, %s, %s)" % (self.to, self.ts, self.so, self.data or "") + + #{ Interface + + def rbound(self): + return self.to + self.ts + + def has_data(self): + """:return: True if the instance has data to add to the target stream""" + return self.data is not None + + #} END interface + + +def _closest_index(dcl, absofs): + """:return: index at which the given absofs should be inserted. The index points + to the DeltaChunk with a target buffer absofs that equals or is greater than + absofs. + **Note:** global method for performance only, it belongs to DeltaChunkList""" + lo = 0 + hi = len(dcl) + while lo < hi: + mid = (lo + hi) / 2 + dc = dcl[mid] + if dc.to > absofs: + hi = mid + elif dc.rbound() > absofs or dc.to == absofs: + return mid + else: + lo = mid + 1 + # END handle bound + # END for each delta absofs + return len(dcl) - 1 + + +def delta_list_apply(dcl, bbuf, write): + """Apply the chain's changes and write the final result using the passed + write function. + :param bbuf: base buffer containing the base of all deltas contained in this + list. It will only be used if the chunk in question does not have a base + chain. + :param write: function taking a string of bytes to write to the output""" + for dc in dcl: + delta_chunk_apply(dc, bbuf, write) + # END for each dc + + +def delta_list_slice(dcl, absofs, size, ndcl): + """:return: Subsection of this list at the given absolute offset, with the given + size in bytes. + :return: None""" + cdi = _closest_index(dcl, absofs) # delta start index + cd = dcl[cdi] + slen = len(dcl) + lappend = ndcl.append + + if cd.to != absofs: + tcd = DeltaChunk(cd.to, cd.ts, cd.so, cd.data) + _move_delta_lbound(tcd, absofs - cd.to) + tcd.ts = min(tcd.ts, size) + lappend(tcd) + size -= tcd.ts + cdi += 1 + # END lbound overlap handling + + while cdi < slen and size: + # are we larger than the current block + cd = dcl[cdi] + if cd.ts <= size: + lappend(DeltaChunk(cd.to, cd.ts, cd.so, cd.data)) + size -= cd.ts + else: + tcd = DeltaChunk(cd.to, cd.ts, cd.so, cd.data) + tcd.ts = size + lappend(tcd) + size -= tcd.ts + break + # END hadle size + cdi += 1 + # END for each chunk + + +class DeltaChunkList(list): + + """List with special functionality to deal with DeltaChunks. + There are two types of lists we represent. The one was created bottom-up, working + towards the latest delta, the other kind was created top-down, working from the + latest delta down to the earliest ancestor. This attribute is queryable + after all processing with is_reversed.""" + + __slots__ = tuple() + + def rbound(self): + """:return: rightmost extend in bytes, absolute""" + if len(self) == 0: + return 0 + return self[-1].rbound() + + def lbound(self): + """:return: leftmost byte at which this chunklist starts""" + if len(self) == 0: + return 0 + return self[0].to + + def size(self): + """:return: size of bytes as measured by our delta chunks""" + return self.rbound() - self.lbound() + + def apply(self, bbuf, write): + """Only used by public clients, internally we only use the global routines + for performance""" + return delta_list_apply(self, bbuf, write) + + def compress(self): + """Alter the list to reduce the amount of nodes. Currently we concatenate + add-chunks + :return: self""" + slen = len(self) + if slen < 2: + return self + i = 0 + + first_data_index = None + while i < slen: + dc = self[i] + i += 1 + if dc.data is None: + if first_data_index is not None and i - 2 - first_data_index > 1: + # if first_data_index is not None: + nd = StringIO() # new data + so = self[first_data_index].to # start offset in target buffer + for x in range(first_data_index, i - 1): + xdc = self[x] + nd.write(xdc.data[:xdc.ts]) + # END collect data + + del(self[first_data_index:i - 1]) + buf = nd.getvalue() + self.insert(first_data_index, DeltaChunk(so, len(buf), 0, buf)) + + slen = len(self) + i = first_data_index + 1 + + # END concatenate data + first_data_index = None + continue + # END skip non-data chunks + + if first_data_index is None: + first_data_index = i - 1 + # END iterate list + + # if slen_orig != len(self): + # print "INFO: Reduced delta list len to %f %% of former size" % ((float(len(self)) / slen_orig) * 100) + return self + + def check_integrity(self, target_size=-1): + """Verify the list has non-overlapping chunks only, and the total size matches + target_size + :param target_size: if not -1, the total size of the chain must be target_size + :raise AssertionError: if the size doesn't match""" + if target_size > -1: + assert self[-1].rbound() == target_size + assert reduce(lambda x, y: x + y, (d.ts for d in self), 0) == target_size + # END target size verification + + if len(self) < 2: + return + + # check data + for dc in self: + assert dc.ts > 0 + if dc.has_data(): + assert len(dc.data) >= dc.ts + # END for each dc + + left = islice(self, 0, len(self) - 1) + right = iter(self) + right.next() + # this is very pythonic - we might have just use index based access here, + # but this could actually be faster + for lft, rgt in zip(left, right): + assert lft.rbound() == rgt.to + assert lft.to + lft.ts == rgt.to + # END for each pair + + +class TopdownDeltaChunkList(DeltaChunkList): + + """Represents a list which is generated by feeding its ancestor streams one by + one""" + __slots__ = tuple() + + def connect_with_next_base(self, bdcl): + """Connect this chain with the next level of our base delta chunklist. + The goal in this game is to mark as many of our chunks rigid, hence they + cannot be changed by any of the upcoming bases anymore. Once all our + chunks are marked like that, we can stop all processing + :param bdcl: data chunk list being one of our bases. They must be fed in + consecutively and in order, towards the earliest ancestor delta + :return: True if processing was done. Use it to abort processing of + remaining streams if False is returned""" + nfc = 0 # number of frozen chunks + dci = 0 # delta chunk index + slen = len(self) # len of self + ccl = list() # temporary list + while dci < slen: + dc = self[dci] + dci += 1 + + # all add-chunks which are already topmost don't need additional processing + if dc.data is not None: + nfc += 1 + continue + # END skip add chunks + + # copy chunks + # integrate the portion of the base list into ourselves. Lists + # dont support efficient insertion ( just one at a time ), but for now + # we live with it. Internally, its all just a 32/64bit pointer, and + # the portions of moved memory should be smallish. Maybe we just rebuild + # ourselves in order to reduce the amount of insertions ... + del(ccl[:]) + delta_list_slice(bdcl, dc.so, dc.ts, ccl) + + # move the target bounds into place to match with our chunk + ofs = dc.to - dc.so + for cdc in ccl: + cdc.to += ofs + # END update target bounds + + if len(ccl) == 1: + self[dci - 1] = ccl[0] + else: + # maybe try to compute the expenses here, and pick the right algorithm + # It would normally be faster than copying everything physically though + # TODO: Use a deque here, and decide by the index whether to extend + # or extend left ! + post_dci = self[dci:] + del(self[dci - 1:]) # include deletion of dc + self.extend(ccl) + self.extend(post_dci) + + slen = len(self) + dci += len(ccl) - 1 # deleted dc, added rest + + # END handle chunk replacement + # END for each chunk + + if nfc == slen: + return False + # END handle completeness + return True + + +#} END structures + +#{ Routines + +def is_loose_object(m): + """ + :return: True the file contained in memory map m appears to be a loose object. + Only the first two bytes are needed""" + b0, b1 = map(ord, m[:2]) + word = (b0 << 8) + b1 + return b0 == 0x78 and (word % 31) == 0 + + +def loose_object_header_info(m): + """ + :return: tuple(type_string, uncompressed_size_in_bytes) the type string of the + object as well as its uncompressed size in bytes. + :param m: memory map from which to read the compressed object data""" + decompress_size = 8192 # is used in cgit as well + hdr = decompressobj().decompress(m, decompress_size) + type_name, size = hdr[:hdr.find(NULL_BYTE)].split(BYTE_SPACE) + + return type_name, int(size) + + +def pack_object_header_info(data): + """ + :return: tuple(type_id, uncompressed_size_in_bytes, byte_offset) + The type_id should be interpreted according to the ``type_id_to_type_map`` map + The byte-offset specifies the start of the actual zlib compressed datastream + :param m: random-access memory, like a string or memory map""" + c = byte_ord(data[0]) # first byte + i = 1 # next char to read + type_id = (c >> 4) & 7 # numeric type + size = c & 15 # starting size + s = 4 # starting bit-shift size + while c & 0x80: + c = byte_ord(data[i]) + i += 1 + size += (c & 0x7f) << s + s += 7 + # END character loop + # end performance at expense of maintenance ... + return (type_id, size, i) + + +def create_pack_object_header(obj_type, obj_size): + """ + :return: string defining the pack header comprised of the object type + and its incompressed size in bytes + + :param obj_type: pack type_id of the object + :param obj_size: uncompressed size in bytes of the following object stream""" + c = 0 # 1 byte + hdr = bytearray() # output string + + c = (obj_type << 4) | (obj_size & 0xf) + obj_size >>= 4 + while obj_size: + hdr.append(c | 0x80) + c = obj_size & 0x7f + obj_size >>= 7 + # END until size is consumed + hdr.append(c) + # end handle interpreter + return hdr + + +def msb_size(data, offset=0): + """ + :return: tuple(read_bytes, size) read the msb size from the given random + access data starting at the given byte offset""" + size = 0 + i = 0 + l = len(data) + hit_msb = False + while i < l: + c = data[i + offset] + size |= (c & 0x7f) << i * 7 + i += 1 + if not c & 0x80: + hit_msb = True + break + # END check msb bit + # END while in range + # end performance ... + if not hit_msb: + raise AssertionError("Could not find terminating MSB byte in data stream") + return i + offset, size + + +def loose_object_header(type, size): + """ + :return: bytes representing the loose object header, which is immediately + followed by the content stream of size 'size'""" + return ('%s %i\0' % (force_text(type), size)).encode('ascii') + + +def write_object(type, size, read, write, chunk_size=chunk_size): + """ + Write the object as identified by type, size and source_stream into the + target_stream + + :param type: type string of the object + :param size: amount of bytes to write from source_stream + :param read: read method of a stream providing the content data + :param write: write method of the output stream + :param close_target_stream: if True, the target stream will be closed when + the routine exits, even if an error is thrown + :return: The actual amount of bytes written to stream, which includes the header and a trailing newline""" + tbw = 0 # total num bytes written + + # WRITE HEADER: type SP size NULL + tbw += write(loose_object_header(type, size)) + tbw += stream_copy(read, write, size, chunk_size) + + return tbw + + +def stream_copy(read, write, size, chunk_size): + """ + Copy a stream up to size bytes using the provided read and write methods, + in chunks of chunk_size + + **Note:** its much like stream_copy utility, but operates just using methods""" + dbw = 0 # num data bytes written + + # WRITE ALL DATA UP TO SIZE + while True: + cs = min(chunk_size, size - dbw) + # NOTE: not all write methods return the amount of written bytes, like + # mmap.write. Its bad, but we just deal with it ... perhaps its not + # even less efficient + # data_len = write(read(cs)) + # dbw += data_len + data = read(cs) + data_len = len(data) + dbw += data_len + write(data) + if data_len < cs or dbw == size: + break + # END check for stream end + # END duplicate data + return dbw + + +def connect_deltas(dstreams): + """ + Read the condensed delta chunk information from dstream and merge its information + into a list of existing delta chunks + + :param dstreams: iterable of delta stream objects, the delta to be applied last + comes first, then all its ancestors in order + :return: DeltaChunkList, containing all operations to apply""" + tdcl = None # topmost dcl + + dcl = tdcl = TopdownDeltaChunkList() + for dsi, ds in enumerate(dstreams): + # print "Stream", dsi + db = ds.read() + delta_buf_size = ds.size + + # read header + i, base_size = msb_size(db) + i, target_size = msb_size(db, i) + + # interpret opcodes + tbw = 0 # amount of target bytes written + while i < delta_buf_size: + c = ord(db[i]) + i += 1 + if c & 0x80: + cp_off, cp_size = 0, 0 + if (c & 0x01): + cp_off = ord(db[i]) + i += 1 + if (c & 0x02): + cp_off |= (ord(db[i]) << 8) + i += 1 + if (c & 0x04): + cp_off |= (ord(db[i]) << 16) + i += 1 + if (c & 0x08): + cp_off |= (ord(db[i]) << 24) + i += 1 + if (c & 0x10): + cp_size = ord(db[i]) + i += 1 + if (c & 0x20): + cp_size |= (ord(db[i]) << 8) + i += 1 + if (c & 0x40): + cp_size |= (ord(db[i]) << 16) + i += 1 + + if not cp_size: + cp_size = 0x10000 + + rbound = cp_off + cp_size + if (rbound < cp_size or + rbound > base_size): + break + + dcl.append(DeltaChunk(tbw, cp_size, cp_off, None)) + tbw += cp_size + elif c: + # NOTE: in C, the data chunks should probably be concatenated here. + # In python, we do it as a post-process + dcl.append(DeltaChunk(tbw, c, 0, db[i:i + c])) + i += c + tbw += c + else: + raise ValueError("unexpected delta opcode 0") + # END handle command byte + # END while processing delta data + + dcl.compress() + + # merge the lists ! + if dsi > 0: + if not tdcl.connect_with_next_base(dcl): + break + # END handle merge + + # prepare next base + dcl = DeltaChunkList() + # END for each delta stream + + return tdcl + + +def apply_delta_data(src_buf, src_buf_size, delta_buf, delta_buf_size, write): + """ + Apply data from a delta buffer using a source buffer to the target file + + :param src_buf: random access data from which the delta was created + :param src_buf_size: size of the source buffer in bytes + :param delta_buf_size: size for the delta buffer in bytes + :param delta_buf: random access delta data + :param write: write method taking a chunk of bytes + + **Note:** transcribed to python from the similar routine in patch-delta.c""" + i = 0 + db = delta_buf + while i < delta_buf_size: + c = db[i] + i += 1 + if c & 0x80: + cp_off, cp_size = 0, 0 + if (c & 0x01): + cp_off = db[i] + i += 1 + if (c & 0x02): + cp_off |= (db[i] << 8) + i += 1 + if (c & 0x04): + cp_off |= (db[i] << 16) + i += 1 + if (c & 0x08): + cp_off |= (db[i] << 24) + i += 1 + if (c & 0x10): + cp_size = db[i] + i += 1 + if (c & 0x20): + cp_size |= (db[i] << 8) + i += 1 + if (c & 0x40): + cp_size |= (db[i] << 16) + i += 1 + + if not cp_size: + cp_size = 0x10000 + + rbound = cp_off + cp_size + if (rbound < cp_size or + rbound > src_buf_size): + break + write(src_buf[cp_off:cp_off + cp_size]) + elif c: + write(db[i:i + c]) + i += c + else: + raise ValueError("unexpected delta opcode 0") + # END handle command byte + # END while processing delta data + + # yes, lets use the exact same error message that git uses :) + assert i == delta_buf_size, "delta replay has gone wild" + + +def is_equal_canonical_sha(canonical_length, match, sha1): + """ + :return: True if the given lhs and rhs 20 byte binary shas + The comparison will take the canonical_length of the match sha into account, + hence the comparison will only use the last 4 bytes for uneven canonical representations + :param match: less than 20 byte sha + :param sha1: 20 byte sha""" + binary_length = canonical_length // 2 + if match[:binary_length] != sha1[:binary_length]: + return False + + if canonical_length - binary_length and \ + (byte_ord(match[-1]) ^ byte_ord(sha1[len(match) - 1])) & 0xf0: + return False + # END handle uneven canonnical length + return True + +#} END routines + + +try: + from gitdb_speedups._perf import connect_deltas +except ImportError: + pass diff --git a/gitdb/gitdb/pack.py b/gitdb/gitdb/pack.py new file mode 100644 index 000000000..2e0707947 --- /dev/null +++ b/gitdb/gitdb/pack.py @@ -0,0 +1,1028 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: https://opensource.org/license/bsd-3-clause/ +"""Contains PackIndexFile and PackFile implementations""" +import zlib + +from gitdb.exc import ( + BadObject, + AmbiguousObjectName, + UnsupportedOperation, + ParseError +) + +from gitdb.util import ( + mman, + LazyMixin, + unpack_from, + bin_to_hex, + byte_ord, +) + +from gitdb.fun import ( + create_pack_object_header, + pack_object_header_info, + is_equal_canonical_sha, + type_id_to_type_map, + write_object, + stream_copy, + chunk_size, + delta_types, + OFS_DELTA, + REF_DELTA, + msb_size +) + +try: + from gitdb_speedups._perf import PackIndexFile_sha_to_index +except ImportError: + pass +# END try c module + +from gitdb.base import ( # Amazing ! + OInfo, + OStream, + OPackInfo, + OPackStream, + ODeltaStream, + ODeltaPackInfo, + ODeltaPackStream, +) + +from gitdb.stream import ( + DecompressMemMapReader, + DeltaApplyReader, + Sha1Writer, + NullStream, + FlexibleSha1Writer +) + +from struct import pack +from binascii import crc32 + +from gitdb.const import NULL_BYTE + +import tempfile +import array +import os +import sys + +__all__ = ('PackIndexFile', 'PackFile', 'PackEntity') + + +#{ Utilities + +def pack_object_at(cursor, offset, as_stream): + """ + :return: Tuple(abs_data_offset, PackInfo|PackStream) + an object of the correct type according to the type_id of the object. + If as_stream is True, the object will contain a stream, allowing the + data to be read decompressed. + :param data: random accessible data containing all required information + :parma offset: offset in to the data at which the object information is located + :param as_stream: if True, a stream object will be returned that can read + the data, otherwise you receive an info object only""" + data = cursor.use_region(offset).buffer() + type_id, uncomp_size, data_rela_offset = pack_object_header_info(data) + total_rela_offset = None # set later, actual offset until data stream begins + delta_info = None + + # OFFSET DELTA + if type_id == OFS_DELTA: + i = data_rela_offset + c = byte_ord(data[i]) + i += 1 + delta_offset = c & 0x7f + while c & 0x80: + c = byte_ord(data[i]) + i += 1 + delta_offset += 1 + delta_offset = (delta_offset << 7) + (c & 0x7f) + # END character loop + delta_info = delta_offset + total_rela_offset = i + # REF DELTA + elif type_id == REF_DELTA: + total_rela_offset = data_rela_offset + 20 + delta_info = data[data_rela_offset:total_rela_offset] + # BASE OBJECT + else: + # assume its a base object + total_rela_offset = data_rela_offset + # END handle type id + abs_data_offset = offset + total_rela_offset + if as_stream: + stream = DecompressMemMapReader(data[total_rela_offset:], False, uncomp_size) + if delta_info is None: + return abs_data_offset, OPackStream(offset, type_id, uncomp_size, stream) + else: + return abs_data_offset, ODeltaPackStream(offset, type_id, uncomp_size, delta_info, stream) + else: + if delta_info is None: + return abs_data_offset, OPackInfo(offset, type_id, uncomp_size) + else: + return abs_data_offset, ODeltaPackInfo(offset, type_id, uncomp_size, delta_info) + # END handle info + # END handle stream + + +def write_stream_to_pack(read, write, zstream, base_crc=None): + """Copy a stream as read from read function, zip it, and write the result. + Count the number of written bytes and return it + :param base_crc: if not None, the crc will be the base for all compressed data + we consecutively write and generate a crc32 from. If None, no crc will be generated + :return: tuple(no bytes read, no bytes written, crc32) crc might be 0 if base_crc + was false""" + br = 0 # bytes read + bw = 0 # bytes written + want_crc = base_crc is not None + crc = 0 + if want_crc: + crc = base_crc + # END initialize crc + + while True: + chunk = read(chunk_size) + br += len(chunk) + compressed = zstream.compress(chunk) + bw += len(compressed) + write(compressed) # cannot assume return value + + if want_crc: + crc = crc32(compressed, crc) + # END handle crc + + if len(chunk) != chunk_size: + break + # END copy loop + + compressed = zstream.flush() + bw += len(compressed) + write(compressed) + if want_crc: + crc = crc32(compressed, crc) + # END handle crc + + return (br, bw, crc) + + +#} END utilities + + +class IndexWriter: + + """Utility to cache index information, allowing to write all information later + in one go to the given stream + **Note:** currently only writes v2 indices""" + __slots__ = '_objs' + + def __init__(self): + self._objs = list() + + def append(self, binsha, crc, offset): + """Append one piece of object information""" + self._objs.append((binsha, crc, offset)) + + def write(self, pack_sha, write): + """Write the index file using the given write method + :param pack_sha: binary sha over the whole pack that we index + :return: sha1 binary sha over all index file contents""" + # sort for sha1 hash + self._objs.sort(key=lambda o: o[0]) + + sha_writer = FlexibleSha1Writer(write) + sha_write = sha_writer.write + sha_write(PackIndexFile.index_v2_signature) + sha_write(pack(">L", PackIndexFile.index_version_default)) + + # fanout + tmplist = list((0,) * 256) # fanout or list with 64 bit offsets + for t in self._objs: + tmplist[byte_ord(t[0][0])] += 1 + # END prepare fanout + for i in range(255): + v = tmplist[i] + sha_write(pack('>L', v)) + tmplist[i + 1] += v + # END write each fanout entry + sha_write(pack('>L', tmplist[255])) + + # sha1 ordered + # save calls, that is push them into c + sha_write(b''.join(t[0] for t in self._objs)) + + # crc32 + for t in self._objs: + sha_write(pack('>L', t[1] & 0xffffffff)) + # END for each crc + + tmplist = list() + # offset 32 + for t in self._objs: + ofs = t[2] + if ofs > 0x7fffffff: + tmplist.append(ofs) + ofs = 0x80000000 + len(tmplist) - 1 + # END handle 64 bit offsets + sha_write(pack('>L', ofs & 0xffffffff)) + # END for each offset + + # offset 64 + for ofs in tmplist: + sha_write(pack(">Q", ofs)) + # END for each offset + + # trailer + assert(len(pack_sha) == 20) + sha_write(pack_sha) + sha = sha_writer.sha(as_hex=False) + write(sha) + return sha + + +class PackIndexFile(LazyMixin): + + """A pack index provides offsets into the corresponding pack, allowing to find + locations for offsets faster.""" + + # Dont use slots as we dynamically bind functions for each version, need a dict for this + # The slots you see here are just to keep track of our instance variables + # __slots__ = ('_indexpath', '_fanout_table', '_cursor', '_version', + # '_sha_list_offset', '_crc_list_offset', '_pack_offset', '_pack_64_offset') + + # used in v2 indices + _sha_list_offset = 8 + 1024 + index_v2_signature = b'\xfftOc' + index_version_default = 2 + + def __init__(self, indexpath): + super().__init__() + self._indexpath = indexpath + + def close(self): + mman.force_map_handle_removal_win(self._indexpath) + self._cursor = None + + def _set_cache_(self, attr): + if attr == "_packfile_checksum": + self._packfile_checksum = self._cursor.map()[-40:-20] + elif attr == "_cursor": + # Note: We don't lock the file when reading as we cannot be sure + # that we can actually write to the location - it could be a read-only + # alternate for instance + self._cursor = mman.make_cursor(self._indexpath).use_region() + # We will assume that the index will always fully fit into memory ! + if mman.window_size() > 0 and self._cursor.file_size() > mman.window_size(): + raise AssertionError("The index file at %s is too large to fit into a mapped window (%i > %i). This is a limitation of the implementation" % ( + self._indexpath, self._cursor.file_size(), mman.window_size())) + # END assert window size + else: + # now its time to initialize everything - if we are here, someone wants + # to access the fanout table or related properties + + # CHECK VERSION + mmap = self._cursor.map() + self._version = (mmap[:4] == self.index_v2_signature and 2) or 1 + if self._version == 2: + version_id = unpack_from(">L", mmap, 4)[0] + assert version_id == self._version, "Unsupported index version: %i" % version_id + # END assert version + + # SETUP FUNCTIONS + # setup our functions according to the actual version + for fname in ('entry', 'offset', 'sha', 'crc'): + setattr(self, fname, getattr(self, "_%s_v%i" % (fname, self._version))) + # END for each function to initialize + + # INITIALIZE DATA + # byte offset is 8 if version is 2, 0 otherwise + self._initialize() + # END handle attributes + + #{ Access V1 + + def _entry_v1(self, i): + """:return: tuple(offset, binsha, 0)""" + return unpack_from(">L20s", self._cursor.map(), 1024 + i * 24) + (0, ) + + def _offset_v1(self, i): + """see ``_offset_v2``""" + return unpack_from(">L", self._cursor.map(), 1024 + i * 24)[0] + + def _sha_v1(self, i): + """see ``_sha_v2``""" + base = 1024 + (i * 24) + 4 + return self._cursor.map()[base:base + 20] + + def _crc_v1(self, i): + """unsupported""" + return 0 + + #} END access V1 + + #{ Access V2 + def _entry_v2(self, i): + """:return: tuple(offset, binsha, crc)""" + return (self._offset_v2(i), self._sha_v2(i), self._crc_v2(i)) + + def _offset_v2(self, i): + """:return: 32 or 64 byte offset into pack files. 64 byte offsets will only + be returned if the pack is larger than 4 GiB, or 2^32""" + offset = unpack_from(">L", self._cursor.map(), self._pack_offset + i * 4)[0] + + # if the high-bit is set, this indicates that we have to lookup the offset + # in the 64 bit region of the file. The current offset ( lower 31 bits ) + # are the index into it + if offset & 0x80000000: + offset = unpack_from(">Q", self._cursor.map(), self._pack_64_offset + (offset & ~0x80000000) * 8)[0] + # END handle 64 bit offset + + return offset + + def _sha_v2(self, i): + """:return: sha at the given index of this file index instance""" + base = self._sha_list_offset + i * 20 + return self._cursor.map()[base:base + 20] + + def _crc_v2(self, i): + """:return: 4 bytes crc for the object at index i""" + return unpack_from(">L", self._cursor.map(), self._crc_list_offset + i * 4)[0] + + #} END access V2 + + #{ Initialization + + def _initialize(self): + """initialize base data""" + self._fanout_table = self._read_fanout((self._version == 2) * 8) + + if self._version == 2: + self._crc_list_offset = self._sha_list_offset + self.size() * 20 + self._pack_offset = self._crc_list_offset + self.size() * 4 + self._pack_64_offset = self._pack_offset + self.size() * 4 + # END setup base + + def _read_fanout(self, byte_offset): + """Generate a fanout table from our data""" + d = self._cursor.map() + out = list() + append = out.append + for i in range(256): + append(unpack_from('>L', d, byte_offset + i * 4)[0]) + # END for each entry + return out + + #} END initialization + + #{ Properties + def version(self): + return self._version + + def size(self): + """:return: amount of objects referred to by this index""" + return self._fanout_table[255] + + def path(self): + """:return: path to the packindexfile""" + return self._indexpath + + def packfile_checksum(self): + """:return: 20 byte sha representing the sha1 hash of the pack file""" + return self._cursor.map()[-40:-20] + + def indexfile_checksum(self): + """:return: 20 byte sha representing the sha1 hash of this index file""" + return self._cursor.map()[-20:] + + def offsets(self): + """:return: sequence of all offsets in the order in which they were written + + **Note:** return value can be random accessed, but may be immmutable""" + if self._version == 2: + # read stream to array, convert to tuple + a = array.array('I') # 4 byte unsigned int, long are 8 byte on 64 bit it appears + a.frombytes(self._cursor.map()[self._pack_offset:self._pack_64_offset]) + + # networkbyteorder to something array likes more + if sys.byteorder == 'little': + a.byteswap() + return a + else: + return tuple(self.offset(index) for index in range(self.size())) + # END handle version + + def sha_to_index(self, sha): + """ + :return: index usable with the ``offset`` or ``entry`` method, or None + if the sha was not found in this pack index + :param sha: 20 byte sha to lookup""" + first_byte = byte_ord(sha[0]) + get_sha = self.sha + lo = 0 # lower index, the left bound of the bisection + if first_byte != 0: + lo = self._fanout_table[first_byte - 1] + hi = self._fanout_table[first_byte] # the upper, right bound of the bisection + + # bisect until we have the sha + while lo < hi: + mid = (lo + hi) // 2 + mid_sha = get_sha(mid) + if sha < mid_sha: + hi = mid + elif sha == mid_sha: + return mid + else: + lo = mid + 1 + # END handle midpoint + # END bisect + return None + + def partial_sha_to_index(self, partial_bin_sha, canonical_length): + """ + :return: index as in `sha_to_index` or None if the sha was not found in this + index file + :param partial_bin_sha: an at least two bytes of a partial binary sha as bytes + :param canonical_length: length of the original hexadecimal representation of the + given partial binary sha + :raise AmbiguousObjectName:""" + if len(partial_bin_sha) < 2: + raise ValueError("Require at least 2 bytes of partial sha") + + assert isinstance(partial_bin_sha, bytes), "partial_bin_sha must be bytes" + first_byte = byte_ord(partial_bin_sha[0]) + + get_sha = self.sha + lo = 0 # lower index, the left bound of the bisection + if first_byte != 0: + lo = self._fanout_table[first_byte - 1] + hi = self._fanout_table[first_byte] # the upper, right bound of the bisection + + # fill the partial to full 20 bytes + filled_sha = partial_bin_sha + NULL_BYTE * (20 - len(partial_bin_sha)) + + # find lowest + while lo < hi: + mid = (lo + hi) // 2 + mid_sha = get_sha(mid) + if filled_sha < mid_sha: + hi = mid + elif filled_sha == mid_sha: + # perfect match + lo = mid + break + else: + lo = mid + 1 + # END handle midpoint + # END bisect + + if lo < self.size(): + cur_sha = get_sha(lo) + if is_equal_canonical_sha(canonical_length, partial_bin_sha, cur_sha): + next_sha = None + if lo + 1 < self.size(): + next_sha = get_sha(lo + 1) + if next_sha and next_sha == cur_sha: + raise AmbiguousObjectName(partial_bin_sha) + return lo + # END if we have a match + # END if we found something + return None + + if 'PackIndexFile_sha_to_index' in globals(): + # NOTE: Its just about 25% faster, the major bottleneck might be the attr + # accesses + def sha_to_index(self, sha): + return PackIndexFile_sha_to_index(self, sha) + # END redefine heavy-hitter with c version + + #} END properties + + +class PackFile(LazyMixin): + + """A pack is a file written according to the Version 2 for git packs + + As we currently use memory maps, it could be assumed that the maximum size of + packs therefore is 32 bit on 32 bit systems. On 64 bit systems, this should be + fine though. + + **Note:** at some point, this might be implemented using streams as well, or + streams are an alternate path in the case memory maps cannot be created + for some reason - one clearly doesn't want to read 10GB at once in that + case""" + + __slots__ = ('_packpath', '_cursor', '_size', '_version') + pack_signature = 0x5041434b # 'PACK' + pack_version_default = 2 + + # offset into our data at which the first object starts + first_object_offset = 3 * 4 # header bytes + footer_size = 20 # final sha + + def __init__(self, packpath): + self._packpath = packpath + + def close(self): + mman.force_map_handle_removal_win(self._packpath) + self._cursor = None + + def _set_cache_(self, attr): + # we fill the whole cache, whichever attribute gets queried first + self._cursor = mman.make_cursor(self._packpath).use_region() + + # read the header information + type_id, self._version, self._size = unpack_from(">LLL", self._cursor.map(), 0) + + # TODO: figure out whether we should better keep the lock, or maybe + # add a .keep file instead ? + if type_id != self.pack_signature: + raise ParseError("Invalid pack signature: %i" % type_id) + + def _iter_objects(self, start_offset, as_stream=True): + """Handle the actual iteration of objects within this pack""" + c = self._cursor + content_size = c.file_size() - self.footer_size + cur_offset = start_offset or self.first_object_offset + + null = NullStream() + while cur_offset < content_size: + data_offset, ostream = pack_object_at(c, cur_offset, True) + # scrub the stream to the end - this decompresses the object, but yields + # the amount of compressed bytes we need to get to the next offset + + stream_copy(ostream.read, null.write, ostream.size, chunk_size) + assert ostream.stream._br == ostream.size + cur_offset += (data_offset - ostream.pack_offset) + ostream.stream.compressed_bytes_read() + + # if a stream is requested, reset it beforehand + # Otherwise return the Stream object directly, its derived from the + # info object + if as_stream: + ostream.stream.seek(0) + yield ostream + # END until we have read everything + + #{ Pack Information + + def size(self): + """:return: The amount of objects stored in this pack""" + return self._size + + def version(self): + """:return: the version of this pack""" + return self._version + + def data(self): + """ + :return: read-only data of this pack. It provides random access and usually + is a memory map. + :note: This method is unsafe as it returns a window into a file which might be larger than than the actual window size""" + # can use map as we are starting at offset 0. Otherwise we would have to use buffer() + return self._cursor.use_region().map() + + def checksum(self): + """:return: 20 byte sha1 hash on all object sha's contained in this file""" + return self._cursor.use_region(self._cursor.file_size() - 20).buffer()[:] + + def path(self): + """:return: path to the packfile""" + return self._packpath + #} END pack information + + #{ Pack Specific + + def collect_streams(self, offset): + """ + :return: list of pack streams which are required to build the object + at the given offset. The first entry of the list is the object at offset, + the last one is either a full object, or a REF_Delta stream. The latter + type needs its reference object to be locked up in an ODB to form a valid + delta chain. + If the object at offset is no delta, the size of the list is 1. + :param offset: specifies the first byte of the object within this pack""" + out = list() + c = self._cursor + while True: + ostream = pack_object_at(c, offset, True)[1] + out.append(ostream) + if ostream.type_id == OFS_DELTA: + offset = ostream.pack_offset - ostream.delta_info + else: + # the only thing we can lookup are OFFSET deltas. Everything + # else is either an object, or a ref delta, in the latter + # case someone else has to find it + break + # END handle type + # END while chaining streams + return out + + #} END pack specific + + #{ Read-Database like Interface + + def info(self, offset): + """Retrieve information about the object at the given file-absolute offset + + :param offset: byte offset + :return: OPackInfo instance, the actual type differs depending on the type_id attribute""" + return pack_object_at(self._cursor, offset or self.first_object_offset, False)[1] + + def stream(self, offset): + """Retrieve an object at the given file-relative offset as stream along with its information + + :param offset: byte offset + :return: OPackStream instance, the actual type differs depending on the type_id attribute""" + return pack_object_at(self._cursor, offset or self.first_object_offset, True)[1] + + def stream_iter(self, start_offset=0): + """ + :return: iterator yielding OPackStream compatible instances, allowing + to access the data in the pack directly. + :param start_offset: offset to the first object to iterate. If 0, iteration + starts at the very first object in the pack. + + **Note:** Iterating a pack directly is costly as the datastream has to be decompressed + to determine the bounds between the objects""" + return self._iter_objects(start_offset, as_stream=True) + + #} END Read-Database like Interface + + +class PackEntity(LazyMixin): + + """Combines the PackIndexFile and the PackFile into one, allowing the + actual objects to be resolved and iterated""" + + __slots__ = ('_index', # our index file + '_pack', # our pack file + '_offset_map' # on demand dict mapping one offset to the next consecutive one + ) + + IndexFileCls = PackIndexFile + PackFileCls = PackFile + + def __init__(self, pack_or_index_path): + """Initialize ourselves with the path to the respective pack or index file""" + basename, ext = os.path.splitext(pack_or_index_path) + self._index = self.IndexFileCls("%s.idx" % basename) # PackIndexFile instance + self._pack = self.PackFileCls("%s.pack" % basename) # corresponding PackFile instance + + def close(self): + self._index.close() + self._pack.close() + + def _set_cache_(self, attr): + # currently this can only be _offset_map + # TODO: make this a simple sorted offset array which can be bisected + # to find the respective entry, from which we can take a +1 easily + # This might be slower, but should also be much lighter in memory ! + offsets_sorted = sorted(self._index.offsets()) + last_offset = len(self._pack.data()) - self._pack.footer_size + assert offsets_sorted, "Cannot handle empty indices" + + offset_map = None + if len(offsets_sorted) == 1: + offset_map = {offsets_sorted[0]: last_offset} + else: + iter_offsets = iter(offsets_sorted) + iter_offsets_plus_one = iter(offsets_sorted) + next(iter_offsets_plus_one) + consecutive = zip(iter_offsets, iter_offsets_plus_one) + + offset_map = dict(consecutive) + + # the last offset is not yet set + offset_map[offsets_sorted[-1]] = last_offset + # END handle offset amount + self._offset_map = offset_map + + def _sha_to_index(self, sha): + """:return: index for the given sha, or raise""" + index = self._index.sha_to_index(sha) + if index is None: + raise BadObject(sha) + return index + + def _iter_objects(self, as_stream): + """Iterate over all objects in our index and yield their OInfo or OStream instences""" + _sha = self._index.sha + _object = self._object + for index in range(self._index.size()): + yield _object(_sha(index), as_stream, index) + # END for each index + + def _object(self, sha, as_stream, index=-1): + """:return: OInfo or OStream object providing information about the given sha + :param index: if not -1, its assumed to be the sha's index in the IndexFile""" + # its a little bit redundant here, but it needs to be efficient + if index < 0: + index = self._sha_to_index(sha) + if sha is None: + sha = self._index.sha(index) + # END assure sha is present ( in output ) + offset = self._index.offset(index) + type_id, uncomp_size, data_rela_offset = pack_object_header_info(self._pack._cursor.use_region(offset).buffer()) + if as_stream: + if type_id not in delta_types: + packstream = self._pack.stream(offset) + return OStream(sha, packstream.type, packstream.size, packstream.stream) + # END handle non-deltas + + # produce a delta stream containing all info + # To prevent it from applying the deltas when querying the size, + # we extract it from the delta stream ourselves + streams = self.collect_streams_at_offset(offset) + dstream = DeltaApplyReader.new(streams) + + return ODeltaStream(sha, dstream.type, None, dstream) + else: + if type_id not in delta_types: + return OInfo(sha, type_id_to_type_map[type_id], uncomp_size) + # END handle non-deltas + + # deltas are a little tougher - unpack the first bytes to obtain + # the actual target size, as opposed to the size of the delta data + streams = self.collect_streams_at_offset(offset) + buf = streams[0].read(512) + offset, src_size = msb_size(buf) + offset, target_size = msb_size(buf, offset) + + # collect the streams to obtain the actual object type + if streams[-1].type_id in delta_types: + raise BadObject(sha, "Could not resolve delta object") + return OInfo(sha, streams[-1].type, target_size) + # END handle stream + + #{ Read-Database like Interface + + def info(self, sha): + """Retrieve information about the object identified by the given sha + + :param sha: 20 byte sha1 + :raise BadObject: + :return: OInfo instance, with 20 byte sha""" + return self._object(sha, False) + + def stream(self, sha): + """Retrieve an object stream along with its information as identified by the given sha + + :param sha: 20 byte sha1 + :raise BadObject: + :return: OStream instance, with 20 byte sha""" + return self._object(sha, True) + + def info_at_index(self, index): + """As ``info``, but uses a PackIndexFile compatible index to refer to the object""" + return self._object(None, False, index) + + def stream_at_index(self, index): + """As ``stream``, but uses a PackIndexFile compatible index to refer to the + object""" + return self._object(None, True, index) + + #} END Read-Database like Interface + + #{ Interface + + def pack(self): + """:return: the underlying pack file instance""" + return self._pack + + def index(self): + """:return: the underlying pack index file instance""" + return self._index + + def is_valid_stream(self, sha, use_crc=False): + """ + Verify that the stream at the given sha is valid. + + :param use_crc: if True, the index' crc is run over the compressed stream of + the object, which is much faster than checking the sha1. It is also + more prone to unnoticed corruption or manipulation. + :param sha: 20 byte sha1 of the object whose stream to verify + whether the compressed stream of the object is valid. If it is + a delta, this only verifies that the delta's data is valid, not the + data of the actual undeltified object, as it depends on more than + just this stream. + If False, the object will be decompressed and the sha generated. It must + match the given sha + + :return: True if the stream is valid + :raise UnsupportedOperation: If the index is version 1 only + :raise BadObject: sha was not found""" + if use_crc: + if self._index.version() < 2: + raise UnsupportedOperation("Version 1 indices do not contain crc's, verify by sha instead") + # END handle index version + + index = self._sha_to_index(sha) + offset = self._index.offset(index) + next_offset = self._offset_map[offset] + crc_value = self._index.crc(index) + + # create the current crc value, on the compressed object data + # Read it in chunks, without copying the data + crc_update = zlib.crc32 + pack_data = self._pack.data() + cur_pos = offset + this_crc_value = 0 + while cur_pos < next_offset: + rbound = min(cur_pos + chunk_size, next_offset) + size = rbound - cur_pos + this_crc_value = crc_update(pack_data[cur_pos:cur_pos + size], this_crc_value) + cur_pos += size + # END window size loop + + # crc returns signed 32 bit numbers, the AND op forces it into unsigned + # mode ... wow, sneaky, from dulwich. + return (this_crc_value & 0xffffffff) == crc_value + else: + shawriter = Sha1Writer() + stream = self._object(sha, as_stream=True) + # write a loose object, which is the basis for the sha + write_object(stream.type, stream.size, stream.read, shawriter.write) + + assert shawriter.sha(as_hex=False) == sha + return shawriter.sha(as_hex=False) == sha + # END handle crc/sha verification + + def info_iter(self): + """ + :return: Iterator over all objects in this pack. The iterator yields + OInfo instances""" + return self._iter_objects(as_stream=False) + + def stream_iter(self): + """ + :return: iterator over all objects in this pack. The iterator yields + OStream instances""" + return self._iter_objects(as_stream=True) + + def collect_streams_at_offset(self, offset): + """ + As the version in the PackFile, but can resolve REF deltas within this pack + For more info, see ``collect_streams`` + + :param offset: offset into the pack file at which the object can be found""" + streams = self._pack.collect_streams(offset) + + # try to resolve the last one if needed. It is assumed to be either + # a REF delta, or a base object, as OFFSET deltas are resolved by the pack + if streams[-1].type_id == REF_DELTA: + stream = streams[-1] + while stream.type_id in delta_types: + if stream.type_id == REF_DELTA: + # smmap can return memory view objects, which can't be compared as buffers/bytes can ... + if isinstance(stream.delta_info, memoryview): + sindex = self._index.sha_to_index(stream.delta_info.tobytes()) + else: + sindex = self._index.sha_to_index(stream.delta_info) + if sindex is None: + break + stream = self._pack.stream(self._index.offset(sindex)) + streams.append(stream) + else: + # must be another OFS DELTA - this could happen if a REF + # delta we resolve previously points to an OFS delta. Who + # would do that ;) ? We can handle it though + stream = self._pack.stream(stream.delta_info) + streams.append(stream) + # END handle ref delta + # END resolve ref streams + # END resolve streams + + return streams + + def collect_streams(self, sha): + """ + As ``PackFile.collect_streams``, but takes a sha instead of an offset. + Additionally, ref_delta streams will be resolved within this pack. + If this is not possible, the stream will be left alone, hence it is adivsed + to check for unresolved ref-deltas and resolve them before attempting to + construct a delta stream. + + :param sha: 20 byte sha1 specifying the object whose related streams you want to collect + :return: list of streams, first being the actual object delta, the last being + a possibly unresolved base object. + :raise BadObject:""" + return self.collect_streams_at_offset(self._index.offset(self._sha_to_index(sha))) + + @classmethod + def write_pack(cls, object_iter, pack_write, index_write=None, + object_count=None, zlib_compression=zlib.Z_BEST_SPEED): + """ + Create a new pack by putting all objects obtained by the object_iterator + into a pack which is written using the pack_write method. + The respective index is produced as well if index_write is not Non. + + :param object_iter: iterator yielding odb output objects + :param pack_write: function to receive strings to write into the pack stream + :param indx_write: if not None, the function writes the index file corresponding + to the pack. + :param object_count: if you can provide the amount of objects in your iteration, + this would be the place to put it. Otherwise we have to pre-iterate and store + all items into a list to get the number, which uses more memory than necessary. + :param zlib_compression: the zlib compression level to use + :return: tuple(pack_sha, index_binsha) binary sha over all the contents of the pack + and over all contents of the index. If index_write was None, index_binsha will be None + + **Note:** The destination of the write functions is up to the user. It could + be a socket, or a file for instance + + **Note:** writes only undeltified objects""" + objs = object_iter + if not object_count: + if not isinstance(object_iter, (tuple, list)): + objs = list(object_iter) + # END handle list type + object_count = len(objs) + # END handle object + + pack_writer = FlexibleSha1Writer(pack_write) + pwrite = pack_writer.write + ofs = 0 # current offset into the pack file + index = None + wants_index = index_write is not None + + # write header + pwrite(pack('>LLL', PackFile.pack_signature, PackFile.pack_version_default, object_count)) + ofs += 12 + + if wants_index: + index = IndexWriter() + # END handle index header + + actual_count = 0 + for obj in objs: + actual_count += 1 + crc = 0 + + # object header + hdr = create_pack_object_header(obj.type_id, obj.size) + if index_write: + crc = crc32(hdr) + else: + crc = None + # END handle crc + pwrite(hdr) + + # data stream + zstream = zlib.compressobj(zlib_compression) + ostream = obj.stream + br, bw, crc = write_stream_to_pack(ostream.read, pwrite, zstream, base_crc=crc) + assert(br == obj.size) + if wants_index: + index.append(obj.binsha, crc, ofs) + # END handle index + + ofs += len(hdr) + bw + if actual_count == object_count: + break + # END abort once we are done + # END for each object + + if actual_count != object_count: + raise ValueError( + "Expected to write %i objects into pack, but received only %i from iterators" % (object_count, actual_count)) + # END count assertion + + # write footer + pack_sha = pack_writer.sha(as_hex=False) + assert len(pack_sha) == 20 + pack_write(pack_sha) + ofs += len(pack_sha) # just for completeness ;) + + index_sha = None + if wants_index: + index_sha = index.write(pack_sha, index_write) + # END handle index + + return pack_sha, index_sha + + @classmethod + def create(cls, object_iter, base_dir, object_count=None, zlib_compression=zlib.Z_BEST_SPEED): + """Create a new on-disk entity comprised of a properly named pack file and a properly named + and corresponding index file. The pack contains all OStream objects contained in object iter. + :param base_dir: directory which is to contain the files + :return: PackEntity instance initialized with the new pack + + **Note:** for more information on the other parameters see the write_pack method""" + pack_fd, pack_path = tempfile.mkstemp('', 'pack', base_dir) + index_fd, index_path = tempfile.mkstemp('', 'index', base_dir) + pack_write = lambda d: os.write(pack_fd, d) + index_write = lambda d: os.write(index_fd, d) + + pack_binsha, index_binsha = cls.write_pack(object_iter, pack_write, index_write, object_count, zlib_compression) + os.close(pack_fd) + os.close(index_fd) + + fmt = "pack-%s.%s" + new_pack_path = os.path.join(base_dir, fmt % (bin_to_hex(pack_binsha), 'pack')) + new_index_path = os.path.join(base_dir, fmt % (bin_to_hex(pack_binsha), 'idx')) + os.rename(pack_path, new_pack_path) + os.rename(index_path, new_index_path) + + return cls(new_pack_path) + + #} END interface diff --git a/gitdb/gitdb/stream.py b/gitdb/gitdb/stream.py new file mode 100644 index 000000000..7c59f0b3c --- /dev/null +++ b/gitdb/gitdb/stream.py @@ -0,0 +1,755 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: https://opensource.org/license/bsd-3-clause/ + +from io import BytesIO + +import mmap +import os +import sys +import zlib + +from gitdb.fun import ( + msb_size, + stream_copy, + apply_delta_data, + connect_deltas, + delta_types +) + +from gitdb.util import ( + allocate_memory, + LazyMixin, + make_sha, + write, + close, +) + +from gitdb.const import NULL_BYTE, BYTE_SPACE +from gitdb.utils.encoding import force_bytes + +has_perf_mod = False +try: + from gitdb_speedups._perf import apply_delta as c_apply_delta + has_perf_mod = True +except ImportError: + pass + +__all__ = ('DecompressMemMapReader', 'FDCompressedSha1Writer', 'DeltaApplyReader', + 'Sha1Writer', 'FlexibleSha1Writer', 'ZippedStoreShaWriter', 'FDCompressedSha1Writer', + 'FDStream', 'NullStream') + + +#{ RO Streams + +class DecompressMemMapReader(LazyMixin): + + """Reads data in chunks from a memory map and decompresses it. The client sees + only the uncompressed data, respective file-like read calls are handling on-demand + buffered decompression accordingly + + A constraint on the total size of bytes is activated, simulating + a logical file within a possibly larger physical memory area + + To read efficiently, you clearly don't want to read individual bytes, instead, + read a few kilobytes at least. + + **Note:** The chunk-size should be carefully selected as it will involve quite a bit + of string copying due to the way the zlib is implemented. Its very wasteful, + hence we try to find a good tradeoff between allocation time and number of + times we actually allocate. An own zlib implementation would be good here + to better support streamed reading - it would only need to keep the mmap + and decompress it into chunks, that's all ... """ + __slots__ = ('_m', '_zip', '_buf', '_buflen', '_br', '_cws', '_cwe', '_s', '_close', + '_cbr', '_phi') + + max_read_size = 512 * 1024 # currently unused + + def __init__(self, m, close_on_deletion, size=None): + """Initialize with mmap for stream reading + :param m: must be content data - use new if you have object data and no size""" + self._m = m + self._zip = zlib.decompressobj() + self._buf = None # buffer of decompressed bytes + self._buflen = 0 # length of bytes in buffer + if size is not None: + self._s = size # size of uncompressed data to read in total + self._br = 0 # num uncompressed bytes read + self._cws = 0 # start byte of compression window + self._cwe = 0 # end byte of compression window + self._cbr = 0 # number of compressed bytes read + self._phi = False # is True if we parsed the header info + self._close = close_on_deletion # close the memmap on deletion ? + + def _set_cache_(self, attr): + assert attr == '_s' + # only happens for size, which is a marker to indicate we still + # have to parse the header from the stream + self._parse_header_info() + + def __del__(self): + self.close() + + def _parse_header_info(self): + """If this stream contains object data, parse the header info and skip the + stream to a point where each read will yield object content + + :return: parsed type_string, size""" + # read header + # should really be enough, cgit uses 8192 I believe + # And for good reason !! This needs to be that high for the header to be read correctly in all cases + maxb = 8192 + self._s = maxb + hdr = self.read(maxb) + hdrend = hdr.find(NULL_BYTE) + typ, size = hdr[:hdrend].split(BYTE_SPACE) + size = int(size) + self._s = size + + # adjust internal state to match actual header length that we ignore + # The buffer will be depleted first on future reads + self._br = 0 + hdrend += 1 + self._buf = BytesIO(hdr[hdrend:]) + self._buflen = len(hdr) - hdrend + + self._phi = True + + return typ, size + + #{ Interface + + @classmethod + def new(self, m, close_on_deletion=False): + """Create a new DecompressMemMapReader instance for acting as a read-only stream + This method parses the object header from m and returns the parsed + type and size, as well as the created stream instance. + + :param m: memory map on which to operate. It must be object data ( header + contents ) + :param close_on_deletion: if True, the memory map will be closed once we are + being deleted""" + inst = DecompressMemMapReader(m, close_on_deletion, 0) + typ, size = inst._parse_header_info() + return typ, size, inst + + def data(self): + """:return: random access compatible data we are working on""" + return self._m + + def close(self): + """Close our underlying stream of compressed bytes if this was allowed during initialization + :return: True if we closed the underlying stream + :note: can be called safely + """ + if self._close: + if hasattr(self._m, 'close'): + self._m.close() + self._close = False + # END handle resource freeing + + def compressed_bytes_read(self): + """ + :return: number of compressed bytes read. This includes the bytes it + took to decompress the header ( if there was one )""" + # ABSTRACT: When decompressing a byte stream, it can be that the first + # x bytes which were requested match the first x bytes in the loosely + # compressed datastream. This is the worst-case assumption that the reader + # does, it assumes that it will get at least X bytes from X compressed bytes + # in call cases. + # The caveat is that the object, according to our known uncompressed size, + # is already complete, but there are still some bytes left in the compressed + # stream that contribute to the amount of compressed bytes. + # How can we know that we are truly done, and have read all bytes we need + # to read ? + # Without help, we cannot know, as we need to obtain the status of the + # decompression. If it is not finished, we need to decompress more data + # until it is finished, to yield the actual number of compressed bytes + # belonging to the decompressed object + # We are using a custom zlib module for this, if its not present, + # we try to put in additional bytes up for decompression if feasible + # and check for the unused_data. + + # Only scrub the stream forward if we are officially done with the + # bytes we were to have. + if self._br == self._s and not self._zip.unused_data: + # manipulate the bytes-read to allow our own read method to continue + # but keep the window at its current position + self._br = 0 + if hasattr(self._zip, 'status'): + while self._zip.status == zlib.Z_OK: + self.read(mmap.PAGESIZE) + # END scrub-loop custom zlib + else: + # pass in additional pages, until we have unused data + while not self._zip.unused_data and self._cbr != len(self._m): + self.read(mmap.PAGESIZE) + # END scrub-loop default zlib + # END handle stream scrubbing + + # reset bytes read, just to be sure + self._br = self._s + # END handle stream scrubbing + + # unused data ends up in the unconsumed tail, which was removed + # from the count already + return self._cbr + + #} END interface + + def seek(self, offset, whence=getattr(os, 'SEEK_SET', 0)): + """Allows to reset the stream to restart reading + :raise ValueError: If offset and whence are not 0""" + if offset != 0 or whence != getattr(os, 'SEEK_SET', 0): + raise ValueError("Can only seek to position 0") + # END handle offset + + self._zip = zlib.decompressobj() + self._br = self._cws = self._cwe = self._cbr = 0 + if self._phi: + self._phi = False + del(self._s) # trigger header parsing on first access + # END skip header + + def read(self, size=-1): + if size < 1: + size = self._s - self._br + else: + size = min(size, self._s - self._br) + # END clamp size + + if size == 0: + return b'' + # END handle depletion + + # deplete the buffer, then just continue using the decompress object + # which has an own buffer. We just need this to transparently parse the + # header from the zlib stream + dat = b'' + if self._buf: + if self._buflen >= size: + # have enough data + dat = self._buf.read(size) + self._buflen -= size + self._br += size + return dat + else: + dat = self._buf.read() # ouch, duplicates data + size -= self._buflen + self._br += self._buflen + + self._buflen = 0 + self._buf = None + # END handle buffer len + # END handle buffer + + # decompress some data + # Abstract: zlib needs to operate on chunks of our memory map ( which may + # be large ), as it will otherwise and always fill in the 'unconsumed_tail' + # attribute which possible reads our whole map to the end, forcing + # everything to be read from disk even though just a portion was requested. + # As this would be a nogo, we workaround it by passing only chunks of data, + # moving the window into the memory map along as we decompress, which keeps + # the tail smaller than our chunk-size. This causes 'only' the chunk to be + # copied once, and another copy of a part of it when it creates the unconsumed + # tail. We have to use it to hand in the appropriate amount of bytes during + # the next read. + # + # Decompress in a loop until we have produced `size` bytes or run out + # of progress. Iteration (instead of recursion) keeps the call bounded + # for streams that consume many input bytes per produced output byte + # (e.g. zlib stored blocks of length zero); the previous recursive + # form blew the stack on inputs > ~1500 empty blocks (issue #120 + # follow-up). + dcompdat = b'' + while True: + tail = self._zip.unconsumed_tail + remaining = size - len(dcompdat) + if tail: + # move the window, make it as large as size demands. For code-clarity, + # we just take the chunk from our map again instead of reusing the unconsumed + # tail. The latter one would save some memory copying, but we could end up + # with not getting enough data uncompressed, so we had to sort that out as well. + # Now we just assume the worst case, hence the data is uncompressed and the window + # needs to be as large as the uncompressed bytes we want to read. + self._cws = self._cwe - len(tail) + self._cwe = self._cws + remaining + else: + cws = self._cws + self._cws = self._cwe + self._cwe = cws + remaining + # END handle tail + + # if window is too small, make it larger so zip can decompress something + if self._cwe - self._cws < 8: + self._cwe = self._cws + 8 + # END adjust winsize + + # takes a slice, but doesn't copy the data, it says ... + indata = self._m[self._cws:self._cwe] + + # get the actual window end to be sure we don't use it for computations + self._cwe = self._cws + len(indata) + chunk = self._zip.decompress(indata, remaining) + # update the amount of compressed bytes read + # We feed possibly overlapping chunks, which is why the unconsumed tail + # has to be taken into consideration, as well as the unused data + # if we hit the end of the stream + # NOTE: Behavior changed in PY2.7 onward, which requires special handling to make the tests work properly. + # They are thorough, and I assume it is truly working. + # Why is this logic as convoluted as it is ? Please look at the table in + # https://github.com/gitpython-developers/gitdb/issues/19 to learn about the test-results. + # Basically, on py2.6, you want to use branch 1, whereas on all other python version, the second branch + # will be the one that works. + # However, the zlib VERSIONs as well as the platform check is used to further match the entries in the + # table in the github issue. This is it ... it was the only way I could make this work everywhere. + # IT's CERTAINLY GOING TO BITE US IN THE FUTURE ... . + if getattr(zlib, 'ZLIB_RUNTIME_VERSION', zlib.ZLIB_VERSION) in ('1.2.7', '1.2.5') and not sys.platform == 'darwin': + unused_datalen = len(self._zip.unconsumed_tail) + else: + unused_datalen = len(self._zip.unconsumed_tail) + len(self._zip.unused_data) + # # end handle very special case ... + + consumed = len(indata) - unused_datalen + self._cbr += consumed + self._br += len(chunk) + if chunk: + if not isinstance(dcompdat, bytearray): + dcompdat = bytearray(dcompdat) + dcompdat.extend(chunk) + + # Stop when we have enough or there is no path to more output. + # `chunk` may legitimately be empty mid-stream when zlib is + # consuming header / dictionary frames; in that case we keep + # iterating as long as we are still feeding zlib new bytes + # (consumed > 0) and zlib has not flagged end-of-stream. The + # compressed_bytes_read() scrub loop drives this same code with + # _br manipulated to 0 past zip EOF; it terminates here because + # `getattr(_zip, 'eof', False)` is True or no compressed bytes + # are consumed. The empty-block recursion attack from issue #120 + # follow-up is bounded by the iteration; each empty block does + # consume input, so the loop walks the stream forward a constant + # amount per iteration without growing the call stack. + if len(dcompdat) >= size or self._br >= self._s: + break + zip_eof = getattr(self._zip, 'eof', False) + if not chunk and (zip_eof or len(indata) == 0 or consumed == 0): + break + # END iterative decompress + + if dat: + dcompdat = dat + dcompdat + # END prepend our cached data + + return dcompdat + + +class DeltaApplyReader(LazyMixin): + + """A reader which dynamically applies pack deltas to a base object, keeping the + memory demands to a minimum. + + The size of the final object is only obtainable once all deltas have been + applied, unless it is retrieved from a pack index. + + The uncompressed Delta has the following layout (MSB being a most significant + bit encoded dynamic size): + + * MSB Source Size - the size of the base against which the delta was created + * MSB Target Size - the size of the resulting data after the delta was applied + * A list of one byte commands (cmd) which are followed by a specific protocol: + + * cmd & 0x80 - copy delta_data[offset:offset+size] + + * Followed by an encoded offset into the delta data + * Followed by an encoded size of the chunk to copy + + * cmd & 0x7f - insert + + * insert cmd bytes from the delta buffer into the output stream + + * cmd == 0 - invalid operation ( or error in delta stream ) + """ + __slots__ = ( + "_bstream", # base stream to which to apply the deltas + "_dstreams", # tuple of delta stream readers + "_mm_target", # memory map of the delta-applied data + "_size", # actual number of bytes in _mm_target + "_br" # number of bytes read + ) + + #{ Configuration + k_max_memory_move = 250 * 1000 * 1000 + #} END configuration + + def __init__(self, stream_list): + """Initialize this instance with a list of streams, the first stream being + the delta to apply on top of all following deltas, the last stream being the + base object onto which to apply the deltas""" + assert len(stream_list) > 1, "Need at least one delta and one base stream" + + self._bstream = stream_list[-1] + self._dstreams = tuple(stream_list[:-1]) + self._br = 0 + + def _set_cache_too_slow_without_c(self, attr): + # the direct algorithm is fastest and most direct if there is only one + # delta. Also, the extra overhead might not be worth it for items smaller + # than X - definitely the case in python, every function call costs + # huge amounts of time + # if len(self._dstreams) * self._bstream.size < self.k_max_memory_move: + if len(self._dstreams) == 1: + return self._set_cache_brute_(attr) + + # Aggregate all deltas into one delta in reverse order. Hence we take + # the last delta, and reverse-merge its ancestor delta, until we receive + # the final delta data stream. + dcl = connect_deltas(self._dstreams) + + # call len directly, as the (optional) c version doesn't implement the sequence + # protocol + if dcl.rbound() == 0: + self._size = 0 + self._mm_target = allocate_memory(0) + return + # END handle empty list + + self._size = dcl.rbound() + self._mm_target = allocate_memory(self._size) + + bbuf = allocate_memory(self._bstream.size) + stream_copy(self._bstream.read, bbuf.write, self._bstream.size, 256 * mmap.PAGESIZE) + + # APPLY CHUNKS + write = self._mm_target.write + dcl.apply(bbuf, write) + + self._mm_target.seek(0) + + def _set_cache_brute_(self, attr): + """If we are here, we apply the actual deltas""" + # TODO: There should be a special case if there is only one stream + # Then the default-git algorithm should perform a tad faster, as the + # delta is not peaked into, causing less overhead. + buffer_info_list = list() + max_target_size = 0 + for dstream in self._dstreams: + buf = dstream.read(512) # read the header information + X + offset, src_size = msb_size(buf) + offset, target_size = msb_size(buf, offset) + buffer_info_list.append((buf[offset:], offset, src_size, target_size)) + max_target_size = max(max_target_size, target_size) + # END for each delta stream + + # sanity check - the first delta to apply should have the same source + # size as our actual base stream + base_size = self._bstream.size + target_size = max_target_size + + # if we have more than 1 delta to apply, we will swap buffers, hence we must + # assure that all buffers we use are large enough to hold all the results + if len(self._dstreams) > 1: + base_size = target_size = max(base_size, max_target_size) + # END adjust buffer sizes + + # Allocate private memory map big enough to hold the first base buffer + # We need random access to it + bbuf = allocate_memory(base_size) + stream_copy(self._bstream.read, bbuf.write, base_size, 256 * mmap.PAGESIZE) + + # allocate memory map large enough for the largest (intermediate) target + # We will use it as scratch space for all delta ops. If the final + # target buffer is smaller than our allocated space, we just use parts + # of it upon return. + tbuf = allocate_memory(target_size) + + # for each delta to apply, memory map the decompressed delta and + # work on the op-codes to reconstruct everything. + # For the actual copying, we use a seek and write pattern of buffer + # slices. + final_target_size = None + for (dbuf, offset, src_size, target_size), dstream in zip(reversed(buffer_info_list), reversed(self._dstreams)): + # allocate a buffer to hold all delta data - fill in the data for + # fast access. We do this as we know that reading individual bytes + # from our stream would be slower than necessary ( although possible ) + # The dbuf buffer contains commands after the first two MSB sizes, the + # offset specifies the amount of bytes read to get the sizes. + ddata = allocate_memory(dstream.size - offset) + ddata.write(dbuf) + # read the rest from the stream. The size we give is larger than necessary + stream_copy(dstream.read, ddata.write, dstream.size, 256 * mmap.PAGESIZE) + + ####################################################################### + if 'c_apply_delta' in globals(): + c_apply_delta(bbuf, ddata, tbuf) + else: + apply_delta_data(bbuf, src_size, ddata, len(ddata), tbuf.write) + ####################################################################### + + # finally, swap out source and target buffers. The target is now the + # base for the next delta to apply + bbuf, tbuf = tbuf, bbuf + bbuf.seek(0) + tbuf.seek(0) + final_target_size = target_size + # END for each delta to apply + + # its already seeked to 0, constrain it to the actual size + # NOTE: in the end of the loop, it swaps buffers, hence our target buffer + # is not tbuf, but bbuf ! + self._mm_target = bbuf + self._size = final_target_size + + #{ Configuration + if not has_perf_mod: + _set_cache_ = _set_cache_brute_ + else: + _set_cache_ = _set_cache_too_slow_without_c + + #} END configuration + + def read(self, count=0): + bl = self._size - self._br # bytes left + if count < 1 or count > bl: + count = bl + # NOTE: we could check for certain size limits, and possibly + # return buffers instead of strings to prevent byte copying + data = self._mm_target.read(count) + self._br += len(data) + return data + + def seek(self, offset, whence=getattr(os, 'SEEK_SET', 0)): + """Allows to reset the stream to restart reading + + :raise ValueError: If offset and whence are not 0""" + if offset != 0 or whence != getattr(os, 'SEEK_SET', 0): + raise ValueError("Can only seek to position 0") + # END handle offset + self._br = 0 + self._mm_target.seek(0) + + #{ Interface + + @classmethod + def new(cls, stream_list): + """ + Convert the given list of streams into a stream which resolves deltas + when reading from it. + + :param stream_list: two or more stream objects, first stream is a Delta + to the object that you want to resolve, followed by N additional delta + streams. The list's last stream must be a non-delta stream. + + :return: Non-Delta OPackStream object whose stream can be used to obtain + the decompressed resolved data + :raise ValueError: if the stream list cannot be handled""" + if len(stream_list) < 2: + raise ValueError("Need at least two streams") + # END single object special handling + + if stream_list[-1].type_id in delta_types: + raise ValueError( + "Cannot resolve deltas if there is no base object stream, last one was type: %s" % stream_list[-1].type) + # END check stream + return cls(stream_list) + + #} END interface + + #{ OInfo like Interface + + @property + def type(self): + return self._bstream.type + + @property + def type_id(self): + return self._bstream.type_id + + @property + def size(self): + """:return: number of uncompressed bytes in the stream""" + return self._size + + #} END oinfo like interface + + +#} END RO streams + + +#{ W Streams + +class Sha1Writer: + + """Simple stream writer which produces a sha whenever you like as it degests + everything it is supposed to write""" + __slots__ = "sha1" + + def __init__(self): + self.sha1 = make_sha() + + #{ Stream Interface + + def write(self, data): + """:raise IOError: If not all bytes could be written + :param data: byte object + :return: length of incoming data""" + + self.sha1.update(data) + + return len(data) + + # END stream interface + + #{ Interface + + def sha(self, as_hex=False): + """:return: sha so far + :param as_hex: if True, sha will be hex-encoded, binary otherwise""" + if as_hex: + return self.sha1.hexdigest() + return self.sha1.digest() + + #} END interface + + +class FlexibleSha1Writer(Sha1Writer): + + """Writer producing a sha1 while passing on the written bytes to the given + write function""" + __slots__ = 'writer' + + def __init__(self, writer): + Sha1Writer.__init__(self) + self.writer = writer + + def write(self, data): + Sha1Writer.write(self, data) + self.writer(data) + + +class ZippedStoreShaWriter(Sha1Writer): + + """Remembers everything someone writes to it and generates a sha""" + __slots__ = ('buf', 'zip') + + def __init__(self): + Sha1Writer.__init__(self) + self.buf = BytesIO() + self.zip = zlib.compressobj(zlib.Z_BEST_SPEED) + + def __getattr__(self, attr): + return getattr(self.buf, attr) + + def write(self, data): + alen = Sha1Writer.write(self, data) + self.buf.write(self.zip.compress(data)) + + return alen + + def close(self): + self.buf.write(self.zip.flush()) + + def seek(self, offset, whence=getattr(os, 'SEEK_SET', 0)): + """Seeking currently only supports to rewind written data + Multiple writes are not supported""" + if offset != 0 or whence != getattr(os, 'SEEK_SET', 0): + raise ValueError("Can only seek to position 0") + # END handle offset + self.buf.seek(0) + + def getvalue(self): + """:return: string value from the current stream position to the end""" + return self.buf.getvalue() + + +class FDCompressedSha1Writer(Sha1Writer): + + """Digests data written to it, making the sha available, then compress the + data and write it to the file descriptor + + **Note:** operates on raw file descriptors + **Note:** for this to work, you have to use the close-method of this instance""" + __slots__ = ("fd", "sha1", "zip") + + # default exception + exc = IOError("Failed to write all bytes to filedescriptor") + + def __init__(self, fd): + super().__init__() + self.fd = fd + self.zip = zlib.compressobj(zlib.Z_BEST_SPEED) + + #{ Stream Interface + + def write(self, data): + """:raise IOError: If not all bytes could be written + :return: length of incoming data""" + self.sha1.update(data) + cdata = self.zip.compress(data) + bytes_written = write(self.fd, cdata) + + if bytes_written != len(cdata): + raise self.exc + + return len(data) + + def close(self): + remainder = self.zip.flush() + if write(self.fd, remainder) != len(remainder): + raise self.exc + return close(self.fd) + + #} END stream interface + + +class FDStream: + + """A simple wrapper providing the most basic functions on a file descriptor + with the fileobject interface. Cannot use os.fdopen as the resulting stream + takes ownership""" + __slots__ = ("_fd", '_pos') + + def __init__(self, fd): + self._fd = fd + self._pos = 0 + + def write(self, data): + self._pos += len(data) + os.write(self._fd, data) + + def read(self, count=0): + if count == 0: + count = os.path.getsize(self._filepath) + # END handle read everything + + bytes = os.read(self._fd, count) + self._pos += len(bytes) + return bytes + + def fileno(self): + return self._fd + + def tell(self): + return self._pos + + def close(self): + close(self._fd) + + +class NullStream: + + """A stream that does nothing but providing a stream interface. + Use it like /dev/null""" + __slots__ = tuple() + + def read(self, size=0): + return '' + + def close(self): + pass + + def write(self, data): + return len(data) + + +#} END W streams diff --git a/gitdb/gitdb/test/__init__.py b/gitdb/gitdb/test/__init__.py new file mode 100644 index 000000000..03bd406be --- /dev/null +++ b/gitdb/gitdb/test/__init__.py @@ -0,0 +1,4 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: https://opensource.org/license/bsd-3-clause/ diff --git a/gitdb/gitdb/test/db/__init__.py b/gitdb/gitdb/test/db/__init__.py new file mode 100644 index 000000000..03bd406be --- /dev/null +++ b/gitdb/gitdb/test/db/__init__.py @@ -0,0 +1,4 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: https://opensource.org/license/bsd-3-clause/ diff --git a/gitdb/gitdb/test/db/lib.py b/gitdb/gitdb/test/db/lib.py new file mode 100644 index 000000000..408dd8c66 --- /dev/null +++ b/gitdb/gitdb/test/db/lib.py @@ -0,0 +1,128 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: https://opensource.org/license/bsd-3-clause/ +"""Base classes for object db testing""" +from gitdb.test.lib import ( + with_rw_directory, + with_packs_rw, + fixture_path, + TestBase +) + +from gitdb.stream import ( + Sha1Writer, + ZippedStoreShaWriter +) + +from gitdb.base import ( + IStream, + OStream, + OInfo +) + +from gitdb.exc import BadObject +from gitdb.typ import str_blob_type + +from io import BytesIO + +from struct import pack + + +__all__ = ('TestDBBase', 'with_rw_directory', 'with_packs_rw', 'fixture_path') + + +class TestDBBase(TestBase): + + """Base class providing testing routines on databases""" + + # data + two_lines = b'1234\nhello world' + all_data = (two_lines, ) + + def _assert_object_writing_simple(self, db): + # write a bunch of objects and query their streams and info + null_objs = db.size() + ni = 250 + for i in range(ni): + data = pack(">L", i) + istream = IStream(str_blob_type, len(data), BytesIO(data)) + new_istream = db.store(istream) + assert new_istream is istream + assert db.has_object(istream.binsha) + + info = db.info(istream.binsha) + assert isinstance(info, OInfo) + assert info.type == istream.type and info.size == istream.size + + stream = db.stream(istream.binsha) + assert isinstance(stream, OStream) + assert stream.binsha == info.binsha and stream.type == info.type + assert stream.read() == data + # END for each item + + assert db.size() == null_objs + ni + shas = list(db.sha_iter()) + assert len(shas) == db.size() + assert len(shas[0]) == 20 + + def _assert_object_writing(self, db): + """General tests to verify object writing, compatible to ObjectDBW + **Note:** requires write access to the database""" + # start in 'dry-run' mode, using a simple sha1 writer + ostreams = (ZippedStoreShaWriter, None) + for ostreamcls in ostreams: + for data in self.all_data: + dry_run = ostreamcls is not None + ostream = None + if ostreamcls is not None: + ostream = ostreamcls() + assert isinstance(ostream, Sha1Writer) + # END create ostream + + prev_ostream = db.set_ostream(ostream) + assert type(prev_ostream) in ostreams or prev_ostream in ostreams + istream = IStream(str_blob_type, len(data), BytesIO(data)) + + # store returns same istream instance, with new sha set + my_istream = db.store(istream) + sha = istream.binsha + assert my_istream is istream + assert db.has_object(sha) != dry_run + assert len(sha) == 20 + + # verify data - the slow way, we want to run code + if not dry_run: + info = db.info(sha) + assert str_blob_type == info.type + assert info.size == len(data) + + ostream = db.stream(sha) + assert ostream.read() == data + assert ostream.type == str_blob_type + assert ostream.size == len(data) + else: + self.assertRaises(BadObject, db.info, sha) + self.assertRaises(BadObject, db.stream, sha) + + # DIRECT STREAM COPY + # our data hase been written in object format to the StringIO + # we passed as output stream. No physical database representation + # was created. + # Test direct stream copy of object streams, the result must be + # identical to what we fed in + ostream.seek(0) + istream.stream = ostream + assert istream.binsha is not None + prev_sha = istream.binsha + + db.set_ostream(ZippedStoreShaWriter()) + db.store(istream) + assert istream.binsha == prev_sha + new_ostream = db.ostream() + + # note: only works as long our store write uses the same compression + # level, which is zip_best + assert ostream.getvalue() == new_ostream.getvalue() + # END for each data set + # END for each dry_run mode diff --git a/gitdb/gitdb/test/db/test_git.py b/gitdb/gitdb/test/db/test_git.py new file mode 100644 index 000000000..73ac1a08c --- /dev/null +++ b/gitdb/gitdb/test/db/test_git.py @@ -0,0 +1,53 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: https://opensource.org/license/bsd-3-clause/ +import os +from gitdb.test.db.lib import ( + TestDBBase, + with_rw_directory +) +from gitdb.exc import BadObject +from gitdb.db import GitDB +from gitdb.base import OStream, OInfo +from gitdb.util import bin_to_hex + + +class TestGitDB(TestDBBase): + + def test_reading(self): + gdb = GitDB(os.path.join(self.gitrepopath, 'objects')) + + # we have packs and loose objects, alternates doesn't necessarily exist + assert 1 < len(gdb.databases()) < 4 + + # access should be possible + gitdb_sha = next(gdb.sha_iter()) + assert isinstance(gdb.info(gitdb_sha), OInfo) + assert isinstance(gdb.stream(gitdb_sha), OStream) + ni = 50 + assert gdb.size() >= ni + sha_list = list(gdb.sha_iter()) + assert len(sha_list) == gdb.size() + sha_list = sha_list[:ni] # speed up tests ... + + # This is actually a test for compound functionality, but it doesn't + # have a separate test module + # test partial shas + # this one as uneven and quite short + gitdb_sha_hex = bin_to_hex(gitdb_sha) + assert gdb.partial_to_complete_sha_hex(gitdb_sha_hex[:5]) == gitdb_sha + + # mix even/uneven hexshas + for i, binsha in enumerate(sha_list): + assert gdb.partial_to_complete_sha_hex(bin_to_hex(binsha)[:8 - (i % 2)]) == binsha + # END for each sha + + self.assertRaises(BadObject, gdb.partial_to_complete_sha_hex, "0000") + + @with_rw_directory + def test_writing(self, path): + gdb = GitDB(path) + + # its possible to write objects + self._assert_object_writing(gdb) diff --git a/gitdb/gitdb/test/db/test_loose.py b/gitdb/gitdb/test/db/test_loose.py new file mode 100644 index 000000000..295e2eecd --- /dev/null +++ b/gitdb/gitdb/test/db/test_loose.py @@ -0,0 +1,36 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: https://opensource.org/license/bsd-3-clause/ +from gitdb.test.db.lib import ( + TestDBBase, + with_rw_directory +) +from gitdb.db import LooseObjectDB +from gitdb.exc import BadObject +from gitdb.util import bin_to_hex + + +class TestLooseDB(TestDBBase): + + @with_rw_directory + def test_basics(self, path): + ldb = LooseObjectDB(path) + + # write data + self._assert_object_writing(ldb) + + # verify sha iteration and size + shas = list(ldb.sha_iter()) + assert shas and len(shas[0]) == 20 + + assert len(shas) == ldb.size() + + # verify find short object + long_sha = bin_to_hex(shas[-1]) + for short_sha in (long_sha[:20], long_sha[:5]): + assert bin_to_hex(ldb.partial_to_complete_sha_hex(short_sha)) == long_sha + # END for each sha + + self.assertRaises(BadObject, ldb.partial_to_complete_sha_hex, '0000') + # raises if no object could be found diff --git a/gitdb/gitdb/test/db/test_mem.py b/gitdb/gitdb/test/db/test_mem.py new file mode 100644 index 000000000..882e54fe5 --- /dev/null +++ b/gitdb/gitdb/test/db/test_mem.py @@ -0,0 +1,34 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: https://opensource.org/license/bsd-3-clause/ +from gitdb.test.db.lib import ( + TestDBBase, + with_rw_directory +) +from gitdb.db import ( + MemoryDB, + LooseObjectDB +) + + +class TestMemoryDB(TestDBBase): + + @with_rw_directory + def test_writing(self, path): + mdb = MemoryDB() + + # write data + self._assert_object_writing_simple(mdb) + + # test stream copy + ldb = LooseObjectDB(path) + assert ldb.size() == 0 + num_streams_copied = mdb.stream_copy(mdb.sha_iter(), ldb) + assert num_streams_copied == mdb.size() + + assert ldb.size() == mdb.size() + for sha in mdb.sha_iter(): + assert ldb.has_object(sha) + assert ldb.stream(sha).read() == mdb.stream(sha).read() + # END verify objects where copied and are equal diff --git a/gitdb/gitdb/test/db/test_pack.py b/gitdb/gitdb/test/db/test_pack.py new file mode 100644 index 000000000..bd07906da --- /dev/null +++ b/gitdb/gitdb/test/db/test_pack.py @@ -0,0 +1,87 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: https://opensource.org/license/bsd-3-clause/ +from gitdb.test.db.lib import ( + TestDBBase, + with_rw_directory, + with_packs_rw +) +from gitdb.db import PackedDB + +from gitdb.exc import BadObject, AmbiguousObjectName +from gitdb.util import mman + +import os +import random +import sys + +import pytest + +class TestPackDB(TestDBBase): + + @with_rw_directory + @with_packs_rw + def test_writing(self, path): + if sys.platform == "win32": + pytest.skip("FIXME: Currently fail on windows") + + pdb = PackedDB(path) + + # on demand, we init our pack cache + num_packs = len(pdb.entities()) + assert pdb._st_mtime != 0 + + # test pack directory changed: + # packs removed - rename a file, should affect the glob + pack_path = pdb.entities()[0].pack().path() + new_pack_path = pack_path + "renamed" + if sys.platform == "win32": + # While using this function, we are not allowed to have any handle + # to this path, which is currently not the case. The pack caching + # does still have a handle :-( + mman.force_map_handle_removal_win(pack_path) + os.rename(pack_path, new_pack_path) + + pdb.update_cache(force=True) + assert len(pdb.entities()) == num_packs - 1 + + # packs added + os.rename(new_pack_path, pack_path) + pdb.update_cache(force=True) + assert len(pdb.entities()) == num_packs + + # bang on the cache + # access the Entities directly, as there is no iteration interface + # yet ( or required for now ) + sha_list = list(pdb.sha_iter()) + assert len(sha_list) == pdb.size() + + # hit all packs in random order + random.shuffle(sha_list) + + for sha in sha_list: + pdb.info(sha) + pdb.stream(sha) + # END for each sha to query + + # test short finding - be a bit more brutal here + max_bytes = 19 + min_bytes = 2 + num_ambiguous = 0 + for i, sha in enumerate(sha_list): + short_sha = sha[:max((i % max_bytes), min_bytes)] + try: + assert pdb.partial_to_complete_sha(short_sha, len(short_sha) * 2) == sha + except AmbiguousObjectName: + num_ambiguous += 1 + pass # valid, we can have short objects + # END exception handling + # END for each sha to find + + # we should have at least one ambiguous, considering the small sizes + # but in our pack, there is no ambiguous ... + # assert num_ambiguous + + # non-existing + self.assertRaises(BadObject, pdb.partial_to_complete_sha, b'\0\0', 4) diff --git a/gitdb/gitdb/test/db/test_ref.py b/gitdb/gitdb/test/db/test_ref.py new file mode 100644 index 000000000..0816e649f --- /dev/null +++ b/gitdb/gitdb/test/db/test_ref.py @@ -0,0 +1,58 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: https://opensource.org/license/bsd-3-clause/ +from gitdb.test.db.lib import ( + TestDBBase, + with_rw_directory, +) +from gitdb.db import ReferenceDB + +from gitdb.util import ( + NULL_BIN_SHA, + hex_to_bin +) + +import os + + +class TestReferenceDB(TestDBBase): + + def make_alt_file(self, alt_path, alt_list): + """Create an alternates file which contains the given alternates. + The list can be empty""" + with open(alt_path, "wb") as alt_file: + for alt in alt_list: + alt_file.write(alt.encode("utf-8") + b"\n") + + @with_rw_directory + def test_writing(self, path): + alt_path = os.path.join(path, 'alternates') + rdb = ReferenceDB(alt_path) + assert len(rdb.databases()) == 0 + assert rdb.size() == 0 + assert len(list(rdb.sha_iter())) == 0 + + # try empty, non-existing + assert not rdb.has_object(NULL_BIN_SHA) + + # setup alternate file + # add two, one is invalid + own_repo_path = os.path.join(self.gitrepopath, 'objects') # use own repo + self.make_alt_file(alt_path, [own_repo_path, "invalid/path"]) + rdb.update_cache() + assert len(rdb.databases()) == 1 + + # we should now find a default revision of ours + gitdb_sha = next(rdb.sha_iter()) + assert rdb.has_object(gitdb_sha) + + # remove valid + self.make_alt_file(alt_path, ["just/one/invalid/path"]) + rdb.update_cache() + assert len(rdb.databases()) == 0 + + # add valid + self.make_alt_file(alt_path, [own_repo_path]) + rdb.update_cache() + assert len(rdb.databases()) == 1 diff --git a/gitdb/gitdb/test/fixtures/objects/7b/b839852ed5e3a069966281bb08d50012fb309b b/gitdb/gitdb/test/fixtures/objects/7b/b839852ed5e3a069966281bb08d50012fb309b new file mode 100644 index 000000000..021c2db34 Binary files /dev/null and b/gitdb/gitdb/test/fixtures/objects/7b/b839852ed5e3a069966281bb08d50012fb309b differ diff --git a/gitdb/gitdb/test/fixtures/objects/88/8401851f15db0eed60eb1bc29dec5ddcace911 b/gitdb/gitdb/test/fixtures/objects/88/8401851f15db0eed60eb1bc29dec5ddcace911 new file mode 100644 index 000000000..d60aeeff7 Binary files /dev/null and b/gitdb/gitdb/test/fixtures/objects/88/8401851f15db0eed60eb1bc29dec5ddcace911 differ diff --git a/gitdb/gitdb/test/fixtures/packs/pack-11fdfa9e156ab73caae3b6da867192221f2089c2.idx b/gitdb/gitdb/test/fixtures/packs/pack-11fdfa9e156ab73caae3b6da867192221f2089c2.idx new file mode 100644 index 000000000..fda5969bc Binary files /dev/null and b/gitdb/gitdb/test/fixtures/packs/pack-11fdfa9e156ab73caae3b6da867192221f2089c2.idx differ diff --git a/gitdb/gitdb/test/fixtures/packs/pack-11fdfa9e156ab73caae3b6da867192221f2089c2.pack b/gitdb/gitdb/test/fixtures/packs/pack-11fdfa9e156ab73caae3b6da867192221f2089c2.pack new file mode 100644 index 000000000..a3209d2be Binary files /dev/null and b/gitdb/gitdb/test/fixtures/packs/pack-11fdfa9e156ab73caae3b6da867192221f2089c2.pack differ diff --git a/gitdb/gitdb/test/fixtures/packs/pack-a2bf8e71d8c18879e499335762dd95119d93d9f1.idx b/gitdb/gitdb/test/fixtures/packs/pack-a2bf8e71d8c18879e499335762dd95119d93d9f1.idx new file mode 100644 index 000000000..a7d6c7177 Binary files /dev/null and b/gitdb/gitdb/test/fixtures/packs/pack-a2bf8e71d8c18879e499335762dd95119d93d9f1.idx differ diff --git a/gitdb/gitdb/test/fixtures/packs/pack-a2bf8e71d8c18879e499335762dd95119d93d9f1.pack b/gitdb/gitdb/test/fixtures/packs/pack-a2bf8e71d8c18879e499335762dd95119d93d9f1.pack new file mode 100644 index 000000000..955c424c7 Binary files /dev/null and b/gitdb/gitdb/test/fixtures/packs/pack-a2bf8e71d8c18879e499335762dd95119d93d9f1.pack differ diff --git a/gitdb/gitdb/test/fixtures/packs/pack-c0438c19fb16422b6bbcce24387b3264416d485b.idx b/gitdb/gitdb/test/fixtures/packs/pack-c0438c19fb16422b6bbcce24387b3264416d485b.idx new file mode 100644 index 000000000..87c635f48 Binary files /dev/null and b/gitdb/gitdb/test/fixtures/packs/pack-c0438c19fb16422b6bbcce24387b3264416d485b.idx differ diff --git a/gitdb/gitdb/test/fixtures/packs/pack-c0438c19fb16422b6bbcce24387b3264416d485b.pack b/gitdb/gitdb/test/fixtures/packs/pack-c0438c19fb16422b6bbcce24387b3264416d485b.pack new file mode 100644 index 000000000..a69b28ac6 Binary files /dev/null and b/gitdb/gitdb/test/fixtures/packs/pack-c0438c19fb16422b6bbcce24387b3264416d485b.pack differ diff --git a/gitdb/gitdb/test/lib.py b/gitdb/gitdb/test/lib.py new file mode 100644 index 000000000..8e602342c --- /dev/null +++ b/gitdb/gitdb/test/lib.py @@ -0,0 +1,192 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: https://opensource.org/license/bsd-3-clause/ +"""Utilities used in ODB testing""" +from gitdb import OStream + +import sys +import random +from array import array + +from io import BytesIO + +import glob +import unittest +import tempfile +import shutil +import os +import gc +import logging +from functools import wraps + + +#{ Bases + +class TestBase(unittest.TestCase): + """Base class for all tests + + TestCase providing access to readonly repositories using the following member variables. + + * gitrepopath + + * read-only base path of the git source repository, i.e. .../git/.git + """ + + #{ Invvariants + k_env_git_repo = "GITDB_TEST_GIT_REPO_BASE" + #} END invariants + + @classmethod + def setUpClass(cls): + try: + super().setUpClass() + except AttributeError: + pass + + cls.gitrepopath = os.environ.get(cls.k_env_git_repo) + if not cls.gitrepopath: + logging.info( + "You can set the %s environment variable to a .git repository of your choice - defaulting to the gitdb repository", cls.k_env_git_repo) + ospd = os.path.dirname + cls.gitrepopath = os.path.join(ospd(ospd(ospd(__file__))), '.git') + # end assure gitrepo is set + assert cls.gitrepopath.endswith('.git') + + +#} END bases + +#{ Decorators + +def with_rw_directory(func): + """Create a temporary directory which can be written to, remove it if the + test succeeds, but leave it otherwise to aid additional debugging""" + + def wrapper(self): + path = tempfile.mktemp(prefix=func.__name__) + os.mkdir(path) + keep = False + try: + try: + return func(self, path) + except Exception: + sys.stderr.write(f"Test {type(self).__name__}.{func.__name__} failed, output is at {path!r}\n") + keep = True + raise + finally: + # Need to collect here to be sure all handles have been closed. It appears + # a windows-only issue. In fact things should be deleted, as well as + # memory maps closed, once objects go out of scope. For some reason + # though this is not the case here unless we collect explicitly. + if not keep: + gc.collect() + shutil.rmtree(path) + # END handle exception + # END wrapper + + wrapper.__name__ = func.__name__ + return wrapper + + +def with_packs_rw(func): + """Function that provides a path into which the packs for testing should be + copied. Will pass on the path to the actual function afterwards""" + + def wrapper(self, path): + src_pack_glob = fixture_path('packs/*') + copy_files_globbed(src_pack_glob, path, hard_link_ok=True) + return func(self, path) + # END wrapper + + wrapper.__name__ = func.__name__ + return wrapper + +#} END decorators + +#{ Routines + + +def fixture_path(relapath=''): + """:return: absolute path into the fixture directory + :param relapath: relative path into the fixtures directory, or '' + to obtain the fixture directory itself""" + return os.path.join(os.path.dirname(__file__), 'fixtures', relapath) + + +def copy_files_globbed(source_glob, target_dir, hard_link_ok=False): + """Copy all files found according to the given source glob into the target directory + :param hard_link_ok: if True, hard links will be created if possible. Otherwise + the files will be copied""" + for src_file in glob.glob(source_glob): + if hard_link_ok and hasattr(os, 'link'): + target = os.path.join(target_dir, os.path.basename(src_file)) + try: + os.link(src_file, target) + except OSError: + shutil.copy(src_file, target_dir) + # END handle cross device links ( and resulting failure ) + else: + shutil.copy(src_file, target_dir) + # END try hard link + # END for each file to copy + + +def make_bytes(size_in_bytes, randomize=False): + """:return: string with given size in bytes + :param randomize: try to produce a very random stream""" + actual_size = size_in_bytes // 4 + producer = range(actual_size) + if randomize: + producer = list(producer) + random.shuffle(producer) + # END randomize + a = array('i', producer) + return a.tobytes() + + +def make_object(type, data): + """:return: bytes resembling an uncompressed object""" + odata = "blob %i\0" % len(data) + return odata.encode("ascii") + data + + +def make_memory_file(size_in_bytes, randomize=False): + """:return: tuple(size_of_stream, stream) + :param randomize: try to produce a very random stream""" + d = make_bytes(size_in_bytes, randomize) + return len(d), BytesIO(d) + +#} END routines + +#{ Stream Utilities + + +class DummyStream: + + def __init__(self): + self.was_read = False + self.bytes = 0 + self.closed = False + + def read(self, size): + self.was_read = True + self.bytes = size + + def close(self): + self.closed = True + + def _assert(self): + assert self.was_read + + +class DeriveTest(OStream): + + def __init__(self, sha, type, size, stream, *args, **kwargs): + self.myarg = kwargs.pop('myarg') + self.args = args + + def _assert(self): + assert self.args + assert self.myarg + +#} END stream utilitiess diff --git a/gitdb/gitdb/test/performance/__init__.py b/gitdb/gitdb/test/performance/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/gitdb/gitdb/test/performance/lib.py b/gitdb/gitdb/test/performance/lib.py new file mode 100644 index 000000000..36916ede3 --- /dev/null +++ b/gitdb/gitdb/test/performance/lib.py @@ -0,0 +1,17 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: https://opensource.org/license/bsd-3-clause/ +"""Contains library functions""" +from gitdb.test.lib import TestBase + + + +#{ Base Classes + +class TestBigRepoR(TestBase): + """A placeholder in case we want to add additional functionality to all performance test-cases + """ + + +#} END base classes diff --git a/gitdb/gitdb/test/performance/test_pack.py b/gitdb/gitdb/test/performance/test_pack.py new file mode 100644 index 000000000..fc3d3349f --- /dev/null +++ b/gitdb/gitdb/test/performance/test_pack.py @@ -0,0 +1,127 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: https://opensource.org/license/bsd-3-clause/ +"""Performance tests for object store""" + +from gitdb.test.performance.lib import ( + TestBigRepoR +) + +from gitdb import ( + MemoryDB, + GitDB, + IStream, +) +from gitdb.typ import str_blob_type +from gitdb.exc import UnsupportedOperation +from gitdb.db.pack import PackedDB + +import sys +import os +from time import time + + +class TestPackedDBPerformance(TestBigRepoR): + + def test_pack_random_access(self): + pdb = PackedDB(os.path.join(self.gitrepopath, "objects/pack")) + + # sha lookup + st = time() + sha_list = list(pdb.sha_iter()) + elapsed = time() - st + ns = len(sha_list) + print("PDB: looked up %i shas by index in %f s ( %f shas/s )" % (ns, elapsed, ns / (elapsed or 1)), file=sys.stderr) + + # sha lookup: best-case and worst case access + pdb_pack_info = pdb._pack_info + # END shuffle shas + st = time() + for sha in sha_list: + pdb_pack_info(sha) + # END for each sha to look up + elapsed = time() - st + + # discard cache + del(pdb._entities) + pdb.entities() + print("PDB: looked up %i sha in %i packs in %f s ( %f shas/s )" % + (ns, len(pdb.entities()), elapsed, ns / (elapsed or 1)), file=sys.stderr) + # END for each random mode + + # query info and streams only + max_items = 10000 # can wait longer when testing memory + for pdb_fun in (pdb.info, pdb.stream): + st = time() + for sha in sha_list[:max_items]: + pdb_fun(sha) + elapsed = time() - st + print("PDB: Obtained %i object %s by sha in %f s ( %f items/s )" % + (max_items, pdb_fun.__name__.upper(), elapsed, max_items / (elapsed or 1)), file=sys.stderr) + # END for each function + + # retrieve stream and read all + max_items = 5000 + pdb_stream = pdb.stream + total_size = 0 + st = time() + for sha in sha_list[:max_items]: + stream = pdb_stream(sha) + read_len = len(stream.read()) + assert read_len == stream.size + total_size += stream.size + elapsed = time() - st + total_kib = total_size / 1000 + print("PDB: Obtained %i streams by sha and read all bytes totallying %i KiB ( %f KiB / s ) in %f s ( %f streams/s )" % + (max_items, total_kib, total_kib / (elapsed or 1), elapsed, max_items / (elapsed or 1)), file=sys.stderr) + + def test_loose_correctness(self): + """based on the pack(s) of our packed object DB, we will just copy and verify all objects in the back + into the loose object db (memory). + This should help finding dormant issues like this one https://github.com/gitpython-developers/GitPython/issues/220 + faster + :note: It doesn't seem this test can find the issue unless the given pack contains highly compressed + data files, like archives.""" + from gitdb.util import bin_to_hex + pdb = GitDB(os.path.join(self.gitrepopath, 'objects')) + mdb = MemoryDB() + for c, sha in enumerate(pdb.sha_iter()): + ostream = pdb.stream(sha) + # the issue only showed on larger files which are hardly compressible ... + if ostream.type != str_blob_type: + continue + istream = IStream(ostream.type, ostream.size, ostream.stream) + mdb.store(istream) + assert istream.binsha == sha, "Failed on object %s" % bin_to_hex(sha).decode('ascii') + # this can fail ... sometimes, so the packs dataset should be huge + assert len(mdb.stream(sha).read()) == ostream.size + + if c and c % 1000 == 0: + print("Verified %i loose object compression/decompression cycles" % c, file=sys.stderr) + mdb._cache.clear() + # end for each sha to copy + + def test_correctness(self): + pdb = PackedDB(os.path.join(self.gitrepopath, "objects/pack")) + # disabled for now as it used to work perfectly, checking big repositories takes a long time + print("Endurance run: verify streaming of objects (crc and sha)", file=sys.stderr) + for crc in range(2): + count = 0 + st = time() + for entity in pdb.entities(): + pack_verify = entity.is_valid_stream + sha_by_index = entity.index().sha + for index in range(entity.index().size()): + try: + assert pack_verify(sha_by_index(index), use_crc=crc) + count += 1 + except UnsupportedOperation: + pass + # END ignore old indices + # END for each index + # END for each entity + elapsed = time() - st + print("PDB: verified %i objects (crc=%i) in %f s ( %f objects/s )" % + (count, crc, elapsed, count / (elapsed or 1)), file=sys.stderr) + # END for each verify mode diff --git a/gitdb/gitdb/test/performance/test_pack_streaming.py b/gitdb/gitdb/test/performance/test_pack_streaming.py new file mode 100644 index 000000000..80c798bbc --- /dev/null +++ b/gitdb/gitdb/test/performance/test_pack_streaming.py @@ -0,0 +1,81 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: https://opensource.org/license/bsd-3-clause/ +"""Specific test for pack streams only""" + +from gitdb.test.performance.lib import ( + TestBigRepoR +) + +from gitdb.db.pack import PackedDB +from gitdb.stream import NullStream +from gitdb.pack import PackEntity + +import os +import sys +from time import time + + +class CountedNullStream(NullStream): + __slots__ = '_bw' + + def __init__(self): + self._bw = 0 + + def bytes_written(self): + return self._bw + + def write(self, d): + self._bw += NullStream.write(self, d) + + +class TestPackStreamingPerformance(TestBigRepoR): + + def test_pack_writing(self): + # see how fast we can write a pack from object streams. + # This will not be fast, as we take time for decompressing the streams as well + ostream = CountedNullStream() + pdb = PackedDB(os.path.join(self.gitrepopath, "objects/pack")) + + ni = 1000 + count = 0 + st = time() + for sha in pdb.sha_iter(): + count += 1 + pdb.stream(sha) + if count == ni: + break + # END gather objects for pack-writing + elapsed = time() - st + print("PDB Streaming: Got %i streams by sha in in %f s ( %f streams/s )" % + (ni, elapsed, ni / (elapsed or 1)), file=sys.stderr) + + st = time() + PackEntity.write_pack((pdb.stream(sha) for sha in pdb.sha_iter()), ostream.write, object_count=ni) + elapsed = time() - st + total_kb = ostream.bytes_written() / 1000 + print(sys.stderr, "PDB Streaming: Wrote pack of size %i kb in %f s (%f kb/s)" % + (total_kb, elapsed, total_kb / (elapsed or 1)), sys.stderr) + + def test_stream_reading(self): + # raise SkipTest() + pdb = PackedDB(os.path.join(self.gitrepopath, "objects/pack")) + + # streaming only, meant for --with-profile runs + ni = 5000 + count = 0 + pdb_stream = pdb.stream + total_size = 0 + st = time() + for sha in pdb.sha_iter(): + if count == ni: + break + stream = pdb_stream(sha) + stream.read() + total_size += stream.size + count += 1 + elapsed = time() - st + total_kib = total_size / 1000 + print(sys.stderr, "PDB Streaming: Got %i streams by sha and read all bytes totallying %i KiB ( %f KiB / s ) in %f s ( %f streams/s )" % + (ni, total_kib, total_kib / (elapsed or 1), elapsed, ni / (elapsed or 1)), sys.stderr) diff --git a/gitdb/gitdb/test/performance/test_stream.py b/gitdb/gitdb/test/performance/test_stream.py new file mode 100644 index 000000000..fb10871ae --- /dev/null +++ b/gitdb/gitdb/test/performance/test_stream.py @@ -0,0 +1,106 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: https://opensource.org/license/bsd-3-clause/ +"""Performance data streaming performance""" + +from gitdb.test.performance.lib import TestBigRepoR +from gitdb.db import LooseObjectDB +from gitdb import IStream + +from gitdb.util import bin_to_hex, remove +from gitdb.fun import chunk_size + +from time import time +import os +import sys + + +from gitdb.test.lib import ( + make_memory_file, + with_rw_directory, +) + + +#{ Utilities +def read_chunked_stream(stream): + total = 0 + while True: + chunk = stream.read(chunk_size) + total += len(chunk) + if len(chunk) < chunk_size: + break + # END read stream loop + assert total == stream.size + return stream + + +#} END utilities + +class TestObjDBPerformance(TestBigRepoR): + + large_data_size_bytes = 1000 * 1000 * 50 # some MiB should do it + moderate_data_size_bytes = 1000 * 1000 * 1 # just 1 MiB + + @with_rw_directory + def test_large_data_streaming(self, path): + ldb = LooseObjectDB(path) + string_ios = list() # list of streams we previously created + + # serial mode + for randomize in range(2): + desc = (randomize and 'random ') or '' + print("Creating %s data ..." % desc, file=sys.stderr) + st = time() + size, stream = make_memory_file(self.large_data_size_bytes, randomize) + elapsed = time() - st + print("Done (in %f s)" % elapsed, file=sys.stderr) + string_ios.append(stream) + + # writing - due to the compression it will seem faster than it is + st = time() + sha = ldb.store(IStream('blob', size, stream)).binsha + elapsed_add = time() - st + assert ldb.has_object(sha) + db_file = ldb.readable_db_object_path(bin_to_hex(sha)) + fsize_kib = os.path.getsize(db_file) / 1000 + + size_kib = size / 1000 + print("Added %i KiB (filesize = %i KiB) of %s data to loose odb in %f s ( %f Write KiB / s)" % + (size_kib, fsize_kib, desc, elapsed_add, size_kib / (elapsed_add or 1)), file=sys.stderr) + + # reading all at once + st = time() + ostream = ldb.stream(sha) + shadata = ostream.read() + elapsed_readall = time() - st + + stream.seek(0) + assert shadata == stream.getvalue() + print("Read %i KiB of %s data at once from loose odb in %f s ( %f Read KiB / s)" % + (size_kib, desc, elapsed_readall, size_kib / (elapsed_readall or 1)), file=sys.stderr) + + # reading in chunks of 1 MiB + cs = 512 * 1000 + chunks = list() + st = time() + ostream = ldb.stream(sha) + while True: + data = ostream.read(cs) + chunks.append(data) + if len(data) < cs: + break + # END read in chunks + elapsed_readchunks = time() - st + + stream.seek(0) + assert b''.join(chunks) == stream.getvalue() + + cs_kib = cs / 1000 + print("Read %i KiB of %s data in %i KiB chunks from loose odb in %f s ( %f Read KiB / s)" % + (size_kib, desc, cs_kib, elapsed_readchunks, size_kib / (elapsed_readchunks or 1)), file=sys.stderr) + + # del db file so we keep something to do + ostream = None # To release the file handle (win) + remove(db_file) + # END for each randomization factor diff --git a/gitdb/gitdb/test/test_base.py b/gitdb/gitdb/test/test_base.py new file mode 100644 index 000000000..17906c9c2 --- /dev/null +++ b/gitdb/gitdb/test/test_base.py @@ -0,0 +1,105 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: https://opensource.org/license/bsd-3-clause/ +"""Test for object db""" +from gitdb.test.lib import ( + TestBase, + DummyStream, + DeriveTest, +) + +from gitdb import ( + OInfo, + OPackInfo, + ODeltaPackInfo, + OStream, + OPackStream, + ODeltaPackStream, + IStream +) +from gitdb.util import ( + NULL_BIN_SHA +) + +from gitdb.typ import ( + str_blob_type +) + + +class TestBaseTypes(TestBase): + + def test_streams(self): + # test info + sha = NULL_BIN_SHA + s = 20 + blob_id = 3 + + info = OInfo(sha, str_blob_type, s) + assert info.binsha == sha + assert info.type == str_blob_type + assert info.type_id == blob_id + assert info.size == s + + # test pack info + # provides type_id + pinfo = OPackInfo(0, blob_id, s) + assert pinfo.type == str_blob_type + assert pinfo.type_id == blob_id + assert pinfo.pack_offset == 0 + + dpinfo = ODeltaPackInfo(0, blob_id, s, sha) + assert dpinfo.type == str_blob_type + assert dpinfo.type_id == blob_id + assert dpinfo.delta_info == sha + assert dpinfo.pack_offset == 0 + + # test ostream + stream = DummyStream() + ostream = OStream(*(info + (stream, ))) + assert ostream.stream is stream + ostream.read(15) + stream._assert() + assert stream.bytes == 15 + ostream.read(20) + assert stream.bytes == 20 + + # test packstream + postream = OPackStream(*(pinfo + (stream, ))) + assert postream.stream is stream + postream.read(10) + stream._assert() + assert stream.bytes == 10 + + # test deltapackstream + dpostream = ODeltaPackStream(*(dpinfo + (stream, ))) + assert dpostream.stream is stream + dpostream.read(5) + stream._assert() + assert stream.bytes == 5 + + # derive with own args + DeriveTest(sha, str_blob_type, s, stream, 'mine', myarg=3)._assert() + + # test istream + istream = IStream(str_blob_type, s, stream) + assert istream.binsha == None + istream.binsha = sha + assert istream.binsha == sha + + assert len(istream.binsha) == 20 + assert len(istream.hexsha) == 40 + + assert istream.size == s + istream.size = s * 2 + assert istream.size == s * 2 + assert istream.type == str_blob_type + istream.type = "something" + assert istream.type == "something" + assert istream.stream is stream + istream.stream = None + assert istream.stream is None + + assert istream.error is None + istream.error = Exception() + assert isinstance(istream.error, Exception) diff --git a/gitdb/gitdb/test/test_example.py b/gitdb/gitdb/test/test_example.py new file mode 100644 index 000000000..3b4c9084b --- /dev/null +++ b/gitdb/gitdb/test/test_example.py @@ -0,0 +1,43 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: https://opensource.org/license/bsd-3-clause/ +"""Module with examples from the tutorial section of the docs""" +import os +from gitdb.test.lib import TestBase +from gitdb import IStream +from gitdb.db import LooseObjectDB + +from io import BytesIO + + +class TestExamples(TestBase): + + def test_base(self): + ldb = LooseObjectDB(os.path.join(self.gitrepopath, 'objects')) + + for sha1 in ldb.sha_iter(): + oinfo = ldb.info(sha1) + ostream = ldb.stream(sha1) + assert oinfo[:3] == ostream[:3] + + assert len(ostream.read()) == ostream.size + assert ldb.has_object(oinfo.binsha) + # END for each sha in database + # assure we close all files + try: + del(ostream) + del(oinfo) + except UnboundLocalError: + pass + # END ignore exception if there are no loose objects + + data = b"my data" + istream = IStream("blob", len(data), BytesIO(data)) + + # the object does not yet have a sha + assert istream.binsha is None + ldb.store(istream) + # now the sha is set + assert len(istream.binsha) == 20 + assert ldb.has_object(istream.binsha) diff --git a/gitdb/gitdb/test/test_pack.py b/gitdb/gitdb/test/test_pack.py new file mode 100644 index 000000000..e72348228 --- /dev/null +++ b/gitdb/gitdb/test/test_pack.py @@ -0,0 +1,249 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: https://opensource.org/license/bsd-3-clause/ +"""Test everything about packs reading and writing""" +from gitdb.test.lib import ( + TestBase, + with_rw_directory, + fixture_path +) + +from gitdb.stream import DeltaApplyReader + +from gitdb.pack import ( + PackEntity, + PackIndexFile, + PackFile +) + +from gitdb.base import ( + OInfo, + OStream, +) + +from gitdb.fun import delta_types +from gitdb.exc import UnsupportedOperation +from gitdb.util import to_bin_sha + +import pytest + +import os +import tempfile + + +#{ Utilities +def bin_sha_from_filename(filename): + return to_bin_sha(os.path.splitext(os.path.basename(filename))[0][5:]) +#} END utilities + + +class TestPack(TestBase): + + packindexfile_v1 = (fixture_path('packs/pack-c0438c19fb16422b6bbcce24387b3264416d485b.idx'), 1, 67) + packindexfile_v2 = (fixture_path('packs/pack-11fdfa9e156ab73caae3b6da867192221f2089c2.idx'), 2, 30) + packindexfile_v2_3_ascii = (fixture_path('packs/pack-a2bf8e71d8c18879e499335762dd95119d93d9f1.idx'), 2, 42) + packfile_v2_1 = (fixture_path('packs/pack-c0438c19fb16422b6bbcce24387b3264416d485b.pack'), 2, packindexfile_v1[2]) + packfile_v2_2 = (fixture_path('packs/pack-11fdfa9e156ab73caae3b6da867192221f2089c2.pack'), 2, packindexfile_v2[2]) + packfile_v2_3_ascii = ( + fixture_path('packs/pack-a2bf8e71d8c18879e499335762dd95119d93d9f1.pack'), 2, packindexfile_v2_3_ascii[2]) + + def _assert_index_file(self, index, version, size): + assert index.packfile_checksum() != index.indexfile_checksum() + assert len(index.packfile_checksum()) == 20 + assert len(index.indexfile_checksum()) == 20 + assert index.version() == version + assert index.size() == size + assert len(index.offsets()) == size + + # get all data of all objects + for oidx in range(index.size()): + sha = index.sha(oidx) + assert oidx == index.sha_to_index(sha) + + entry = index.entry(oidx) + assert len(entry) == 3 + + assert entry[0] == index.offset(oidx) + assert entry[1] == sha + assert entry[2] == index.crc(oidx) + + # verify partial sha + for l in (4, 8, 11, 17, 20): + assert index.partial_sha_to_index(sha[:l], l * 2) == oidx + + # END for each object index in indexfile + self.assertRaises(ValueError, index.partial_sha_to_index, "\0", 2) + + def _assert_pack_file(self, pack, version, size): + assert pack.version() == 2 + assert pack.size() == size + assert len(pack.checksum()) == 20 + + num_obj = 0 + for obj in pack.stream_iter(): + num_obj += 1 + info = pack.info(obj.pack_offset) + stream = pack.stream(obj.pack_offset) + + assert info.pack_offset == stream.pack_offset + assert info.type_id == stream.type_id + assert hasattr(stream, 'read') + + # it should be possible to read from both streams + assert obj.read() == stream.read() + + streams = pack.collect_streams(obj.pack_offset) + assert streams + + # read the stream + try: + dstream = DeltaApplyReader.new(streams) + except ValueError: + # ignore these, old git versions use only ref deltas, + # which we haven't resolved ( as we are without an index ) + # Also ignore non-delta streams + continue + # END get deltastream + + # read all + data = dstream.read() + assert len(data) == dstream.size + + # test seek + dstream.seek(0) + assert dstream.read() == data + + # read chunks + # NOTE: the current implementation is safe, it basically transfers + # all calls to the underlying memory map + + # END for each object + assert num_obj == size + + def test_pack_index(self): + # check version 1 and 2 + for indexfile, version, size in (self.packindexfile_v1, self.packindexfile_v2): + index = PackIndexFile(indexfile) + self._assert_index_file(index, version, size) + # END run tests + + def test_pack(self): + # there is this special version 3, but apparently its like 2 ... + for packfile, version, size in (self.packfile_v2_3_ascii, self.packfile_v2_1, self.packfile_v2_2): + pack = PackFile(packfile) + self._assert_pack_file(pack, version, size) + # END for each pack to test + + @with_rw_directory + def test_pack_entity(self, rw_dir): + pack_objs = list() + for packinfo, indexinfo in ((self.packfile_v2_1, self.packindexfile_v1), + (self.packfile_v2_2, self.packindexfile_v2), + (self.packfile_v2_3_ascii, self.packindexfile_v2_3_ascii)): + packfile, version, size = packinfo + indexfile, version, size = indexinfo + entity = PackEntity(packfile) + assert entity.pack().path() == packfile + assert entity.index().path() == indexfile + pack_objs.extend(entity.stream_iter()) + + count = 0 + for info, stream in zip(entity.info_iter(), entity.stream_iter()): + count += 1 + assert info.binsha == stream.binsha + assert len(info.binsha) == 20 + assert info.type_id == stream.type_id + assert info.size == stream.size + + # we return fully resolved items, which is implied by the sha centric access + assert not info.type_id in delta_types + + # try all calls + assert len(entity.collect_streams(info.binsha)) + oinfo = entity.info(info.binsha) + assert isinstance(oinfo, OInfo) + assert oinfo.binsha is not None + ostream = entity.stream(info.binsha) + assert isinstance(ostream, OStream) + assert ostream.binsha is not None + + # verify the stream + try: + assert entity.is_valid_stream(info.binsha, use_crc=True) + except UnsupportedOperation: + pass + # END ignore version issues + assert entity.is_valid_stream(info.binsha, use_crc=False) + # END for each info, stream tuple + assert count == size + + # END for each entity + + # pack writing - write all packs into one + # index path can be None + pack_path1 = tempfile.mktemp('', "pack1", rw_dir) + pack_path2 = tempfile.mktemp('', "pack2", rw_dir) + index_path = tempfile.mktemp('', 'index', rw_dir) + iteration = 0 + + def rewind_streams(): + for obj in pack_objs: + obj.stream.seek(0) + # END utility + for ppath, ipath, num_obj in zip((pack_path1, pack_path2), + (index_path, None), + (len(pack_objs), None)): + iwrite = None + if ipath: + ifile = open(ipath, 'wb') + iwrite = ifile.write + # END handle ip + + # make sure we rewind the streams ... we work on the same objects over and over again + if iteration > 0: + rewind_streams() + # END rewind streams + iteration += 1 + + with open(ppath, 'wb') as pfile: + pack_sha, index_sha = PackEntity.write_pack(pack_objs, pfile.write, iwrite, object_count=num_obj) + assert os.path.getsize(ppath) > 100 + + # verify pack + pf = PackFile(ppath) + assert pf.size() == len(pack_objs) + assert pf.version() == PackFile.pack_version_default + assert pf.checksum() == pack_sha + pf.close() + + # verify index + if ipath is not None: + ifile.close() + assert os.path.getsize(ipath) > 100 + idx = PackIndexFile(ipath) + assert idx.version() == PackIndexFile.index_version_default + assert idx.packfile_checksum() == pack_sha + assert idx.indexfile_checksum() == index_sha + assert idx.size() == len(pack_objs) + idx.close() + # END verify files exist + # END for each packpath, indexpath pair + + # verify the packs thoroughly + rewind_streams() + entity = PackEntity.create(pack_objs, rw_dir) + count = 0 + for info in entity.info_iter(): + count += 1 + for use_crc in range(2): + assert entity.is_valid_stream(info.binsha, use_crc) + # END for each crc mode + # END for each info + assert count == len(pack_objs) + entity.close() + + def test_pack_64(self): + # TODO: hex-edit a pack helping us to verify that we can handle 64 byte offsets + # of course without really needing such a huge pack + pytest.skip('not implemented') diff --git a/gitdb/gitdb/test/test_stream.py b/gitdb/gitdb/test/test_stream.py new file mode 100644 index 000000000..f36b06b96 --- /dev/null +++ b/gitdb/gitdb/test/test_stream.py @@ -0,0 +1,200 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: https://opensource.org/license/bsd-3-clause/ +"""Test for object db""" + +from gitdb.test.lib import ( + TestBase, + DummyStream, + make_bytes, + make_object, + fixture_path +) + +from gitdb import ( + DecompressMemMapReader, + FDCompressedSha1Writer, + LooseObjectDB, + Sha1Writer, + MemoryDB, + IStream, +) +from gitdb.util import hex_to_bin + +import zlib +from gitdb.typ import ( + str_blob_type +) + +import tempfile +import os +from io import BytesIO + + +class TestStream(TestBase): + + """Test stream classes""" + + data_sizes = (15, 10000, 1000 * 1024 + 512) + + def _assert_stream_reader(self, stream, cdata, rewind_stream=lambda s: None): + """Make stream tests - the orig_stream is seekable, allowing it to be + rewound and reused + :param cdata: the data we expect to read from stream, the contents + :param rewind_stream: function called to rewind the stream to make it ready + for reuse""" + ns = 10 + assert len(cdata) > ns - 1, "Data must be larger than %i, was %i" % (ns, len(cdata)) + + # read in small steps + ss = len(cdata) // ns + for i in range(ns): + data = stream.read(ss) + chunk = cdata[i * ss:(i + 1) * ss] + assert data == chunk + # END for each step + rest = stream.read() + if rest: + assert rest == cdata[-len(rest):] + # END handle rest + + if isinstance(stream, DecompressMemMapReader): + assert len(stream.data()) == stream.compressed_bytes_read() + # END handle special type + + rewind_stream(stream) + + # read everything + rdata = stream.read() + assert rdata == cdata + + if isinstance(stream, DecompressMemMapReader): + assert len(stream.data()) == stream.compressed_bytes_read() + # END handle special type + + def test_decompress_reader(self): + for close_on_deletion in range(2): + for with_size in range(2): + for ds in self.data_sizes: + cdata = make_bytes(ds, randomize=False) + + # zdata = zipped actual data + # cdata = original content data + + # create reader + if with_size: + # need object data + zdata = zlib.compress(make_object(str_blob_type, cdata)) + typ, size, reader = DecompressMemMapReader.new(zdata, close_on_deletion) + assert size == len(cdata) + assert typ == str_blob_type + + # even if we don't set the size, it will be set automatically on first read + test_reader = DecompressMemMapReader(zdata, close_on_deletion=False) + assert test_reader._s == len(cdata) + else: + # here we need content data + zdata = zlib.compress(cdata) + reader = DecompressMemMapReader(zdata, close_on_deletion, len(cdata)) + assert reader._s == len(cdata) + # END get reader + + self._assert_stream_reader(reader, cdata, lambda r: r.seek(0)) + + # put in a dummy stream for closing + dummy = DummyStream() + reader._m = dummy + + assert not dummy.closed + del(reader) + assert dummy.closed == close_on_deletion + # END for each datasize + # END whether size should be used + # END whether stream should be closed when deleted + + def test_sha_writer(self): + writer = Sha1Writer() + assert 2 == writer.write(b"hi") + assert len(writer.sha(as_hex=1)) == 40 + assert len(writer.sha(as_hex=0)) == 20 + + # make sure it does something ;) + prev_sha = writer.sha() + writer.write(b"hi again") + assert writer.sha() != prev_sha + + def test_compressed_writer(self): + for ds in self.data_sizes: + fd, path = tempfile.mkstemp() + ostream = FDCompressedSha1Writer(fd) + data = make_bytes(ds, randomize=False) + + # for now, just a single write, code doesn't care about chunking + assert len(data) == ostream.write(data) + ostream.close() + + # its closed already + self.assertRaises(OSError, os.close, fd) + + # read everything back, compare to data we zip + fd = os.open(path, os.O_RDONLY | getattr(os, 'O_BINARY', 0)) + written_data = os.read(fd, os.path.getsize(path)) + assert len(written_data) == os.path.getsize(path) + os.close(fd) + assert written_data == zlib.compress(data, 1) # best speed + + os.remove(path) + # END for each os + + def test_decompress_reader_special_case(self): + odb = LooseObjectDB(fixture_path('objects')) + mdb = MemoryDB() + for sha in (b'888401851f15db0eed60eb1bc29dec5ddcace911', + b'7bb839852ed5e3a069966281bb08d50012fb309b',): + ostream = odb.stream(hex_to_bin(sha)) + + # if there is a bug, we will be missing one byte exactly ! + data = ostream.read() + assert len(data) == ostream.size + + # Putting it back in should yield nothing new - after all, we have + dump = mdb.store(IStream(ostream.type, ostream.size, BytesIO(data))) + assert dump.hexsha == sha + # end for each loose object sha to test + + def test_decompress_reader_chunked_read_does_not_terminate_early(self): + """Regression test for #120: read(N) must not return b'' before EOF. + + zlib can consume input without producing decompressed output (e.g. + while ingesting block headers). The reader's internal recursion + previously bailed on any empty zip output, so a caller reading in + small chunks via the standard `while chunk := stream.read(N)` idiom + would terminate at the first empty chunk -- before the actual end + of the uncompressed stream. + """ + # Highly compressible data exposes the bug because each zlib chunk + # spans many uncompressed bytes -- intermediate decompress() calls + # often return empty while consuming input. + data = b"hello world! " * 1000 + zdata = zlib.compress(data) + + # Loop with a small chunk size to force many internal read/decompression + # iterations before EOF. + for chunk_size in (1, 4, 16, 64): + reader = DecompressMemMapReader( + zdata, close_on_deletion=False, size=len(data) + ) + out = bytearray() + while True: + chunk = reader.read(chunk_size) + if not chunk: + break + out.extend(chunk) + assert bytes(out) == data, ( + f"chunk_size={chunk_size}: got {len(out)}/{len(data)} bytes" + ) + assert reader._br == reader._s, ( + f"chunk_size={chunk_size}: stream stopped at " + f"{reader._br}/{reader._s}" + ) diff --git a/gitdb/gitdb/test/test_util.py b/gitdb/gitdb/test/test_util.py new file mode 100644 index 000000000..166b33c3a --- /dev/null +++ b/gitdb/gitdb/test/test_util.py @@ -0,0 +1,100 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: https://opensource.org/license/bsd-3-clause/ +"""Test for object db""" +import tempfile +import os + +from gitdb.test.lib import TestBase +from gitdb.util import ( + to_hex_sha, + to_bin_sha, + NULL_HEX_SHA, + LockedFD +) + + +class TestUtils(TestBase): + + def test_basics(self): + assert to_hex_sha(NULL_HEX_SHA) == NULL_HEX_SHA + assert len(to_bin_sha(NULL_HEX_SHA)) == 20 + assert to_hex_sha(to_bin_sha(NULL_HEX_SHA)) == NULL_HEX_SHA.encode("ascii") + + def _cmp_contents(self, file_path, data): + # raise if data from file at file_path + # does not match data string + with open(file_path, "rb") as fp: + assert fp.read() == data.encode("ascii") + + def test_lockedfd(self): + my_file = tempfile.mktemp() + orig_data = "hello" + new_data = "world" + with open(my_file, "wb") as my_file_fp: + my_file_fp.write(orig_data.encode("ascii")) + + try: + lfd = LockedFD(my_file) + lockfilepath = lfd._lockfilepath() + + # cannot end before it was started + self.assertRaises(AssertionError, lfd.rollback) + self.assertRaises(AssertionError, lfd.commit) + + # open for writing + assert not os.path.isfile(lockfilepath) + wfd = lfd.open(write=True) + assert lfd._fd is wfd + assert os.path.isfile(lockfilepath) + + # write data and fail + os.write(wfd, new_data.encode("ascii")) + lfd.rollback() + assert lfd._fd is None + self._cmp_contents(my_file, orig_data) + assert not os.path.isfile(lockfilepath) + + # additional call doesn't fail + lfd.commit() + lfd.rollback() + + # test reading + lfd = LockedFD(my_file) + rfd = lfd.open(write=False) + assert os.read(rfd, len(orig_data)) == orig_data.encode("ascii") + + assert os.path.isfile(lockfilepath) + # deletion rolls back + del(lfd) + assert not os.path.isfile(lockfilepath) + + # write data - concurrently + lfd = LockedFD(my_file) + olfd = LockedFD(my_file) + assert not os.path.isfile(lockfilepath) + wfdstream = lfd.open(write=True, stream=True) # this time as stream + assert os.path.isfile(lockfilepath) + # another one fails + self.assertRaises(IOError, olfd.open) + + wfdstream.write(new_data.encode("ascii")) + lfd.commit() + assert not os.path.isfile(lockfilepath) + self._cmp_contents(my_file, new_data) + + # could test automatic _end_writing on destruction + finally: + os.remove(my_file) + # END final cleanup + + # try non-existing file for reading + lfd = LockedFD(tempfile.mktemp()) + try: + lfd.open(write=False) + except OSError: + assert not os.path.exists(lfd._lockfilepath()) + else: + self.fail("expected OSError") + # END handle exceptions diff --git a/gitdb/gitdb/typ.py b/gitdb/gitdb/typ.py new file mode 100644 index 000000000..314db50a7 --- /dev/null +++ b/gitdb/gitdb/typ.py @@ -0,0 +1,10 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: https://opensource.org/license/bsd-3-clause/ +"""Module containing information about types known to the database""" + +str_blob_type = b'blob' +str_commit_type = b'commit' +str_tree_type = b'tree' +str_tag_type = b'tag' diff --git a/gitdb/gitdb/util.py b/gitdb/gitdb/util.py new file mode 100644 index 000000000..bb6d8797a --- /dev/null +++ b/gitdb/gitdb/util.py @@ -0,0 +1,398 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: https://opensource.org/license/bsd-3-clause/ +import binascii +import os +import mmap +import sys +import time +import errno + +from io import BytesIO + +from smmap import ( + StaticWindowMapManager, + SlidingWindowMapManager, + SlidingWindowMapBuffer +) + +# initialize our global memory manager instance +# Use it to free cached (and unused) resources. +mman = SlidingWindowMapManager() +# END handle mman + +import hashlib + +try: + from struct import unpack_from +except ImportError: + from struct import unpack, calcsize + __calcsize_cache = dict() + + def unpack_from(fmt, data, offset=0): + try: + size = __calcsize_cache[fmt] + except KeyError: + size = calcsize(fmt) + __calcsize_cache[fmt] = size + # END exception handling + return unpack(fmt, data[offset: offset + size]) + # END own unpack_from implementation + + +#{ Aliases + +hex_to_bin = binascii.a2b_hex +bin_to_hex = binascii.b2a_hex + +# errors +ENOENT = errno.ENOENT + +# os shortcuts +exists = os.path.exists +mkdir = os.mkdir +chmod = os.chmod +isdir = os.path.isdir +isfile = os.path.isfile +rename = os.rename +dirname = os.path.dirname +basename = os.path.basename +join = os.path.join +read = os.read +write = os.write +close = os.close +fsync = os.fsync + + +def _retry(func, *args, **kwargs): + # Wrapper around functions, that are problematic on "Windows". Sometimes + # the OS or someone else has still a handle to the file + if sys.platform == "win32": + for _ in range(10): + try: + return func(*args, **kwargs) + except Exception: + time.sleep(0.1) + return func(*args, **kwargs) + else: + return func(*args, **kwargs) + + +def remove(*args, **kwargs): + return _retry(os.remove, *args, **kwargs) + + +# Backwards compatibility imports +from gitdb.const import ( + NULL_BIN_SHA, + NULL_HEX_SHA +) + +#} END Aliases + +#{ compatibility stuff ... + + +class _RandomAccessBytesIO: + + """Wrapper to provide required functionality in case memory maps cannot or may + not be used. This is only really required in python 2.4""" + __slots__ = '_sio' + + def __init__(self, buf=''): + self._sio = BytesIO(buf) + + def __getattr__(self, attr): + return getattr(self._sio, attr) + + def __len__(self): + return len(self.getvalue()) + + def __getitem__(self, i): + return self.getvalue()[i] + + def __getslice__(self, start, end): + return self.getvalue()[start:end] + + +def byte_ord(b): + """ + Return the integer representation of the byte string. This supports Python + 3 byte arrays as well as standard strings. + """ + try: + return ord(b) + except TypeError: + return b + +#} END compatibility stuff ... + +#{ Routines + + +def make_sha(source=b''): + """A python2.4 workaround for the sha/hashlib module fiasco + + **Note** From the dulwich project """ + try: + return hashlib.sha1(source) + except NameError: + import sha + sha1 = sha.sha(source) + return sha1 + + +def allocate_memory(size): + """:return: a file-protocol accessible memory block of the given size""" + if size == 0: + return _RandomAccessBytesIO(b'') + # END handle empty chunks gracefully + + try: + return mmap.mmap(-1, size) # read-write by default + except OSError: + # setup real memory instead + # this of course may fail if the amount of memory is not available in + # one chunk - would only be the case in python 2.4, being more likely on + # 32 bit systems. + return _RandomAccessBytesIO(b"\0" * size) + # END handle memory allocation + + +def file_contents_ro(fd, stream=False, allow_mmap=True): + """:return: read-only contents of the file represented by the file descriptor fd + + :param fd: file descriptor opened for reading + :param stream: if False, random access is provided, otherwise the stream interface + is provided. + :param allow_mmap: if True, its allowed to map the contents into memory, which + allows large files to be handled and accessed efficiently. The file-descriptor + will change its position if this is False""" + try: + if allow_mmap: + # supports stream and random access + try: + return mmap.mmap(fd, 0, access=mmap.ACCESS_READ) + except OSError: + # python 2.4 issue, 0 wants to be the actual size + return mmap.mmap(fd, os.fstat(fd).st_size, access=mmap.ACCESS_READ) + # END handle python 2.4 + except OSError: + pass + # END exception handling + + # read manually + contents = os.read(fd, os.fstat(fd).st_size) + if stream: + return _RandomAccessBytesIO(contents) + return contents + + +def file_contents_ro_filepath(filepath, stream=False, allow_mmap=True, flags=0): + """Get the file contents at filepath as fast as possible + + :return: random access compatible memory of the given filepath + :param stream: see ``file_contents_ro`` + :param allow_mmap: see ``file_contents_ro`` + :param flags: additional flags to pass to os.open + :raise OSError: If the file could not be opened + + **Note** for now we don't try to use O_NOATIME directly as the right value needs to be + shared per database in fact. It only makes a real difference for loose object + databases anyway, and they use it with the help of the ``flags`` parameter""" + fd = os.open(filepath, os.O_RDONLY | getattr(os, 'O_BINARY', 0) | flags) + try: + return file_contents_ro(fd, stream, allow_mmap) + finally: + close(fd) + # END assure file is closed + + +def sliding_ro_buffer(filepath, flags=0): + """ + :return: a buffer compatible object which uses our mapped memory manager internally + ready to read the whole given filepath""" + return SlidingWindowMapBuffer(mman.make_cursor(filepath), flags=flags) + + +def to_hex_sha(sha): + """:return: hexified version of sha""" + if len(sha) == 40: + return sha + return bin_to_hex(sha) + + +def to_bin_sha(sha): + if len(sha) == 20: + return sha + return hex_to_bin(sha) + + +#} END routines + + +#{ Utilities + +class LazyMixin: + + """ + Base class providing an interface to lazily retrieve attribute values upon + first access. If slots are used, memory will only be reserved once the attribute + is actually accessed and retrieved the first time. All future accesses will + return the cached value as stored in the Instance's dict or slot. + """ + + __slots__ = tuple() + + def __getattr__(self, attr): + """ + Whenever an attribute is requested that we do not know, we allow it + to be created and set. Next time the same attribute is requested, it is simply + returned from our dict/slots. """ + self._set_cache_(attr) + # will raise in case the cache was not created + return object.__getattribute__(self, attr) + + def _set_cache_(self, attr): + """ + This method should be overridden in the derived class. + It should check whether the attribute named by attr can be created + and cached. Do nothing if you do not know the attribute or call your subclass + + The derived class may create as many additional attributes as it deems + necessary in case a git command returns more information than represented + in the single attribute.""" + pass + + +class LockedFD: + + """ + This class facilitates a safe read and write operation to a file on disk. + If we write to 'file', we obtain a lock file at 'file.lock' and write to + that instead. If we succeed, the lock file will be renamed to overwrite + the original file. + + When reading, we obtain a lock file, but to prevent other writers from + succeeding while we are reading the file. + + This type handles error correctly in that it will assure a consistent state + on destruction. + + **note** with this setup, parallel reading is not possible""" + __slots__ = ("_filepath", '_fd', '_write') + + def __init__(self, filepath): + """Initialize an instance with the givne filepath""" + self._filepath = filepath + self._fd = None + self._write = None # if True, we write a file + + def __del__(self): + # will do nothing if the file descriptor is already closed + if self._fd is not None: + self.rollback() + + def _lockfilepath(self): + return "%s.lock" % self._filepath + + def open(self, write=False, stream=False): + """ + Open the file descriptor for reading or writing, both in binary mode. + + :param write: if True, the file descriptor will be opened for writing. Other + wise it will be opened read-only. + :param stream: if True, the file descriptor will be wrapped into a simple stream + object which supports only reading or writing + :return: fd to read from or write to. It is still maintained by this instance + and must not be closed directly + :raise IOError: if the lock could not be retrieved + :raise OSError: If the actual file could not be opened for reading + + **note** must only be called once""" + if self._write is not None: + raise AssertionError("Called %s multiple times" % self.open) + + self._write = write + + # try to open the lock file + binary = getattr(os, 'O_BINARY', 0) + lockmode = os.O_WRONLY | os.O_CREAT | os.O_EXCL | binary + try: + fd = os.open(self._lockfilepath(), lockmode, int("600", 8)) + if not write: + os.close(fd) + else: + self._fd = fd + # END handle file descriptor + except OSError as e: + raise OSError("Lock at %r could not be obtained" % self._lockfilepath()) from e + # END handle lock retrieval + + # open actual file if required + if self._fd is None: + # we could specify exclusive here, as we obtained the lock anyway + try: + self._fd = os.open(self._filepath, os.O_RDONLY | binary) + except: + # assure we release our lockfile + remove(self._lockfilepath()) + raise + # END handle lockfile + # END open descriptor for reading + + if stream: + # need delayed import + from gitdb.stream import FDStream + return FDStream(self._fd) + else: + return self._fd + # END handle stream + + def commit(self): + """When done writing, call this function to commit your changes into the + actual file. + The file descriptor will be closed, and the lockfile handled. + + **Note** can be called multiple times""" + self._end_writing(successful=True) + + def rollback(self): + """Abort your operation without any changes. The file descriptor will be + closed, and the lock released. + + **Note** can be called multiple times""" + self._end_writing(successful=False) + + def _end_writing(self, successful=True): + """Handle the lock according to the write mode """ + if self._write is None: + raise AssertionError("Cannot end operation if it wasn't started yet") + + if self._fd is None: + return + + os.close(self._fd) + self._fd = None + + lockfile = self._lockfilepath() + if self._write and successful: + # on windows, rename does not silently overwrite the existing one + if sys.platform == "win32": + if isfile(self._filepath): + remove(self._filepath) + # END remove if exists + # END win32 special handling + os.rename(lockfile, self._filepath) + + # assure others can at least read the file - the tmpfile left it at rw-- + # We may also write that file, on windows that boils down to a remove- + # protection as well + chmod(self._filepath, int("644", 8)) + else: + # just delete the file so far, we failed + remove(lockfile) + # END successful handling + +#} END utilities diff --git a/gitdb/gitdb/utils/__init__.py b/gitdb/gitdb/utils/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/gitdb/gitdb/utils/encoding.py b/gitdb/gitdb/utils/encoding.py new file mode 100644 index 000000000..b534ef763 --- /dev/null +++ b/gitdb/gitdb/utils/encoding.py @@ -0,0 +1,18 @@ +def force_bytes(data, encoding="utf-8"): + if isinstance(data, bytes): + return data + + if isinstance(data, str): + return data.encode(encoding) + + return data + + +def force_text(data, encoding="utf-8"): + if isinstance(data, str): + return data + + if isinstance(data, bytes): + return data.decode(encoding) + + return str(data, encoding) diff --git a/gitdb/requirements.txt b/gitdb/requirements.txt new file mode 100644 index 000000000..1b2e11db7 --- /dev/null +++ b/gitdb/requirements.txt @@ -0,0 +1 @@ +smmap>=3.0.1,<6 diff --git a/gitdb/setup.py b/gitdb/setup.py new file mode 100755 index 000000000..924f026b8 --- /dev/null +++ b/gitdb/setup.py @@ -0,0 +1,46 @@ +from setuptools import setup + +# NOTE: This is currently duplicated from the gitdb.__init__ module, because +# that's just how you write a setup.py (nobody reads this stuff out of the +# module) + +__author__ = "Sebastian Thiel" +__contact__ = "byronimo@gmail.com" +__homepage__ = "https://github.com/gitpython-developers/GitPython/tree/main/gitdb" +version_info = (4, 0, 12) +__version__ = '.'.join(str(i) for i in version_info) + +setup( + name="gitdb", + version=__version__, + description="Git Object Database", + author=__author__, + author_email=__contact__, + url=__homepage__, + packages=('gitdb', 'gitdb.db', 'gitdb.utils', 'gitdb.test'), + license="BSD License", + zip_safe=False, + install_requires=['smmap>=3.0.1,<6'], + long_description="""GitDB is a pure-Python git object database""", + python_requires='>=3.7', + # See https://pypi.python.org/pypi?%3Aaction=list_classifiers + classifiers=[ + "Development Status :: 5 - Production/Stable", + "Environment :: Console", + "Intended Audience :: Developers", + "Operating System :: OS Independent", + "Operating System :: POSIX", + "Operating System :: Microsoft :: Windows", + "Operating System :: MacOS :: MacOS X", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3 :: Only", + ] +) diff --git a/init-tests-after-clone.sh b/init-tests-after-clone.sh index a88f983fc..db3540857 100755 --- a/init-tests-after-clone.sh +++ b/init-tests-after-clone.sh @@ -55,9 +55,6 @@ git reset --hard HEAD~1 # Point the master branch where we started, so we test the correct code. git reset --hard __testing_point__ -# The tests need submodules, including a submodule with a submodule. -git submodule update --init --recursive - # The tests need some version tags. Try to get them even in forks. This fetches # other objects too. So, locally, we always do it, for a consistent experience. if ! ci || no_version_tags; then diff --git a/pyproject.toml b/pyproject.toml index 324630002..00662dceb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,20 +28,22 @@ warn_unreachable = true implicit_reexport = true # strict = true # TODO: Remove when 'gitdb' is fully annotated. -exclude = ["^git/ext/gitdb"] +exclude = ["^(gitdb|smmap)/"] [[tool.mypy.overrides]] -module = "gitdb.*" +module = ["gitdb", "gitdb.*"] +follow_imports = "skip" ignore_missing_imports = true [tool.basedpyright] typeCheckingMode = "standard" pythonVersion = "3.7" extraPaths = [ - "git/ext/gitdb", - "git/ext/gitdb/gitdb/ext/smmap", + "gitdb", + "smmap", ] exclude = [ - "git/ext/gitdb", + "gitdb", + "smmap", ] [tool.coverage.run] @@ -57,6 +59,8 @@ line-length = 120 # Exclude a variety of commonly ignored directories. exclude = [ "git/ext/", + "gitdb/", + "smmap/", "build", "dist", ] diff --git a/setup.py b/setup.py index a7b1eab00..6f95fef18 100755 --- a/setup.py +++ b/setup.py @@ -71,7 +71,7 @@ def _stamp_version(filename: str) -> None: author_email="byronimo@gmail.com, mtrier@gmail.com", license="BSD-3-Clause", url="https://github.com/gitpython-developers/GitPython", - packages=find_packages(exclude=["test", "test.*"]), + packages=find_packages(include=["git", "git.*"]), include_package_data=True, package_dir={"git": "git"}, python_requires=">=3.7", diff --git a/smmap/.coveragerc b/smmap/.coveragerc new file mode 100644 index 000000000..e61d27ba9 --- /dev/null +++ b/smmap/.coveragerc @@ -0,0 +1,10 @@ +[run] +source = smmap + +; to make nosetests happy +[report] +omit = + */yaml* + */tests/* + */python?.?/* + */site-packages/nose/* \ No newline at end of file diff --git a/smmap/.gitignore b/smmap/.gitignore new file mode 100644 index 000000000..85cc74fdf --- /dev/null +++ b/smmap/.gitignore @@ -0,0 +1,12 @@ +*.pyc +build/ +.coverage +coverage +cover/ +dist/ +MANIFEST +.tox +*.egg-info +.noseids +*.sublime-workspace +/env/ diff --git a/smmap/LICENSE b/smmap/LICENSE new file mode 100644 index 000000000..710010f1f --- /dev/null +++ b/smmap/LICENSE @@ -0,0 +1,30 @@ +Copyright (C) 2010, 2011 Sebastian Thiel and contributors +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + +* Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. + +* Neither the name of the async project nor the names of +its contributors may be used to endorse or promote products derived +from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED +TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + diff --git a/smmap/MANIFEST.in b/smmap/MANIFEST.in new file mode 100644 index 000000000..399a207e6 --- /dev/null +++ b/smmap/MANIFEST.in @@ -0,0 +1,2 @@ +# Include the license file +include LICENSE diff --git a/smmap/Makefile b/smmap/Makefile new file mode 100644 index 000000000..ad0912ffe --- /dev/null +++ b/smmap/Makefile @@ -0,0 +1,7 @@ +.PHONY: all clean + +all: + @grep -Ee '^[a-z].*:' Makefile | cut -d: -f1 | grep -vF all + +clean: + rm -rf build/ dist/ .eggs/ .tox/ diff --git a/smmap/README.md b/smmap/README.md new file mode 100644 index 000000000..748f0f0db --- /dev/null +++ b/smmap/README.md @@ -0,0 +1,82 @@ +## Motivation + +When reading from many possibly large files in a fashion similar to random access, it is usually the fastest and most efficient to use memory maps. + +Although memory maps have many advantages, they represent a very limited system resource as every map uses one file descriptor, whose amount is limited per process. On 32 bit systems, the amount of memory you can have mapped at a time is naturally limited to theoretical 4GB of memory, which may not be enough for some applications. + + +## Limitations + +* **System resources (file-handles) are likely to be leaked!** This is due to the library authors reliance on a deterministic `__del__()` destructor. +* The memory access is read-only by design. + + +## Overview + +![Dependency packages](https://github.com/gitpython-developers/GitPython/actions/workflows/dependencies.yml/badge.svg) + +Smmap wraps an interface around mmap and tracks the mapped files as well as the amount of clients who use it. If the system runs out of resources, or if a memory limit is reached, it will automatically unload unused maps to allow continued operation. + +To allow processing large files even on 32 bit systems, it allows only portions of the file to be mapped. Once the user reads beyond the mapped region, smmap will automatically map the next required region, unloading unused regions using a LRU algorithm. + +Although the library can be used most efficiently with its native interface, a Buffer implementation is provided to hide these details behind a simple string-like interface. + +For performance critical 64 bit applications, a simplified version of memory mapping is provided which always maps the whole file, but still provides the benefit of unloading unused mappings on demand. + + + +## Prerequisites + +* Python 3.7+ +* OSX, Windows or Linux + +The package was tested on all of the previously mentioned configurations. + +## Installing smmap + +[![Documentation Status](https://readthedocs.org/projects/smmap/badge/?version=latest)](https://readthedocs.org/projects/smmap/?badge=latest) + +Its easiest to install smmap using the [pip](http://www.pip-installer.org/en/latest) program: + +```bash +$ pip install smmap +``` + +As the command will install smmap in your respective python distribution, you will most likely need root permissions to authorize the required changes. + +If you have downloaded the source archive, the package can be installed by running the `setup.py` script: + +```bash +$ python setup.py install +``` + +It is advised to have a look at the **Usage Guide** for a brief introduction on the different database implementations. + + + +## Homepage and Links + +The project is maintained in the GitPython repository at https://github.com/gitpython-developers/GitPython/tree/main/smmap . + +The latest source can be cloned from github as well: + +* https://github.com/gitpython-developers/GitPython.git + + +For support, please use the git-python mailing list: + +* http://groups.google.com/group/git-python + + +Issues can be filed on github: + +* https://github.com/gitpython-developers/GitPython/issues + +A link to the pypi page related to this repository: + +* https://pypi.org/project/smmap/ + + +## License Information + +*smmap* is licensed under the New BSD License. diff --git a/smmap/SECURITY.md b/smmap/SECURITY.md new file mode 100644 index 000000000..9e0c0d16a --- /dev/null +++ b/smmap/SECURITY.md @@ -0,0 +1,3 @@ +# Security Policy + +See [GitPython](https://github.com/gitpython-developers/GitPython/blob/main/SECURITY.md). Vulnerabilities found in `smmap` can be reported there. diff --git a/smmap/doc/.gitignore b/smmap/doc/.gitignore new file mode 100644 index 000000000..32060acdd --- /dev/null +++ b/smmap/doc/.gitignore @@ -0,0 +1,2 @@ +build +*.version_info diff --git a/smmap/doc/Makefile b/smmap/doc/Makefile new file mode 100644 index 000000000..675ec2094 --- /dev/null +++ b/smmap/doc/Makefile @@ -0,0 +1,89 @@ +# Makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = sphinx-build +PAPER = +BUILDDIR = build + +# Internal variables. +PAPEROPT_a4 = -D latex_paper_size=a4 +PAPEROPT_letter = -D latex_paper_size=letter +ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source + +.PHONY: help clean html dirhtml pickle json htmlhelp qthelp latex changes linkcheck doctest + +help: + @echo "Please use \`make ' where is one of" + @echo " html to make standalone HTML files" + @echo " dirhtml to make HTML files named index.html in directories" + @echo " pickle to make pickle files" + @echo " json to make JSON files" + @echo " htmlhelp to make HTML files and a HTML help project" + @echo " qthelp to make HTML files and a qthelp project" + @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" + @echo " changes to make an overview of all changed/added/deprecated items" + @echo " linkcheck to check all external links for integrity" + @echo " doctest to run all doctests embedded in the documentation (if enabled)" + +clean: + -rm -rf $(BUILDDIR)/* + +html: + $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." + +dirhtml: + $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." + +pickle: + $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle + @echo + @echo "Build finished; now you can process the pickle files." + +json: + $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json + @echo + @echo "Build finished; now you can process the JSON files." + +htmlhelp: + $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp + @echo + @echo "Build finished; now you can run HTML Help Workshop with the" \ + ".hhp project file in $(BUILDDIR)/htmlhelp." + +qthelp: + $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp + @echo + @echo "Build finished; now you can run "qcollectiongenerator" with the" \ + ".qhcp project file in $(BUILDDIR)/qthelp, like this:" + @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/smmap.qhcp" + @echo "To view the help file:" + @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/smmap.qhc" + +latex: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo + @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." + @echo "Run \`make all-pdf' or \`make all-ps' in that directory to" \ + "run these through (pdf)latex." + +changes: + $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes + @echo + @echo "The overview file is in $(BUILDDIR)/changes." + +linkcheck: + $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck + @echo + @echo "Link check complete; look for any errors in the above output " \ + "or in $(BUILDDIR)/linkcheck/output.txt." + +doctest: + $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest + @echo "Testing of doctests in the sources finished, look at the " \ + "results in $(BUILDDIR)/doctest/output.txt." diff --git a/smmap/doc/make.bat b/smmap/doc/make.bat new file mode 100644 index 000000000..6900a2a46 --- /dev/null +++ b/smmap/doc/make.bat @@ -0,0 +1,113 @@ +@ECHO OFF + +REM Command file for Sphinx documentation + +set SPHINXBUILD=sphinx-build +set BUILDDIR=build +set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% source +if NOT "%PAPER%" == "" ( + set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS% +) + +if "%1" == "" goto help + +if "%1" == "help" ( + :help + echo.Please use `make ^` where ^ is one of + echo. html to make standalone HTML files + echo. dirhtml to make HTML files named index.html in directories + echo. pickle to make pickle files + echo. json to make JSON files + echo. htmlhelp to make HTML files and a HTML help project + echo. qthelp to make HTML files and a qthelp project + echo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter + echo. changes to make an overview over all changed/added/deprecated items + echo. linkcheck to check all external links for integrity + echo. doctest to run all doctests embedded in the documentation if enabled + goto end +) + +if "%1" == "clean" ( + for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i + del /q /s %BUILDDIR%\* + goto end +) + +if "%1" == "html" ( + %SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html + echo. + echo.Build finished. The HTML pages are in %BUILDDIR%/html. + goto end +) + +if "%1" == "dirhtml" ( + %SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml + echo. + echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml. + goto end +) + +if "%1" == "pickle" ( + %SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle + echo. + echo.Build finished; now you can process the pickle files. + goto end +) + +if "%1" == "json" ( + %SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json + echo. + echo.Build finished; now you can process the JSON files. + goto end +) + +if "%1" == "htmlhelp" ( + %SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp + echo. + echo.Build finished; now you can run HTML Help Workshop with the ^ +.hhp project file in %BUILDDIR%/htmlhelp. + goto end +) + +if "%1" == "qthelp" ( + %SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp + echo. + echo.Build finished; now you can run "qcollectiongenerator" with the ^ +.qhcp project file in %BUILDDIR%/qthelp, like this: + echo.^> qcollectiongenerator %BUILDDIR%\qthelp\smmap.qhcp + echo.To view the help file: + echo.^> assistant -collectionFile %BUILDDIR%\qthelp\smmap.ghc + goto end +) + +if "%1" == "latex" ( + %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex + echo. + echo.Build finished; the LaTeX files are in %BUILDDIR%/latex. + goto end +) + +if "%1" == "changes" ( + %SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes + echo. + echo.The overview file is in %BUILDDIR%/changes. + goto end +) + +if "%1" == "linkcheck" ( + %SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck + echo. + echo.Link check complete; look for any errors in the above output ^ +or in %BUILDDIR%/linkcheck/output.txt. + goto end +) + +if "%1" == "doctest" ( + %SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest + echo. + echo.Testing of doctests in the sources finished, look at the ^ +results in %BUILDDIR%/doctest/output.txt. + goto end +) + +:end diff --git a/smmap/doc/source/api.rst b/smmap/doc/source/api.rst new file mode 100644 index 000000000..2e2dac41a --- /dev/null +++ b/smmap/doc/source/api.rst @@ -0,0 +1,34 @@ +.. _api-label: + +############# +API Reference +############# + +*********************** +Mapped Memory Managers +*********************** + +.. automodule:: smmap.mman + :members: + :undoc-members: + +******* +Buffers +******* + +.. automodule:: smmap.buf + :members: + :undoc-members: + +********* +Utilities +********* + +.. automodule:: smmap.util + :members: + :undoc-members: + + + + + diff --git a/smmap/doc/source/changes.rst b/smmap/doc/source/changes.rst new file mode 100644 index 000000000..faed6085d --- /dev/null +++ b/smmap/doc/source/changes.rst @@ -0,0 +1,108 @@ +######### +Changelog +######### + +****** +v5.0.3 +****** + +- declare support for Python 3.13 + +For more, see https://github.com/gitpython-developers/smmap/compare/v5.0.3...v5.0.2 + +****** +v5.0.2 +****** + +- remove a usage of mktemp + +****** +v5.0.1 +****** + +- Added support for Python 3.12 + +****** +v6.0.0 +****** + +YANKED + +- Dropped support 3.6 and 3.7 +- Declared support for Python 3.11 and 3.12 + +****** +v5.0.0 +****** + +- Dropped support 3.5 +- Added support for Python 3.10 + +****** +v4.0.0 +****** + +- Dropped support for Python 2.7 and 3.4 +- Added support for Python 3.7, 3.8, and 3.9 +- Removed unused exc.MemoryManagerError and exc.RegionCollectionError + +****** +v3.0.5 +****** + +- Restored Python 2 support removed in v3.0.2 +- Changed release signature key to 27C50E7F590947D7273A741E85194C08421980C9. + See https://keybase.io/byronbates for proof of ownership. + +****** +v3.0.4 +****** + +- Signed release (with correct key this time) + +****** +v3.0.2 +****** + +- Signed release +- Switched to GitHub Actions for CI + +****** +v3.0.1 +****** +- Switched back to the smmap package name on PyPI and fixed the smmap2 mirror package + (`#44 `_) +- Fixed setup.py ``long_description`` rendering + (`#40 `_) + +********** +v0.9.0 +********** +- Fixed issue with resources never being freed as mmaps were never closed. +- Client counting is now done manually, instead of relying on pyton's reference count + +********** +v0.8.5 +********** +- Fixed Python 3.0-3.3 regression, which also causes smmap to become about 3 times slower depending on the code path. It's related to this bug (http://bugs.python.org/issue15958), which was fixed in python 3.4 + +********** +v0.8.4 +********** +- Fixed Python 3 performance regression + +********** +v0.8.3 +********** +- Cleaned up code and assured it works sufficiently well with python 3 + +********** +v0.8.1 +********** +- A single bugfix + +********** +v0.8.0 +********** + +- Initial Release diff --git a/smmap/doc/source/conf.py b/smmap/doc/source/conf.py new file mode 100644 index 000000000..55dfc5c46 --- /dev/null +++ b/smmap/doc/source/conf.py @@ -0,0 +1,194 @@ +# +# smmap documentation build configuration file, created by +# sphinx-quickstart on Wed Jun 8 15:14:25 2011. +# +# This file is execfile()d with the current directory set to its containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +import sys +import os + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +sys.path.append(os.path.abspath('../../')) + +# -- General configuration ----------------------------------------------------- + +# Add any Sphinx extension module names here, as strings. They can be extensions +# coming with Sphinx (named 'sphinx.ext.*') or your custom ones. +extensions = ['sphinx.ext.autodoc', 'sphinx.ext.todo'] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['.templates'] + +# The suffix of source filenames. +source_suffix = '.rst' + +# The encoding of source files. +#source_encoding = 'utf-8' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = 'smmap' +copyright = '2011, Sebastian Thiel' + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# +# The short X.Y version. +version = '0.8.0' +# The full version, including alpha/beta/rc tags. +release = '0.8.0' + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +#language = None + +# There are two options for replacing |today|: either, you set today to some +# non-false value, then it is used: +#today = '' +# Else, today_fmt is used as the format for a strftime call. +#today_fmt = '%B %d, %Y' + +# List of documents that shouldn't be included in the build. +#unused_docs = [] + +# List of directories, relative to source directory, that shouldn't be searched +# for source files. +exclude_trees = [] + +# The reST default role (used for this markup: `text`) to use for all documents. +#default_role = None + +# If true, '()' will be appended to :func: etc. cross-reference text. +#add_function_parentheses = True + +# If true, the current module name will be prepended to all description +# unit titles (such as .. function::). +#add_module_names = True + +# If true, sectionauthor and moduleauthor directives will be shown in the +# output. They are ignored by default. +#show_authors = False + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +# A list of ignored prefixes for module index sorting. +#modindex_common_prefix = [] + + +# -- Options for HTML output --------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. Major themes that come with +# Sphinx are currently 'default' and 'sphinxdoc'. +html_theme = 'default' + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +#html_theme_options = {} + +# Add any paths that contain custom themes here, relative to this directory. +#html_theme_path = [] + +# The name for this set of Sphinx documents. If None, it defaults to +# " v documentation". +#html_title = None + +# A shorter title for the navigation bar. Default is the same as html_title. +#html_short_title = None + +# The name of an image file (relative to this directory) to place at the top +# of the sidebar. +#html_logo = None + +# The name of an image file (within the static path) to use as favicon of the +# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 +# pixels large. +#html_favicon = None + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['.static'] + +# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, +# using the given strftime format. +#html_last_updated_fmt = '%b %d, %Y' + +# If true, SmartyPants will be used to convert quotes and dashes to +# typographically correct entities. +#html_use_smartypants = True + +# Custom sidebar templates, maps document names to template names. +#html_sidebars = {} + +# Additional templates that should be rendered to pages, maps page names to +# template names. +#html_additional_pages = {} + +# If false, no module index is generated. +#html_use_modindex = True + +# If false, no index is generated. +#html_use_index = True + +# If true, the index is split into individual pages for each letter. +#html_split_index = False + +# If true, links to the reST sources are added to the pages. +#html_show_sourcelink = True + +# If true, an OpenSearch description file will be output, and all pages will +# contain a tag referring to it. The value of this option must be the +# base URL from which the finished HTML is served. +#html_use_opensearch = '' + +# If nonempty, this is the file name suffix for HTML files (e.g. ".xhtml"). +#html_file_suffix = '' + +# Output file base name for HTML help builder. +htmlhelp_basename = 'smmapdoc' + + +# -- Options for LaTeX output -------------------------------------------------- + +# The paper size ('letter' or 'a4'). +#latex_paper_size = 'letter' + +# The font size ('10pt', '11pt' or '12pt'). +#latex_font_size = '10pt' + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, author, documentclass [howto/manual]). +latex_documents = [ + ('index', 'smmap.tex', 'smmap Documentation', + 'Sebastian Thiel', 'manual'), +] + +# The name of an image file (relative to this directory) to place at the top of +# the title page. +#latex_logo = None + +# For "manual" documents, if this is true, then toplevel headings are parts, +# not chapters. +#latex_use_parts = False + +# Additional stuff for the LaTeX preamble. +#latex_preamble = '' + +# Documents to append as an appendix to all manuals. +#latex_appendices = [] + +# If false, no module index is generated. +#latex_use_modindex = True diff --git a/smmap/doc/source/index.rst b/smmap/doc/source/index.rst new file mode 100644 index 000000000..d25ef8244 --- /dev/null +++ b/smmap/doc/source/index.rst @@ -0,0 +1,26 @@ +.. smmap documentation master file, created by + sphinx-quickstart on Wed Jun 8 15:14:25 2011. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +Welcome to smmap's documentation! +================================= +**smmap** is a pure python implementation of a sliding memory map to help unifying memory mapped access on 32 and 64 bit systems and to help managing resources more efficiently. + +Contents: + +.. toctree:: + :maxdepth: 2 + + intro + tutorial + api + changes + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` + diff --git a/smmap/doc/source/intro.rst b/smmap/doc/source/intro.rst new file mode 100644 index 000000000..109fec247 --- /dev/null +++ b/smmap/doc/source/intro.rst @@ -0,0 +1,72 @@ +########### +Motivation +########### +When reading from many possibly large files in a fashion similar to random access, it is usually the fastest and most efficient to use memory maps. + +Although memory maps have many advantages, they represent a very limited system resource as every map uses one file descriptor, whose amount is limited per process. On 32 bit systems, the amount of memory you can have mapped at a time is naturally limited to theoretical 4GB of memory, which may not be enough for some applications. + +######## +Overview +######## + +Smmap wraps an interface around mmap and tracks the mapped files as well as the amount of clients who use it. If the system runs out of resources, or if a memory limit is reached, it will automatically unload unused maps to allow continued operation. + +To allow processing large files even on 32 bit systems, it allows only portions of the file to be mapped. Once the user reads beyond the mapped region, smmap will automatically map the next required region, unloading unused regions using a LRU algorithm. + +Although the library can be used most efficiently with its native interface, a Buffer implementation is provided to hide these details behind a simple string-like interface. + +For performance critical 64 bit applications, a simplified version of memory mapping is provided which always maps the whole file, but still provides the benefit of unloading unused mappings on demand. + +############# +Prerequisites +############# +* Python 3.7+ +* OSX, Windows or Linux + +The package was tested on all of the previously mentioned configurations. + +########### +Limitations +########### +* The memory access is read-only by design. + +################ +Installing smmap +################ +Its easiest to install smmap using the *pip* program:: + + $ pip install smmap + +As the command will install smmap in your respective python distribution, you will most likely need root permissions to authorize the required changes. + +If you have downloaded the source archive, the package can be installed by running the ``setup.py`` script:: + + $ python setup.py install + +It is advised to have a look at the :ref:`Usage Guide ` for a brief introduction on the different database implementations. + +################## +Homepage and Links +################## +The project is home on github at `https://github.com/gitpython-developers/smmap `_. + +The latest source can be cloned from github as well: + + * git://github.com/gitpython-developers/smmap.git + + +For support, please use the git-python mailing list: + + * http://groups.google.com/group/git-python + + +Issues can be filed on github: + + * https://github.com/gitpython-developers/smmap/issues + +################### +License Information +################### +*smmap* is licensed under the New BSD License. + +.. _pip: http://www.pip-installer.org/en/latest/ diff --git a/smmap/doc/source/tutorial.rst b/smmap/doc/source/tutorial.rst new file mode 100644 index 000000000..917b24594 --- /dev/null +++ b/smmap/doc/source/tutorial.rst @@ -0,0 +1,118 @@ +.. _tutorial-label: + +########### +Usage Guide +########### +This text briefly introduces you to the basic design decisions and accompanying classes. + +****** +Design +****** +Per application, there is *MemoryManager* which is held as static instance and used throughout the application. It can be configured to keep your resources within certain limits. + +To access mapped regions, you require a cursor. Cursors point to exactly one file and serve as handles into it. As long as it exists, the respective memory region will remain available. + +For convenience, a buffer implementation is provided which handles cursors and resource allocation behind its simple buffer like interface. + +*************** +Memory Managers +*************** +There are two types of memory managers, one uses *static* windows, the other one uses *sliding* windows. A window is a region of a file mapped into memory. Although the names might be somewhat misleading as technically windows are always static, the *sliding* version will allocate relatively small windows whereas the *static* version will always map the whole file. + +The *static* manager does nothing more than keeping a client count on the respective memory maps which always map the whole file, which allows to make some assumptions that can lead to simplified data access and increased performance, but reduces the compatibility to 32 bit systems or giant files. + +The *sliding* memory manager therefore should be the default manager when preparing an application for handling huge amounts of data on 32 bit and 64 bit platforms:: + + import smmap + # This instance should be globally available in your application + # It is configured to be well suitable for 32-bit or 64 bit applications. + mman = smmap.SlidingWindowMapManager() + + # the manager provides much useful information about its current state + # like the amount of open file handles or the amount of mapped memory + mman.num_file_handles() + mman.mapped_memory_size() + # and many more ... + + +Cursors +******* +*Cursors* are handles that point onto a window, i.e. a region of a file mapped into memory. From them you may obtain a buffer through which the data of that window can actually be accessed:: + + import smmap.test.lib + fc = smmap.test.lib.FileCreator(1024*1024*8, "test_file") + + # obtain a cursor to access some file. + c = mman.make_cursor(fc.path) + + # the cursor is now associated with the file, but not yet usable + assert c.is_associated() + assert not c.is_valid() + + # before you can use the cursor, you have to specify a window you want to + # access. The following just says you want as much data as possible starting + # from offset 0. + # To be sure your region could be mapped, query for validity + assert c.use_region().is_valid() # use_region returns self + + # once a region was mapped, you must query its dimension regularly + # to assure you don't try to access its buffer out of its bounds + assert c.size() + c.buffer()[0] # first byte + c.buffer()[1:10] # first 9 bytes + c.buffer()[c.size()-1] # last byte + + # its recommended not to create big slices when feeding the buffer + # into consumers (e.g. struct or zlib). + # Instead, either give the buffer directly, or use pythons buffer command. + buffer(c.buffer(), 1, 9) # first 9 bytes without copying them + + # you can query absolute offsets, and check whether an offset is included + # in the cursor's data. + assert c.ofs_begin() < c.ofs_end() + assert c.includes_ofs(100) + + # If you are over out of bounds with one of your region requests, the + # cursor will be come invalid. It cannot be used in that state + assert not c.use_region(fc.size, 100).is_valid() + # map as much as possible after skipping the first 100 bytes + assert c.use_region(100).is_valid() + + # You can explicitly free cursor resources by unusing the cursor's region + c.unuse_region() + assert not c.is_valid() + + +Now you would have to write your algorithms around this interface to properly slide through huge amounts of data. + +Alternatively you can use a convenience interface. + +******* +Buffers +******* +To make first use easier, at the expense of performance, there is a Buffer implementation which uses a cursor underneath. + +With it, you can access all data in a possibly huge file without having to take care of setting the cursor to different regions yourself:: + + # Create a default buffer which can operate on the whole file + buf = smmap.SlidingWindowMapBuffer(mman.make_cursor(fc.path)) + + # you can use it right away + assert buf.cursor().is_valid() + + buf[0] # access the first byte + buf[-1] # access the last ten bytes on the file + buf[-10:]# access the last ten bytes + + # If you want to keep the instance between different accesses, use the + # dedicated methods + buf.end_access() + assert not buf.cursor().is_valid() # you cannot use the buffer anymore + assert buf.begin_access(offset=10) # start using the buffer at an offset + + # it will stop using resources automatically once it goes out of scope + +Disadvantages +************* +Buffers cannot be used in place of strings or maps, hence you have to slice them to have valid input for the sorts of struct and zlib. A slice means a lot of data handling overhead which makes buffers slower compared to using cursors directly. + diff --git a/smmap/setup.cfg b/smmap/setup.cfg new file mode 100644 index 000000000..38c535ba5 --- /dev/null +++ b/smmap/setup.cfg @@ -0,0 +1,2 @@ +[flake8] +exclude = .tox,.venv,build,dist,doc diff --git a/smmap/setup.py b/smmap/setup.py new file mode 100755 index 000000000..585f0e76c --- /dev/null +++ b/smmap/setup.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python +import os + +try: + from setuptools import setup, find_packages +except ImportError: + from ez_setup import use_setuptools + use_setuptools() + from setuptools import setup, find_packages + +import smmap + +if os.path.exists("README.md"): + with open("README.md", encoding="utf-8") as readme: + long_description = readme.read().replace("\r\n", "\n") +else: + long_description = "See https://github.com/gitpython-developers/GitPython/tree/main/smmap" + +setup( + name="smmap", + version=smmap.__version__, + description="A pure Python implementation of a sliding window memory map manager", + author=smmap.__author__, + author_email=smmap.__contact__, + url=smmap.__homepage__, + platforms=["any"], + license="BSD-3-Clause", + packages=find_packages(), + zip_safe=True, + python_requires=">=3.7", + classifiers=[ + "Development Status :: 5 - Production/Stable", + "Environment :: Console", + "Intended Audience :: Developers", + "Operating System :: OS Independent", + "Operating System :: POSIX", + "Operating System :: Microsoft :: Windows", + "Operating System :: MacOS :: MacOS X", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3 :: Only", + ], + long_description=long_description, + long_description_content_type='text/markdown', +) diff --git a/smmap/smmap/__init__.py b/smmap/smmap/__init__.py new file mode 100644 index 000000000..79ec0674d --- /dev/null +++ b/smmap/smmap/__init__.py @@ -0,0 +1,11 @@ +"""Initialize the smmap package""" + +__author__ = "Sebastian Thiel" +__contact__ = "byronimo@gmail.com" +__homepage__ = "https://github.com/gitpython-developers/GitPython/tree/main/smmap" +version_info = (5, 0, 3) +__version__ = '.'.join(str(i) for i in version_info) + +# make everything available in root package for convenience +from .mman import * +from .buf import * diff --git a/smmap/smmap/buf.py b/smmap/smmap/buf.py new file mode 100644 index 000000000..731b0644b --- /dev/null +++ b/smmap/smmap/buf.py @@ -0,0 +1,143 @@ +"""Module with a simple buffer implementation using the memory manager""" +import sys + +__all__ = ["SlidingWindowMapBuffer"] + + +class SlidingWindowMapBuffer: + + """A buffer like object which allows direct byte-wise object and slicing into + memory of a mapped file. The mapping is controlled by the provided cursor. + + The buffer is relative, that is if you map an offset, index 0 will map to the + first byte at the offset you used during initialization or begin_access + + **Note:** Although this type effectively hides the fact that there are mapped windows + underneath, it can unfortunately not be used in any non-pure python method which + needs a buffer or string""" + __slots__ = ( + '_c', # our cursor + '_size', # our supposed size + ) + + def __init__(self, cursor=None, offset=0, size=sys.maxsize, flags=0): + """Initialize the instance to operate on the given cursor. + :param cursor: if not None, the associated cursor to the file you want to access + If None, you have call begin_access before using the buffer and provide a cursor + :param offset: absolute offset in bytes + :param size: the total size of the mapping. Defaults to the maximum possible size + From that point on, the __len__ of the buffer will be the given size or the file size. + If the size is larger than the mappable area, you can only access the actually available + area, although the length of the buffer is reported to be your given size. + Hence it is in your own interest to provide a proper size ! + :param flags: Additional flags to be passed to os.open + :raise ValueError: if the buffer could not achieve a valid state""" + self._c = cursor + if cursor and not self.begin_access(cursor, offset, size, flags): + raise ValueError("Failed to allocate the buffer - probably the given offset is out of bounds") + # END handle offset + + def __del__(self): + self.end_access() + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, traceback): + self.end_access() + + def __len__(self): + return self._size + + def __getitem__(self, i): + if isinstance(i, slice): + return self.__getslice__(i.start or 0, i.stop or self._size) + c = self._c + assert c.is_valid() + if i < 0: + i = self._size + i + if not c.includes_ofs(i): + c.use_region(i, 1) + # END handle region usage + return c.buffer()[i - c.ofs_begin()] + + def __getslice__(self, i, j): + c = self._c + # fast path, slice fully included - safes a concatenate operation and + # should be the default + assert c.is_valid() + if i < 0: + i = self._size + i + if j == sys.maxsize: + j = self._size + if j < 0: + j = self._size + j + if (c.ofs_begin() <= i) and (j < c.ofs_end()): + b = c.ofs_begin() + return c.buffer()[i - b:j - b] + else: + l = j - i # total length + ofs = i + # It's fastest to keep tokens and join later, especially in py3, which was 7 times slower + # in the previous iteration of this code + md = list() + while l: + c.use_region(ofs, l) + assert c.is_valid() + d = c.buffer()[:l] + ofs += len(d) + l -= len(d) + # Make sure we don't keep references, as c.use_region() might attempt to free resources, but + # can't unless we use pure bytes + if hasattr(d, 'tobytes'): + d = d.tobytes() + md.append(d) + # END while there are bytes to read + return b''.join(md) + # END fast or slow path + #{ Interface + + def begin_access(self, cursor=None, offset=0, size=sys.maxsize, flags=0): + """Call this before the first use of this instance. The method was already + called by the constructor in case sufficient information was provided. + + For more information no the parameters, see the __init__ method + :param path: if cursor is None the existing one will be used. + :return: True if the buffer can be used""" + if cursor: + self._c = cursor + # END update our cursor + + # reuse existing cursors if possible + if self._c is not None and self._c.is_associated(): + res = self._c.use_region(offset, size, flags).is_valid() + if res: + # if given size is too large or default, we computer a proper size + # If its smaller, we assume the combination between offset and size + # as chosen by the user is correct and use it ! + # If not, the user is in trouble. + if size > self._c.file_size(): + size = self._c.file_size() - offset + # END handle size + self._size = size + # END set size + return res + # END use our cursor + return False + + def end_access(self): + """Call this method once you are done using the instance. It is automatically + called on destruction, and should be called just in time to allow system + resources to be freed. + + Once you called end_access, you must call begin access before reusing this instance!""" + self._size = 0 + if self._c is not None: + self._c.unuse_region() + # END unuse region + + def cursor(self): + """:return: the currently set cursor which provides access to the data""" + return self._c + + #}END interface diff --git a/smmap/smmap/mman.py b/smmap/smmap/mman.py new file mode 100644 index 000000000..873f687fd --- /dev/null +++ b/smmap/smmap/mman.py @@ -0,0 +1,588 @@ +"""Module containing a memory memory manager which provides a sliding window on a number of memory mapped files""" +from .util import ( + MapWindow, + MapRegion, + MapRegionList, + is_64_bit, +) + +import sys +from functools import reduce + +__all__ = ["StaticWindowMapManager", "SlidingWindowMapManager", "WindowCursor"] +#{ Utilities + +#}END utilities + + +class WindowCursor: + + """ + Pointer into the mapped region of the memory manager, keeping the map + alive until it is destroyed and no other client uses it. + + Cursors should not be created manually, but are instead returned by the SlidingWindowMapManager + + **Note:**: The current implementation is suited for static and sliding window managers, but it also means + that it must be suited for the somewhat quite different sliding manager. It could be improved, but + I see no real need to do so.""" + __slots__ = ( + '_manager', # the manager keeping all file regions + '_rlist', # a regions list with regions for our file + '_region', # our current class:`MapRegion` or None + '_ofs', # relative offset from the actually mapped area to our start area + '_size' # maximum size we should provide + ) + + def __init__(self, manager=None, regions=None): + self._manager = manager + self._rlist = regions + self._region = None + self._ofs = 0 + self._size = 0 + + def __del__(self): + self._destroy() + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, traceback): + self._destroy() + + def _destroy(self): + """Destruction code to decrement counters""" + self.unuse_region() + + if self._rlist is not None: + # Actual client count, which doesn't include the reference kept by the manager, nor ours + # as we are about to be deleted + try: + if len(self._rlist) == 0: + # Free all resources associated with the mapped file + self._manager._fdict.pop(self._rlist.path_or_fd()) + # END remove regions list from manager + except (TypeError, KeyError): + # sometimes, during shutdown, getrefcount is None. Its possible + # to re-import it, however, its probably better to just ignore + # this python problem (for now). + # The next step is to get rid of the error prone getrefcount altogether. + pass + # END exception handling + # END handle regions + + def _copy_from(self, rhs): + """Copy all data from rhs into this instance, handles usage count""" + self._manager = rhs._manager + self._rlist = type(rhs._rlist)(rhs._rlist) + self._region = rhs._region + self._ofs = rhs._ofs + self._size = rhs._size + + for region in self._rlist: + region.increment_client_count() + + if self._region is not None: + self._region.increment_client_count() + # END handle regions + + def __copy__(self): + """copy module interface""" + cpy = type(self)() + cpy._copy_from(self) + return cpy + + #{ Interface + def assign(self, rhs): + """Assign rhs to this instance. This is required in order to get a real copy. + Alternatively, you can copy an existing instance using the copy module""" + self._destroy() + self._copy_from(rhs) + + def use_region(self, offset=0, size=0, flags=0): + """Assure we point to a window which allows access to the given offset into the file + + :param offset: absolute offset in bytes into the file + :param size: amount of bytes to map. If 0, all available bytes will be mapped + :param flags: additional flags to be given to os.open in case a file handle is initially opened + for mapping. Has no effect if a region can actually be reused. + :return: this instance - it should be queried for whether it points to a valid memory region. + This is not the case if the mapping failed because we reached the end of the file + + **Note:**: The size actually mapped may be smaller than the given size. If that is the case, + either the file has reached its end, or the map was created between two existing regions""" + need_region = True + man = self._manager + fsize = self._rlist.file_size() + size = min(size or fsize, man.window_size() or fsize) # clamp size to window size + + if self._region is not None: + if self._region.includes_ofs(offset): + need_region = False + else: + self.unuse_region() + # END handle existing region + # END check existing region + + # offset too large ? + if offset >= fsize: + return self + # END handle offset + + if need_region: + self._region = man._obtain_region(self._rlist, offset, size, flags, False) + self._region.increment_client_count() + # END need region handling + + self._ofs = offset - self._region._b + self._size = min(size, self._region.ofs_end() - offset) + + return self + + def unuse_region(self): + """Unuse the current region. Does nothing if we have no current region + + **Note:** the cursor unuses the region automatically upon destruction. It is recommended + to un-use the region once you are done reading from it in persistent cursors as it + helps to free up resource more quickly""" + if self._region is not None: + self._region.increment_client_count(-1) + self._region = None + # note: should reset ofs and size, but we spare that for performance. Its not + # allowed to query information if we are not valid ! + + def buffer(self): + """Return a buffer object which allows access to our memory region from our offset + to the window size. Please note that it might be smaller than you requested when calling use_region() + + **Note:** You can only obtain a buffer if this instance is_valid() ! + + **Note:** buffers should not be cached passed the duration of your access as it will + prevent resources from being freed even though they might not be accounted for anymore !""" + return memoryview(self._region.buffer())[self._ofs:self._ofs+self._size] + + def map(self): + """ + :return: the underlying raw memory map. Please not that the offset and size is likely to be different + to what you set as offset and size. Use it only if you are sure about the region it maps, which is the whole + file in case of StaticWindowMapManager""" + return self._region.map() + + def is_valid(self): + """:return: True if we have a valid and usable region""" + return self._region is not None + + def is_associated(self): + """:return: True if we are associated with a specific file already""" + return self._rlist is not None + + def ofs_begin(self): + """:return: offset to the first byte pointed to by our cursor + + **Note:** only if is_valid() is True""" + return self._region._b + self._ofs + + def ofs_end(self): + """:return: offset to one past the last available byte""" + # unroll method calls for performance ! + return self._region._b + self._ofs + self._size + + def size(self): + """:return: amount of bytes we point to""" + return self._size + + def region(self): + """:return: our mapped region, or None if nothing is mapped yet + :raise AssertionError: if we have no current region. This is only useful for debugging""" + return self._region + + def includes_ofs(self, ofs): + """:return: True if the given absolute offset is contained in the cursors + current region + + **Note:** cursor must be valid for this to work""" + # unroll methods + return (self._region._b + self._ofs) <= ofs < (self._region._b + self._ofs + self._size) + + def file_size(self): + """:return: size of the underlying file""" + return self._rlist.file_size() + + def path_or_fd(self): + """:return: path or file descriptor of the underlying mapped file""" + return self._rlist.path_or_fd() + + def path(self): + """:return: path of the underlying mapped file + :raise ValueError: if attached path is not a path""" + if isinstance(self._rlist.path_or_fd(), int): + raise ValueError("Path queried although mapping was applied to a file descriptor") + # END handle type + return self._rlist.path_or_fd() + + def fd(self): + """:return: file descriptor used to create the underlying mapping. + + **Note:** it is not required to be valid anymore + :raise ValueError: if the mapping was not created by a file descriptor""" + if isinstance(self._rlist.path_or_fd(), str): + raise ValueError("File descriptor queried although mapping was generated from path") + # END handle type + return self._rlist.path_or_fd() + + #} END interface + + +class StaticWindowMapManager: + + """Provides a manager which will produce single size cursors that are allowed + to always map the whole file. + + Clients must be written to specifically know that they are accessing their data + through a StaticWindowMapManager, as they otherwise have to deal with their window size. + + These clients would have to use a SlidingWindowMapBuffer to hide this fact. + + This type will always use a maximum window size, and optimize certain methods to + accommodate this fact""" + + __slots__ = [ + '_fdict', # mapping of path -> StorageHelper (of some kind + '_window_size', # maximum size of a window + '_max_memory_size', # maximum amount of memory we may allocate + '_max_handle_count', # maximum amount of handles to keep open + '_memory_size', # currently allocated memory size + '_handle_count', # amount of currently allocated file handles + ] + + #{ Configuration + MapRegionListCls = MapRegionList + MapWindowCls = MapWindow + MapRegionCls = MapRegion + WindowCursorCls = WindowCursor + #} END configuration + + _MB_in_bytes = 1024 * 1024 + + def __init__(self, window_size=0, max_memory_size=0, max_open_handles=sys.maxsize): + """initialize the manager with the given parameters. + :param window_size: if -1, a default window size will be chosen depending on + the operating system's architecture. It will internally be quantified to a multiple of the page size + If 0, the window may have any size, which basically results in mapping the whole file at one + :param max_memory_size: maximum amount of memory we may map at once before releasing mapped regions. + If 0, a viable default will be set depending on the system's architecture. + It is a soft limit that is tried to be kept, but nothing bad happens if we have to over-allocate + :param max_open_handles: if not maxint, limit the amount of open file handles to the given number. + Otherwise the amount is only limited by the system itself. If a system or soft limit is hit, + the manager will free as many handles as possible""" + self._fdict = dict() + self._window_size = window_size + self._max_memory_size = max_memory_size + self._max_handle_count = max_open_handles + self._memory_size = 0 + self._handle_count = 0 + + if window_size < 0: + coeff = 64 + if is_64_bit(): + coeff = 1024 + # END handle arch + self._window_size = coeff * self._MB_in_bytes + # END handle max window size + + if max_memory_size == 0: + coeff = 1024 + if is_64_bit(): + coeff = 8192 + # END handle arch + self._max_memory_size = coeff * self._MB_in_bytes + # END handle max memory size + + #{ Internal Methods + + def _collect_lru_region(self, size): + """Unmap the region which was least-recently used and has no client + :param size: size of the region we want to map next (assuming its not already mapped partially or full + if 0, we try to free any available region + :return: Amount of freed regions + + .. Note:: + We don't raise exceptions anymore, in order to keep the system working, allowing temporary overallocation. + If the system runs out of memory, it will tell. + + .. TODO:: + implement a case where all unusued regions are discarded efficiently. + Currently its only brute force + """ + num_found = 0 + while (size == 0) or (self._memory_size + size > self._max_memory_size): + lru_region = None + lru_list = None + for regions in self._fdict.values(): + for region in regions: + # check client count - if it's 1, it's just us + if (region.client_count() == 1 and + (lru_region is None or region._uc < lru_region._uc)): + lru_region = region + lru_list = regions + # END update lru_region + # END for each region + # END for each regions list + + if lru_region is None: + break + # END handle region not found + + num_found += 1 + del(lru_list[lru_list.index(lru_region)]) + lru_region.increment_client_count(-1) + self._memory_size -= lru_region.size() + self._handle_count -= 1 + # END while there is more memory to free + return num_found + + def _obtain_region(self, a, offset, size, flags, is_recursive): + """Utility to create a new region - for more information on the parameters, + see MapCursor.use_region. + :param a: A regions (a)rray + :return: The newly created region""" + if self._memory_size + size > self._max_memory_size: + self._collect_lru_region(size) + # END handle collection + + r = None + if a: + assert len(a) == 1 + r = a[0] + else: + try: + r = self.MapRegionCls(a.path_or_fd(), 0, sys.maxsize, flags) + except Exception: + # apparently we are out of system resources or hit a limit + # As many more operations are likely to fail in that condition ( + # like reading a file from disk, etc) we free up as much as possible + # As this invalidates our insert position, we have to recurse here + if is_recursive: + # we already tried this, and still have no success in obtaining + # a mapping. This is an exception, so we propagate it + raise + # END handle existing recursion + self._collect_lru_region(0) + return self._obtain_region(a, offset, size, flags, True) + # END handle exceptions + + self._handle_count += 1 + self._memory_size += r.size() + a.append(r) + # END handle array + + assert r.includes_ofs(offset) + return r + + #}END internal methods + + #{ Interface + def make_cursor(self, path_or_fd): + """ + :return: a cursor pointing to the given path or file descriptor. + It can be used to map new regions of the file into memory + + **Note:** if a file descriptor is given, it is assumed to be open and valid, + but may be closed afterwards. To refer to the same file, you may reuse + your existing file descriptor, but keep in mind that new windows can only + be mapped as long as it stays valid. This is why the using actual file paths + are preferred unless you plan to keep the file descriptor open. + + **Note:** file descriptors are problematic as they are not necessarily unique, as two + different files opened and closed in succession might have the same file descriptor id. + + **Note:** Using file descriptors directly is faster once new windows are mapped as it + prevents the file to be opened again just for the purpose of mapping it.""" + regions = self._fdict.get(path_or_fd) + if regions is None: + regions = self.MapRegionListCls(path_or_fd) + self._fdict[path_or_fd] = regions + # END obtain region for path + return self.WindowCursorCls(self, regions) + + def collect(self): + """Collect all available free-to-collect mapped regions + :return: Amount of freed handles""" + return self._collect_lru_region(0) + + def num_file_handles(self): + """:return: amount of file handles in use. Each mapped region uses one file handle""" + return self._handle_count + + def num_open_files(self): + """Amount of opened files in the system""" + return reduce(lambda x, y: x + y, (1 for rlist in self._fdict.values() if len(rlist) > 0), 0) + + def window_size(self): + """:return: size of each window when allocating new regions""" + return self._window_size + + def mapped_memory_size(self): + """:return: amount of bytes currently mapped in total""" + return self._memory_size + + def max_file_handles(self): + """:return: maximum amount of handles we may have opened""" + return self._max_handle_count + + def max_mapped_memory_size(self): + """:return: maximum amount of memory we may allocate""" + return self._max_memory_size + + #} END interface + + #{ Special Purpose Interface + + def force_map_handle_removal_win(self, base_path): + """ONLY AVAILABLE ON WINDOWS + On windows removing files is not allowed if anybody still has it opened. + If this process is ourselves, and if the whole process uses this memory + manager (as far as the parent framework is concerned) we can enforce + closing all memory maps whose path matches the given base path to + allow the respective operation after all. + The respective system must NOT access the closed memory regions anymore ! + This really may only be used if you know that the items which keep + the cursors alive will not be using it anymore. They need to be recreated ! + :return: Amount of closed handles + + **Note:** does nothing on non-windows platforms""" + if sys.platform != 'win32': + return + # END early bailout + + num_closed = 0 + for path, rlist in self._fdict.items(): + if path.startswith(base_path): + for region in rlist: + region.release() + num_closed += 1 + # END path matches + # END for each path + return num_closed + #} END special purpose interface + + +class SlidingWindowMapManager(StaticWindowMapManager): + + """Maintains a list of ranges of mapped memory regions in one or more files and allows to easily + obtain additional regions assuring there is no overlap. + Once a certain memory limit is reached globally, or if there cannot be more open file handles + which result from each mmap call, the least recently used, and currently unused mapped regions + are unloaded automatically. + + **Note:** currently not thread-safe ! + + **Note:** in the current implementation, we will automatically unload windows if we either cannot + create more memory maps (as the open file handles limit is hit) or if we have allocated more than + a safe amount of memory already, which would possibly cause memory allocations to fail as our address + space is full.""" + + __slots__ = tuple() + + def __init__(self, window_size=-1, max_memory_size=0, max_open_handles=sys.maxsize): + """Adjusts the default window size to -1""" + super().__init__(window_size, max_memory_size, max_open_handles) + + def _obtain_region(self, a, offset, size, flags, is_recursive): + # bisect to find an existing region. The c++ implementation cannot + # do that as it uses a linked list for regions. + r = None + lo = 0 + hi = len(a) + while lo < hi: + mid = (lo + hi) // 2 + ofs = a[mid]._b + if ofs <= offset: + if a[mid].includes_ofs(offset): + r = a[mid] + break + # END have region + lo = mid + 1 + else: + hi = mid + # END handle position + # END while bisecting + + if r is None: + window_size = self._window_size + left = self.MapWindowCls(0, 0) + mid = self.MapWindowCls(offset, size) + right = self.MapWindowCls(a.file_size(), 0) + + # we want to honor the max memory size, and assure we have anough + # memory available + # Save calls ! + if self._memory_size + window_size > self._max_memory_size: + self._collect_lru_region(window_size) + # END handle collection + + # we assume the list remains sorted by offset + insert_pos = 0 + len_regions = len(a) + if len_regions == 1: + if a[0]._b <= offset: + insert_pos = 1 + # END maintain sort + else: + # find insert position + insert_pos = len_regions + for i, region in enumerate(a): + if region._b > offset: + insert_pos = i + break + # END if insert position is correct + # END for each region + # END obtain insert pos + + # adjust the actual offset and size values to create the largest + # possible mapping + if insert_pos == 0: + if len_regions: + right = self.MapWindowCls.from_region(a[insert_pos]) + # END adjust right side + else: + if insert_pos != len_regions: + right = self.MapWindowCls.from_region(a[insert_pos]) + # END adjust right window + left = self.MapWindowCls.from_region(a[insert_pos - 1]) + # END adjust surrounding windows + + mid.extend_left_to(left, window_size) + mid.extend_right_to(right, window_size) + mid.align() + + # it can happen that we align beyond the end of the file + if mid.ofs_end() > right.ofs: + mid.size = right.ofs - mid.ofs + # END readjust size + + # insert new region at the right offset to keep the order + try: + if self._handle_count >= self._max_handle_count: + raise Exception + # END assert own imposed max file handles + r = self.MapRegionCls(a.path_or_fd(), mid.ofs, mid.size, flags) + except Exception: + # apparently we are out of system resources or hit a limit + # As many more operations are likely to fail in that condition ( + # like reading a file from disk, etc) we free up as much as possible + # As this invalidates our insert position, we have to recurse here + if is_recursive: + # we already tried this, and still have no success in obtaining + # a mapping. This is an exception, so we propagate it + raise + # END handle existing recursion + self._collect_lru_region(0) + return self._obtain_region(a, offset, size, flags, True) + # END handle exceptions + + self._handle_count += 1 + self._memory_size += r.size() + a.insert(insert_pos, r) + # END create new region + return r diff --git a/smmap/smmap/test/__init__.py b/smmap/smmap/test/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/smmap/smmap/test/lib.py b/smmap/smmap/test/lib.py new file mode 100644 index 000000000..b15b0ec6a --- /dev/null +++ b/smmap/smmap/test/lib.py @@ -0,0 +1,72 @@ +"""Provide base classes for the test system""" +from unittest import TestCase +import os +import tempfile + +__all__ = ['TestBase', 'FileCreator'] + + +#{ Utilities + +class FileCreator: + + """A instance which creates a temporary file with a prefix and a given size + and provides this info to the user. + Once it gets deleted, it will remove the temporary file as well.""" + __slots__ = ("_size", "_path") + + def __init__(self, size, prefix=''): + assert size, "Require size to be larger 0" + + self._size = size + + with tempfile.NamedTemporaryFile("wb", prefix=prefix, delete=False) as file: + self._path = file.name + file.seek(size - 1) + file.write(b'1') + + assert os.path.getsize(self.path) == size + + def __del__(self): + try: + os.remove(self.path) + except OSError: + pass + # END exception handling + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, traceback): + self.__del__() + + @property + def path(self): + return self._path + + @property + def size(self): + return self._size + +#} END utilities + + +class TestBase(TestCase): + + """Foundation used by all tests""" + + #{ Configuration + k_window_test_size = 1000 * 1000 * 8 + 5195 + #} END configuration + + #{ Overrides + @classmethod + def setUpAll(cls): + # nothing for now + pass + + # END overrides + + #{ Interface + + #} END interface diff --git a/smmap/smmap/test/test_buf.py b/smmap/smmap/test/test_buf.py new file mode 100644 index 000000000..f0a86fb64 --- /dev/null +++ b/smmap/smmap/test/test_buf.py @@ -0,0 +1,126 @@ +from .lib import TestBase, FileCreator + +from smmap.mman import ( + SlidingWindowMapManager, + StaticWindowMapManager +) +from smmap.buf import SlidingWindowMapBuffer + +from random import randint +from time import time +import sys +import os + + +man_optimal = SlidingWindowMapManager() +man_worst_case = SlidingWindowMapManager( + window_size=TestBase.k_window_test_size // 100, + max_memory_size=TestBase.k_window_test_size // 3, + max_open_handles=15) +static_man = StaticWindowMapManager() + + +class TestBuf(TestBase): + + def test_basics(self): + with FileCreator(self.k_window_test_size, "buffer_test") as fc: + + # invalid paths fail upon construction + c = man_optimal.make_cursor(fc.path) + self.assertRaises(ValueError, SlidingWindowMapBuffer, type(c)()) # invalid cursor + self.assertRaises(ValueError, SlidingWindowMapBuffer, c, fc.size) # offset too large + + buf = SlidingWindowMapBuffer() # can create uninitialized buffers + assert buf.cursor() is None + + # can call end access any time + buf.end_access() + buf.end_access() + assert len(buf) == 0 + + # begin access can revive it, if the offset is suitable + offset = 100 + assert buf.begin_access(c, fc.size) == False + assert buf.begin_access(c, offset) == True + assert len(buf) == fc.size - offset + assert buf.cursor().is_valid() + + # empty begin access keeps it valid on the same path, but alters the offset + assert buf.begin_access() == True + assert len(buf) == fc.size + assert buf.cursor().is_valid() + + # simple access + with open(fc.path, 'rb') as fp: + data = fp.read() + assert data[offset] == buf[0] + assert data[offset:offset * 2] == buf[0:offset] + + # negative indices, partial slices + assert buf[-1] == buf[len(buf) - 1] + assert buf[-10:] == buf[len(buf) - 10:len(buf)] + + # end access makes its cursor invalid + buf.end_access() + assert not buf.cursor().is_valid() + assert buf.cursor().is_associated() # but it remains associated + + # an empty begin access fixes it up again + assert buf.begin_access() == True and buf.cursor().is_valid() + del(buf) # ends access automatically + del(c) + + assert man_optimal.num_file_handles() == 1 + + # PERFORMANCE + # blast away with random access and a full mapping - we don't want to + # exaggerate the manager's overhead, but measure the buffer overhead + # We do it once with an optimal setting, and with a worse manager which + # will produce small mappings only ! + max_num_accesses = 100 + fd = os.open(fc.path, os.O_RDONLY) + for item in (fc.path, fd): + for manager, man_id in ((man_optimal, 'optimal'), + (man_worst_case, 'worst case'), + (static_man, 'static optimal')): + buf = SlidingWindowMapBuffer(manager.make_cursor(item)) + assert manager.num_file_handles() == 1 + for access_mode in range(2): # single, multi + num_accesses_left = max_num_accesses + num_bytes = 0 + fsize = fc.size + + st = time() + buf.begin_access() + while num_accesses_left: + num_accesses_left -= 1 + if access_mode: # multi + ofs_start = randint(0, fsize) + ofs_end = randint(ofs_start, fsize) + d = buf[ofs_start:ofs_end] + assert len(d) == ofs_end - ofs_start + assert d == data[ofs_start:ofs_end] + num_bytes += len(d) + del d + else: + pos = randint(0, fsize) + assert buf[pos] == data[pos] + num_bytes += 1 + # END handle mode + # END handle num accesses + + buf.end_access() + assert manager.num_file_handles() + assert manager.collect() + assert manager.num_file_handles() == 0 + elapsed = max(time() - st, 0.001) # prevent zero division errors on windows + mb = float(1000 * 1000) + mode_str = (access_mode and "slice") or "single byte" + print("%s: Made %i random %s accesses to buffer created from %s reading a total of %f mb in %f s (%f mb/s)" + % (man_id, max_num_accesses, mode_str, type(item), num_bytes / mb, elapsed, (num_bytes / mb) / elapsed), + file=sys.stderr) + # END handle access mode + del buf + # END for each manager + # END for each input + os.close(fd) diff --git a/smmap/smmap/test/test_mman.py b/smmap/smmap/test/test_mman.py new file mode 100644 index 000000000..7a5f4092c --- /dev/null +++ b/smmap/smmap/test/test_mman.py @@ -0,0 +1,224 @@ +from .lib import TestBase, FileCreator + +from smmap.mman import ( + WindowCursor, + SlidingWindowMapManager, + StaticWindowMapManager +) +from smmap.util import align_to_mmap + +from random import randint +from time import time +import os +import sys +from copy import copy + + +class TestMMan(TestBase): + + def test_cursor(self): + with FileCreator(self.k_window_test_size, "cursor_test") as fc: + man = SlidingWindowMapManager() + ci = WindowCursor(man) # invalid cursor + assert not ci.is_valid() + assert not ci.is_associated() + assert ci.size() == 0 # this is cached, so we can query it in invalid state + + cv = man.make_cursor(fc.path) + assert not cv.is_valid() # no region mapped yet + assert cv.is_associated() # but it know where to map it from + assert cv.file_size() == fc.size + assert cv.path() == fc.path + + # copy module + cio = copy(cv) + assert not cio.is_valid() and cio.is_associated() + + # assign method + assert not ci.is_associated() + ci.assign(cv) + assert not ci.is_valid() and ci.is_associated() + + # unuse non-existing region is fine + cv.unuse_region() + cv.unuse_region() + + # destruction is fine (even multiple times) + cv._destroy() + WindowCursor(man)._destroy() + + def test_memory_manager(self): + slide_man = SlidingWindowMapManager() + static_man = StaticWindowMapManager() + + for man in (static_man, slide_man): + assert man.num_file_handles() == 0 + assert man.num_open_files() == 0 + winsize_cmp_val = 0 + if isinstance(man, StaticWindowMapManager): + winsize_cmp_val = -1 + # END handle window size + assert man.window_size() > winsize_cmp_val + assert man.mapped_memory_size() == 0 + assert man.max_mapped_memory_size() > 0 + + # collection doesn't raise in 'any' mode + man._collect_lru_region(0) + # doesn't raise if we are within the limit + man._collect_lru_region(10) + + # doesn't fail if we over-allocate + assert man._collect_lru_region(sys.maxsize) == 0 + + # use a region, verify most basic functionality + with FileCreator(self.k_window_test_size, "manager_test") as fc: + fd = os.open(fc.path, os.O_RDONLY) + try: + for item in (fc.path, fd): + c = man.make_cursor(item) + assert c.path_or_fd() is item + assert c.use_region(10, 10).is_valid() + assert c.ofs_begin() == 10 + assert c.size() == 10 + with open(fc.path, 'rb') as fp: + assert c.buffer()[:] == fp.read(20)[10:] + + if isinstance(item, int): + self.assertRaises(ValueError, c.path) + else: + self.assertRaises(ValueError, c.fd) + # END handle value error + # END for each input + finally: + os.close(fd) + # END for each manasger type + + def test_memman_operation(self): + # test more access, force it to actually unmap regions + with FileCreator(self.k_window_test_size, "manager_operation_test") as fc: + with open(fc.path, 'rb') as fp: + data = fp.read() + fd = os.open(fc.path, os.O_RDONLY) + try: + max_num_handles = 15 + # small_size = + for mtype, args in ((StaticWindowMapManager, (0, fc.size // 3, max_num_handles)), + (SlidingWindowMapManager, (fc.size // 100, fc.size // 3, max_num_handles)),): + for item in (fc.path, fd): + assert len(data) == fc.size + + # small windows, a reasonable max memory. Not too many regions at once + man = mtype(window_size=args[0], max_memory_size=args[1], max_open_handles=args[2]) + c = man.make_cursor(item) + + # still empty (more about that is tested in test_memory_manager() + assert man.num_open_files() == 0 + assert man.mapped_memory_size() == 0 + + base_offset = 5000 + # window size is 0 for static managers, hence size will be 0. We take that into consideration + size = man.window_size() // 2 + assert c.use_region(base_offset, size).is_valid() + rr = c.region() + assert rr.client_count() == 2 # the manager and the cursor and us + + assert man.num_open_files() == 1 + assert man.num_file_handles() == 1 + assert man.mapped_memory_size() == rr.size() + + # assert c.size() == size # the cursor may overallocate in its static version + assert c.ofs_begin() == base_offset + assert rr.ofs_begin() == 0 # it was aligned and expanded + if man.window_size(): + # but isn't larger than the max window (aligned) + assert rr.size() == align_to_mmap(man.window_size(), True) + else: + assert rr.size() == fc.size + # END ignore static managers which dont use windows and are aligned to file boundaries + + assert c.buffer()[:] == data[base_offset:base_offset + (size or c.size())] + + # obtain second window, which spans the first part of the file - it is a still the same window + nsize = (size or fc.size) - 10 + assert c.use_region(0, nsize).is_valid() + assert c.region() == rr + assert man.num_file_handles() == 1 + assert c.size() == nsize + assert c.ofs_begin() == 0 + assert c.buffer()[:] == data[:nsize] + + # map some part at the end, our requested size cannot be kept + overshoot = 4000 + base_offset = fc.size - (size or c.size()) + overshoot + assert c.use_region(base_offset, size).is_valid() + if man.window_size(): + assert man.num_file_handles() == 2 + assert c.size() < size + assert c.region() is not rr # old region is still available, but has not cursor ref anymore + assert rr.client_count() == 1 # only held by manager + else: + assert c.size() < fc.size + # END ignore static managers which only have one handle per file + rr = c.region() + assert rr.client_count() == 2 # manager + cursor + assert rr.ofs_begin() < c.ofs_begin() # it should have extended itself to the left + assert rr.ofs_end() <= fc.size # it cannot be larger than the file + assert c.buffer()[:] == data[base_offset:base_offset + (size or c.size())] + + # unising a region makes the cursor invalid + c.unuse_region() + assert not c.is_valid() + if man.window_size(): + # but doesn't change anything regarding the handle count - we cache it and only + # remove mapped regions if we have to + assert man.num_file_handles() == 2 + # END ignore this for static managers + + # iterate through the windows, verify data contents + # this will trigger map collection after a while + max_random_accesses = 5000 + num_random_accesses = max_random_accesses + memory_read = 0 + st = time() + + # cache everything to get some more performance + includes_ofs = c.includes_ofs + max_mapped_memory_size = man.max_mapped_memory_size() + max_file_handles = man.max_file_handles() + mapped_memory_size = man.mapped_memory_size + num_file_handles = man.num_file_handles + while num_random_accesses: + num_random_accesses -= 1 + base_offset = randint(0, fc.size - 1) + + # precondition + if man.window_size(): + assert max_mapped_memory_size >= mapped_memory_size() + # END statistics will overshoot, which is fine + assert max_file_handles >= num_file_handles() + assert c.use_region(base_offset, (size or c.size())).is_valid() + csize = c.size() + assert c.buffer()[:] == data[base_offset:base_offset + csize] + memory_read += csize + + assert includes_ofs(base_offset) + assert includes_ofs(base_offset + csize - 1) + assert not includes_ofs(base_offset + csize) + # END while we should do an access + elapsed = max(time() - st, 0.001) # prevent zero division errors on windows + mb = float(1000 * 1000) + print("%s: Read %i mb of memory with %i random on cursor initialized with %s accesses in %fs (%f mb/s)\n" + % (mtype, memory_read / mb, max_random_accesses, type(item), elapsed, (memory_read / mb) / elapsed), + file=sys.stderr) + + # an offset as large as the size doesn't work ! + assert not c.use_region(fc.size, size).is_valid() + + # collection - it should be able to collect all + assert man.num_file_handles() + assert man.collect() + assert man.num_file_handles() == 0 + # END for each item + # END for each manager type + finally: + os.close(fd) diff --git a/smmap/smmap/test/test_tutorial.py b/smmap/smmap/test/test_tutorial.py new file mode 100644 index 000000000..31c272abb --- /dev/null +++ b/smmap/smmap/test/test_tutorial.py @@ -0,0 +1,75 @@ +from .lib import TestBase + + +class TestTutorial(TestBase): + + def test_example(self): + # Memory Managers + ################## + import smmap + # This instance should be globally available in your application + # It is configured to be well suitable for 32-bit or 64 bit applications. + mman = smmap.SlidingWindowMapManager() + + # the manager provides much useful information about its current state + # like the amount of open file handles or the amount of mapped memory + assert mman.num_file_handles() == 0 + assert mman.mapped_memory_size() == 0 + # and many more ... + + # Cursors + ########## + import smmap.test.lib + with smmap.test.lib.FileCreator(1024 * 1024 * 8, "test_file") as fc: + # obtain a cursor to access some file. + c = mman.make_cursor(fc.path) + + # the cursor is now associated with the file, but not yet usable + assert c.is_associated() + assert not c.is_valid() + + # before you can use the cursor, you have to specify a window you want to + # access. The following just says you want as much data as possible starting + # from offset 0. + # To be sure your region could be mapped, query for validity + assert c.use_region().is_valid() # use_region returns self + + # once a region was mapped, you must query its dimension regularly + # to assure you don't try to access its buffer out of its bounds + assert c.size() + c.buffer()[0] # first byte + c.buffer()[1:10] # first 9 bytes + c.buffer()[c.size() - 1] # last byte + + # you can query absolute offsets, and check whether an offset is included + # in the cursor's data. + assert c.ofs_begin() < c.ofs_end() + assert c.includes_ofs(100) + + # If you are over out of bounds with one of your region requests, the + # cursor will be come invalid. It cannot be used in that state + assert not c.use_region(fc.size, 100).is_valid() + # map as much as possible after skipping the first 100 bytes + assert c.use_region(100).is_valid() + + # You can explicitly free cursor resources by unusing the cursor's region + c.unuse_region() + assert not c.is_valid() + + # Buffers + ######### + # Create a default buffer which can operate on the whole file + buf = smmap.SlidingWindowMapBuffer(mman.make_cursor(fc.path)) + + # you can use it right away + assert buf.cursor().is_valid() + + buf[0] # access the first byte + buf[-1] # access the last ten bytes on the file + buf[-10:] # access the last ten bytes + + # If you want to keep the instance between different accesses, use the + # dedicated methods + buf.end_access() + assert not buf.cursor().is_valid() # you cannot use the buffer anymore + assert buf.begin_access(offset=10) # start using the buffer at an offset diff --git a/smmap/smmap/test/test_util.py b/smmap/smmap/test/test_util.py new file mode 100644 index 000000000..e6ac10f35 --- /dev/null +++ b/smmap/smmap/test/test_util.py @@ -0,0 +1,105 @@ +from .lib import TestBase, FileCreator + +from smmap.util import ( + MapWindow, + MapRegion, + MapRegionList, + ALLOCATIONGRANULARITY, + is_64_bit, + align_to_mmap +) + +import os +import sys + + +class TestMMan(TestBase): + + def test_window(self): + wl = MapWindow(0, 1) # left + wc = MapWindow(1, 1) # center + wc2 = MapWindow(10, 5) # another center + wr = MapWindow(8000, 50) # right + + assert wl.ofs_end() == 1 + assert wc.ofs_end() == 2 + assert wr.ofs_end() == 8050 + + # extension does nothing if already in place + maxsize = 100 + wc.extend_left_to(wl, maxsize) + assert wc.ofs == 1 and wc.size == 1 + wl.extend_right_to(wc, maxsize) + wl.extend_right_to(wc, maxsize) + assert wl.ofs == 0 and wl.size == 1 + + # an actual left extension + pofs_end = wc2.ofs_end() + wc2.extend_left_to(wc, maxsize) + assert wc2.ofs == wc.ofs_end() and pofs_end == wc2.ofs_end() + + # respects maxsize + wc.extend_right_to(wr, maxsize) + assert wc.ofs == 1 and wc.size == maxsize + wc.extend_right_to(wr, maxsize) + assert wc.ofs == 1 and wc.size == maxsize + + # without maxsize + wc.extend_right_to(wr, sys.maxsize) + assert wc.ofs_end() == wr.ofs and wc.ofs == 1 + + # extend left + wr.extend_left_to(wc2, maxsize) + wr.extend_left_to(wc2, maxsize) + assert wr.size == maxsize + + wr.extend_left_to(wc2, sys.maxsize) + assert wr.ofs == wc2.ofs_end() + + wc.align() + assert wc.ofs == 0 and wc.size == align_to_mmap(wc.size, True) + + def test_region(self): + with FileCreator(self.k_window_test_size, "window_test") as fc: + half_size = fc.size // 2 + rofs = align_to_mmap(4200, False) + rfull = MapRegion(fc.path, 0, fc.size) + rhalfofs = MapRegion(fc.path, rofs, fc.size) + rhalfsize = MapRegion(fc.path, 0, half_size) + + # offsets + assert rfull.ofs_begin() == 0 and rfull.size() == fc.size + assert rfull.ofs_end() == fc.size # if this method works, it works always + + assert rhalfofs.ofs_begin() == rofs and rhalfofs.size() == fc.size - rofs + assert rhalfsize.ofs_begin() == 0 and rhalfsize.size() == half_size + + assert rfull.includes_ofs(0) and rfull.includes_ofs(fc.size - 1) and rfull.includes_ofs(half_size) + assert not rfull.includes_ofs(-1) and not rfull.includes_ofs(sys.maxsize) + + # auto-refcount + assert rfull.client_count() == 1 + rfull2 = rfull + assert rfull.client_count() == 1, "no auto-counting" + + # window constructor + w = MapWindow.from_region(rfull) + assert w.ofs == rfull.ofs_begin() and w.ofs_end() == rfull.ofs_end() + + def test_region_list(self): + with FileCreator(100, "sample_file") as fc: + fd = os.open(fc.path, os.O_RDONLY) + try: + for item in (fc.path, fd): + ml = MapRegionList(item) + + assert len(ml) == 0 + assert ml.path_or_fd() == item + assert ml.file_size() == fc.size + finally: + os.close(fd) + + def test_util(self): + assert isinstance(is_64_bit(), bool) # just call it + assert align_to_mmap(1, False) == 0 + assert align_to_mmap(1, True) == ALLOCATIONGRANULARITY diff --git a/smmap/smmap/util.py b/smmap/smmap/util.py new file mode 100644 index 000000000..fbb387211 --- /dev/null +++ b/smmap/smmap/util.py @@ -0,0 +1,222 @@ +"""Module containing a memory memory manager which provides a sliding window on a number of memory mapped files""" +import os +import sys + +from mmap import mmap, ACCESS_READ +from mmap import ALLOCATIONGRANULARITY + +__all__ = ["align_to_mmap", "is_64_bit", + "MapWindow", "MapRegion", "MapRegionList", "ALLOCATIONGRANULARITY"] + +#{ Utilities + + +def align_to_mmap(num, round_up): + """ + Align the given integer number to the closest page offset, which usually is 4096 bytes. + + :param round_up: if True, the next higher multiple of page size is used, otherwise + the lower page_size will be used (i.e. if True, 1 becomes 4096, otherwise it becomes 0) + :return: num rounded to closest page""" + res = (num // ALLOCATIONGRANULARITY) * ALLOCATIONGRANULARITY + if round_up and (res != num): + res += ALLOCATIONGRANULARITY + # END handle size + return res + + +def is_64_bit(): + """:return: True if the system is 64 bit. Otherwise it can be assumed to be 32 bit""" + return sys.maxsize > (1 << 32) - 1 + +#}END utilities + + +#{ Utility Classes + +class MapWindow: + + """Utility type which is used to snap windows towards each other, and to adjust their size""" + __slots__ = ( + 'ofs', # offset into the file in bytes + 'size' # size of the window in bytes + ) + + def __init__(self, offset, size): + self.ofs = offset + self.size = size + + def __repr__(self): + return "MapWindow(%i, %i)" % (self.ofs, self.size) + + @classmethod + def from_region(cls, region): + """:return: new window from a region""" + return cls(region._b, region.size()) + + def ofs_end(self): + return self.ofs + self.size + + def align(self): + """Assures the previous window area is contained in the new one""" + nofs = align_to_mmap(self.ofs, 0) + self.size += self.ofs - nofs # keep size constant + self.ofs = nofs + self.size = align_to_mmap(self.size, 1) + + def extend_left_to(self, window, max_size): + """Adjust the offset to start where the given window on our left ends if possible, + but don't make yourself larger than max_size. + The resize will assure that the new window still contains the old window area""" + rofs = self.ofs - window.ofs_end() + nsize = rofs + self.size + rofs -= nsize - min(nsize, max_size) + self.ofs -= rofs + self.size += rofs + + def extend_right_to(self, window, max_size): + """Adjust the size to make our window end where the right window begins, but don't + get larger than max_size""" + self.size = min(self.size + (window.ofs - self.ofs_end()), max_size) + + +class MapRegion: + + """Defines a mapped region of memory, aligned to pagesizes + + **Note:** deallocates used region automatically on destruction""" + __slots__ = [ + '_b', # beginning of mapping + '_mf', # mapped memory chunk (as returned by mmap) + '_uc', # total amount of usages + '_size', # cached size of our memory map + '__weakref__' + ] + + #{ Configuration + #} END configuration + + def __init__(self, path_or_fd, ofs, size, flags=0): + """Initialize a region, allocate the memory map + :param path_or_fd: path to the file to map, or the opened file descriptor + :param ofs: **aligned** offset into the file to be mapped + :param size: if size is larger then the file on disk, the whole file will be + allocated the the size automatically adjusted + :param flags: additional flags to be given when opening the file. + :raise Exception: if no memory can be allocated""" + self._b = ofs + self._size = 0 + self._uc = 0 + + if isinstance(path_or_fd, int): + fd = path_or_fd + else: + fd = os.open(path_or_fd, os.O_RDONLY | getattr(os, 'O_BINARY', 0) | flags) + # END handle fd + + try: + kwargs = dict(access=ACCESS_READ, offset=ofs) + corrected_size = size + sizeofs = ofs + + # have to correct size, otherwise (instead of the c version) it will + # bark that the size is too large ... many extra file accesses because + # if this ... argh ! + actual_size = min(os.fstat(fd).st_size - sizeofs, corrected_size) + self._mf = mmap(fd, actual_size, **kwargs) + # END handle memory mode + + self._size = len(self._mf) + finally: + if isinstance(path_or_fd, str): + os.close(fd) + # END only close it if we opened it + # END close file handle + # We assume the first one to use us keeps us around + self.increment_client_count() + + def __repr__(self): + return "MapRegion<%i, %i>" % (self._b, self.size()) + + #{ Interface + + def buffer(self): + """:return: a buffer containing the memory""" + return self._mf + + def map(self): + """:return: a memory map containing the memory""" + return self._mf + + def ofs_begin(self): + """:return: absolute byte offset to the first byte of the mapping""" + return self._b + + def size(self): + """:return: total size of the mapped region in bytes""" + return self._size + + def ofs_end(self): + """:return: Absolute offset to one byte beyond the mapping into the file""" + return self._b + self._size + + def includes_ofs(self, ofs): + """:return: True if the given offset can be read in our mapped region""" + return self._b <= ofs < self._b + self._size + + def client_count(self): + """:return: number of clients currently using this region""" + return self._uc + + def increment_client_count(self, ofs = 1): + """Adjust the usage count by the given positive or negative offset. + If usage count equals 0, we will auto-release our resources + :return: True if we released resources, False otherwise. In the latter case, we can still be used""" + self._uc += ofs + assert self._uc > -1, "Increments must match decrements, usage counter negative: %i" % self._uc + + if self.client_count() == 0: + self.release() + return True + else: + return False + # end handle release + + def release(self): + """Release all resources this instance might hold. Must only be called if there usage_count() is zero""" + self._mf.close() + + #} END interface + + +class MapRegionList(list): + + """List of MapRegion instances associating a path with a list of regions.""" + __slots__ = ( + '_path_or_fd', # path or file descriptor which is mapped by all our regions + '_file_size' # total size of the file we map + ) + + def __new__(cls, path): + return super().__new__(cls) + + def __init__(self, path_or_fd): + self._path_or_fd = path_or_fd + self._file_size = None + + def path_or_fd(self): + """:return: path or file descriptor we are attached to""" + return self._path_or_fd + + def file_size(self): + """:return: size of file we manager""" + if self._file_size is None: + if isinstance(self._path_or_fd, str): + self._file_size = os.stat(self._path_or_fd).st_size + else: + self._file_size = os.fstat(self._path_or_fd).st_size + # END handle path type + # END update file size + return self._file_size + +#} END utility classes diff --git a/smmap/tox.ini b/smmap/tox.ini new file mode 100644 index 000000000..f7205c5a3 --- /dev/null +++ b/smmap/tox.ini @@ -0,0 +1,17 @@ +# Tox (http://tox.testrun.org/) is a tool for running tests +# in multiple virtualenvs. This configuration file will run the +# test suite on all supported python versions. To use it, "pip install tox" +# and then run "tox" from this directory. + +[tox] +envlist = flake8, py{37, 38, 39, 310, 311, 312} + +[testenv] +commands = {envpython} -m pytest --cov smmap --cov-report xml {posargs} +deps = + pytest + pytest-cov + +[testenv:flake8] +commands = flake8 {posargs} +deps = flake8 diff --git a/test/lib/helper.py b/test/lib/helper.py index 1c110e103..deddced2b 100644 --- a/test/lib/helper.py +++ b/test/lib/helper.py @@ -382,7 +382,13 @@ def _small_repo_url(self): """:return: A path to a small, clonable repository""" from git.cmd import Git - return Git.polish_url(osp.join(self.rorepo.working_tree_dir, "git/ext/gitdb/gitdb/ext/smmap")) + return Git.polish_url(self._dependency_repo_dirs["smmap"]) + + def _gitdb_repo_url(self): + """:return: A local gitdb repository reconstructed from merged history""" + from git.cmd import Git + + return Git.polish_url(self._dependency_repo_dirs["gitdb"]) @classmethod def setUpClass(cls): @@ -394,11 +400,26 @@ def setUpClass(cls): gc.collect() cls.rorepo = Repo(GIT_REPO) + cls._dependency_repo_root = tempfile.mkdtemp(prefix="gitpython-dependency-repos-") + cls._dependency_repo_dirs = {} + for name, rev in { + "gitdb": "2da3232f9d58e7761e384ac6d32f7b1ed77a74a2", + "smmap": "8ce61ad5cc4016bffaf25080bc0d69b3acbe8555", + }.items(): + path = osp.join(cls._dependency_repo_root, name) + repo = cls.rorepo.clone(path, shared=True, no_checkout=True) + repo.create_head("master", repo.commit(rev), force=True).checkout() + if name == "smmap": + repo.create_tag("v0.8.1", ref="master~10", message="Test fixture tag", force=True) + repo.git.gc() + repo.close() + cls._dependency_repo_dirs[name] = path @classmethod def tearDownClass(cls): cls.rorepo.git.clear_cache() cls.rorepo.git = None + rmtree(cls._dependency_repo_root) def _make_file(self, rela_path, data, repo=None): """ diff --git a/test/test_clone.py b/test/test_clone.py index 5fd59b3a3..24ab04c62 100644 --- a/test/test_clone.py +++ b/test/test_clone.py @@ -87,6 +87,7 @@ def test_clone_from_with_path_contains_unicode(self): Repo.clone_from( url=self._small_repo_url(), to_path=path_with_unicode, + no_hardlinks=True, ) except UnicodeEncodeError: self.fail("Raised UnicodeEncodeError") diff --git a/test/test_docs.py b/test/test_docs.py index c3426a807..c3cfec3e0 100644 --- a/test/test_docs.py +++ b/test/test_docs.py @@ -8,7 +8,7 @@ import os.path from test.lib import TestBase -from test.lib.helper import with_rw_directory +from test.lib.helper import with_rw_directory, with_rw_repo class Tutorials(TestBase): @@ -475,13 +475,20 @@ def test_references_and_objects(self, rw_dir): repo.git.clear_cache() - def test_submodules(self): + @with_rw_repo("c15a6e1923a14bc760851913858a3942a4193cdb") + def test_submodules(self, repo): # [1-test_submodules] - repo = self.rorepo sms = repo.submodules assert len(sms) == 1 sm = sms[0] + with sm.config_writer() as writer: + writer.set_value("url", self._gitdb_repo_url()) + sm.update(recursive=False) + child = sm.children()[0] + with child.config_writer() as writer: + writer.set_value("url", self._small_repo_url()) + child.update() self.assertEqual(sm.name, "gitdb") # GitPython has gitdb as its one and only (direct) submodule... self.assertEqual(sm.children()[0].name, "smmap") # ...which has smmap as its one and only submodule. diff --git a/test/test_fixture_health.py b/test/test_fixture_health.py index b18d5e8f9..8301001c4 100644 --- a/test/test_fixture_health.py +++ b/test/test_fixture_health.py @@ -1,18 +1,7 @@ # This module is part of GitPython and is released under the # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ -"""Verify that fixture directories are usable by git. - -If a fixture directory is missing, isn't an initialized git repository, -or is rejected by git for "dubious ownership", dependent tests -elsewhere in the suite fail in opaque ways. The checks here name the -preconditions directly so a misconfigured environment is recognizable -from the test output rather than from a cascade of unrelated-seeming -failures. - -These tests do not exercise GitPython's production code. They verify -the conditions under which production code is exercised are valid. -""" +"""Verify that the source repository is usable by git.""" import subprocess from pathlib import Path @@ -21,32 +10,25 @@ REPO_ROOT = Path(__file__).resolve().parent.parent -# Directories git must trust for the test suite to operate normally. The -# current set is the GitPython working tree plus the working trees of its -# gitdb submodule and the smmap submodule nested inside gitdb. New entries -# should be added here whenever the test suite gains a dependency on git -# accepting another directory. -FIXTURE_DIRS = [ - pytest.param(REPO_ROOT, id="repo_root"), - pytest.param(REPO_ROOT / "git" / "ext" / "gitdb", id="gitdb"), - pytest.param( - REPO_ROOT / "git" / "ext" / "gitdb" / "gitdb" / "ext" / "smmap", - id="smmap", - ), -] +FIXTURE_DIRS = [pytest.param(REPO_ROOT, id="repo_root")] + -# Submodule working trees that must be present and initialized for the -# test suite to operate normally: gitdb at `git/ext/gitdb`, and smmap -# nested inside gitdb at `git/ext/gitdb/gitdb/ext/smmap`. The paths -# below are anchored at REPO_ROOT (the GitPython source tree), not at -# any rorepo redirection target. -SUBMODULE_DIRS = [ - pytest.param(REPO_ROOT / "git" / "ext" / "gitdb", id="gitdb"), - pytest.param( - REPO_ROOT / "git" / "ext" / "gitdb" / "gitdb" / "ext" / "smmap", - id="smmap", - ), -] +def test_source_tree_has_no_gitlinks() -> None: + """All formerly external dependencies are regular monorepo directories.""" + if not (REPO_ROOT / ".git").exists(): + pytest.skip(f"{REPO_ROOT} is not a git checkout") + try: + result = subprocess.run( + ["git", "ls-files", "--stage"], + cwd=REPO_ROOT, + capture_output=True, + text=True, + check=True, + ) + except FileNotFoundError: + pytest.skip("git is not installed or not on PATH") + gitlinks = [line for line in result.stdout.splitlines() if line.startswith("160000 ")] + assert not gitlinks, "Unexpected submodules:\n" + "\n".join(gitlinks) @pytest.mark.parametrize("fixture_dir", FIXTURE_DIRS) @@ -59,14 +41,8 @@ def test_fixture_dir_is_trusted_by_git(fixture_dir: Path) -> None: running user and the CI workflow's ``safe.directory`` list is missing an entry that would override the check. """ - if not fixture_dir.exists(): - pytest.skip(f"{fixture_dir} not present (run `git submodule update --init --recursive` from the repo root)") if not (fixture_dir / ".git").exists(): - pytest.skip( - f"{fixture_dir} has no .git marker " - "(submodule not initialized; run " - "`git submodule update --init --recursive` from the repo root)" - ) + pytest.skip(f"{fixture_dir} is not a git checkout") try: result = subprocess.run( ["git", "-C", str(fixture_dir), "rev-parse", "--show-toplevel"], @@ -92,40 +68,3 @@ def test_fixture_dir_is_trusted_by_git(fixture_dir: Path) -> None: "This usually means the directory is not an initialized git " "repository (its `.git` marker may be stale or pointing elsewhere)." ) - - -@pytest.mark.parametrize("submodule_dir", SUBMODULE_DIRS) -def test_required_submodule_is_initialized(submodule_dir: Path) -> None: - """The submodule's working tree is present and initialized. - - Failure means the source tree is a git clone but the submodule's - working tree hasn't been populated. Skipped when the source tree - itself isn't a git clone (e.g. an extracted release tarball), since - ``git submodule update`` cannot operate there; setups that handle - submodules in a separately-prepared tree (via - ``GIT_PYTHON_TEST_GIT_REPO_BASE``) are exempted from this check. - """ - if not (REPO_ROOT / ".git").exists(): - pytest.skip( - "Source tree is not a git clone (no .git in REPO_ROOT); submodules " - "cannot be initialized via `git submodule update` here. Setups " - "that prepare submodules in a separately-pointed tree (via " - "GIT_PYTHON_TEST_GIT_REPO_BASE) are exempted from this check." - ) - # The assertion messages below recommend `git submodule update --init - # --recursive` rather than `init-tests-after-clone.sh`, even though the - # latter is the documented entry point for first-time test setup. Two - # reasons: the script performs `git reset --hard` operations that can - # destroy local work, and #1713 showed the script itself can carry - # submodule-init regressions, in which case recommending it would lead - # developers in a circle. The direct git command is a safe minimal fix - # for this test's specific failure mode and bypasses any such regression. - assert submodule_dir.is_dir(), ( - f"Submodule working tree missing: {submodule_dir}.\n" - "Run `git submodule update --init --recursive` from the repo root." - ) - assert (submodule_dir / ".git").exists(), ( - f"Submodule directory exists but has no .git marker: {submodule_dir}.\n" - "The submodule hasn't been initialized. " - "Run `git submodule update --init --recursive` from the repo root." - ) diff --git a/test/test_fun.py b/test/test_fun.py index a456b8aab..38f506ff4 100644 --- a/test/test_fun.py +++ b/test/test_fun.py @@ -306,3 +306,7 @@ def test_linked_worktree_traversal(self, rw_dir): def test_tree_entries_from_data_with_failing_name_decode_py3(self): r = tree_entries_from_data(b"100644 \x9f\0aaa") assert r == [(b"aaa", 33188, "\udc9f")], r + + def test_tree_entries_from_bytearray(self): + r = tree_entries_from_data(bytearray(b"100644 name\0abcdefghijklmnopqrst")) + assert r == [(b"abcdefghijklmnopqrst", 33188, "name")], r diff --git a/test/test_installation.py b/test/test_installation.py index 7c82bd403..656ef9787 100644 --- a/test/test_installation.py +++ b/test/test_installation.py @@ -14,8 +14,9 @@ class TestInstallation(TestBase): def test_installation(self, rw_dir): venv, run = self._set_up_venv(rw_dir) - result = run([venv.pip, "install", "."]) - self._check_result(result, "Can't install project") + for project in ("./smmap", "./gitdb", "."): + result = run([venv.pip, "install", project]) + self._check_result(result, f"Can't install {project}") result = run([venv.python, "-c", "import git"]) self._check_result(result, "Self-test failed") diff --git a/test/test_repo.py b/test/test_repo.py index cfad73489..27246db88 100644 --- a/test/test_repo.py +++ b/test/test_repo.py @@ -948,12 +948,11 @@ def test_repo_odbtype(self): target_type = GitCmdObjectDB self.assertIsInstance(self.rorepo.odb, target_type) - def test_submodules(self): - self.assertEqual(len(self.rorepo.submodules), 1) # non-recursive - self.assertGreaterEqual(len(list(self.rorepo.iter_submodules())), 2) - - self.assertIsInstance(self.rorepo.submodule("gitdb"), Submodule) - self.assertRaises(ValueError, self.rorepo.submodule, "doesn't exist") + @with_rw_repo("c15a6e1923a14bc760851913858a3942a4193cdb", bare=True) + def test_submodules(self, rwrepo): + self.assertEqual(len(rwrepo.submodules), 1) + self.assertIsInstance(rwrepo.submodule("gitdb"), Submodule) + self.assertRaises(ValueError, rwrepo.submodule, "doesn't exist") @with_rw_repo("HEAD", bare=False) def test_submodule_update(self, rwrepo): @@ -963,11 +962,10 @@ def test_submodule_update(self, rwrepo): rwrepo._bare = False # Test submodule creation. - sm = rwrepo.submodules[0] sm = rwrepo.create_submodule( "my_new_sub", "some_path", - join_path_native(self.rorepo.working_tree_dir, sm.path), + self._small_repo_url(), ) self.assertIsInstance(sm, Submodule) diff --git a/test/test_submodule.py b/test/test_submodule.py index 265ee2ffb..f9870e08d 100644 --- a/test/test_submodule.py +++ b/test/test_submodule.py @@ -124,7 +124,7 @@ def _do_base_tests(self, rwrepo): else: with sm.config_writer() as writer: # For faster checkout, set the url to the local path. - new_smclone_path = Git.polish_url(osp.join(self.rorepo.working_tree_dir, sm.path)) + new_smclone_path = self._gitdb_repo_url() writer.set_value("url", new_smclone_path) writer.release() assert sm.config_reader().get_value("url") == new_smclone_path @@ -239,7 +239,7 @@ def _do_base_tests(self, rwrepo): # Adjust the path of the submodules module to point to the local # destination. - new_csmclone_path = Git.polish_url(osp.join(self.rorepo.working_tree_dir, sm.path, csm.path)) + new_csmclone_path = self._small_repo_url() with csm.config_writer() as writer: writer.set_value("url", new_csmclone_path) assert csm.url == new_csmclone_path @@ -491,15 +491,15 @@ def test_base_bare(self, rwrepo): @with_rw_repo(k_subm_current, bare=False) def test_root_module(self, rwrepo): # Can query everything without problems. - rm = RootModule(self.rorepo) - assert rm.module() is self.rorepo + rm = RootModule(rwrepo) + assert rm.module() is rwrepo # Try attributes. rm.binsha rm.mode rm.path assert rm.name == rm.k_root_name - assert rm.parent_commit == self.rorepo.head.commit + assert rm.parent_commit == rwrepo.head.commit rm.url rm.branch @@ -508,10 +508,6 @@ def test_root_module(self, rwrepo): with rm.config_writer(): pass - # Deep traversal yields gitdb and its nested smmap. - rsmsp = [sm.path for sm in rm.traverse()] - assert rsmsp == ["git/ext/gitdb", "gitdb/ext/smmap"] - # Cannot set the parent commit as root module's path didn't exist. self.assertRaises(ValueError, rm.set_parent_commit, "HEAD") @@ -533,7 +529,7 @@ def test_root_module(self, rwrepo): # Ensure we clone from a local source. with sm.config_writer() as writer: - writer.set_value("url", Git.polish_url(osp.join(self.rorepo.working_tree_dir, sm.path))) + writer.set_value("url", self._gitdb_repo_url()) # Dry-run does nothing. sm.update(recursive=False, dry_run=True, progress=prog) @@ -568,7 +564,7 @@ def test_root_module(self, rwrepo): # ============== nsmn = "newsubmodule" nsmp = "submrepo" - subrepo_url = Git.polish_url(osp.join(self.rorepo.working_tree_dir, rsmsp[0], rsmsp[1])) + subrepo_url = self._small_repo_url() nsm = Submodule.add(rwrepo, nsmn, nsmp, url=subrepo_url) csmadded = rwrepo.index.commit("Added submodule").hexsha # Make sure we don't keep the repo reference. nsm.set_parent_commit(csmadded) @@ -633,7 +629,7 @@ def test_root_module(self, rwrepo): # ...to the first repository. This way we have a fast checkout, and a completely # different repository at the different url. nsm.set_parent_commit(csmremoved) - nsmurl = Git.polish_url(osp.join(self.rorepo.working_tree_dir, rsmsp[0])) + nsmurl = self._gitdb_repo_url() with nsm.config_writer() as writer: writer.set_value("url", nsmurl) csmpathchange = rwrepo.index.commit("changed url")