upload
This commit is contained in:
HamsterMimi
@@ -0,0 +1,54 @@
|
||||
GitPython was originally written by Michael Trier.
|
||||
GitPython 0.2 was partially (re)written by Sebastian Thiel, based on 0.1.6 and git-dulwich.
|
||||
|
||||
Contributors are:
|
||||
|
||||
-Michael Trier <mtrier _at_ gmail.com>
|
||||
-Alan Briolat
|
||||
-Florian Apolloner <florian _at_ apolloner.eu>
|
||||
-David Aguilar <davvid _at_ gmail.com>
|
||||
-Jelmer Vernooij <jelmer _at_ samba.org>
|
||||
-Steve Frécinaux <code _at_ istique.net>
|
||||
-Kai Lautaportti <kai _at_ lautaportti.fi>
|
||||
-Paul Sowden <paul _at_ idontsmoke.co.uk>
|
||||
-Sebastian Thiel <byronimo _at_ gmail.com>
|
||||
-Jonathan Chu <jonathan.chu _at_ me.com>
|
||||
-Vincent Driessen <me _at_ nvie.com>
|
||||
-Phil Elson <pelson _dot_ pub _at_ gmail.com>
|
||||
-Bernard `Guyzmo` Pratz <guyzmo+gitpython+pub@m0g.net>
|
||||
-Timothy B. Hartman <tbhartman _at_ gmail.com>
|
||||
-Konstantin Popov <konstantin.popov.89 _at_ yandex.ru>
|
||||
-Peter Jones <pjones _at_ redhat.com>
|
||||
-Anson Mansfield <anson.mansfield _at_ gmail.com>
|
||||
-Ken Odegard <ken.odegard _at_ gmail.com>
|
||||
-Alexis Horgix Chotard
|
||||
-Piotr Babij <piotr.babij _at_ gmail.com>
|
||||
-Mikuláš Poul <mikulaspoul _at_ gmail.com>
|
||||
-Charles Bouchard-Légaré <cblegare.atl _at_ ntis.ca>
|
||||
-Yaroslav Halchenko <debian _at_ onerussian.com>
|
||||
-Tim Swast <swast _at_ google.com>
|
||||
-William Luc Ritchie
|
||||
-David Host <hostdm _at_ outlook.com>
|
||||
-A. Jesse Jiryu Davis <jesse _at_ emptysquare.net>
|
||||
-Steven Whitman <ninloot _at_ gmail.com>
|
||||
-Stefan Stancu <stefan.stancu _at_ gmail.com>
|
||||
-César Izurieta <cesar _at_ caih.org>
|
||||
-Arthur Milchior <arthur _at_ milchior.fr>
|
||||
-Anil Khatri <anil.soccer.khatri _at_ gmail.com>
|
||||
-JJ Graham <thetwoj _at_ gmail.com>
|
||||
-Ben Thayer <ben _at_ benthayer.com>
|
||||
-Dries Kennes <admin _at_ dries007.net>
|
||||
-Pratik Anurag <panurag247365 _at_ gmail.com>
|
||||
-Harmon <harmon.public _at_ gmail.com>
|
||||
-Liam Beguin <liambeguin _at_ gmail.com>
|
||||
-Ram Rachum <ram _at_ rachum.com>
|
||||
-Alba Mendez <me _at_ alba.sh>
|
||||
-Robert Westman <robert _at_ byteflux.io>
|
||||
-Hugo van Kemenade
|
||||
-Hiroki Tokunaga <tokusan441 _at_ gmail.com>
|
||||
-Julien Mauroy <pro.julien.mauroy _at_ gmail.com>
|
||||
-Patrick Gerard
|
||||
-Luke Twist <itsluketwist@gmail.com>
|
||||
-Joseph Hale <me _at_ jhale.dev>
|
||||
-Santos Gallegos <stsewd _at_ proton.me>
|
||||
Portions derived from other open source works and are clearly marked.
|
||||
@@ -0,0 +1,30 @@
|
||||
Copyright (C) 2008, 2009 Michael Trier and contributors
|
||||
All rights reserved.
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions
|
||||
are met:
|
||||
|
||||
* Redistributions of source code must retain the above copyright
|
||||
notice, this list of conditions and the following disclaimer.
|
||||
|
||||
* Redistributions in binary form must reproduce the above copyright
|
||||
notice, this list of conditions and the following disclaimer in the
|
||||
documentation and/or other materials provided with the distribution.
|
||||
|
||||
* Neither the name of the GitPython project nor the names of
|
||||
its contributors may be used to endorse or promote products derived
|
||||
from this software without specific prior written permission.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
|
||||
TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
|
||||
PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
|
||||
LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
|
||||
NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
|
||||
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
@@ -0,0 +1,34 @@
|
||||
Metadata-Version: 2.1
|
||||
Name: GitPython
|
||||
Version: 3.1.31
|
||||
Summary: GitPython is a Python library used to interact with Git repositories
|
||||
Home-page: https://github.com/gitpython-developers/GitPython
|
||||
Author: Sebastian Thiel, Michael Trier
|
||||
Author-email: byronimo@gmail.com, mtrier@gmail.com
|
||||
License: BSD
|
||||
Platform: UNKNOWN
|
||||
Classifier: Development Status :: 5 - Production/Stable
|
||||
Classifier: Environment :: Console
|
||||
Classifier: Intended Audience :: Developers
|
||||
Classifier: License :: OSI Approved :: BSD License
|
||||
Classifier: Operating System :: OS Independent
|
||||
Classifier: Operating System :: POSIX
|
||||
Classifier: Operating System :: Microsoft :: Windows
|
||||
Classifier: Operating System :: MacOS :: MacOS X
|
||||
Classifier: Typing :: Typed
|
||||
Classifier: Programming Language :: Python
|
||||
Classifier: Programming Language :: Python :: 3
|
||||
Classifier: Programming Language :: Python :: 3.7
|
||||
Classifier: Programming Language :: Python :: 3.8
|
||||
Classifier: Programming Language :: Python :: 3.9
|
||||
Classifier: Programming Language :: Python :: 3.10
|
||||
Classifier: Programming Language :: Python :: 3.11
|
||||
Requires-Python: >=3.7
|
||||
Description-Content-Type: text/markdown
|
||||
License-File: LICENSE
|
||||
License-File: AUTHORS
|
||||
Requires-Dist: gitdb (<5,>=4.0.1)
|
||||
Requires-Dist: typing-extensions (>=3.7.4.3) ; python_version < "3.8"
|
||||
|
||||
GitPython is a Python library used to interact with Git repositories
|
||||
|
||||
@@ -0,0 +1,44 @@
|
||||
git/__init__.py,sha256=O2tZaGpLYVQiK9lN3NucvyEoZcSFig13tAB6d2TTTL0,2342
|
||||
git/cmd.py,sha256=i4IyhmCTP-72NPO5aVeWhDT6_jLmA1C2qzhsS7G2UVw,53712
|
||||
git/compat.py,sha256=3wWLkD9QrZvLiV6NtNxJILwGrLE2nw_SoLqaTEPH364,2256
|
||||
git/config.py,sha256=PO6qicfkKwRFlKJr9AUuDrWV0rimlmb5S2wIVLlOd7w,34581
|
||||
git/db.py,sha256=dEs2Bn-iDuHyero9afw8mrXHrLE7_CDExv943iWU9WI,2244
|
||||
git/diff.py,sha256=DOWd26Dk7FqnKt79zpniv19muBzdYa949TcQPqVbZGg,23434
|
||||
git/exc.py,sha256=ys5ZYuvzvNN3TfcB5R_bUNRy3OEvURS5pJMdfy0Iws4,6446
|
||||
git/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
||||
git/remote.py,sha256=H88bonpIjnfozWScpQIFIccE7Soq2hfHO9ldnRCmGUY,45069
|
||||
git/types.py,sha256=bA4El-NC7YNwQ9jNtkbWgT0QmmAfVs4PVSwBOE_D1Bo,3020
|
||||
git/util.py,sha256=j5cjyeFibLs4Ed_ErkePf6sx1VWb95OQ4GlJUWgq6PU,39874
|
||||
git/index/__init__.py,sha256=43ovvVNocVRNiQd4fLqvUMuGGmwhBQ9SsiQ46vkvk1E,89
|
||||
git/index/base.py,sha256=5GnqwmhLNF9f12hUq4rQyOvqzxPF1Fdc0QOETT5r010,57523
|
||||
git/index/fun.py,sha256=Y41IGlu8XqnradQXFjTGMISM45m8J256bTKs4xWR4qY,16406
|
||||
git/index/typ.py,sha256=QnyWeqzU7_xnyiwOki5W633Jp9g5COqEf6B4PeW3hK8,6252
|
||||
git/index/util.py,sha256=ISsWZjGiflooNr6XtElP4AhWUxQOakouvgeXC2PEepI,3475
|
||||
git/objects/__init__.py,sha256=NW8HBfdZvBYe9W6IjMWafSj_DVlV2REmmrpWKrHkGVw,692
|
||||
git/objects/base.py,sha256=N2NTL9hLwgKqY-VQiar8Hvn4a41Y8o_Tmi_SR0mGAS8,7857
|
||||
git/objects/blob.py,sha256=FIbZTYniJ7nLsdrHuwhagFVs9tYoUIyXodRaHYLaQqs,986
|
||||
git/objects/commit.py,sha256=ji9ityweewpr12mHh9w3s3ubouYNNCRTBr-LBrjrPbs,27304
|
||||
git/objects/fun.py,sha256=SV3_G_jnEb_wEa5doF6AapX58StH3OGBxCAKeMyuA0I,8612
|
||||
git/objects/tag.py,sha256=ZXOLK_lV9E5G2aDl5t0hYDN2hhIhGF23HILHBnZgRX0,3840
|
||||
git/objects/tree.py,sha256=cSQbt3nn3cIrbVrBasB1wm2r-vzotYWhka1yDjOHf-k,14230
|
||||
git/objects/util.py,sha256=M8h53ueOV32nXE6XcnKhCHzXznT7pi8JpEEGgCNicXo,22275
|
||||
git/objects/submodule/__init__.py,sha256=OsMeiex7cG6ev2f35IaJ5csH-eXchSoNKCt4HXUG5Ws,93
|
||||
git/objects/submodule/base.py,sha256=R4jTjBJyMjFOfDAYwsA6Q3Lt6qeFYERPE4PABACW6GE,61539
|
||||
git/objects/submodule/root.py,sha256=Ev_RnGzv4hi3UqEFMHuSR-uGR7kYpwOgwZFUG31X-Hc,19568
|
||||
git/objects/submodule/util.py,sha256=u2zQGFWBmryqET0XWf9BuiY1OOgWB8YCU3Wz0xdp4E4,3380
|
||||
git/refs/__init__.py,sha256=PMF97jMUcivbCCEJnl2zTs-YtECNFp8rL8GHK8AitXU,203
|
||||
git/refs/head.py,sha256=rZ4LbFd05Gs9sAuSU5VQRDmJZfrwMwWtBpLlmiUQ-Zg,9756
|
||||
git/refs/log.py,sha256=Z8X9_ZGZrVTWz9p_-fk1N3m47G-HTRPwozoZBDd70DI,11892
|
||||
git/refs/reference.py,sha256=DUx7QvYqTBeVxG53ntPfKCp3wuJyDBRIZcPCy1OD22s,5414
|
||||
git/refs/remote.py,sha256=E63Bh5ig1GYrk6FE46iNtS5P6ZgODyPXot8eJw-mxts,2556
|
||||
git/refs/symbolic.py,sha256=XwfeYr1Zp-fuHAoGuVAXKk4EYlsuUMVu99OjJWuWDTQ,29967
|
||||
git/refs/tag.py,sha256=FNoCZ3BdDl2i5kD3si2P9hoXU9rDAZ_YK0Rn84TmKT8,4419
|
||||
git/repo/__init__.py,sha256=XMpdeowJRtTEd80jAcrKSQfMu2JZGMfPlpuIYHG2ZCk,80
|
||||
git/repo/base.py,sha256=uD4EL2AWUMSCHCqIk7voXoZ2iChaf5VJ1t1Abr7Zk10,54937
|
||||
git/repo/fun.py,sha256=VTRODXAb_x8bazkSd8g-Pkk8M2iLVK4kPoKQY9HXjZc,12962
|
||||
GitPython-3.1.31.dist-info/AUTHORS,sha256=0F09KKrRmwH3zJ4gqo1tJMVlalC9bSunDNKlRvR6q2c,2158
|
||||
GitPython-3.1.31.dist-info/LICENSE,sha256=_WV__CzvY9JceMq3gI1BTdA6KC5jiTSR_RHDL5i-Z_s,1521
|
||||
GitPython-3.1.31.dist-info/METADATA,sha256=zFy5SrG7Ur2UItx3seZXELCST9LBEX72wZa7Y7z7FSY,1340
|
||||
GitPython-3.1.31.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
|
||||
GitPython-3.1.31.dist-info/top_level.txt,sha256=0hzDuIp8obv624V3GmbqsagBWkk8ohtGU-Bc1PmTT0o,4
|
||||
GitPython-3.1.31.dist-info/RECORD,,
|
||||
@@ -0,0 +1,5 @@
|
||||
Wheel-Version: 1.0
|
||||
Generator: bdist_wheel (0.37.0)
|
||||
Root-Is-Purelib: true
|
||||
Tag: py3-none-any
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
gitdb<5,>=4.0.1
|
||||
@@ -0,0 +1 @@
|
||||
git
|
||||
@@ -0,0 +1,92 @@
|
||||
# __init__.py
|
||||
# Copyright (C) 2008, 2009 Michael Trier (mtrier@gmail.com) and contributors
|
||||
#
|
||||
# This module is part of GitPython and is released under
|
||||
# the BSD License: http://www.opensource.org/licenses/bsd-license.php
|
||||
# flake8: noqa
|
||||
# @PydevCodeAnalysisIgnore
|
||||
from git.exc import * # @NoMove @IgnorePep8
|
||||
import inspect
|
||||
import os
|
||||
import sys
|
||||
import os.path as osp
|
||||
|
||||
from typing import Optional
|
||||
from git.types import PathLike
|
||||
|
||||
__version__ = '3.1.31'
|
||||
|
||||
|
||||
# { Initialization
|
||||
def _init_externals() -> None:
|
||||
"""Initialize external projects by putting them into the path"""
|
||||
if __version__ == '3.1.31' and "PYOXIDIZER" not in os.environ:
|
||||
sys.path.insert(1, osp.join(osp.dirname(__file__), "ext", "gitdb"))
|
||||
|
||||
try:
|
||||
import gitdb
|
||||
except ImportError as e:
|
||||
raise ImportError("'gitdb' could not be found in your PYTHONPATH") from e
|
||||
# END verify import
|
||||
|
||||
|
||||
# } END initialization
|
||||
|
||||
|
||||
#################
|
||||
_init_externals()
|
||||
#################
|
||||
|
||||
# { Imports
|
||||
|
||||
try:
|
||||
from git.config import GitConfigParser # @NoMove @IgnorePep8
|
||||
from git.objects import * # @NoMove @IgnorePep8
|
||||
from git.refs import * # @NoMove @IgnorePep8
|
||||
from git.diff import * # @NoMove @IgnorePep8
|
||||
from git.db import * # @NoMove @IgnorePep8
|
||||
from git.cmd import Git # @NoMove @IgnorePep8
|
||||
from git.repo import Repo # @NoMove @IgnorePep8
|
||||
from git.remote import * # @NoMove @IgnorePep8
|
||||
from git.index import * # @NoMove @IgnorePep8
|
||||
from git.util import ( # @NoMove @IgnorePep8
|
||||
LockFile,
|
||||
BlockingLockFile,
|
||||
Stats,
|
||||
Actor,
|
||||
rmtree,
|
||||
)
|
||||
except GitError as exc:
|
||||
raise ImportError("%s: %s" % (exc.__class__.__name__, exc)) from exc
|
||||
|
||||
# } END imports
|
||||
|
||||
__all__ = [name for name, obj in locals().items() if not (name.startswith("_") or inspect.ismodule(obj))]
|
||||
|
||||
|
||||
# { Initialize git executable path
|
||||
GIT_OK = None
|
||||
|
||||
|
||||
def refresh(path: Optional[PathLike] = None) -> None:
|
||||
"""Convenience method for setting the git executable path."""
|
||||
global GIT_OK
|
||||
GIT_OK = False
|
||||
|
||||
if not Git.refresh(path=path):
|
||||
return
|
||||
if not FetchInfo.refresh():
|
||||
return
|
||||
|
||||
GIT_OK = True
|
||||
|
||||
|
||||
# } END initialize git executable path
|
||||
|
||||
|
||||
#################
|
||||
try:
|
||||
refresh()
|
||||
except Exception as exc:
|
||||
raise ImportError("Failed to initialize: {0}".format(exc)) from exc
|
||||
#################
|
||||
1417
zero-cost-nas/.eggs/GitPython-3.1.31-py3.8.egg/git/cmd.py
Normal file
1417
zero-cost-nas/.eggs/GitPython-3.1.31-py3.8.egg/git/cmd.py
Normal file
File diff suppressed because it is too large
Load Diff
104
zero-cost-nas/.eggs/GitPython-3.1.31-py3.8.egg/git/compat.py
Normal file
104
zero-cost-nas/.eggs/GitPython-3.1.31-py3.8.egg/git/compat.py
Normal file
@@ -0,0 +1,104 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
# config.py
|
||||
# Copyright (C) 2008, 2009 Michael Trier (mtrier@gmail.com) and contributors
|
||||
#
|
||||
# This module is part of GitPython and is released under
|
||||
# the BSD License: http://www.opensource.org/licenses/bsd-license.php
|
||||
"""utilities to help provide compatibility with python 3"""
|
||||
# flake8: noqa
|
||||
|
||||
import locale
|
||||
import os
|
||||
import sys
|
||||
|
||||
from gitdb.utils.encoding import (
|
||||
force_bytes, # @UnusedImport
|
||||
force_text, # @UnusedImport
|
||||
)
|
||||
|
||||
# typing --------------------------------------------------------------------
|
||||
|
||||
from typing import (
|
||||
Any,
|
||||
AnyStr,
|
||||
Dict,
|
||||
IO,
|
||||
Optional,
|
||||
Tuple,
|
||||
Type,
|
||||
Union,
|
||||
overload,
|
||||
)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
is_win: bool = os.name == "nt"
|
||||
is_posix = os.name == "posix"
|
||||
is_darwin = os.name == "darwin"
|
||||
defenc = sys.getfilesystemencoding()
|
||||
|
||||
|
||||
@overload
|
||||
def safe_decode(s: None) -> None:
|
||||
...
|
||||
|
||||
|
||||
@overload
|
||||
def safe_decode(s: AnyStr) -> str:
|
||||
...
|
||||
|
||||
|
||||
def safe_decode(s: Union[AnyStr, None]) -> Optional[str]:
|
||||
"""Safely decodes a binary string to unicode"""
|
||||
if isinstance(s, str):
|
||||
return s
|
||||
elif isinstance(s, bytes):
|
||||
return s.decode(defenc, "surrogateescape")
|
||||
elif s is None:
|
||||
return None
|
||||
else:
|
||||
raise TypeError("Expected bytes or text, but got %r" % (s,))
|
||||
|
||||
|
||||
@overload
|
||||
def safe_encode(s: None) -> None:
|
||||
...
|
||||
|
||||
|
||||
@overload
|
||||
def safe_encode(s: AnyStr) -> bytes:
|
||||
...
|
||||
|
||||
|
||||
def safe_encode(s: Optional[AnyStr]) -> Optional[bytes]:
|
||||
"""Safely encodes a binary string to unicode"""
|
||||
if isinstance(s, str):
|
||||
return s.encode(defenc)
|
||||
elif isinstance(s, bytes):
|
||||
return s
|
||||
elif s is None:
|
||||
return None
|
||||
else:
|
||||
raise TypeError("Expected bytes or text, but got %r" % (s,))
|
||||
|
||||
|
||||
@overload
|
||||
def win_encode(s: None) -> None:
|
||||
...
|
||||
|
||||
|
||||
@overload
|
||||
def win_encode(s: AnyStr) -> bytes:
|
||||
...
|
||||
|
||||
|
||||
def win_encode(s: Optional[AnyStr]) -> Optional[bytes]:
|
||||
"""Encode unicodes for process arguments on Windows."""
|
||||
if isinstance(s, str):
|
||||
return s.encode(locale.getpreferredencoding(False))
|
||||
elif isinstance(s, bytes):
|
||||
return s
|
||||
elif s is not None:
|
||||
raise TypeError("Expected bytes or text, but got %r" % (s,))
|
||||
return None
|
||||
897
zero-cost-nas/.eggs/GitPython-3.1.31-py3.8.egg/git/config.py
Normal file
897
zero-cost-nas/.eggs/GitPython-3.1.31-py3.8.egg/git/config.py
Normal file
@@ -0,0 +1,897 @@
|
||||
# config.py
|
||||
# Copyright (C) 2008, 2009 Michael Trier (mtrier@gmail.com) and contributors
|
||||
#
|
||||
# This module is part of GitPython and is released under
|
||||
# the BSD License: http://www.opensource.org/licenses/bsd-license.php
|
||||
"""Module containing module parser implementation able to properly read and write
|
||||
configuration files"""
|
||||
|
||||
import sys
|
||||
import abc
|
||||
from functools import wraps
|
||||
import inspect
|
||||
from io import BufferedReader, IOBase
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import fnmatch
|
||||
|
||||
from git.compat import (
|
||||
defenc,
|
||||
force_text,
|
||||
is_win,
|
||||
)
|
||||
|
||||
from git.util import LockFile
|
||||
|
||||
import os.path as osp
|
||||
|
||||
import configparser as cp
|
||||
|
||||
# typing-------------------------------------------------------
|
||||
|
||||
from typing import (
|
||||
Any,
|
||||
Callable,
|
||||
Generic,
|
||||
IO,
|
||||
List,
|
||||
Dict,
|
||||
Sequence,
|
||||
TYPE_CHECKING,
|
||||
Tuple,
|
||||
TypeVar,
|
||||
Union,
|
||||
cast,
|
||||
)
|
||||
|
||||
from git.types import Lit_config_levels, ConfigLevels_Tup, PathLike, assert_never, _T
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from git.repo.base import Repo
|
||||
from io import BytesIO
|
||||
|
||||
T_ConfigParser = TypeVar("T_ConfigParser", bound="GitConfigParser")
|
||||
T_OMD_value = TypeVar("T_OMD_value", str, bytes, int, float, bool)
|
||||
|
||||
if sys.version_info[:3] < (3, 7, 2):
|
||||
# typing.Ordereddict not added until py 3.7.2
|
||||
from collections import OrderedDict
|
||||
|
||||
OrderedDict_OMD = OrderedDict
|
||||
else:
|
||||
from typing import OrderedDict
|
||||
|
||||
OrderedDict_OMD = OrderedDict[str, List[T_OMD_value]] # type: ignore[assignment, misc]
|
||||
|
||||
# -------------------------------------------------------------
|
||||
|
||||
__all__ = ("GitConfigParser", "SectionConstraint")
|
||||
|
||||
|
||||
log = logging.getLogger("git.config")
|
||||
log.addHandler(logging.NullHandler())
|
||||
|
||||
# invariants
|
||||
# represents the configuration level of a configuration file
|
||||
|
||||
|
||||
CONFIG_LEVELS: ConfigLevels_Tup = ("system", "user", "global", "repository")
|
||||
|
||||
|
||||
# Section pattern to detect conditional includes.
|
||||
# https://git-scm.com/docs/git-config#_conditional_includes
|
||||
CONDITIONAL_INCLUDE_REGEXP = re.compile(r"(?<=includeIf )\"(gitdir|gitdir/i|onbranch):(.+)\"")
|
||||
|
||||
|
||||
class MetaParserBuilder(abc.ABCMeta): # noqa: B024
|
||||
"""Utility class wrapping base-class methods into decorators that assure read-only properties"""
|
||||
|
||||
def __new__(cls, name: str, bases: Tuple, clsdict: Dict[str, Any]) -> "MetaParserBuilder":
|
||||
"""
|
||||
Equip all base-class methods with a needs_values decorator, and all non-const methods
|
||||
with a set_dirty_and_flush_changes decorator in addition to that."""
|
||||
kmm = "_mutating_methods_"
|
||||
if kmm in clsdict:
|
||||
mutating_methods = clsdict[kmm]
|
||||
for base in bases:
|
||||
methods = (t for t in inspect.getmembers(base, inspect.isroutine) if not t[0].startswith("_"))
|
||||
for name, method in methods:
|
||||
if name in clsdict:
|
||||
continue
|
||||
method_with_values = needs_values(method)
|
||||
if name in mutating_methods:
|
||||
method_with_values = set_dirty_and_flush_changes(method_with_values)
|
||||
# END mutating methods handling
|
||||
|
||||
clsdict[name] = method_with_values
|
||||
# END for each name/method pair
|
||||
# END for each base
|
||||
# END if mutating methods configuration is set
|
||||
|
||||
new_type = super(MetaParserBuilder, cls).__new__(cls, name, bases, clsdict)
|
||||
return new_type
|
||||
|
||||
|
||||
def needs_values(func: Callable[..., _T]) -> Callable[..., _T]:
|
||||
"""Returns method assuring we read values (on demand) before we try to access them"""
|
||||
|
||||
@wraps(func)
|
||||
def assure_data_present(self: "GitConfigParser", *args: Any, **kwargs: Any) -> _T:
|
||||
self.read()
|
||||
return func(self, *args, **kwargs)
|
||||
|
||||
# END wrapper method
|
||||
return assure_data_present
|
||||
|
||||
|
||||
def set_dirty_and_flush_changes(non_const_func: Callable[..., _T]) -> Callable[..., _T]:
|
||||
"""Return method that checks whether given non constant function may be called.
|
||||
If so, the instance will be set dirty.
|
||||
Additionally, we flush the changes right to disk"""
|
||||
|
||||
def flush_changes(self: "GitConfigParser", *args: Any, **kwargs: Any) -> _T:
|
||||
rval = non_const_func(self, *args, **kwargs)
|
||||
self._dirty = True
|
||||
self.write()
|
||||
return rval
|
||||
|
||||
# END wrapper method
|
||||
flush_changes.__name__ = non_const_func.__name__
|
||||
return flush_changes
|
||||
|
||||
|
||||
class SectionConstraint(Generic[T_ConfigParser]):
|
||||
|
||||
"""Constrains a ConfigParser to only option commands which are constrained to
|
||||
always use the section we have been initialized with.
|
||||
|
||||
It supports all ConfigParser methods that operate on an option.
|
||||
|
||||
:note:
|
||||
If used as a context manager, will release the wrapped ConfigParser."""
|
||||
|
||||
__slots__ = ("_config", "_section_name")
|
||||
_valid_attrs_ = (
|
||||
"get_value",
|
||||
"set_value",
|
||||
"get",
|
||||
"set",
|
||||
"getint",
|
||||
"getfloat",
|
||||
"getboolean",
|
||||
"has_option",
|
||||
"remove_section",
|
||||
"remove_option",
|
||||
"options",
|
||||
)
|
||||
|
||||
def __init__(self, config: T_ConfigParser, section: str) -> None:
|
||||
self._config = config
|
||||
self._section_name = section
|
||||
|
||||
def __del__(self) -> None:
|
||||
# Yes, for some reason, we have to call it explicitly for it to work in PY3 !
|
||||
# Apparently __del__ doesn't get call anymore if refcount becomes 0
|
||||
# Ridiculous ... .
|
||||
self._config.release()
|
||||
|
||||
def __getattr__(self, attr: str) -> Any:
|
||||
if attr in self._valid_attrs_:
|
||||
return lambda *args, **kwargs: self._call_config(attr, *args, **kwargs)
|
||||
return super(SectionConstraint, self).__getattribute__(attr)
|
||||
|
||||
def _call_config(self, method: str, *args: Any, **kwargs: Any) -> Any:
|
||||
"""Call the configuration at the given method which must take a section name
|
||||
as first argument"""
|
||||
return getattr(self._config, method)(self._section_name, *args, **kwargs)
|
||||
|
||||
@property
|
||||
def config(self) -> T_ConfigParser:
|
||||
"""return: Configparser instance we constrain"""
|
||||
return self._config
|
||||
|
||||
def release(self) -> None:
|
||||
"""Equivalent to GitConfigParser.release(), which is called on our underlying parser instance"""
|
||||
return self._config.release()
|
||||
|
||||
def __enter__(self) -> "SectionConstraint[T_ConfigParser]":
|
||||
self._config.__enter__()
|
||||
return self
|
||||
|
||||
def __exit__(self, exception_type: str, exception_value: str, traceback: str) -> None:
|
||||
self._config.__exit__(exception_type, exception_value, traceback)
|
||||
|
||||
|
||||
class _OMD(OrderedDict_OMD):
|
||||
"""Ordered multi-dict."""
|
||||
|
||||
def __setitem__(self, key: str, value: _T) -> None:
|
||||
super(_OMD, self).__setitem__(key, [value])
|
||||
|
||||
def add(self, key: str, value: Any) -> None:
|
||||
if key not in self:
|
||||
super(_OMD, self).__setitem__(key, [value])
|
||||
return None
|
||||
super(_OMD, self).__getitem__(key).append(value)
|
||||
|
||||
def setall(self, key: str, values: List[_T]) -> None:
|
||||
super(_OMD, self).__setitem__(key, values)
|
||||
|
||||
def __getitem__(self, key: str) -> Any:
|
||||
return super(_OMD, self).__getitem__(key)[-1]
|
||||
|
||||
def getlast(self, key: str) -> Any:
|
||||
return super(_OMD, self).__getitem__(key)[-1]
|
||||
|
||||
def setlast(self, key: str, value: Any) -> None:
|
||||
if key not in self:
|
||||
super(_OMD, self).__setitem__(key, [value])
|
||||
return
|
||||
|
||||
prior = super(_OMD, self).__getitem__(key)
|
||||
prior[-1] = value
|
||||
|
||||
def get(self, key: str, default: Union[_T, None] = None) -> Union[_T, None]:
|
||||
return super(_OMD, self).get(key, [default])[-1]
|
||||
|
||||
def getall(self, key: str) -> List[_T]:
|
||||
return super(_OMD, self).__getitem__(key)
|
||||
|
||||
def items(self) -> List[Tuple[str, _T]]: # type: ignore[override]
|
||||
"""List of (key, last value for key)."""
|
||||
return [(k, self[k]) for k in self]
|
||||
|
||||
def items_all(self) -> List[Tuple[str, List[_T]]]:
|
||||
"""List of (key, list of values for key)."""
|
||||
return [(k, self.getall(k)) for k in self]
|
||||
|
||||
|
||||
def get_config_path(config_level: Lit_config_levels) -> str:
|
||||
|
||||
# we do not support an absolute path of the gitconfig on windows ,
|
||||
# use the global config instead
|
||||
if is_win and config_level == "system":
|
||||
config_level = "global"
|
||||
|
||||
if config_level == "system":
|
||||
return "/etc/gitconfig"
|
||||
elif config_level == "user":
|
||||
config_home = os.environ.get("XDG_CONFIG_HOME") or osp.join(os.environ.get("HOME", "~"), ".config")
|
||||
return osp.normpath(osp.expanduser(osp.join(config_home, "git", "config")))
|
||||
elif config_level == "global":
|
||||
return osp.normpath(osp.expanduser("~/.gitconfig"))
|
||||
elif config_level == "repository":
|
||||
raise ValueError("No repo to get repository configuration from. Use Repo._get_config_path")
|
||||
else:
|
||||
# Should not reach here. Will raise ValueError if does. Static typing will warn missing elifs
|
||||
assert_never(
|
||||
config_level, # type: ignore[unreachable]
|
||||
ValueError(f"Invalid configuration level: {config_level!r}"),
|
||||
)
|
||||
|
||||
|
||||
class GitConfigParser(cp.RawConfigParser, metaclass=MetaParserBuilder):
|
||||
|
||||
"""Implements specifics required to read git style configuration files.
|
||||
|
||||
This variation behaves much like the git.config command such that the configuration
|
||||
will be read on demand based on the filepath given during initialization.
|
||||
|
||||
The changes will automatically be written once the instance goes out of scope, but
|
||||
can be triggered manually as well.
|
||||
|
||||
The configuration file will be locked if you intend to change values preventing other
|
||||
instances to write concurrently.
|
||||
|
||||
:note:
|
||||
The config is case-sensitive even when queried, hence section and option names
|
||||
must match perfectly.
|
||||
If used as a context manager, will release the locked file."""
|
||||
|
||||
# { Configuration
|
||||
# The lock type determines the type of lock to use in new configuration readers.
|
||||
# They must be compatible to the LockFile interface.
|
||||
# A suitable alternative would be the BlockingLockFile
|
||||
t_lock = LockFile
|
||||
re_comment = re.compile(r"^\s*[#;]")
|
||||
|
||||
# } END configuration
|
||||
|
||||
optvalueonly_source = r"\s*(?P<option>[^:=\s][^:=]*)"
|
||||
|
||||
OPTVALUEONLY = re.compile(optvalueonly_source)
|
||||
|
||||
OPTCRE = re.compile(optvalueonly_source + r"\s*(?P<vi>[:=])\s*" + r"(?P<value>.*)$")
|
||||
|
||||
del optvalueonly_source
|
||||
|
||||
# list of RawConfigParser methods able to change the instance
|
||||
_mutating_methods_ = ("add_section", "remove_section", "remove_option", "set")
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
file_or_files: Union[None, PathLike, "BytesIO", Sequence[Union[PathLike, "BytesIO"]]] = None,
|
||||
read_only: bool = True,
|
||||
merge_includes: bool = True,
|
||||
config_level: Union[Lit_config_levels, None] = None,
|
||||
repo: Union["Repo", None] = None,
|
||||
) -> None:
|
||||
"""Initialize a configuration reader to read the given file_or_files and to
|
||||
possibly allow changes to it by setting read_only False
|
||||
|
||||
:param file_or_files:
|
||||
A single file path or file objects or multiple of these
|
||||
|
||||
:param read_only:
|
||||
If True, the ConfigParser may only read the data , but not change it.
|
||||
If False, only a single file path or file object may be given. We will write back the changes
|
||||
when they happen, or when the ConfigParser is released. This will not happen if other
|
||||
configuration files have been included
|
||||
:param merge_includes: if True, we will read files mentioned in [include] sections and merge their
|
||||
contents into ours. This makes it impossible to write back an individual configuration file.
|
||||
Thus, if you want to modify a single configuration file, turn this off to leave the original
|
||||
dataset unaltered when reading it.
|
||||
:param repo: Reference to repository to use if [includeIf] sections are found in configuration files.
|
||||
|
||||
"""
|
||||
cp.RawConfigParser.__init__(self, dict_type=_OMD)
|
||||
self._dict: Callable[..., _OMD] # type: ignore # mypy/typeshed bug?
|
||||
self._defaults: _OMD
|
||||
self._sections: _OMD # type: ignore # mypy/typeshed bug?
|
||||
|
||||
# Used in python 3, needs to stay in sync with sections for underlying implementation to work
|
||||
if not hasattr(self, "_proxies"):
|
||||
self._proxies = self._dict()
|
||||
|
||||
if file_or_files is not None:
|
||||
self._file_or_files: Union[PathLike, "BytesIO", Sequence[Union[PathLike, "BytesIO"]]] = file_or_files
|
||||
else:
|
||||
if config_level is None:
|
||||
if read_only:
|
||||
self._file_or_files = [
|
||||
get_config_path(cast(Lit_config_levels, f)) for f in CONFIG_LEVELS if f != "repository"
|
||||
]
|
||||
else:
|
||||
raise ValueError("No configuration level or configuration files specified")
|
||||
else:
|
||||
self._file_or_files = [get_config_path(config_level)]
|
||||
|
||||
self._read_only = read_only
|
||||
self._dirty = False
|
||||
self._is_initialized = False
|
||||
self._merge_includes = merge_includes
|
||||
self._repo = repo
|
||||
self._lock: Union["LockFile", None] = None
|
||||
self._acquire_lock()
|
||||
|
||||
def _acquire_lock(self) -> None:
|
||||
if not self._read_only:
|
||||
if not self._lock:
|
||||
if isinstance(self._file_or_files, (str, os.PathLike)):
|
||||
file_or_files = self._file_or_files
|
||||
elif isinstance(self._file_or_files, (tuple, list, Sequence)):
|
||||
raise ValueError(
|
||||
"Write-ConfigParsers can operate on a single file only, multiple files have been passed"
|
||||
)
|
||||
else:
|
||||
file_or_files = self._file_or_files.name
|
||||
|
||||
# END get filename from handle/stream
|
||||
# initialize lock base - we want to write
|
||||
self._lock = self.t_lock(file_or_files)
|
||||
# END lock check
|
||||
|
||||
self._lock._obtain_lock()
|
||||
# END read-only check
|
||||
|
||||
def __del__(self) -> None:
|
||||
"""Write pending changes if required and release locks"""
|
||||
# NOTE: only consistent in PY2
|
||||
self.release()
|
||||
|
||||
def __enter__(self) -> "GitConfigParser":
|
||||
self._acquire_lock()
|
||||
return self
|
||||
|
||||
def __exit__(self, *args: Any) -> None:
|
||||
self.release()
|
||||
|
||||
def release(self) -> None:
|
||||
"""Flush changes and release the configuration write lock. This instance must not be used anymore afterwards.
|
||||
In Python 3, it's required to explicitly release locks and flush changes, as __del__ is not called
|
||||
deterministically anymore."""
|
||||
# checking for the lock here makes sure we do not raise during write()
|
||||
# in case an invalid parser was created who could not get a lock
|
||||
if self.read_only or (self._lock and not self._lock._has_lock()):
|
||||
return
|
||||
|
||||
try:
|
||||
try:
|
||||
self.write()
|
||||
except IOError:
|
||||
log.error("Exception during destruction of GitConfigParser", exc_info=True)
|
||||
except ReferenceError:
|
||||
# This happens in PY3 ... and usually means that some state cannot be written
|
||||
# as the sections dict cannot be iterated
|
||||
# Usually when shutting down the interpreter, don'y know how to fix this
|
||||
pass
|
||||
finally:
|
||||
if self._lock is not None:
|
||||
self._lock._release_lock()
|
||||
|
||||
def optionxform(self, optionstr: str) -> str:
|
||||
"""Do not transform options in any way when writing"""
|
||||
return optionstr
|
||||
|
||||
def _read(self, fp: Union[BufferedReader, IO[bytes]], fpname: str) -> None:
|
||||
"""A direct copy of the py2.4 version of the super class's _read method
|
||||
to assure it uses ordered dicts. Had to change one line to make it work.
|
||||
|
||||
Future versions have this fixed, but in fact its quite embarrassing for the
|
||||
guys not to have done it right in the first place !
|
||||
|
||||
Removed big comments to make it more compact.
|
||||
|
||||
Made sure it ignores initial whitespace as git uses tabs"""
|
||||
cursect = None # None, or a dictionary
|
||||
optname = None
|
||||
lineno = 0
|
||||
is_multi_line = False
|
||||
e = None # None, or an exception
|
||||
|
||||
def string_decode(v: str) -> str:
|
||||
if v[-1] == "\\":
|
||||
v = v[:-1]
|
||||
# end cut trailing escapes to prevent decode error
|
||||
|
||||
return v.encode(defenc).decode("unicode_escape")
|
||||
# end
|
||||
|
||||
# end
|
||||
|
||||
while True:
|
||||
# we assume to read binary !
|
||||
line = fp.readline().decode(defenc)
|
||||
if not line:
|
||||
break
|
||||
lineno = lineno + 1
|
||||
# comment or blank line?
|
||||
if line.strip() == "" or self.re_comment.match(line):
|
||||
continue
|
||||
if line.split(None, 1)[0].lower() == "rem" and line[0] in "rR":
|
||||
# no leading whitespace
|
||||
continue
|
||||
|
||||
# is it a section header?
|
||||
mo = self.SECTCRE.match(line.strip())
|
||||
if not is_multi_line and mo:
|
||||
sectname: str = mo.group("header").strip()
|
||||
if sectname in self._sections:
|
||||
cursect = self._sections[sectname]
|
||||
elif sectname == cp.DEFAULTSECT:
|
||||
cursect = self._defaults
|
||||
else:
|
||||
cursect = self._dict((("__name__", sectname),))
|
||||
self._sections[sectname] = cursect
|
||||
self._proxies[sectname] = None
|
||||
# So sections can't start with a continuation line
|
||||
optname = None
|
||||
# no section header in the file?
|
||||
elif cursect is None:
|
||||
raise cp.MissingSectionHeaderError(fpname, lineno, line)
|
||||
# an option line?
|
||||
elif not is_multi_line:
|
||||
mo = self.OPTCRE.match(line)
|
||||
if mo:
|
||||
# We might just have handled the last line, which could contain a quotation we want to remove
|
||||
optname, vi, optval = mo.group("option", "vi", "value")
|
||||
if vi in ("=", ":") and ";" in optval and not optval.strip().startswith('"'):
|
||||
pos = optval.find(";")
|
||||
if pos != -1 and optval[pos - 1].isspace():
|
||||
optval = optval[:pos]
|
||||
optval = optval.strip()
|
||||
if optval == '""':
|
||||
optval = ""
|
||||
# end handle empty string
|
||||
optname = self.optionxform(optname.rstrip())
|
||||
if len(optval) > 1 and optval[0] == '"' and optval[-1] != '"':
|
||||
is_multi_line = True
|
||||
optval = string_decode(optval[1:])
|
||||
# end handle multi-line
|
||||
# preserves multiple values for duplicate optnames
|
||||
cursect.add(optname, optval)
|
||||
else:
|
||||
# check if it's an option with no value - it's just ignored by git
|
||||
if not self.OPTVALUEONLY.match(line):
|
||||
if not e:
|
||||
e = cp.ParsingError(fpname)
|
||||
e.append(lineno, repr(line))
|
||||
continue
|
||||
else:
|
||||
line = line.rstrip()
|
||||
if line.endswith('"'):
|
||||
is_multi_line = False
|
||||
line = line[:-1]
|
||||
# end handle quotations
|
||||
optval = cursect.getlast(optname)
|
||||
cursect.setlast(optname, optval + string_decode(line))
|
||||
# END parse section or option
|
||||
# END while reading
|
||||
|
||||
# if any parsing errors occurred, raise an exception
|
||||
if e:
|
||||
raise e
|
||||
|
||||
def _has_includes(self) -> Union[bool, int]:
|
||||
return self._merge_includes and len(self._included_paths())
|
||||
|
||||
def _included_paths(self) -> List[Tuple[str, str]]:
|
||||
"""Return List all paths that must be included to configuration
|
||||
as Tuples of (option, value).
|
||||
"""
|
||||
paths = []
|
||||
|
||||
for section in self.sections():
|
||||
if section == "include":
|
||||
paths += self.items(section)
|
||||
|
||||
match = CONDITIONAL_INCLUDE_REGEXP.search(section)
|
||||
if match is None or self._repo is None:
|
||||
continue
|
||||
|
||||
keyword = match.group(1)
|
||||
value = match.group(2).strip()
|
||||
|
||||
if keyword in ["gitdir", "gitdir/i"]:
|
||||
value = osp.expanduser(value)
|
||||
|
||||
if not any(value.startswith(s) for s in ["./", "/"]):
|
||||
value = "**/" + value
|
||||
if value.endswith("/"):
|
||||
value += "**"
|
||||
|
||||
# Ensure that glob is always case insensitive if required.
|
||||
if keyword.endswith("/i"):
|
||||
value = re.sub(
|
||||
r"[a-zA-Z]",
|
||||
lambda m: "[{}{}]".format(m.group().lower(), m.group().upper()),
|
||||
value,
|
||||
)
|
||||
if self._repo.git_dir:
|
||||
if fnmatch.fnmatchcase(str(self._repo.git_dir), value):
|
||||
paths += self.items(section)
|
||||
|
||||
elif keyword == "onbranch":
|
||||
try:
|
||||
branch_name = self._repo.active_branch.name
|
||||
except TypeError:
|
||||
# Ignore section if active branch cannot be retrieved.
|
||||
continue
|
||||
|
||||
if fnmatch.fnmatchcase(branch_name, value):
|
||||
paths += self.items(section)
|
||||
|
||||
return paths
|
||||
|
||||
def read(self) -> None: # type: ignore[override]
|
||||
"""Reads the data stored in the files we have been initialized with. It will
|
||||
ignore files that cannot be read, possibly leaving an empty configuration
|
||||
|
||||
:return: Nothing
|
||||
:raise IOError: if a file cannot be handled"""
|
||||
if self._is_initialized:
|
||||
return None
|
||||
self._is_initialized = True
|
||||
|
||||
files_to_read: List[Union[PathLike, IO]] = [""]
|
||||
if isinstance(self._file_or_files, (str, os.PathLike)):
|
||||
# for str or Path, as str is a type of Sequence
|
||||
files_to_read = [self._file_or_files]
|
||||
elif not isinstance(self._file_or_files, (tuple, list, Sequence)):
|
||||
# could merge with above isinstance once runtime type known
|
||||
files_to_read = [self._file_or_files]
|
||||
else: # for lists or tuples
|
||||
files_to_read = list(self._file_or_files)
|
||||
# end assure we have a copy of the paths to handle
|
||||
|
||||
seen = set(files_to_read)
|
||||
num_read_include_files = 0
|
||||
while files_to_read:
|
||||
file_path = files_to_read.pop(0)
|
||||
file_ok = False
|
||||
|
||||
if hasattr(file_path, "seek"):
|
||||
# must be a file objectfile-object
|
||||
file_path = cast(IO[bytes], file_path) # replace with assert to narrow type, once sure
|
||||
self._read(file_path, file_path.name)
|
||||
else:
|
||||
# assume a path if it is not a file-object
|
||||
file_path = cast(PathLike, file_path)
|
||||
try:
|
||||
with open(file_path, "rb") as fp:
|
||||
file_ok = True
|
||||
self._read(fp, fp.name)
|
||||
except IOError:
|
||||
continue
|
||||
|
||||
# Read includes and append those that we didn't handle yet
|
||||
# We expect all paths to be normalized and absolute (and will assure that is the case)
|
||||
if self._has_includes():
|
||||
for _, include_path in self._included_paths():
|
||||
if include_path.startswith("~"):
|
||||
include_path = osp.expanduser(include_path)
|
||||
if not osp.isabs(include_path):
|
||||
if not file_ok:
|
||||
continue
|
||||
# end ignore relative paths if we don't know the configuration file path
|
||||
file_path = cast(PathLike, file_path)
|
||||
assert osp.isabs(file_path), "Need absolute paths to be sure our cycle checks will work"
|
||||
include_path = osp.join(osp.dirname(file_path), include_path)
|
||||
# end make include path absolute
|
||||
include_path = osp.normpath(include_path)
|
||||
if include_path in seen or not os.access(include_path, os.R_OK):
|
||||
continue
|
||||
seen.add(include_path)
|
||||
# insert included file to the top to be considered first
|
||||
files_to_read.insert(0, include_path)
|
||||
num_read_include_files += 1
|
||||
# each include path in configuration file
|
||||
# end handle includes
|
||||
# END for each file object to read
|
||||
|
||||
# If there was no file included, we can safely write back (potentially) the configuration file
|
||||
# without altering it's meaning
|
||||
if num_read_include_files == 0:
|
||||
self._merge_includes = False
|
||||
# end
|
||||
|
||||
def _write(self, fp: IO) -> None:
|
||||
"""Write an .ini-format representation of the configuration state in
|
||||
git compatible format"""
|
||||
|
||||
def write_section(name: str, section_dict: _OMD) -> None:
|
||||
fp.write(("[%s]\n" % name).encode(defenc))
|
||||
|
||||
values: Sequence[str] # runtime only gets str in tests, but should be whatever _OMD stores
|
||||
v: str
|
||||
for (key, values) in section_dict.items_all():
|
||||
if key == "__name__":
|
||||
continue
|
||||
|
||||
for v in values:
|
||||
fp.write(("\t%s = %s\n" % (key, self._value_to_string(v).replace("\n", "\n\t"))).encode(defenc))
|
||||
# END if key is not __name__
|
||||
|
||||
# END section writing
|
||||
|
||||
if self._defaults:
|
||||
write_section(cp.DEFAULTSECT, self._defaults)
|
||||
value: _OMD
|
||||
|
||||
for name, value in self._sections.items():
|
||||
write_section(name, value)
|
||||
|
||||
def items(self, section_name: str) -> List[Tuple[str, str]]: # type: ignore[override]
|
||||
""":return: list((option, value), ...) pairs of all items in the given section"""
|
||||
return [(k, v) for k, v in super(GitConfigParser, self).items(section_name) if k != "__name__"]
|
||||
|
||||
def items_all(self, section_name: str) -> List[Tuple[str, List[str]]]:
|
||||
""":return: list((option, [values...]), ...) pairs of all items in the given section"""
|
||||
rv = _OMD(self._defaults)
|
||||
|
||||
for k, vs in self._sections[section_name].items_all():
|
||||
if k == "__name__":
|
||||
continue
|
||||
|
||||
if k in rv and rv.getall(k) == vs:
|
||||
continue
|
||||
|
||||
for v in vs:
|
||||
rv.add(k, v)
|
||||
|
||||
return rv.items_all()
|
||||
|
||||
@needs_values
|
||||
def write(self) -> None:
|
||||
"""Write changes to our file, if there are changes at all
|
||||
|
||||
:raise IOError: if this is a read-only writer instance or if we could not obtain
|
||||
a file lock"""
|
||||
self._assure_writable("write")
|
||||
if not self._dirty:
|
||||
return None
|
||||
|
||||
if isinstance(self._file_or_files, (list, tuple)):
|
||||
raise AssertionError(
|
||||
"Cannot write back if there is not exactly a single file to write to, have %i files"
|
||||
% len(self._file_or_files)
|
||||
)
|
||||
# end assert multiple files
|
||||
|
||||
if self._has_includes():
|
||||
log.debug(
|
||||
"Skipping write-back of configuration file as include files were merged in."
|
||||
+ "Set merge_includes=False to prevent this."
|
||||
)
|
||||
return None
|
||||
# end
|
||||
|
||||
fp = self._file_or_files
|
||||
|
||||
# we have a physical file on disk, so get a lock
|
||||
is_file_lock = isinstance(fp, (str, os.PathLike, IOBase)) # can't use Pathlike until 3.5 dropped
|
||||
if is_file_lock and self._lock is not None: # else raise Error?
|
||||
self._lock._obtain_lock()
|
||||
|
||||
if not hasattr(fp, "seek"):
|
||||
fp = cast(PathLike, fp)
|
||||
with open(fp, "wb") as fp_open:
|
||||
self._write(fp_open)
|
||||
else:
|
||||
fp = cast("BytesIO", fp)
|
||||
fp.seek(0)
|
||||
# make sure we do not overwrite into an existing file
|
||||
if hasattr(fp, "truncate"):
|
||||
fp.truncate()
|
||||
self._write(fp)
|
||||
|
||||
def _assure_writable(self, method_name: str) -> None:
|
||||
if self.read_only:
|
||||
raise IOError("Cannot execute non-constant method %s.%s" % (self, method_name))
|
||||
|
||||
def add_section(self, section: str) -> None:
|
||||
"""Assures added options will stay in order"""
|
||||
return super(GitConfigParser, self).add_section(section)
|
||||
|
||||
@property
|
||||
def read_only(self) -> bool:
|
||||
""":return: True if this instance may change the configuration file"""
|
||||
return self._read_only
|
||||
|
||||
def get_value(
|
||||
self,
|
||||
section: str,
|
||||
option: str,
|
||||
default: Union[int, float, str, bool, None] = None,
|
||||
) -> Union[int, float, str, bool]:
|
||||
# can default or return type include bool?
|
||||
"""Get an option's value.
|
||||
|
||||
If multiple values are specified for this option in the section, the
|
||||
last one specified is returned.
|
||||
|
||||
:param default:
|
||||
If not None, the given default value will be returned in case
|
||||
the option did not exist
|
||||
:return: a properly typed value, either int, float or string
|
||||
|
||||
:raise TypeError: in case the value could not be understood
|
||||
Otherwise the exceptions known to the ConfigParser will be raised."""
|
||||
try:
|
||||
valuestr = self.get(section, option)
|
||||
except Exception:
|
||||
if default is not None:
|
||||
return default
|
||||
raise
|
||||
|
||||
return self._string_to_value(valuestr)
|
||||
|
||||
def get_values(
|
||||
self,
|
||||
section: str,
|
||||
option: str,
|
||||
default: Union[int, float, str, bool, None] = None,
|
||||
) -> List[Union[int, float, str, bool]]:
|
||||
"""Get an option's values.
|
||||
|
||||
If multiple values are specified for this option in the section, all are
|
||||
returned.
|
||||
|
||||
:param default:
|
||||
If not None, a list containing the given default value will be
|
||||
returned in case the option did not exist
|
||||
:return: a list of properly typed values, either int, float or string
|
||||
|
||||
:raise TypeError: in case the value could not be understood
|
||||
Otherwise the exceptions known to the ConfigParser will be raised."""
|
||||
try:
|
||||
self.sections()
|
||||
lst = self._sections[section].getall(option)
|
||||
except Exception:
|
||||
if default is not None:
|
||||
return [default]
|
||||
raise
|
||||
|
||||
return [self._string_to_value(valuestr) for valuestr in lst]
|
||||
|
||||
def _string_to_value(self, valuestr: str) -> Union[int, float, str, bool]:
|
||||
types = (int, float)
|
||||
for numtype in types:
|
||||
try:
|
||||
val = numtype(valuestr)
|
||||
# truncated value ?
|
||||
if val != float(valuestr):
|
||||
continue
|
||||
return val
|
||||
except (ValueError, TypeError):
|
||||
continue
|
||||
# END for each numeric type
|
||||
|
||||
# try boolean values as git uses them
|
||||
vl = valuestr.lower()
|
||||
if vl == "false":
|
||||
return False
|
||||
if vl == "true":
|
||||
return True
|
||||
|
||||
if not isinstance(valuestr, str):
|
||||
raise TypeError(
|
||||
"Invalid value type: only int, long, float and str are allowed",
|
||||
valuestr,
|
||||
)
|
||||
|
||||
return valuestr
|
||||
|
||||
def _value_to_string(self, value: Union[str, bytes, int, float, bool]) -> str:
|
||||
if isinstance(value, (int, float, bool)):
|
||||
return str(value)
|
||||
return force_text(value)
|
||||
|
||||
@needs_values
|
||||
@set_dirty_and_flush_changes
|
||||
def set_value(self, section: str, option: str, value: Union[str, bytes, int, float, bool]) -> "GitConfigParser":
|
||||
"""Sets the given option in section to the given value.
|
||||
It will create the section if required, and will not throw as opposed to the default
|
||||
ConfigParser 'set' method.
|
||||
|
||||
:param section: Name of the section in which the option resides or should reside
|
||||
:param option: Name of the options whose value to set
|
||||
|
||||
:param value: Value to set the option to. It must be a string or convertible
|
||||
to a string
|
||||
:return: this instance"""
|
||||
if not self.has_section(section):
|
||||
self.add_section(section)
|
||||
self.set(section, option, self._value_to_string(value))
|
||||
return self
|
||||
|
||||
@needs_values
|
||||
@set_dirty_and_flush_changes
|
||||
def add_value(self, section: str, option: str, value: Union[str, bytes, int, float, bool]) -> "GitConfigParser":
|
||||
"""Adds a value for the given option in section.
|
||||
It will create the section if required, and will not throw as opposed to the default
|
||||
ConfigParser 'set' method. The value becomes the new value of the option as returned
|
||||
by 'get_value', and appends to the list of values returned by 'get_values`'.
|
||||
|
||||
:param section: Name of the section in which the option resides or should reside
|
||||
:param option: Name of the option
|
||||
|
||||
:param value: Value to add to option. It must be a string or convertible
|
||||
to a string
|
||||
:return: this instance"""
|
||||
if not self.has_section(section):
|
||||
self.add_section(section)
|
||||
self._sections[section].add(option, self._value_to_string(value))
|
||||
return self
|
||||
|
||||
def rename_section(self, section: str, new_name: str) -> "GitConfigParser":
|
||||
"""rename the given section to new_name
|
||||
:raise ValueError: if section doesn't exit
|
||||
:raise ValueError: if a section with new_name does already exist
|
||||
:return: this instance
|
||||
"""
|
||||
if not self.has_section(section):
|
||||
raise ValueError("Source section '%s' doesn't exist" % section)
|
||||
if self.has_section(new_name):
|
||||
raise ValueError("Destination section '%s' already exists" % new_name)
|
||||
|
||||
super(GitConfigParser, self).add_section(new_name)
|
||||
new_section = self._sections[new_name]
|
||||
for k, vs in self.items_all(section):
|
||||
new_section.setall(k, vs)
|
||||
# end for each value to copy
|
||||
|
||||
# This call writes back the changes, which is why we don't have the respective decorator
|
||||
self.remove_section(section)
|
||||
return self
|
||||
63
zero-cost-nas/.eggs/GitPython-3.1.31-py3.8.egg/git/db.py
Normal file
63
zero-cost-nas/.eggs/GitPython-3.1.31-py3.8.egg/git/db.py
Normal file
@@ -0,0 +1,63 @@
|
||||
"""Module with our own gitdb implementation - it uses the git command"""
|
||||
from git.util import bin_to_hex, hex_to_bin
|
||||
from gitdb.base import OInfo, OStream
|
||||
from gitdb.db import GitDB # @UnusedImport
|
||||
from gitdb.db import LooseObjectDB
|
||||
|
||||
from gitdb.exc import BadObject
|
||||
from git.exc import GitCommandError
|
||||
|
||||
# typing-------------------------------------------------
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
from git.types import PathLike
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from git.cmd import Git
|
||||
|
||||
|
||||
# --------------------------------------------------------
|
||||
|
||||
__all__ = ("GitCmdObjectDB", "GitDB")
|
||||
|
||||
|
||||
class GitCmdObjectDB(LooseObjectDB):
|
||||
|
||||
"""A database representing the default git object store, which includes loose
|
||||
objects, pack files and an alternates file
|
||||
|
||||
It will create objects only in the loose object database.
|
||||
:note: for now, we use the git command to do all the lookup, just until he
|
||||
have packs and the other implementations
|
||||
"""
|
||||
|
||||
def __init__(self, root_path: PathLike, git: "Git") -> None:
|
||||
"""Initialize this instance with the root and a git command"""
|
||||
super(GitCmdObjectDB, self).__init__(root_path)
|
||||
self._git = git
|
||||
|
||||
def info(self, binsha: bytes) -> OInfo:
|
||||
hexsha, typename, size = self._git.get_object_header(bin_to_hex(binsha))
|
||||
return OInfo(hex_to_bin(hexsha), typename, size)
|
||||
|
||||
def stream(self, binsha: bytes) -> OStream:
|
||||
"""For now, all lookup is done by git itself"""
|
||||
hexsha, typename, size, stream = self._git.stream_object_data(bin_to_hex(binsha))
|
||||
return OStream(hex_to_bin(hexsha), typename, size, stream)
|
||||
|
||||
# { Interface
|
||||
|
||||
def partial_to_complete_sha_hex(self, partial_hexsha: str) -> bytes:
|
||||
""":return: Full binary 20 byte sha from the given partial hexsha
|
||||
:raise AmbiguousObjectName:
|
||||
:raise BadObject:
|
||||
:note: currently we only raise BadObject as git does not communicate
|
||||
AmbiguousObjects separately"""
|
||||
try:
|
||||
hexsha, _typename, _size = self._git.get_object_header(partial_hexsha)
|
||||
return hex_to_bin(hexsha)
|
||||
except (GitCommandError, ValueError) as e:
|
||||
raise BadObject(partial_hexsha) from e
|
||||
# END handle exceptions
|
||||
|
||||
# } END interface
|
||||
662
zero-cost-nas/.eggs/GitPython-3.1.31-py3.8.egg/git/diff.py
Normal file
662
zero-cost-nas/.eggs/GitPython-3.1.31-py3.8.egg/git/diff.py
Normal file
@@ -0,0 +1,662 @@
|
||||
# diff.py
|
||||
# Copyright (C) 2008, 2009 Michael Trier (mtrier@gmail.com) and contributors
|
||||
#
|
||||
# This module is part of GitPython and is released under
|
||||
# the BSD License: http://www.opensource.org/licenses/bsd-license.php
|
||||
|
||||
import re
|
||||
from git.cmd import handle_process_output
|
||||
from git.compat import defenc
|
||||
from git.util import finalize_process, hex_to_bin
|
||||
|
||||
from .objects.blob import Blob
|
||||
from .objects.util import mode_str_to_int
|
||||
|
||||
|
||||
# typing ------------------------------------------------------------------
|
||||
|
||||
from typing import (
|
||||
Any,
|
||||
Iterator,
|
||||
List,
|
||||
Match,
|
||||
Optional,
|
||||
Tuple,
|
||||
Type,
|
||||
TypeVar,
|
||||
Union,
|
||||
TYPE_CHECKING,
|
||||
cast,
|
||||
)
|
||||
from git.types import PathLike, Literal
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from .objects.tree import Tree
|
||||
from .objects import Commit
|
||||
from git.repo.base import Repo
|
||||
from git.objects.base import IndexObject
|
||||
from subprocess import Popen
|
||||
from git import Git
|
||||
|
||||
Lit_change_type = Literal["A", "D", "C", "M", "R", "T", "U"]
|
||||
|
||||
|
||||
# def is_change_type(inp: str) -> TypeGuard[Lit_change_type]:
|
||||
# # return True
|
||||
# return inp in ['A', 'D', 'C', 'M', 'R', 'T', 'U']
|
||||
|
||||
# ------------------------------------------------------------------------
|
||||
|
||||
|
||||
__all__ = ("Diffable", "DiffIndex", "Diff", "NULL_TREE")
|
||||
|
||||
# Special object to compare against the empty tree in diffs
|
||||
NULL_TREE = object()
|
||||
|
||||
_octal_byte_re = re.compile(b"\\\\([0-9]{3})")
|
||||
|
||||
|
||||
def _octal_repl(matchobj: Match) -> bytes:
|
||||
value = matchobj.group(1)
|
||||
value = int(value, 8)
|
||||
value = bytes(bytearray((value,)))
|
||||
return value
|
||||
|
||||
|
||||
def decode_path(path: bytes, has_ab_prefix: bool = True) -> Optional[bytes]:
|
||||
if path == b"/dev/null":
|
||||
return None
|
||||
|
||||
if path.startswith(b'"') and path.endswith(b'"'):
|
||||
path = path[1:-1].replace(b"\\n", b"\n").replace(b"\\t", b"\t").replace(b'\\"', b'"').replace(b"\\\\", b"\\")
|
||||
|
||||
path = _octal_byte_re.sub(_octal_repl, path)
|
||||
|
||||
if has_ab_prefix:
|
||||
assert path.startswith(b"a/") or path.startswith(b"b/")
|
||||
path = path[2:]
|
||||
|
||||
return path
|
||||
|
||||
|
||||
class Diffable(object):
|
||||
|
||||
"""Common interface for all object that can be diffed against another object of compatible type.
|
||||
|
||||
:note:
|
||||
Subclasses require a repo member as it is the case for Object instances, for practical
|
||||
reasons we do not derive from Object."""
|
||||
|
||||
__slots__ = ()
|
||||
|
||||
# standin indicating you want to diff against the index
|
||||
class Index(object):
|
||||
pass
|
||||
|
||||
def _process_diff_args(
|
||||
self, args: List[Union[str, "Diffable", Type["Diffable.Index"], object]]
|
||||
) -> List[Union[str, "Diffable", Type["Diffable.Index"], object]]:
|
||||
"""
|
||||
:return:
|
||||
possibly altered version of the given args list.
|
||||
Method is called right before git command execution.
|
||||
Subclasses can use it to alter the behaviour of the superclass"""
|
||||
return args
|
||||
|
||||
def diff(
|
||||
self,
|
||||
other: Union[Type["Index"], "Tree", "Commit", None, str, object] = Index,
|
||||
paths: Union[PathLike, List[PathLike], Tuple[PathLike, ...], None] = None,
|
||||
create_patch: bool = False,
|
||||
**kwargs: Any,
|
||||
) -> "DiffIndex":
|
||||
"""Creates diffs between two items being trees, trees and index or an
|
||||
index and the working tree. It will detect renames automatically.
|
||||
|
||||
:param other:
|
||||
Is the item to compare us with.
|
||||
If None, we will be compared to the working tree.
|
||||
If Treeish, it will be compared against the respective tree
|
||||
If Index ( type ), it will be compared against the index.
|
||||
If git.NULL_TREE, it will compare against the empty tree.
|
||||
It defaults to Index to assure the method will not by-default fail
|
||||
on bare repositories.
|
||||
|
||||
:param paths:
|
||||
is a list of paths or a single path to limit the diff to.
|
||||
It will only include at least one of the given path or paths.
|
||||
|
||||
:param create_patch:
|
||||
If True, the returned Diff contains a detailed patch that if applied
|
||||
makes the self to other. Patches are somewhat costly as blobs have to be read
|
||||
and diffed.
|
||||
|
||||
:param kwargs:
|
||||
Additional arguments passed to git-diff, such as
|
||||
R=True to swap both sides of the diff.
|
||||
|
||||
:return: git.DiffIndex
|
||||
|
||||
:note:
|
||||
On a bare repository, 'other' needs to be provided as Index or as
|
||||
as Tree/Commit, or a git command error will occur"""
|
||||
args: List[Union[PathLike, Diffable, Type["Diffable.Index"], object]] = []
|
||||
args.append("--abbrev=40") # we need full shas
|
||||
args.append("--full-index") # get full index paths, not only filenames
|
||||
|
||||
# remove default '-M' arg (check for renames) if user is overriding it
|
||||
if not any(x in kwargs for x in ('find_renames', 'no_renames', 'M')):
|
||||
args.append("-M")
|
||||
|
||||
if create_patch:
|
||||
args.append("-p")
|
||||
else:
|
||||
args.append("--raw")
|
||||
args.append("-z")
|
||||
|
||||
# in any way, assure we don't see colored output,
|
||||
# fixes https://github.com/gitpython-developers/GitPython/issues/172
|
||||
args.append("--no-color")
|
||||
|
||||
if paths is not None and not isinstance(paths, (tuple, list)):
|
||||
paths = [paths]
|
||||
|
||||
if hasattr(self, "Has_Repo"):
|
||||
self.repo: "Repo" = self.repo
|
||||
|
||||
diff_cmd = self.repo.git.diff
|
||||
if other is self.Index:
|
||||
args.insert(0, "--cached")
|
||||
elif other is NULL_TREE:
|
||||
args.insert(0, "-r") # recursive diff-tree
|
||||
args.insert(0, "--root")
|
||||
diff_cmd = self.repo.git.diff_tree
|
||||
elif other is not None:
|
||||
args.insert(0, "-r") # recursive diff-tree
|
||||
args.insert(0, other)
|
||||
diff_cmd = self.repo.git.diff_tree
|
||||
|
||||
args.insert(0, self)
|
||||
|
||||
# paths is list here or None
|
||||
if paths:
|
||||
args.append("--")
|
||||
args.extend(paths)
|
||||
# END paths handling
|
||||
|
||||
kwargs["as_process"] = True
|
||||
proc = diff_cmd(*self._process_diff_args(args), **kwargs)
|
||||
|
||||
diff_method = Diff._index_from_patch_format if create_patch else Diff._index_from_raw_format
|
||||
index = diff_method(self.repo, proc)
|
||||
|
||||
proc.wait()
|
||||
return index
|
||||
|
||||
|
||||
T_Diff = TypeVar("T_Diff", bound="Diff")
|
||||
|
||||
|
||||
class DiffIndex(List[T_Diff]):
|
||||
|
||||
"""Implements an Index for diffs, allowing a list of Diffs to be queried by
|
||||
the diff properties.
|
||||
|
||||
The class improves the diff handling convenience"""
|
||||
|
||||
# change type invariant identifying possible ways a blob can have changed
|
||||
# A = Added
|
||||
# D = Deleted
|
||||
# R = Renamed
|
||||
# M = Modified
|
||||
# T = Changed in the type
|
||||
change_type = ("A", "C", "D", "R", "M", "T")
|
||||
|
||||
def iter_change_type(self, change_type: Lit_change_type) -> Iterator[T_Diff]:
|
||||
"""
|
||||
:return:
|
||||
iterator yielding Diff instances that match the given change_type
|
||||
|
||||
:param change_type:
|
||||
Member of DiffIndex.change_type, namely:
|
||||
|
||||
* 'A' for added paths
|
||||
* 'D' for deleted paths
|
||||
* 'R' for renamed paths
|
||||
* 'M' for paths with modified data
|
||||
* 'T' for changed in the type paths
|
||||
"""
|
||||
if change_type not in self.change_type:
|
||||
raise ValueError("Invalid change type: %s" % change_type)
|
||||
|
||||
for diffidx in self:
|
||||
if diffidx.change_type == change_type:
|
||||
yield diffidx
|
||||
elif change_type == "A" and diffidx.new_file:
|
||||
yield diffidx
|
||||
elif change_type == "D" and diffidx.deleted_file:
|
||||
yield diffidx
|
||||
elif change_type == "C" and diffidx.copied_file:
|
||||
yield diffidx
|
||||
elif change_type == "R" and diffidx.renamed:
|
||||
yield diffidx
|
||||
elif change_type == "M" and diffidx.a_blob and diffidx.b_blob and diffidx.a_blob != diffidx.b_blob:
|
||||
yield diffidx
|
||||
# END for each diff
|
||||
|
||||
|
||||
class Diff(object):
|
||||
|
||||
"""A Diff contains diff information between two Trees.
|
||||
|
||||
It contains two sides a and b of the diff, members are prefixed with
|
||||
"a" and "b" respectively to inidcate that.
|
||||
|
||||
Diffs keep information about the changed blob objects, the file mode, renames,
|
||||
deletions and new files.
|
||||
|
||||
There are a few cases where None has to be expected as member variable value:
|
||||
|
||||
``New File``::
|
||||
|
||||
a_mode is None
|
||||
a_blob is None
|
||||
a_path is None
|
||||
|
||||
``Deleted File``::
|
||||
|
||||
b_mode is None
|
||||
b_blob is None
|
||||
b_path is None
|
||||
|
||||
``Working Tree Blobs``
|
||||
|
||||
When comparing to working trees, the working tree blob will have a null hexsha
|
||||
as a corresponding object does not yet exist. The mode will be null as well.
|
||||
But the path will be available though.
|
||||
If it is listed in a diff the working tree version of the file must
|
||||
be different to the version in the index or tree, and hence has been modified."""
|
||||
|
||||
# precompiled regex
|
||||
re_header = re.compile(
|
||||
rb"""
|
||||
^diff[ ]--git
|
||||
[ ](?P<a_path_fallback>"?[ab]/.+?"?)[ ](?P<b_path_fallback>"?[ab]/.+?"?)\n
|
||||
(?:^old[ ]mode[ ](?P<old_mode>\d+)\n
|
||||
^new[ ]mode[ ](?P<new_mode>\d+)(?:\n|$))?
|
||||
(?:^similarity[ ]index[ ]\d+%\n
|
||||
^rename[ ]from[ ](?P<rename_from>.*)\n
|
||||
^rename[ ]to[ ](?P<rename_to>.*)(?:\n|$))?
|
||||
(?:^new[ ]file[ ]mode[ ](?P<new_file_mode>.+)(?:\n|$))?
|
||||
(?:^deleted[ ]file[ ]mode[ ](?P<deleted_file_mode>.+)(?:\n|$))?
|
||||
(?:^similarity[ ]index[ ]\d+%\n
|
||||
^copy[ ]from[ ].*\n
|
||||
^copy[ ]to[ ](?P<copied_file_name>.*)(?:\n|$))?
|
||||
(?:^index[ ](?P<a_blob_id>[0-9A-Fa-f]+)
|
||||
\.\.(?P<b_blob_id>[0-9A-Fa-f]+)[ ]?(?P<b_mode>.+)?(?:\n|$))?
|
||||
(?:^---[ ](?P<a_path>[^\t\n\r\f\v]*)[\t\r\f\v]*(?:\n|$))?
|
||||
(?:^\+\+\+[ ](?P<b_path>[^\t\n\r\f\v]*)[\t\r\f\v]*(?:\n|$))?
|
||||
""",
|
||||
re.VERBOSE | re.MULTILINE,
|
||||
)
|
||||
# can be used for comparisons
|
||||
NULL_HEX_SHA = "0" * 40
|
||||
NULL_BIN_SHA = b"\0" * 20
|
||||
|
||||
__slots__ = (
|
||||
"a_blob",
|
||||
"b_blob",
|
||||
"a_mode",
|
||||
"b_mode",
|
||||
"a_rawpath",
|
||||
"b_rawpath",
|
||||
"new_file",
|
||||
"deleted_file",
|
||||
"copied_file",
|
||||
"raw_rename_from",
|
||||
"raw_rename_to",
|
||||
"diff",
|
||||
"change_type",
|
||||
"score",
|
||||
)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
repo: "Repo",
|
||||
a_rawpath: Optional[bytes],
|
||||
b_rawpath: Optional[bytes],
|
||||
a_blob_id: Union[str, bytes, None],
|
||||
b_blob_id: Union[str, bytes, None],
|
||||
a_mode: Union[bytes, str, None],
|
||||
b_mode: Union[bytes, str, None],
|
||||
new_file: bool,
|
||||
deleted_file: bool,
|
||||
copied_file: bool,
|
||||
raw_rename_from: Optional[bytes],
|
||||
raw_rename_to: Optional[bytes],
|
||||
diff: Union[str, bytes, None],
|
||||
change_type: Optional[Lit_change_type],
|
||||
score: Optional[int],
|
||||
) -> None:
|
||||
|
||||
assert a_rawpath is None or isinstance(a_rawpath, bytes)
|
||||
assert b_rawpath is None or isinstance(b_rawpath, bytes)
|
||||
self.a_rawpath = a_rawpath
|
||||
self.b_rawpath = b_rawpath
|
||||
|
||||
self.a_mode = mode_str_to_int(a_mode) if a_mode else None
|
||||
self.b_mode = mode_str_to_int(b_mode) if b_mode else None
|
||||
|
||||
# Determine whether this diff references a submodule, if it does then
|
||||
# we need to overwrite "repo" to the corresponding submodule's repo instead
|
||||
if repo and a_rawpath:
|
||||
for submodule in repo.submodules:
|
||||
if submodule.path == a_rawpath.decode(defenc, "replace"):
|
||||
if submodule.module_exists():
|
||||
repo = submodule.module()
|
||||
break
|
||||
|
||||
self.a_blob: Union["IndexObject", None]
|
||||
if a_blob_id is None or a_blob_id == self.NULL_HEX_SHA:
|
||||
self.a_blob = None
|
||||
else:
|
||||
self.a_blob = Blob(repo, hex_to_bin(a_blob_id), mode=self.a_mode, path=self.a_path)
|
||||
|
||||
self.b_blob: Union["IndexObject", None]
|
||||
if b_blob_id is None or b_blob_id == self.NULL_HEX_SHA:
|
||||
self.b_blob = None
|
||||
else:
|
||||
self.b_blob = Blob(repo, hex_to_bin(b_blob_id), mode=self.b_mode, path=self.b_path)
|
||||
|
||||
self.new_file: bool = new_file
|
||||
self.deleted_file: bool = deleted_file
|
||||
self.copied_file: bool = copied_file
|
||||
|
||||
# be clear and use None instead of empty strings
|
||||
assert raw_rename_from is None or isinstance(raw_rename_from, bytes)
|
||||
assert raw_rename_to is None or isinstance(raw_rename_to, bytes)
|
||||
self.raw_rename_from = raw_rename_from or None
|
||||
self.raw_rename_to = raw_rename_to or None
|
||||
|
||||
self.diff = diff
|
||||
self.change_type: Union[Lit_change_type, None] = change_type
|
||||
self.score = score
|
||||
|
||||
def __eq__(self, other: object) -> bool:
|
||||
for name in self.__slots__:
|
||||
if getattr(self, name) != getattr(other, name):
|
||||
return False
|
||||
# END for each name
|
||||
return True
|
||||
|
||||
def __ne__(self, other: object) -> bool:
|
||||
return not (self == other)
|
||||
|
||||
def __hash__(self) -> int:
|
||||
return hash(tuple(getattr(self, n) for n in self.__slots__))
|
||||
|
||||
def __str__(self) -> str:
|
||||
h: str = "%s"
|
||||
if self.a_blob:
|
||||
h %= self.a_blob.path
|
||||
elif self.b_blob:
|
||||
h %= self.b_blob.path
|
||||
|
||||
msg: str = ""
|
||||
line = None # temp line
|
||||
line_length = 0 # line length
|
||||
for b, n in zip((self.a_blob, self.b_blob), ("lhs", "rhs")):
|
||||
if b:
|
||||
line = "\n%s: %o | %s" % (n, b.mode, b.hexsha)
|
||||
else:
|
||||
line = "\n%s: None" % n
|
||||
# END if blob is not None
|
||||
line_length = max(len(line), line_length)
|
||||
msg += line
|
||||
# END for each blob
|
||||
|
||||
# add headline
|
||||
h += "\n" + "=" * line_length
|
||||
|
||||
if self.deleted_file:
|
||||
msg += "\nfile deleted in rhs"
|
||||
if self.new_file:
|
||||
msg += "\nfile added in rhs"
|
||||
if self.copied_file:
|
||||
msg += "\nfile %r copied from %r" % (self.b_path, self.a_path)
|
||||
if self.rename_from:
|
||||
msg += "\nfile renamed from %r" % self.rename_from
|
||||
if self.rename_to:
|
||||
msg += "\nfile renamed to %r" % self.rename_to
|
||||
if self.diff:
|
||||
msg += "\n---"
|
||||
try:
|
||||
msg += self.diff.decode(defenc) if isinstance(self.diff, bytes) else self.diff
|
||||
except UnicodeDecodeError:
|
||||
msg += "OMITTED BINARY DATA"
|
||||
# end handle encoding
|
||||
msg += "\n---"
|
||||
# END diff info
|
||||
|
||||
# Python2 silliness: have to assure we convert our likely to be unicode object to a string with the
|
||||
# right encoding. Otherwise it tries to convert it using ascii, which may fail ungracefully
|
||||
res = h + msg
|
||||
# end
|
||||
return res
|
||||
|
||||
@property
|
||||
def a_path(self) -> Optional[str]:
|
||||
return self.a_rawpath.decode(defenc, "replace") if self.a_rawpath else None
|
||||
|
||||
@property
|
||||
def b_path(self) -> Optional[str]:
|
||||
return self.b_rawpath.decode(defenc, "replace") if self.b_rawpath else None
|
||||
|
||||
@property
|
||||
def rename_from(self) -> Optional[str]:
|
||||
return self.raw_rename_from.decode(defenc, "replace") if self.raw_rename_from else None
|
||||
|
||||
@property
|
||||
def rename_to(self) -> Optional[str]:
|
||||
return self.raw_rename_to.decode(defenc, "replace") if self.raw_rename_to else None
|
||||
|
||||
@property
|
||||
def renamed(self) -> bool:
|
||||
""":returns: True if the blob of our diff has been renamed
|
||||
:note: This property is deprecated, please use ``renamed_file`` instead.
|
||||
"""
|
||||
return self.renamed_file
|
||||
|
||||
@property
|
||||
def renamed_file(self) -> bool:
|
||||
""":returns: True if the blob of our diff has been renamed"""
|
||||
return self.rename_from != self.rename_to
|
||||
|
||||
@classmethod
|
||||
def _pick_best_path(cls, path_match: bytes, rename_match: bytes, path_fallback_match: bytes) -> Optional[bytes]:
|
||||
if path_match:
|
||||
return decode_path(path_match)
|
||||
|
||||
if rename_match:
|
||||
return decode_path(rename_match, has_ab_prefix=False)
|
||||
|
||||
if path_fallback_match:
|
||||
return decode_path(path_fallback_match)
|
||||
|
||||
return None
|
||||
|
||||
@classmethod
|
||||
def _index_from_patch_format(cls, repo: "Repo", proc: Union["Popen", "Git.AutoInterrupt"]) -> DiffIndex:
|
||||
"""Create a new DiffIndex from the given text which must be in patch format
|
||||
:param repo: is the repository we are operating on - it is required
|
||||
:param stream: result of 'git diff' as a stream (supporting file protocol)
|
||||
:return: git.DiffIndex"""
|
||||
|
||||
## FIXME: Here SLURPING raw, need to re-phrase header-regexes linewise.
|
||||
text_list: List[bytes] = []
|
||||
handle_process_output(proc, text_list.append, None, finalize_process, decode_streams=False)
|
||||
|
||||
# for now, we have to bake the stream
|
||||
text = b"".join(text_list)
|
||||
index: "DiffIndex" = DiffIndex()
|
||||
previous_header: Union[Match[bytes], None] = None
|
||||
header: Union[Match[bytes], None] = None
|
||||
a_path, b_path = None, None # for mypy
|
||||
a_mode, b_mode = None, None # for mypy
|
||||
for _header in cls.re_header.finditer(text):
|
||||
(
|
||||
a_path_fallback,
|
||||
b_path_fallback,
|
||||
old_mode,
|
||||
new_mode,
|
||||
rename_from,
|
||||
rename_to,
|
||||
new_file_mode,
|
||||
deleted_file_mode,
|
||||
copied_file_name,
|
||||
a_blob_id,
|
||||
b_blob_id,
|
||||
b_mode,
|
||||
a_path,
|
||||
b_path,
|
||||
) = _header.groups()
|
||||
|
||||
new_file, deleted_file, copied_file = (
|
||||
bool(new_file_mode),
|
||||
bool(deleted_file_mode),
|
||||
bool(copied_file_name),
|
||||
)
|
||||
|
||||
a_path = cls._pick_best_path(a_path, rename_from, a_path_fallback)
|
||||
b_path = cls._pick_best_path(b_path, rename_to, b_path_fallback)
|
||||
|
||||
# Our only means to find the actual text is to see what has not been matched by our regex,
|
||||
# and then retro-actively assign it to our index
|
||||
if previous_header is not None:
|
||||
index[-1].diff = text[previous_header.end() : _header.start()]
|
||||
# end assign actual diff
|
||||
|
||||
# Make sure the mode is set if the path is set. Otherwise the resulting blob is invalid
|
||||
# We just use the one mode we should have parsed
|
||||
a_mode = old_mode or deleted_file_mode or (a_path and (b_mode or new_mode or new_file_mode))
|
||||
b_mode = b_mode or new_mode or new_file_mode or (b_path and a_mode)
|
||||
index.append(
|
||||
Diff(
|
||||
repo,
|
||||
a_path,
|
||||
b_path,
|
||||
a_blob_id and a_blob_id.decode(defenc),
|
||||
b_blob_id and b_blob_id.decode(defenc),
|
||||
a_mode and a_mode.decode(defenc),
|
||||
b_mode and b_mode.decode(defenc),
|
||||
new_file,
|
||||
deleted_file,
|
||||
copied_file,
|
||||
rename_from,
|
||||
rename_to,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
)
|
||||
)
|
||||
|
||||
previous_header = _header
|
||||
header = _header
|
||||
# end for each header we parse
|
||||
if index and header:
|
||||
index[-1].diff = text[header.end() :]
|
||||
# end assign last diff
|
||||
|
||||
return index
|
||||
|
||||
@staticmethod
|
||||
def _handle_diff_line(lines_bytes: bytes, repo: "Repo", index: DiffIndex) -> None:
|
||||
lines = lines_bytes.decode(defenc)
|
||||
|
||||
# Discard everything before the first colon, and the colon itself.
|
||||
_, _, lines = lines.partition(":")
|
||||
|
||||
for line in lines.split("\x00:"):
|
||||
if not line:
|
||||
# The line data is empty, skip
|
||||
continue
|
||||
meta, _, path = line.partition("\x00")
|
||||
path = path.rstrip("\x00")
|
||||
a_blob_id: Optional[str]
|
||||
b_blob_id: Optional[str]
|
||||
old_mode, new_mode, a_blob_id, b_blob_id, _change_type = meta.split(None, 4)
|
||||
# Change type can be R100
|
||||
# R: status letter
|
||||
# 100: score (in case of copy and rename)
|
||||
# assert is_change_type(_change_type[0]), f"Unexpected value for change_type received: {_change_type[0]}"
|
||||
change_type: Lit_change_type = cast(Lit_change_type, _change_type[0])
|
||||
score_str = "".join(_change_type[1:])
|
||||
score = int(score_str) if score_str.isdigit() else None
|
||||
path = path.strip()
|
||||
a_path = path.encode(defenc)
|
||||
b_path = path.encode(defenc)
|
||||
deleted_file = False
|
||||
new_file = False
|
||||
copied_file = False
|
||||
rename_from = None
|
||||
rename_to = None
|
||||
|
||||
# NOTE: We cannot conclude from the existence of a blob to change type
|
||||
# as diffs with the working do not have blobs yet
|
||||
if change_type == "D":
|
||||
b_blob_id = None # Optional[str]
|
||||
deleted_file = True
|
||||
elif change_type == "A":
|
||||
a_blob_id = None
|
||||
new_file = True
|
||||
elif change_type == "C":
|
||||
copied_file = True
|
||||
a_path_str, b_path_str = path.split("\x00", 1)
|
||||
a_path = a_path_str.encode(defenc)
|
||||
b_path = b_path_str.encode(defenc)
|
||||
elif change_type == "R":
|
||||
a_path_str, b_path_str = path.split("\x00", 1)
|
||||
a_path = a_path_str.encode(defenc)
|
||||
b_path = b_path_str.encode(defenc)
|
||||
rename_from, rename_to = a_path, b_path
|
||||
elif change_type == "T":
|
||||
# Nothing to do
|
||||
pass
|
||||
# END add/remove handling
|
||||
|
||||
diff = Diff(
|
||||
repo,
|
||||
a_path,
|
||||
b_path,
|
||||
a_blob_id,
|
||||
b_blob_id,
|
||||
old_mode,
|
||||
new_mode,
|
||||
new_file,
|
||||
deleted_file,
|
||||
copied_file,
|
||||
rename_from,
|
||||
rename_to,
|
||||
"",
|
||||
change_type,
|
||||
score,
|
||||
)
|
||||
index.append(diff)
|
||||
|
||||
@classmethod
|
||||
def _index_from_raw_format(cls, repo: "Repo", proc: "Popen") -> "DiffIndex":
|
||||
"""Create a new DiffIndex from the given stream which must be in raw format.
|
||||
:return: git.DiffIndex"""
|
||||
# handles
|
||||
# :100644 100644 687099101... 37c5e30c8... M .gitignore
|
||||
|
||||
index: "DiffIndex" = DiffIndex()
|
||||
handle_process_output(
|
||||
proc,
|
||||
lambda byt: cls._handle_diff_line(byt, repo, index),
|
||||
None,
|
||||
finalize_process,
|
||||
decode_streams=False,
|
||||
)
|
||||
|
||||
return index
|
||||
186
zero-cost-nas/.eggs/GitPython-3.1.31-py3.8.egg/git/exc.py
Normal file
186
zero-cost-nas/.eggs/GitPython-3.1.31-py3.8.egg/git/exc.py
Normal file
@@ -0,0 +1,186 @@
|
||||
# exc.py
|
||||
# Copyright (C) 2008, 2009 Michael Trier (mtrier@gmail.com) and contributors
|
||||
#
|
||||
# This module is part of GitPython and is released under
|
||||
# the BSD License: http://www.opensource.org/licenses/bsd-license.php
|
||||
""" Module containing all exceptions thrown throughout the git package, """
|
||||
|
||||
from gitdb.exc import BadName # NOQA @UnusedWildImport skipcq: PYL-W0401, PYL-W0614
|
||||
from gitdb.exc import * # NOQA @UnusedWildImport skipcq: PYL-W0401, PYL-W0614
|
||||
from git.compat import safe_decode
|
||||
from git.util import remove_password_if_present
|
||||
|
||||
# typing ----------------------------------------------------
|
||||
|
||||
from typing import List, Sequence, Tuple, Union, TYPE_CHECKING
|
||||
from git.types import PathLike
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from git.repo.base import Repo
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
|
||||
class GitError(Exception):
|
||||
"""Base class for all package exceptions"""
|
||||
|
||||
|
||||
class InvalidGitRepositoryError(GitError):
|
||||
"""Thrown if the given repository appears to have an invalid format."""
|
||||
|
||||
|
||||
class WorkTreeRepositoryUnsupported(InvalidGitRepositoryError):
|
||||
"""Thrown to indicate we can't handle work tree repositories"""
|
||||
|
||||
|
||||
class NoSuchPathError(GitError, OSError):
|
||||
"""Thrown if a path could not be access by the system."""
|
||||
|
||||
|
||||
class UnsafeProtocolError(GitError):
|
||||
"""Thrown if unsafe protocols are passed without being explicitly allowed."""
|
||||
|
||||
|
||||
class UnsafeOptionError(GitError):
|
||||
"""Thrown if unsafe options are passed without being explicitly allowed."""
|
||||
|
||||
|
||||
class CommandError(GitError):
|
||||
"""Base class for exceptions thrown at every stage of `Popen()` execution.
|
||||
|
||||
:param command:
|
||||
A non-empty list of argv comprising the command-line.
|
||||
"""
|
||||
|
||||
#: A unicode print-format with 2 `%s for `<cmdline>` and the rest,
|
||||
#: e.g.
|
||||
#: "'%s' failed%s"
|
||||
_msg = "Cmd('%s') failed%s"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
command: Union[List[str], Tuple[str, ...], str],
|
||||
status: Union[str, int, None, Exception] = None,
|
||||
stderr: Union[bytes, str, None] = None,
|
||||
stdout: Union[bytes, str, None] = None,
|
||||
) -> None:
|
||||
if not isinstance(command, (tuple, list)):
|
||||
command = command.split()
|
||||
self.command = remove_password_if_present(command)
|
||||
self.status = status
|
||||
if status:
|
||||
if isinstance(status, Exception):
|
||||
status = "%s('%s')" % (type(status).__name__, safe_decode(str(status)))
|
||||
else:
|
||||
try:
|
||||
status = "exit code(%s)" % int(status)
|
||||
except (ValueError, TypeError):
|
||||
s = safe_decode(str(status))
|
||||
status = "'%s'" % s if isinstance(status, str) else s
|
||||
|
||||
self._cmd = safe_decode(self.command[0])
|
||||
self._cmdline = " ".join(safe_decode(i) for i in self.command)
|
||||
self._cause = status and " due to: %s" % status or "!"
|
||||
stdout_decode = safe_decode(stdout)
|
||||
stderr_decode = safe_decode(stderr)
|
||||
self.stdout = stdout_decode and "\n stdout: '%s'" % stdout_decode or ""
|
||||
self.stderr = stderr_decode and "\n stderr: '%s'" % stderr_decode or ""
|
||||
|
||||
def __str__(self) -> str:
|
||||
return (self._msg + "\n cmdline: %s%s%s") % (
|
||||
self._cmd,
|
||||
self._cause,
|
||||
self._cmdline,
|
||||
self.stdout,
|
||||
self.stderr,
|
||||
)
|
||||
|
||||
|
||||
class GitCommandNotFound(CommandError):
|
||||
"""Thrown if we cannot find the `git` executable in the PATH or at the path given by
|
||||
the GIT_PYTHON_GIT_EXECUTABLE environment variable"""
|
||||
|
||||
def __init__(self, command: Union[List[str], Tuple[str], str], cause: Union[str, Exception]) -> None:
|
||||
super(GitCommandNotFound, self).__init__(command, cause)
|
||||
self._msg = "Cmd('%s') not found%s"
|
||||
|
||||
|
||||
class GitCommandError(CommandError):
|
||||
"""Thrown if execution of the git command fails with non-zero status code."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
command: Union[List[str], Tuple[str, ...], str],
|
||||
status: Union[str, int, None, Exception] = None,
|
||||
stderr: Union[bytes, str, None] = None,
|
||||
stdout: Union[bytes, str, None] = None,
|
||||
) -> None:
|
||||
super(GitCommandError, self).__init__(command, status, stderr, stdout)
|
||||
|
||||
|
||||
class CheckoutError(GitError):
|
||||
"""Thrown if a file could not be checked out from the index as it contained
|
||||
changes.
|
||||
|
||||
The .failed_files attribute contains a list of relative paths that failed
|
||||
to be checked out as they contained changes that did not exist in the index.
|
||||
|
||||
The .failed_reasons attribute contains a string informing about the actual
|
||||
cause of the issue.
|
||||
|
||||
The .valid_files attribute contains a list of relative paths to files that
|
||||
were checked out successfully and hence match the version stored in the
|
||||
index"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
message: str,
|
||||
failed_files: Sequence[PathLike],
|
||||
valid_files: Sequence[PathLike],
|
||||
failed_reasons: List[str],
|
||||
) -> None:
|
||||
|
||||
Exception.__init__(self, message)
|
||||
self.failed_files = failed_files
|
||||
self.failed_reasons = failed_reasons
|
||||
self.valid_files = valid_files
|
||||
|
||||
def __str__(self) -> str:
|
||||
return Exception.__str__(self) + ":%s" % self.failed_files
|
||||
|
||||
|
||||
class CacheError(GitError):
|
||||
|
||||
"""Base for all errors related to the git index, which is called cache internally"""
|
||||
|
||||
|
||||
class UnmergedEntriesError(CacheError):
|
||||
"""Thrown if an operation cannot proceed as there are still unmerged
|
||||
entries in the cache"""
|
||||
|
||||
|
||||
class HookExecutionError(CommandError):
|
||||
"""Thrown if a hook exits with a non-zero exit code. It provides access to the exit code and the string returned
|
||||
via standard output"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
command: Union[List[str], Tuple[str, ...], str],
|
||||
status: Union[str, int, None, Exception],
|
||||
stderr: Union[bytes, str, None] = None,
|
||||
stdout: Union[bytes, str, None] = None,
|
||||
) -> None:
|
||||
|
||||
super(HookExecutionError, self).__init__(command, status, stderr, stdout)
|
||||
self._msg = "Hook('%s') failed%s"
|
||||
|
||||
|
||||
class RepositoryDirtyError(GitError):
|
||||
"""Thrown whenever an operation on a repository fails as it has uncommitted changes that would be overwritten"""
|
||||
|
||||
def __init__(self, repo: "Repo", message: str) -> None:
|
||||
self.repo = repo
|
||||
self.message = message
|
||||
|
||||
def __str__(self) -> str:
|
||||
return "Operation cannot be performed on %r: %s" % (self.repo, self.message)
|
||||
@@ -0,0 +1,4 @@
|
||||
"""Initialize the index package"""
|
||||
# flake8: noqa
|
||||
from .base import *
|
||||
from .typ import *
|
||||
1401
zero-cost-nas/.eggs/GitPython-3.1.31-py3.8.egg/git/index/base.py
Normal file
1401
zero-cost-nas/.eggs/GitPython-3.1.31-py3.8.egg/git/index/base.py
Normal file
File diff suppressed because it is too large
Load Diff
444
zero-cost-nas/.eggs/GitPython-3.1.31-py3.8.egg/git/index/fun.py
Normal file
444
zero-cost-nas/.eggs/GitPython-3.1.31-py3.8.egg/git/index/fun.py
Normal file
@@ -0,0 +1,444 @@
|
||||
# Contains standalone functions to accompany the index implementation and make it
|
||||
# more versatile
|
||||
# NOTE: Autodoc hates it if this is a docstring
|
||||
|
||||
from io import BytesIO
|
||||
from pathlib import Path
|
||||
import os
|
||||
from stat import (
|
||||
S_IFDIR,
|
||||
S_IFLNK,
|
||||
S_ISLNK,
|
||||
S_ISDIR,
|
||||
S_IFMT,
|
||||
S_IFREG,
|
||||
S_IXUSR,
|
||||
)
|
||||
import subprocess
|
||||
|
||||
from git.cmd import PROC_CREATIONFLAGS, handle_process_output
|
||||
from git.compat import (
|
||||
defenc,
|
||||
force_text,
|
||||
force_bytes,
|
||||
is_posix,
|
||||
is_win,
|
||||
safe_decode,
|
||||
)
|
||||
from git.exc import UnmergedEntriesError, HookExecutionError
|
||||
from git.objects.fun import (
|
||||
tree_to_stream,
|
||||
traverse_tree_recursive,
|
||||
traverse_trees_recursive,
|
||||
)
|
||||
from git.util import IndexFileSHA1Writer, finalize_process
|
||||
from gitdb.base import IStream
|
||||
from gitdb.typ import str_tree_type
|
||||
|
||||
import os.path as osp
|
||||
|
||||
from .typ import BaseIndexEntry, IndexEntry, CE_NAMEMASK, CE_STAGESHIFT
|
||||
from .util import pack, unpack
|
||||
|
||||
# typing -----------------------------------------------------------------------------
|
||||
|
||||
from typing import Dict, IO, List, Sequence, TYPE_CHECKING, Tuple, Type, Union, cast
|
||||
|
||||
from git.types import PathLike
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from .base import IndexFile
|
||||
from git.db import GitCmdObjectDB
|
||||
from git.objects.tree import TreeCacheTup
|
||||
|
||||
# from git.objects.fun import EntryTupOrNone
|
||||
|
||||
# ------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
S_IFGITLINK = S_IFLNK | S_IFDIR # a submodule
|
||||
CE_NAMEMASK_INV = ~CE_NAMEMASK
|
||||
|
||||
__all__ = (
|
||||
"write_cache",
|
||||
"read_cache",
|
||||
"write_tree_from_cache",
|
||||
"entry_key",
|
||||
"stat_mode_to_index_mode",
|
||||
"S_IFGITLINK",
|
||||
"run_commit_hook",
|
||||
"hook_path",
|
||||
)
|
||||
|
||||
|
||||
def hook_path(name: str, git_dir: PathLike) -> str:
|
||||
""":return: path to the given named hook in the given git repository directory"""
|
||||
return osp.join(git_dir, "hooks", name)
|
||||
|
||||
|
||||
def _has_file_extension(path):
|
||||
return osp.splitext(path)[1]
|
||||
|
||||
|
||||
def run_commit_hook(name: str, index: "IndexFile", *args: str) -> None:
|
||||
"""Run the commit hook of the given name. Silently ignores hooks that do not exist.
|
||||
|
||||
:param name: name of hook, like 'pre-commit'
|
||||
:param index: IndexFile instance
|
||||
:param args: arguments passed to hook file
|
||||
:raises HookExecutionError:"""
|
||||
hp = hook_path(name, index.repo.git_dir)
|
||||
if not os.access(hp, os.X_OK):
|
||||
return None
|
||||
|
||||
env = os.environ.copy()
|
||||
env["GIT_INDEX_FILE"] = safe_decode(str(index.path))
|
||||
env["GIT_EDITOR"] = ":"
|
||||
cmd = [hp]
|
||||
try:
|
||||
if is_win and not _has_file_extension(hp):
|
||||
# Windows only uses extensions to determine how to open files
|
||||
# (doesn't understand shebangs). Try using bash to run the hook.
|
||||
relative_hp = Path(hp).relative_to(index.repo.working_dir).as_posix()
|
||||
cmd = ["bash.exe", relative_hp]
|
||||
|
||||
cmd = subprocess.Popen(
|
||||
cmd + list(args),
|
||||
env=env,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
cwd=index.repo.working_dir,
|
||||
close_fds=is_posix,
|
||||
creationflags=PROC_CREATIONFLAGS,
|
||||
)
|
||||
except Exception as ex:
|
||||
raise HookExecutionError(hp, ex) from ex
|
||||
else:
|
||||
stdout_list: List[str] = []
|
||||
stderr_list: List[str] = []
|
||||
handle_process_output(cmd, stdout_list.append, stderr_list.append, finalize_process)
|
||||
stdout = "".join(stdout_list)
|
||||
stderr = "".join(stderr_list)
|
||||
if cmd.returncode != 0:
|
||||
stdout = force_text(stdout, defenc)
|
||||
stderr = force_text(stderr, defenc)
|
||||
raise HookExecutionError(hp, cmd.returncode, stderr, stdout)
|
||||
# end handle return code
|
||||
|
||||
|
||||
def stat_mode_to_index_mode(mode: int) -> int:
|
||||
"""Convert the given mode from a stat call to the corresponding index mode
|
||||
and return it"""
|
||||
if S_ISLNK(mode): # symlinks
|
||||
return S_IFLNK
|
||||
if S_ISDIR(mode) or S_IFMT(mode) == S_IFGITLINK: # submodules
|
||||
return S_IFGITLINK
|
||||
return S_IFREG | (mode & S_IXUSR and 0o755 or 0o644) # blobs with or without executable bit
|
||||
|
||||
|
||||
def write_cache(
|
||||
entries: Sequence[Union[BaseIndexEntry, "IndexEntry"]],
|
||||
stream: IO[bytes],
|
||||
extension_data: Union[None, bytes] = None,
|
||||
ShaStreamCls: Type[IndexFileSHA1Writer] = IndexFileSHA1Writer,
|
||||
) -> None:
|
||||
"""Write the cache represented by entries to a stream
|
||||
|
||||
:param entries: **sorted** list of entries
|
||||
:param stream: stream to wrap into the AdapterStreamCls - it is used for
|
||||
final output.
|
||||
|
||||
:param ShaStreamCls: Type to use when writing to the stream. It produces a sha
|
||||
while writing to it, before the data is passed on to the wrapped stream
|
||||
|
||||
:param extension_data: any kind of data to write as a trailer, it must begin
|
||||
a 4 byte identifier, followed by its size ( 4 bytes )"""
|
||||
# wrap the stream into a compatible writer
|
||||
stream_sha = ShaStreamCls(stream)
|
||||
|
||||
tell = stream_sha.tell
|
||||
write = stream_sha.write
|
||||
|
||||
# header
|
||||
version = 2
|
||||
write(b"DIRC")
|
||||
write(pack(">LL", version, len(entries)))
|
||||
|
||||
# body
|
||||
for entry in entries:
|
||||
beginoffset = tell()
|
||||
write(entry.ctime_bytes) # ctime
|
||||
write(entry.mtime_bytes) # mtime
|
||||
path_str = str(entry.path)
|
||||
path: bytes = force_bytes(path_str, encoding=defenc)
|
||||
plen = len(path) & CE_NAMEMASK # path length
|
||||
assert plen == len(path), "Path %s too long to fit into index" % entry.path
|
||||
flags = plen | (entry.flags & CE_NAMEMASK_INV) # clear possible previous values
|
||||
write(
|
||||
pack(
|
||||
">LLLLLL20sH",
|
||||
entry.dev,
|
||||
entry.inode,
|
||||
entry.mode,
|
||||
entry.uid,
|
||||
entry.gid,
|
||||
entry.size,
|
||||
entry.binsha,
|
||||
flags,
|
||||
)
|
||||
)
|
||||
write(path)
|
||||
real_size = (tell() - beginoffset + 8) & ~7
|
||||
write(b"\0" * ((beginoffset + real_size) - tell()))
|
||||
# END for each entry
|
||||
|
||||
# write previously cached extensions data
|
||||
if extension_data is not None:
|
||||
stream_sha.write(extension_data)
|
||||
|
||||
# write the sha over the content
|
||||
stream_sha.write_sha()
|
||||
|
||||
|
||||
def read_header(stream: IO[bytes]) -> Tuple[int, int]:
|
||||
"""Return tuple(version_long, num_entries) from the given stream"""
|
||||
type_id = stream.read(4)
|
||||
if type_id != b"DIRC":
|
||||
raise AssertionError("Invalid index file header: %r" % type_id)
|
||||
unpacked = cast(Tuple[int, int], unpack(">LL", stream.read(4 * 2)))
|
||||
version, num_entries = unpacked
|
||||
|
||||
# TODO: handle version 3: extended data, see read-cache.c
|
||||
assert version in (1, 2)
|
||||
return version, num_entries
|
||||
|
||||
|
||||
def entry_key(*entry: Union[BaseIndexEntry, PathLike, int]) -> Tuple[PathLike, int]:
|
||||
""":return: Key suitable to be used for the index.entries dictionary
|
||||
:param entry: One instance of type BaseIndexEntry or the path and the stage"""
|
||||
|
||||
# def is_entry_key_tup(entry_key: Tuple) -> TypeGuard[Tuple[PathLike, int]]:
|
||||
# return isinstance(entry_key, tuple) and len(entry_key) == 2
|
||||
|
||||
if len(entry) == 1:
|
||||
entry_first = entry[0]
|
||||
assert isinstance(entry_first, BaseIndexEntry)
|
||||
return (entry_first.path, entry_first.stage)
|
||||
else:
|
||||
# assert is_entry_key_tup(entry)
|
||||
entry = cast(Tuple[PathLike, int], entry)
|
||||
return entry
|
||||
# END handle entry
|
||||
|
||||
|
||||
def read_cache(
|
||||
stream: IO[bytes],
|
||||
) -> Tuple[int, Dict[Tuple[PathLike, int], "IndexEntry"], bytes, bytes]:
|
||||
"""Read a cache file from the given stream
|
||||
|
||||
:return: tuple(version, entries_dict, extension_data, content_sha)
|
||||
|
||||
* version is the integer version number
|
||||
* entries dict is a dictionary which maps IndexEntry instances to a path at a stage
|
||||
* extension_data is '' or 4 bytes of type + 4 bytes of size + size bytes
|
||||
* content_sha is a 20 byte sha on all cache file contents"""
|
||||
version, num_entries = read_header(stream)
|
||||
count = 0
|
||||
entries: Dict[Tuple[PathLike, int], "IndexEntry"] = {}
|
||||
|
||||
read = stream.read
|
||||
tell = stream.tell
|
||||
while count < num_entries:
|
||||
beginoffset = tell()
|
||||
ctime = unpack(">8s", read(8))[0]
|
||||
mtime = unpack(">8s", read(8))[0]
|
||||
(dev, ino, mode, uid, gid, size, sha, flags) = unpack(">LLLLLL20sH", read(20 + 4 * 6 + 2))
|
||||
path_size = flags & CE_NAMEMASK
|
||||
path = read(path_size).decode(defenc)
|
||||
|
||||
real_size = (tell() - beginoffset + 8) & ~7
|
||||
read((beginoffset + real_size) - tell())
|
||||
entry = IndexEntry((mode, sha, flags, path, ctime, mtime, dev, ino, uid, gid, size))
|
||||
# entry_key would be the method to use, but we safe the effort
|
||||
entries[(path, entry.stage)] = entry
|
||||
count += 1
|
||||
# END for each entry
|
||||
|
||||
# the footer contains extension data and a sha on the content so far
|
||||
# Keep the extension footer,and verify we have a sha in the end
|
||||
# Extension data format is:
|
||||
# 4 bytes ID
|
||||
# 4 bytes length of chunk
|
||||
# repeated 0 - N times
|
||||
extension_data = stream.read(~0)
|
||||
assert (
|
||||
len(extension_data) > 19
|
||||
), "Index Footer was not at least a sha on content as it was only %i bytes in size" % len(extension_data)
|
||||
|
||||
content_sha = extension_data[-20:]
|
||||
|
||||
# truncate the sha in the end as we will dynamically create it anyway
|
||||
extension_data = extension_data[:-20]
|
||||
|
||||
return (version, entries, extension_data, content_sha)
|
||||
|
||||
|
||||
def write_tree_from_cache(
|
||||
entries: List[IndexEntry], odb: "GitCmdObjectDB", sl: slice, si: int = 0
|
||||
) -> Tuple[bytes, List["TreeCacheTup"]]:
|
||||
"""Create a tree from the given sorted list of entries and put the respective
|
||||
trees into the given object database
|
||||
|
||||
:param entries: **sorted** list of IndexEntries
|
||||
:param odb: object database to store the trees in
|
||||
:param si: start index at which we should start creating subtrees
|
||||
:param sl: slice indicating the range we should process on the entries list
|
||||
:return: tuple(binsha, list(tree_entry, ...)) a tuple of a sha and a list of
|
||||
tree entries being a tuple of hexsha, mode, name"""
|
||||
tree_items: List["TreeCacheTup"] = []
|
||||
|
||||
ci = sl.start
|
||||
end = sl.stop
|
||||
while ci < end:
|
||||
entry = entries[ci]
|
||||
if entry.stage != 0:
|
||||
raise UnmergedEntriesError(entry)
|
||||
# END abort on unmerged
|
||||
ci += 1
|
||||
rbound = entry.path.find("/", si)
|
||||
if rbound == -1:
|
||||
# its not a tree
|
||||
tree_items.append((entry.binsha, entry.mode, entry.path[si:]))
|
||||
else:
|
||||
# find common base range
|
||||
base = entry.path[si:rbound]
|
||||
xi = ci
|
||||
while xi < end:
|
||||
oentry = entries[xi]
|
||||
orbound = oentry.path.find("/", si)
|
||||
if orbound == -1 or oentry.path[si:orbound] != base:
|
||||
break
|
||||
# END abort on base mismatch
|
||||
xi += 1
|
||||
# END find common base
|
||||
|
||||
# enter recursion
|
||||
# ci - 1 as we want to count our current item as well
|
||||
sha, _tree_entry_list = write_tree_from_cache(entries, odb, slice(ci - 1, xi), rbound + 1)
|
||||
tree_items.append((sha, S_IFDIR, base))
|
||||
|
||||
# skip ahead
|
||||
ci = xi
|
||||
# END handle bounds
|
||||
# END for each entry
|
||||
|
||||
# finally create the tree
|
||||
sio = BytesIO()
|
||||
tree_to_stream(tree_items, sio.write) # writes to stream as bytes, but doesn't change tree_items
|
||||
sio.seek(0)
|
||||
|
||||
istream = odb.store(IStream(str_tree_type, len(sio.getvalue()), sio))
|
||||
return (istream.binsha, tree_items)
|
||||
|
||||
|
||||
def _tree_entry_to_baseindexentry(tree_entry: "TreeCacheTup", stage: int) -> BaseIndexEntry:
|
||||
return BaseIndexEntry((tree_entry[1], tree_entry[0], stage << CE_STAGESHIFT, tree_entry[2]))
|
||||
|
||||
|
||||
def aggressive_tree_merge(odb: "GitCmdObjectDB", tree_shas: Sequence[bytes]) -> List[BaseIndexEntry]:
|
||||
"""
|
||||
:return: list of BaseIndexEntries representing the aggressive merge of the given
|
||||
trees. All valid entries are on stage 0, whereas the conflicting ones are left
|
||||
on stage 1, 2 or 3, whereas stage 1 corresponds to the common ancestor tree,
|
||||
2 to our tree and 3 to 'their' tree.
|
||||
:param tree_shas: 1, 2 or 3 trees as identified by their binary 20 byte shas
|
||||
If 1 or two, the entries will effectively correspond to the last given tree
|
||||
If 3 are given, a 3 way merge is performed"""
|
||||
out: List[BaseIndexEntry] = []
|
||||
|
||||
# one and two way is the same for us, as we don't have to handle an existing
|
||||
# index, instrea
|
||||
if len(tree_shas) in (1, 2):
|
||||
for entry in traverse_tree_recursive(odb, tree_shas[-1], ""):
|
||||
out.append(_tree_entry_to_baseindexentry(entry, 0))
|
||||
# END for each entry
|
||||
return out
|
||||
# END handle single tree
|
||||
|
||||
if len(tree_shas) > 3:
|
||||
raise ValueError("Cannot handle %i trees at once" % len(tree_shas))
|
||||
|
||||
# three trees
|
||||
for base, ours, theirs in traverse_trees_recursive(odb, tree_shas, ""):
|
||||
if base is not None:
|
||||
# base version exists
|
||||
if ours is not None:
|
||||
# ours exists
|
||||
if theirs is not None:
|
||||
# it exists in all branches, if it was changed in both
|
||||
# its a conflict, otherwise we take the changed version
|
||||
# This should be the most common branch, so it comes first
|
||||
if (base[0] != ours[0] and base[0] != theirs[0] and ours[0] != theirs[0]) or (
|
||||
base[1] != ours[1] and base[1] != theirs[1] and ours[1] != theirs[1]
|
||||
):
|
||||
# changed by both
|
||||
out.append(_tree_entry_to_baseindexentry(base, 1))
|
||||
out.append(_tree_entry_to_baseindexentry(ours, 2))
|
||||
out.append(_tree_entry_to_baseindexentry(theirs, 3))
|
||||
elif base[0] != ours[0] or base[1] != ours[1]:
|
||||
# only we changed it
|
||||
out.append(_tree_entry_to_baseindexentry(ours, 0))
|
||||
else:
|
||||
# either nobody changed it, or they did. In either
|
||||
# case, use theirs
|
||||
out.append(_tree_entry_to_baseindexentry(theirs, 0))
|
||||
# END handle modification
|
||||
else:
|
||||
|
||||
if ours[0] != base[0] or ours[1] != base[1]:
|
||||
# they deleted it, we changed it, conflict
|
||||
out.append(_tree_entry_to_baseindexentry(base, 1))
|
||||
out.append(_tree_entry_to_baseindexentry(ours, 2))
|
||||
# else:
|
||||
# we didn't change it, ignore
|
||||
# pass
|
||||
# END handle our change
|
||||
# END handle theirs
|
||||
else:
|
||||
if theirs is None:
|
||||
# deleted in both, its fine - its out
|
||||
pass
|
||||
else:
|
||||
if theirs[0] != base[0] or theirs[1] != base[1]:
|
||||
# deleted in ours, changed theirs, conflict
|
||||
out.append(_tree_entry_to_baseindexentry(base, 1))
|
||||
out.append(_tree_entry_to_baseindexentry(theirs, 3))
|
||||
# END theirs changed
|
||||
# else:
|
||||
# theirs didn't change
|
||||
# pass
|
||||
# END handle theirs
|
||||
# END handle ours
|
||||
else:
|
||||
# all three can't be None
|
||||
if ours is None:
|
||||
# added in their branch
|
||||
assert theirs is not None
|
||||
out.append(_tree_entry_to_baseindexentry(theirs, 0))
|
||||
elif theirs is None:
|
||||
# added in our branch
|
||||
out.append(_tree_entry_to_baseindexentry(ours, 0))
|
||||
else:
|
||||
# both have it, except for the base, see whether it changed
|
||||
if ours[0] != theirs[0] or ours[1] != theirs[1]:
|
||||
out.append(_tree_entry_to_baseindexentry(ours, 2))
|
||||
out.append(_tree_entry_to_baseindexentry(theirs, 3))
|
||||
else:
|
||||
# it was added the same in both
|
||||
out.append(_tree_entry_to_baseindexentry(ours, 0))
|
||||
# END handle two items
|
||||
# END handle heads
|
||||
# END handle base exists
|
||||
# END for each entries tuple
|
||||
|
||||
return out
|
||||
191
zero-cost-nas/.eggs/GitPython-3.1.31-py3.8.egg/git/index/typ.py
Normal file
191
zero-cost-nas/.eggs/GitPython-3.1.31-py3.8.egg/git/index/typ.py
Normal file
@@ -0,0 +1,191 @@
|
||||
"""Module with additional types used by the index"""
|
||||
|
||||
from binascii import b2a_hex
|
||||
from pathlib import Path
|
||||
|
||||
from .util import pack, unpack
|
||||
from git.objects import Blob
|
||||
|
||||
|
||||
# typing ----------------------------------------------------------------------
|
||||
|
||||
from typing import NamedTuple, Sequence, TYPE_CHECKING, Tuple, Union, cast, List
|
||||
|
||||
from git.types import PathLike
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from git.repo import Repo
|
||||
|
||||
StageType = int
|
||||
|
||||
# ---------------------------------------------------------------------------------
|
||||
|
||||
__all__ = ("BlobFilter", "BaseIndexEntry", "IndexEntry", "StageType")
|
||||
|
||||
# { Invariants
|
||||
CE_NAMEMASK = 0x0FFF
|
||||
CE_STAGEMASK = 0x3000
|
||||
CE_EXTENDED = 0x4000
|
||||
CE_VALID = 0x8000
|
||||
CE_STAGESHIFT = 12
|
||||
|
||||
# } END invariants
|
||||
|
||||
|
||||
class BlobFilter(object):
|
||||
|
||||
"""
|
||||
Predicate to be used by iter_blobs allowing to filter only return blobs which
|
||||
match the given list of directories or files.
|
||||
|
||||
The given paths are given relative to the repository.
|
||||
"""
|
||||
|
||||
__slots__ = "paths"
|
||||
|
||||
def __init__(self, paths: Sequence[PathLike]) -> None:
|
||||
"""
|
||||
:param paths:
|
||||
tuple or list of paths which are either pointing to directories or
|
||||
to files relative to the current repository
|
||||
"""
|
||||
self.paths = paths
|
||||
|
||||
def __call__(self, stage_blob: Tuple[StageType, Blob]) -> bool:
|
||||
blob_pathlike: PathLike = stage_blob[1].path
|
||||
blob_path: Path = blob_pathlike if isinstance(blob_pathlike, Path) else Path(blob_pathlike)
|
||||
for pathlike in self.paths:
|
||||
path: Path = pathlike if isinstance(pathlike, Path) else Path(pathlike)
|
||||
# TODO: Change to use `PosixPath.is_relative_to` once Python 3.8 is no longer supported.
|
||||
filter_parts: List[str] = path.parts
|
||||
blob_parts: List[str] = blob_path.parts
|
||||
if len(filter_parts) > len(blob_parts):
|
||||
continue
|
||||
if all(i == j for i, j in zip(filter_parts, blob_parts)):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
class BaseIndexEntryHelper(NamedTuple):
|
||||
"""Typed namedtuple to provide named attribute access for BaseIndexEntry.
|
||||
Needed to allow overriding __new__ in child class to preserve backwards compat."""
|
||||
|
||||
mode: int
|
||||
binsha: bytes
|
||||
flags: int
|
||||
path: PathLike
|
||||
ctime_bytes: bytes = pack(">LL", 0, 0)
|
||||
mtime_bytes: bytes = pack(">LL", 0, 0)
|
||||
dev: int = 0
|
||||
inode: int = 0
|
||||
uid: int = 0
|
||||
gid: int = 0
|
||||
size: int = 0
|
||||
|
||||
|
||||
class BaseIndexEntry(BaseIndexEntryHelper):
|
||||
|
||||
"""Small Brother of an index entry which can be created to describe changes
|
||||
done to the index in which case plenty of additional information is not required.
|
||||
|
||||
As the first 4 data members match exactly to the IndexEntry type, methods
|
||||
expecting a BaseIndexEntry can also handle full IndexEntries even if they
|
||||
use numeric indices for performance reasons.
|
||||
"""
|
||||
|
||||
def __new__(
|
||||
cls,
|
||||
inp_tuple: Union[
|
||||
Tuple[int, bytes, int, PathLike],
|
||||
Tuple[int, bytes, int, PathLike, bytes, bytes, int, int, int, int, int],
|
||||
],
|
||||
) -> "BaseIndexEntry":
|
||||
"""Override __new__ to allow construction from a tuple for backwards compatibility"""
|
||||
return super().__new__(cls, *inp_tuple)
|
||||
|
||||
def __str__(self) -> str:
|
||||
return "%o %s %i\t%s" % (self.mode, self.hexsha, self.stage, self.path)
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return "(%o, %s, %i, %s)" % (self.mode, self.hexsha, self.stage, self.path)
|
||||
|
||||
@property
|
||||
def hexsha(self) -> str:
|
||||
"""hex version of our sha"""
|
||||
return b2a_hex(self.binsha).decode("ascii")
|
||||
|
||||
@property
|
||||
def stage(self) -> int:
|
||||
"""Stage of the entry, either:
|
||||
|
||||
* 0 = default stage
|
||||
* 1 = stage before a merge or common ancestor entry in case of a 3 way merge
|
||||
* 2 = stage of entries from the 'left' side of the merge
|
||||
* 3 = stage of entries from the right side of the merge
|
||||
|
||||
:note: For more information, see http://www.kernel.org/pub/software/scm/git/docs/git-read-tree.html
|
||||
"""
|
||||
return (self.flags & CE_STAGEMASK) >> CE_STAGESHIFT
|
||||
|
||||
@classmethod
|
||||
def from_blob(cls, blob: Blob, stage: int = 0) -> "BaseIndexEntry":
|
||||
""":return: Fully equipped BaseIndexEntry at the given stage"""
|
||||
return cls((blob.mode, blob.binsha, stage << CE_STAGESHIFT, blob.path))
|
||||
|
||||
def to_blob(self, repo: "Repo") -> Blob:
|
||||
""":return: Blob using the information of this index entry"""
|
||||
return Blob(repo, self.binsha, self.mode, self.path)
|
||||
|
||||
|
||||
class IndexEntry(BaseIndexEntry):
|
||||
|
||||
"""Allows convenient access to IndexEntry data without completely unpacking it.
|
||||
|
||||
Attributes usully accessed often are cached in the tuple whereas others are
|
||||
unpacked on demand.
|
||||
|
||||
See the properties for a mapping between names and tuple indices."""
|
||||
|
||||
@property
|
||||
def ctime(self) -> Tuple[int, int]:
|
||||
"""
|
||||
:return:
|
||||
Tuple(int_time_seconds_since_epoch, int_nano_seconds) of the
|
||||
file's creation time"""
|
||||
return cast(Tuple[int, int], unpack(">LL", self.ctime_bytes))
|
||||
|
||||
@property
|
||||
def mtime(self) -> Tuple[int, int]:
|
||||
"""See ctime property, but returns modification time"""
|
||||
return cast(Tuple[int, int], unpack(">LL", self.mtime_bytes))
|
||||
|
||||
@classmethod
|
||||
def from_base(cls, base: "BaseIndexEntry") -> "IndexEntry":
|
||||
"""
|
||||
:return:
|
||||
Minimal entry as created from the given BaseIndexEntry instance.
|
||||
Missing values will be set to null-like values
|
||||
|
||||
:param base: Instance of type BaseIndexEntry"""
|
||||
time = pack(">LL", 0, 0)
|
||||
return IndexEntry((base.mode, base.binsha, base.flags, base.path, time, time, 0, 0, 0, 0, 0))
|
||||
|
||||
@classmethod
|
||||
def from_blob(cls, blob: Blob, stage: int = 0) -> "IndexEntry":
|
||||
""":return: Minimal entry resembling the given blob object"""
|
||||
time = pack(">LL", 0, 0)
|
||||
return IndexEntry(
|
||||
(
|
||||
blob.mode,
|
||||
blob.binsha,
|
||||
stage << CE_STAGESHIFT,
|
||||
blob.path,
|
||||
time,
|
||||
time,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
blob.size,
|
||||
)
|
||||
)
|
||||
119
zero-cost-nas/.eggs/GitPython-3.1.31-py3.8.egg/git/index/util.py
Normal file
119
zero-cost-nas/.eggs/GitPython-3.1.31-py3.8.egg/git/index/util.py
Normal file
@@ -0,0 +1,119 @@
|
||||
"""Module containing index utilities"""
|
||||
from functools import wraps
|
||||
import os
|
||||
import struct
|
||||
import tempfile
|
||||
|
||||
from git.compat import is_win
|
||||
|
||||
import os.path as osp
|
||||
|
||||
|
||||
# typing ----------------------------------------------------------------------
|
||||
|
||||
from typing import Any, Callable, TYPE_CHECKING
|
||||
|
||||
from git.types import PathLike, _T
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from git.index import IndexFile
|
||||
|
||||
# ---------------------------------------------------------------------------------
|
||||
|
||||
|
||||
__all__ = ("TemporaryFileSwap", "post_clear_cache", "default_index", "git_working_dir")
|
||||
|
||||
# { Aliases
|
||||
pack = struct.pack
|
||||
unpack = struct.unpack
|
||||
|
||||
|
||||
# } END aliases
|
||||
|
||||
|
||||
class TemporaryFileSwap(object):
|
||||
|
||||
"""Utility class moving a file to a temporary location within the same directory
|
||||
and moving it back on to where on object deletion."""
|
||||
|
||||
__slots__ = ("file_path", "tmp_file_path")
|
||||
|
||||
def __init__(self, file_path: PathLike) -> None:
|
||||
self.file_path = file_path
|
||||
self.tmp_file_path = str(self.file_path) + tempfile.mktemp("", "", "")
|
||||
# it may be that the source does not exist
|
||||
try:
|
||||
os.rename(self.file_path, self.tmp_file_path)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
def __del__(self) -> None:
|
||||
if osp.isfile(self.tmp_file_path):
|
||||
if is_win and osp.exists(self.file_path):
|
||||
os.remove(self.file_path)
|
||||
os.rename(self.tmp_file_path, self.file_path)
|
||||
# END temp file exists
|
||||
|
||||
|
||||
# { Decorators
|
||||
|
||||
|
||||
def post_clear_cache(func: Callable[..., _T]) -> Callable[..., _T]:
|
||||
"""Decorator for functions that alter the index using the git command. This would
|
||||
invalidate our possibly existing entries dictionary which is why it must be
|
||||
deleted to allow it to be lazily reread later.
|
||||
|
||||
:note:
|
||||
This decorator will not be required once all functions are implemented
|
||||
natively which in fact is possible, but probably not feasible performance wise.
|
||||
"""
|
||||
|
||||
@wraps(func)
|
||||
def post_clear_cache_if_not_raised(self: "IndexFile", *args: Any, **kwargs: Any) -> _T:
|
||||
rval = func(self, *args, **kwargs)
|
||||
self._delete_entries_cache()
|
||||
return rval
|
||||
|
||||
# END wrapper method
|
||||
|
||||
return post_clear_cache_if_not_raised
|
||||
|
||||
|
||||
def default_index(func: Callable[..., _T]) -> Callable[..., _T]:
|
||||
"""Decorator assuring the wrapped method may only run if we are the default
|
||||
repository index. This is as we rely on git commands that operate
|
||||
on that index only."""
|
||||
|
||||
@wraps(func)
|
||||
def check_default_index(self: "IndexFile", *args: Any, **kwargs: Any) -> _T:
|
||||
if self._file_path != self._index_path():
|
||||
raise AssertionError(
|
||||
"Cannot call %r on indices that do not represent the default git index" % func.__name__
|
||||
)
|
||||
return func(self, *args, **kwargs)
|
||||
|
||||
# END wrapper method
|
||||
|
||||
return check_default_index
|
||||
|
||||
|
||||
def git_working_dir(func: Callable[..., _T]) -> Callable[..., _T]:
|
||||
"""Decorator which changes the current working dir to the one of the git
|
||||
repository in order to assure relative paths are handled correctly"""
|
||||
|
||||
@wraps(func)
|
||||
def set_git_working_dir(self: "IndexFile", *args: Any, **kwargs: Any) -> _T:
|
||||
cur_wd = os.getcwd()
|
||||
os.chdir(str(self.repo.working_tree_dir))
|
||||
try:
|
||||
return func(self, *args, **kwargs)
|
||||
finally:
|
||||
os.chdir(cur_wd)
|
||||
# END handle working dir
|
||||
|
||||
# END wrapper
|
||||
|
||||
return set_git_working_dir
|
||||
|
||||
|
||||
# } END decorators
|
||||
@@ -0,0 +1,24 @@
|
||||
"""
|
||||
Import all submodules main classes into the package space
|
||||
"""
|
||||
# flake8: noqa
|
||||
import inspect
|
||||
|
||||
from .base import *
|
||||
from .blob import *
|
||||
from .commit import *
|
||||
from .submodule import util as smutil
|
||||
from .submodule.base import *
|
||||
from .submodule.root import *
|
||||
from .tag import *
|
||||
from .tree import *
|
||||
|
||||
# Fix import dependency - add IndexObject to the util module, so that it can be
|
||||
# imported by the submodule.base
|
||||
smutil.IndexObject = IndexObject # type: ignore[attr-defined]
|
||||
smutil.Object = Object # type: ignore[attr-defined]
|
||||
del smutil
|
||||
|
||||
# must come after submodule was made available
|
||||
|
||||
__all__ = [name for name, obj in locals().items() if not (name.startswith("_") or inspect.ismodule(obj))]
|
||||
@@ -0,0 +1,224 @@
|
||||
# base.py
|
||||
# Copyright (C) 2008, 2009 Michael Trier (mtrier@gmail.com) and contributors
|
||||
#
|
||||
# This module is part of GitPython and is released under
|
||||
# the BSD License: http://www.opensource.org/licenses/bsd-license.php
|
||||
|
||||
from git.exc import WorkTreeRepositoryUnsupported
|
||||
from git.util import LazyMixin, join_path_native, stream_copy, bin_to_hex
|
||||
|
||||
import gitdb.typ as dbtyp
|
||||
import os.path as osp
|
||||
|
||||
from .util import get_object_type_by_name
|
||||
|
||||
|
||||
# typing ------------------------------------------------------------------
|
||||
|
||||
from typing import Any, TYPE_CHECKING, Union
|
||||
|
||||
from git.types import PathLike, Commit_ish, Lit_commit_ish
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from git.repo import Repo
|
||||
from gitdb.base import OStream
|
||||
from .tree import Tree
|
||||
from .blob import Blob
|
||||
from .submodule.base import Submodule
|
||||
from git.refs.reference import Reference
|
||||
|
||||
IndexObjUnion = Union["Tree", "Blob", "Submodule"]
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
_assertion_msg_format = "Created object %r whose python type %r disagrees with the actual git object type %r"
|
||||
|
||||
__all__ = ("Object", "IndexObject")
|
||||
|
||||
|
||||
class Object(LazyMixin):
|
||||
|
||||
"""Implements an Object which may be Blobs, Trees, Commits and Tags"""
|
||||
|
||||
NULL_HEX_SHA = "0" * 40
|
||||
NULL_BIN_SHA = b"\0" * 20
|
||||
|
||||
TYPES = (
|
||||
dbtyp.str_blob_type,
|
||||
dbtyp.str_tree_type,
|
||||
dbtyp.str_commit_type,
|
||||
dbtyp.str_tag_type,
|
||||
)
|
||||
__slots__ = ("repo", "binsha", "size")
|
||||
type: Union[Lit_commit_ish, None] = None
|
||||
|
||||
def __init__(self, repo: "Repo", binsha: bytes):
|
||||
"""Initialize an object by identifying it by its binary sha.
|
||||
All keyword arguments will be set on demand if None.
|
||||
|
||||
:param repo: repository this object is located in
|
||||
|
||||
:param binsha: 20 byte SHA1"""
|
||||
super(Object, self).__init__()
|
||||
self.repo = repo
|
||||
self.binsha = binsha
|
||||
assert len(binsha) == 20, "Require 20 byte binary sha, got %r, len = %i" % (
|
||||
binsha,
|
||||
len(binsha),
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def new(cls, repo: "Repo", id: Union[str, "Reference"]) -> Commit_ish:
|
||||
"""
|
||||
:return: New Object instance of a type appropriate to the object type behind
|
||||
id. The id of the newly created object will be a binsha even though
|
||||
the input id may have been a Reference or Rev-Spec
|
||||
|
||||
:param id: reference, rev-spec, or hexsha
|
||||
|
||||
:note: This cannot be a __new__ method as it would always call __init__
|
||||
with the input id which is not necessarily a binsha."""
|
||||
return repo.rev_parse(str(id))
|
||||
|
||||
@classmethod
|
||||
def new_from_sha(cls, repo: "Repo", sha1: bytes) -> Commit_ish:
|
||||
"""
|
||||
:return: new object instance of a type appropriate to represent the given
|
||||
binary sha1
|
||||
:param sha1: 20 byte binary sha1"""
|
||||
if sha1 == cls.NULL_BIN_SHA:
|
||||
# the NULL binsha is always the root commit
|
||||
return get_object_type_by_name(b"commit")(repo, sha1)
|
||||
# END handle special case
|
||||
oinfo = repo.odb.info(sha1)
|
||||
inst = get_object_type_by_name(oinfo.type)(repo, oinfo.binsha)
|
||||
inst.size = oinfo.size
|
||||
return inst
|
||||
|
||||
def _set_cache_(self, attr: str) -> None:
|
||||
"""Retrieve object information"""
|
||||
if attr == "size":
|
||||
oinfo = self.repo.odb.info(self.binsha)
|
||||
self.size = oinfo.size # type: int
|
||||
# assert oinfo.type == self.type, _assertion_msg_format % (self.binsha, oinfo.type, self.type)
|
||||
else:
|
||||
super(Object, self)._set_cache_(attr)
|
||||
|
||||
def __eq__(self, other: Any) -> bool:
|
||||
""":return: True if the objects have the same SHA1"""
|
||||
if not hasattr(other, "binsha"):
|
||||
return False
|
||||
return self.binsha == other.binsha
|
||||
|
||||
def __ne__(self, other: Any) -> bool:
|
||||
""":return: True if the objects do not have the same SHA1"""
|
||||
if not hasattr(other, "binsha"):
|
||||
return True
|
||||
return self.binsha != other.binsha
|
||||
|
||||
def __hash__(self) -> int:
|
||||
""":return: Hash of our id allowing objects to be used in dicts and sets"""
|
||||
return hash(self.binsha)
|
||||
|
||||
def __str__(self) -> str:
|
||||
""":return: string of our SHA1 as understood by all git commands"""
|
||||
return self.hexsha
|
||||
|
||||
def __repr__(self) -> str:
|
||||
""":return: string with pythonic representation of our object"""
|
||||
return '<git.%s "%s">' % (self.__class__.__name__, self.hexsha)
|
||||
|
||||
@property
|
||||
def hexsha(self) -> str:
|
||||
""":return: 40 byte hex version of our 20 byte binary sha"""
|
||||
# b2a_hex produces bytes
|
||||
return bin_to_hex(self.binsha).decode("ascii")
|
||||
|
||||
@property
|
||||
def data_stream(self) -> "OStream":
|
||||
""":return: File Object compatible stream to the uncompressed raw data of the object
|
||||
:note: returned streams must be read in order"""
|
||||
return self.repo.odb.stream(self.binsha)
|
||||
|
||||
def stream_data(self, ostream: "OStream") -> "Object":
|
||||
"""Writes our data directly to the given output stream
|
||||
|
||||
:param ostream: File object compatible stream object.
|
||||
:return: self"""
|
||||
istream = self.repo.odb.stream(self.binsha)
|
||||
stream_copy(istream, ostream)
|
||||
return self
|
||||
|
||||
|
||||
class IndexObject(Object):
|
||||
|
||||
"""Base for all objects that can be part of the index file , namely Tree, Blob and
|
||||
SubModule objects"""
|
||||
|
||||
__slots__ = ("path", "mode")
|
||||
|
||||
# for compatibility with iterable lists
|
||||
_id_attribute_ = "path"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
repo: "Repo",
|
||||
binsha: bytes,
|
||||
mode: Union[None, int] = None,
|
||||
path: Union[None, PathLike] = None,
|
||||
) -> None:
|
||||
"""Initialize a newly instanced IndexObject
|
||||
|
||||
:param repo: is the Repo we are located in
|
||||
:param binsha: 20 byte sha1
|
||||
:param mode:
|
||||
is the stat compatible file mode as int, use the stat module
|
||||
to evaluate the information
|
||||
:param path:
|
||||
is the path to the file in the file system, relative to the git repository root, i.e.
|
||||
file.ext or folder/other.ext
|
||||
:note:
|
||||
Path may not be set of the index object has been created directly as it cannot
|
||||
be retrieved without knowing the parent tree."""
|
||||
super(IndexObject, self).__init__(repo, binsha)
|
||||
if mode is not None:
|
||||
self.mode = mode
|
||||
if path is not None:
|
||||
self.path = path
|
||||
|
||||
def __hash__(self) -> int:
|
||||
"""
|
||||
:return:
|
||||
Hash of our path as index items are uniquely identifiable by path, not
|
||||
by their data !"""
|
||||
return hash(self.path)
|
||||
|
||||
def _set_cache_(self, attr: str) -> None:
|
||||
if attr in IndexObject.__slots__:
|
||||
# they cannot be retrieved lateron ( not without searching for them )
|
||||
raise AttributeError(
|
||||
"Attribute '%s' unset: path and mode attributes must have been set during %s object creation"
|
||||
% (attr, type(self).__name__)
|
||||
)
|
||||
else:
|
||||
super(IndexObject, self)._set_cache_(attr)
|
||||
# END handle slot attribute
|
||||
|
||||
@property
|
||||
def name(self) -> str:
|
||||
""":return: Name portion of the path, effectively being the basename"""
|
||||
return osp.basename(self.path)
|
||||
|
||||
@property
|
||||
def abspath(self) -> PathLike:
|
||||
"""
|
||||
:return:
|
||||
Absolute path to this index object in the file system ( as opposed to the
|
||||
.path field which is a path relative to the git repository ).
|
||||
|
||||
The returned path will be native to the system and contains '\' on windows."""
|
||||
if self.repo.working_tree_dir is not None:
|
||||
return join_path_native(self.repo.working_tree_dir, self.path)
|
||||
else:
|
||||
raise WorkTreeRepositoryUnsupported("Working_tree_dir was None or empty")
|
||||
@@ -0,0 +1,36 @@
|
||||
# blob.py
|
||||
# Copyright (C) 2008, 2009 Michael Trier (mtrier@gmail.com) and contributors
|
||||
#
|
||||
# This module is part of GitPython and is released under
|
||||
# the BSD License: http://www.opensource.org/licenses/bsd-license.php
|
||||
from mimetypes import guess_type
|
||||
from . import base
|
||||
|
||||
from git.types import Literal
|
||||
|
||||
__all__ = ("Blob",)
|
||||
|
||||
|
||||
class Blob(base.IndexObject):
|
||||
|
||||
"""A Blob encapsulates a git blob object"""
|
||||
|
||||
DEFAULT_MIME_TYPE = "text/plain"
|
||||
type: Literal["blob"] = "blob"
|
||||
|
||||
# valid blob modes
|
||||
executable_mode = 0o100755
|
||||
file_mode = 0o100644
|
||||
link_mode = 0o120000
|
||||
|
||||
__slots__ = ()
|
||||
|
||||
@property
|
||||
def mime_type(self) -> str:
|
||||
"""
|
||||
:return: String describing the mime type of this file (based on the filename)
|
||||
:note: Defaults to 'text/plain' in case the actual file type is unknown."""
|
||||
guesses = None
|
||||
if self.path:
|
||||
guesses = guess_type(str(self.path))
|
||||
return guesses and guesses[0] or self.DEFAULT_MIME_TYPE
|
||||
@@ -0,0 +1,762 @@
|
||||
# commit.py
|
||||
# Copyright (C) 2008, 2009 Michael Trier (mtrier@gmail.com) and contributors
|
||||
#
|
||||
# This module is part of GitPython and is released under
|
||||
# the BSD License: http://www.opensource.org/licenses/bsd-license.php
|
||||
import datetime
|
||||
import re
|
||||
from subprocess import Popen, PIPE
|
||||
from gitdb import IStream
|
||||
from git.util import hex_to_bin, Actor, Stats, finalize_process
|
||||
from git.diff import Diffable
|
||||
from git.cmd import Git
|
||||
|
||||
from .tree import Tree
|
||||
from . import base
|
||||
from .util import (
|
||||
Serializable,
|
||||
TraversableIterableObj,
|
||||
parse_date,
|
||||
altz_to_utctz_str,
|
||||
parse_actor_and_date,
|
||||
from_timestamp,
|
||||
)
|
||||
|
||||
from time import time, daylight, altzone, timezone, localtime
|
||||
import os
|
||||
from io import BytesIO
|
||||
import logging
|
||||
|
||||
|
||||
# typing ------------------------------------------------------------------
|
||||
|
||||
from typing import (
|
||||
Any,
|
||||
IO,
|
||||
Iterator,
|
||||
List,
|
||||
Sequence,
|
||||
Tuple,
|
||||
Union,
|
||||
TYPE_CHECKING,
|
||||
cast,
|
||||
Dict,
|
||||
)
|
||||
|
||||
from git.types import PathLike, Literal
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from git.repo import Repo
|
||||
from git.refs import SymbolicReference
|
||||
|
||||
# ------------------------------------------------------------------------
|
||||
|
||||
log = logging.getLogger("git.objects.commit")
|
||||
log.addHandler(logging.NullHandler())
|
||||
|
||||
__all__ = ("Commit",)
|
||||
|
||||
|
||||
class Commit(base.Object, TraversableIterableObj, Diffable, Serializable):
|
||||
|
||||
"""Wraps a git Commit object.
|
||||
|
||||
This class will act lazily on some of its attributes and will query the
|
||||
value on demand only if it involves calling the git binary."""
|
||||
|
||||
# ENVIRONMENT VARIABLES
|
||||
# read when creating new commits
|
||||
env_author_date = "GIT_AUTHOR_DATE"
|
||||
env_committer_date = "GIT_COMMITTER_DATE"
|
||||
|
||||
# CONFIGURATION KEYS
|
||||
conf_encoding = "i18n.commitencoding"
|
||||
|
||||
# INVARIANTS
|
||||
default_encoding = "UTF-8"
|
||||
|
||||
# object configuration
|
||||
type: Literal["commit"] = "commit"
|
||||
__slots__ = (
|
||||
"tree",
|
||||
"author",
|
||||
"authored_date",
|
||||
"author_tz_offset",
|
||||
"committer",
|
||||
"committed_date",
|
||||
"committer_tz_offset",
|
||||
"message",
|
||||
"parents",
|
||||
"encoding",
|
||||
"gpgsig",
|
||||
)
|
||||
_id_attribute_ = "hexsha"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
repo: "Repo",
|
||||
binsha: bytes,
|
||||
tree: Union[Tree, None] = None,
|
||||
author: Union[Actor, None] = None,
|
||||
authored_date: Union[int, None] = None,
|
||||
author_tz_offset: Union[None, float] = None,
|
||||
committer: Union[Actor, None] = None,
|
||||
committed_date: Union[int, None] = None,
|
||||
committer_tz_offset: Union[None, float] = None,
|
||||
message: Union[str, bytes, None] = None,
|
||||
parents: Union[Sequence["Commit"], None] = None,
|
||||
encoding: Union[str, None] = None,
|
||||
gpgsig: Union[str, None] = None,
|
||||
) -> None:
|
||||
"""Instantiate a new Commit. All keyword arguments taking None as default will
|
||||
be implicitly set on first query.
|
||||
|
||||
:param binsha: 20 byte sha1
|
||||
:param parents: tuple( Commit, ... )
|
||||
is a tuple of commit ids or actual Commits
|
||||
:param tree: Tree object
|
||||
:param author: Actor
|
||||
is the author Actor object
|
||||
:param authored_date: int_seconds_since_epoch
|
||||
is the authored DateTime - use time.gmtime() to convert it into a
|
||||
different format
|
||||
:param author_tz_offset: int_seconds_west_of_utc
|
||||
is the timezone that the authored_date is in
|
||||
:param committer: Actor
|
||||
is the committer string
|
||||
:param committed_date: int_seconds_since_epoch
|
||||
is the committed DateTime - use time.gmtime() to convert it into a
|
||||
different format
|
||||
:param committer_tz_offset: int_seconds_west_of_utc
|
||||
is the timezone that the committed_date is in
|
||||
:param message: string
|
||||
is the commit message
|
||||
:param encoding: string
|
||||
encoding of the message, defaults to UTF-8
|
||||
:param parents:
|
||||
List or tuple of Commit objects which are our parent(s) in the commit
|
||||
dependency graph
|
||||
:return: git.Commit
|
||||
|
||||
:note:
|
||||
Timezone information is in the same format and in the same sign
|
||||
as what time.altzone returns. The sign is inverted compared to git's
|
||||
UTC timezone."""
|
||||
super(Commit, self).__init__(repo, binsha)
|
||||
self.binsha = binsha
|
||||
if tree is not None:
|
||||
assert isinstance(tree, Tree), "Tree needs to be a Tree instance, was %s" % type(tree)
|
||||
if tree is not None:
|
||||
self.tree = tree
|
||||
if author is not None:
|
||||
self.author = author
|
||||
if authored_date is not None:
|
||||
self.authored_date = authored_date
|
||||
if author_tz_offset is not None:
|
||||
self.author_tz_offset = author_tz_offset
|
||||
if committer is not None:
|
||||
self.committer = committer
|
||||
if committed_date is not None:
|
||||
self.committed_date = committed_date
|
||||
if committer_tz_offset is not None:
|
||||
self.committer_tz_offset = committer_tz_offset
|
||||
if message is not None:
|
||||
self.message = message
|
||||
if parents is not None:
|
||||
self.parents = parents
|
||||
if encoding is not None:
|
||||
self.encoding = encoding
|
||||
if gpgsig is not None:
|
||||
self.gpgsig = gpgsig
|
||||
|
||||
@classmethod
|
||||
def _get_intermediate_items(cls, commit: "Commit") -> Tuple["Commit", ...]:
|
||||
return tuple(commit.parents)
|
||||
|
||||
@classmethod
|
||||
def _calculate_sha_(cls, repo: "Repo", commit: "Commit") -> bytes:
|
||||
"""Calculate the sha of a commit.
|
||||
|
||||
:param repo: Repo object the commit should be part of
|
||||
:param commit: Commit object for which to generate the sha
|
||||
"""
|
||||
|
||||
stream = BytesIO()
|
||||
commit._serialize(stream)
|
||||
streamlen = stream.tell()
|
||||
stream.seek(0)
|
||||
|
||||
istream = repo.odb.store(IStream(cls.type, streamlen, stream))
|
||||
return istream.binsha
|
||||
|
||||
def replace(self, **kwargs: Any) -> "Commit":
|
||||
"""Create new commit object from existing commit object.
|
||||
|
||||
Any values provided as keyword arguments will replace the
|
||||
corresponding attribute in the new object.
|
||||
"""
|
||||
|
||||
attrs = {k: getattr(self, k) for k in self.__slots__}
|
||||
|
||||
for attrname in kwargs:
|
||||
if attrname not in self.__slots__:
|
||||
raise ValueError("invalid attribute name")
|
||||
|
||||
attrs.update(kwargs)
|
||||
new_commit = self.__class__(self.repo, self.NULL_BIN_SHA, **attrs)
|
||||
new_commit.binsha = self._calculate_sha_(self.repo, new_commit)
|
||||
|
||||
return new_commit
|
||||
|
||||
def _set_cache_(self, attr: str) -> None:
|
||||
if attr in Commit.__slots__:
|
||||
# read the data in a chunk, its faster - then provide a file wrapper
|
||||
_binsha, _typename, self.size, stream = self.repo.odb.stream(self.binsha)
|
||||
self._deserialize(BytesIO(stream.read()))
|
||||
else:
|
||||
super(Commit, self)._set_cache_(attr)
|
||||
# END handle attrs
|
||||
|
||||
@property
|
||||
def authored_datetime(self) -> datetime.datetime:
|
||||
return from_timestamp(self.authored_date, self.author_tz_offset)
|
||||
|
||||
@property
|
||||
def committed_datetime(self) -> datetime.datetime:
|
||||
return from_timestamp(self.committed_date, self.committer_tz_offset)
|
||||
|
||||
@property
|
||||
def summary(self) -> Union[str, bytes]:
|
||||
""":return: First line of the commit message"""
|
||||
if isinstance(self.message, str):
|
||||
return self.message.split("\n", 1)[0]
|
||||
else:
|
||||
return self.message.split(b"\n", 1)[0]
|
||||
|
||||
def count(self, paths: Union[PathLike, Sequence[PathLike]] = "", **kwargs: Any) -> int:
|
||||
"""Count the number of commits reachable from this commit
|
||||
|
||||
:param paths:
|
||||
is an optional path or a list of paths restricting the return value
|
||||
to commits actually containing the paths
|
||||
|
||||
:param kwargs:
|
||||
Additional options to be passed to git-rev-list. They must not alter
|
||||
the output style of the command, or parsing will yield incorrect results
|
||||
:return: int defining the number of reachable commits"""
|
||||
# yes, it makes a difference whether empty paths are given or not in our case
|
||||
# as the empty paths version will ignore merge commits for some reason.
|
||||
if paths:
|
||||
return len(self.repo.git.rev_list(self.hexsha, "--", paths, **kwargs).splitlines())
|
||||
return len(self.repo.git.rev_list(self.hexsha, **kwargs).splitlines())
|
||||
|
||||
@property
|
||||
def name_rev(self) -> str:
|
||||
"""
|
||||
:return:
|
||||
String describing the commits hex sha based on the closest Reference.
|
||||
Mostly useful for UI purposes"""
|
||||
return self.repo.git.name_rev(self)
|
||||
|
||||
@classmethod
|
||||
def iter_items(
|
||||
cls,
|
||||
repo: "Repo",
|
||||
rev: Union[str, "Commit", "SymbolicReference"], # type: ignore
|
||||
paths: Union[PathLike, Sequence[PathLike]] = "",
|
||||
**kwargs: Any,
|
||||
) -> Iterator["Commit"]:
|
||||
"""Find all commits matching the given criteria.
|
||||
|
||||
:param repo: is the Repo
|
||||
:param rev: revision specifier, see git-rev-parse for viable options
|
||||
:param paths:
|
||||
is an optional path or list of paths, if set only Commits that include the path
|
||||
or paths will be considered
|
||||
:param kwargs:
|
||||
optional keyword arguments to git rev-list where
|
||||
``max_count`` is the maximum number of commits to fetch
|
||||
``skip`` is the number of commits to skip
|
||||
``since`` all commits since i.e. '1970-01-01'
|
||||
:return: iterator yielding Commit items"""
|
||||
if "pretty" in kwargs:
|
||||
raise ValueError("--pretty cannot be used as parsing expects single sha's only")
|
||||
# END handle pretty
|
||||
|
||||
# use -- in any case, to prevent possibility of ambiguous arguments
|
||||
# see https://github.com/gitpython-developers/GitPython/issues/264
|
||||
|
||||
args_list: List[PathLike] = ["--"]
|
||||
|
||||
if paths:
|
||||
paths_tup: Tuple[PathLike, ...]
|
||||
if isinstance(paths, (str, os.PathLike)):
|
||||
paths_tup = (paths,)
|
||||
else:
|
||||
paths_tup = tuple(paths)
|
||||
|
||||
args_list.extend(paths_tup)
|
||||
# END if paths
|
||||
|
||||
proc = repo.git.rev_list(rev, args_list, as_process=True, **kwargs)
|
||||
return cls._iter_from_process_or_stream(repo, proc)
|
||||
|
||||
def iter_parents(self, paths: Union[PathLike, Sequence[PathLike]] = "", **kwargs: Any) -> Iterator["Commit"]:
|
||||
"""Iterate _all_ parents of this commit.
|
||||
|
||||
:param paths:
|
||||
Optional path or list of paths limiting the Commits to those that
|
||||
contain at least one of the paths
|
||||
:param kwargs: All arguments allowed by git-rev-list
|
||||
:return: Iterator yielding Commit objects which are parents of self"""
|
||||
# skip ourselves
|
||||
skip = kwargs.get("skip", 1)
|
||||
if skip == 0: # skip ourselves
|
||||
skip = 1
|
||||
kwargs["skip"] = skip
|
||||
|
||||
return self.iter_items(self.repo, self, paths, **kwargs)
|
||||
|
||||
@property
|
||||
def stats(self) -> Stats:
|
||||
"""Create a git stat from changes between this commit and its first parent
|
||||
or from all changes done if this is the very first commit.
|
||||
|
||||
:return: git.Stats"""
|
||||
if not self.parents:
|
||||
text = self.repo.git.diff_tree(self.hexsha, "--", numstat=True, no_renames=True, root=True)
|
||||
text2 = ""
|
||||
for line in text.splitlines()[1:]:
|
||||
(insertions, deletions, filename) = line.split("\t")
|
||||
text2 += "%s\t%s\t%s\n" % (insertions, deletions, filename)
|
||||
text = text2
|
||||
else:
|
||||
text = self.repo.git.diff(self.parents[0].hexsha, self.hexsha, "--", numstat=True, no_renames=True)
|
||||
return Stats._list_from_string(self.repo, text)
|
||||
|
||||
@property
|
||||
def trailers(self) -> Dict:
|
||||
"""Get the trailers of the message as dictionary
|
||||
|
||||
Git messages can contain trailer information that are similar to RFC 822
|
||||
e-mail headers (see: https://git-scm.com/docs/git-interpret-trailers).
|
||||
|
||||
This functions calls ``git interpret-trailers --parse`` onto the message
|
||||
to extract the trailer information. The key value pairs are stripped of
|
||||
leading and trailing whitespaces before they get saved into a dictionary.
|
||||
|
||||
Valid message with trailer:
|
||||
|
||||
.. code-block::
|
||||
|
||||
Subject line
|
||||
|
||||
some body information
|
||||
|
||||
another information
|
||||
|
||||
key1: value1
|
||||
key2 : value 2 with inner spaces
|
||||
|
||||
dictionary will look like this:
|
||||
|
||||
.. code-block::
|
||||
|
||||
{
|
||||
"key1": "value1",
|
||||
"key2": "value 2 with inner spaces"
|
||||
}
|
||||
|
||||
:return: Dictionary containing whitespace stripped trailer information
|
||||
|
||||
"""
|
||||
d = {}
|
||||
cmd = ["git", "interpret-trailers", "--parse"]
|
||||
proc: Git.AutoInterrupt = self.repo.git.execute(cmd, as_process=True, istream=PIPE) # type: ignore
|
||||
trailer: str = proc.communicate(str(self.message).encode())[0].decode()
|
||||
if trailer.endswith("\n"):
|
||||
trailer = trailer[0:-1]
|
||||
if trailer != "":
|
||||
for line in trailer.split("\n"):
|
||||
key, value = line.split(":", 1)
|
||||
d[key.strip()] = value.strip()
|
||||
return d
|
||||
|
||||
@classmethod
|
||||
def _iter_from_process_or_stream(cls, repo: "Repo", proc_or_stream: Union[Popen, IO]) -> Iterator["Commit"]:
|
||||
"""Parse out commit information into a list of Commit objects
|
||||
We expect one-line per commit, and parse the actual commit information directly
|
||||
from our lighting fast object database
|
||||
|
||||
:param proc: git-rev-list process instance - one sha per line
|
||||
:return: iterator returning Commit objects"""
|
||||
|
||||
# def is_proc(inp) -> TypeGuard[Popen]:
|
||||
# return hasattr(proc_or_stream, 'wait') and not hasattr(proc_or_stream, 'readline')
|
||||
|
||||
# def is_stream(inp) -> TypeGuard[IO]:
|
||||
# return hasattr(proc_or_stream, 'readline')
|
||||
|
||||
if hasattr(proc_or_stream, "wait"):
|
||||
proc_or_stream = cast(Popen, proc_or_stream)
|
||||
if proc_or_stream.stdout is not None:
|
||||
stream = proc_or_stream.stdout
|
||||
elif hasattr(proc_or_stream, "readline"):
|
||||
proc_or_stream = cast(IO, proc_or_stream)
|
||||
stream = proc_or_stream
|
||||
|
||||
readline = stream.readline
|
||||
while True:
|
||||
line = readline()
|
||||
if not line:
|
||||
break
|
||||
hexsha = line.strip()
|
||||
if len(hexsha) > 40:
|
||||
# split additional information, as returned by bisect for instance
|
||||
hexsha, _ = line.split(None, 1)
|
||||
# END handle extra info
|
||||
|
||||
assert len(hexsha) == 40, "Invalid line: %s" % hexsha
|
||||
yield cls(repo, hex_to_bin(hexsha))
|
||||
# END for each line in stream
|
||||
# TODO: Review this - it seems process handling got a bit out of control
|
||||
# due to many developers trying to fix the open file handles issue
|
||||
if hasattr(proc_or_stream, "wait"):
|
||||
proc_or_stream = cast(Popen, proc_or_stream)
|
||||
finalize_process(proc_or_stream)
|
||||
|
||||
@classmethod
|
||||
def create_from_tree(
|
||||
cls,
|
||||
repo: "Repo",
|
||||
tree: Union[Tree, str],
|
||||
message: str,
|
||||
parent_commits: Union[None, List["Commit"]] = None,
|
||||
head: bool = False,
|
||||
author: Union[None, Actor] = None,
|
||||
committer: Union[None, Actor] = None,
|
||||
author_date: Union[None, str, datetime.datetime] = None,
|
||||
commit_date: Union[None, str, datetime.datetime] = None,
|
||||
) -> "Commit":
|
||||
"""Commit the given tree, creating a commit object.
|
||||
|
||||
:param repo: Repo object the commit should be part of
|
||||
:param tree: Tree object or hex or bin sha
|
||||
the tree of the new commit
|
||||
:param message: Commit message. It may be an empty string if no message is provided.
|
||||
It will be converted to a string , in any case.
|
||||
:param parent_commits:
|
||||
Optional Commit objects to use as parents for the new commit.
|
||||
If empty list, the commit will have no parents at all and become
|
||||
a root commit.
|
||||
If None , the current head commit will be the parent of the
|
||||
new commit object
|
||||
:param head:
|
||||
If True, the HEAD will be advanced to the new commit automatically.
|
||||
Else the HEAD will remain pointing on the previous commit. This could
|
||||
lead to undesired results when diffing files.
|
||||
:param author: The name of the author, optional. If unset, the repository
|
||||
configuration is used to obtain this value.
|
||||
:param committer: The name of the committer, optional. If unset, the
|
||||
repository configuration is used to obtain this value.
|
||||
:param author_date: The timestamp for the author field
|
||||
:param commit_date: The timestamp for the committer field
|
||||
|
||||
:return: Commit object representing the new commit
|
||||
|
||||
:note:
|
||||
Additional information about the committer and Author are taken from the
|
||||
environment or from the git configuration, see git-commit-tree for
|
||||
more information"""
|
||||
if parent_commits is None:
|
||||
try:
|
||||
parent_commits = [repo.head.commit]
|
||||
except ValueError:
|
||||
# empty repositories have no head commit
|
||||
parent_commits = []
|
||||
# END handle parent commits
|
||||
else:
|
||||
for p in parent_commits:
|
||||
if not isinstance(p, cls):
|
||||
raise ValueError(f"Parent commit '{p!r}' must be of type {cls}")
|
||||
# end check parent commit types
|
||||
# END if parent commits are unset
|
||||
|
||||
# retrieve all additional information, create a commit object, and
|
||||
# serialize it
|
||||
# Generally:
|
||||
# * Environment variables override configuration values
|
||||
# * Sensible defaults are set according to the git documentation
|
||||
|
||||
# COMMITTER AND AUTHOR INFO
|
||||
cr = repo.config_reader()
|
||||
env = os.environ
|
||||
|
||||
committer = committer or Actor.committer(cr)
|
||||
author = author or Actor.author(cr)
|
||||
|
||||
# PARSE THE DATES
|
||||
unix_time = int(time())
|
||||
is_dst = daylight and localtime().tm_isdst > 0
|
||||
offset = altzone if is_dst else timezone
|
||||
|
||||
author_date_str = env.get(cls.env_author_date, "")
|
||||
if author_date:
|
||||
author_time, author_offset = parse_date(author_date)
|
||||
elif author_date_str:
|
||||
author_time, author_offset = parse_date(author_date_str)
|
||||
else:
|
||||
author_time, author_offset = unix_time, offset
|
||||
# END set author time
|
||||
|
||||
committer_date_str = env.get(cls.env_committer_date, "")
|
||||
if commit_date:
|
||||
committer_time, committer_offset = parse_date(commit_date)
|
||||
elif committer_date_str:
|
||||
committer_time, committer_offset = parse_date(committer_date_str)
|
||||
else:
|
||||
committer_time, committer_offset = unix_time, offset
|
||||
# END set committer time
|
||||
|
||||
# assume utf8 encoding
|
||||
enc_section, enc_option = cls.conf_encoding.split(".")
|
||||
conf_encoding = cr.get_value(enc_section, enc_option, cls.default_encoding)
|
||||
if not isinstance(conf_encoding, str):
|
||||
raise TypeError("conf_encoding could not be coerced to str")
|
||||
|
||||
# if the tree is no object, make sure we create one - otherwise
|
||||
# the created commit object is invalid
|
||||
if isinstance(tree, str):
|
||||
tree = repo.tree(tree)
|
||||
# END tree conversion
|
||||
|
||||
# CREATE NEW COMMIT
|
||||
new_commit = cls(
|
||||
repo,
|
||||
cls.NULL_BIN_SHA,
|
||||
tree,
|
||||
author,
|
||||
author_time,
|
||||
author_offset,
|
||||
committer,
|
||||
committer_time,
|
||||
committer_offset,
|
||||
message,
|
||||
parent_commits,
|
||||
conf_encoding,
|
||||
)
|
||||
|
||||
new_commit.binsha = cls._calculate_sha_(repo, new_commit)
|
||||
|
||||
if head:
|
||||
# need late import here, importing git at the very beginning throws
|
||||
# as well ...
|
||||
import git.refs
|
||||
|
||||
try:
|
||||
repo.head.set_commit(new_commit, logmsg=message)
|
||||
except ValueError:
|
||||
# head is not yet set to the ref our HEAD points to
|
||||
# Happens on first commit
|
||||
master = git.refs.Head.create(
|
||||
repo,
|
||||
repo.head.ref,
|
||||
new_commit,
|
||||
logmsg="commit (initial): %s" % message,
|
||||
)
|
||||
repo.head.set_reference(master, logmsg="commit: Switching to %s" % master)
|
||||
# END handle empty repositories
|
||||
# END advance head handling
|
||||
|
||||
return new_commit
|
||||
|
||||
# { Serializable Implementation
|
||||
|
||||
def _serialize(self, stream: BytesIO) -> "Commit":
|
||||
write = stream.write
|
||||
write(("tree %s\n" % self.tree).encode("ascii"))
|
||||
for p in self.parents:
|
||||
write(("parent %s\n" % p).encode("ascii"))
|
||||
|
||||
a = self.author
|
||||
aname = a.name
|
||||
c = self.committer
|
||||
fmt = "%s %s <%s> %s %s\n"
|
||||
write(
|
||||
(
|
||||
fmt
|
||||
% (
|
||||
"author",
|
||||
aname,
|
||||
a.email,
|
||||
self.authored_date,
|
||||
altz_to_utctz_str(self.author_tz_offset),
|
||||
)
|
||||
).encode(self.encoding)
|
||||
)
|
||||
|
||||
# encode committer
|
||||
aname = c.name
|
||||
write(
|
||||
(
|
||||
fmt
|
||||
% (
|
||||
"committer",
|
||||
aname,
|
||||
c.email,
|
||||
self.committed_date,
|
||||
altz_to_utctz_str(self.committer_tz_offset),
|
||||
)
|
||||
).encode(self.encoding)
|
||||
)
|
||||
|
||||
if self.encoding != self.default_encoding:
|
||||
write(("encoding %s\n" % self.encoding).encode("ascii"))
|
||||
|
||||
try:
|
||||
if self.__getattribute__("gpgsig"):
|
||||
write(b"gpgsig")
|
||||
for sigline in self.gpgsig.rstrip("\n").split("\n"):
|
||||
write((" " + sigline + "\n").encode("ascii"))
|
||||
except AttributeError:
|
||||
pass
|
||||
|
||||
write(b"\n")
|
||||
|
||||
# write plain bytes, be sure its encoded according to our encoding
|
||||
if isinstance(self.message, str):
|
||||
write(self.message.encode(self.encoding))
|
||||
else:
|
||||
write(self.message)
|
||||
# END handle encoding
|
||||
return self
|
||||
|
||||
def _deserialize(self, stream: BytesIO) -> "Commit":
|
||||
"""
|
||||
:param from_rev_list: if true, the stream format is coming from the rev-list command
|
||||
Otherwise it is assumed to be a plain data stream from our object
|
||||
"""
|
||||
readline = stream.readline
|
||||
self.tree = Tree(self.repo, hex_to_bin(readline().split()[1]), Tree.tree_id << 12, "")
|
||||
|
||||
self.parents = []
|
||||
next_line = None
|
||||
while True:
|
||||
parent_line = readline()
|
||||
if not parent_line.startswith(b"parent"):
|
||||
next_line = parent_line
|
||||
break
|
||||
# END abort reading parents
|
||||
self.parents.append(type(self)(self.repo, hex_to_bin(parent_line.split()[-1].decode("ascii"))))
|
||||
# END for each parent line
|
||||
self.parents = tuple(self.parents)
|
||||
|
||||
# we don't know actual author encoding before we have parsed it, so keep the lines around
|
||||
author_line = next_line
|
||||
committer_line = readline()
|
||||
|
||||
# we might run into one or more mergetag blocks, skip those for now
|
||||
next_line = readline()
|
||||
while next_line.startswith(b"mergetag "):
|
||||
next_line = readline()
|
||||
while next_line.startswith(b" "):
|
||||
next_line = readline()
|
||||
# end skip mergetags
|
||||
|
||||
# now we can have the encoding line, or an empty line followed by the optional
|
||||
# message.
|
||||
self.encoding = self.default_encoding
|
||||
self.gpgsig = ""
|
||||
|
||||
# read headers
|
||||
enc = next_line
|
||||
buf = enc.strip()
|
||||
while buf:
|
||||
if buf[0:10] == b"encoding ":
|
||||
self.encoding = buf[buf.find(b" ") + 1 :].decode(self.encoding, "ignore")
|
||||
elif buf[0:7] == b"gpgsig ":
|
||||
sig = buf[buf.find(b" ") + 1 :] + b"\n"
|
||||
is_next_header = False
|
||||
while True:
|
||||
sigbuf = readline()
|
||||
if not sigbuf:
|
||||
break
|
||||
if sigbuf[0:1] != b" ":
|
||||
buf = sigbuf.strip()
|
||||
is_next_header = True
|
||||
break
|
||||
sig += sigbuf[1:]
|
||||
# end read all signature
|
||||
self.gpgsig = sig.rstrip(b"\n").decode(self.encoding, "ignore")
|
||||
if is_next_header:
|
||||
continue
|
||||
buf = readline().strip()
|
||||
# decode the authors name
|
||||
|
||||
try:
|
||||
(
|
||||
self.author,
|
||||
self.authored_date,
|
||||
self.author_tz_offset,
|
||||
) = parse_actor_and_date(author_line.decode(self.encoding, "replace"))
|
||||
except UnicodeDecodeError:
|
||||
log.error(
|
||||
"Failed to decode author line '%s' using encoding %s",
|
||||
author_line,
|
||||
self.encoding,
|
||||
exc_info=True,
|
||||
)
|
||||
|
||||
try:
|
||||
(
|
||||
self.committer,
|
||||
self.committed_date,
|
||||
self.committer_tz_offset,
|
||||
) = parse_actor_and_date(committer_line.decode(self.encoding, "replace"))
|
||||
except UnicodeDecodeError:
|
||||
log.error(
|
||||
"Failed to decode committer line '%s' using encoding %s",
|
||||
committer_line,
|
||||
self.encoding,
|
||||
exc_info=True,
|
||||
)
|
||||
# END handle author's encoding
|
||||
|
||||
# a stream from our data simply gives us the plain message
|
||||
# The end of our message stream is marked with a newline that we strip
|
||||
self.message = stream.read()
|
||||
try:
|
||||
self.message = self.message.decode(self.encoding, "replace")
|
||||
except UnicodeDecodeError:
|
||||
log.error(
|
||||
"Failed to decode message '%s' using encoding %s",
|
||||
self.message,
|
||||
self.encoding,
|
||||
exc_info=True,
|
||||
)
|
||||
# END exception handling
|
||||
|
||||
return self
|
||||
|
||||
# } END serializable implementation
|
||||
|
||||
@property
|
||||
def co_authors(self) -> List[Actor]:
|
||||
"""
|
||||
Search the commit message for any co-authors of this commit.
|
||||
Details on co-authors: https://github.blog/2018-01-29-commit-together-with-co-authors/
|
||||
|
||||
:return: List of co-authors for this commit (as Actor objects).
|
||||
"""
|
||||
co_authors = []
|
||||
|
||||
if self.message:
|
||||
results = re.findall(
|
||||
r"^Co-authored-by: (.*) <(.*?)>$",
|
||||
self.message,
|
||||
re.MULTILINE,
|
||||
)
|
||||
for author in results:
|
||||
co_authors.append(Actor(*author))
|
||||
|
||||
return co_authors
|
||||
@@ -0,0 +1,254 @@
|
||||
"""Module with functions which are supposed to be as fast as possible"""
|
||||
from stat import S_ISDIR
|
||||
|
||||
|
||||
from git.compat import safe_decode, defenc
|
||||
|
||||
# typing ----------------------------------------------
|
||||
|
||||
from typing import (
|
||||
Callable,
|
||||
List,
|
||||
MutableSequence,
|
||||
Sequence,
|
||||
Tuple,
|
||||
TYPE_CHECKING,
|
||||
Union,
|
||||
overload,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from _typeshed import ReadableBuffer
|
||||
from git import GitCmdObjectDB
|
||||
|
||||
EntryTup = Tuple[bytes, int, str] # same as TreeCacheTup in tree.py
|
||||
EntryTupOrNone = Union[EntryTup, None]
|
||||
|
||||
# ---------------------------------------------------
|
||||
|
||||
|
||||
__all__ = (
|
||||
"tree_to_stream",
|
||||
"tree_entries_from_data",
|
||||
"traverse_trees_recursive",
|
||||
"traverse_tree_recursive",
|
||||
)
|
||||
|
||||
|
||||
def tree_to_stream(entries: Sequence[EntryTup], write: Callable[["ReadableBuffer"], Union[int, None]]) -> None:
|
||||
"""Write the give list of entries into a stream using its write method
|
||||
|
||||
:param entries: **sorted** list of tuples with (binsha, mode, name)
|
||||
:param write: write method which takes a data string"""
|
||||
ord_zero = ord("0")
|
||||
bit_mask = 7 # 3 bits set
|
||||
|
||||
for binsha, mode, name in entries:
|
||||
mode_str = b""
|
||||
for i in range(6):
|
||||
mode_str = bytes([((mode >> (i * 3)) & bit_mask) + ord_zero]) + mode_str
|
||||
# END for each 8 octal value
|
||||
|
||||
# git slices away the first octal if its zero
|
||||
if mode_str[0] == ord_zero:
|
||||
mode_str = mode_str[1:]
|
||||
# END save a byte
|
||||
|
||||
# here it comes: if the name is actually unicode, the replacement below
|
||||
# will not work as the binsha is not part of the ascii unicode encoding -
|
||||
# hence we must convert to an utf8 string for it to work properly.
|
||||
# According to my tests, this is exactly what git does, that is it just
|
||||
# takes the input literally, which appears to be utf8 on linux.
|
||||
if isinstance(name, str):
|
||||
name_bytes = name.encode(defenc)
|
||||
else:
|
||||
name_bytes = name # type: ignore[unreachable] # check runtime types - is always str?
|
||||
write(b"".join((mode_str, b" ", name_bytes, b"\0", binsha)))
|
||||
# END for each item
|
||||
|
||||
|
||||
def tree_entries_from_data(data: bytes) -> List[EntryTup]:
|
||||
"""Reads the binary representation of a tree and returns tuples of Tree items
|
||||
|
||||
:param data: data block with tree data (as bytes)
|
||||
:return: list(tuple(binsha, mode, tree_relative_path), ...)"""
|
||||
ord_zero = ord("0")
|
||||
space_ord = ord(" ")
|
||||
len_data = len(data)
|
||||
i = 0
|
||||
out = []
|
||||
while i < len_data:
|
||||
mode = 0
|
||||
|
||||
# read mode
|
||||
# Some git versions truncate the leading 0, some don't
|
||||
# The type will be extracted from the mode later
|
||||
while data[i] != space_ord:
|
||||
# move existing mode integer up one level being 3 bits
|
||||
# and add the actual ordinal value of the character
|
||||
mode = (mode << 3) + (data[i] - ord_zero)
|
||||
i += 1
|
||||
# END while reading mode
|
||||
|
||||
# byte is space now, skip it
|
||||
i += 1
|
||||
|
||||
# parse name, it is NULL separated
|
||||
|
||||
ns = i
|
||||
while data[i] != 0:
|
||||
i += 1
|
||||
# END while not reached NULL
|
||||
|
||||
# default encoding for strings in git is utf8
|
||||
# Only use the respective unicode object if the byte stream was encoded
|
||||
name_bytes = data[ns:i]
|
||||
name = safe_decode(name_bytes)
|
||||
|
||||
# byte is NULL, get next 20
|
||||
i += 1
|
||||
sha = data[i : i + 20]
|
||||
i = i + 20
|
||||
out.append((sha, mode, name))
|
||||
# END for each byte in data stream
|
||||
return out
|
||||
|
||||
|
||||
def _find_by_name(tree_data: MutableSequence[EntryTupOrNone], name: str, is_dir: bool, start_at: int) -> EntryTupOrNone:
|
||||
"""return data entry matching the given name and tree mode
|
||||
or None.
|
||||
Before the item is returned, the respective data item is set
|
||||
None in the tree_data list to mark it done"""
|
||||
|
||||
try:
|
||||
item = tree_data[start_at]
|
||||
if item and item[2] == name and S_ISDIR(item[1]) == is_dir:
|
||||
tree_data[start_at] = None
|
||||
return item
|
||||
except IndexError:
|
||||
pass
|
||||
# END exception handling
|
||||
for index, item in enumerate(tree_data):
|
||||
if item and item[2] == name and S_ISDIR(item[1]) == is_dir:
|
||||
tree_data[index] = None
|
||||
return item
|
||||
# END if item matches
|
||||
# END for each item
|
||||
return None
|
||||
|
||||
|
||||
@overload
|
||||
def _to_full_path(item: None, path_prefix: str) -> None:
|
||||
...
|
||||
|
||||
|
||||
@overload
|
||||
def _to_full_path(item: EntryTup, path_prefix: str) -> EntryTup:
|
||||
...
|
||||
|
||||
|
||||
def _to_full_path(item: EntryTupOrNone, path_prefix: str) -> EntryTupOrNone:
|
||||
"""Rebuild entry with given path prefix"""
|
||||
if not item:
|
||||
return item
|
||||
return (item[0], item[1], path_prefix + item[2])
|
||||
|
||||
|
||||
def traverse_trees_recursive(
|
||||
odb: "GitCmdObjectDB", tree_shas: Sequence[Union[bytes, None]], path_prefix: str
|
||||
) -> List[Tuple[EntryTupOrNone, ...]]:
|
||||
"""
|
||||
:return: list of list with entries according to the given binary tree-shas.
|
||||
The result is encoded in a list
|
||||
of n tuple|None per blob/commit, (n == len(tree_shas)), where
|
||||
* [0] == 20 byte sha
|
||||
* [1] == mode as int
|
||||
* [2] == path relative to working tree root
|
||||
The entry tuple is None if the respective blob/commit did not
|
||||
exist in the given tree.
|
||||
:param tree_shas: iterable of shas pointing to trees. All trees must
|
||||
be on the same level. A tree-sha may be None in which case None
|
||||
:param path_prefix: a prefix to be added to the returned paths on this level,
|
||||
set it '' for the first iteration
|
||||
:note: The ordering of the returned items will be partially lost"""
|
||||
trees_data: List[List[EntryTupOrNone]] = []
|
||||
|
||||
nt = len(tree_shas)
|
||||
for tree_sha in tree_shas:
|
||||
if tree_sha is None:
|
||||
data: List[EntryTupOrNone] = []
|
||||
else:
|
||||
# make new list for typing as list invariant
|
||||
data = list(tree_entries_from_data(odb.stream(tree_sha).read()))
|
||||
# END handle muted trees
|
||||
trees_data.append(data)
|
||||
# END for each sha to get data for
|
||||
|
||||
out: List[Tuple[EntryTupOrNone, ...]] = []
|
||||
|
||||
# find all matching entries and recursively process them together if the match
|
||||
# is a tree. If the match is a non-tree item, put it into the result.
|
||||
# Processed items will be set None
|
||||
for ti, tree_data in enumerate(trees_data):
|
||||
|
||||
for ii, item in enumerate(tree_data):
|
||||
if not item:
|
||||
continue
|
||||
# END skip already done items
|
||||
entries: List[EntryTupOrNone]
|
||||
entries = [None for _ in range(nt)]
|
||||
entries[ti] = item
|
||||
_sha, mode, name = item
|
||||
is_dir = S_ISDIR(mode) # type mode bits
|
||||
|
||||
# find this item in all other tree data items
|
||||
# wrap around, but stop one before our current index, hence
|
||||
# ti+nt, not ti+1+nt
|
||||
for tio in range(ti + 1, ti + nt):
|
||||
tio = tio % nt
|
||||
entries[tio] = _find_by_name(trees_data[tio], name, is_dir, ii)
|
||||
|
||||
# END for each other item data
|
||||
# if we are a directory, enter recursion
|
||||
if is_dir:
|
||||
out.extend(
|
||||
traverse_trees_recursive(
|
||||
odb,
|
||||
[((ei and ei[0]) or None) for ei in entries],
|
||||
path_prefix + name + "/",
|
||||
)
|
||||
)
|
||||
else:
|
||||
out.append(tuple(_to_full_path(e, path_prefix) for e in entries))
|
||||
|
||||
# END handle recursion
|
||||
# finally mark it done
|
||||
tree_data[ii] = None
|
||||
# END for each item
|
||||
|
||||
# we are done with one tree, set all its data empty
|
||||
del tree_data[:]
|
||||
# END for each tree_data chunk
|
||||
return out
|
||||
|
||||
|
||||
def traverse_tree_recursive(odb: "GitCmdObjectDB", tree_sha: bytes, path_prefix: str) -> List[EntryTup]:
|
||||
"""
|
||||
:return: list of entries of the tree pointed to by the binary tree_sha. An entry
|
||||
has the following format:
|
||||
* [0] 20 byte sha
|
||||
* [1] mode as int
|
||||
* [2] path relative to the repository
|
||||
:param path_prefix: prefix to prepend to the front of all returned paths"""
|
||||
entries = []
|
||||
data = tree_entries_from_data(odb.stream(tree_sha).read())
|
||||
|
||||
# unpacking/packing is faster than accessing individual items
|
||||
for sha, mode, name in data:
|
||||
if S_ISDIR(mode):
|
||||
entries.extend(traverse_tree_recursive(odb, sha, path_prefix + name + "/"))
|
||||
else:
|
||||
entries.append((sha, mode, path_prefix + name))
|
||||
# END for each item
|
||||
|
||||
return entries
|
||||
@@ -0,0 +1,2 @@
|
||||
# NOTE: Cannot import anything here as the top-level _init_ has to handle
|
||||
# our dependencies
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,426 @@
|
||||
from .base import Submodule, UpdateProgress
|
||||
from .util import find_first_remote_branch
|
||||
from git.exc import InvalidGitRepositoryError
|
||||
import git
|
||||
|
||||
import logging
|
||||
|
||||
# typing -------------------------------------------------------------------
|
||||
|
||||
from typing import TYPE_CHECKING, Union
|
||||
|
||||
from git.types import Commit_ish
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from git.repo import Repo
|
||||
from git.util import IterableList
|
||||
|
||||
# ----------------------------------------------------------------------------
|
||||
|
||||
__all__ = ["RootModule", "RootUpdateProgress"]
|
||||
|
||||
log = logging.getLogger("git.objects.submodule.root")
|
||||
log.addHandler(logging.NullHandler())
|
||||
|
||||
|
||||
class RootUpdateProgress(UpdateProgress):
|
||||
"""Utility class which adds more opcodes to the UpdateProgress"""
|
||||
|
||||
REMOVE, PATHCHANGE, BRANCHCHANGE, URLCHANGE = [
|
||||
1 << x for x in range(UpdateProgress._num_op_codes, UpdateProgress._num_op_codes + 4)
|
||||
]
|
||||
_num_op_codes = UpdateProgress._num_op_codes + 4
|
||||
|
||||
__slots__ = ()
|
||||
|
||||
|
||||
BEGIN = RootUpdateProgress.BEGIN
|
||||
END = RootUpdateProgress.END
|
||||
REMOVE = RootUpdateProgress.REMOVE
|
||||
BRANCHCHANGE = RootUpdateProgress.BRANCHCHANGE
|
||||
URLCHANGE = RootUpdateProgress.URLCHANGE
|
||||
PATHCHANGE = RootUpdateProgress.PATHCHANGE
|
||||
|
||||
|
||||
class RootModule(Submodule):
|
||||
|
||||
"""A (virtual) Root of all submodules in the given repository. It can be used
|
||||
to more easily traverse all submodules of the master repository"""
|
||||
|
||||
__slots__ = ()
|
||||
|
||||
k_root_name = "__ROOT__"
|
||||
|
||||
def __init__(self, repo: "Repo"):
|
||||
# repo, binsha, mode=None, path=None, name = None, parent_commit=None, url=None, ref=None)
|
||||
super(RootModule, self).__init__(
|
||||
repo,
|
||||
binsha=self.NULL_BIN_SHA,
|
||||
mode=self.k_default_mode,
|
||||
path="",
|
||||
name=self.k_root_name,
|
||||
parent_commit=repo.head.commit,
|
||||
url="",
|
||||
branch_path=git.Head.to_full_path(self.k_head_default),
|
||||
)
|
||||
|
||||
def _clear_cache(self) -> None:
|
||||
"""May not do anything"""
|
||||
pass
|
||||
|
||||
# { Interface
|
||||
|
||||
def update(
|
||||
self,
|
||||
previous_commit: Union[Commit_ish, None] = None, # type: ignore[override]
|
||||
recursive: bool = True,
|
||||
force_remove: bool = False,
|
||||
init: bool = True,
|
||||
to_latest_revision: bool = False,
|
||||
progress: Union[None, "RootUpdateProgress"] = None,
|
||||
dry_run: bool = False,
|
||||
force_reset: bool = False,
|
||||
keep_going: bool = False,
|
||||
) -> "RootModule":
|
||||
"""Update the submodules of this repository to the current HEAD commit.
|
||||
This method behaves smartly by determining changes of the path of a submodules
|
||||
repository, next to changes to the to-be-checked-out commit or the branch to be
|
||||
checked out. This works if the submodules ID does not change.
|
||||
Additionally it will detect addition and removal of submodules, which will be handled
|
||||
gracefully.
|
||||
|
||||
:param previous_commit: If set to a commit'ish, the commit we should use
|
||||
as the previous commit the HEAD pointed to before it was set to the commit it points to now.
|
||||
If None, it defaults to HEAD@{1} otherwise
|
||||
:param recursive: if True, the children of submodules will be updated as well
|
||||
using the same technique
|
||||
:param force_remove: If submodules have been deleted, they will be forcibly removed.
|
||||
Otherwise the update may fail if a submodule's repository cannot be deleted as
|
||||
changes have been made to it (see Submodule.update() for more information)
|
||||
:param init: If we encounter a new module which would need to be initialized, then do it.
|
||||
:param to_latest_revision: If True, instead of checking out the revision pointed to
|
||||
by this submodule's sha, the checked out tracking branch will be merged with the
|
||||
latest remote branch fetched from the repository's origin.
|
||||
Unless force_reset is specified, a local tracking branch will never be reset into its past, therefore
|
||||
the remote branch must be in the future for this to have an effect.
|
||||
:param force_reset: if True, submodules may checkout or reset their branch even if the repository has
|
||||
pending changes that would be overwritten, or if the local tracking branch is in the future of the
|
||||
remote tracking branch and would be reset into its past.
|
||||
:param progress: RootUpdateProgress instance or None if no progress should be sent
|
||||
:param dry_run: if True, operations will not actually be performed. Progress messages
|
||||
will change accordingly to indicate the WOULD DO state of the operation.
|
||||
:param keep_going: if True, we will ignore but log all errors, and keep going recursively.
|
||||
Unless dry_run is set as well, keep_going could cause subsequent/inherited errors you wouldn't see
|
||||
otherwise.
|
||||
In conjunction with dry_run, it can be useful to anticipate all errors when updating submodules
|
||||
:return: self"""
|
||||
if self.repo.bare:
|
||||
raise InvalidGitRepositoryError("Cannot update submodules in bare repositories")
|
||||
# END handle bare
|
||||
|
||||
if progress is None:
|
||||
progress = RootUpdateProgress()
|
||||
# END assure progress is set
|
||||
|
||||
prefix = ""
|
||||
if dry_run:
|
||||
prefix = "DRY-RUN: "
|
||||
|
||||
repo = self.repo
|
||||
|
||||
try:
|
||||
# SETUP BASE COMMIT
|
||||
###################
|
||||
cur_commit = repo.head.commit
|
||||
if previous_commit is None:
|
||||
try:
|
||||
previous_commit = repo.commit(repo.head.log_entry(-1).oldhexsha)
|
||||
if previous_commit.binsha == previous_commit.NULL_BIN_SHA:
|
||||
raise IndexError
|
||||
# END handle initial commit
|
||||
except IndexError:
|
||||
# in new repositories, there is no previous commit
|
||||
previous_commit = cur_commit
|
||||
# END exception handling
|
||||
else:
|
||||
previous_commit = repo.commit(previous_commit) # obtain commit object
|
||||
# END handle previous commit
|
||||
|
||||
psms: "IterableList[Submodule]" = self.list_items(repo, parent_commit=previous_commit)
|
||||
sms: "IterableList[Submodule]" = self.list_items(repo)
|
||||
spsms = set(psms)
|
||||
ssms = set(sms)
|
||||
|
||||
# HANDLE REMOVALS
|
||||
###################
|
||||
rrsm = spsms - ssms
|
||||
len_rrsm = len(rrsm)
|
||||
|
||||
for i, rsm in enumerate(rrsm):
|
||||
op = REMOVE
|
||||
if i == 0:
|
||||
op |= BEGIN
|
||||
# END handle begin
|
||||
|
||||
# fake it into thinking its at the current commit to allow deletion
|
||||
# of previous module. Trigger the cache to be updated before that
|
||||
progress.update(
|
||||
op,
|
||||
i,
|
||||
len_rrsm,
|
||||
prefix + "Removing submodule %r at %s" % (rsm.name, rsm.abspath),
|
||||
)
|
||||
rsm._parent_commit = repo.head.commit
|
||||
rsm.remove(
|
||||
configuration=False,
|
||||
module=True,
|
||||
force=force_remove,
|
||||
dry_run=dry_run,
|
||||
)
|
||||
|
||||
if i == len_rrsm - 1:
|
||||
op |= END
|
||||
# END handle end
|
||||
progress.update(op, i, len_rrsm, prefix + "Done removing submodule %r" % rsm.name)
|
||||
# END for each removed submodule
|
||||
|
||||
# HANDLE PATH RENAMES
|
||||
#####################
|
||||
# url changes + branch changes
|
||||
csms = spsms & ssms
|
||||
len_csms = len(csms)
|
||||
for i, csm in enumerate(csms):
|
||||
psm: "Submodule" = psms[csm.name]
|
||||
sm: "Submodule" = sms[csm.name]
|
||||
|
||||
# PATH CHANGES
|
||||
##############
|
||||
if sm.path != psm.path and psm.module_exists():
|
||||
progress.update(
|
||||
BEGIN | PATHCHANGE,
|
||||
i,
|
||||
len_csms,
|
||||
prefix + "Moving repository of submodule %r from %s to %s" % (sm.name, psm.abspath, sm.abspath),
|
||||
)
|
||||
# move the module to the new path
|
||||
if not dry_run:
|
||||
psm.move(sm.path, module=True, configuration=False)
|
||||
# END handle dry_run
|
||||
progress.update(
|
||||
END | PATHCHANGE,
|
||||
i,
|
||||
len_csms,
|
||||
prefix + "Done moving repository of submodule %r" % sm.name,
|
||||
)
|
||||
# END handle path changes
|
||||
|
||||
if sm.module_exists():
|
||||
# HANDLE URL CHANGE
|
||||
###################
|
||||
if sm.url != psm.url:
|
||||
# Add the new remote, remove the old one
|
||||
# This way, if the url just changes, the commits will not
|
||||
# have to be re-retrieved
|
||||
nn = "__new_origin__"
|
||||
smm = sm.module()
|
||||
rmts = smm.remotes
|
||||
|
||||
# don't do anything if we already have the url we search in place
|
||||
if len([r for r in rmts if r.url == sm.url]) == 0:
|
||||
progress.update(
|
||||
BEGIN | URLCHANGE,
|
||||
i,
|
||||
len_csms,
|
||||
prefix + "Changing url of submodule %r from %s to %s" % (sm.name, psm.url, sm.url),
|
||||
)
|
||||
|
||||
if not dry_run:
|
||||
assert nn not in [r.name for r in rmts]
|
||||
smr = smm.create_remote(nn, sm.url)
|
||||
smr.fetch(progress=progress)
|
||||
|
||||
# If we have a tracking branch, it should be available
|
||||
# in the new remote as well.
|
||||
if len([r for r in smr.refs if r.remote_head == sm.branch_name]) == 0:
|
||||
raise ValueError(
|
||||
"Submodule branch named %r was not available in new submodule remote at %r"
|
||||
% (sm.branch_name, sm.url)
|
||||
)
|
||||
# END head is not detached
|
||||
|
||||
# now delete the changed one
|
||||
rmt_for_deletion = None
|
||||
for remote in rmts:
|
||||
if remote.url == psm.url:
|
||||
rmt_for_deletion = remote
|
||||
break
|
||||
# END if urls match
|
||||
# END for each remote
|
||||
|
||||
# if we didn't find a matching remote, but have exactly one,
|
||||
# we can safely use this one
|
||||
if rmt_for_deletion is None:
|
||||
if len(rmts) == 1:
|
||||
rmt_for_deletion = rmts[0]
|
||||
else:
|
||||
# if we have not found any remote with the original url
|
||||
# we may not have a name. This is a special case,
|
||||
# and its okay to fail here
|
||||
# Alternatively we could just generate a unique name and leave all
|
||||
# existing ones in place
|
||||
raise InvalidGitRepositoryError(
|
||||
"Couldn't find original remote-repo at url %r" % psm.url
|
||||
)
|
||||
# END handle one single remote
|
||||
# END handle check we found a remote
|
||||
|
||||
orig_name = rmt_for_deletion.name
|
||||
smm.delete_remote(rmt_for_deletion)
|
||||
# NOTE: Currently we leave tags from the deleted remotes
|
||||
# as well as separate tracking branches in the possibly totally
|
||||
# changed repository ( someone could have changed the url to
|
||||
# another project ). At some point, one might want to clean
|
||||
# it up, but the danger is high to remove stuff the user
|
||||
# has added explicitly
|
||||
|
||||
# rename the new remote back to what it was
|
||||
smr.rename(orig_name)
|
||||
|
||||
# early on, we verified that the our current tracking branch
|
||||
# exists in the remote. Now we have to assure that the
|
||||
# sha we point to is still contained in the new remote
|
||||
# tracking branch.
|
||||
smsha = sm.binsha
|
||||
found = False
|
||||
rref = smr.refs[self.branch_name]
|
||||
for c in rref.commit.traverse():
|
||||
if c.binsha == smsha:
|
||||
found = True
|
||||
break
|
||||
# END traverse all commits in search for sha
|
||||
# END for each commit
|
||||
|
||||
if not found:
|
||||
# adjust our internal binsha to use the one of the remote
|
||||
# this way, it will be checked out in the next step
|
||||
# This will change the submodule relative to us, so
|
||||
# the user will be able to commit the change easily
|
||||
log.warning(
|
||||
"Current sha %s was not contained in the tracking\
|
||||
branch at the new remote, setting it the the remote's tracking branch",
|
||||
sm.hexsha,
|
||||
)
|
||||
sm.binsha = rref.commit.binsha
|
||||
# END reset binsha
|
||||
|
||||
# NOTE: All checkout is performed by the base implementation of update
|
||||
# END handle dry_run
|
||||
progress.update(
|
||||
END | URLCHANGE,
|
||||
i,
|
||||
len_csms,
|
||||
prefix + "Done adjusting url of submodule %r" % (sm.name),
|
||||
)
|
||||
# END skip remote handling if new url already exists in module
|
||||
# END handle url
|
||||
|
||||
# HANDLE PATH CHANGES
|
||||
#####################
|
||||
if sm.branch_path != psm.branch_path:
|
||||
# finally, create a new tracking branch which tracks the
|
||||
# new remote branch
|
||||
progress.update(
|
||||
BEGIN | BRANCHCHANGE,
|
||||
i,
|
||||
len_csms,
|
||||
prefix
|
||||
+ "Changing branch of submodule %r from %s to %s"
|
||||
% (sm.name, psm.branch_path, sm.branch_path),
|
||||
)
|
||||
if not dry_run:
|
||||
smm = sm.module()
|
||||
smmr = smm.remotes
|
||||
# As the branch might not exist yet, we will have to fetch all remotes to be sure ... .
|
||||
for remote in smmr:
|
||||
remote.fetch(progress=progress)
|
||||
# end for each remote
|
||||
|
||||
try:
|
||||
tbr = git.Head.create(
|
||||
smm,
|
||||
sm.branch_name,
|
||||
logmsg="branch: Created from HEAD",
|
||||
)
|
||||
except OSError:
|
||||
# ... or reuse the existing one
|
||||
tbr = git.Head(smm, sm.branch_path)
|
||||
# END assure tracking branch exists
|
||||
|
||||
tbr.set_tracking_branch(find_first_remote_branch(smmr, sm.branch_name))
|
||||
# NOTE: All head-resetting is done in the base implementation of update
|
||||
# but we will have to checkout the new branch here. As it still points to the currently
|
||||
# checkout out commit, we don't do any harm.
|
||||
# As we don't want to update working-tree or index, changing the ref is all there is to do
|
||||
smm.head.reference = tbr
|
||||
# END handle dry_run
|
||||
|
||||
progress.update(
|
||||
END | BRANCHCHANGE,
|
||||
i,
|
||||
len_csms,
|
||||
prefix + "Done changing branch of submodule %r" % sm.name,
|
||||
)
|
||||
# END handle branch
|
||||
# END handle
|
||||
# END for each common submodule
|
||||
except Exception as err:
|
||||
if not keep_going:
|
||||
raise
|
||||
log.error(str(err))
|
||||
# end handle keep_going
|
||||
|
||||
# FINALLY UPDATE ALL ACTUAL SUBMODULES
|
||||
######################################
|
||||
for sm in sms:
|
||||
# update the submodule using the default method
|
||||
sm.update(
|
||||
recursive=False,
|
||||
init=init,
|
||||
to_latest_revision=to_latest_revision,
|
||||
progress=progress,
|
||||
dry_run=dry_run,
|
||||
force=force_reset,
|
||||
keep_going=keep_going,
|
||||
)
|
||||
|
||||
# update recursively depth first - question is which inconsistent
|
||||
# state will be better in case it fails somewhere. Defective branch
|
||||
# or defective depth. The RootSubmodule type will never process itself,
|
||||
# which was done in the previous expression
|
||||
if recursive:
|
||||
# the module would exist by now if we are not in dry_run mode
|
||||
if sm.module_exists():
|
||||
type(self)(sm.module()).update(
|
||||
recursive=True,
|
||||
force_remove=force_remove,
|
||||
init=init,
|
||||
to_latest_revision=to_latest_revision,
|
||||
progress=progress,
|
||||
dry_run=dry_run,
|
||||
force_reset=force_reset,
|
||||
keep_going=keep_going,
|
||||
)
|
||||
# END handle dry_run
|
||||
# END handle recursive
|
||||
# END for each submodule to update
|
||||
|
||||
return self
|
||||
|
||||
def module(self) -> "Repo":
|
||||
""":return: the actual repository containing the submodules"""
|
||||
return self.repo
|
||||
|
||||
# } END interface
|
||||
|
||||
|
||||
# } END classes
|
||||
@@ -0,0 +1,118 @@
|
||||
import git
|
||||
from git.exc import InvalidGitRepositoryError
|
||||
from git.config import GitConfigParser
|
||||
from io import BytesIO
|
||||
import weakref
|
||||
|
||||
|
||||
# typing -----------------------------------------------------------------------
|
||||
|
||||
from typing import Any, Sequence, TYPE_CHECKING, Union
|
||||
|
||||
from git.types import PathLike
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from .base import Submodule
|
||||
from weakref import ReferenceType
|
||||
from git.repo import Repo
|
||||
from git.refs import Head
|
||||
from git import Remote
|
||||
from git.refs import RemoteReference
|
||||
|
||||
|
||||
__all__ = (
|
||||
"sm_section",
|
||||
"sm_name",
|
||||
"mkhead",
|
||||
"find_first_remote_branch",
|
||||
"SubmoduleConfigParser",
|
||||
)
|
||||
|
||||
# { Utilities
|
||||
|
||||
|
||||
def sm_section(name: str) -> str:
|
||||
""":return: section title used in .gitmodules configuration file"""
|
||||
return f'submodule "{name}"'
|
||||
|
||||
|
||||
def sm_name(section: str) -> str:
|
||||
""":return: name of the submodule as parsed from the section name"""
|
||||
section = section.strip()
|
||||
return section[11:-1]
|
||||
|
||||
|
||||
def mkhead(repo: "Repo", path: PathLike) -> "Head":
|
||||
""":return: New branch/head instance"""
|
||||
return git.Head(repo, git.Head.to_full_path(path))
|
||||
|
||||
|
||||
def find_first_remote_branch(remotes: Sequence["Remote"], branch_name: str) -> "RemoteReference":
|
||||
"""Find the remote branch matching the name of the given branch or raise InvalidGitRepositoryError"""
|
||||
for remote in remotes:
|
||||
try:
|
||||
return remote.refs[branch_name]
|
||||
except IndexError:
|
||||
continue
|
||||
# END exception handling
|
||||
# END for remote
|
||||
raise InvalidGitRepositoryError("Didn't find remote branch '%r' in any of the given remotes" % branch_name)
|
||||
|
||||
|
||||
# } END utilities
|
||||
|
||||
|
||||
# { Classes
|
||||
|
||||
|
||||
class SubmoduleConfigParser(GitConfigParser):
|
||||
|
||||
"""
|
||||
Catches calls to _write, and updates the .gitmodules blob in the index
|
||||
with the new data, if we have written into a stream. Otherwise it will
|
||||
add the local file to the index to make it correspond with the working tree.
|
||||
Additionally, the cache must be cleared
|
||||
|
||||
Please note that no mutating method will work in bare mode
|
||||
"""
|
||||
|
||||
def __init__(self, *args: Any, **kwargs: Any) -> None:
|
||||
self._smref: Union["ReferenceType[Submodule]", None] = None
|
||||
self._index = None
|
||||
self._auto_write = True
|
||||
super(SubmoduleConfigParser, self).__init__(*args, **kwargs)
|
||||
|
||||
# { Interface
|
||||
def set_submodule(self, submodule: "Submodule") -> None:
|
||||
"""Set this instance's submodule. It must be called before
|
||||
the first write operation begins"""
|
||||
self._smref = weakref.ref(submodule)
|
||||
|
||||
def flush_to_index(self) -> None:
|
||||
"""Flush changes in our configuration file to the index"""
|
||||
assert self._smref is not None
|
||||
# should always have a file here
|
||||
assert not isinstance(self._file_or_files, BytesIO)
|
||||
|
||||
sm = self._smref()
|
||||
if sm is not None:
|
||||
index = self._index
|
||||
if index is None:
|
||||
index = sm.repo.index
|
||||
# END handle index
|
||||
index.add([sm.k_modules_file], write=self._auto_write)
|
||||
sm._clear_cache()
|
||||
# END handle weakref
|
||||
|
||||
# } END interface
|
||||
|
||||
# { Overridden Methods
|
||||
def write(self) -> None: # type: ignore[override]
|
||||
rval: None = super(SubmoduleConfigParser, self).write()
|
||||
self.flush_to_index()
|
||||
return rval
|
||||
|
||||
# END overridden methods
|
||||
|
||||
|
||||
# } END classes
|
||||
@@ -0,0 +1,107 @@
|
||||
# objects.py
|
||||
# Copyright (C) 2008, 2009 Michael Trier (mtrier@gmail.com) and contributors
|
||||
#
|
||||
# This module is part of GitPython and is released under
|
||||
# the BSD License: http://www.opensource.org/licenses/bsd-license.php
|
||||
""" Module containing all object based types. """
|
||||
from . import base
|
||||
from .util import get_object_type_by_name, parse_actor_and_date
|
||||
from ..util import hex_to_bin
|
||||
from ..compat import defenc
|
||||
|
||||
from typing import List, TYPE_CHECKING, Union
|
||||
|
||||
from git.types import Literal
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from git.repo import Repo
|
||||
from git.util import Actor
|
||||
from .commit import Commit
|
||||
from .blob import Blob
|
||||
from .tree import Tree
|
||||
|
||||
__all__ = ("TagObject",)
|
||||
|
||||
|
||||
class TagObject(base.Object):
|
||||
|
||||
"""Non-Lightweight tag carrying additional information about an object we are pointing to."""
|
||||
|
||||
type: Literal["tag"] = "tag"
|
||||
__slots__ = (
|
||||
"object",
|
||||
"tag",
|
||||
"tagger",
|
||||
"tagged_date",
|
||||
"tagger_tz_offset",
|
||||
"message",
|
||||
)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
repo: "Repo",
|
||||
binsha: bytes,
|
||||
object: Union[None, base.Object] = None,
|
||||
tag: Union[None, str] = None,
|
||||
tagger: Union[None, "Actor"] = None,
|
||||
tagged_date: Union[int, None] = None,
|
||||
tagger_tz_offset: Union[int, None] = None,
|
||||
message: Union[str, None] = None,
|
||||
) -> None: # @ReservedAssignment
|
||||
"""Initialize a tag object with additional data
|
||||
|
||||
:param repo: repository this object is located in
|
||||
:param binsha: 20 byte SHA1
|
||||
:param object: Object instance of object we are pointing to
|
||||
:param tag: name of this tag
|
||||
:param tagger: Actor identifying the tagger
|
||||
:param tagged_date: int_seconds_since_epoch
|
||||
is the DateTime of the tag creation - use time.gmtime to convert
|
||||
it into a different format
|
||||
:param tagged_tz_offset: int_seconds_west_of_utc is the timezone that the
|
||||
authored_date is in, in a format similar to time.altzone"""
|
||||
super(TagObject, self).__init__(repo, binsha)
|
||||
if object is not None:
|
||||
self.object: Union["Commit", "Blob", "Tree", "TagObject"] = object
|
||||
if tag is not None:
|
||||
self.tag = tag
|
||||
if tagger is not None:
|
||||
self.tagger = tagger
|
||||
if tagged_date is not None:
|
||||
self.tagged_date = tagged_date
|
||||
if tagger_tz_offset is not None:
|
||||
self.tagger_tz_offset = tagger_tz_offset
|
||||
if message is not None:
|
||||
self.message = message
|
||||
|
||||
def _set_cache_(self, attr: str) -> None:
|
||||
"""Cache all our attributes at once"""
|
||||
if attr in TagObject.__slots__:
|
||||
ostream = self.repo.odb.stream(self.binsha)
|
||||
lines: List[str] = ostream.read().decode(defenc, "replace").splitlines()
|
||||
|
||||
_obj, hexsha = lines[0].split(" ")
|
||||
_type_token, type_name = lines[1].split(" ")
|
||||
object_type = get_object_type_by_name(type_name.encode("ascii"))
|
||||
self.object = object_type(self.repo, hex_to_bin(hexsha))
|
||||
|
||||
self.tag = lines[2][4:] # tag <tag name>
|
||||
|
||||
if len(lines) > 3:
|
||||
tagger_info = lines[3] # tagger <actor> <date>
|
||||
(
|
||||
self.tagger,
|
||||
self.tagged_date,
|
||||
self.tagger_tz_offset,
|
||||
) = parse_actor_and_date(tagger_info)
|
||||
|
||||
# line 4 empty - it could mark the beginning of the next header
|
||||
# in case there really is no message, it would not exist. Otherwise
|
||||
# a newline separates header from message
|
||||
if len(lines) > 5:
|
||||
self.message = "\n".join(lines[5:])
|
||||
else:
|
||||
self.message = ""
|
||||
# END check our attributes
|
||||
else:
|
||||
super(TagObject, self)._set_cache_(attr)
|
||||
@@ -0,0 +1,424 @@
|
||||
# tree.py
|
||||
# Copyright (C) 2008, 2009 Michael Trier (mtrier@gmail.com) and contributors
|
||||
#
|
||||
# This module is part of GitPython and is released under
|
||||
# the BSD License: http://www.opensource.org/licenses/bsd-license.php
|
||||
|
||||
from git.util import IterableList, join_path
|
||||
import git.diff as git_diff
|
||||
from git.util import to_bin_sha
|
||||
|
||||
from . import util
|
||||
from .base import IndexObject, IndexObjUnion
|
||||
from .blob import Blob
|
||||
from .submodule.base import Submodule
|
||||
|
||||
from .fun import tree_entries_from_data, tree_to_stream
|
||||
|
||||
|
||||
# typing -------------------------------------------------
|
||||
|
||||
from typing import (
|
||||
Any,
|
||||
Callable,
|
||||
Dict,
|
||||
Iterable,
|
||||
Iterator,
|
||||
List,
|
||||
Tuple,
|
||||
Type,
|
||||
Union,
|
||||
cast,
|
||||
TYPE_CHECKING,
|
||||
)
|
||||
|
||||
from git.types import PathLike, Literal
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from git.repo import Repo
|
||||
from io import BytesIO
|
||||
|
||||
TreeCacheTup = Tuple[bytes, int, str]
|
||||
|
||||
TraversedTreeTup = Union[Tuple[Union["Tree", None], IndexObjUnion, Tuple["Submodule", "Submodule"]]]
|
||||
|
||||
|
||||
# def is_tree_cache(inp: Tuple[bytes, int, str]) -> TypeGuard[TreeCacheTup]:
|
||||
# return isinstance(inp[0], bytes) and isinstance(inp[1], int) and isinstance([inp], str)
|
||||
|
||||
# --------------------------------------------------------
|
||||
|
||||
|
||||
cmp: Callable[[str, str], int] = lambda a, b: (a > b) - (a < b)
|
||||
|
||||
__all__ = ("TreeModifier", "Tree")
|
||||
|
||||
|
||||
def git_cmp(t1: TreeCacheTup, t2: TreeCacheTup) -> int:
|
||||
a, b = t1[2], t2[2]
|
||||
# assert isinstance(a, str) and isinstance(b, str)
|
||||
len_a, len_b = len(a), len(b)
|
||||
min_len = min(len_a, len_b)
|
||||
min_cmp = cmp(a[:min_len], b[:min_len])
|
||||
|
||||
if min_cmp:
|
||||
return min_cmp
|
||||
|
||||
return len_a - len_b
|
||||
|
||||
|
||||
def merge_sort(a: List[TreeCacheTup], cmp: Callable[[TreeCacheTup, TreeCacheTup], int]) -> None:
|
||||
if len(a) < 2:
|
||||
return None
|
||||
|
||||
mid = len(a) // 2
|
||||
lefthalf = a[:mid]
|
||||
righthalf = a[mid:]
|
||||
|
||||
merge_sort(lefthalf, cmp)
|
||||
merge_sort(righthalf, cmp)
|
||||
|
||||
i = 0
|
||||
j = 0
|
||||
k = 0
|
||||
|
||||
while i < len(lefthalf) and j < len(righthalf):
|
||||
if cmp(lefthalf[i], righthalf[j]) <= 0:
|
||||
a[k] = lefthalf[i]
|
||||
i = i + 1
|
||||
else:
|
||||
a[k] = righthalf[j]
|
||||
j = j + 1
|
||||
k = k + 1
|
||||
|
||||
while i < len(lefthalf):
|
||||
a[k] = lefthalf[i]
|
||||
i = i + 1
|
||||
k = k + 1
|
||||
|
||||
while j < len(righthalf):
|
||||
a[k] = righthalf[j]
|
||||
j = j + 1
|
||||
k = k + 1
|
||||
|
||||
|
||||
class TreeModifier(object):
|
||||
|
||||
"""A utility class providing methods to alter the underlying cache in a list-like fashion.
|
||||
|
||||
Once all adjustments are complete, the _cache, which really is a reference to
|
||||
the cache of a tree, will be sorted. Assuring it will be in a serializable state"""
|
||||
|
||||
__slots__ = "_cache"
|
||||
|
||||
def __init__(self, cache: List[TreeCacheTup]) -> None:
|
||||
self._cache = cache
|
||||
|
||||
def _index_by_name(self, name: str) -> int:
|
||||
""":return: index of an item with name, or -1 if not found"""
|
||||
for i, t in enumerate(self._cache):
|
||||
if t[2] == name:
|
||||
return i
|
||||
# END found item
|
||||
# END for each item in cache
|
||||
return -1
|
||||
|
||||
# { Interface
|
||||
def set_done(self) -> "TreeModifier":
|
||||
"""Call this method once you are done modifying the tree information.
|
||||
It may be called several times, but be aware that each call will cause
|
||||
a sort operation
|
||||
|
||||
:return self:"""
|
||||
merge_sort(self._cache, git_cmp)
|
||||
return self
|
||||
|
||||
# } END interface
|
||||
|
||||
# { Mutators
|
||||
def add(self, sha: bytes, mode: int, name: str, force: bool = False) -> "TreeModifier":
|
||||
"""Add the given item to the tree. If an item with the given name already
|
||||
exists, nothing will be done, but a ValueError will be raised if the
|
||||
sha and mode of the existing item do not match the one you add, unless
|
||||
force is True
|
||||
|
||||
:param sha: The 20 or 40 byte sha of the item to add
|
||||
:param mode: int representing the stat compatible mode of the item
|
||||
:param force: If True, an item with your name and information will overwrite
|
||||
any existing item with the same name, no matter which information it has
|
||||
:return: self"""
|
||||
if "/" in name:
|
||||
raise ValueError("Name must not contain '/' characters")
|
||||
if (mode >> 12) not in Tree._map_id_to_type:
|
||||
raise ValueError("Invalid object type according to mode %o" % mode)
|
||||
|
||||
sha = to_bin_sha(sha)
|
||||
index = self._index_by_name(name)
|
||||
|
||||
item = (sha, mode, name)
|
||||
# assert is_tree_cache(item)
|
||||
|
||||
if index == -1:
|
||||
self._cache.append(item)
|
||||
else:
|
||||
if force:
|
||||
self._cache[index] = item
|
||||
else:
|
||||
ex_item = self._cache[index]
|
||||
if ex_item[0] != sha or ex_item[1] != mode:
|
||||
raise ValueError("Item %r existed with different properties" % name)
|
||||
# END handle mismatch
|
||||
# END handle force
|
||||
# END handle name exists
|
||||
return self
|
||||
|
||||
def add_unchecked(self, binsha: bytes, mode: int, name: str) -> None:
|
||||
"""Add the given item to the tree, its correctness is assumed, which
|
||||
puts the caller into responsibility to assure the input is correct.
|
||||
For more information on the parameters, see ``add``
|
||||
|
||||
:param binsha: 20 byte binary sha"""
|
||||
assert isinstance(binsha, bytes) and isinstance(mode, int) and isinstance(name, str)
|
||||
tree_cache = (binsha, mode, name)
|
||||
|
||||
self._cache.append(tree_cache)
|
||||
|
||||
def __delitem__(self, name: str) -> None:
|
||||
"""Deletes an item with the given name if it exists"""
|
||||
index = self._index_by_name(name)
|
||||
if index > -1:
|
||||
del self._cache[index]
|
||||
|
||||
# } END mutators
|
||||
|
||||
|
||||
class Tree(IndexObject, git_diff.Diffable, util.Traversable, util.Serializable):
|
||||
|
||||
"""Tree objects represent an ordered list of Blobs and other Trees.
|
||||
|
||||
``Tree as a list``::
|
||||
|
||||
Access a specific blob using the
|
||||
tree['filename'] notation.
|
||||
|
||||
You may as well access by index
|
||||
blob = tree[0]
|
||||
"""
|
||||
|
||||
type: Literal["tree"] = "tree"
|
||||
__slots__ = "_cache"
|
||||
|
||||
# actual integer ids for comparison
|
||||
commit_id = 0o16 # equals stat.S_IFDIR | stat.S_IFLNK - a directory link
|
||||
blob_id = 0o10
|
||||
symlink_id = 0o12
|
||||
tree_id = 0o04
|
||||
|
||||
_map_id_to_type: Dict[int, Type[IndexObjUnion]] = {
|
||||
commit_id: Submodule,
|
||||
blob_id: Blob,
|
||||
symlink_id: Blob
|
||||
# tree id added once Tree is defined
|
||||
}
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
repo: "Repo",
|
||||
binsha: bytes,
|
||||
mode: int = tree_id << 12,
|
||||
path: Union[PathLike, None] = None,
|
||||
):
|
||||
super(Tree, self).__init__(repo, binsha, mode, path)
|
||||
|
||||
@classmethod
|
||||
def _get_intermediate_items(
|
||||
cls,
|
||||
index_object: IndexObjUnion,
|
||||
) -> Union[Tuple["Tree", ...], Tuple[()]]:
|
||||
if index_object.type == "tree":
|
||||
return tuple(index_object._iter_convert_to_object(index_object._cache))
|
||||
return ()
|
||||
|
||||
def _set_cache_(self, attr: str) -> None:
|
||||
if attr == "_cache":
|
||||
# Set the data when we need it
|
||||
ostream = self.repo.odb.stream(self.binsha)
|
||||
self._cache: List[TreeCacheTup] = tree_entries_from_data(ostream.read())
|
||||
else:
|
||||
super(Tree, self)._set_cache_(attr)
|
||||
# END handle attribute
|
||||
|
||||
def _iter_convert_to_object(self, iterable: Iterable[TreeCacheTup]) -> Iterator[IndexObjUnion]:
|
||||
"""Iterable yields tuples of (binsha, mode, name), which will be converted
|
||||
to the respective object representation"""
|
||||
for binsha, mode, name in iterable:
|
||||
path = join_path(self.path, name)
|
||||
try:
|
||||
yield self._map_id_to_type[mode >> 12](self.repo, binsha, mode, path)
|
||||
except KeyError as e:
|
||||
raise TypeError("Unknown mode %o found in tree data for path '%s'" % (mode, path)) from e
|
||||
# END for each item
|
||||
|
||||
def join(self, file: str) -> IndexObjUnion:
|
||||
"""Find the named object in this tree's contents
|
||||
|
||||
:return: ``git.Blob`` or ``git.Tree`` or ``git.Submodule``
|
||||
:raise KeyError: if given file or tree does not exist in tree"""
|
||||
msg = "Blob or Tree named %r not found"
|
||||
if "/" in file:
|
||||
tree = self
|
||||
item = self
|
||||
tokens = file.split("/")
|
||||
for i, token in enumerate(tokens):
|
||||
item = tree[token]
|
||||
if item.type == "tree":
|
||||
tree = item
|
||||
else:
|
||||
# safety assertion - blobs are at the end of the path
|
||||
if i != len(tokens) - 1:
|
||||
raise KeyError(msg % file)
|
||||
return item
|
||||
# END handle item type
|
||||
# END for each token of split path
|
||||
if item == self:
|
||||
raise KeyError(msg % file)
|
||||
return item
|
||||
else:
|
||||
for info in self._cache:
|
||||
if info[2] == file: # [2] == name
|
||||
return self._map_id_to_type[info[1] >> 12](
|
||||
self.repo, info[0], info[1], join_path(self.path, info[2])
|
||||
)
|
||||
# END for each obj
|
||||
raise KeyError(msg % file)
|
||||
# END handle long paths
|
||||
|
||||
def __truediv__(self, file: str) -> IndexObjUnion:
|
||||
"""For PY3 only"""
|
||||
return self.join(file)
|
||||
|
||||
@property
|
||||
def trees(self) -> List["Tree"]:
|
||||
""":return: list(Tree, ...) list of trees directly below this tree"""
|
||||
return [i for i in self if i.type == "tree"]
|
||||
|
||||
@property
|
||||
def blobs(self) -> List[Blob]:
|
||||
""":return: list(Blob, ...) list of blobs directly below this tree"""
|
||||
return [i for i in self if i.type == "blob"]
|
||||
|
||||
@property
|
||||
def cache(self) -> TreeModifier:
|
||||
"""
|
||||
:return: An object allowing to modify the internal cache. This can be used
|
||||
to change the tree's contents. When done, make sure you call ``set_done``
|
||||
on the tree modifier, or serialization behaviour will be incorrect.
|
||||
See the ``TreeModifier`` for more information on how to alter the cache"""
|
||||
return TreeModifier(self._cache)
|
||||
|
||||
def traverse(
|
||||
self, # type: ignore[override]
|
||||
predicate: Callable[[Union[IndexObjUnion, TraversedTreeTup], int], bool] = lambda i, d: True,
|
||||
prune: Callable[[Union[IndexObjUnion, TraversedTreeTup], int], bool] = lambda i, d: False,
|
||||
depth: int = -1,
|
||||
branch_first: bool = True,
|
||||
visit_once: bool = False,
|
||||
ignore_self: int = 1,
|
||||
as_edge: bool = False,
|
||||
) -> Union[Iterator[IndexObjUnion], Iterator[TraversedTreeTup]]:
|
||||
"""For documentation, see util.Traversable._traverse()
|
||||
Trees are set to visit_once = False to gain more performance in the traversal"""
|
||||
|
||||
# """
|
||||
# # To typecheck instead of using cast.
|
||||
# import itertools
|
||||
# def is_tree_traversed(inp: Tuple) -> TypeGuard[Tuple[Iterator[Union['Tree', 'Blob', 'Submodule']]]]:
|
||||
# return all(isinstance(x, (Blob, Tree, Submodule)) for x in inp[1])
|
||||
|
||||
# ret = super(Tree, self).traverse(predicate, prune, depth, branch_first, visit_once, ignore_self)
|
||||
# ret_tup = itertools.tee(ret, 2)
|
||||
# assert is_tree_traversed(ret_tup), f"Type is {[type(x) for x in list(ret_tup[0])]}"
|
||||
# return ret_tup[0]"""
|
||||
return cast(
|
||||
Union[Iterator[IndexObjUnion], Iterator[TraversedTreeTup]],
|
||||
super(Tree, self)._traverse(
|
||||
predicate,
|
||||
prune,
|
||||
depth, # type: ignore
|
||||
branch_first,
|
||||
visit_once,
|
||||
ignore_self,
|
||||
),
|
||||
)
|
||||
|
||||
def list_traverse(self, *args: Any, **kwargs: Any) -> IterableList[IndexObjUnion]:
|
||||
"""
|
||||
:return: IterableList with the results of the traversal as produced by
|
||||
traverse()
|
||||
Tree -> IterableList[Union['Submodule', 'Tree', 'Blob']]
|
||||
"""
|
||||
return super(Tree, self)._list_traverse(*args, **kwargs)
|
||||
|
||||
# List protocol
|
||||
|
||||
def __getslice__(self, i: int, j: int) -> List[IndexObjUnion]:
|
||||
return list(self._iter_convert_to_object(self._cache[i:j]))
|
||||
|
||||
def __iter__(self) -> Iterator[IndexObjUnion]:
|
||||
return self._iter_convert_to_object(self._cache)
|
||||
|
||||
def __len__(self) -> int:
|
||||
return len(self._cache)
|
||||
|
||||
def __getitem__(self, item: Union[str, int, slice]) -> IndexObjUnion:
|
||||
if isinstance(item, int):
|
||||
info = self._cache[item]
|
||||
return self._map_id_to_type[info[1] >> 12](self.repo, info[0], info[1], join_path(self.path, info[2]))
|
||||
|
||||
if isinstance(item, str):
|
||||
# compatibility
|
||||
return self.join(item)
|
||||
# END index is basestring
|
||||
|
||||
raise TypeError("Invalid index type: %r" % item)
|
||||
|
||||
def __contains__(self, item: Union[IndexObjUnion, PathLike]) -> bool:
|
||||
if isinstance(item, IndexObject):
|
||||
for info in self._cache:
|
||||
if item.binsha == info[0]:
|
||||
return True
|
||||
# END compare sha
|
||||
# END for each entry
|
||||
# END handle item is index object
|
||||
# compatibility
|
||||
|
||||
# treat item as repo-relative path
|
||||
else:
|
||||
path = self.path
|
||||
for info in self._cache:
|
||||
if item == join_path(path, info[2]):
|
||||
return True
|
||||
# END for each item
|
||||
return False
|
||||
|
||||
def __reversed__(self) -> Iterator[IndexObjUnion]:
|
||||
return reversed(self._iter_convert_to_object(self._cache)) # type: ignore
|
||||
|
||||
def _serialize(self, stream: "BytesIO") -> "Tree":
|
||||
"""Serialize this tree into the stream. Please note that we will assume
|
||||
our tree data to be in a sorted state. If this is not the case, serialization
|
||||
will not generate a correct tree representation as these are assumed to be sorted
|
||||
by algorithms"""
|
||||
tree_to_stream(self._cache, stream.write)
|
||||
return self
|
||||
|
||||
def _deserialize(self, stream: "BytesIO") -> "Tree":
|
||||
self._cache = tree_entries_from_data(stream.read())
|
||||
return self
|
||||
|
||||
|
||||
# END tree
|
||||
|
||||
# finalize map definition
|
||||
Tree._map_id_to_type[Tree.tree_id] = Tree
|
||||
#
|
||||
@@ -0,0 +1,637 @@
|
||||
# util.py
|
||||
# Copyright (C) 2008, 2009 Michael Trier (mtrier@gmail.com) and contributors
|
||||
#
|
||||
# This module is part of GitPython and is released under
|
||||
# the BSD License: http://www.opensource.org/licenses/bsd-license.php
|
||||
"""Module for general utility functions"""
|
||||
# flake8: noqa F401
|
||||
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
import warnings
|
||||
from git.util import IterableList, IterableObj, Actor
|
||||
|
||||
import re
|
||||
from collections import deque
|
||||
|
||||
from string import digits
|
||||
import time
|
||||
import calendar
|
||||
from datetime import datetime, timedelta, tzinfo
|
||||
|
||||
# typing ------------------------------------------------------------
|
||||
from typing import (
|
||||
Any,
|
||||
Callable,
|
||||
Deque,
|
||||
Iterator,
|
||||
Generic,
|
||||
NamedTuple,
|
||||
overload,
|
||||
Sequence, # NOQA: F401
|
||||
TYPE_CHECKING,
|
||||
Tuple,
|
||||
Type,
|
||||
TypeVar,
|
||||
Union,
|
||||
cast,
|
||||
)
|
||||
|
||||
from git.types import Has_id_attribute, Literal, _T # NOQA: F401
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from io import BytesIO, StringIO
|
||||
from .commit import Commit
|
||||
from .blob import Blob
|
||||
from .tag import TagObject
|
||||
from .tree import Tree, TraversedTreeTup
|
||||
from subprocess import Popen
|
||||
from .submodule.base import Submodule
|
||||
from git.types import Protocol, runtime_checkable
|
||||
else:
|
||||
# Protocol = Generic[_T] # Needed for typing bug #572?
|
||||
Protocol = ABC
|
||||
|
||||
def runtime_checkable(f):
|
||||
return f
|
||||
|
||||
|
||||
class TraverseNT(NamedTuple):
|
||||
depth: int
|
||||
item: Union["Traversable", "Blob"]
|
||||
src: Union["Traversable", None]
|
||||
|
||||
|
||||
T_TIobj = TypeVar("T_TIobj", bound="TraversableIterableObj") # for TraversableIterableObj.traverse()
|
||||
|
||||
TraversedTup = Union[
|
||||
Tuple[Union["Traversable", None], "Traversable"], # for commit, submodule
|
||||
"TraversedTreeTup",
|
||||
] # for tree.traverse()
|
||||
|
||||
# --------------------------------------------------------------------
|
||||
|
||||
__all__ = (
|
||||
"get_object_type_by_name",
|
||||
"parse_date",
|
||||
"parse_actor_and_date",
|
||||
"ProcessStreamAdapter",
|
||||
"Traversable",
|
||||
"altz_to_utctz_str",
|
||||
"utctz_to_altz",
|
||||
"verify_utctz",
|
||||
"Actor",
|
||||
"tzoffset",
|
||||
"utc",
|
||||
)
|
||||
|
||||
ZERO = timedelta(0)
|
||||
|
||||
# { Functions
|
||||
|
||||
|
||||
def mode_str_to_int(modestr: Union[bytes, str]) -> int:
|
||||
"""
|
||||
:param modestr: string like 755 or 644 or 100644 - only the last 6 chars will be used
|
||||
:return:
|
||||
String identifying a mode compatible to the mode methods ids of the
|
||||
stat module regarding the rwx permissions for user, group and other,
|
||||
special flags and file system flags, i.e. whether it is a symlink
|
||||
for example."""
|
||||
mode = 0
|
||||
for iteration, char in enumerate(reversed(modestr[-6:])):
|
||||
char = cast(Union[str, int], char)
|
||||
mode += int(char) << iteration * 3
|
||||
# END for each char
|
||||
return mode
|
||||
|
||||
|
||||
def get_object_type_by_name(
|
||||
object_type_name: bytes,
|
||||
) -> Union[Type["Commit"], Type["TagObject"], Type["Tree"], Type["Blob"]]:
|
||||
"""
|
||||
:return: type suitable to handle the given object type name.
|
||||
Use the type to create new instances.
|
||||
|
||||
:param object_type_name: Member of TYPES
|
||||
|
||||
:raise ValueError: In case object_type_name is unknown"""
|
||||
if object_type_name == b"commit":
|
||||
from . import commit
|
||||
|
||||
return commit.Commit
|
||||
elif object_type_name == b"tag":
|
||||
from . import tag
|
||||
|
||||
return tag.TagObject
|
||||
elif object_type_name == b"blob":
|
||||
from . import blob
|
||||
|
||||
return blob.Blob
|
||||
elif object_type_name == b"tree":
|
||||
from . import tree
|
||||
|
||||
return tree.Tree
|
||||
else:
|
||||
raise ValueError("Cannot handle unknown object type: %s" % object_type_name.decode())
|
||||
|
||||
|
||||
def utctz_to_altz(utctz: str) -> int:
|
||||
"""Convert a git timezone offset into a timezone offset west of
|
||||
UTC in seconds (compatible with time.altzone).
|
||||
|
||||
:param utctz: git utc timezone string, i.e. +0200
|
||||
"""
|
||||
int_utctz = int(utctz)
|
||||
seconds = ((abs(int_utctz) // 100) * 3600 + (abs(int_utctz) % 100) * 60)
|
||||
return seconds if int_utctz < 0 else -seconds
|
||||
|
||||
|
||||
def altz_to_utctz_str(altz: int) -> str:
|
||||
"""Convert a timezone offset west of UTC in seconds into a git timezone offset string
|
||||
|
||||
:param altz: timezone offset in seconds west of UTC
|
||||
"""
|
||||
hours = abs(altz) // 3600
|
||||
minutes = (abs(altz) % 3600) // 60
|
||||
sign = "-" if altz >= 60 else "+"
|
||||
return "{}{:02}{:02}".format(sign, hours, minutes)
|
||||
|
||||
|
||||
def verify_utctz(offset: str) -> str:
|
||||
""":raise ValueError: if offset is incorrect
|
||||
:return: offset"""
|
||||
fmt_exc = ValueError("Invalid timezone offset format: %s" % offset)
|
||||
if len(offset) != 5:
|
||||
raise fmt_exc
|
||||
if offset[0] not in "+-":
|
||||
raise fmt_exc
|
||||
if offset[1] not in digits or offset[2] not in digits or offset[3] not in digits or offset[4] not in digits:
|
||||
raise fmt_exc
|
||||
# END for each char
|
||||
return offset
|
||||
|
||||
|
||||
class tzoffset(tzinfo):
|
||||
def __init__(self, secs_west_of_utc: float, name: Union[None, str] = None) -> None:
|
||||
self._offset = timedelta(seconds=-secs_west_of_utc)
|
||||
self._name = name or "fixed"
|
||||
|
||||
def __reduce__(self) -> Tuple[Type["tzoffset"], Tuple[float, str]]:
|
||||
return tzoffset, (-self._offset.total_seconds(), self._name)
|
||||
|
||||
def utcoffset(self, dt: Union[datetime, None]) -> timedelta:
|
||||
return self._offset
|
||||
|
||||
def tzname(self, dt: Union[datetime, None]) -> str:
|
||||
return self._name
|
||||
|
||||
def dst(self, dt: Union[datetime, None]) -> timedelta:
|
||||
return ZERO
|
||||
|
||||
|
||||
utc = tzoffset(0, "UTC")
|
||||
|
||||
|
||||
def from_timestamp(timestamp: float, tz_offset: float) -> datetime:
|
||||
"""Converts a timestamp + tz_offset into an aware datetime instance."""
|
||||
utc_dt = datetime.fromtimestamp(timestamp, utc)
|
||||
try:
|
||||
local_dt = utc_dt.astimezone(tzoffset(tz_offset))
|
||||
return local_dt
|
||||
except ValueError:
|
||||
return utc_dt
|
||||
|
||||
|
||||
def parse_date(string_date: Union[str, datetime]) -> Tuple[int, int]:
|
||||
"""
|
||||
Parse the given date as one of the following
|
||||
|
||||
* aware datetime instance
|
||||
* Git internal format: timestamp offset
|
||||
* RFC 2822: Thu, 07 Apr 2005 22:13:13 +0200.
|
||||
* ISO 8601 2005-04-07T22:13:13
|
||||
The T can be a space as well
|
||||
|
||||
:return: Tuple(int(timestamp_UTC), int(offset)), both in seconds since epoch
|
||||
:raise ValueError: If the format could not be understood
|
||||
:note: Date can also be YYYY.MM.DD, MM/DD/YYYY and DD.MM.YYYY.
|
||||
"""
|
||||
if isinstance(string_date, datetime):
|
||||
if string_date.tzinfo:
|
||||
utcoffset = cast(timedelta, string_date.utcoffset()) # typeguard, if tzinfoand is not None
|
||||
offset = -int(utcoffset.total_seconds())
|
||||
return int(string_date.astimezone(utc).timestamp()), offset
|
||||
else:
|
||||
raise ValueError(f"string_date datetime object without tzinfo, {string_date}")
|
||||
|
||||
# git time
|
||||
try:
|
||||
if string_date.count(" ") == 1 and string_date.rfind(":") == -1:
|
||||
timestamp, offset_str = string_date.split()
|
||||
if timestamp.startswith("@"):
|
||||
timestamp = timestamp[1:]
|
||||
timestamp_int = int(timestamp)
|
||||
return timestamp_int, utctz_to_altz(verify_utctz(offset_str))
|
||||
else:
|
||||
offset_str = "+0000" # local time by default
|
||||
if string_date[-5] in "-+":
|
||||
offset_str = verify_utctz(string_date[-5:])
|
||||
string_date = string_date[:-6] # skip space as well
|
||||
# END split timezone info
|
||||
offset = utctz_to_altz(offset_str)
|
||||
|
||||
# now figure out the date and time portion - split time
|
||||
date_formats = []
|
||||
splitter = -1
|
||||
if "," in string_date:
|
||||
date_formats.append("%a, %d %b %Y")
|
||||
splitter = string_date.rfind(" ")
|
||||
else:
|
||||
# iso plus additional
|
||||
date_formats.append("%Y-%m-%d")
|
||||
date_formats.append("%Y.%m.%d")
|
||||
date_formats.append("%m/%d/%Y")
|
||||
date_formats.append("%d.%m.%Y")
|
||||
|
||||
splitter = string_date.rfind("T")
|
||||
if splitter == -1:
|
||||
splitter = string_date.rfind(" ")
|
||||
# END handle 'T' and ' '
|
||||
# END handle rfc or iso
|
||||
|
||||
assert splitter > -1
|
||||
|
||||
# split date and time
|
||||
time_part = string_date[splitter + 1 :] # skip space
|
||||
date_part = string_date[:splitter]
|
||||
|
||||
# parse time
|
||||
tstruct = time.strptime(time_part, "%H:%M:%S")
|
||||
|
||||
for fmt in date_formats:
|
||||
try:
|
||||
dtstruct = time.strptime(date_part, fmt)
|
||||
utctime = calendar.timegm(
|
||||
(
|
||||
dtstruct.tm_year,
|
||||
dtstruct.tm_mon,
|
||||
dtstruct.tm_mday,
|
||||
tstruct.tm_hour,
|
||||
tstruct.tm_min,
|
||||
tstruct.tm_sec,
|
||||
dtstruct.tm_wday,
|
||||
dtstruct.tm_yday,
|
||||
tstruct.tm_isdst,
|
||||
)
|
||||
)
|
||||
return int(utctime), offset
|
||||
except ValueError:
|
||||
continue
|
||||
# END exception handling
|
||||
# END for each fmt
|
||||
|
||||
# still here ? fail
|
||||
raise ValueError("no format matched")
|
||||
# END handle format
|
||||
except Exception as e:
|
||||
raise ValueError(f"Unsupported date format or type: {string_date}, type={type(string_date)}") from e
|
||||
# END handle exceptions
|
||||
|
||||
|
||||
# precompiled regex
|
||||
_re_actor_epoch = re.compile(r"^.+? (.*) (\d+) ([+-]\d+).*$")
|
||||
_re_only_actor = re.compile(r"^.+? (.*)$")
|
||||
|
||||
|
||||
def parse_actor_and_date(line: str) -> Tuple[Actor, int, int]:
|
||||
"""Parse out the actor (author or committer) info from a line like::
|
||||
|
||||
author Tom Preston-Werner <tom@mojombo.com> 1191999972 -0700
|
||||
|
||||
:return: [Actor, int_seconds_since_epoch, int_timezone_offset]"""
|
||||
actor, epoch, offset = "", "0", "0"
|
||||
m = _re_actor_epoch.search(line)
|
||||
if m:
|
||||
actor, epoch, offset = m.groups()
|
||||
else:
|
||||
m = _re_only_actor.search(line)
|
||||
actor = m.group(1) if m else line or ""
|
||||
return (Actor._from_string(actor), int(epoch), utctz_to_altz(offset))
|
||||
|
||||
|
||||
# } END functions
|
||||
|
||||
|
||||
# { Classes
|
||||
|
||||
|
||||
class ProcessStreamAdapter(object):
|
||||
|
||||
"""Class wireing all calls to the contained Process instance.
|
||||
|
||||
Use this type to hide the underlying process to provide access only to a specified
|
||||
stream. The process is usually wrapped into an AutoInterrupt class to kill
|
||||
it if the instance goes out of scope."""
|
||||
|
||||
__slots__ = ("_proc", "_stream")
|
||||
|
||||
def __init__(self, process: "Popen", stream_name: str) -> None:
|
||||
self._proc = process
|
||||
self._stream: StringIO = getattr(process, stream_name) # guessed type
|
||||
|
||||
def __getattr__(self, attr: str) -> Any:
|
||||
return getattr(self._stream, attr)
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class Traversable(Protocol):
|
||||
|
||||
"""Simple interface to perform depth-first or breadth-first traversals
|
||||
into one direction.
|
||||
Subclasses only need to implement one function.
|
||||
Instances of the Subclass must be hashable
|
||||
|
||||
Defined subclasses = [Commit, Tree, SubModule]
|
||||
"""
|
||||
|
||||
__slots__ = ()
|
||||
|
||||
@classmethod
|
||||
@abstractmethod
|
||||
def _get_intermediate_items(cls, item: Any) -> Sequence["Traversable"]:
|
||||
"""
|
||||
Returns:
|
||||
Tuple of items connected to the given item.
|
||||
Must be implemented in subclass
|
||||
|
||||
class Commit:: (cls, Commit) -> Tuple[Commit, ...]
|
||||
class Submodule:: (cls, Submodule) -> Iterablelist[Submodule]
|
||||
class Tree:: (cls, Tree) -> Tuple[Tree, ...]
|
||||
"""
|
||||
raise NotImplementedError("To be implemented in subclass")
|
||||
|
||||
@abstractmethod
|
||||
def list_traverse(self, *args: Any, **kwargs: Any) -> Any:
|
||||
""" """
|
||||
warnings.warn(
|
||||
"list_traverse() method should only be called from subclasses."
|
||||
"Calling from Traversable abstract class will raise NotImplementedError in 3.1.20"
|
||||
"Builtin sublclasses are 'Submodule', 'Tree' and 'Commit",
|
||||
DeprecationWarning,
|
||||
stacklevel=2,
|
||||
)
|
||||
return self._list_traverse(*args, **kwargs)
|
||||
|
||||
def _list_traverse(
|
||||
self, as_edge: bool = False, *args: Any, **kwargs: Any
|
||||
) -> IterableList[Union["Commit", "Submodule", "Tree", "Blob"]]:
|
||||
"""
|
||||
:return: IterableList with the results of the traversal as produced by
|
||||
traverse()
|
||||
Commit -> IterableList['Commit']
|
||||
Submodule -> IterableList['Submodule']
|
||||
Tree -> IterableList[Union['Submodule', 'Tree', 'Blob']]
|
||||
"""
|
||||
# Commit and Submodule have id.__attribute__ as IterableObj
|
||||
# Tree has id.__attribute__ inherited from IndexObject
|
||||
if isinstance(self, Has_id_attribute):
|
||||
id = self._id_attribute_
|
||||
else:
|
||||
id = "" # shouldn't reach here, unless Traversable subclass created with no _id_attribute_
|
||||
# could add _id_attribute_ to Traversable, or make all Traversable also Iterable?
|
||||
|
||||
if not as_edge:
|
||||
out: IterableList[Union["Commit", "Submodule", "Tree", "Blob"]] = IterableList(id)
|
||||
out.extend(self.traverse(as_edge=as_edge, *args, **kwargs))
|
||||
return out
|
||||
# overloads in subclasses (mypy doesn't allow typing self: subclass)
|
||||
# Union[IterableList['Commit'], IterableList['Submodule'], IterableList[Union['Submodule', 'Tree', 'Blob']]]
|
||||
else:
|
||||
# Raise deprecationwarning, doesn't make sense to use this
|
||||
out_list: IterableList = IterableList(self.traverse(*args, **kwargs))
|
||||
return out_list
|
||||
|
||||
@abstractmethod
|
||||
def traverse(self, *args: Any, **kwargs: Any) -> Any:
|
||||
""" """
|
||||
warnings.warn(
|
||||
"traverse() method should only be called from subclasses."
|
||||
"Calling from Traversable abstract class will raise NotImplementedError in 3.1.20"
|
||||
"Builtin sublclasses are 'Submodule', 'Tree' and 'Commit",
|
||||
DeprecationWarning,
|
||||
stacklevel=2,
|
||||
)
|
||||
return self._traverse(*args, **kwargs)
|
||||
|
||||
def _traverse(
|
||||
self,
|
||||
predicate: Callable[[Union["Traversable", "Blob", TraversedTup], int], bool] = lambda i, d: True,
|
||||
prune: Callable[[Union["Traversable", "Blob", TraversedTup], int], bool] = lambda i, d: False,
|
||||
depth: int = -1,
|
||||
branch_first: bool = True,
|
||||
visit_once: bool = True,
|
||||
ignore_self: int = 1,
|
||||
as_edge: bool = False,
|
||||
) -> Union[Iterator[Union["Traversable", "Blob"]], Iterator[TraversedTup]]:
|
||||
""":return: iterator yielding of items found when traversing self
|
||||
:param predicate: f(i,d) returns False if item i at depth d should not be included in the result
|
||||
|
||||
:param prune:
|
||||
f(i,d) return True if the search should stop at item i at depth d.
|
||||
Item i will not be returned.
|
||||
|
||||
:param depth:
|
||||
define at which level the iteration should not go deeper
|
||||
if -1, there is no limit
|
||||
if 0, you would effectively only get self, the root of the iteration
|
||||
i.e. if 1, you would only get the first level of predecessors/successors
|
||||
|
||||
:param branch_first:
|
||||
if True, items will be returned branch first, otherwise depth first
|
||||
|
||||
:param visit_once:
|
||||
if True, items will only be returned once, although they might be encountered
|
||||
several times. Loops are prevented that way.
|
||||
|
||||
:param ignore_self:
|
||||
if True, self will be ignored and automatically pruned from
|
||||
the result. Otherwise it will be the first item to be returned.
|
||||
If as_edge is True, the source of the first edge is None
|
||||
|
||||
:param as_edge:
|
||||
if True, return a pair of items, first being the source, second the
|
||||
destination, i.e. tuple(src, dest) with the edge spanning from
|
||||
source to destination"""
|
||||
|
||||
"""
|
||||
Commit -> Iterator[Union[Commit, Tuple[Commit, Commit]]
|
||||
Submodule -> Iterator[Submodule, Tuple[Submodule, Submodule]]
|
||||
Tree -> Iterator[Union[Blob, Tree, Submodule,
|
||||
Tuple[Union[Submodule, Tree], Union[Blob, Tree, Submodule]]]
|
||||
|
||||
ignore_self=True is_edge=True -> Iterator[item]
|
||||
ignore_self=True is_edge=False --> Iterator[item]
|
||||
ignore_self=False is_edge=True -> Iterator[item] | Iterator[Tuple[src, item]]
|
||||
ignore_self=False is_edge=False -> Iterator[Tuple[src, item]]"""
|
||||
|
||||
visited = set()
|
||||
stack: Deque[TraverseNT] = deque()
|
||||
stack.append(TraverseNT(0, self, None)) # self is always depth level 0
|
||||
|
||||
def addToStack(
|
||||
stack: Deque[TraverseNT],
|
||||
src_item: "Traversable",
|
||||
branch_first: bool,
|
||||
depth: int,
|
||||
) -> None:
|
||||
lst = self._get_intermediate_items(item)
|
||||
if not lst: # empty list
|
||||
return None
|
||||
if branch_first:
|
||||
stack.extendleft(TraverseNT(depth, i, src_item) for i in lst)
|
||||
else:
|
||||
reviter = (TraverseNT(depth, lst[i], src_item) for i in range(len(lst) - 1, -1, -1))
|
||||
stack.extend(reviter)
|
||||
|
||||
# END addToStack local method
|
||||
|
||||
while stack:
|
||||
d, item, src = stack.pop() # depth of item, item, item_source
|
||||
|
||||
if visit_once and item in visited:
|
||||
continue
|
||||
|
||||
if visit_once:
|
||||
visited.add(item)
|
||||
|
||||
rval: Union[TraversedTup, "Traversable", "Blob"]
|
||||
if as_edge: # if as_edge return (src, item) unless rrc is None (e.g. for first item)
|
||||
rval = (src, item)
|
||||
else:
|
||||
rval = item
|
||||
|
||||
if prune(rval, d):
|
||||
continue
|
||||
|
||||
skipStartItem = ignore_self and (item is self)
|
||||
if not skipStartItem and predicate(rval, d):
|
||||
yield rval
|
||||
|
||||
# only continue to next level if this is appropriate !
|
||||
nd = d + 1
|
||||
if depth > -1 and nd > depth:
|
||||
continue
|
||||
|
||||
addToStack(stack, item, branch_first, nd)
|
||||
# END for each item on work stack
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class Serializable(Protocol):
|
||||
|
||||
"""Defines methods to serialize and deserialize objects from and into a data stream"""
|
||||
|
||||
__slots__ = ()
|
||||
|
||||
# @abstractmethod
|
||||
def _serialize(self, stream: "BytesIO") -> "Serializable":
|
||||
"""Serialize the data of this object into the given data stream
|
||||
:note: a serialized object would ``_deserialize`` into the same object
|
||||
:param stream: a file-like object
|
||||
:return: self"""
|
||||
raise NotImplementedError("To be implemented in subclass")
|
||||
|
||||
# @abstractmethod
|
||||
def _deserialize(self, stream: "BytesIO") -> "Serializable":
|
||||
"""Deserialize all information regarding this object from the stream
|
||||
:param stream: a file-like object
|
||||
:return: self"""
|
||||
raise NotImplementedError("To be implemented in subclass")
|
||||
|
||||
|
||||
class TraversableIterableObj(IterableObj, Traversable):
|
||||
__slots__ = ()
|
||||
|
||||
TIobj_tuple = Tuple[Union[T_TIobj, None], T_TIobj]
|
||||
|
||||
def list_traverse(self: T_TIobj, *args: Any, **kwargs: Any) -> IterableList[T_TIobj]:
|
||||
return super(TraversableIterableObj, self)._list_traverse(*args, **kwargs)
|
||||
|
||||
@overload # type: ignore
|
||||
def traverse(self: T_TIobj) -> Iterator[T_TIobj]:
|
||||
...
|
||||
|
||||
@overload
|
||||
def traverse(
|
||||
self: T_TIobj,
|
||||
predicate: Callable[[Union[T_TIobj, Tuple[Union[T_TIobj, None], T_TIobj]], int], bool],
|
||||
prune: Callable[[Union[T_TIobj, Tuple[Union[T_TIobj, None], T_TIobj]], int], bool],
|
||||
depth: int,
|
||||
branch_first: bool,
|
||||
visit_once: bool,
|
||||
ignore_self: Literal[True],
|
||||
as_edge: Literal[False],
|
||||
) -> Iterator[T_TIobj]:
|
||||
...
|
||||
|
||||
@overload
|
||||
def traverse(
|
||||
self: T_TIobj,
|
||||
predicate: Callable[[Union[T_TIobj, Tuple[Union[T_TIobj, None], T_TIobj]], int], bool],
|
||||
prune: Callable[[Union[T_TIobj, Tuple[Union[T_TIobj, None], T_TIobj]], int], bool],
|
||||
depth: int,
|
||||
branch_first: bool,
|
||||
visit_once: bool,
|
||||
ignore_self: Literal[False],
|
||||
as_edge: Literal[True],
|
||||
) -> Iterator[Tuple[Union[T_TIobj, None], T_TIobj]]:
|
||||
...
|
||||
|
||||
@overload
|
||||
def traverse(
|
||||
self: T_TIobj,
|
||||
predicate: Callable[[Union[T_TIobj, TIobj_tuple], int], bool],
|
||||
prune: Callable[[Union[T_TIobj, TIobj_tuple], int], bool],
|
||||
depth: int,
|
||||
branch_first: bool,
|
||||
visit_once: bool,
|
||||
ignore_self: Literal[True],
|
||||
as_edge: Literal[True],
|
||||
) -> Iterator[Tuple[T_TIobj, T_TIobj]]:
|
||||
...
|
||||
|
||||
def traverse(
|
||||
self: T_TIobj,
|
||||
predicate: Callable[[Union[T_TIobj, TIobj_tuple], int], bool] = lambda i, d: True,
|
||||
prune: Callable[[Union[T_TIobj, TIobj_tuple], int], bool] = lambda i, d: False,
|
||||
depth: int = -1,
|
||||
branch_first: bool = True,
|
||||
visit_once: bool = True,
|
||||
ignore_self: int = 1,
|
||||
as_edge: bool = False,
|
||||
) -> Union[Iterator[T_TIobj], Iterator[Tuple[T_TIobj, T_TIobj]], Iterator[TIobj_tuple]]:
|
||||
"""For documentation, see util.Traversable._traverse()"""
|
||||
|
||||
"""
|
||||
# To typecheck instead of using cast.
|
||||
import itertools
|
||||
from git.types import TypeGuard
|
||||
def is_commit_traversed(inp: Tuple) -> TypeGuard[Tuple[Iterator[Tuple['Commit', 'Commit']]]]:
|
||||
for x in inp[1]:
|
||||
if not isinstance(x, tuple) and len(x) != 2:
|
||||
if all(isinstance(inner, Commit) for inner in x):
|
||||
continue
|
||||
return True
|
||||
|
||||
ret = super(Commit, self).traverse(predicate, prune, depth, branch_first, visit_once, ignore_self, as_edge)
|
||||
ret_tup = itertools.tee(ret, 2)
|
||||
assert is_commit_traversed(ret_tup), f"{[type(x) for x in list(ret_tup[0])]}"
|
||||
return ret_tup[0]
|
||||
"""
|
||||
return cast(
|
||||
Union[Iterator[T_TIobj], Iterator[Tuple[Union[None, T_TIobj], T_TIobj]]],
|
||||
super(TraversableIterableObj, self)._traverse(
|
||||
predicate, prune, depth, branch_first, visit_once, ignore_self, as_edge # type: ignore
|
||||
),
|
||||
)
|
||||
@@ -0,0 +1,9 @@
|
||||
# flake8: noqa
|
||||
# import all modules in order, fix the names they require
|
||||
from .symbolic import *
|
||||
from .reference import *
|
||||
from .head import *
|
||||
from .tag import *
|
||||
from .remote import *
|
||||
|
||||
from .log import *
|
||||
277
zero-cost-nas/.eggs/GitPython-3.1.31-py3.8.egg/git/refs/head.py
Normal file
277
zero-cost-nas/.eggs/GitPython-3.1.31-py3.8.egg/git/refs/head.py
Normal file
@@ -0,0 +1,277 @@
|
||||
from git.config import GitConfigParser, SectionConstraint
|
||||
from git.util import join_path
|
||||
from git.exc import GitCommandError
|
||||
|
||||
from .symbolic import SymbolicReference
|
||||
from .reference import Reference
|
||||
|
||||
# typinng ---------------------------------------------------
|
||||
|
||||
from typing import Any, Sequence, Union, TYPE_CHECKING
|
||||
|
||||
from git.types import PathLike, Commit_ish
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from git.repo import Repo
|
||||
from git.objects import Commit
|
||||
from git.refs import RemoteReference
|
||||
|
||||
# -------------------------------------------------------------------
|
||||
|
||||
__all__ = ["HEAD", "Head"]
|
||||
|
||||
|
||||
def strip_quotes(string: str) -> str:
|
||||
if string.startswith('"') and string.endswith('"'):
|
||||
return string[1:-1]
|
||||
return string
|
||||
|
||||
|
||||
class HEAD(SymbolicReference):
|
||||
|
||||
"""Special case of a Symbolic Reference as it represents the repository's
|
||||
HEAD reference."""
|
||||
|
||||
_HEAD_NAME = "HEAD"
|
||||
_ORIG_HEAD_NAME = "ORIG_HEAD"
|
||||
__slots__ = ()
|
||||
|
||||
def __init__(self, repo: "Repo", path: PathLike = _HEAD_NAME):
|
||||
if path != self._HEAD_NAME:
|
||||
raise ValueError("HEAD instance must point to %r, got %r" % (self._HEAD_NAME, path))
|
||||
super(HEAD, self).__init__(repo, path)
|
||||
self.commit: "Commit"
|
||||
|
||||
def orig_head(self) -> SymbolicReference:
|
||||
"""
|
||||
:return: SymbolicReference pointing at the ORIG_HEAD, which is maintained
|
||||
to contain the previous value of HEAD"""
|
||||
return SymbolicReference(self.repo, self._ORIG_HEAD_NAME)
|
||||
|
||||
def reset(
|
||||
self,
|
||||
commit: Union[Commit_ish, SymbolicReference, str] = "HEAD",
|
||||
index: bool = True,
|
||||
working_tree: bool = False,
|
||||
paths: Union[PathLike, Sequence[PathLike], None] = None,
|
||||
**kwargs: Any,
|
||||
) -> "HEAD":
|
||||
"""Reset our HEAD to the given commit optionally synchronizing
|
||||
the index and working tree. The reference we refer to will be set to
|
||||
commit as well.
|
||||
|
||||
:param commit:
|
||||
Commit object, Reference Object or string identifying a revision we
|
||||
should reset HEAD to.
|
||||
|
||||
:param index:
|
||||
If True, the index will be set to match the given commit. Otherwise
|
||||
it will not be touched.
|
||||
|
||||
:param working_tree:
|
||||
If True, the working tree will be forcefully adjusted to match the given
|
||||
commit, possibly overwriting uncommitted changes without warning.
|
||||
If working_tree is True, index must be true as well
|
||||
|
||||
:param paths:
|
||||
Single path or list of paths relative to the git root directory
|
||||
that are to be reset. This allows to partially reset individual files.
|
||||
|
||||
:param kwargs:
|
||||
Additional arguments passed to git-reset.
|
||||
|
||||
:return: self"""
|
||||
mode: Union[str, None]
|
||||
mode = "--soft"
|
||||
if index:
|
||||
mode = "--mixed"
|
||||
|
||||
# it appears, some git-versions declare mixed and paths deprecated
|
||||
# see http://github.com/Byron/GitPython/issues#issue/2
|
||||
if paths:
|
||||
mode = None
|
||||
# END special case
|
||||
# END handle index
|
||||
|
||||
if working_tree:
|
||||
mode = "--hard"
|
||||
if not index:
|
||||
raise ValueError("Cannot reset the working tree if the index is not reset as well")
|
||||
|
||||
# END working tree handling
|
||||
|
||||
try:
|
||||
self.repo.git.reset(mode, commit, "--", paths, **kwargs)
|
||||
except GitCommandError as e:
|
||||
# git nowadays may use 1 as status to indicate there are still unstaged
|
||||
# modifications after the reset
|
||||
if e.status != 1:
|
||||
raise
|
||||
# END handle exception
|
||||
|
||||
return self
|
||||
|
||||
|
||||
class Head(Reference):
|
||||
|
||||
"""A Head is a named reference to a Commit. Every Head instance contains a name
|
||||
and a Commit object.
|
||||
|
||||
Examples::
|
||||
|
||||
>>> repo = Repo("/path/to/repo")
|
||||
>>> head = repo.heads[0]
|
||||
|
||||
>>> head.name
|
||||
'master'
|
||||
|
||||
>>> head.commit
|
||||
<git.Commit "1c09f116cbc2cb4100fb6935bb162daa4723f455">
|
||||
|
||||
>>> head.commit.hexsha
|
||||
'1c09f116cbc2cb4100fb6935bb162daa4723f455'"""
|
||||
|
||||
_common_path_default = "refs/heads"
|
||||
k_config_remote = "remote"
|
||||
k_config_remote_ref = "merge" # branch to merge from remote
|
||||
|
||||
@classmethod
|
||||
def delete(cls, repo: "Repo", *heads: "Union[Head, str]", force: bool = False, **kwargs: Any) -> None:
|
||||
"""Delete the given heads
|
||||
|
||||
:param force:
|
||||
If True, the heads will be deleted even if they are not yet merged into
|
||||
the main development stream.
|
||||
Default False"""
|
||||
flag = "-d"
|
||||
if force:
|
||||
flag = "-D"
|
||||
repo.git.branch(flag, *heads)
|
||||
|
||||
def set_tracking_branch(self, remote_reference: Union["RemoteReference", None]) -> "Head":
|
||||
"""
|
||||
Configure this branch to track the given remote reference. This will alter
|
||||
this branch's configuration accordingly.
|
||||
|
||||
:param remote_reference: The remote reference to track or None to untrack
|
||||
any references
|
||||
:return: self"""
|
||||
from .remote import RemoteReference
|
||||
|
||||
if remote_reference is not None and not isinstance(remote_reference, RemoteReference):
|
||||
raise ValueError("Incorrect parameter type: %r" % remote_reference)
|
||||
# END handle type
|
||||
|
||||
with self.config_writer() as writer:
|
||||
if remote_reference is None:
|
||||
writer.remove_option(self.k_config_remote)
|
||||
writer.remove_option(self.k_config_remote_ref)
|
||||
if len(writer.options()) == 0:
|
||||
writer.remove_section()
|
||||
else:
|
||||
writer.set_value(self.k_config_remote, remote_reference.remote_name)
|
||||
writer.set_value(
|
||||
self.k_config_remote_ref,
|
||||
Head.to_full_path(remote_reference.remote_head),
|
||||
)
|
||||
|
||||
return self
|
||||
|
||||
def tracking_branch(self) -> Union["RemoteReference", None]:
|
||||
"""
|
||||
:return: The remote_reference we are tracking, or None if we are
|
||||
not a tracking branch"""
|
||||
from .remote import RemoteReference
|
||||
|
||||
reader = self.config_reader()
|
||||
if reader.has_option(self.k_config_remote) and reader.has_option(self.k_config_remote_ref):
|
||||
ref = Head(
|
||||
self.repo,
|
||||
Head.to_full_path(strip_quotes(reader.get_value(self.k_config_remote_ref))),
|
||||
)
|
||||
remote_refpath = RemoteReference.to_full_path(join_path(reader.get_value(self.k_config_remote), ref.name))
|
||||
return RemoteReference(self.repo, remote_refpath)
|
||||
# END handle have tracking branch
|
||||
|
||||
# we are not a tracking branch
|
||||
return None
|
||||
|
||||
def rename(self, new_path: PathLike, force: bool = False) -> "Head":
|
||||
"""Rename self to a new path
|
||||
|
||||
:param new_path:
|
||||
Either a simple name or a path, i.e. new_name or features/new_name.
|
||||
The prefix refs/heads is implied
|
||||
|
||||
:param force:
|
||||
If True, the rename will succeed even if a head with the target name
|
||||
already exists.
|
||||
|
||||
:return: self
|
||||
:note: respects the ref log as git commands are used"""
|
||||
flag = "-m"
|
||||
if force:
|
||||
flag = "-M"
|
||||
|
||||
self.repo.git.branch(flag, self, new_path)
|
||||
self.path = "%s/%s" % (self._common_path_default, new_path)
|
||||
return self
|
||||
|
||||
def checkout(self, force: bool = False, **kwargs: Any) -> Union["HEAD", "Head"]:
|
||||
"""Checkout this head by setting the HEAD to this reference, by updating the index
|
||||
to reflect the tree we point to and by updating the working tree to reflect
|
||||
the latest index.
|
||||
|
||||
The command will fail if changed working tree files would be overwritten.
|
||||
|
||||
:param force:
|
||||
If True, changes to the index and the working tree will be discarded.
|
||||
If False, GitCommandError will be raised in that situation.
|
||||
|
||||
:param kwargs:
|
||||
Additional keyword arguments to be passed to git checkout, i.e.
|
||||
b='new_branch' to create a new branch at the given spot.
|
||||
|
||||
:return:
|
||||
The active branch after the checkout operation, usually self unless
|
||||
a new branch has been created.
|
||||
If there is no active branch, as the HEAD is now detached, the HEAD
|
||||
reference will be returned instead.
|
||||
|
||||
:note:
|
||||
By default it is only allowed to checkout heads - everything else
|
||||
will leave the HEAD detached which is allowed and possible, but remains
|
||||
a special state that some tools might not be able to handle."""
|
||||
kwargs["f"] = force
|
||||
if kwargs["f"] is False:
|
||||
kwargs.pop("f")
|
||||
|
||||
self.repo.git.checkout(self, **kwargs)
|
||||
if self.repo.head.is_detached:
|
||||
return self.repo.head
|
||||
else:
|
||||
return self.repo.active_branch
|
||||
|
||||
# { Configuration
|
||||
def _config_parser(self, read_only: bool) -> SectionConstraint[GitConfigParser]:
|
||||
if read_only:
|
||||
parser = self.repo.config_reader()
|
||||
else:
|
||||
parser = self.repo.config_writer()
|
||||
# END handle parser instance
|
||||
|
||||
return SectionConstraint(parser, 'branch "%s"' % self.name)
|
||||
|
||||
def config_reader(self) -> SectionConstraint[GitConfigParser]:
|
||||
"""
|
||||
:return: A configuration parser instance constrained to only read
|
||||
this instance's values"""
|
||||
return self._config_parser(read_only=True)
|
||||
|
||||
def config_writer(self) -> SectionConstraint[GitConfigParser]:
|
||||
"""
|
||||
:return: A configuration writer instance with read-and write access
|
||||
to options of this head"""
|
||||
return self._config_parser(read_only=False)
|
||||
|
||||
# } END configuration
|
||||
353
zero-cost-nas/.eggs/GitPython-3.1.31-py3.8.egg/git/refs/log.py
Normal file
353
zero-cost-nas/.eggs/GitPython-3.1.31-py3.8.egg/git/refs/log.py
Normal file
@@ -0,0 +1,353 @@
|
||||
from mmap import mmap
|
||||
import re
|
||||
import time as _time
|
||||
|
||||
from git.compat import defenc
|
||||
from git.objects.util import (
|
||||
parse_date,
|
||||
Serializable,
|
||||
altz_to_utctz_str,
|
||||
)
|
||||
from git.util import (
|
||||
Actor,
|
||||
LockedFD,
|
||||
LockFile,
|
||||
assure_directory_exists,
|
||||
to_native_path,
|
||||
bin_to_hex,
|
||||
file_contents_ro_filepath,
|
||||
)
|
||||
|
||||
import os.path as osp
|
||||
|
||||
|
||||
# typing ------------------------------------------------------------------
|
||||
|
||||
from typing import Iterator, List, Tuple, Union, TYPE_CHECKING
|
||||
|
||||
from git.types import PathLike
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from git.refs import SymbolicReference
|
||||
from io import BytesIO
|
||||
from git.config import GitConfigParser, SectionConstraint # NOQA
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
|
||||
__all__ = ["RefLog", "RefLogEntry"]
|
||||
|
||||
|
||||
class RefLogEntry(Tuple[str, str, Actor, Tuple[int, int], str]):
|
||||
|
||||
"""Named tuple allowing easy access to the revlog data fields"""
|
||||
|
||||
_re_hexsha_only = re.compile("^[0-9A-Fa-f]{40}$")
|
||||
__slots__ = ()
|
||||
|
||||
def __repr__(self) -> str:
|
||||
"""Representation of ourselves in git reflog format"""
|
||||
return self.format()
|
||||
|
||||
def format(self) -> str:
|
||||
""":return: a string suitable to be placed in a reflog file"""
|
||||
act = self.actor
|
||||
time = self.time
|
||||
return "{} {} {} <{}> {!s} {}\t{}\n".format(
|
||||
self.oldhexsha,
|
||||
self.newhexsha,
|
||||
act.name,
|
||||
act.email,
|
||||
time[0],
|
||||
altz_to_utctz_str(time[1]),
|
||||
self.message,
|
||||
)
|
||||
|
||||
@property
|
||||
def oldhexsha(self) -> str:
|
||||
"""The hexsha to the commit the ref pointed to before the change"""
|
||||
return self[0]
|
||||
|
||||
@property
|
||||
def newhexsha(self) -> str:
|
||||
"""The hexsha to the commit the ref now points to, after the change"""
|
||||
return self[1]
|
||||
|
||||
@property
|
||||
def actor(self) -> Actor:
|
||||
"""Actor instance, providing access"""
|
||||
return self[2]
|
||||
|
||||
@property
|
||||
def time(self) -> Tuple[int, int]:
|
||||
"""time as tuple:
|
||||
|
||||
* [0] = int(time)
|
||||
* [1] = int(timezone_offset) in time.altzone format"""
|
||||
return self[3]
|
||||
|
||||
@property
|
||||
def message(self) -> str:
|
||||
"""Message describing the operation that acted on the reference"""
|
||||
return self[4]
|
||||
|
||||
@classmethod
|
||||
def new(
|
||||
cls,
|
||||
oldhexsha: str,
|
||||
newhexsha: str,
|
||||
actor: Actor,
|
||||
time: int,
|
||||
tz_offset: int,
|
||||
message: str,
|
||||
) -> "RefLogEntry": # skipcq: PYL-W0621
|
||||
""":return: New instance of a RefLogEntry"""
|
||||
if not isinstance(actor, Actor):
|
||||
raise ValueError("Need actor instance, got %s" % actor)
|
||||
# END check types
|
||||
return RefLogEntry((oldhexsha, newhexsha, actor, (time, tz_offset), message))
|
||||
|
||||
@classmethod
|
||||
def from_line(cls, line: bytes) -> "RefLogEntry":
|
||||
""":return: New RefLogEntry instance from the given revlog line.
|
||||
:param line: line bytes without trailing newline
|
||||
:raise ValueError: If line could not be parsed"""
|
||||
line_str = line.decode(defenc)
|
||||
fields = line_str.split("\t", 1)
|
||||
if len(fields) == 1:
|
||||
info, msg = fields[0], None
|
||||
elif len(fields) == 2:
|
||||
info, msg = fields
|
||||
else:
|
||||
raise ValueError("Line must have up to two TAB-separated fields." " Got %s" % repr(line_str))
|
||||
# END handle first split
|
||||
|
||||
oldhexsha = info[:40]
|
||||
newhexsha = info[41:81]
|
||||
for hexsha in (oldhexsha, newhexsha):
|
||||
if not cls._re_hexsha_only.match(hexsha):
|
||||
raise ValueError("Invalid hexsha: %r" % (hexsha,))
|
||||
# END if hexsha re doesn't match
|
||||
# END for each hexsha
|
||||
|
||||
email_end = info.find(">", 82)
|
||||
if email_end == -1:
|
||||
raise ValueError("Missing token: >")
|
||||
# END handle missing end brace
|
||||
|
||||
actor = Actor._from_string(info[82 : email_end + 1])
|
||||
time, tz_offset = parse_date(info[email_end + 2 :]) # skipcq: PYL-W0621
|
||||
|
||||
return RefLogEntry((oldhexsha, newhexsha, actor, (time, tz_offset), msg))
|
||||
|
||||
|
||||
class RefLog(List[RefLogEntry], Serializable):
|
||||
|
||||
"""A reflog contains RefLogEntrys, each of which defines a certain state
|
||||
of the head in question. Custom query methods allow to retrieve log entries
|
||||
by date or by other criteria.
|
||||
|
||||
Reflog entries are ordered, the first added entry is first in the list, the last
|
||||
entry, i.e. the last change of the head or reference, is last in the list."""
|
||||
|
||||
__slots__ = ("_path",)
|
||||
|
||||
def __new__(cls, filepath: Union[PathLike, None] = None) -> "RefLog":
|
||||
inst = super(RefLog, cls).__new__(cls)
|
||||
return inst
|
||||
|
||||
def __init__(self, filepath: Union[PathLike, None] = None):
|
||||
"""Initialize this instance with an optional filepath, from which we will
|
||||
initialize our data. The path is also used to write changes back using
|
||||
the write() method"""
|
||||
self._path = filepath
|
||||
if filepath is not None:
|
||||
self._read_from_file()
|
||||
# END handle filepath
|
||||
|
||||
def _read_from_file(self) -> None:
|
||||
try:
|
||||
fmap = file_contents_ro_filepath(self._path, stream=True, allow_mmap=True)
|
||||
except OSError:
|
||||
# it is possible and allowed that the file doesn't exist !
|
||||
return
|
||||
# END handle invalid log
|
||||
|
||||
try:
|
||||
self._deserialize(fmap)
|
||||
finally:
|
||||
fmap.close()
|
||||
# END handle closing of handle
|
||||
|
||||
# { Interface
|
||||
|
||||
@classmethod
|
||||
def from_file(cls, filepath: PathLike) -> "RefLog":
|
||||
"""
|
||||
:return: a new RefLog instance containing all entries from the reflog
|
||||
at the given filepath
|
||||
:param filepath: path to reflog
|
||||
:raise ValueError: If the file could not be read or was corrupted in some way"""
|
||||
return cls(filepath)
|
||||
|
||||
@classmethod
|
||||
def path(cls, ref: "SymbolicReference") -> str:
|
||||
"""
|
||||
:return: string to absolute path at which the reflog of the given ref
|
||||
instance would be found. The path is not guaranteed to point to a valid
|
||||
file though.
|
||||
:param ref: SymbolicReference instance"""
|
||||
return osp.join(ref.repo.git_dir, "logs", to_native_path(ref.path))
|
||||
|
||||
@classmethod
|
||||
def iter_entries(cls, stream: Union[str, "BytesIO", mmap]) -> Iterator[RefLogEntry]:
|
||||
"""
|
||||
:return: Iterator yielding RefLogEntry instances, one for each line read
|
||||
sfrom the given stream.
|
||||
:param stream: file-like object containing the revlog in its native format
|
||||
or string instance pointing to a file to read"""
|
||||
new_entry = RefLogEntry.from_line
|
||||
if isinstance(stream, str):
|
||||
# default args return mmap on py>3
|
||||
_stream = file_contents_ro_filepath(stream)
|
||||
assert isinstance(_stream, mmap)
|
||||
else:
|
||||
_stream = stream
|
||||
# END handle stream type
|
||||
while True:
|
||||
line = _stream.readline()
|
||||
if not line:
|
||||
return
|
||||
yield new_entry(line.strip())
|
||||
# END endless loop
|
||||
|
||||
@classmethod
|
||||
def entry_at(cls, filepath: PathLike, index: int) -> "RefLogEntry":
|
||||
"""
|
||||
:return: RefLogEntry at the given index
|
||||
|
||||
:param filepath: full path to the index file from which to read the entry
|
||||
|
||||
:param index: python list compatible index, i.e. it may be negative to
|
||||
specify an entry counted from the end of the list
|
||||
|
||||
:raise IndexError: If the entry didn't exist
|
||||
|
||||
.. note:: This method is faster as it only parses the entry at index, skipping
|
||||
all other lines. Nonetheless, the whole file has to be read if
|
||||
the index is negative
|
||||
"""
|
||||
with open(filepath, "rb") as fp:
|
||||
if index < 0:
|
||||
return RefLogEntry.from_line(fp.readlines()[index].strip())
|
||||
# read until index is reached
|
||||
|
||||
for i in range(index + 1):
|
||||
line = fp.readline()
|
||||
if not line:
|
||||
raise IndexError(f"Index file ended at line {i+1}, before given index was reached")
|
||||
# END abort on eof
|
||||
# END handle runup
|
||||
|
||||
return RefLogEntry.from_line(line.strip())
|
||||
# END handle index
|
||||
|
||||
def to_file(self, filepath: PathLike) -> None:
|
||||
"""Write the contents of the reflog instance to a file at the given filepath.
|
||||
|
||||
:param filepath: path to file, parent directories are assumed to exist"""
|
||||
lfd = LockedFD(filepath)
|
||||
assure_directory_exists(filepath, is_file=True)
|
||||
|
||||
fp = lfd.open(write=True, stream=True)
|
||||
try:
|
||||
self._serialize(fp)
|
||||
lfd.commit()
|
||||
except Exception:
|
||||
# on failure it rolls back automatically, but we make it clear
|
||||
lfd.rollback()
|
||||
raise
|
||||
# END handle change
|
||||
|
||||
@classmethod
|
||||
def append_entry(
|
||||
cls,
|
||||
config_reader: Union[Actor, "GitConfigParser", "SectionConstraint", None],
|
||||
filepath: PathLike,
|
||||
oldbinsha: bytes,
|
||||
newbinsha: bytes,
|
||||
message: str,
|
||||
write: bool = True,
|
||||
) -> "RefLogEntry":
|
||||
"""Append a new log entry to the revlog at filepath.
|
||||
|
||||
:param config_reader: configuration reader of the repository - used to obtain
|
||||
user information. May also be an Actor instance identifying the committer directly or None.
|
||||
:param filepath: full path to the log file
|
||||
:param oldbinsha: binary sha of the previous commit
|
||||
:param newbinsha: binary sha of the current commit
|
||||
:param message: message describing the change to the reference
|
||||
:param write: If True, the changes will be written right away. Otherwise
|
||||
the change will not be written
|
||||
|
||||
:return: RefLogEntry objects which was appended to the log
|
||||
|
||||
:note: As we are append-only, concurrent access is not a problem as we
|
||||
do not interfere with readers."""
|
||||
|
||||
if len(oldbinsha) != 20 or len(newbinsha) != 20:
|
||||
raise ValueError("Shas need to be given in binary format")
|
||||
# END handle sha type
|
||||
assure_directory_exists(filepath, is_file=True)
|
||||
first_line = message.split("\n")[0]
|
||||
if isinstance(config_reader, Actor):
|
||||
committer = config_reader # mypy thinks this is Actor | Gitconfigparser, but why?
|
||||
else:
|
||||
committer = Actor.committer(config_reader)
|
||||
entry = RefLogEntry(
|
||||
(
|
||||
bin_to_hex(oldbinsha).decode("ascii"),
|
||||
bin_to_hex(newbinsha).decode("ascii"),
|
||||
committer,
|
||||
(int(_time.time()), _time.altzone),
|
||||
first_line,
|
||||
)
|
||||
)
|
||||
|
||||
if write:
|
||||
lf = LockFile(filepath)
|
||||
lf._obtain_lock_or_raise()
|
||||
fd = open(filepath, "ab")
|
||||
try:
|
||||
fd.write(entry.format().encode(defenc))
|
||||
finally:
|
||||
fd.close()
|
||||
lf._release_lock()
|
||||
# END handle write operation
|
||||
return entry
|
||||
|
||||
def write(self) -> "RefLog":
|
||||
"""Write this instance's data to the file we are originating from
|
||||
|
||||
:return: self"""
|
||||
if self._path is None:
|
||||
raise ValueError("Instance was not initialized with a path, use to_file(...) instead")
|
||||
# END assert path
|
||||
self.to_file(self._path)
|
||||
return self
|
||||
|
||||
# } END interface
|
||||
|
||||
# { Serializable Interface
|
||||
def _serialize(self, stream: "BytesIO") -> "RefLog":
|
||||
write = stream.write
|
||||
|
||||
# write all entries
|
||||
for e in self:
|
||||
write(e.format().encode(defenc))
|
||||
# END for each entry
|
||||
return self
|
||||
|
||||
def _deserialize(self, stream: "BytesIO") -> "RefLog":
|
||||
self.extend(self.iter_entries(stream))
|
||||
# } END serializable interface
|
||||
return self
|
||||
@@ -0,0 +1,154 @@
|
||||
from git.util import (
|
||||
LazyMixin,
|
||||
IterableObj,
|
||||
)
|
||||
from .symbolic import SymbolicReference, T_References
|
||||
|
||||
|
||||
# typing ------------------------------------------------------------------
|
||||
|
||||
from typing import Any, Callable, Iterator, Type, Union, TYPE_CHECKING # NOQA
|
||||
from git.types import Commit_ish, PathLike, _T # NOQA
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from git.repo import Repo
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
|
||||
|
||||
__all__ = ["Reference"]
|
||||
|
||||
# { Utilities
|
||||
|
||||
|
||||
def require_remote_ref_path(func: Callable[..., _T]) -> Callable[..., _T]:
|
||||
"""A decorator raising a TypeError if we are not a valid remote, based on the path"""
|
||||
|
||||
def wrapper(self: T_References, *args: Any) -> _T:
|
||||
if not self.is_remote():
|
||||
raise ValueError("ref path does not point to a remote reference: %s" % self.path)
|
||||
return func(self, *args)
|
||||
|
||||
# END wrapper
|
||||
wrapper.__name__ = func.__name__
|
||||
return wrapper
|
||||
|
||||
|
||||
# }END utilities
|
||||
|
||||
|
||||
class Reference(SymbolicReference, LazyMixin, IterableObj):
|
||||
|
||||
"""Represents a named reference to any object. Subclasses may apply restrictions though,
|
||||
i.e. Heads can only point to commits."""
|
||||
|
||||
__slots__ = ()
|
||||
_points_to_commits_only = False
|
||||
_resolve_ref_on_create = True
|
||||
_common_path_default = "refs"
|
||||
|
||||
def __init__(self, repo: "Repo", path: PathLike, check_path: bool = True) -> None:
|
||||
"""Initialize this instance
|
||||
|
||||
:param repo: Our parent repository
|
||||
:param path:
|
||||
Path relative to the .git/ directory pointing to the ref in question, i.e.
|
||||
refs/heads/master
|
||||
:param check_path: if False, you can provide any path. Otherwise the path must start with the
|
||||
default path prefix of this type."""
|
||||
if check_path and not str(path).startswith(self._common_path_default + "/"):
|
||||
raise ValueError(f"Cannot instantiate {self.__class__.__name__!r} from path {path}")
|
||||
self.path: str # SymbolicReference converts to string atm
|
||||
super(Reference, self).__init__(repo, path)
|
||||
|
||||
def __str__(self) -> str:
|
||||
return self.name
|
||||
|
||||
# { Interface
|
||||
|
||||
# @ReservedAssignment
|
||||
def set_object(
|
||||
self,
|
||||
object: Union[Commit_ish, "SymbolicReference", str],
|
||||
logmsg: Union[str, None] = None,
|
||||
) -> "Reference":
|
||||
"""Special version which checks if the head-log needs an update as well
|
||||
|
||||
:return: self"""
|
||||
oldbinsha = None
|
||||
if logmsg is not None:
|
||||
head = self.repo.head
|
||||
if not head.is_detached and head.ref == self:
|
||||
oldbinsha = self.commit.binsha
|
||||
# END handle commit retrieval
|
||||
# END handle message is set
|
||||
|
||||
super(Reference, self).set_object(object, logmsg)
|
||||
|
||||
if oldbinsha is not None:
|
||||
# /* from refs.c in git-source
|
||||
# * Special hack: If a branch is updated directly and HEAD
|
||||
# * points to it (may happen on the remote side of a push
|
||||
# * for example) then logically the HEAD reflog should be
|
||||
# * updated too.
|
||||
# * A generic solution implies reverse symref information,
|
||||
# * but finding all symrefs pointing to the given branch
|
||||
# * would be rather costly for this rare event (the direct
|
||||
# * update of a branch) to be worth it. So let's cheat and
|
||||
# * check with HEAD only which should cover 99% of all usage
|
||||
# * scenarios (even 100% of the default ones).
|
||||
# */
|
||||
self.repo.head.log_append(oldbinsha, logmsg)
|
||||
# END check if the head
|
||||
|
||||
return self
|
||||
|
||||
# NOTE: Don't have to overwrite properties as the will only work without a the log
|
||||
|
||||
@property
|
||||
def name(self) -> str:
|
||||
""":return: (shortest) Name of this reference - it may contain path components"""
|
||||
# first two path tokens are can be removed as they are
|
||||
# refs/heads or refs/tags or refs/remotes
|
||||
tokens = self.path.split("/")
|
||||
if len(tokens) < 3:
|
||||
return self.path # could be refs/HEAD
|
||||
return "/".join(tokens[2:])
|
||||
|
||||
@classmethod
|
||||
def iter_items(
|
||||
cls: Type[T_References],
|
||||
repo: "Repo",
|
||||
common_path: Union[PathLike, None] = None,
|
||||
*args: Any,
|
||||
**kwargs: Any,
|
||||
) -> Iterator[T_References]:
|
||||
"""Equivalent to SymbolicReference.iter_items, but will return non-detached
|
||||
references as well."""
|
||||
return cls._iter_items(repo, common_path)
|
||||
|
||||
# }END interface
|
||||
|
||||
# { Remote Interface
|
||||
|
||||
@property # type: ignore ## mypy cannot deal with properties with an extra decorator (2021-04-21)
|
||||
@require_remote_ref_path
|
||||
def remote_name(self) -> str:
|
||||
"""
|
||||
:return:
|
||||
Name of the remote we are a reference of, such as 'origin' for a reference
|
||||
named 'origin/master'"""
|
||||
tokens = self.path.split("/")
|
||||
# /refs/remotes/<remote name>/<branch_name>
|
||||
return tokens[2]
|
||||
|
||||
@property # type: ignore ## mypy cannot deal with properties with an extra decorator (2021-04-21)
|
||||
@require_remote_ref_path
|
||||
def remote_head(self) -> str:
|
||||
""":return: Name of the remote head itself, i.e. master.
|
||||
:note: The returned name is usually not qualified enough to uniquely identify
|
||||
a branch"""
|
||||
tokens = self.path.split("/")
|
||||
return "/".join(tokens[3:])
|
||||
|
||||
# } END remote interface
|
||||
@@ -0,0 +1,75 @@
|
||||
import os
|
||||
|
||||
from git.util import join_path
|
||||
|
||||
from .head import Head
|
||||
|
||||
|
||||
__all__ = ["RemoteReference"]
|
||||
|
||||
# typing ------------------------------------------------------------------
|
||||
|
||||
from typing import Any, Iterator, NoReturn, Union, TYPE_CHECKING
|
||||
from git.types import PathLike
|
||||
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from git.repo import Repo
|
||||
from git import Remote
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
|
||||
|
||||
class RemoteReference(Head):
|
||||
|
||||
"""Represents a reference pointing to a remote head."""
|
||||
|
||||
_common_path_default = Head._remote_common_path_default
|
||||
|
||||
@classmethod
|
||||
def iter_items(
|
||||
cls,
|
||||
repo: "Repo",
|
||||
common_path: Union[PathLike, None] = None,
|
||||
remote: Union["Remote", None] = None,
|
||||
*args: Any,
|
||||
**kwargs: Any,
|
||||
) -> Iterator["RemoteReference"]:
|
||||
"""Iterate remote references, and if given, constrain them to the given remote"""
|
||||
common_path = common_path or cls._common_path_default
|
||||
if remote is not None:
|
||||
common_path = join_path(common_path, str(remote))
|
||||
# END handle remote constraint
|
||||
# super is Reference
|
||||
return super(RemoteReference, cls).iter_items(repo, common_path)
|
||||
|
||||
# The Head implementation of delete also accepts strs, but this
|
||||
# implementation does not. mypy doesn't have a way of representing
|
||||
# tightening the types of arguments in subclasses and recommends Any or
|
||||
# "type: ignore". (See https://github.com/python/typing/issues/241)
|
||||
@classmethod
|
||||
def delete(cls, repo: "Repo", *refs: "RemoteReference", **kwargs: Any) -> None: # type: ignore
|
||||
"""Delete the given remote references
|
||||
|
||||
:note:
|
||||
kwargs are given for comparability with the base class method as we
|
||||
should not narrow the signature."""
|
||||
repo.git.branch("-d", "-r", *refs)
|
||||
# the official deletion method will ignore remote symbolic refs - these
|
||||
# are generally ignored in the refs/ folder. We don't though
|
||||
# and delete remainders manually
|
||||
for ref in refs:
|
||||
try:
|
||||
os.remove(os.path.join(repo.common_dir, ref.path))
|
||||
except OSError:
|
||||
pass
|
||||
try:
|
||||
os.remove(os.path.join(repo.git_dir, ref.path))
|
||||
except OSError:
|
||||
pass
|
||||
# END for each ref
|
||||
|
||||
@classmethod
|
||||
def create(cls, *args: Any, **kwargs: Any) -> NoReturn:
|
||||
"""Used to disable this method"""
|
||||
raise TypeError("Cannot explicitly create remote references")
|
||||
@@ -0,0 +1,767 @@
|
||||
from git.types import PathLike
|
||||
import os
|
||||
|
||||
from git.compat import defenc
|
||||
from git.objects import Object
|
||||
from git.objects.commit import Commit
|
||||
from git.util import (
|
||||
join_path,
|
||||
join_path_native,
|
||||
to_native_path_linux,
|
||||
assure_directory_exists,
|
||||
hex_to_bin,
|
||||
LockedFD,
|
||||
)
|
||||
from gitdb.exc import BadObject, BadName
|
||||
|
||||
from .log import RefLog
|
||||
|
||||
# typing ------------------------------------------------------------------
|
||||
|
||||
from typing import (
|
||||
Any,
|
||||
Iterator,
|
||||
List,
|
||||
Tuple,
|
||||
Type,
|
||||
TypeVar,
|
||||
Union,
|
||||
TYPE_CHECKING,
|
||||
cast,
|
||||
) # NOQA
|
||||
from git.types import Commit_ish, PathLike # NOQA
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from git.repo import Repo
|
||||
from git.refs import Head, TagReference, RemoteReference, Reference
|
||||
from .log import RefLogEntry
|
||||
from git.config import GitConfigParser
|
||||
from git.objects.commit import Actor
|
||||
|
||||
|
||||
T_References = TypeVar("T_References", bound="SymbolicReference")
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
|
||||
|
||||
__all__ = ["SymbolicReference"]
|
||||
|
||||
|
||||
def _git_dir(repo: "Repo", path: Union[PathLike, None]) -> PathLike:
|
||||
"""Find the git dir that's appropriate for the path"""
|
||||
name = f"{path}"
|
||||
if name in ["HEAD", "ORIG_HEAD", "FETCH_HEAD", "index", "logs"]:
|
||||
return repo.git_dir
|
||||
return repo.common_dir
|
||||
|
||||
|
||||
class SymbolicReference(object):
|
||||
|
||||
"""Represents a special case of a reference such that this reference is symbolic.
|
||||
It does not point to a specific commit, but to another Head, which itself
|
||||
specifies a commit.
|
||||
|
||||
A typical example for a symbolic reference is HEAD."""
|
||||
|
||||
__slots__ = ("repo", "path")
|
||||
_resolve_ref_on_create = False
|
||||
_points_to_commits_only = True
|
||||
_common_path_default = ""
|
||||
_remote_common_path_default = "refs/remotes"
|
||||
_id_attribute_ = "name"
|
||||
|
||||
def __init__(self, repo: "Repo", path: PathLike, check_path: bool = False):
|
||||
self.repo = repo
|
||||
self.path = path
|
||||
|
||||
def __str__(self) -> str:
|
||||
return str(self.path)
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return '<git.%s "%s">' % (self.__class__.__name__, self.path)
|
||||
|
||||
def __eq__(self, other: object) -> bool:
|
||||
if hasattr(other, "path"):
|
||||
other = cast(SymbolicReference, other)
|
||||
return self.path == other.path
|
||||
return False
|
||||
|
||||
def __ne__(self, other: object) -> bool:
|
||||
return not (self == other)
|
||||
|
||||
def __hash__(self) -> int:
|
||||
return hash(self.path)
|
||||
|
||||
@property
|
||||
def name(self) -> str:
|
||||
"""
|
||||
:return:
|
||||
In case of symbolic references, the shortest assumable name
|
||||
is the path itself."""
|
||||
return str(self.path)
|
||||
|
||||
@property
|
||||
def abspath(self) -> PathLike:
|
||||
return join_path_native(_git_dir(self.repo, self.path), self.path)
|
||||
|
||||
@classmethod
|
||||
def _get_packed_refs_path(cls, repo: "Repo") -> str:
|
||||
return os.path.join(repo.common_dir, "packed-refs")
|
||||
|
||||
@classmethod
|
||||
def _iter_packed_refs(cls, repo: "Repo") -> Iterator[Tuple[str, str]]:
|
||||
"""Returns an iterator yielding pairs of sha1/path pairs (as strings) for the corresponding refs.
|
||||
:note: The packed refs file will be kept open as long as we iterate"""
|
||||
try:
|
||||
with open(cls._get_packed_refs_path(repo), "rt", encoding="UTF-8") as fp:
|
||||
for line in fp:
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
if line.startswith("#"):
|
||||
# "# pack-refs with: peeled fully-peeled sorted"
|
||||
# the git source code shows "peeled",
|
||||
# "fully-peeled" and "sorted" as the keywords
|
||||
# that can go on this line, as per comments in git file
|
||||
# refs/packed-backend.c
|
||||
# I looked at master on 2017-10-11,
|
||||
# commit 111ef79afe, after tag v2.15.0-rc1
|
||||
# from repo https://github.com/git/git.git
|
||||
if line.startswith("# pack-refs with:") and "peeled" not in line:
|
||||
raise TypeError("PackingType of packed-Refs not understood: %r" % line)
|
||||
# END abort if we do not understand the packing scheme
|
||||
continue
|
||||
# END parse comment
|
||||
|
||||
# skip dereferenced tag object entries - previous line was actual
|
||||
# tag reference for it
|
||||
if line[0] == "^":
|
||||
continue
|
||||
|
||||
yield cast(Tuple[str, str], tuple(line.split(" ", 1)))
|
||||
# END for each line
|
||||
except OSError:
|
||||
return None
|
||||
# END no packed-refs file handling
|
||||
# NOTE: Had try-finally block around here to close the fp,
|
||||
# but some python version wouldn't allow yields within that.
|
||||
# I believe files are closing themselves on destruction, so it is
|
||||
# alright.
|
||||
|
||||
@classmethod
|
||||
def dereference_recursive(cls, repo: "Repo", ref_path: Union[PathLike, None]) -> str:
|
||||
"""
|
||||
:return: hexsha stored in the reference at the given ref_path, recursively dereferencing all
|
||||
intermediate references as required
|
||||
:param repo: the repository containing the reference at ref_path"""
|
||||
|
||||
while True:
|
||||
hexsha, ref_path = cls._get_ref_info(repo, ref_path)
|
||||
if hexsha is not None:
|
||||
return hexsha
|
||||
# END recursive dereferencing
|
||||
|
||||
@classmethod
|
||||
def _get_ref_info_helper(
|
||||
cls, repo: "Repo", ref_path: Union[PathLike, None]
|
||||
) -> Union[Tuple[str, None], Tuple[None, str]]:
|
||||
"""Return: (str(sha), str(target_ref_path)) if available, the sha the file at
|
||||
rela_path points to, or None. target_ref_path is the reference we
|
||||
point to, or None"""
|
||||
tokens: Union[None, List[str], Tuple[str, str]] = None
|
||||
repodir = _git_dir(repo, ref_path)
|
||||
try:
|
||||
with open(os.path.join(repodir, str(ref_path)), "rt", encoding="UTF-8") as fp:
|
||||
value = fp.read().rstrip()
|
||||
# Don't only split on spaces, but on whitespace, which allows to parse lines like
|
||||
# 60b64ef992065e2600bfef6187a97f92398a9144 branch 'master' of git-server:/path/to/repo
|
||||
tokens = value.split()
|
||||
assert len(tokens) != 0
|
||||
except OSError:
|
||||
# Probably we are just packed, find our entry in the packed refs file
|
||||
# NOTE: We are not a symbolic ref if we are in a packed file, as these
|
||||
# are excluded explicitly
|
||||
for sha, path in cls._iter_packed_refs(repo):
|
||||
if path != ref_path:
|
||||
continue
|
||||
# sha will be used
|
||||
tokens = sha, path
|
||||
break
|
||||
# END for each packed ref
|
||||
# END handle packed refs
|
||||
if tokens is None:
|
||||
raise ValueError("Reference at %r does not exist" % ref_path)
|
||||
|
||||
# is it a reference ?
|
||||
if tokens[0] == "ref:":
|
||||
return (None, tokens[1])
|
||||
|
||||
# its a commit
|
||||
if repo.re_hexsha_only.match(tokens[0]):
|
||||
return (tokens[0], None)
|
||||
|
||||
raise ValueError("Failed to parse reference information from %r" % ref_path)
|
||||
|
||||
@classmethod
|
||||
def _get_ref_info(cls, repo: "Repo", ref_path: Union[PathLike, None]) -> Union[Tuple[str, None], Tuple[None, str]]:
|
||||
"""Return: (str(sha), str(target_ref_path)) if available, the sha the file at
|
||||
rela_path points to, or None. target_ref_path is the reference we
|
||||
point to, or None"""
|
||||
return cls._get_ref_info_helper(repo, ref_path)
|
||||
|
||||
def _get_object(self) -> Commit_ish:
|
||||
"""
|
||||
:return:
|
||||
The object our ref currently refers to. Refs can be cached, they will
|
||||
always point to the actual object as it gets re-created on each query"""
|
||||
# have to be dynamic here as we may be a tag which can point to anything
|
||||
# Our path will be resolved to the hexsha which will be used accordingly
|
||||
return Object.new_from_sha(self.repo, hex_to_bin(self.dereference_recursive(self.repo, self.path)))
|
||||
|
||||
def _get_commit(self) -> "Commit":
|
||||
"""
|
||||
:return:
|
||||
Commit object we point to, works for detached and non-detached
|
||||
SymbolicReferences. The symbolic reference will be dereferenced recursively."""
|
||||
obj = self._get_object()
|
||||
if obj.type == "tag":
|
||||
obj = obj.object
|
||||
# END dereference tag
|
||||
|
||||
if obj.type != Commit.type:
|
||||
raise TypeError("Symbolic Reference pointed to object %r, commit was required" % obj)
|
||||
# END handle type
|
||||
return obj
|
||||
|
||||
def set_commit(
|
||||
self,
|
||||
commit: Union[Commit, "SymbolicReference", str],
|
||||
logmsg: Union[str, None] = None,
|
||||
) -> "SymbolicReference":
|
||||
"""As set_object, but restricts the type of object to be a Commit
|
||||
|
||||
:raise ValueError: If commit is not a Commit object or doesn't point to
|
||||
a commit
|
||||
:return: self"""
|
||||
# check the type - assume the best if it is a base-string
|
||||
invalid_type = False
|
||||
if isinstance(commit, Object):
|
||||
invalid_type = commit.type != Commit.type
|
||||
elif isinstance(commit, SymbolicReference):
|
||||
invalid_type = commit.object.type != Commit.type
|
||||
else:
|
||||
try:
|
||||
invalid_type = self.repo.rev_parse(commit).type != Commit.type
|
||||
except (BadObject, BadName) as e:
|
||||
raise ValueError("Invalid object: %s" % commit) from e
|
||||
# END handle exception
|
||||
# END verify type
|
||||
|
||||
if invalid_type:
|
||||
raise ValueError("Need commit, got %r" % commit)
|
||||
# END handle raise
|
||||
|
||||
# we leave strings to the rev-parse method below
|
||||
self.set_object(commit, logmsg)
|
||||
|
||||
return self
|
||||
|
||||
def set_object(
|
||||
self,
|
||||
object: Union[Commit_ish, "SymbolicReference", str],
|
||||
logmsg: Union[str, None] = None,
|
||||
) -> "SymbolicReference":
|
||||
"""Set the object we point to, possibly dereference our symbolic reference first.
|
||||
If the reference does not exist, it will be created
|
||||
|
||||
:param object: a refspec, a SymbolicReference or an Object instance. SymbolicReferences
|
||||
will be dereferenced beforehand to obtain the object they point to
|
||||
:param logmsg: If not None, the message will be used in the reflog entry to be
|
||||
written. Otherwise the reflog is not altered
|
||||
:note: plain SymbolicReferences may not actually point to objects by convention
|
||||
:return: self"""
|
||||
if isinstance(object, SymbolicReference):
|
||||
object = object.object # @ReservedAssignment
|
||||
# END resolve references
|
||||
|
||||
is_detached = True
|
||||
try:
|
||||
is_detached = self.is_detached
|
||||
except ValueError:
|
||||
pass
|
||||
# END handle non-existing ones
|
||||
|
||||
if is_detached:
|
||||
return self.set_reference(object, logmsg)
|
||||
|
||||
# set the commit on our reference
|
||||
return self._get_reference().set_object(object, logmsg)
|
||||
|
||||
commit = property(_get_commit, set_commit, doc="Query or set commits directly") # type: ignore
|
||||
object = property(_get_object, set_object, doc="Return the object our ref currently refers to") # type: ignore
|
||||
|
||||
def _get_reference(self) -> "SymbolicReference":
|
||||
""":return: Reference Object we point to
|
||||
:raise TypeError: If this symbolic reference is detached, hence it doesn't point
|
||||
to a reference, but to a commit"""
|
||||
sha, target_ref_path = self._get_ref_info(self.repo, self.path)
|
||||
if target_ref_path is None:
|
||||
raise TypeError("%s is a detached symbolic reference as it points to %r" % (self, sha))
|
||||
return self.from_path(self.repo, target_ref_path)
|
||||
|
||||
def set_reference(
|
||||
self,
|
||||
ref: Union[Commit_ish, "SymbolicReference", str],
|
||||
logmsg: Union[str, None] = None,
|
||||
) -> "SymbolicReference":
|
||||
"""Set ourselves to the given ref. It will stay a symbol if the ref is a Reference.
|
||||
Otherwise an Object, given as Object instance or refspec, is assumed and if valid,
|
||||
will be set which effectively detaches the reference if it was a purely
|
||||
symbolic one.
|
||||
|
||||
:param ref: SymbolicReference instance, Object instance or refspec string
|
||||
Only if the ref is a SymbolicRef instance, we will point to it. Everything
|
||||
else is dereferenced to obtain the actual object.
|
||||
:param logmsg: If set to a string, the message will be used in the reflog.
|
||||
Otherwise, a reflog entry is not written for the changed reference.
|
||||
The previous commit of the entry will be the commit we point to now.
|
||||
|
||||
See also: log_append()
|
||||
|
||||
:return: self
|
||||
:note: This symbolic reference will not be dereferenced. For that, see
|
||||
``set_object(...)``"""
|
||||
write_value = None
|
||||
obj = None
|
||||
if isinstance(ref, SymbolicReference):
|
||||
write_value = "ref: %s" % ref.path
|
||||
elif isinstance(ref, Object):
|
||||
obj = ref
|
||||
write_value = ref.hexsha
|
||||
elif isinstance(ref, str):
|
||||
try:
|
||||
obj = self.repo.rev_parse(ref + "^{}") # optionally deref tags
|
||||
write_value = obj.hexsha
|
||||
except (BadObject, BadName) as e:
|
||||
raise ValueError("Could not extract object from %s" % ref) from e
|
||||
# END end try string
|
||||
else:
|
||||
raise ValueError("Unrecognized Value: %r" % ref)
|
||||
# END try commit attribute
|
||||
|
||||
# typecheck
|
||||
if obj is not None and self._points_to_commits_only and obj.type != Commit.type:
|
||||
raise TypeError("Require commit, got %r" % obj)
|
||||
# END verify type
|
||||
|
||||
oldbinsha: bytes = b""
|
||||
if logmsg is not None:
|
||||
try:
|
||||
oldbinsha = self.commit.binsha
|
||||
except ValueError:
|
||||
oldbinsha = Commit.NULL_BIN_SHA
|
||||
# END handle non-existing
|
||||
# END retrieve old hexsha
|
||||
|
||||
fpath = self.abspath
|
||||
assure_directory_exists(fpath, is_file=True)
|
||||
|
||||
lfd = LockedFD(fpath)
|
||||
fd = lfd.open(write=True, stream=True)
|
||||
ok = True
|
||||
try:
|
||||
fd.write(write_value.encode("utf-8") + b"\n")
|
||||
lfd.commit()
|
||||
ok = True
|
||||
finally:
|
||||
if not ok:
|
||||
lfd.rollback()
|
||||
# Adjust the reflog
|
||||
if logmsg is not None:
|
||||
self.log_append(oldbinsha, logmsg)
|
||||
|
||||
return self
|
||||
|
||||
# aliased reference
|
||||
reference: Union["Head", "TagReference", "RemoteReference", "Reference"]
|
||||
reference = property(_get_reference, set_reference, doc="Returns the Reference we point to") # type: ignore
|
||||
ref = reference
|
||||
|
||||
def is_valid(self) -> bool:
|
||||
"""
|
||||
:return:
|
||||
True if the reference is valid, hence it can be read and points to
|
||||
a valid object or reference."""
|
||||
try:
|
||||
self.object
|
||||
except (OSError, ValueError):
|
||||
return False
|
||||
else:
|
||||
return True
|
||||
|
||||
@property
|
||||
def is_detached(self) -> bool:
|
||||
"""
|
||||
:return:
|
||||
True if we are a detached reference, hence we point to a specific commit
|
||||
instead to another reference"""
|
||||
try:
|
||||
self.ref
|
||||
return False
|
||||
except TypeError:
|
||||
return True
|
||||
|
||||
def log(self) -> "RefLog":
|
||||
"""
|
||||
:return: RefLog for this reference. Its last entry reflects the latest change
|
||||
applied to this reference
|
||||
|
||||
.. note:: As the log is parsed every time, its recommended to cache it for use
|
||||
instead of calling this method repeatedly. It should be considered read-only."""
|
||||
return RefLog.from_file(RefLog.path(self))
|
||||
|
||||
def log_append(
|
||||
self,
|
||||
oldbinsha: bytes,
|
||||
message: Union[str, None],
|
||||
newbinsha: Union[bytes, None] = None,
|
||||
) -> "RefLogEntry":
|
||||
"""Append a logentry to the logfile of this ref
|
||||
|
||||
:param oldbinsha: binary sha this ref used to point to
|
||||
:param message: A message describing the change
|
||||
:param newbinsha: The sha the ref points to now. If None, our current commit sha
|
||||
will be used
|
||||
:return: added RefLogEntry instance"""
|
||||
# NOTE: we use the committer of the currently active commit - this should be
|
||||
# correct to allow overriding the committer on a per-commit level.
|
||||
# See https://github.com/gitpython-developers/GitPython/pull/146
|
||||
try:
|
||||
committer_or_reader: Union["Actor", "GitConfigParser"] = self.commit.committer
|
||||
except ValueError:
|
||||
committer_or_reader = self.repo.config_reader()
|
||||
# end handle newly cloned repositories
|
||||
if newbinsha is None:
|
||||
newbinsha = self.commit.binsha
|
||||
|
||||
if message is None:
|
||||
message = ""
|
||||
|
||||
return RefLog.append_entry(committer_or_reader, RefLog.path(self), oldbinsha, newbinsha, message)
|
||||
|
||||
def log_entry(self, index: int) -> "RefLogEntry":
|
||||
""":return: RefLogEntry at the given index
|
||||
:param index: python list compatible positive or negative index
|
||||
|
||||
.. note:: This method must read part of the reflog during execution, hence
|
||||
it should be used sparringly, or only if you need just one index.
|
||||
In that case, it will be faster than the ``log()`` method"""
|
||||
return RefLog.entry_at(RefLog.path(self), index)
|
||||
|
||||
@classmethod
|
||||
def to_full_path(cls, path: Union[PathLike, "SymbolicReference"]) -> PathLike:
|
||||
"""
|
||||
:return: string with a full repository-relative path which can be used to initialize
|
||||
a Reference instance, for instance by using ``Reference.from_path``"""
|
||||
if isinstance(path, SymbolicReference):
|
||||
path = path.path
|
||||
full_ref_path = path
|
||||
if not cls._common_path_default:
|
||||
return full_ref_path
|
||||
if not str(path).startswith(cls._common_path_default + "/"):
|
||||
full_ref_path = "%s/%s" % (cls._common_path_default, path)
|
||||
return full_ref_path
|
||||
|
||||
@classmethod
|
||||
def delete(cls, repo: "Repo", path: PathLike) -> None:
|
||||
"""Delete the reference at the given path
|
||||
|
||||
:param repo:
|
||||
Repository to delete the reference from
|
||||
|
||||
:param path:
|
||||
Short or full path pointing to the reference, i.e. refs/myreference
|
||||
or just "myreference", hence 'refs/' is implied.
|
||||
Alternatively the symbolic reference to be deleted"""
|
||||
full_ref_path = cls.to_full_path(path)
|
||||
abs_path = os.path.join(repo.common_dir, full_ref_path)
|
||||
if os.path.exists(abs_path):
|
||||
os.remove(abs_path)
|
||||
else:
|
||||
# check packed refs
|
||||
pack_file_path = cls._get_packed_refs_path(repo)
|
||||
try:
|
||||
with open(pack_file_path, "rb") as reader:
|
||||
new_lines = []
|
||||
made_change = False
|
||||
dropped_last_line = False
|
||||
for line_bytes in reader:
|
||||
line = line_bytes.decode(defenc)
|
||||
_, _, line_ref = line.partition(" ")
|
||||
line_ref = line_ref.strip()
|
||||
# keep line if it is a comment or if the ref to delete is not
|
||||
# in the line
|
||||
# If we deleted the last line and this one is a tag-reference object,
|
||||
# we drop it as well
|
||||
if (line.startswith("#") or full_ref_path != line_ref) and (
|
||||
not dropped_last_line or dropped_last_line and not line.startswith("^")
|
||||
):
|
||||
new_lines.append(line)
|
||||
dropped_last_line = False
|
||||
continue
|
||||
# END skip comments and lines without our path
|
||||
|
||||
# drop this line
|
||||
made_change = True
|
||||
dropped_last_line = True
|
||||
|
||||
# write the new lines
|
||||
if made_change:
|
||||
# write-binary is required, otherwise windows will
|
||||
# open the file in text mode and change LF to CRLF !
|
||||
with open(pack_file_path, "wb") as fd:
|
||||
fd.writelines(line.encode(defenc) for line in new_lines)
|
||||
|
||||
except OSError:
|
||||
pass # it didn't exist at all
|
||||
|
||||
# delete the reflog
|
||||
reflog_path = RefLog.path(cls(repo, full_ref_path))
|
||||
if os.path.isfile(reflog_path):
|
||||
os.remove(reflog_path)
|
||||
# END remove reflog
|
||||
|
||||
@classmethod
|
||||
def _create(
|
||||
cls: Type[T_References],
|
||||
repo: "Repo",
|
||||
path: PathLike,
|
||||
resolve: bool,
|
||||
reference: Union["SymbolicReference", str],
|
||||
force: bool,
|
||||
logmsg: Union[str, None] = None,
|
||||
) -> T_References:
|
||||
"""internal method used to create a new symbolic reference.
|
||||
If resolve is False, the reference will be taken as is, creating
|
||||
a proper symbolic reference. Otherwise it will be resolved to the
|
||||
corresponding object and a detached symbolic reference will be created
|
||||
instead"""
|
||||
git_dir = _git_dir(repo, path)
|
||||
full_ref_path = cls.to_full_path(path)
|
||||
abs_ref_path = os.path.join(git_dir, full_ref_path)
|
||||
|
||||
# figure out target data
|
||||
target = reference
|
||||
if resolve:
|
||||
target = repo.rev_parse(str(reference))
|
||||
|
||||
if not force and os.path.isfile(abs_ref_path):
|
||||
target_data = str(target)
|
||||
if isinstance(target, SymbolicReference):
|
||||
target_data = str(target.path)
|
||||
if not resolve:
|
||||
target_data = "ref: " + target_data
|
||||
with open(abs_ref_path, "rb") as fd:
|
||||
existing_data = fd.read().decode(defenc).strip()
|
||||
if existing_data != target_data:
|
||||
raise OSError(
|
||||
"Reference at %r does already exist, pointing to %r, requested was %r"
|
||||
% (full_ref_path, existing_data, target_data)
|
||||
)
|
||||
# END no force handling
|
||||
|
||||
ref = cls(repo, full_ref_path)
|
||||
ref.set_reference(target, logmsg)
|
||||
return ref
|
||||
|
||||
@classmethod
|
||||
def create(
|
||||
cls: Type[T_References],
|
||||
repo: "Repo",
|
||||
path: PathLike,
|
||||
reference: Union["SymbolicReference", str] = "HEAD",
|
||||
logmsg: Union[str, None] = None,
|
||||
force: bool = False,
|
||||
**kwargs: Any,
|
||||
) -> T_References:
|
||||
"""Create a new symbolic reference, hence a reference pointing , to another reference.
|
||||
|
||||
:param repo:
|
||||
Repository to create the reference in
|
||||
|
||||
:param path:
|
||||
full path at which the new symbolic reference is supposed to be
|
||||
created at, i.e. "NEW_HEAD" or "symrefs/my_new_symref"
|
||||
|
||||
:param reference:
|
||||
The reference to which the new symbolic reference should point to.
|
||||
If it is a commit'ish, the symbolic ref will be detached.
|
||||
|
||||
:param force:
|
||||
if True, force creation even if a symbolic reference with that name already exists.
|
||||
Raise OSError otherwise
|
||||
|
||||
:param logmsg:
|
||||
If not None, the message to append to the reflog. Otherwise no reflog
|
||||
entry is written.
|
||||
|
||||
:return: Newly created symbolic Reference
|
||||
|
||||
:raise OSError:
|
||||
If a (Symbolic)Reference with the same name but different contents
|
||||
already exists.
|
||||
|
||||
:note: This does not alter the current HEAD, index or Working Tree"""
|
||||
return cls._create(repo, path, cls._resolve_ref_on_create, reference, force, logmsg)
|
||||
|
||||
def rename(self, new_path: PathLike, force: bool = False) -> "SymbolicReference":
|
||||
"""Rename self to a new path
|
||||
|
||||
:param new_path:
|
||||
Either a simple name or a full path, i.e. new_name or features/new_name.
|
||||
The prefix refs/ is implied for references and will be set as needed.
|
||||
In case this is a symbolic ref, there is no implied prefix
|
||||
|
||||
:param force:
|
||||
If True, the rename will succeed even if a head with the target name
|
||||
already exists. It will be overwritten in that case
|
||||
|
||||
:return: self
|
||||
:raise OSError: In case a file at path but a different contents already exists"""
|
||||
new_path = self.to_full_path(new_path)
|
||||
if self.path == new_path:
|
||||
return self
|
||||
|
||||
new_abs_path = os.path.join(_git_dir(self.repo, new_path), new_path)
|
||||
cur_abs_path = os.path.join(_git_dir(self.repo, self.path), self.path)
|
||||
if os.path.isfile(new_abs_path):
|
||||
if not force:
|
||||
# if they point to the same file, its not an error
|
||||
with open(new_abs_path, "rb") as fd1:
|
||||
f1 = fd1.read().strip()
|
||||
with open(cur_abs_path, "rb") as fd2:
|
||||
f2 = fd2.read().strip()
|
||||
if f1 != f2:
|
||||
raise OSError("File at path %r already exists" % new_abs_path)
|
||||
# else: we could remove ourselves and use the otherone, but
|
||||
# but clarity we just continue as usual
|
||||
# END not force handling
|
||||
os.remove(new_abs_path)
|
||||
# END handle existing target file
|
||||
|
||||
dname = os.path.dirname(new_abs_path)
|
||||
if not os.path.isdir(dname):
|
||||
os.makedirs(dname)
|
||||
# END create directory
|
||||
|
||||
os.rename(cur_abs_path, new_abs_path)
|
||||
self.path = new_path
|
||||
|
||||
return self
|
||||
|
||||
@classmethod
|
||||
def _iter_items(
|
||||
cls: Type[T_References], repo: "Repo", common_path: Union[PathLike, None] = None
|
||||
) -> Iterator[T_References]:
|
||||
if common_path is None:
|
||||
common_path = cls._common_path_default
|
||||
rela_paths = set()
|
||||
|
||||
# walk loose refs
|
||||
# Currently we do not follow links
|
||||
for root, dirs, files in os.walk(join_path_native(repo.common_dir, common_path)):
|
||||
if "refs" not in root.split(os.sep): # skip non-refs subfolders
|
||||
refs_id = [d for d in dirs if d == "refs"]
|
||||
if refs_id:
|
||||
dirs[0:] = ["refs"]
|
||||
# END prune non-refs folders
|
||||
|
||||
for f in files:
|
||||
if f == "packed-refs":
|
||||
continue
|
||||
abs_path = to_native_path_linux(join_path(root, f))
|
||||
rela_paths.add(abs_path.replace(to_native_path_linux(repo.common_dir) + "/", ""))
|
||||
# END for each file in root directory
|
||||
# END for each directory to walk
|
||||
|
||||
# read packed refs
|
||||
for _sha, rela_path in cls._iter_packed_refs(repo):
|
||||
if rela_path.startswith(str(common_path)):
|
||||
rela_paths.add(rela_path)
|
||||
# END relative path matches common path
|
||||
# END packed refs reading
|
||||
|
||||
# return paths in sorted order
|
||||
for path in sorted(rela_paths):
|
||||
try:
|
||||
yield cls.from_path(repo, path)
|
||||
except ValueError:
|
||||
continue
|
||||
# END for each sorted relative refpath
|
||||
|
||||
@classmethod
|
||||
def iter_items(
|
||||
cls: Type[T_References],
|
||||
repo: "Repo",
|
||||
common_path: Union[PathLike, None] = None,
|
||||
*args: Any,
|
||||
**kwargs: Any,
|
||||
) -> Iterator[T_References]:
|
||||
"""Find all refs in the repository
|
||||
|
||||
:param repo: is the Repo
|
||||
|
||||
:param common_path:
|
||||
Optional keyword argument to the path which is to be shared by all
|
||||
returned Ref objects.
|
||||
Defaults to class specific portion if None assuring that only
|
||||
refs suitable for the actual class are returned.
|
||||
|
||||
:return:
|
||||
git.SymbolicReference[], each of them is guaranteed to be a symbolic
|
||||
ref which is not detached and pointing to a valid ref
|
||||
|
||||
List is lexicographically sorted
|
||||
The returned objects represent actual subclasses, such as Head or TagReference"""
|
||||
return (r for r in cls._iter_items(repo, common_path) if r.__class__ == SymbolicReference or not r.is_detached)
|
||||
|
||||
@classmethod
|
||||
def from_path(cls: Type[T_References], repo: "Repo", path: PathLike) -> T_References:
|
||||
"""
|
||||
:param path: full .git-directory-relative path name to the Reference to instantiate
|
||||
:note: use to_full_path() if you only have a partial path of a known Reference Type
|
||||
:return:
|
||||
Instance of type Reference, Head, or Tag
|
||||
depending on the given path"""
|
||||
if not path:
|
||||
raise ValueError("Cannot create Reference from %r" % path)
|
||||
|
||||
# Names like HEAD are inserted after the refs module is imported - we have an import dependency
|
||||
# cycle and don't want to import these names in-function
|
||||
from . import HEAD, Head, RemoteReference, TagReference, Reference
|
||||
|
||||
for ref_type in (
|
||||
HEAD,
|
||||
Head,
|
||||
RemoteReference,
|
||||
TagReference,
|
||||
Reference,
|
||||
SymbolicReference,
|
||||
):
|
||||
try:
|
||||
instance: T_References
|
||||
instance = ref_type(repo, path)
|
||||
if instance.__class__ == SymbolicReference and instance.is_detached:
|
||||
raise ValueError("SymbolRef was detached, we drop it")
|
||||
else:
|
||||
return instance
|
||||
|
||||
except ValueError:
|
||||
pass
|
||||
# END exception handling
|
||||
# END for each type to try
|
||||
raise ValueError("Could not find reference type suitable to handle path %r" % path)
|
||||
|
||||
def is_remote(self) -> bool:
|
||||
""":return: True if this symbolic reference points to a remote branch"""
|
||||
return str(self.path).startswith(self._remote_common_path_default + "/")
|
||||
138
zero-cost-nas/.eggs/GitPython-3.1.31-py3.8.egg/git/refs/tag.py
Normal file
138
zero-cost-nas/.eggs/GitPython-3.1.31-py3.8.egg/git/refs/tag.py
Normal file
@@ -0,0 +1,138 @@
|
||||
from .reference import Reference
|
||||
|
||||
__all__ = ["TagReference", "Tag"]
|
||||
|
||||
# typing ------------------------------------------------------------------
|
||||
|
||||
from typing import Any, Type, Union, TYPE_CHECKING
|
||||
from git.types import Commit_ish, PathLike
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from git.repo import Repo
|
||||
from git.objects import Commit
|
||||
from git.objects import TagObject
|
||||
from git.refs import SymbolicReference
|
||||
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TagReference(Reference):
|
||||
|
||||
"""Class representing a lightweight tag reference which either points to a commit
|
||||
,a tag object or any other object. In the latter case additional information,
|
||||
like the signature or the tag-creator, is available.
|
||||
|
||||
This tag object will always point to a commit object, but may carry additional
|
||||
information in a tag object::
|
||||
|
||||
tagref = TagReference.list_items(repo)[0]
|
||||
print(tagref.commit.message)
|
||||
if tagref.tag is not None:
|
||||
print(tagref.tag.message)"""
|
||||
|
||||
__slots__ = ()
|
||||
_common_default = "tags"
|
||||
_common_path_default = Reference._common_path_default + "/" + _common_default
|
||||
|
||||
@property
|
||||
def commit(self) -> "Commit": # type: ignore[override] # LazyMixin has unrelated commit method
|
||||
""":return: Commit object the tag ref points to
|
||||
|
||||
:raise ValueError: if the tag points to a tree or blob"""
|
||||
obj = self.object
|
||||
while obj.type != "commit":
|
||||
if obj.type == "tag":
|
||||
# it is a tag object which carries the commit as an object - we can point to anything
|
||||
obj = obj.object
|
||||
else:
|
||||
raise ValueError(
|
||||
(
|
||||
"Cannot resolve commit as tag %s points to a %s object - "
|
||||
+ "use the `.object` property instead to access it"
|
||||
)
|
||||
% (self, obj.type)
|
||||
)
|
||||
return obj
|
||||
|
||||
@property
|
||||
def tag(self) -> Union["TagObject", None]:
|
||||
"""
|
||||
:return: Tag object this tag ref points to or None in case
|
||||
we are a light weight tag"""
|
||||
obj = self.object
|
||||
if obj.type == "tag":
|
||||
return obj
|
||||
return None
|
||||
|
||||
# make object read-only
|
||||
# It should be reasonably hard to adjust an existing tag
|
||||
|
||||
# object = property(Reference._get_object)
|
||||
@property
|
||||
def object(self) -> Commit_ish: # type: ignore[override]
|
||||
return Reference._get_object(self)
|
||||
|
||||
@classmethod
|
||||
def create(
|
||||
cls: Type["TagReference"],
|
||||
repo: "Repo",
|
||||
path: PathLike,
|
||||
reference: Union[str, "SymbolicReference"] = "HEAD",
|
||||
logmsg: Union[str, None] = None,
|
||||
force: bool = False,
|
||||
**kwargs: Any,
|
||||
) -> "TagReference":
|
||||
"""Create a new tag reference.
|
||||
|
||||
:param path:
|
||||
The name of the tag, i.e. 1.0 or releases/1.0.
|
||||
The prefix refs/tags is implied
|
||||
|
||||
:param ref:
|
||||
A reference to the Object you want to tag. The Object can be a commit, tree or
|
||||
blob.
|
||||
|
||||
:param logmsg:
|
||||
If not None, the message will be used in your tag object. This will also
|
||||
create an additional tag object that allows to obtain that information, i.e.::
|
||||
|
||||
tagref.tag.message
|
||||
|
||||
:param message:
|
||||
Synonym for :param logmsg:
|
||||
Included for backwards compatibility. :param logmsg is used in preference if both given.
|
||||
|
||||
:param force:
|
||||
If True, to force creation of a tag even though that tag already exists.
|
||||
|
||||
:param kwargs:
|
||||
Additional keyword arguments to be passed to git-tag
|
||||
|
||||
:return: A new TagReference"""
|
||||
if "ref" in kwargs and kwargs["ref"]:
|
||||
reference = kwargs["ref"]
|
||||
|
||||
if "message" in kwargs and kwargs["message"]:
|
||||
kwargs["m"] = kwargs["message"]
|
||||
del kwargs["message"]
|
||||
|
||||
if logmsg:
|
||||
kwargs["m"] = logmsg
|
||||
|
||||
if force:
|
||||
kwargs["f"] = True
|
||||
|
||||
args = (path, reference)
|
||||
|
||||
repo.git.tag(*args, **kwargs)
|
||||
return TagReference(repo, "%s/%s" % (cls._common_path_default, path))
|
||||
|
||||
@classmethod
|
||||
def delete(cls, repo: "Repo", *tags: "TagReference") -> None: # type: ignore[override]
|
||||
"""Delete the given existing tag or tags"""
|
||||
repo.git.tag("-d", *tags)
|
||||
|
||||
|
||||
# provide an alias
|
||||
Tag = TagReference
|
||||
1150
zero-cost-nas/.eggs/GitPython-3.1.31-py3.8.egg/git/remote.py
Normal file
1150
zero-cost-nas/.eggs/GitPython-3.1.31-py3.8.egg/git/remote.py
Normal file
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,3 @@
|
||||
"""Initialize the Repo package"""
|
||||
# flake8: noqa
|
||||
from .base import Repo as Repo
|
||||
1402
zero-cost-nas/.eggs/GitPython-3.1.31-py3.8.egg/git/repo/base.py
Normal file
1402
zero-cost-nas/.eggs/GitPython-3.1.31-py3.8.egg/git/repo/base.py
Normal file
File diff suppressed because it is too large
Load Diff
388
zero-cost-nas/.eggs/GitPython-3.1.31-py3.8.egg/git/repo/fun.py
Normal file
388
zero-cost-nas/.eggs/GitPython-3.1.31-py3.8.egg/git/repo/fun.py
Normal file
@@ -0,0 +1,388 @@
|
||||
"""Package with general repository related functions"""
|
||||
from __future__ import annotations
|
||||
import os
|
||||
import stat
|
||||
from pathlib import Path
|
||||
from string import digits
|
||||
|
||||
from git.exc import WorkTreeRepositoryUnsupported
|
||||
from git.objects import Object
|
||||
from git.refs import SymbolicReference
|
||||
from git.util import hex_to_bin, bin_to_hex, cygpath
|
||||
from gitdb.exc import (
|
||||
BadObject,
|
||||
BadName,
|
||||
)
|
||||
|
||||
import os.path as osp
|
||||
from git.cmd import Git
|
||||
|
||||
# Typing ----------------------------------------------------------------------
|
||||
|
||||
from typing import Union, Optional, cast, TYPE_CHECKING
|
||||
from git.types import Commit_ish
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from git.types import PathLike
|
||||
from .base import Repo
|
||||
from git.db import GitCmdObjectDB
|
||||
from git.refs.reference import Reference
|
||||
from git.objects import Commit, TagObject, Blob, Tree
|
||||
from git.refs.tag import Tag
|
||||
|
||||
# ----------------------------------------------------------------------------
|
||||
|
||||
__all__ = (
|
||||
"rev_parse",
|
||||
"is_git_dir",
|
||||
"touch",
|
||||
"find_submodule_git_dir",
|
||||
"name_to_object",
|
||||
"short_to_long",
|
||||
"deref_tag",
|
||||
"to_commit",
|
||||
"find_worktree_git_dir",
|
||||
)
|
||||
|
||||
|
||||
def touch(filename: str) -> str:
|
||||
with open(filename, "ab"):
|
||||
pass
|
||||
return filename
|
||||
|
||||
|
||||
def is_git_dir(d: "PathLike") -> bool:
|
||||
"""This is taken from the git setup.c:is_git_directory
|
||||
function.
|
||||
|
||||
@throws WorkTreeRepositoryUnsupported if it sees a worktree directory. It's quite hacky to do that here,
|
||||
but at least clearly indicates that we don't support it.
|
||||
There is the unlikely danger to throw if we see directories which just look like a worktree dir,
|
||||
but are none."""
|
||||
if osp.isdir(d):
|
||||
if (osp.isdir(osp.join(d, "objects")) or "GIT_OBJECT_DIRECTORY" in os.environ) and osp.isdir(
|
||||
osp.join(d, "refs")
|
||||
):
|
||||
headref = osp.join(d, "HEAD")
|
||||
return osp.isfile(headref) or (osp.islink(headref) and os.readlink(headref).startswith("refs"))
|
||||
elif (
|
||||
osp.isfile(osp.join(d, "gitdir"))
|
||||
and osp.isfile(osp.join(d, "commondir"))
|
||||
and osp.isfile(osp.join(d, "gitfile"))
|
||||
):
|
||||
raise WorkTreeRepositoryUnsupported(d)
|
||||
return False
|
||||
|
||||
|
||||
def find_worktree_git_dir(dotgit: "PathLike") -> Optional[str]:
|
||||
"""Search for a gitdir for this worktree."""
|
||||
try:
|
||||
statbuf = os.stat(dotgit)
|
||||
except OSError:
|
||||
return None
|
||||
if not stat.S_ISREG(statbuf.st_mode):
|
||||
return None
|
||||
|
||||
try:
|
||||
lines = Path(dotgit).read_text().splitlines()
|
||||
for key, value in [line.strip().split(": ") for line in lines]:
|
||||
if key == "gitdir":
|
||||
return value
|
||||
except ValueError:
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
def find_submodule_git_dir(d: "PathLike") -> Optional["PathLike"]:
|
||||
"""Search for a submodule repo."""
|
||||
if is_git_dir(d):
|
||||
return d
|
||||
|
||||
try:
|
||||
with open(d) as fp:
|
||||
content = fp.read().rstrip()
|
||||
except IOError:
|
||||
# it's probably not a file
|
||||
pass
|
||||
else:
|
||||
if content.startswith("gitdir: "):
|
||||
path = content[8:]
|
||||
|
||||
if Git.is_cygwin():
|
||||
## Cygwin creates submodules prefixed with `/cygdrive/...` suffixes.
|
||||
# Cygwin git understands Cygwin paths much better than Windows ones
|
||||
# Also the Cygwin tests are assuming Cygwin paths.
|
||||
path = cygpath(path)
|
||||
if not osp.isabs(path):
|
||||
path = osp.normpath(osp.join(osp.dirname(d), path))
|
||||
return find_submodule_git_dir(path)
|
||||
# end handle exception
|
||||
return None
|
||||
|
||||
|
||||
def short_to_long(odb: "GitCmdObjectDB", hexsha: str) -> Optional[bytes]:
|
||||
""":return: long hexadecimal sha1 from the given less-than-40 byte hexsha
|
||||
or None if no candidate could be found.
|
||||
:param hexsha: hexsha with less than 40 byte"""
|
||||
try:
|
||||
return bin_to_hex(odb.partial_to_complete_sha_hex(hexsha))
|
||||
except BadObject:
|
||||
return None
|
||||
# END exception handling
|
||||
|
||||
|
||||
def name_to_object(
|
||||
repo: "Repo", name: str, return_ref: bool = False
|
||||
) -> Union[SymbolicReference, "Commit", "TagObject", "Blob", "Tree"]:
|
||||
"""
|
||||
:return: object specified by the given name, hexshas ( short and long )
|
||||
as well as references are supported
|
||||
:param return_ref: if name specifies a reference, we will return the reference
|
||||
instead of the object. Otherwise it will raise BadObject or BadName
|
||||
"""
|
||||
hexsha: Union[None, str, bytes] = None
|
||||
|
||||
# is it a hexsha ? Try the most common ones, which is 7 to 40
|
||||
if repo.re_hexsha_shortened.match(name):
|
||||
if len(name) != 40:
|
||||
# find long sha for short sha
|
||||
hexsha = short_to_long(repo.odb, name)
|
||||
else:
|
||||
hexsha = name
|
||||
# END handle short shas
|
||||
# END find sha if it matches
|
||||
|
||||
# if we couldn't find an object for what seemed to be a short hexsha
|
||||
# try to find it as reference anyway, it could be named 'aaa' for instance
|
||||
if hexsha is None:
|
||||
for base in (
|
||||
"%s",
|
||||
"refs/%s",
|
||||
"refs/tags/%s",
|
||||
"refs/heads/%s",
|
||||
"refs/remotes/%s",
|
||||
"refs/remotes/%s/HEAD",
|
||||
):
|
||||
try:
|
||||
hexsha = SymbolicReference.dereference_recursive(repo, base % name)
|
||||
if return_ref:
|
||||
return SymbolicReference(repo, base % name)
|
||||
# END handle symbolic ref
|
||||
break
|
||||
except ValueError:
|
||||
pass
|
||||
# END for each base
|
||||
# END handle hexsha
|
||||
|
||||
# didn't find any ref, this is an error
|
||||
if return_ref:
|
||||
raise BadObject("Couldn't find reference named %r" % name)
|
||||
# END handle return ref
|
||||
|
||||
# tried everything ? fail
|
||||
if hexsha is None:
|
||||
raise BadName(name)
|
||||
# END assert hexsha was found
|
||||
|
||||
return Object.new_from_sha(repo, hex_to_bin(hexsha))
|
||||
|
||||
|
||||
def deref_tag(tag: "Tag") -> "TagObject":
|
||||
"""Recursively dereference a tag and return the resulting object"""
|
||||
while True:
|
||||
try:
|
||||
tag = tag.object
|
||||
except AttributeError:
|
||||
break
|
||||
# END dereference tag
|
||||
return tag
|
||||
|
||||
|
||||
def to_commit(obj: Object) -> Union["Commit", "TagObject"]:
|
||||
"""Convert the given object to a commit if possible and return it"""
|
||||
if obj.type == "tag":
|
||||
obj = deref_tag(obj)
|
||||
|
||||
if obj.type != "commit":
|
||||
raise ValueError("Cannot convert object %r to type commit" % obj)
|
||||
# END verify type
|
||||
return obj
|
||||
|
||||
|
||||
def rev_parse(repo: "Repo", rev: str) -> Union["Commit", "Tag", "Tree", "Blob"]:
|
||||
"""
|
||||
:return: Object at the given revision, either Commit, Tag, Tree or Blob
|
||||
:param rev: git-rev-parse compatible revision specification as string, please see
|
||||
http://www.kernel.org/pub/software/scm/git/docs/git-rev-parse.html
|
||||
for details
|
||||
:raise BadObject: if the given revision could not be found
|
||||
:raise ValueError: If rev couldn't be parsed
|
||||
:raise IndexError: If invalid reflog index is specified"""
|
||||
|
||||
# colon search mode ?
|
||||
if rev.startswith(":/"):
|
||||
# colon search mode
|
||||
raise NotImplementedError("commit by message search ( regex )")
|
||||
# END handle search
|
||||
|
||||
obj: Union[Commit_ish, "Reference", None] = None
|
||||
ref = None
|
||||
output_type = "commit"
|
||||
start = 0
|
||||
parsed_to = 0
|
||||
lr = len(rev)
|
||||
while start < lr:
|
||||
if rev[start] not in "^~:@":
|
||||
start += 1
|
||||
continue
|
||||
# END handle start
|
||||
|
||||
token = rev[start]
|
||||
|
||||
if obj is None:
|
||||
# token is a rev name
|
||||
if start == 0:
|
||||
ref = repo.head.ref
|
||||
else:
|
||||
if token == "@":
|
||||
ref = cast("Reference", name_to_object(repo, rev[:start], return_ref=True))
|
||||
else:
|
||||
obj = cast(Commit_ish, name_to_object(repo, rev[:start]))
|
||||
# END handle token
|
||||
# END handle refname
|
||||
else:
|
||||
assert obj is not None
|
||||
|
||||
if ref is not None:
|
||||
obj = cast("Commit", ref.commit)
|
||||
# END handle ref
|
||||
# END initialize obj on first token
|
||||
|
||||
start += 1
|
||||
|
||||
# try to parse {type}
|
||||
if start < lr and rev[start] == "{":
|
||||
end = rev.find("}", start)
|
||||
if end == -1:
|
||||
raise ValueError("Missing closing brace to define type in %s" % rev)
|
||||
output_type = rev[start + 1 : end] # exclude brace
|
||||
|
||||
# handle type
|
||||
if output_type == "commit":
|
||||
pass # default
|
||||
elif output_type == "tree":
|
||||
try:
|
||||
obj = cast(Commit_ish, obj)
|
||||
obj = to_commit(obj).tree
|
||||
except (AttributeError, ValueError):
|
||||
pass # error raised later
|
||||
# END exception handling
|
||||
elif output_type in ("", "blob"):
|
||||
obj = cast("TagObject", obj)
|
||||
if obj and obj.type == "tag":
|
||||
obj = deref_tag(obj)
|
||||
else:
|
||||
# cannot do anything for non-tags
|
||||
pass
|
||||
# END handle tag
|
||||
elif token == "@":
|
||||
# try single int
|
||||
assert ref is not None, "Require Reference to access reflog"
|
||||
revlog_index = None
|
||||
try:
|
||||
# transform reversed index into the format of our revlog
|
||||
revlog_index = -(int(output_type) + 1)
|
||||
except ValueError as e:
|
||||
# TODO: Try to parse the other date options, using parse_date
|
||||
# maybe
|
||||
raise NotImplementedError("Support for additional @{...} modes not implemented") from e
|
||||
# END handle revlog index
|
||||
|
||||
try:
|
||||
entry = ref.log_entry(revlog_index)
|
||||
except IndexError as e:
|
||||
raise IndexError("Invalid revlog index: %i" % revlog_index) from e
|
||||
# END handle index out of bound
|
||||
|
||||
obj = Object.new_from_sha(repo, hex_to_bin(entry.newhexsha))
|
||||
|
||||
# make it pass the following checks
|
||||
output_type = ""
|
||||
else:
|
||||
raise ValueError("Invalid output type: %s ( in %s )" % (output_type, rev))
|
||||
# END handle output type
|
||||
|
||||
# empty output types don't require any specific type, its just about dereferencing tags
|
||||
if output_type and obj and obj.type != output_type:
|
||||
raise ValueError("Could not accommodate requested object type %r, got %s" % (output_type, obj.type))
|
||||
# END verify output type
|
||||
|
||||
start = end + 1 # skip brace
|
||||
parsed_to = start
|
||||
continue
|
||||
# END parse type
|
||||
|
||||
# try to parse a number
|
||||
num = 0
|
||||
if token != ":":
|
||||
found_digit = False
|
||||
while start < lr:
|
||||
if rev[start] in digits:
|
||||
num = num * 10 + int(rev[start])
|
||||
start += 1
|
||||
found_digit = True
|
||||
else:
|
||||
break
|
||||
# END handle number
|
||||
# END number parse loop
|
||||
|
||||
# no explicit number given, 1 is the default
|
||||
# It could be 0 though
|
||||
if not found_digit:
|
||||
num = 1
|
||||
# END set default num
|
||||
# END number parsing only if non-blob mode
|
||||
|
||||
parsed_to = start
|
||||
# handle hierarchy walk
|
||||
try:
|
||||
obj = cast(Commit_ish, obj)
|
||||
if token == "~":
|
||||
obj = to_commit(obj)
|
||||
for _ in range(num):
|
||||
obj = obj.parents[0]
|
||||
# END for each history item to walk
|
||||
elif token == "^":
|
||||
obj = to_commit(obj)
|
||||
# must be n'th parent
|
||||
if num:
|
||||
obj = obj.parents[num - 1]
|
||||
elif token == ":":
|
||||
if obj.type != "tree":
|
||||
obj = obj.tree
|
||||
# END get tree type
|
||||
obj = obj[rev[start:]]
|
||||
parsed_to = lr
|
||||
else:
|
||||
raise ValueError("Invalid token: %r" % token)
|
||||
# END end handle tag
|
||||
except (IndexError, AttributeError) as e:
|
||||
raise BadName(
|
||||
f"Invalid revision spec '{rev}' - not enough " f"parent commits to reach '{token}{int(num)}'"
|
||||
) from e
|
||||
# END exception handling
|
||||
# END parse loop
|
||||
|
||||
# still no obj ? Its probably a simple name
|
||||
if obj is None:
|
||||
obj = cast(Commit_ish, name_to_object(repo, rev))
|
||||
parsed_to = lr
|
||||
# END handle simple name
|
||||
|
||||
if obj is None:
|
||||
raise ValueError("Revision specifier could not be parsed: %s" % rev)
|
||||
|
||||
if parsed_to != lr:
|
||||
raise ValueError("Didn't consume complete rev spec %s, consumed part: %s" % (rev, rev[:parsed_to]))
|
||||
|
||||
return obj
|
||||
117
zero-cost-nas/.eggs/GitPython-3.1.31-py3.8.egg/git/types.py
Normal file
117
zero-cost-nas/.eggs/GitPython-3.1.31-py3.8.egg/git/types.py
Normal file
@@ -0,0 +1,117 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
# This module is part of GitPython and is released under
|
||||
# the BSD License: http://www.opensource.org/licenses/bsd-license.php
|
||||
# flake8: noqa
|
||||
|
||||
import os
|
||||
import sys
|
||||
from typing import (
|
||||
Dict,
|
||||
NoReturn,
|
||||
Sequence,
|
||||
Tuple,
|
||||
Union,
|
||||
Any,
|
||||
TYPE_CHECKING,
|
||||
TypeVar,
|
||||
) # noqa: F401
|
||||
|
||||
if sys.version_info[:2] >= (3, 8):
|
||||
from typing import (
|
||||
Literal,
|
||||
SupportsIndex,
|
||||
TypedDict,
|
||||
Protocol,
|
||||
runtime_checkable,
|
||||
) # noqa: F401
|
||||
else:
|
||||
from typing_extensions import (
|
||||
Literal,
|
||||
SupportsIndex, # noqa: F401
|
||||
TypedDict,
|
||||
Protocol,
|
||||
runtime_checkable,
|
||||
) # noqa: F401
|
||||
|
||||
# if sys.version_info[:2] >= (3, 10):
|
||||
# from typing import TypeGuard # noqa: F401
|
||||
# else:
|
||||
# from typing_extensions import TypeGuard # noqa: F401
|
||||
|
||||
|
||||
if sys.version_info[:2] < (3, 9):
|
||||
PathLike = Union[str, os.PathLike]
|
||||
else:
|
||||
# os.PathLike only becomes subscriptable from Python 3.9 onwards
|
||||
PathLike = Union[str, os.PathLike[str]]
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from git.repo import Repo
|
||||
from git.objects import Commit, Tree, TagObject, Blob
|
||||
|
||||
# from git.refs import SymbolicReference
|
||||
|
||||
TBD = Any
|
||||
_T = TypeVar("_T")
|
||||
|
||||
Tree_ish = Union["Commit", "Tree"]
|
||||
Commit_ish = Union["Commit", "TagObject", "Blob", "Tree"]
|
||||
Lit_commit_ish = Literal["commit", "tag", "blob", "tree"]
|
||||
|
||||
# Config_levels ---------------------------------------------------------
|
||||
|
||||
Lit_config_levels = Literal["system", "global", "user", "repository"]
|
||||
|
||||
|
||||
# def is_config_level(inp: str) -> TypeGuard[Lit_config_levels]:
|
||||
# # return inp in get_args(Lit_config_level) # only py >= 3.8
|
||||
# return inp in ("system", "user", "global", "repository")
|
||||
|
||||
|
||||
ConfigLevels_Tup = Tuple[Literal["system"], Literal["user"], Literal["global"], Literal["repository"]]
|
||||
|
||||
# -----------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def assert_never(inp: NoReturn, raise_error: bool = True, exc: Union[Exception, None] = None) -> None:
|
||||
"""For use in exhaustive checking of literal or Enum in if/else chain.
|
||||
Should only be reached if all members not handled OR attempt to pass non-members through chain.
|
||||
|
||||
If all members handled, type is Empty. Otherwise, will cause mypy error.
|
||||
If non-members given, should cause mypy error at variable creation.
|
||||
|
||||
If raise_error is True, will also raise AssertionError or the Exception passed to exc.
|
||||
"""
|
||||
if raise_error:
|
||||
if exc is None:
|
||||
raise ValueError(f"An unhandled Literal ({inp}) in an if/else chain was found")
|
||||
else:
|
||||
raise exc
|
||||
|
||||
|
||||
class Files_TD(TypedDict):
|
||||
insertions: int
|
||||
deletions: int
|
||||
lines: int
|
||||
|
||||
|
||||
class Total_TD(TypedDict):
|
||||
insertions: int
|
||||
deletions: int
|
||||
lines: int
|
||||
files: int
|
||||
|
||||
|
||||
class HSH_TD(TypedDict):
|
||||
total: Total_TD
|
||||
files: Dict[PathLike, Files_TD]
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class Has_Repo(Protocol):
|
||||
repo: "Repo"
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class Has_id_attribute(Protocol):
|
||||
_id_attribute_: str
|
||||
1206
zero-cost-nas/.eggs/GitPython-3.1.31-py3.8.egg/git/util.py
Normal file
1206
zero-cost-nas/.eggs/GitPython-3.1.31-py3.8.egg/git/util.py
Normal file
File diff suppressed because it is too large
Load Diff
6
zero-cost-nas/.eggs/README.txt
Normal file
6
zero-cost-nas/.eggs/README.txt
Normal file
@@ -0,0 +1,6 @@
|
||||
This directory contains eggs that were downloaded by setuptools to build, test, and run plug-ins.
|
||||
|
||||
This directory caches those eggs to prevent repeated downloads.
|
||||
|
||||
However, it is safe to delete this directory.
|
||||
|
||||
@@ -0,0 +1,4 @@
|
||||
Creator: Sebastian Thiel
|
||||
|
||||
Contributors:
|
||||
- Ram Rachum (@cool-RR)
|
||||
42
zero-cost-nas/.eggs/gitdb-4.0.10-py3.8.egg/EGG-INFO/LICENSE
Normal file
42
zero-cost-nas/.eggs/gitdb-4.0.10-py3.8.egg/EGG-INFO/LICENSE
Normal file
@@ -0,0 +1,42 @@
|
||||
Copyright (C) 2010, 2011 Sebastian Thiel and contributors
|
||||
All rights reserved.
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions
|
||||
are met:
|
||||
|
||||
* Redistributions of source code must retain the above copyright
|
||||
notice, this list of conditions and the following disclaimer.
|
||||
|
||||
* Redistributions in binary form must reproduce the above copyright
|
||||
notice, this list of conditions and the following disclaimer in the
|
||||
documentation and/or other materials provided with the distribution.
|
||||
|
||||
* Neither the name of the GitDB project nor the names of
|
||||
its contributors may be used to endorse or promote products derived
|
||||
from this software without specific prior written permission.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
|
||||
TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
|
||||
PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
|
||||
LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
|
||||
NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
|
||||
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
|
||||
Additional Licenses
|
||||
-------------------
|
||||
The files at
|
||||
gitdb/test/fixtures/packs/pack-11fdfa9e156ab73caae3b6da867192221f2089c2.idx
|
||||
and
|
||||
gitdb/test/fixtures/packs/pack-11fdfa9e156ab73caae3b6da867192221f2089c2.pack
|
||||
are licensed under GNU GPL as part of the git source repository,
|
||||
see http://en.wikipedia.org/wiki/Git_%28software%29 for more information.
|
||||
|
||||
They are not required for the actual operation, which is why they are not found
|
||||
in the distribution package.
|
||||
32
zero-cost-nas/.eggs/gitdb-4.0.10-py3.8.egg/EGG-INFO/PKG-INFO
Normal file
32
zero-cost-nas/.eggs/gitdb-4.0.10-py3.8.egg/EGG-INFO/PKG-INFO
Normal file
@@ -0,0 +1,32 @@
|
||||
Metadata-Version: 2.1
|
||||
Name: gitdb
|
||||
Version: 4.0.10
|
||||
Summary: Git Object Database
|
||||
Home-page: https://github.com/gitpython-developers/gitdb
|
||||
Author: Sebastian Thiel
|
||||
Author-email: byronimo@gmail.com
|
||||
License: BSD License
|
||||
Platform: UNKNOWN
|
||||
Classifier: Development Status :: 5 - Production/Stable
|
||||
Classifier: Environment :: Console
|
||||
Classifier: Intended Audience :: Developers
|
||||
Classifier: License :: OSI Approved :: BSD License
|
||||
Classifier: Operating System :: OS Independent
|
||||
Classifier: Operating System :: POSIX
|
||||
Classifier: Operating System :: Microsoft :: Windows
|
||||
Classifier: Operating System :: MacOS :: MacOS X
|
||||
Classifier: Programming Language :: Python
|
||||
Classifier: Programming Language :: Python :: 3
|
||||
Classifier: Programming Language :: Python :: 3.7
|
||||
Classifier: Programming Language :: Python :: 3.8
|
||||
Classifier: Programming Language :: Python :: 3.9
|
||||
Classifier: Programming Language :: Python :: 3.10
|
||||
Classifier: Programming Language :: Python :: 3.11
|
||||
Classifier: Programming Language :: Python :: 3 :: Only
|
||||
Requires-Python: >=3.7
|
||||
License-File: LICENSE
|
||||
License-File: AUTHORS
|
||||
Requires-Dist: smmap (<6,>=3.0.1)
|
||||
|
||||
GitDB is a pure-Python git object database
|
||||
|
||||
31
zero-cost-nas/.eggs/gitdb-4.0.10-py3.8.egg/EGG-INFO/RECORD
Normal file
31
zero-cost-nas/.eggs/gitdb-4.0.10-py3.8.egg/EGG-INFO/RECORD
Normal file
@@ -0,0 +1,31 @@
|
||||
gitdb/__init__.py,sha256=4GfyWmyHAntCxBC4UzHrmZMU0yL_vq7fQ1GxC_Au_YI,972
|
||||
gitdb/base.py,sha256=UQEnspMcsv1k47FEcPOyAtrrodtrdMq-_qoQEY00dt0,8029
|
||||
gitdb/const.py,sha256=WWmEYKNDdm3J9fxYTFT_B6-QLDSMBClbz0LSBa1D1S8,90
|
||||
gitdb/exc.py,sha256=QmO9u8CMzBNJdB_oL_8Zkjjk__2DIFcMzeeSjn5_zzE,1307
|
||||
gitdb/fun.py,sha256=S3fMKTGqWJcy48PInlgdbgPQ0Gm9VWx_bG8RO5inNOU,23249
|
||||
gitdb/pack.py,sha256=UVwcGwuz0j1kOskP1bgk_CAsUypjpAMf7MCV62PtIt0,39250
|
||||
gitdb/stream.py,sha256=9ybR0ftBR1wRebFLXXm90UqKT6EtNNJoycr54uKSevA,27512
|
||||
gitdb/typ.py,sha256=VVpMfwE0Hrwfg67zx3CDYtJK3kME7qOGwaSq3-7Ia34,379
|
||||
gitdb/util.py,sha256=Ij8YmhrfIr49wvqf2dux7S7gwwKxkiTbqKfUoQ_c4zA,12308
|
||||
gitdb/db/__init__.py,sha256=E9KSdtGIQzq8KHKRl2X5eOlFWglfYz7v76M5z3h1GnA,377
|
||||
gitdb/db/base.py,sha256=SVIOI11GhF1JrMkAic9JK7B_Wu3G6hb_OpG_TgqK_OQ,9067
|
||||
gitdb/db/git.py,sha256=ne_aHMOEp7omBFcs1PZvZH1Q3gM8Na4LaKUYVuU3wWE,2672
|
||||
gitdb/db/loose.py,sha256=cAEh4MSruFe89OdkEXf3BGiiXnlrSHWslOcJqZ-AM8s,8083
|
||||
gitdb/db/mem.py,sha256=9OGx2t-ld17yQMkgYBj-w0JDbaQYOGCjxsSDf8sVVgQ,3349
|
||||
gitdb/db/pack.py,sha256=zc4VOd7S4_2BTXTQLeLdySncyYbdbfunwMTHTlM-bys,7291
|
||||
gitdb/db/ref.py,sha256=xMGjwGGg2-RK7f93PL4sVY-nVOfLFu5WNVDIfXkVtTg,2597
|
||||
gitdb/test/__init__.py,sha256=m2nfHtiAdFArkDi-o6Ek7zVPmtKprUwlNIa19ZQR6Cs,210
|
||||
gitdb/test/lib.py,sha256=L3b8FP_RRGzuGar6ukgPFmdvnzaL-y_ruE9454sa2aE,5495
|
||||
gitdb/test/test_base.py,sha256=548K9hQBmKPH1_16gJPpJHTYrJ2TX-HtCZ7FBiHhFys,2828
|
||||
gitdb/test/test_example.py,sha256=RFb0Tq9uN_qNERj0paOnDGCFeyEp16LTV25FZwfRmwU,1356
|
||||
gitdb/test/test_pack.py,sha256=fCNLRpB5kZu0QpUpiXmmStDl97tl38HyMbLLb0WBVaQ,9242
|
||||
gitdb/test/test_stream.py,sha256=Mgi5hNXBB57ZqKzNr9nQMF-y3spvv_fjp8gvCHQ_f88,5733
|
||||
gitdb/test/test_util.py,sha256=KN8E7gHPCz7B0dOyoXKB5stwoHKOtHm1IhRvPf65rJI,3249
|
||||
gitdb/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
||||
gitdb/utils/encoding.py,sha256=ceZZFb86LGJ71cwW6qkq_BFquAlNE7jaafNbwxYRSXk,372
|
||||
gitdb-4.0.10.dist-info/AUTHORS,sha256=aUmmuuKGJrGDzN5i-dDIbj00R1IOPcFTZDWznhEwZuM,66
|
||||
gitdb-4.0.10.dist-info/LICENSE,sha256=79KfWWoI6IV-aOdpSlC82nKDl5LafD8EG8v_XxgAkjk,1984
|
||||
gitdb-4.0.10.dist-info/METADATA,sha256=ony-DlT0WxzieNVSijpUeB0kBz3Q6z_rapXINO6All4,1150
|
||||
gitdb-4.0.10.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
|
||||
gitdb-4.0.10.dist-info/top_level.txt,sha256=ss6atT8cG4mQuAYXO6PokJ0r4Mm5cBiDbKsu2e3YHfs,6
|
||||
gitdb-4.0.10.dist-info/RECORD,,
|
||||
@@ -0,0 +1,5 @@
|
||||
Wheel-Version: 1.0
|
||||
Generator: bdist_wheel (0.37.0)
|
||||
Root-Is-Purelib: true
|
||||
Tag: py3-none-any
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
smmap<6,>=3.0.1
|
||||
@@ -0,0 +1 @@
|
||||
gitdb
|
||||
38
zero-cost-nas/.eggs/gitdb-4.0.10-py3.8.egg/gitdb/__init__.py
Normal file
38
zero-cost-nas/.eggs/gitdb-4.0.10-py3.8.egg/gitdb/__init__.py
Normal file
@@ -0,0 +1,38 @@
|
||||
# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors
|
||||
#
|
||||
# This module is part of GitDB and is released under
|
||||
# the New BSD License: http://www.opensource.org/licenses/bsd-license.php
|
||||
"""Initialize the object database module"""
|
||||
|
||||
import sys
|
||||
import os
|
||||
|
||||
#{ Initialization
|
||||
|
||||
|
||||
def _init_externals():
|
||||
"""Initialize external projects by putting them into the path"""
|
||||
if 'PYOXIDIZER' not in os.environ:
|
||||
where = os.path.join(os.path.dirname(__file__), 'ext', 'smmap')
|
||||
if os.path.exists(where):
|
||||
sys.path.append(where)
|
||||
|
||||
import smmap
|
||||
del smmap
|
||||
# END handle imports
|
||||
|
||||
#} END initialization
|
||||
|
||||
_init_externals()
|
||||
|
||||
__author__ = "Sebastian Thiel"
|
||||
__contact__ = "byronimo@gmail.com"
|
||||
__homepage__ = "https://github.com/gitpython-developers/gitdb"
|
||||
version_info = (4, 0, 10)
|
||||
__version__ = '.'.join(str(i) for i in version_info)
|
||||
|
||||
|
||||
# default imports
|
||||
from gitdb.base import *
|
||||
from gitdb.db import *
|
||||
from gitdb.stream import *
|
||||
315
zero-cost-nas/.eggs/gitdb-4.0.10-py3.8.egg/gitdb/base.py
Normal file
315
zero-cost-nas/.eggs/gitdb-4.0.10-py3.8.egg/gitdb/base.py
Normal file
@@ -0,0 +1,315 @@
|
||||
# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors
|
||||
#
|
||||
# This module is part of GitDB and is released under
|
||||
# the New BSD License: http://www.opensource.org/licenses/bsd-license.php
|
||||
"""Module with basic data structures - they are designed to be lightweight and fast"""
|
||||
from gitdb.util import bin_to_hex
|
||||
|
||||
from gitdb.fun import (
|
||||
type_id_to_type_map,
|
||||
type_to_type_id_map
|
||||
)
|
||||
|
||||
__all__ = ('OInfo', 'OPackInfo', 'ODeltaPackInfo',
|
||||
'OStream', 'OPackStream', 'ODeltaPackStream',
|
||||
'IStream', 'InvalidOInfo', 'InvalidOStream')
|
||||
|
||||
#{ ODB Bases
|
||||
|
||||
|
||||
class OInfo(tuple):
|
||||
|
||||
"""Carries information about an object in an ODB, providing information
|
||||
about the binary sha of the object, the type_string as well as the uncompressed size
|
||||
in bytes.
|
||||
|
||||
It can be accessed using tuple notation and using attribute access notation::
|
||||
|
||||
assert dbi[0] == dbi.binsha
|
||||
assert dbi[1] == dbi.type
|
||||
assert dbi[2] == dbi.size
|
||||
|
||||
The type is designed to be as lightweight as possible."""
|
||||
__slots__ = tuple()
|
||||
|
||||
def __new__(cls, sha, type, size):
|
||||
return tuple.__new__(cls, (sha, type, size))
|
||||
|
||||
def __init__(self, *args):
|
||||
tuple.__init__(self)
|
||||
|
||||
#{ Interface
|
||||
@property
|
||||
def binsha(self):
|
||||
""":return: our sha as binary, 20 bytes"""
|
||||
return self[0]
|
||||
|
||||
@property
|
||||
def hexsha(self):
|
||||
""":return: our sha, hex encoded, 40 bytes"""
|
||||
return bin_to_hex(self[0])
|
||||
|
||||
@property
|
||||
def type(self):
|
||||
return self[1]
|
||||
|
||||
@property
|
||||
def type_id(self):
|
||||
return type_to_type_id_map[self[1]]
|
||||
|
||||
@property
|
||||
def size(self):
|
||||
return self[2]
|
||||
#} END interface
|
||||
|
||||
|
||||
class OPackInfo(tuple):
|
||||
|
||||
"""As OInfo, but provides a type_id property to retrieve the numerical type id, and
|
||||
does not include a sha.
|
||||
|
||||
Additionally, the pack_offset is the absolute offset into the packfile at which
|
||||
all object information is located. The data_offset property points to the absolute
|
||||
location in the pack at which that actual data stream can be found."""
|
||||
__slots__ = tuple()
|
||||
|
||||
def __new__(cls, packoffset, type, size):
|
||||
return tuple.__new__(cls, (packoffset, type, size))
|
||||
|
||||
def __init__(self, *args):
|
||||
tuple.__init__(self)
|
||||
|
||||
#{ Interface
|
||||
|
||||
@property
|
||||
def pack_offset(self):
|
||||
return self[0]
|
||||
|
||||
@property
|
||||
def type(self):
|
||||
return type_id_to_type_map[self[1]]
|
||||
|
||||
@property
|
||||
def type_id(self):
|
||||
return self[1]
|
||||
|
||||
@property
|
||||
def size(self):
|
||||
return self[2]
|
||||
|
||||
#} END interface
|
||||
|
||||
|
||||
class ODeltaPackInfo(OPackInfo):
|
||||
|
||||
"""Adds delta specific information,
|
||||
Either the 20 byte sha which points to some object in the database,
|
||||
or the negative offset from the pack_offset, so that pack_offset - delta_info yields
|
||||
the pack offset of the base object"""
|
||||
__slots__ = tuple()
|
||||
|
||||
def __new__(cls, packoffset, type, size, delta_info):
|
||||
return tuple.__new__(cls, (packoffset, type, size, delta_info))
|
||||
|
||||
#{ Interface
|
||||
@property
|
||||
def delta_info(self):
|
||||
return self[3]
|
||||
#} END interface
|
||||
|
||||
|
||||
class OStream(OInfo):
|
||||
|
||||
"""Base for object streams retrieved from the database, providing additional
|
||||
information about the stream.
|
||||
Generally, ODB streams are read-only as objects are immutable"""
|
||||
__slots__ = tuple()
|
||||
|
||||
def __new__(cls, sha, type, size, stream, *args, **kwargs):
|
||||
"""Helps with the initialization of subclasses"""
|
||||
return tuple.__new__(cls, (sha, type, size, stream))
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
tuple.__init__(self)
|
||||
|
||||
#{ Stream Reader Interface
|
||||
|
||||
def read(self, size=-1):
|
||||
return self[3].read(size)
|
||||
|
||||
@property
|
||||
def stream(self):
|
||||
return self[3]
|
||||
|
||||
#} END stream reader interface
|
||||
|
||||
|
||||
class ODeltaStream(OStream):
|
||||
|
||||
"""Uses size info of its stream, delaying reads"""
|
||||
|
||||
def __new__(cls, sha, type, size, stream, *args, **kwargs):
|
||||
"""Helps with the initialization of subclasses"""
|
||||
return tuple.__new__(cls, (sha, type, size, stream))
|
||||
|
||||
#{ Stream Reader Interface
|
||||
|
||||
@property
|
||||
def size(self):
|
||||
return self[3].size
|
||||
|
||||
#} END stream reader interface
|
||||
|
||||
|
||||
class OPackStream(OPackInfo):
|
||||
|
||||
"""Next to pack object information, a stream outputting an undeltified base object
|
||||
is provided"""
|
||||
__slots__ = tuple()
|
||||
|
||||
def __new__(cls, packoffset, type, size, stream, *args):
|
||||
"""Helps with the initialization of subclasses"""
|
||||
return tuple.__new__(cls, (packoffset, type, size, stream))
|
||||
|
||||
#{ Stream Reader Interface
|
||||
def read(self, size=-1):
|
||||
return self[3].read(size)
|
||||
|
||||
@property
|
||||
def stream(self):
|
||||
return self[3]
|
||||
#} END stream reader interface
|
||||
|
||||
|
||||
class ODeltaPackStream(ODeltaPackInfo):
|
||||
|
||||
"""Provides a stream outputting the uncompressed offset delta information"""
|
||||
__slots__ = tuple()
|
||||
|
||||
def __new__(cls, packoffset, type, size, delta_info, stream):
|
||||
return tuple.__new__(cls, (packoffset, type, size, delta_info, stream))
|
||||
|
||||
#{ Stream Reader Interface
|
||||
def read(self, size=-1):
|
||||
return self[4].read(size)
|
||||
|
||||
@property
|
||||
def stream(self):
|
||||
return self[4]
|
||||
#} END stream reader interface
|
||||
|
||||
|
||||
class IStream(list):
|
||||
|
||||
"""Represents an input content stream to be fed into the ODB. It is mutable to allow
|
||||
the ODB to record information about the operations outcome right in this instance.
|
||||
|
||||
It provides interfaces for the OStream and a StreamReader to allow the instance
|
||||
to blend in without prior conversion.
|
||||
|
||||
The only method your content stream must support is 'read'"""
|
||||
__slots__ = tuple()
|
||||
|
||||
def __new__(cls, type, size, stream, sha=None):
|
||||
return list.__new__(cls, (sha, type, size, stream, None))
|
||||
|
||||
def __init__(self, type, size, stream, sha=None):
|
||||
list.__init__(self, (sha, type, size, stream, None))
|
||||
|
||||
#{ Interface
|
||||
@property
|
||||
def hexsha(self):
|
||||
""":return: our sha, hex encoded, 40 bytes"""
|
||||
return bin_to_hex(self[0])
|
||||
|
||||
def _error(self):
|
||||
""":return: the error that occurred when processing the stream, or None"""
|
||||
return self[4]
|
||||
|
||||
def _set_error(self, exc):
|
||||
"""Set this input stream to the given exc, may be None to reset the error"""
|
||||
self[4] = exc
|
||||
|
||||
error = property(_error, _set_error)
|
||||
|
||||
#} END interface
|
||||
|
||||
#{ Stream Reader Interface
|
||||
|
||||
def read(self, size=-1):
|
||||
"""Implements a simple stream reader interface, passing the read call on
|
||||
to our internal stream"""
|
||||
return self[3].read(size)
|
||||
|
||||
#} END stream reader interface
|
||||
|
||||
#{ interface
|
||||
|
||||
def _set_binsha(self, binsha):
|
||||
self[0] = binsha
|
||||
|
||||
def _binsha(self):
|
||||
return self[0]
|
||||
|
||||
binsha = property(_binsha, _set_binsha)
|
||||
|
||||
def _type(self):
|
||||
return self[1]
|
||||
|
||||
def _set_type(self, type):
|
||||
self[1] = type
|
||||
|
||||
type = property(_type, _set_type)
|
||||
|
||||
def _size(self):
|
||||
return self[2]
|
||||
|
||||
def _set_size(self, size):
|
||||
self[2] = size
|
||||
|
||||
size = property(_size, _set_size)
|
||||
|
||||
def _stream(self):
|
||||
return self[3]
|
||||
|
||||
def _set_stream(self, stream):
|
||||
self[3] = stream
|
||||
|
||||
stream = property(_stream, _set_stream)
|
||||
|
||||
#} END odb info interface
|
||||
|
||||
|
||||
class InvalidOInfo(tuple):
|
||||
|
||||
"""Carries information about a sha identifying an object which is invalid in
|
||||
the queried database. The exception attribute provides more information about
|
||||
the cause of the issue"""
|
||||
__slots__ = tuple()
|
||||
|
||||
def __new__(cls, sha, exc):
|
||||
return tuple.__new__(cls, (sha, exc))
|
||||
|
||||
def __init__(self, sha, exc):
|
||||
tuple.__init__(self, (sha, exc))
|
||||
|
||||
@property
|
||||
def binsha(self):
|
||||
return self[0]
|
||||
|
||||
@property
|
||||
def hexsha(self):
|
||||
return bin_to_hex(self[0])
|
||||
|
||||
@property
|
||||
def error(self):
|
||||
""":return: exception instance explaining the failure"""
|
||||
return self[1]
|
||||
|
||||
|
||||
class InvalidOStream(InvalidOInfo):
|
||||
|
||||
"""Carries information about an invalid ODB stream"""
|
||||
__slots__ = tuple()
|
||||
|
||||
#} END ODB Bases
|
||||
@@ -0,0 +1,4 @@
|
||||
BYTE_SPACE = b' '
|
||||
NULL_BYTE = b'\0'
|
||||
NULL_HEX_SHA = "0" * 40
|
||||
NULL_BIN_SHA = NULL_BYTE * 20
|
||||
@@ -0,0 +1,11 @@
|
||||
# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors
|
||||
#
|
||||
# This module is part of GitDB and is released under
|
||||
# the New BSD License: http://www.opensource.org/licenses/bsd-license.php
|
||||
|
||||
from gitdb.db.base import *
|
||||
from gitdb.db.loose import *
|
||||
from gitdb.db.mem import *
|
||||
from gitdb.db.pack import *
|
||||
from gitdb.db.git import *
|
||||
from gitdb.db.ref import *
|
||||
278
zero-cost-nas/.eggs/gitdb-4.0.10-py3.8.egg/gitdb/db/base.py
Normal file
278
zero-cost-nas/.eggs/gitdb-4.0.10-py3.8.egg/gitdb/db/base.py
Normal file
@@ -0,0 +1,278 @@
|
||||
# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors
|
||||
#
|
||||
# This module is part of GitDB and is released under
|
||||
# the New BSD License: http://www.opensource.org/licenses/bsd-license.php
|
||||
"""Contains implementations of database retrieveing objects"""
|
||||
from gitdb.util import (
|
||||
join,
|
||||
LazyMixin,
|
||||
hex_to_bin
|
||||
)
|
||||
|
||||
from gitdb.utils.encoding import force_text
|
||||
from gitdb.exc import (
|
||||
BadObject,
|
||||
AmbiguousObjectName
|
||||
)
|
||||
|
||||
from itertools import chain
|
||||
from functools import reduce
|
||||
|
||||
|
||||
__all__ = ('ObjectDBR', 'ObjectDBW', 'FileDBBase', 'CompoundDB', 'CachingDB')
|
||||
|
||||
|
||||
class ObjectDBR:
|
||||
|
||||
"""Defines an interface for object database lookup.
|
||||
Objects are identified either by their 20 byte bin sha"""
|
||||
|
||||
def __contains__(self, sha):
|
||||
return self.has_obj
|
||||
|
||||
#{ Query Interface
|
||||
def has_object(self, sha):
|
||||
"""
|
||||
Whether the object identified by the given 20 bytes
|
||||
binary sha is contained in the database
|
||||
|
||||
:return: True if the object identified by the given 20 bytes
|
||||
binary sha is contained in the database"""
|
||||
raise NotImplementedError("To be implemented in subclass")
|
||||
|
||||
def info(self, sha):
|
||||
""" :return: OInfo instance
|
||||
:param sha: bytes binary sha
|
||||
:raise BadObject:"""
|
||||
raise NotImplementedError("To be implemented in subclass")
|
||||
|
||||
def stream(self, sha):
|
||||
""":return: OStream instance
|
||||
:param sha: 20 bytes binary sha
|
||||
:raise BadObject:"""
|
||||
raise NotImplementedError("To be implemented in subclass")
|
||||
|
||||
def size(self):
|
||||
""":return: amount of objects in this database"""
|
||||
raise NotImplementedError()
|
||||
|
||||
def sha_iter(self):
|
||||
"""Return iterator yielding 20 byte shas for all objects in this data base"""
|
||||
raise NotImplementedError()
|
||||
|
||||
#} END query interface
|
||||
|
||||
|
||||
class ObjectDBW:
|
||||
|
||||
"""Defines an interface to create objects in the database"""
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
self._ostream = None
|
||||
|
||||
#{ Edit Interface
|
||||
def set_ostream(self, stream):
|
||||
"""
|
||||
Adjusts the stream to which all data should be sent when storing new objects
|
||||
|
||||
:param stream: if not None, the stream to use, if None the default stream
|
||||
will be used.
|
||||
:return: previously installed stream, or None if there was no override
|
||||
:raise TypeError: if the stream doesn't have the supported functionality"""
|
||||
cstream = self._ostream
|
||||
self._ostream = stream
|
||||
return cstream
|
||||
|
||||
def ostream(self):
|
||||
"""
|
||||
Return the output stream
|
||||
|
||||
:return: overridden output stream this instance will write to, or None
|
||||
if it will write to the default stream"""
|
||||
return self._ostream
|
||||
|
||||
def store(self, istream):
|
||||
"""
|
||||
Create a new object in the database
|
||||
:return: the input istream object with its sha set to its corresponding value
|
||||
|
||||
:param istream: IStream compatible instance. If its sha is already set
|
||||
to a value, the object will just be stored in the our database format,
|
||||
in which case the input stream is expected to be in object format ( header + contents ).
|
||||
:raise IOError: if data could not be written"""
|
||||
raise NotImplementedError("To be implemented in subclass")
|
||||
|
||||
#} END edit interface
|
||||
|
||||
|
||||
class FileDBBase:
|
||||
|
||||
"""Provides basic facilities to retrieve files of interest, including
|
||||
caching facilities to help mapping hexsha's to objects"""
|
||||
|
||||
def __init__(self, root_path):
|
||||
"""Initialize this instance to look for its files at the given root path
|
||||
All subsequent operations will be relative to this path
|
||||
:raise InvalidDBRoot:
|
||||
**Note:** The base will not perform any accessablity checking as the base
|
||||
might not yet be accessible, but become accessible before the first
|
||||
access."""
|
||||
super().__init__()
|
||||
self._root_path = root_path
|
||||
|
||||
#{ Interface
|
||||
def root_path(self):
|
||||
""":return: path at which this db operates"""
|
||||
return self._root_path
|
||||
|
||||
def db_path(self, rela_path):
|
||||
"""
|
||||
:return: the given relative path relative to our database root, allowing
|
||||
to pontentially access datafiles"""
|
||||
return join(self._root_path, force_text(rela_path))
|
||||
#} END interface
|
||||
|
||||
|
||||
class CachingDB:
|
||||
|
||||
"""A database which uses caches to speed-up access"""
|
||||
|
||||
#{ Interface
|
||||
def update_cache(self, force=False):
|
||||
"""
|
||||
Call this method if the underlying data changed to trigger an update
|
||||
of the internal caching structures.
|
||||
|
||||
:param force: if True, the update must be performed. Otherwise the implementation
|
||||
may decide not to perform an update if it thinks nothing has changed.
|
||||
:return: True if an update was performed as something change indeed"""
|
||||
|
||||
# END interface
|
||||
|
||||
|
||||
def _databases_recursive(database, output):
|
||||
"""Fill output list with database from db, in order. Deals with Loose, Packed
|
||||
and compound databases."""
|
||||
if isinstance(database, CompoundDB):
|
||||
dbs = database.databases()
|
||||
output.extend(db for db in dbs if not isinstance(db, CompoundDB))
|
||||
for cdb in (db for db in dbs if isinstance(db, CompoundDB)):
|
||||
_databases_recursive(cdb, output)
|
||||
else:
|
||||
output.append(database)
|
||||
# END handle database type
|
||||
|
||||
|
||||
class CompoundDB(ObjectDBR, LazyMixin, CachingDB):
|
||||
|
||||
"""A database which delegates calls to sub-databases.
|
||||
|
||||
Databases are stored in the lazy-loaded _dbs attribute.
|
||||
Define _set_cache_ to update it with your databases"""
|
||||
|
||||
def _set_cache_(self, attr):
|
||||
if attr == '_dbs':
|
||||
self._dbs = list()
|
||||
elif attr == '_db_cache':
|
||||
self._db_cache = dict()
|
||||
else:
|
||||
super()._set_cache_(attr)
|
||||
|
||||
def _db_query(self, sha):
|
||||
""":return: database containing the given 20 byte sha
|
||||
:raise BadObject:"""
|
||||
# most databases use binary representations, prevent converting
|
||||
# it every time a database is being queried
|
||||
try:
|
||||
return self._db_cache[sha]
|
||||
except KeyError:
|
||||
pass
|
||||
# END first level cache
|
||||
|
||||
for db in self._dbs:
|
||||
if db.has_object(sha):
|
||||
self._db_cache[sha] = db
|
||||
return db
|
||||
# END for each database
|
||||
raise BadObject(sha)
|
||||
|
||||
#{ ObjectDBR interface
|
||||
|
||||
def has_object(self, sha):
|
||||
try:
|
||||
self._db_query(sha)
|
||||
return True
|
||||
except BadObject:
|
||||
return False
|
||||
# END handle exceptions
|
||||
|
||||
def info(self, sha):
|
||||
return self._db_query(sha).info(sha)
|
||||
|
||||
def stream(self, sha):
|
||||
return self._db_query(sha).stream(sha)
|
||||
|
||||
def size(self):
|
||||
""":return: total size of all contained databases"""
|
||||
return reduce(lambda x, y: x + y, (db.size() for db in self._dbs), 0)
|
||||
|
||||
def sha_iter(self):
|
||||
return chain(*(db.sha_iter() for db in self._dbs))
|
||||
|
||||
#} END object DBR Interface
|
||||
|
||||
#{ Interface
|
||||
|
||||
def databases(self):
|
||||
""":return: tuple of database instances we use for lookups"""
|
||||
return tuple(self._dbs)
|
||||
|
||||
def update_cache(self, force=False):
|
||||
# something might have changed, clear everything
|
||||
self._db_cache.clear()
|
||||
stat = False
|
||||
for db in self._dbs:
|
||||
if isinstance(db, CachingDB):
|
||||
stat |= db.update_cache(force)
|
||||
# END if is caching db
|
||||
# END for each database to update
|
||||
return stat
|
||||
|
||||
def partial_to_complete_sha_hex(self, partial_hexsha):
|
||||
"""
|
||||
:return: 20 byte binary sha1 from the given less-than-40 byte hexsha (bytes or str)
|
||||
:param partial_hexsha: hexsha with less than 40 byte
|
||||
:raise AmbiguousObjectName: """
|
||||
databases = list()
|
||||
_databases_recursive(self, databases)
|
||||
partial_hexsha = force_text(partial_hexsha)
|
||||
len_partial_hexsha = len(partial_hexsha)
|
||||
if len_partial_hexsha % 2 != 0:
|
||||
partial_binsha = hex_to_bin(partial_hexsha + "0")
|
||||
else:
|
||||
partial_binsha = hex_to_bin(partial_hexsha)
|
||||
# END assure successful binary conversion
|
||||
|
||||
candidate = None
|
||||
for db in databases:
|
||||
full_bin_sha = None
|
||||
try:
|
||||
if hasattr(db, 'partial_to_complete_sha_hex'):
|
||||
full_bin_sha = db.partial_to_complete_sha_hex(partial_hexsha)
|
||||
else:
|
||||
full_bin_sha = db.partial_to_complete_sha(partial_binsha, len_partial_hexsha)
|
||||
# END handle database type
|
||||
except BadObject:
|
||||
continue
|
||||
# END ignore bad objects
|
||||
if full_bin_sha:
|
||||
if candidate and candidate != full_bin_sha:
|
||||
raise AmbiguousObjectName(partial_hexsha)
|
||||
candidate = full_bin_sha
|
||||
# END handle candidate
|
||||
# END for each db
|
||||
if not candidate:
|
||||
raise BadObject(partial_binsha)
|
||||
return candidate
|
||||
|
||||
#} END interface
|
||||
85
zero-cost-nas/.eggs/gitdb-4.0.10-py3.8.egg/gitdb/db/git.py
Normal file
85
zero-cost-nas/.eggs/gitdb-4.0.10-py3.8.egg/gitdb/db/git.py
Normal file
@@ -0,0 +1,85 @@
|
||||
# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors
|
||||
#
|
||||
# This module is part of GitDB and is released under
|
||||
# the New BSD License: http://www.opensource.org/licenses/bsd-license.php
|
||||
from gitdb.db.base import (
|
||||
CompoundDB,
|
||||
ObjectDBW,
|
||||
FileDBBase
|
||||
)
|
||||
|
||||
from gitdb.db.loose import LooseObjectDB
|
||||
from gitdb.db.pack import PackedDB
|
||||
from gitdb.db.ref import ReferenceDB
|
||||
|
||||
from gitdb.exc import InvalidDBRoot
|
||||
|
||||
import os
|
||||
|
||||
__all__ = ('GitDB', )
|
||||
|
||||
|
||||
class GitDB(FileDBBase, ObjectDBW, CompoundDB):
|
||||
|
||||
"""A git-style object database, which contains all objects in the 'objects'
|
||||
subdirectory
|
||||
|
||||
``IMPORTANT``: The usage of this implementation is highly discouraged as it fails to release file-handles.
|
||||
This can be a problem with long-running processes and/or big repositories.
|
||||
"""
|
||||
# Configuration
|
||||
PackDBCls = PackedDB
|
||||
LooseDBCls = LooseObjectDB
|
||||
ReferenceDBCls = ReferenceDB
|
||||
|
||||
# Directories
|
||||
packs_dir = 'pack'
|
||||
loose_dir = ''
|
||||
alternates_dir = os.path.join('info', 'alternates')
|
||||
|
||||
def __init__(self, root_path):
|
||||
"""Initialize ourselves on a git objects directory"""
|
||||
super().__init__(root_path)
|
||||
|
||||
def _set_cache_(self, attr):
|
||||
if attr == '_dbs' or attr == '_loose_db':
|
||||
self._dbs = list()
|
||||
loose_db = None
|
||||
for subpath, dbcls in ((self.packs_dir, self.PackDBCls),
|
||||
(self.loose_dir, self.LooseDBCls),
|
||||
(self.alternates_dir, self.ReferenceDBCls)):
|
||||
path = self.db_path(subpath)
|
||||
if os.path.exists(path):
|
||||
self._dbs.append(dbcls(path))
|
||||
if dbcls is self.LooseDBCls:
|
||||
loose_db = self._dbs[-1]
|
||||
# END remember loose db
|
||||
# END check path exists
|
||||
# END for each db type
|
||||
|
||||
# should have at least one subdb
|
||||
if not self._dbs:
|
||||
raise InvalidDBRoot(self.root_path())
|
||||
# END handle error
|
||||
|
||||
# we the first one should have the store method
|
||||
assert loose_db is not None and hasattr(loose_db, 'store'), "First database needs store functionality"
|
||||
|
||||
# finally set the value
|
||||
self._loose_db = loose_db
|
||||
else:
|
||||
super()._set_cache_(attr)
|
||||
# END handle attrs
|
||||
|
||||
#{ ObjectDBW interface
|
||||
|
||||
def store(self, istream):
|
||||
return self._loose_db.store(istream)
|
||||
|
||||
def ostream(self):
|
||||
return self._loose_db.ostream()
|
||||
|
||||
def set_ostream(self, ostream):
|
||||
return self._loose_db.set_ostream(ostream)
|
||||
|
||||
#} END objectdbw interface
|
||||
258
zero-cost-nas/.eggs/gitdb-4.0.10-py3.8.egg/gitdb/db/loose.py
Normal file
258
zero-cost-nas/.eggs/gitdb-4.0.10-py3.8.egg/gitdb/db/loose.py
Normal file
@@ -0,0 +1,258 @@
|
||||
# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors
|
||||
#
|
||||
# This module is part of GitDB and is released under
|
||||
# the New BSD License: http://www.opensource.org/licenses/bsd-license.php
|
||||
from gitdb.db.base import (
|
||||
FileDBBase,
|
||||
ObjectDBR,
|
||||
ObjectDBW
|
||||
)
|
||||
|
||||
|
||||
from gitdb.exc import (
|
||||
BadObject,
|
||||
AmbiguousObjectName
|
||||
)
|
||||
|
||||
from gitdb.stream import (
|
||||
DecompressMemMapReader,
|
||||
FDCompressedSha1Writer,
|
||||
FDStream,
|
||||
Sha1Writer
|
||||
)
|
||||
|
||||
from gitdb.base import (
|
||||
OStream,
|
||||
OInfo
|
||||
)
|
||||
|
||||
from gitdb.util import (
|
||||
file_contents_ro_filepath,
|
||||
ENOENT,
|
||||
hex_to_bin,
|
||||
bin_to_hex,
|
||||
exists,
|
||||
chmod,
|
||||
isdir,
|
||||
isfile,
|
||||
remove,
|
||||
mkdir,
|
||||
rename,
|
||||
dirname,
|
||||
basename,
|
||||
join
|
||||
)
|
||||
|
||||
from gitdb.fun import (
|
||||
chunk_size,
|
||||
loose_object_header_info,
|
||||
write_object,
|
||||
stream_copy
|
||||
)
|
||||
|
||||
from gitdb.utils.encoding import force_bytes
|
||||
|
||||
import tempfile
|
||||
import os
|
||||
import sys
|
||||
|
||||
|
||||
__all__ = ('LooseObjectDB', )
|
||||
|
||||
|
||||
class LooseObjectDB(FileDBBase, ObjectDBR, ObjectDBW):
|
||||
|
||||
"""A database which operates on loose object files"""
|
||||
|
||||
# CONFIGURATION
|
||||
# chunks in which data will be copied between streams
|
||||
stream_chunk_size = chunk_size
|
||||
|
||||
# On windows we need to keep it writable, otherwise it cannot be removed
|
||||
# either
|
||||
new_objects_mode = int("444", 8)
|
||||
if os.name == 'nt':
|
||||
new_objects_mode = int("644", 8)
|
||||
|
||||
def __init__(self, root_path):
|
||||
super().__init__(root_path)
|
||||
self._hexsha_to_file = dict()
|
||||
# Additional Flags - might be set to 0 after the first failure
|
||||
# Depending on the root, this might work for some mounts, for others not, which
|
||||
# is why it is per instance
|
||||
self._fd_open_flags = getattr(os, 'O_NOATIME', 0)
|
||||
|
||||
#{ Interface
|
||||
def object_path(self, hexsha):
|
||||
"""
|
||||
:return: path at which the object with the given hexsha would be stored,
|
||||
relative to the database root"""
|
||||
return join(hexsha[:2], hexsha[2:])
|
||||
|
||||
def readable_db_object_path(self, hexsha):
|
||||
"""
|
||||
:return: readable object path to the object identified by hexsha
|
||||
:raise BadObject: If the object file does not exist"""
|
||||
try:
|
||||
return self._hexsha_to_file[hexsha]
|
||||
except KeyError:
|
||||
pass
|
||||
# END ignore cache misses
|
||||
|
||||
# try filesystem
|
||||
path = self.db_path(self.object_path(hexsha))
|
||||
if exists(path):
|
||||
self._hexsha_to_file[hexsha] = path
|
||||
return path
|
||||
# END handle cache
|
||||
raise BadObject(hexsha)
|
||||
|
||||
def partial_to_complete_sha_hex(self, partial_hexsha):
|
||||
""":return: 20 byte binary sha1 string which matches the given name uniquely
|
||||
:param name: hexadecimal partial name (bytes or ascii string)
|
||||
:raise AmbiguousObjectName:
|
||||
:raise BadObject: """
|
||||
candidate = None
|
||||
for binsha in self.sha_iter():
|
||||
if bin_to_hex(binsha).startswith(force_bytes(partial_hexsha)):
|
||||
# it can't ever find the same object twice
|
||||
if candidate is not None:
|
||||
raise AmbiguousObjectName(partial_hexsha)
|
||||
candidate = binsha
|
||||
# END for each object
|
||||
if candidate is None:
|
||||
raise BadObject(partial_hexsha)
|
||||
return candidate
|
||||
|
||||
#} END interface
|
||||
|
||||
def _map_loose_object(self, sha):
|
||||
"""
|
||||
:return: memory map of that file to allow random read access
|
||||
:raise BadObject: if object could not be located"""
|
||||
db_path = self.db_path(self.object_path(bin_to_hex(sha)))
|
||||
try:
|
||||
return file_contents_ro_filepath(db_path, flags=self._fd_open_flags)
|
||||
except OSError as e:
|
||||
if e.errno != ENOENT:
|
||||
# try again without noatime
|
||||
try:
|
||||
return file_contents_ro_filepath(db_path)
|
||||
except OSError as new_e:
|
||||
raise BadObject(sha) from new_e
|
||||
# didn't work because of our flag, don't try it again
|
||||
self._fd_open_flags = 0
|
||||
else:
|
||||
raise BadObject(sha) from e
|
||||
# END handle error
|
||||
# END exception handling
|
||||
|
||||
def set_ostream(self, stream):
|
||||
""":raise TypeError: if the stream does not support the Sha1Writer interface"""
|
||||
if stream is not None and not isinstance(stream, Sha1Writer):
|
||||
raise TypeError("Output stream musst support the %s interface" % Sha1Writer.__name__)
|
||||
return super().set_ostream(stream)
|
||||
|
||||
def info(self, sha):
|
||||
m = self._map_loose_object(sha)
|
||||
try:
|
||||
typ, size = loose_object_header_info(m)
|
||||
return OInfo(sha, typ, size)
|
||||
finally:
|
||||
if hasattr(m, 'close'):
|
||||
m.close()
|
||||
# END assure release of system resources
|
||||
|
||||
def stream(self, sha):
|
||||
m = self._map_loose_object(sha)
|
||||
type, size, stream = DecompressMemMapReader.new(m, close_on_deletion=True)
|
||||
return OStream(sha, type, size, stream)
|
||||
|
||||
def has_object(self, sha):
|
||||
try:
|
||||
self.readable_db_object_path(bin_to_hex(sha))
|
||||
return True
|
||||
except BadObject:
|
||||
return False
|
||||
# END check existence
|
||||
|
||||
def store(self, istream):
|
||||
"""note: The sha we produce will be hex by nature"""
|
||||
tmp_path = None
|
||||
writer = self.ostream()
|
||||
if writer is None:
|
||||
# open a tmp file to write the data to
|
||||
fd, tmp_path = tempfile.mkstemp(prefix='obj', dir=self._root_path)
|
||||
|
||||
if istream.binsha is None:
|
||||
writer = FDCompressedSha1Writer(fd)
|
||||
else:
|
||||
writer = FDStream(fd)
|
||||
# END handle direct stream copies
|
||||
# END handle custom writer
|
||||
|
||||
try:
|
||||
try:
|
||||
if istream.binsha is not None:
|
||||
# copy as much as possible, the actual uncompressed item size might
|
||||
# be smaller than the compressed version
|
||||
stream_copy(istream.read, writer.write, sys.maxsize, self.stream_chunk_size)
|
||||
else:
|
||||
# write object with header, we have to make a new one
|
||||
write_object(istream.type, istream.size, istream.read, writer.write,
|
||||
chunk_size=self.stream_chunk_size)
|
||||
# END handle direct stream copies
|
||||
finally:
|
||||
if tmp_path:
|
||||
writer.close()
|
||||
# END assure target stream is closed
|
||||
except:
|
||||
if tmp_path:
|
||||
os.remove(tmp_path)
|
||||
raise
|
||||
# END assure tmpfile removal on error
|
||||
|
||||
hexsha = None
|
||||
if istream.binsha:
|
||||
hexsha = istream.hexsha
|
||||
else:
|
||||
hexsha = writer.sha(as_hex=True)
|
||||
# END handle sha
|
||||
|
||||
if tmp_path:
|
||||
obj_path = self.db_path(self.object_path(hexsha))
|
||||
obj_dir = dirname(obj_path)
|
||||
if not isdir(obj_dir):
|
||||
mkdir(obj_dir)
|
||||
# END handle destination directory
|
||||
# rename onto existing doesn't work on NTFS
|
||||
if isfile(obj_path):
|
||||
remove(tmp_path)
|
||||
else:
|
||||
rename(tmp_path, obj_path)
|
||||
# end rename only if needed
|
||||
|
||||
# make sure its readable for all ! It started out as rw-- tmp file
|
||||
# but needs to be rwrr
|
||||
chmod(obj_path, self.new_objects_mode)
|
||||
# END handle dry_run
|
||||
|
||||
istream.binsha = hex_to_bin(hexsha)
|
||||
return istream
|
||||
|
||||
def sha_iter(self):
|
||||
# find all files which look like an object, extract sha from there
|
||||
for root, dirs, files in os.walk(self.root_path()):
|
||||
root_base = basename(root)
|
||||
if len(root_base) != 2:
|
||||
continue
|
||||
|
||||
for f in files:
|
||||
if len(f) != 38:
|
||||
continue
|
||||
yield hex_to_bin(root_base + f)
|
||||
# END for each file
|
||||
# END for each walk iteration
|
||||
|
||||
def size(self):
|
||||
return len(tuple(self.sha_iter()))
|
||||
110
zero-cost-nas/.eggs/gitdb-4.0.10-py3.8.egg/gitdb/db/mem.py
Normal file
110
zero-cost-nas/.eggs/gitdb-4.0.10-py3.8.egg/gitdb/db/mem.py
Normal file
@@ -0,0 +1,110 @@
|
||||
# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors
|
||||
#
|
||||
# This module is part of GitDB and is released under
|
||||
# the New BSD License: http://www.opensource.org/licenses/bsd-license.php
|
||||
"""Contains the MemoryDatabase implementation"""
|
||||
from gitdb.db.loose import LooseObjectDB
|
||||
from gitdb.db.base import (
|
||||
ObjectDBR,
|
||||
ObjectDBW
|
||||
)
|
||||
|
||||
from gitdb.base import (
|
||||
OStream,
|
||||
IStream,
|
||||
)
|
||||
|
||||
from gitdb.exc import (
|
||||
BadObject,
|
||||
UnsupportedOperation
|
||||
)
|
||||
|
||||
from gitdb.stream import (
|
||||
ZippedStoreShaWriter,
|
||||
DecompressMemMapReader,
|
||||
)
|
||||
|
||||
from io import BytesIO
|
||||
|
||||
__all__ = ("MemoryDB", )
|
||||
|
||||
|
||||
class MemoryDB(ObjectDBR, ObjectDBW):
|
||||
|
||||
"""A memory database stores everything to memory, providing fast IO and object
|
||||
retrieval. It should be used to buffer results and obtain SHAs before writing
|
||||
it to the actual physical storage, as it allows to query whether object already
|
||||
exists in the target storage before introducing actual IO"""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self._db = LooseObjectDB("path/doesnt/matter")
|
||||
|
||||
# maps 20 byte shas to their OStream objects
|
||||
self._cache = dict()
|
||||
|
||||
def set_ostream(self, stream):
|
||||
raise UnsupportedOperation("MemoryDB's always stream into memory")
|
||||
|
||||
def store(self, istream):
|
||||
zstream = ZippedStoreShaWriter()
|
||||
self._db.set_ostream(zstream)
|
||||
|
||||
istream = self._db.store(istream)
|
||||
zstream.close() # close to flush
|
||||
zstream.seek(0)
|
||||
|
||||
# don't provide a size, the stream is written in object format, hence the
|
||||
# header needs decompression
|
||||
decomp_stream = DecompressMemMapReader(zstream.getvalue(), close_on_deletion=False)
|
||||
self._cache[istream.binsha] = OStream(istream.binsha, istream.type, istream.size, decomp_stream)
|
||||
|
||||
return istream
|
||||
|
||||
def has_object(self, sha):
|
||||
return sha in self._cache
|
||||
|
||||
def info(self, sha):
|
||||
# we always return streams, which are infos as well
|
||||
return self.stream(sha)
|
||||
|
||||
def stream(self, sha):
|
||||
try:
|
||||
ostream = self._cache[sha]
|
||||
# rewind stream for the next one to read
|
||||
ostream.stream.seek(0)
|
||||
return ostream
|
||||
except KeyError as e:
|
||||
raise BadObject(sha) from e
|
||||
# END exception handling
|
||||
|
||||
def size(self):
|
||||
return len(self._cache)
|
||||
|
||||
def sha_iter(self):
|
||||
return self._cache.keys()
|
||||
|
||||
#{ Interface
|
||||
def stream_copy(self, sha_iter, odb):
|
||||
"""Copy the streams as identified by sha's yielded by sha_iter into the given odb
|
||||
The streams will be copied directly
|
||||
**Note:** the object will only be written if it did not exist in the target db
|
||||
|
||||
:return: amount of streams actually copied into odb. If smaller than the amount
|
||||
of input shas, one or more objects did already exist in odb"""
|
||||
count = 0
|
||||
for sha in sha_iter:
|
||||
if odb.has_object(sha):
|
||||
continue
|
||||
# END check object existence
|
||||
|
||||
ostream = self.stream(sha)
|
||||
# compressed data including header
|
||||
sio = BytesIO(ostream.stream.data())
|
||||
istream = IStream(ostream.type, ostream.size, sio, sha)
|
||||
|
||||
odb.store(istream)
|
||||
count += 1
|
||||
# END for each sha
|
||||
return count
|
||||
#} END interface
|
||||
206
zero-cost-nas/.eggs/gitdb-4.0.10-py3.8.egg/gitdb/db/pack.py
Normal file
206
zero-cost-nas/.eggs/gitdb-4.0.10-py3.8.egg/gitdb/db/pack.py
Normal file
@@ -0,0 +1,206 @@
|
||||
# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors
|
||||
#
|
||||
# This module is part of GitDB and is released under
|
||||
# the New BSD License: http://www.opensource.org/licenses/bsd-license.php
|
||||
"""Module containing a database to deal with packs"""
|
||||
from gitdb.db.base import (
|
||||
FileDBBase,
|
||||
ObjectDBR,
|
||||
CachingDB
|
||||
)
|
||||
|
||||
from gitdb.util import LazyMixin
|
||||
|
||||
from gitdb.exc import (
|
||||
BadObject,
|
||||
UnsupportedOperation,
|
||||
AmbiguousObjectName
|
||||
)
|
||||
|
||||
from gitdb.pack import PackEntity
|
||||
|
||||
from functools import reduce
|
||||
|
||||
import os
|
||||
import glob
|
||||
|
||||
__all__ = ('PackedDB', )
|
||||
|
||||
#{ Utilities
|
||||
|
||||
|
||||
class PackedDB(FileDBBase, ObjectDBR, CachingDB, LazyMixin):
|
||||
|
||||
"""A database operating on a set of object packs"""
|
||||
|
||||
# sort the priority list every N queries
|
||||
# Higher values are better, performance tests don't show this has
|
||||
# any effect, but it should have one
|
||||
_sort_interval = 500
|
||||
|
||||
def __init__(self, root_path):
|
||||
super().__init__(root_path)
|
||||
# list of lists with three items:
|
||||
# * hits - number of times the pack was hit with a request
|
||||
# * entity - Pack entity instance
|
||||
# * sha_to_index - PackIndexFile.sha_to_index method for direct cache query
|
||||
# self._entities = list() # lazy loaded list
|
||||
self._hit_count = 0 # amount of hits
|
||||
self._st_mtime = 0 # last modification data of our root path
|
||||
|
||||
def _set_cache_(self, attr):
|
||||
if attr == '_entities':
|
||||
self._entities = list()
|
||||
self.update_cache(force=True)
|
||||
# END handle entities initialization
|
||||
|
||||
def _sort_entities(self):
|
||||
self._entities.sort(key=lambda l: l[0], reverse=True)
|
||||
|
||||
def _pack_info(self, sha):
|
||||
""":return: tuple(entity, index) for an item at the given sha
|
||||
:param sha: 20 or 40 byte sha
|
||||
:raise BadObject:
|
||||
**Note:** This method is not thread-safe, but may be hit in multi-threaded
|
||||
operation. The worst thing that can happen though is a counter that
|
||||
was not incremented, or the list being in wrong order. So we safe
|
||||
the time for locking here, lets see how that goes"""
|
||||
# presort ?
|
||||
if self._hit_count % self._sort_interval == 0:
|
||||
self._sort_entities()
|
||||
# END update sorting
|
||||
|
||||
for item in self._entities:
|
||||
index = item[2](sha)
|
||||
if index is not None:
|
||||
item[0] += 1 # one hit for you
|
||||
self._hit_count += 1 # general hit count
|
||||
return (item[1], index)
|
||||
# END index found in pack
|
||||
# END for each item
|
||||
|
||||
# no hit, see whether we have to update packs
|
||||
# NOTE: considering packs don't change very often, we safe this call
|
||||
# and leave it to the super-caller to trigger that
|
||||
raise BadObject(sha)
|
||||
|
||||
#{ Object DB Read
|
||||
|
||||
def has_object(self, sha):
|
||||
try:
|
||||
self._pack_info(sha)
|
||||
return True
|
||||
except BadObject:
|
||||
return False
|
||||
# END exception handling
|
||||
|
||||
def info(self, sha):
|
||||
entity, index = self._pack_info(sha)
|
||||
return entity.info_at_index(index)
|
||||
|
||||
def stream(self, sha):
|
||||
entity, index = self._pack_info(sha)
|
||||
return entity.stream_at_index(index)
|
||||
|
||||
def sha_iter(self):
|
||||
for entity in self.entities():
|
||||
index = entity.index()
|
||||
sha_by_index = index.sha
|
||||
for index in range(index.size()):
|
||||
yield sha_by_index(index)
|
||||
# END for each index
|
||||
# END for each entity
|
||||
|
||||
def size(self):
|
||||
sizes = [item[1].index().size() for item in self._entities]
|
||||
return reduce(lambda x, y: x + y, sizes, 0)
|
||||
|
||||
#} END object db read
|
||||
|
||||
#{ object db write
|
||||
|
||||
def store(self, istream):
|
||||
"""Storing individual objects is not feasible as a pack is designed to
|
||||
hold multiple objects. Writing or rewriting packs for single objects is
|
||||
inefficient"""
|
||||
raise UnsupportedOperation()
|
||||
|
||||
#} END object db write
|
||||
|
||||
#{ Interface
|
||||
|
||||
def update_cache(self, force=False):
|
||||
"""
|
||||
Update our cache with the actually existing packs on disk. Add new ones,
|
||||
and remove deleted ones. We keep the unchanged ones
|
||||
|
||||
:param force: If True, the cache will be updated even though the directory
|
||||
does not appear to have changed according to its modification timestamp.
|
||||
:return: True if the packs have been updated so there is new information,
|
||||
False if there was no change to the pack database"""
|
||||
stat = os.stat(self.root_path())
|
||||
if not force and stat.st_mtime <= self._st_mtime:
|
||||
return False
|
||||
# END abort early on no change
|
||||
self._st_mtime = stat.st_mtime
|
||||
|
||||
# packs are supposed to be prefixed with pack- by git-convention
|
||||
# get all pack files, figure out what changed
|
||||
pack_files = set(glob.glob(os.path.join(self.root_path(), "pack-*.pack")))
|
||||
our_pack_files = {item[1].pack().path() for item in self._entities}
|
||||
|
||||
# new packs
|
||||
for pack_file in (pack_files - our_pack_files):
|
||||
# init the hit-counter/priority with the size, a good measure for hit-
|
||||
# probability. Its implemented so that only 12 bytes will be read
|
||||
entity = PackEntity(pack_file)
|
||||
self._entities.append([entity.pack().size(), entity, entity.index().sha_to_index])
|
||||
# END for each new packfile
|
||||
|
||||
# removed packs
|
||||
for pack_file in (our_pack_files - pack_files):
|
||||
del_index = -1
|
||||
for i, item in enumerate(self._entities):
|
||||
if item[1].pack().path() == pack_file:
|
||||
del_index = i
|
||||
break
|
||||
# END found index
|
||||
# END for each entity
|
||||
assert del_index != -1
|
||||
del(self._entities[del_index])
|
||||
# END for each removed pack
|
||||
|
||||
# reinitialize prioritiess
|
||||
self._sort_entities()
|
||||
return True
|
||||
|
||||
def entities(self):
|
||||
""":return: list of pack entities operated upon by this database"""
|
||||
return [item[1] for item in self._entities]
|
||||
|
||||
def partial_to_complete_sha(self, partial_binsha, canonical_length):
|
||||
""":return: 20 byte sha as inferred by the given partial binary sha
|
||||
:param partial_binsha: binary sha with less than 20 bytes
|
||||
:param canonical_length: length of the corresponding canonical representation.
|
||||
It is required as binary sha's cannot display whether the original hex sha
|
||||
had an odd or even number of characters
|
||||
:raise AmbiguousObjectName:
|
||||
:raise BadObject: """
|
||||
candidate = None
|
||||
for item in self._entities:
|
||||
item_index = item[1].index().partial_sha_to_index(partial_binsha, canonical_length)
|
||||
if item_index is not None:
|
||||
sha = item[1].index().sha(item_index)
|
||||
if candidate and candidate != sha:
|
||||
raise AmbiguousObjectName(partial_binsha)
|
||||
candidate = sha
|
||||
# END handle full sha could be found
|
||||
# END for each entity
|
||||
|
||||
if candidate:
|
||||
return candidate
|
||||
|
||||
# still not found ?
|
||||
raise BadObject(partial_binsha)
|
||||
|
||||
#} END interface
|
||||
82
zero-cost-nas/.eggs/gitdb-4.0.10-py3.8.egg/gitdb/db/ref.py
Normal file
82
zero-cost-nas/.eggs/gitdb-4.0.10-py3.8.egg/gitdb/db/ref.py
Normal file
@@ -0,0 +1,82 @@
|
||||
# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors
|
||||
#
|
||||
# This module is part of GitDB and is released under
|
||||
# the New BSD License: http://www.opensource.org/licenses/bsd-license.php
|
||||
import codecs
|
||||
from gitdb.db.base import (
|
||||
CompoundDB,
|
||||
)
|
||||
|
||||
__all__ = ('ReferenceDB', )
|
||||
|
||||
|
||||
class ReferenceDB(CompoundDB):
|
||||
|
||||
"""A database consisting of database referred to in a file"""
|
||||
|
||||
# Configuration
|
||||
# Specifies the object database to use for the paths found in the alternates
|
||||
# file. If None, it defaults to the GitDB
|
||||
ObjectDBCls = None
|
||||
|
||||
def __init__(self, ref_file):
|
||||
super().__init__()
|
||||
self._ref_file = ref_file
|
||||
|
||||
def _set_cache_(self, attr):
|
||||
if attr == '_dbs':
|
||||
self._dbs = list()
|
||||
self._update_dbs_from_ref_file()
|
||||
else:
|
||||
super()._set_cache_(attr)
|
||||
# END handle attrs
|
||||
|
||||
def _update_dbs_from_ref_file(self):
|
||||
dbcls = self.ObjectDBCls
|
||||
if dbcls is None:
|
||||
# late import
|
||||
from gitdb.db.git import GitDB
|
||||
dbcls = GitDB
|
||||
# END get db type
|
||||
|
||||
# try to get as many as possible, don't fail if some are unavailable
|
||||
ref_paths = list()
|
||||
try:
|
||||
with codecs.open(self._ref_file, 'r', encoding="utf-8") as f:
|
||||
ref_paths = [l.strip() for l in f]
|
||||
except OSError:
|
||||
pass
|
||||
# END handle alternates
|
||||
|
||||
ref_paths_set = set(ref_paths)
|
||||
cur_ref_paths_set = {db.root_path() for db in self._dbs}
|
||||
|
||||
# remove existing
|
||||
for path in (cur_ref_paths_set - ref_paths_set):
|
||||
for i, db in enumerate(self._dbs[:]):
|
||||
if db.root_path() == path:
|
||||
del(self._dbs[i])
|
||||
continue
|
||||
# END del matching db
|
||||
# END for each path to remove
|
||||
|
||||
# add new
|
||||
# sort them to maintain order
|
||||
added_paths = sorted(ref_paths_set - cur_ref_paths_set, key=lambda p: ref_paths.index(p))
|
||||
for path in added_paths:
|
||||
try:
|
||||
db = dbcls(path)
|
||||
# force an update to verify path
|
||||
if isinstance(db, CompoundDB):
|
||||
db.databases()
|
||||
# END verification
|
||||
self._dbs.append(db)
|
||||
except Exception:
|
||||
# ignore invalid paths or issues
|
||||
pass
|
||||
# END for each path to add
|
||||
|
||||
def update_cache(self, force=False):
|
||||
# re-read alternates and update databases
|
||||
self._update_dbs_from_ref_file()
|
||||
return super().update_cache(force)
|
||||
46
zero-cost-nas/.eggs/gitdb-4.0.10-py3.8.egg/gitdb/exc.py
Normal file
46
zero-cost-nas/.eggs/gitdb-4.0.10-py3.8.egg/gitdb/exc.py
Normal file
@@ -0,0 +1,46 @@
|
||||
# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors
|
||||
#
|
||||
# This module is part of GitDB and is released under
|
||||
# the New BSD License: http://www.opensource.org/licenses/bsd-license.php
|
||||
"""Module with common exceptions"""
|
||||
from gitdb.util import to_hex_sha
|
||||
|
||||
|
||||
class ODBError(Exception):
|
||||
"""All errors thrown by the object database"""
|
||||
|
||||
|
||||
class InvalidDBRoot(ODBError):
|
||||
"""Thrown if an object database cannot be initialized at the given path"""
|
||||
|
||||
|
||||
class BadObject(ODBError):
|
||||
"""The object with the given SHA does not exist. Instantiate with the
|
||||
failed sha"""
|
||||
|
||||
def __str__(self):
|
||||
return "BadObject: %s" % to_hex_sha(self.args[0])
|
||||
|
||||
|
||||
class BadName(ODBError):
|
||||
"""A name provided to rev_parse wasn't understood"""
|
||||
|
||||
def __str__(self):
|
||||
return "Ref '%s' did not resolve to an object" % self.args[0]
|
||||
|
||||
|
||||
class ParseError(ODBError):
|
||||
"""Thrown if the parsing of a file failed due to an invalid format"""
|
||||
|
||||
|
||||
class AmbiguousObjectName(ODBError):
|
||||
"""Thrown if a possibly shortened name does not uniquely represent a single object
|
||||
in the database"""
|
||||
|
||||
|
||||
class BadObjectType(ODBError):
|
||||
"""The object had an unsupported type"""
|
||||
|
||||
|
||||
class UnsupportedOperation(ODBError):
|
||||
"""Thrown if the given operation cannot be supported by the object database"""
|
||||
704
zero-cost-nas/.eggs/gitdb-4.0.10-py3.8.egg/gitdb/fun.py
Normal file
704
zero-cost-nas/.eggs/gitdb-4.0.10-py3.8.egg/gitdb/fun.py
Normal file
@@ -0,0 +1,704 @@
|
||||
# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors
|
||||
#
|
||||
# This module is part of GitDB and is released under
|
||||
# the New BSD License: http://www.opensource.org/licenses/bsd-license.php
|
||||
"""Contains basic c-functions which usually contain performance critical code
|
||||
Keeping this code separate from the beginning makes it easier to out-source
|
||||
it into c later, if required"""
|
||||
|
||||
import zlib
|
||||
from gitdb.util import byte_ord
|
||||
decompressobj = zlib.decompressobj
|
||||
|
||||
import mmap
|
||||
from itertools import islice
|
||||
from functools import reduce
|
||||
|
||||
from gitdb.const import NULL_BYTE, BYTE_SPACE
|
||||
from gitdb.utils.encoding import force_text
|
||||
from gitdb.typ import (
|
||||
str_blob_type,
|
||||
str_commit_type,
|
||||
str_tree_type,
|
||||
str_tag_type,
|
||||
)
|
||||
|
||||
from io import StringIO
|
||||
|
||||
# INVARIANTS
|
||||
OFS_DELTA = 6
|
||||
REF_DELTA = 7
|
||||
delta_types = (OFS_DELTA, REF_DELTA)
|
||||
|
||||
type_id_to_type_map = {
|
||||
0: b'', # EXT 1
|
||||
1: str_commit_type,
|
||||
2: str_tree_type,
|
||||
3: str_blob_type,
|
||||
4: str_tag_type,
|
||||
5: b'', # EXT 2
|
||||
OFS_DELTA: "OFS_DELTA", # OFFSET DELTA
|
||||
REF_DELTA: "REF_DELTA" # REFERENCE DELTA
|
||||
}
|
||||
|
||||
type_to_type_id_map = {
|
||||
str_commit_type: 1,
|
||||
str_tree_type: 2,
|
||||
str_blob_type: 3,
|
||||
str_tag_type: 4,
|
||||
"OFS_DELTA": OFS_DELTA,
|
||||
"REF_DELTA": REF_DELTA,
|
||||
}
|
||||
|
||||
# used when dealing with larger streams
|
||||
chunk_size = 1000 * mmap.PAGESIZE
|
||||
|
||||
__all__ = ('is_loose_object', 'loose_object_header_info', 'msb_size', 'pack_object_header_info',
|
||||
'write_object', 'loose_object_header', 'stream_copy', 'apply_delta_data',
|
||||
'is_equal_canonical_sha', 'connect_deltas', 'DeltaChunkList', 'create_pack_object_header')
|
||||
|
||||
|
||||
#{ Structures
|
||||
|
||||
def _set_delta_rbound(d, size):
|
||||
"""Truncate the given delta to the given size
|
||||
:param size: size relative to our target offset, may not be 0, must be smaller or equal
|
||||
to our size
|
||||
:return: d"""
|
||||
d.ts = size
|
||||
|
||||
# NOTE: data is truncated automatically when applying the delta
|
||||
# MUST NOT DO THIS HERE
|
||||
return d
|
||||
|
||||
|
||||
def _move_delta_lbound(d, bytes):
|
||||
"""Move the delta by the given amount of bytes, reducing its size so that its
|
||||
right bound stays static
|
||||
:param bytes: amount of bytes to move, must be smaller than delta size
|
||||
:return: d"""
|
||||
if bytes == 0:
|
||||
return
|
||||
|
||||
d.to += bytes
|
||||
d.so += bytes
|
||||
d.ts -= bytes
|
||||
if d.data is not None:
|
||||
d.data = d.data[bytes:]
|
||||
# END handle data
|
||||
|
||||
return d
|
||||
|
||||
|
||||
def delta_duplicate(src):
|
||||
return DeltaChunk(src.to, src.ts, src.so, src.data)
|
||||
|
||||
|
||||
def delta_chunk_apply(dc, bbuf, write):
|
||||
"""Apply own data to the target buffer
|
||||
:param bbuf: buffer providing source bytes for copy operations
|
||||
:param write: write method to call with data to write"""
|
||||
if dc.data is None:
|
||||
# COPY DATA FROM SOURCE
|
||||
write(bbuf[dc.so:dc.so + dc.ts])
|
||||
else:
|
||||
# APPEND DATA
|
||||
# what's faster: if + 4 function calls or just a write with a slice ?
|
||||
# Considering data can be larger than 127 bytes now, it should be worth it
|
||||
if dc.ts < len(dc.data):
|
||||
write(dc.data[:dc.ts])
|
||||
else:
|
||||
write(dc.data)
|
||||
# END handle truncation
|
||||
# END handle chunk mode
|
||||
|
||||
|
||||
class DeltaChunk:
|
||||
|
||||
"""Represents a piece of a delta, it can either add new data, or copy existing
|
||||
one from a source buffer"""
|
||||
__slots__ = (
|
||||
'to', # start offset in the target buffer in bytes
|
||||
'ts', # size of this chunk in the target buffer in bytes
|
||||
'so', # start offset in the source buffer in bytes or None
|
||||
'data', # chunk of bytes to be added to the target buffer,
|
||||
# DeltaChunkList to use as base, or None
|
||||
)
|
||||
|
||||
def __init__(self, to, ts, so, data):
|
||||
self.to = to
|
||||
self.ts = ts
|
||||
self.so = so
|
||||
self.data = data
|
||||
|
||||
def __repr__(self):
|
||||
return "DeltaChunk(%i, %i, %s, %s)" % (self.to, self.ts, self.so, self.data or "")
|
||||
|
||||
#{ Interface
|
||||
|
||||
def rbound(self):
|
||||
return self.to + self.ts
|
||||
|
||||
def has_data(self):
|
||||
""":return: True if the instance has data to add to the target stream"""
|
||||
return self.data is not None
|
||||
|
||||
#} END interface
|
||||
|
||||
|
||||
def _closest_index(dcl, absofs):
|
||||
""":return: index at which the given absofs should be inserted. The index points
|
||||
to the DeltaChunk with a target buffer absofs that equals or is greater than
|
||||
absofs.
|
||||
**Note:** global method for performance only, it belongs to DeltaChunkList"""
|
||||
lo = 0
|
||||
hi = len(dcl)
|
||||
while lo < hi:
|
||||
mid = (lo + hi) / 2
|
||||
dc = dcl[mid]
|
||||
if dc.to > absofs:
|
||||
hi = mid
|
||||
elif dc.rbound() > absofs or dc.to == absofs:
|
||||
return mid
|
||||
else:
|
||||
lo = mid + 1
|
||||
# END handle bound
|
||||
# END for each delta absofs
|
||||
return len(dcl) - 1
|
||||
|
||||
|
||||
def delta_list_apply(dcl, bbuf, write):
|
||||
"""Apply the chain's changes and write the final result using the passed
|
||||
write function.
|
||||
:param bbuf: base buffer containing the base of all deltas contained in this
|
||||
list. It will only be used if the chunk in question does not have a base
|
||||
chain.
|
||||
:param write: function taking a string of bytes to write to the output"""
|
||||
for dc in dcl:
|
||||
delta_chunk_apply(dc, bbuf, write)
|
||||
# END for each dc
|
||||
|
||||
|
||||
def delta_list_slice(dcl, absofs, size, ndcl):
|
||||
""":return: Subsection of this list at the given absolute offset, with the given
|
||||
size in bytes.
|
||||
:return: None"""
|
||||
cdi = _closest_index(dcl, absofs) # delta start index
|
||||
cd = dcl[cdi]
|
||||
slen = len(dcl)
|
||||
lappend = ndcl.append
|
||||
|
||||
if cd.to != absofs:
|
||||
tcd = DeltaChunk(cd.to, cd.ts, cd.so, cd.data)
|
||||
_move_delta_lbound(tcd, absofs - cd.to)
|
||||
tcd.ts = min(tcd.ts, size)
|
||||
lappend(tcd)
|
||||
size -= tcd.ts
|
||||
cdi += 1
|
||||
# END lbound overlap handling
|
||||
|
||||
while cdi < slen and size:
|
||||
# are we larger than the current block
|
||||
cd = dcl[cdi]
|
||||
if cd.ts <= size:
|
||||
lappend(DeltaChunk(cd.to, cd.ts, cd.so, cd.data))
|
||||
size -= cd.ts
|
||||
else:
|
||||
tcd = DeltaChunk(cd.to, cd.ts, cd.so, cd.data)
|
||||
tcd.ts = size
|
||||
lappend(tcd)
|
||||
size -= tcd.ts
|
||||
break
|
||||
# END hadle size
|
||||
cdi += 1
|
||||
# END for each chunk
|
||||
|
||||
|
||||
class DeltaChunkList(list):
|
||||
|
||||
"""List with special functionality to deal with DeltaChunks.
|
||||
There are two types of lists we represent. The one was created bottom-up, working
|
||||
towards the latest delta, the other kind was created top-down, working from the
|
||||
latest delta down to the earliest ancestor. This attribute is queryable
|
||||
after all processing with is_reversed."""
|
||||
|
||||
__slots__ = tuple()
|
||||
|
||||
def rbound(self):
|
||||
""":return: rightmost extend in bytes, absolute"""
|
||||
if len(self) == 0:
|
||||
return 0
|
||||
return self[-1].rbound()
|
||||
|
||||
def lbound(self):
|
||||
""":return: leftmost byte at which this chunklist starts"""
|
||||
if len(self) == 0:
|
||||
return 0
|
||||
return self[0].to
|
||||
|
||||
def size(self):
|
||||
""":return: size of bytes as measured by our delta chunks"""
|
||||
return self.rbound() - self.lbound()
|
||||
|
||||
def apply(self, bbuf, write):
|
||||
"""Only used by public clients, internally we only use the global routines
|
||||
for performance"""
|
||||
return delta_list_apply(self, bbuf, write)
|
||||
|
||||
def compress(self):
|
||||
"""Alter the list to reduce the amount of nodes. Currently we concatenate
|
||||
add-chunks
|
||||
:return: self"""
|
||||
slen = len(self)
|
||||
if slen < 2:
|
||||
return self
|
||||
i = 0
|
||||
|
||||
first_data_index = None
|
||||
while i < slen:
|
||||
dc = self[i]
|
||||
i += 1
|
||||
if dc.data is None:
|
||||
if first_data_index is not None and i - 2 - first_data_index > 1:
|
||||
# if first_data_index is not None:
|
||||
nd = StringIO() # new data
|
||||
so = self[first_data_index].to # start offset in target buffer
|
||||
for x in range(first_data_index, i - 1):
|
||||
xdc = self[x]
|
||||
nd.write(xdc.data[:xdc.ts])
|
||||
# END collect data
|
||||
|
||||
del(self[first_data_index:i - 1])
|
||||
buf = nd.getvalue()
|
||||
self.insert(first_data_index, DeltaChunk(so, len(buf), 0, buf))
|
||||
|
||||
slen = len(self)
|
||||
i = first_data_index + 1
|
||||
|
||||
# END concatenate data
|
||||
first_data_index = None
|
||||
continue
|
||||
# END skip non-data chunks
|
||||
|
||||
if first_data_index is None:
|
||||
first_data_index = i - 1
|
||||
# END iterate list
|
||||
|
||||
# if slen_orig != len(self):
|
||||
# print "INFO: Reduced delta list len to %f %% of former size" % ((float(len(self)) / slen_orig) * 100)
|
||||
return self
|
||||
|
||||
def check_integrity(self, target_size=-1):
|
||||
"""Verify the list has non-overlapping chunks only, and the total size matches
|
||||
target_size
|
||||
:param target_size: if not -1, the total size of the chain must be target_size
|
||||
:raise AssertionError: if the size doesn't match"""
|
||||
if target_size > -1:
|
||||
assert self[-1].rbound() == target_size
|
||||
assert reduce(lambda x, y: x + y, (d.ts for d in self), 0) == target_size
|
||||
# END target size verification
|
||||
|
||||
if len(self) < 2:
|
||||
return
|
||||
|
||||
# check data
|
||||
for dc in self:
|
||||
assert dc.ts > 0
|
||||
if dc.has_data():
|
||||
assert len(dc.data) >= dc.ts
|
||||
# END for each dc
|
||||
|
||||
left = islice(self, 0, len(self) - 1)
|
||||
right = iter(self)
|
||||
right.next()
|
||||
# this is very pythonic - we might have just use index based access here,
|
||||
# but this could actually be faster
|
||||
for lft, rgt in zip(left, right):
|
||||
assert lft.rbound() == rgt.to
|
||||
assert lft.to + lft.ts == rgt.to
|
||||
# END for each pair
|
||||
|
||||
|
||||
class TopdownDeltaChunkList(DeltaChunkList):
|
||||
|
||||
"""Represents a list which is generated by feeding its ancestor streams one by
|
||||
one"""
|
||||
__slots__ = tuple()
|
||||
|
||||
def connect_with_next_base(self, bdcl):
|
||||
"""Connect this chain with the next level of our base delta chunklist.
|
||||
The goal in this game is to mark as many of our chunks rigid, hence they
|
||||
cannot be changed by any of the upcoming bases anymore. Once all our
|
||||
chunks are marked like that, we can stop all processing
|
||||
:param bdcl: data chunk list being one of our bases. They must be fed in
|
||||
consecutively and in order, towards the earliest ancestor delta
|
||||
:return: True if processing was done. Use it to abort processing of
|
||||
remaining streams if False is returned"""
|
||||
nfc = 0 # number of frozen chunks
|
||||
dci = 0 # delta chunk index
|
||||
slen = len(self) # len of self
|
||||
ccl = list() # temporary list
|
||||
while dci < slen:
|
||||
dc = self[dci]
|
||||
dci += 1
|
||||
|
||||
# all add-chunks which are already topmost don't need additional processing
|
||||
if dc.data is not None:
|
||||
nfc += 1
|
||||
continue
|
||||
# END skip add chunks
|
||||
|
||||
# copy chunks
|
||||
# integrate the portion of the base list into ourselves. Lists
|
||||
# dont support efficient insertion ( just one at a time ), but for now
|
||||
# we live with it. Internally, its all just a 32/64bit pointer, and
|
||||
# the portions of moved memory should be smallish. Maybe we just rebuild
|
||||
# ourselves in order to reduce the amount of insertions ...
|
||||
del(ccl[:])
|
||||
delta_list_slice(bdcl, dc.so, dc.ts, ccl)
|
||||
|
||||
# move the target bounds into place to match with our chunk
|
||||
ofs = dc.to - dc.so
|
||||
for cdc in ccl:
|
||||
cdc.to += ofs
|
||||
# END update target bounds
|
||||
|
||||
if len(ccl) == 1:
|
||||
self[dci - 1] = ccl[0]
|
||||
else:
|
||||
# maybe try to compute the expenses here, and pick the right algorithm
|
||||
# It would normally be faster than copying everything physically though
|
||||
# TODO: Use a deque here, and decide by the index whether to extend
|
||||
# or extend left !
|
||||
post_dci = self[dci:]
|
||||
del(self[dci - 1:]) # include deletion of dc
|
||||
self.extend(ccl)
|
||||
self.extend(post_dci)
|
||||
|
||||
slen = len(self)
|
||||
dci += len(ccl) - 1 # deleted dc, added rest
|
||||
|
||||
# END handle chunk replacement
|
||||
# END for each chunk
|
||||
|
||||
if nfc == slen:
|
||||
return False
|
||||
# END handle completeness
|
||||
return True
|
||||
|
||||
|
||||
#} END structures
|
||||
|
||||
#{ Routines
|
||||
|
||||
def is_loose_object(m):
|
||||
"""
|
||||
:return: True the file contained in memory map m appears to be a loose object.
|
||||
Only the first two bytes are needed"""
|
||||
b0, b1 = map(ord, m[:2])
|
||||
word = (b0 << 8) + b1
|
||||
return b0 == 0x78 and (word % 31) == 0
|
||||
|
||||
|
||||
def loose_object_header_info(m):
|
||||
"""
|
||||
:return: tuple(type_string, uncompressed_size_in_bytes) the type string of the
|
||||
object as well as its uncompressed size in bytes.
|
||||
:param m: memory map from which to read the compressed object data"""
|
||||
decompress_size = 8192 # is used in cgit as well
|
||||
hdr = decompressobj().decompress(m, decompress_size)
|
||||
type_name, size = hdr[:hdr.find(NULL_BYTE)].split(BYTE_SPACE)
|
||||
|
||||
return type_name, int(size)
|
||||
|
||||
|
||||
def pack_object_header_info(data):
|
||||
"""
|
||||
:return: tuple(type_id, uncompressed_size_in_bytes, byte_offset)
|
||||
The type_id should be interpreted according to the ``type_id_to_type_map`` map
|
||||
The byte-offset specifies the start of the actual zlib compressed datastream
|
||||
:param m: random-access memory, like a string or memory map"""
|
||||
c = byte_ord(data[0]) # first byte
|
||||
i = 1 # next char to read
|
||||
type_id = (c >> 4) & 7 # numeric type
|
||||
size = c & 15 # starting size
|
||||
s = 4 # starting bit-shift size
|
||||
while c & 0x80:
|
||||
c = byte_ord(data[i])
|
||||
i += 1
|
||||
size += (c & 0x7f) << s
|
||||
s += 7
|
||||
# END character loop
|
||||
# end performance at expense of maintenance ...
|
||||
return (type_id, size, i)
|
||||
|
||||
|
||||
def create_pack_object_header(obj_type, obj_size):
|
||||
"""
|
||||
:return: string defining the pack header comprised of the object type
|
||||
and its incompressed size in bytes
|
||||
|
||||
:param obj_type: pack type_id of the object
|
||||
:param obj_size: uncompressed size in bytes of the following object stream"""
|
||||
c = 0 # 1 byte
|
||||
hdr = bytearray() # output string
|
||||
|
||||
c = (obj_type << 4) | (obj_size & 0xf)
|
||||
obj_size >>= 4
|
||||
while obj_size:
|
||||
hdr.append(c | 0x80)
|
||||
c = obj_size & 0x7f
|
||||
obj_size >>= 7
|
||||
# END until size is consumed
|
||||
hdr.append(c)
|
||||
# end handle interpreter
|
||||
return hdr
|
||||
|
||||
|
||||
def msb_size(data, offset=0):
|
||||
"""
|
||||
:return: tuple(read_bytes, size) read the msb size from the given random
|
||||
access data starting at the given byte offset"""
|
||||
size = 0
|
||||
i = 0
|
||||
l = len(data)
|
||||
hit_msb = False
|
||||
while i < l:
|
||||
c = data[i + offset]
|
||||
size |= (c & 0x7f) << i * 7
|
||||
i += 1
|
||||
if not c & 0x80:
|
||||
hit_msb = True
|
||||
break
|
||||
# END check msb bit
|
||||
# END while in range
|
||||
# end performance ...
|
||||
if not hit_msb:
|
||||
raise AssertionError("Could not find terminating MSB byte in data stream")
|
||||
return i + offset, size
|
||||
|
||||
|
||||
def loose_object_header(type, size):
|
||||
"""
|
||||
:return: bytes representing the loose object header, which is immediately
|
||||
followed by the content stream of size 'size'"""
|
||||
return ('%s %i\0' % (force_text(type), size)).encode('ascii')
|
||||
|
||||
|
||||
def write_object(type, size, read, write, chunk_size=chunk_size):
|
||||
"""
|
||||
Write the object as identified by type, size and source_stream into the
|
||||
target_stream
|
||||
|
||||
:param type: type string of the object
|
||||
:param size: amount of bytes to write from source_stream
|
||||
:param read: read method of a stream providing the content data
|
||||
:param write: write method of the output stream
|
||||
:param close_target_stream: if True, the target stream will be closed when
|
||||
the routine exits, even if an error is thrown
|
||||
:return: The actual amount of bytes written to stream, which includes the header and a trailing newline"""
|
||||
tbw = 0 # total num bytes written
|
||||
|
||||
# WRITE HEADER: type SP size NULL
|
||||
tbw += write(loose_object_header(type, size))
|
||||
tbw += stream_copy(read, write, size, chunk_size)
|
||||
|
||||
return tbw
|
||||
|
||||
|
||||
def stream_copy(read, write, size, chunk_size):
|
||||
"""
|
||||
Copy a stream up to size bytes using the provided read and write methods,
|
||||
in chunks of chunk_size
|
||||
|
||||
**Note:** its much like stream_copy utility, but operates just using methods"""
|
||||
dbw = 0 # num data bytes written
|
||||
|
||||
# WRITE ALL DATA UP TO SIZE
|
||||
while True:
|
||||
cs = min(chunk_size, size - dbw)
|
||||
# NOTE: not all write methods return the amount of written bytes, like
|
||||
# mmap.write. Its bad, but we just deal with it ... perhaps its not
|
||||
# even less efficient
|
||||
# data_len = write(read(cs))
|
||||
# dbw += data_len
|
||||
data = read(cs)
|
||||
data_len = len(data)
|
||||
dbw += data_len
|
||||
write(data)
|
||||
if data_len < cs or dbw == size:
|
||||
break
|
||||
# END check for stream end
|
||||
# END duplicate data
|
||||
return dbw
|
||||
|
||||
|
||||
def connect_deltas(dstreams):
|
||||
"""
|
||||
Read the condensed delta chunk information from dstream and merge its information
|
||||
into a list of existing delta chunks
|
||||
|
||||
:param dstreams: iterable of delta stream objects, the delta to be applied last
|
||||
comes first, then all its ancestors in order
|
||||
:return: DeltaChunkList, containing all operations to apply"""
|
||||
tdcl = None # topmost dcl
|
||||
|
||||
dcl = tdcl = TopdownDeltaChunkList()
|
||||
for dsi, ds in enumerate(dstreams):
|
||||
# print "Stream", dsi
|
||||
db = ds.read()
|
||||
delta_buf_size = ds.size
|
||||
|
||||
# read header
|
||||
i, base_size = msb_size(db)
|
||||
i, target_size = msb_size(db, i)
|
||||
|
||||
# interpret opcodes
|
||||
tbw = 0 # amount of target bytes written
|
||||
while i < delta_buf_size:
|
||||
c = ord(db[i])
|
||||
i += 1
|
||||
if c & 0x80:
|
||||
cp_off, cp_size = 0, 0
|
||||
if (c & 0x01):
|
||||
cp_off = ord(db[i])
|
||||
i += 1
|
||||
if (c & 0x02):
|
||||
cp_off |= (ord(db[i]) << 8)
|
||||
i += 1
|
||||
if (c & 0x04):
|
||||
cp_off |= (ord(db[i]) << 16)
|
||||
i += 1
|
||||
if (c & 0x08):
|
||||
cp_off |= (ord(db[i]) << 24)
|
||||
i += 1
|
||||
if (c & 0x10):
|
||||
cp_size = ord(db[i])
|
||||
i += 1
|
||||
if (c & 0x20):
|
||||
cp_size |= (ord(db[i]) << 8)
|
||||
i += 1
|
||||
if (c & 0x40):
|
||||
cp_size |= (ord(db[i]) << 16)
|
||||
i += 1
|
||||
|
||||
if not cp_size:
|
||||
cp_size = 0x10000
|
||||
|
||||
rbound = cp_off + cp_size
|
||||
if (rbound < cp_size or
|
||||
rbound > base_size):
|
||||
break
|
||||
|
||||
dcl.append(DeltaChunk(tbw, cp_size, cp_off, None))
|
||||
tbw += cp_size
|
||||
elif c:
|
||||
# NOTE: in C, the data chunks should probably be concatenated here.
|
||||
# In python, we do it as a post-process
|
||||
dcl.append(DeltaChunk(tbw, c, 0, db[i:i + c]))
|
||||
i += c
|
||||
tbw += c
|
||||
else:
|
||||
raise ValueError("unexpected delta opcode 0")
|
||||
# END handle command byte
|
||||
# END while processing delta data
|
||||
|
||||
dcl.compress()
|
||||
|
||||
# merge the lists !
|
||||
if dsi > 0:
|
||||
if not tdcl.connect_with_next_base(dcl):
|
||||
break
|
||||
# END handle merge
|
||||
|
||||
# prepare next base
|
||||
dcl = DeltaChunkList()
|
||||
# END for each delta stream
|
||||
|
||||
return tdcl
|
||||
|
||||
|
||||
def apply_delta_data(src_buf, src_buf_size, delta_buf, delta_buf_size, write):
|
||||
"""
|
||||
Apply data from a delta buffer using a source buffer to the target file
|
||||
|
||||
:param src_buf: random access data from which the delta was created
|
||||
:param src_buf_size: size of the source buffer in bytes
|
||||
:param delta_buf_size: size for the delta buffer in bytes
|
||||
:param delta_buf: random access delta data
|
||||
:param write: write method taking a chunk of bytes
|
||||
|
||||
**Note:** transcribed to python from the similar routine in patch-delta.c"""
|
||||
i = 0
|
||||
db = delta_buf
|
||||
while i < delta_buf_size:
|
||||
c = db[i]
|
||||
i += 1
|
||||
if c & 0x80:
|
||||
cp_off, cp_size = 0, 0
|
||||
if (c & 0x01):
|
||||
cp_off = db[i]
|
||||
i += 1
|
||||
if (c & 0x02):
|
||||
cp_off |= (db[i] << 8)
|
||||
i += 1
|
||||
if (c & 0x04):
|
||||
cp_off |= (db[i] << 16)
|
||||
i += 1
|
||||
if (c & 0x08):
|
||||
cp_off |= (db[i] << 24)
|
||||
i += 1
|
||||
if (c & 0x10):
|
||||
cp_size = db[i]
|
||||
i += 1
|
||||
if (c & 0x20):
|
||||
cp_size |= (db[i] << 8)
|
||||
i += 1
|
||||
if (c & 0x40):
|
||||
cp_size |= (db[i] << 16)
|
||||
i += 1
|
||||
|
||||
if not cp_size:
|
||||
cp_size = 0x10000
|
||||
|
||||
rbound = cp_off + cp_size
|
||||
if (rbound < cp_size or
|
||||
rbound > src_buf_size):
|
||||
break
|
||||
write(src_buf[cp_off:cp_off + cp_size])
|
||||
elif c:
|
||||
write(db[i:i + c])
|
||||
i += c
|
||||
else:
|
||||
raise ValueError("unexpected delta opcode 0")
|
||||
# END handle command byte
|
||||
# END while processing delta data
|
||||
|
||||
# yes, lets use the exact same error message that git uses :)
|
||||
assert i == delta_buf_size, "delta replay has gone wild"
|
||||
|
||||
|
||||
def is_equal_canonical_sha(canonical_length, match, sha1):
|
||||
"""
|
||||
:return: True if the given lhs and rhs 20 byte binary shas
|
||||
The comparison will take the canonical_length of the match sha into account,
|
||||
hence the comparison will only use the last 4 bytes for uneven canonical representations
|
||||
:param match: less than 20 byte sha
|
||||
:param sha1: 20 byte sha"""
|
||||
binary_length = canonical_length // 2
|
||||
if match[:binary_length] != sha1[:binary_length]:
|
||||
return False
|
||||
|
||||
if canonical_length - binary_length and \
|
||||
(byte_ord(match[-1]) ^ byte_ord(sha1[len(match) - 1])) & 0xf0:
|
||||
return False
|
||||
# END handle uneven canonnical length
|
||||
return True
|
||||
|
||||
#} END routines
|
||||
|
||||
|
||||
try:
|
||||
from gitdb_speedups._perf import connect_deltas
|
||||
except ImportError:
|
||||
pass
|
||||
1031
zero-cost-nas/.eggs/gitdb-4.0.10-py3.8.egg/gitdb/pack.py
Normal file
1031
zero-cost-nas/.eggs/gitdb-4.0.10-py3.8.egg/gitdb/pack.py
Normal file
File diff suppressed because it is too large
Load Diff
730
zero-cost-nas/.eggs/gitdb-4.0.10-py3.8.egg/gitdb/stream.py
Normal file
730
zero-cost-nas/.eggs/gitdb-4.0.10-py3.8.egg/gitdb/stream.py
Normal file
@@ -0,0 +1,730 @@
|
||||
# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors
|
||||
#
|
||||
# This module is part of GitDB and is released under
|
||||
# the New BSD License: http://www.opensource.org/licenses/bsd-license.php
|
||||
|
||||
from io import BytesIO
|
||||
|
||||
import mmap
|
||||
import os
|
||||
import sys
|
||||
import zlib
|
||||
|
||||
from gitdb.fun import (
|
||||
msb_size,
|
||||
stream_copy,
|
||||
apply_delta_data,
|
||||
connect_deltas,
|
||||
delta_types
|
||||
)
|
||||
|
||||
from gitdb.util import (
|
||||
allocate_memory,
|
||||
LazyMixin,
|
||||
make_sha,
|
||||
write,
|
||||
close,
|
||||
)
|
||||
|
||||
from gitdb.const import NULL_BYTE, BYTE_SPACE
|
||||
from gitdb.utils.encoding import force_bytes
|
||||
|
||||
has_perf_mod = False
|
||||
try:
|
||||
from gitdb_speedups._perf import apply_delta as c_apply_delta
|
||||
has_perf_mod = True
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
__all__ = ('DecompressMemMapReader', 'FDCompressedSha1Writer', 'DeltaApplyReader',
|
||||
'Sha1Writer', 'FlexibleSha1Writer', 'ZippedStoreShaWriter', 'FDCompressedSha1Writer',
|
||||
'FDStream', 'NullStream')
|
||||
|
||||
|
||||
#{ RO Streams
|
||||
|
||||
class DecompressMemMapReader(LazyMixin):
|
||||
|
||||
"""Reads data in chunks from a memory map and decompresses it. The client sees
|
||||
only the uncompressed data, respective file-like read calls are handling on-demand
|
||||
buffered decompression accordingly
|
||||
|
||||
A constraint on the total size of bytes is activated, simulating
|
||||
a logical file within a possibly larger physical memory area
|
||||
|
||||
To read efficiently, you clearly don't want to read individual bytes, instead,
|
||||
read a few kilobytes at least.
|
||||
|
||||
**Note:** The chunk-size should be carefully selected as it will involve quite a bit
|
||||
of string copying due to the way the zlib is implemented. Its very wasteful,
|
||||
hence we try to find a good tradeoff between allocation time and number of
|
||||
times we actually allocate. An own zlib implementation would be good here
|
||||
to better support streamed reading - it would only need to keep the mmap
|
||||
and decompress it into chunks, that's all ... """
|
||||
__slots__ = ('_m', '_zip', '_buf', '_buflen', '_br', '_cws', '_cwe', '_s', '_close',
|
||||
'_cbr', '_phi')
|
||||
|
||||
max_read_size = 512 * 1024 # currently unused
|
||||
|
||||
def __init__(self, m, close_on_deletion, size=None):
|
||||
"""Initialize with mmap for stream reading
|
||||
:param m: must be content data - use new if you have object data and no size"""
|
||||
self._m = m
|
||||
self._zip = zlib.decompressobj()
|
||||
self._buf = None # buffer of decompressed bytes
|
||||
self._buflen = 0 # length of bytes in buffer
|
||||
if size is not None:
|
||||
self._s = size # size of uncompressed data to read in total
|
||||
self._br = 0 # num uncompressed bytes read
|
||||
self._cws = 0 # start byte of compression window
|
||||
self._cwe = 0 # end byte of compression window
|
||||
self._cbr = 0 # number of compressed bytes read
|
||||
self._phi = False # is True if we parsed the header info
|
||||
self._close = close_on_deletion # close the memmap on deletion ?
|
||||
|
||||
def _set_cache_(self, attr):
|
||||
assert attr == '_s'
|
||||
# only happens for size, which is a marker to indicate we still
|
||||
# have to parse the header from the stream
|
||||
self._parse_header_info()
|
||||
|
||||
def __del__(self):
|
||||
self.close()
|
||||
|
||||
def _parse_header_info(self):
|
||||
"""If this stream contains object data, parse the header info and skip the
|
||||
stream to a point where each read will yield object content
|
||||
|
||||
:return: parsed type_string, size"""
|
||||
# read header
|
||||
# should really be enough, cgit uses 8192 I believe
|
||||
# And for good reason !! This needs to be that high for the header to be read correctly in all cases
|
||||
maxb = 8192
|
||||
self._s = maxb
|
||||
hdr = self.read(maxb)
|
||||
hdrend = hdr.find(NULL_BYTE)
|
||||
typ, size = hdr[:hdrend].split(BYTE_SPACE)
|
||||
size = int(size)
|
||||
self._s = size
|
||||
|
||||
# adjust internal state to match actual header length that we ignore
|
||||
# The buffer will be depleted first on future reads
|
||||
self._br = 0
|
||||
hdrend += 1
|
||||
self._buf = BytesIO(hdr[hdrend:])
|
||||
self._buflen = len(hdr) - hdrend
|
||||
|
||||
self._phi = True
|
||||
|
||||
return typ, size
|
||||
|
||||
#{ Interface
|
||||
|
||||
@classmethod
|
||||
def new(self, m, close_on_deletion=False):
|
||||
"""Create a new DecompressMemMapReader instance for acting as a read-only stream
|
||||
This method parses the object header from m and returns the parsed
|
||||
type and size, as well as the created stream instance.
|
||||
|
||||
:param m: memory map on which to operate. It must be object data ( header + contents )
|
||||
:param close_on_deletion: if True, the memory map will be closed once we are
|
||||
being deleted"""
|
||||
inst = DecompressMemMapReader(m, close_on_deletion, 0)
|
||||
typ, size = inst._parse_header_info()
|
||||
return typ, size, inst
|
||||
|
||||
def data(self):
|
||||
""":return: random access compatible data we are working on"""
|
||||
return self._m
|
||||
|
||||
def close(self):
|
||||
"""Close our underlying stream of compressed bytes if this was allowed during initialization
|
||||
:return: True if we closed the underlying stream
|
||||
:note: can be called safely
|
||||
"""
|
||||
if self._close:
|
||||
if hasattr(self._m, 'close'):
|
||||
self._m.close()
|
||||
self._close = False
|
||||
# END handle resource freeing
|
||||
|
||||
def compressed_bytes_read(self):
|
||||
"""
|
||||
:return: number of compressed bytes read. This includes the bytes it
|
||||
took to decompress the header ( if there was one )"""
|
||||
# ABSTRACT: When decompressing a byte stream, it can be that the first
|
||||
# x bytes which were requested match the first x bytes in the loosely
|
||||
# compressed datastream. This is the worst-case assumption that the reader
|
||||
# does, it assumes that it will get at least X bytes from X compressed bytes
|
||||
# in call cases.
|
||||
# The caveat is that the object, according to our known uncompressed size,
|
||||
# is already complete, but there are still some bytes left in the compressed
|
||||
# stream that contribute to the amount of compressed bytes.
|
||||
# How can we know that we are truly done, and have read all bytes we need
|
||||
# to read ?
|
||||
# Without help, we cannot know, as we need to obtain the status of the
|
||||
# decompression. If it is not finished, we need to decompress more data
|
||||
# until it is finished, to yield the actual number of compressed bytes
|
||||
# belonging to the decompressed object
|
||||
# We are using a custom zlib module for this, if its not present,
|
||||
# we try to put in additional bytes up for decompression if feasible
|
||||
# and check for the unused_data.
|
||||
|
||||
# Only scrub the stream forward if we are officially done with the
|
||||
# bytes we were to have.
|
||||
if self._br == self._s and not self._zip.unused_data:
|
||||
# manipulate the bytes-read to allow our own read method to continue
|
||||
# but keep the window at its current position
|
||||
self._br = 0
|
||||
if hasattr(self._zip, 'status'):
|
||||
while self._zip.status == zlib.Z_OK:
|
||||
self.read(mmap.PAGESIZE)
|
||||
# END scrub-loop custom zlib
|
||||
else:
|
||||
# pass in additional pages, until we have unused data
|
||||
while not self._zip.unused_data and self._cbr != len(self._m):
|
||||
self.read(mmap.PAGESIZE)
|
||||
# END scrub-loop default zlib
|
||||
# END handle stream scrubbing
|
||||
|
||||
# reset bytes read, just to be sure
|
||||
self._br = self._s
|
||||
# END handle stream scrubbing
|
||||
|
||||
# unused data ends up in the unconsumed tail, which was removed
|
||||
# from the count already
|
||||
return self._cbr
|
||||
|
||||
#} END interface
|
||||
|
||||
def seek(self, offset, whence=getattr(os, 'SEEK_SET', 0)):
|
||||
"""Allows to reset the stream to restart reading
|
||||
:raise ValueError: If offset and whence are not 0"""
|
||||
if offset != 0 or whence != getattr(os, 'SEEK_SET', 0):
|
||||
raise ValueError("Can only seek to position 0")
|
||||
# END handle offset
|
||||
|
||||
self._zip = zlib.decompressobj()
|
||||
self._br = self._cws = self._cwe = self._cbr = 0
|
||||
if self._phi:
|
||||
self._phi = False
|
||||
del(self._s) # trigger header parsing on first access
|
||||
# END skip header
|
||||
|
||||
def read(self, size=-1):
|
||||
if size < 1:
|
||||
size = self._s - self._br
|
||||
else:
|
||||
size = min(size, self._s - self._br)
|
||||
# END clamp size
|
||||
|
||||
if size == 0:
|
||||
return b''
|
||||
# END handle depletion
|
||||
|
||||
# deplete the buffer, then just continue using the decompress object
|
||||
# which has an own buffer. We just need this to transparently parse the
|
||||
# header from the zlib stream
|
||||
dat = b''
|
||||
if self._buf:
|
||||
if self._buflen >= size:
|
||||
# have enough data
|
||||
dat = self._buf.read(size)
|
||||
self._buflen -= size
|
||||
self._br += size
|
||||
return dat
|
||||
else:
|
||||
dat = self._buf.read() # ouch, duplicates data
|
||||
size -= self._buflen
|
||||
self._br += self._buflen
|
||||
|
||||
self._buflen = 0
|
||||
self._buf = None
|
||||
# END handle buffer len
|
||||
# END handle buffer
|
||||
|
||||
# decompress some data
|
||||
# Abstract: zlib needs to operate on chunks of our memory map ( which may
|
||||
# be large ), as it will otherwise and always fill in the 'unconsumed_tail'
|
||||
# attribute which possible reads our whole map to the end, forcing
|
||||
# everything to be read from disk even though just a portion was requested.
|
||||
# As this would be a nogo, we workaround it by passing only chunks of data,
|
||||
# moving the window into the memory map along as we decompress, which keeps
|
||||
# the tail smaller than our chunk-size. This causes 'only' the chunk to be
|
||||
# copied once, and another copy of a part of it when it creates the unconsumed
|
||||
# tail. We have to use it to hand in the appropriate amount of bytes during
|
||||
# the next read.
|
||||
tail = self._zip.unconsumed_tail
|
||||
if tail:
|
||||
# move the window, make it as large as size demands. For code-clarity,
|
||||
# we just take the chunk from our map again instead of reusing the unconsumed
|
||||
# tail. The latter one would safe some memory copying, but we could end up
|
||||
# with not getting enough data uncompressed, so we had to sort that out as well.
|
||||
# Now we just assume the worst case, hence the data is uncompressed and the window
|
||||
# needs to be as large as the uncompressed bytes we want to read.
|
||||
self._cws = self._cwe - len(tail)
|
||||
self._cwe = self._cws + size
|
||||
else:
|
||||
cws = self._cws
|
||||
self._cws = self._cwe
|
||||
self._cwe = cws + size
|
||||
# END handle tail
|
||||
|
||||
# if window is too small, make it larger so zip can decompress something
|
||||
if self._cwe - self._cws < 8:
|
||||
self._cwe = self._cws + 8
|
||||
# END adjust winsize
|
||||
|
||||
# takes a slice, but doesn't copy the data, it says ...
|
||||
indata = self._m[self._cws:self._cwe]
|
||||
|
||||
# get the actual window end to be sure we don't use it for computations
|
||||
self._cwe = self._cws + len(indata)
|
||||
dcompdat = self._zip.decompress(indata, size)
|
||||
# update the amount of compressed bytes read
|
||||
# We feed possibly overlapping chunks, which is why the unconsumed tail
|
||||
# has to be taken into consideration, as well as the unused data
|
||||
# if we hit the end of the stream
|
||||
# NOTE: Behavior changed in PY2.7 onward, which requires special handling to make the tests work properly.
|
||||
# They are thorough, and I assume it is truly working.
|
||||
# Why is this logic as convoluted as it is ? Please look at the table in
|
||||
# https://github.com/gitpython-developers/gitdb/issues/19 to learn about the test-results.
|
||||
# Basically, on py2.6, you want to use branch 1, whereas on all other python version, the second branch
|
||||
# will be the one that works.
|
||||
# However, the zlib VERSIONs as well as the platform check is used to further match the entries in the
|
||||
# table in the github issue. This is it ... it was the only way I could make this work everywhere.
|
||||
# IT's CERTAINLY GOING TO BITE US IN THE FUTURE ... .
|
||||
if zlib.ZLIB_VERSION in ('1.2.7', '1.2.5') and not sys.platform == 'darwin':
|
||||
unused_datalen = len(self._zip.unconsumed_tail)
|
||||
else:
|
||||
unused_datalen = len(self._zip.unconsumed_tail) + len(self._zip.unused_data)
|
||||
# # end handle very special case ...
|
||||
|
||||
self._cbr += len(indata) - unused_datalen
|
||||
self._br += len(dcompdat)
|
||||
|
||||
if dat:
|
||||
dcompdat = dat + dcompdat
|
||||
# END prepend our cached data
|
||||
|
||||
# it can happen, depending on the compression, that we get less bytes
|
||||
# than ordered as it needs the final portion of the data as well.
|
||||
# Recursively resolve that.
|
||||
# Note: dcompdat can be empty even though we still appear to have bytes
|
||||
# to read, if we are called by compressed_bytes_read - it manipulates
|
||||
# us to empty the stream
|
||||
if dcompdat and (len(dcompdat) - len(dat)) < size and self._br < self._s:
|
||||
dcompdat += self.read(size - len(dcompdat))
|
||||
# END handle special case
|
||||
return dcompdat
|
||||
|
||||
|
||||
class DeltaApplyReader(LazyMixin):
|
||||
|
||||
"""A reader which dynamically applies pack deltas to a base object, keeping the
|
||||
memory demands to a minimum.
|
||||
|
||||
The size of the final object is only obtainable once all deltas have been
|
||||
applied, unless it is retrieved from a pack index.
|
||||
|
||||
The uncompressed Delta has the following layout (MSB being a most significant
|
||||
bit encoded dynamic size):
|
||||
|
||||
* MSB Source Size - the size of the base against which the delta was created
|
||||
* MSB Target Size - the size of the resulting data after the delta was applied
|
||||
* A list of one byte commands (cmd) which are followed by a specific protocol:
|
||||
|
||||
* cmd & 0x80 - copy delta_data[offset:offset+size]
|
||||
|
||||
* Followed by an encoded offset into the delta data
|
||||
* Followed by an encoded size of the chunk to copy
|
||||
|
||||
* cmd & 0x7f - insert
|
||||
|
||||
* insert cmd bytes from the delta buffer into the output stream
|
||||
|
||||
* cmd == 0 - invalid operation ( or error in delta stream )
|
||||
"""
|
||||
__slots__ = (
|
||||
"_bstream", # base stream to which to apply the deltas
|
||||
"_dstreams", # tuple of delta stream readers
|
||||
"_mm_target", # memory map of the delta-applied data
|
||||
"_size", # actual number of bytes in _mm_target
|
||||
"_br" # number of bytes read
|
||||
)
|
||||
|
||||
#{ Configuration
|
||||
k_max_memory_move = 250 * 1000 * 1000
|
||||
#} END configuration
|
||||
|
||||
def __init__(self, stream_list):
|
||||
"""Initialize this instance with a list of streams, the first stream being
|
||||
the delta to apply on top of all following deltas, the last stream being the
|
||||
base object onto which to apply the deltas"""
|
||||
assert len(stream_list) > 1, "Need at least one delta and one base stream"
|
||||
|
||||
self._bstream = stream_list[-1]
|
||||
self._dstreams = tuple(stream_list[:-1])
|
||||
self._br = 0
|
||||
|
||||
def _set_cache_too_slow_without_c(self, attr):
|
||||
# the direct algorithm is fastest and most direct if there is only one
|
||||
# delta. Also, the extra overhead might not be worth it for items smaller
|
||||
# than X - definitely the case in python, every function call costs
|
||||
# huge amounts of time
|
||||
# if len(self._dstreams) * self._bstream.size < self.k_max_memory_move:
|
||||
if len(self._dstreams) == 1:
|
||||
return self._set_cache_brute_(attr)
|
||||
|
||||
# Aggregate all deltas into one delta in reverse order. Hence we take
|
||||
# the last delta, and reverse-merge its ancestor delta, until we receive
|
||||
# the final delta data stream.
|
||||
dcl = connect_deltas(self._dstreams)
|
||||
|
||||
# call len directly, as the (optional) c version doesn't implement the sequence
|
||||
# protocol
|
||||
if dcl.rbound() == 0:
|
||||
self._size = 0
|
||||
self._mm_target = allocate_memory(0)
|
||||
return
|
||||
# END handle empty list
|
||||
|
||||
self._size = dcl.rbound()
|
||||
self._mm_target = allocate_memory(self._size)
|
||||
|
||||
bbuf = allocate_memory(self._bstream.size)
|
||||
stream_copy(self._bstream.read, bbuf.write, self._bstream.size, 256 * mmap.PAGESIZE)
|
||||
|
||||
# APPLY CHUNKS
|
||||
write = self._mm_target.write
|
||||
dcl.apply(bbuf, write)
|
||||
|
||||
self._mm_target.seek(0)
|
||||
|
||||
def _set_cache_brute_(self, attr):
|
||||
"""If we are here, we apply the actual deltas"""
|
||||
# TODO: There should be a special case if there is only one stream
|
||||
# Then the default-git algorithm should perform a tad faster, as the
|
||||
# delta is not peaked into, causing less overhead.
|
||||
buffer_info_list = list()
|
||||
max_target_size = 0
|
||||
for dstream in self._dstreams:
|
||||
buf = dstream.read(512) # read the header information + X
|
||||
offset, src_size = msb_size(buf)
|
||||
offset, target_size = msb_size(buf, offset)
|
||||
buffer_info_list.append((buf[offset:], offset, src_size, target_size))
|
||||
max_target_size = max(max_target_size, target_size)
|
||||
# END for each delta stream
|
||||
|
||||
# sanity check - the first delta to apply should have the same source
|
||||
# size as our actual base stream
|
||||
base_size = self._bstream.size
|
||||
target_size = max_target_size
|
||||
|
||||
# if we have more than 1 delta to apply, we will swap buffers, hence we must
|
||||
# assure that all buffers we use are large enough to hold all the results
|
||||
if len(self._dstreams) > 1:
|
||||
base_size = target_size = max(base_size, max_target_size)
|
||||
# END adjust buffer sizes
|
||||
|
||||
# Allocate private memory map big enough to hold the first base buffer
|
||||
# We need random access to it
|
||||
bbuf = allocate_memory(base_size)
|
||||
stream_copy(self._bstream.read, bbuf.write, base_size, 256 * mmap.PAGESIZE)
|
||||
|
||||
# allocate memory map large enough for the largest (intermediate) target
|
||||
# We will use it as scratch space for all delta ops. If the final
|
||||
# target buffer is smaller than our allocated space, we just use parts
|
||||
# of it upon return.
|
||||
tbuf = allocate_memory(target_size)
|
||||
|
||||
# for each delta to apply, memory map the decompressed delta and
|
||||
# work on the op-codes to reconstruct everything.
|
||||
# For the actual copying, we use a seek and write pattern of buffer
|
||||
# slices.
|
||||
final_target_size = None
|
||||
for (dbuf, offset, src_size, target_size), dstream in zip(reversed(buffer_info_list), reversed(self._dstreams)):
|
||||
# allocate a buffer to hold all delta data - fill in the data for
|
||||
# fast access. We do this as we know that reading individual bytes
|
||||
# from our stream would be slower than necessary ( although possible )
|
||||
# The dbuf buffer contains commands after the first two MSB sizes, the
|
||||
# offset specifies the amount of bytes read to get the sizes.
|
||||
ddata = allocate_memory(dstream.size - offset)
|
||||
ddata.write(dbuf)
|
||||
# read the rest from the stream. The size we give is larger than necessary
|
||||
stream_copy(dstream.read, ddata.write, dstream.size, 256 * mmap.PAGESIZE)
|
||||
|
||||
#######################################################################
|
||||
if 'c_apply_delta' in globals():
|
||||
c_apply_delta(bbuf, ddata, tbuf)
|
||||
else:
|
||||
apply_delta_data(bbuf, src_size, ddata, len(ddata), tbuf.write)
|
||||
#######################################################################
|
||||
|
||||
# finally, swap out source and target buffers. The target is now the
|
||||
# base for the next delta to apply
|
||||
bbuf, tbuf = tbuf, bbuf
|
||||
bbuf.seek(0)
|
||||
tbuf.seek(0)
|
||||
final_target_size = target_size
|
||||
# END for each delta to apply
|
||||
|
||||
# its already seeked to 0, constrain it to the actual size
|
||||
# NOTE: in the end of the loop, it swaps buffers, hence our target buffer
|
||||
# is not tbuf, but bbuf !
|
||||
self._mm_target = bbuf
|
||||
self._size = final_target_size
|
||||
|
||||
#{ Configuration
|
||||
if not has_perf_mod:
|
||||
_set_cache_ = _set_cache_brute_
|
||||
else:
|
||||
_set_cache_ = _set_cache_too_slow_without_c
|
||||
|
||||
#} END configuration
|
||||
|
||||
def read(self, count=0):
|
||||
bl = self._size - self._br # bytes left
|
||||
if count < 1 or count > bl:
|
||||
count = bl
|
||||
# NOTE: we could check for certain size limits, and possibly
|
||||
# return buffers instead of strings to prevent byte copying
|
||||
data = self._mm_target.read(count)
|
||||
self._br += len(data)
|
||||
return data
|
||||
|
||||
def seek(self, offset, whence=getattr(os, 'SEEK_SET', 0)):
|
||||
"""Allows to reset the stream to restart reading
|
||||
|
||||
:raise ValueError: If offset and whence are not 0"""
|
||||
if offset != 0 or whence != getattr(os, 'SEEK_SET', 0):
|
||||
raise ValueError("Can only seek to position 0")
|
||||
# END handle offset
|
||||
self._br = 0
|
||||
self._mm_target.seek(0)
|
||||
|
||||
#{ Interface
|
||||
|
||||
@classmethod
|
||||
def new(cls, stream_list):
|
||||
"""
|
||||
Convert the given list of streams into a stream which resolves deltas
|
||||
when reading from it.
|
||||
|
||||
:param stream_list: two or more stream objects, first stream is a Delta
|
||||
to the object that you want to resolve, followed by N additional delta
|
||||
streams. The list's last stream must be a non-delta stream.
|
||||
|
||||
:return: Non-Delta OPackStream object whose stream can be used to obtain
|
||||
the decompressed resolved data
|
||||
:raise ValueError: if the stream list cannot be handled"""
|
||||
if len(stream_list) < 2:
|
||||
raise ValueError("Need at least two streams")
|
||||
# END single object special handling
|
||||
|
||||
if stream_list[-1].type_id in delta_types:
|
||||
raise ValueError(
|
||||
"Cannot resolve deltas if there is no base object stream, last one was type: %s" % stream_list[-1].type)
|
||||
# END check stream
|
||||
return cls(stream_list)
|
||||
|
||||
#} END interface
|
||||
|
||||
#{ OInfo like Interface
|
||||
|
||||
@property
|
||||
def type(self):
|
||||
return self._bstream.type
|
||||
|
||||
@property
|
||||
def type_id(self):
|
||||
return self._bstream.type_id
|
||||
|
||||
@property
|
||||
def size(self):
|
||||
""":return: number of uncompressed bytes in the stream"""
|
||||
return self._size
|
||||
|
||||
#} END oinfo like interface
|
||||
|
||||
|
||||
#} END RO streams
|
||||
|
||||
|
||||
#{ W Streams
|
||||
|
||||
class Sha1Writer:
|
||||
|
||||
"""Simple stream writer which produces a sha whenever you like as it degests
|
||||
everything it is supposed to write"""
|
||||
__slots__ = "sha1"
|
||||
|
||||
def __init__(self):
|
||||
self.sha1 = make_sha()
|
||||
|
||||
#{ Stream Interface
|
||||
|
||||
def write(self, data):
|
||||
""":raise IOError: If not all bytes could be written
|
||||
:param data: byte object
|
||||
:return: length of incoming data"""
|
||||
|
||||
self.sha1.update(data)
|
||||
|
||||
return len(data)
|
||||
|
||||
# END stream interface
|
||||
|
||||
#{ Interface
|
||||
|
||||
def sha(self, as_hex=False):
|
||||
""":return: sha so far
|
||||
:param as_hex: if True, sha will be hex-encoded, binary otherwise"""
|
||||
if as_hex:
|
||||
return self.sha1.hexdigest()
|
||||
return self.sha1.digest()
|
||||
|
||||
#} END interface
|
||||
|
||||
|
||||
class FlexibleSha1Writer(Sha1Writer):
|
||||
|
||||
"""Writer producing a sha1 while passing on the written bytes to the given
|
||||
write function"""
|
||||
__slots__ = 'writer'
|
||||
|
||||
def __init__(self, writer):
|
||||
Sha1Writer.__init__(self)
|
||||
self.writer = writer
|
||||
|
||||
def write(self, data):
|
||||
Sha1Writer.write(self, data)
|
||||
self.writer(data)
|
||||
|
||||
|
||||
class ZippedStoreShaWriter(Sha1Writer):
|
||||
|
||||
"""Remembers everything someone writes to it and generates a sha"""
|
||||
__slots__ = ('buf', 'zip')
|
||||
|
||||
def __init__(self):
|
||||
Sha1Writer.__init__(self)
|
||||
self.buf = BytesIO()
|
||||
self.zip = zlib.compressobj(zlib.Z_BEST_SPEED)
|
||||
|
||||
def __getattr__(self, attr):
|
||||
return getattr(self.buf, attr)
|
||||
|
||||
def write(self, data):
|
||||
alen = Sha1Writer.write(self, data)
|
||||
self.buf.write(self.zip.compress(data))
|
||||
|
||||
return alen
|
||||
|
||||
def close(self):
|
||||
self.buf.write(self.zip.flush())
|
||||
|
||||
def seek(self, offset, whence=getattr(os, 'SEEK_SET', 0)):
|
||||
"""Seeking currently only supports to rewind written data
|
||||
Multiple writes are not supported"""
|
||||
if offset != 0 or whence != getattr(os, 'SEEK_SET', 0):
|
||||
raise ValueError("Can only seek to position 0")
|
||||
# END handle offset
|
||||
self.buf.seek(0)
|
||||
|
||||
def getvalue(self):
|
||||
""":return: string value from the current stream position to the end"""
|
||||
return self.buf.getvalue()
|
||||
|
||||
|
||||
class FDCompressedSha1Writer(Sha1Writer):
|
||||
|
||||
"""Digests data written to it, making the sha available, then compress the
|
||||
data and write it to the file descriptor
|
||||
|
||||
**Note:** operates on raw file descriptors
|
||||
**Note:** for this to work, you have to use the close-method of this instance"""
|
||||
__slots__ = ("fd", "sha1", "zip")
|
||||
|
||||
# default exception
|
||||
exc = IOError("Failed to write all bytes to filedescriptor")
|
||||
|
||||
def __init__(self, fd):
|
||||
super().__init__()
|
||||
self.fd = fd
|
||||
self.zip = zlib.compressobj(zlib.Z_BEST_SPEED)
|
||||
|
||||
#{ Stream Interface
|
||||
|
||||
def write(self, data):
|
||||
""":raise IOError: If not all bytes could be written
|
||||
:return: length of incoming data"""
|
||||
self.sha1.update(data)
|
||||
cdata = self.zip.compress(data)
|
||||
bytes_written = write(self.fd, cdata)
|
||||
|
||||
if bytes_written != len(cdata):
|
||||
raise self.exc
|
||||
|
||||
return len(data)
|
||||
|
||||
def close(self):
|
||||
remainder = self.zip.flush()
|
||||
if write(self.fd, remainder) != len(remainder):
|
||||
raise self.exc
|
||||
return close(self.fd)
|
||||
|
||||
#} END stream interface
|
||||
|
||||
|
||||
class FDStream:
|
||||
|
||||
"""A simple wrapper providing the most basic functions on a file descriptor
|
||||
with the fileobject interface. Cannot use os.fdopen as the resulting stream
|
||||
takes ownership"""
|
||||
__slots__ = ("_fd", '_pos')
|
||||
|
||||
def __init__(self, fd):
|
||||
self._fd = fd
|
||||
self._pos = 0
|
||||
|
||||
def write(self, data):
|
||||
self._pos += len(data)
|
||||
os.write(self._fd, data)
|
||||
|
||||
def read(self, count=0):
|
||||
if count == 0:
|
||||
count = os.path.getsize(self._filepath)
|
||||
# END handle read everything
|
||||
|
||||
bytes = os.read(self._fd, count)
|
||||
self._pos += len(bytes)
|
||||
return bytes
|
||||
|
||||
def fileno(self):
|
||||
return self._fd
|
||||
|
||||
def tell(self):
|
||||
return self._pos
|
||||
|
||||
def close(self):
|
||||
close(self._fd)
|
||||
|
||||
|
||||
class NullStream:
|
||||
|
||||
"""A stream that does nothing but providing a stream interface.
|
||||
Use it like /dev/null"""
|
||||
__slots__ = tuple()
|
||||
|
||||
def read(self, size=0):
|
||||
return ''
|
||||
|
||||
def close(self):
|
||||
pass
|
||||
|
||||
def write(self, data):
|
||||
return len(data)
|
||||
|
||||
|
||||
#} END W streams
|
||||
@@ -0,0 +1,4 @@
|
||||
# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors
|
||||
#
|
||||
# This module is part of GitDB and is released under
|
||||
# the New BSD License: http://www.opensource.org/licenses/bsd-license.php
|
||||
192
zero-cost-nas/.eggs/gitdb-4.0.10-py3.8.egg/gitdb/test/lib.py
Normal file
192
zero-cost-nas/.eggs/gitdb-4.0.10-py3.8.egg/gitdb/test/lib.py
Normal file
@@ -0,0 +1,192 @@
|
||||
# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors
|
||||
#
|
||||
# This module is part of GitDB and is released under
|
||||
# the New BSD License: http://www.opensource.org/licenses/bsd-license.php
|
||||
"""Utilities used in ODB testing"""
|
||||
from gitdb import OStream
|
||||
|
||||
import sys
|
||||
import random
|
||||
from array import array
|
||||
|
||||
from io import BytesIO
|
||||
|
||||
import glob
|
||||
import unittest
|
||||
import tempfile
|
||||
import shutil
|
||||
import os
|
||||
import gc
|
||||
import logging
|
||||
from functools import wraps
|
||||
|
||||
|
||||
#{ Bases
|
||||
|
||||
class TestBase(unittest.TestCase):
|
||||
"""Base class for all tests
|
||||
|
||||
TestCase providing access to readonly repositories using the following member variables.
|
||||
|
||||
* gitrepopath
|
||||
|
||||
* read-only base path of the git source repository, i.e. .../git/.git
|
||||
"""
|
||||
|
||||
#{ Invvariants
|
||||
k_env_git_repo = "GITDB_TEST_GIT_REPO_BASE"
|
||||
#} END invariants
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
try:
|
||||
super().setUpClass()
|
||||
except AttributeError:
|
||||
pass
|
||||
|
||||
cls.gitrepopath = os.environ.get(cls.k_env_git_repo)
|
||||
if not cls.gitrepopath:
|
||||
logging.info(
|
||||
"You can set the %s environment variable to a .git repository of your choice - defaulting to the gitdb repository", cls.k_env_git_repo)
|
||||
ospd = os.path.dirname
|
||||
cls.gitrepopath = os.path.join(ospd(ospd(ospd(__file__))), '.git')
|
||||
# end assure gitrepo is set
|
||||
assert cls.gitrepopath.endswith('.git')
|
||||
|
||||
|
||||
#} END bases
|
||||
|
||||
#{ Decorators
|
||||
|
||||
def with_rw_directory(func):
|
||||
"""Create a temporary directory which can be written to, remove it if the
|
||||
test succeeds, but leave it otherwise to aid additional debugging"""
|
||||
|
||||
def wrapper(self):
|
||||
path = tempfile.mktemp(prefix=func.__name__)
|
||||
os.mkdir(path)
|
||||
keep = False
|
||||
try:
|
||||
try:
|
||||
return func(self, path)
|
||||
except Exception:
|
||||
sys.stderr.write(f"Test {type(self).__name__}.{func.__name__} failed, output is at {path!r}\n")
|
||||
keep = True
|
||||
raise
|
||||
finally:
|
||||
# Need to collect here to be sure all handles have been closed. It appears
|
||||
# a windows-only issue. In fact things should be deleted, as well as
|
||||
# memory maps closed, once objects go out of scope. For some reason
|
||||
# though this is not the case here unless we collect explicitly.
|
||||
if not keep:
|
||||
gc.collect()
|
||||
shutil.rmtree(path)
|
||||
# END handle exception
|
||||
# END wrapper
|
||||
|
||||
wrapper.__name__ = func.__name__
|
||||
return wrapper
|
||||
|
||||
|
||||
def with_packs_rw(func):
|
||||
"""Function that provides a path into which the packs for testing should be
|
||||
copied. Will pass on the path to the actual function afterwards"""
|
||||
|
||||
def wrapper(self, path):
|
||||
src_pack_glob = fixture_path('packs/*')
|
||||
copy_files_globbed(src_pack_glob, path, hard_link_ok=True)
|
||||
return func(self, path)
|
||||
# END wrapper
|
||||
|
||||
wrapper.__name__ = func.__name__
|
||||
return wrapper
|
||||
|
||||
#} END decorators
|
||||
|
||||
#{ Routines
|
||||
|
||||
|
||||
def fixture_path(relapath=''):
|
||||
""":return: absolute path into the fixture directory
|
||||
:param relapath: relative path into the fixtures directory, or ''
|
||||
to obtain the fixture directory itself"""
|
||||
return os.path.join(os.path.dirname(__file__), 'fixtures', relapath)
|
||||
|
||||
|
||||
def copy_files_globbed(source_glob, target_dir, hard_link_ok=False):
|
||||
"""Copy all files found according to the given source glob into the target directory
|
||||
:param hard_link_ok: if True, hard links will be created if possible. Otherwise
|
||||
the files will be copied"""
|
||||
for src_file in glob.glob(source_glob):
|
||||
if hard_link_ok and hasattr(os, 'link'):
|
||||
target = os.path.join(target_dir, os.path.basename(src_file))
|
||||
try:
|
||||
os.link(src_file, target)
|
||||
except OSError:
|
||||
shutil.copy(src_file, target_dir)
|
||||
# END handle cross device links ( and resulting failure )
|
||||
else:
|
||||
shutil.copy(src_file, target_dir)
|
||||
# END try hard link
|
||||
# END for each file to copy
|
||||
|
||||
|
||||
def make_bytes(size_in_bytes, randomize=False):
|
||||
""":return: string with given size in bytes
|
||||
:param randomize: try to produce a very random stream"""
|
||||
actual_size = size_in_bytes // 4
|
||||
producer = range(actual_size)
|
||||
if randomize:
|
||||
producer = list(producer)
|
||||
random.shuffle(producer)
|
||||
# END randomize
|
||||
a = array('i', producer)
|
||||
return a.tobytes()
|
||||
|
||||
|
||||
def make_object(type, data):
|
||||
""":return: bytes resembling an uncompressed object"""
|
||||
odata = "blob %i\0" % len(data)
|
||||
return odata.encode("ascii") + data
|
||||
|
||||
|
||||
def make_memory_file(size_in_bytes, randomize=False):
|
||||
""":return: tuple(size_of_stream, stream)
|
||||
:param randomize: try to produce a very random stream"""
|
||||
d = make_bytes(size_in_bytes, randomize)
|
||||
return len(d), BytesIO(d)
|
||||
|
||||
#} END routines
|
||||
|
||||
#{ Stream Utilities
|
||||
|
||||
|
||||
class DummyStream:
|
||||
|
||||
def __init__(self):
|
||||
self.was_read = False
|
||||
self.bytes = 0
|
||||
self.closed = False
|
||||
|
||||
def read(self, size):
|
||||
self.was_read = True
|
||||
self.bytes = size
|
||||
|
||||
def close(self):
|
||||
self.closed = True
|
||||
|
||||
def _assert(self):
|
||||
assert self.was_read
|
||||
|
||||
|
||||
class DeriveTest(OStream):
|
||||
|
||||
def __init__(self, sha, type, size, stream, *args, **kwargs):
|
||||
self.myarg = kwargs.pop('myarg')
|
||||
self.args = args
|
||||
|
||||
def _assert(self):
|
||||
assert self.args
|
||||
assert self.myarg
|
||||
|
||||
#} END stream utilitiess
|
||||
@@ -0,0 +1,105 @@
|
||||
# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors
|
||||
#
|
||||
# This module is part of GitDB and is released under
|
||||
# the New BSD License: http://www.opensource.org/licenses/bsd-license.php
|
||||
"""Test for object db"""
|
||||
from gitdb.test.lib import (
|
||||
TestBase,
|
||||
DummyStream,
|
||||
DeriveTest,
|
||||
)
|
||||
|
||||
from gitdb import (
|
||||
OInfo,
|
||||
OPackInfo,
|
||||
ODeltaPackInfo,
|
||||
OStream,
|
||||
OPackStream,
|
||||
ODeltaPackStream,
|
||||
IStream
|
||||
)
|
||||
from gitdb.util import (
|
||||
NULL_BIN_SHA
|
||||
)
|
||||
|
||||
from gitdb.typ import (
|
||||
str_blob_type
|
||||
)
|
||||
|
||||
|
||||
class TestBaseTypes(TestBase):
|
||||
|
||||
def test_streams(self):
|
||||
# test info
|
||||
sha = NULL_BIN_SHA
|
||||
s = 20
|
||||
blob_id = 3
|
||||
|
||||
info = OInfo(sha, str_blob_type, s)
|
||||
assert info.binsha == sha
|
||||
assert info.type == str_blob_type
|
||||
assert info.type_id == blob_id
|
||||
assert info.size == s
|
||||
|
||||
# test pack info
|
||||
# provides type_id
|
||||
pinfo = OPackInfo(0, blob_id, s)
|
||||
assert pinfo.type == str_blob_type
|
||||
assert pinfo.type_id == blob_id
|
||||
assert pinfo.pack_offset == 0
|
||||
|
||||
dpinfo = ODeltaPackInfo(0, blob_id, s, sha)
|
||||
assert dpinfo.type == str_blob_type
|
||||
assert dpinfo.type_id == blob_id
|
||||
assert dpinfo.delta_info == sha
|
||||
assert dpinfo.pack_offset == 0
|
||||
|
||||
# test ostream
|
||||
stream = DummyStream()
|
||||
ostream = OStream(*(info + (stream, )))
|
||||
assert ostream.stream is stream
|
||||
ostream.read(15)
|
||||
stream._assert()
|
||||
assert stream.bytes == 15
|
||||
ostream.read(20)
|
||||
assert stream.bytes == 20
|
||||
|
||||
# test packstream
|
||||
postream = OPackStream(*(pinfo + (stream, )))
|
||||
assert postream.stream is stream
|
||||
postream.read(10)
|
||||
stream._assert()
|
||||
assert stream.bytes == 10
|
||||
|
||||
# test deltapackstream
|
||||
dpostream = ODeltaPackStream(*(dpinfo + (stream, )))
|
||||
dpostream.stream is stream
|
||||
dpostream.read(5)
|
||||
stream._assert()
|
||||
assert stream.bytes == 5
|
||||
|
||||
# derive with own args
|
||||
DeriveTest(sha, str_blob_type, s, stream, 'mine', myarg=3)._assert()
|
||||
|
||||
# test istream
|
||||
istream = IStream(str_blob_type, s, stream)
|
||||
assert istream.binsha == None
|
||||
istream.binsha = sha
|
||||
assert istream.binsha == sha
|
||||
|
||||
assert len(istream.binsha) == 20
|
||||
assert len(istream.hexsha) == 40
|
||||
|
||||
assert istream.size == s
|
||||
istream.size = s * 2
|
||||
istream.size == s * 2
|
||||
assert istream.type == str_blob_type
|
||||
istream.type = "something"
|
||||
assert istream.type == "something"
|
||||
assert istream.stream is stream
|
||||
istream.stream = None
|
||||
assert istream.stream is None
|
||||
|
||||
assert istream.error is None
|
||||
istream.error = Exception()
|
||||
assert isinstance(istream.error, Exception)
|
||||
@@ -0,0 +1,43 @@
|
||||
# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors
|
||||
#
|
||||
# This module is part of GitDB and is released under
|
||||
# the New BSD License: http://www.opensource.org/licenses/bsd-license.php
|
||||
"""Module with examples from the tutorial section of the docs"""
|
||||
import os
|
||||
from gitdb.test.lib import TestBase
|
||||
from gitdb import IStream
|
||||
from gitdb.db import LooseObjectDB
|
||||
|
||||
from io import BytesIO
|
||||
|
||||
|
||||
class TestExamples(TestBase):
|
||||
|
||||
def test_base(self):
|
||||
ldb = LooseObjectDB(os.path.join(self.gitrepopath, 'objects'))
|
||||
|
||||
for sha1 in ldb.sha_iter():
|
||||
oinfo = ldb.info(sha1)
|
||||
ostream = ldb.stream(sha1)
|
||||
assert oinfo[:3] == ostream[:3]
|
||||
|
||||
assert len(ostream.read()) == ostream.size
|
||||
assert ldb.has_object(oinfo.binsha)
|
||||
# END for each sha in database
|
||||
# assure we close all files
|
||||
try:
|
||||
del(ostream)
|
||||
del(oinfo)
|
||||
except UnboundLocalError:
|
||||
pass
|
||||
# END ignore exception if there are no loose objects
|
||||
|
||||
data = b"my data"
|
||||
istream = IStream("blob", len(data), BytesIO(data))
|
||||
|
||||
# the object does not yet have a sha
|
||||
assert istream.binsha is None
|
||||
ldb.store(istream)
|
||||
# now the sha is set
|
||||
assert len(istream.binsha) == 20
|
||||
assert ldb.has_object(istream.binsha)
|
||||
@@ -0,0 +1,249 @@
|
||||
# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors
|
||||
#
|
||||
# This module is part of GitDB and is released under
|
||||
# the New BSD License: http://www.opensource.org/licenses/bsd-license.php
|
||||
"""Test everything about packs reading and writing"""
|
||||
from gitdb.test.lib import (
|
||||
TestBase,
|
||||
with_rw_directory,
|
||||
fixture_path
|
||||
)
|
||||
|
||||
from gitdb.stream import DeltaApplyReader
|
||||
|
||||
from gitdb.pack import (
|
||||
PackEntity,
|
||||
PackIndexFile,
|
||||
PackFile
|
||||
)
|
||||
|
||||
from gitdb.base import (
|
||||
OInfo,
|
||||
OStream,
|
||||
)
|
||||
|
||||
from gitdb.fun import delta_types
|
||||
from gitdb.exc import UnsupportedOperation
|
||||
from gitdb.util import to_bin_sha
|
||||
|
||||
import pytest
|
||||
|
||||
import os
|
||||
import tempfile
|
||||
|
||||
|
||||
#{ Utilities
|
||||
def bin_sha_from_filename(filename):
|
||||
return to_bin_sha(os.path.splitext(os.path.basename(filename))[0][5:])
|
||||
#} END utilities
|
||||
|
||||
|
||||
class TestPack(TestBase):
|
||||
|
||||
packindexfile_v1 = (fixture_path('packs/pack-c0438c19fb16422b6bbcce24387b3264416d485b.idx'), 1, 67)
|
||||
packindexfile_v2 = (fixture_path('packs/pack-11fdfa9e156ab73caae3b6da867192221f2089c2.idx'), 2, 30)
|
||||
packindexfile_v2_3_ascii = (fixture_path('packs/pack-a2bf8e71d8c18879e499335762dd95119d93d9f1.idx'), 2, 42)
|
||||
packfile_v2_1 = (fixture_path('packs/pack-c0438c19fb16422b6bbcce24387b3264416d485b.pack'), 2, packindexfile_v1[2])
|
||||
packfile_v2_2 = (fixture_path('packs/pack-11fdfa9e156ab73caae3b6da867192221f2089c2.pack'), 2, packindexfile_v2[2])
|
||||
packfile_v2_3_ascii = (
|
||||
fixture_path('packs/pack-a2bf8e71d8c18879e499335762dd95119d93d9f1.pack'), 2, packindexfile_v2_3_ascii[2])
|
||||
|
||||
def _assert_index_file(self, index, version, size):
|
||||
assert index.packfile_checksum() != index.indexfile_checksum()
|
||||
assert len(index.packfile_checksum()) == 20
|
||||
assert len(index.indexfile_checksum()) == 20
|
||||
assert index.version() == version
|
||||
assert index.size() == size
|
||||
assert len(index.offsets()) == size
|
||||
|
||||
# get all data of all objects
|
||||
for oidx in range(index.size()):
|
||||
sha = index.sha(oidx)
|
||||
assert oidx == index.sha_to_index(sha)
|
||||
|
||||
entry = index.entry(oidx)
|
||||
assert len(entry) == 3
|
||||
|
||||
assert entry[0] == index.offset(oidx)
|
||||
assert entry[1] == sha
|
||||
assert entry[2] == index.crc(oidx)
|
||||
|
||||
# verify partial sha
|
||||
for l in (4, 8, 11, 17, 20):
|
||||
assert index.partial_sha_to_index(sha[:l], l * 2) == oidx
|
||||
|
||||
# END for each object index in indexfile
|
||||
self.assertRaises(ValueError, index.partial_sha_to_index, "\0", 2)
|
||||
|
||||
def _assert_pack_file(self, pack, version, size):
|
||||
assert pack.version() == 2
|
||||
assert pack.size() == size
|
||||
assert len(pack.checksum()) == 20
|
||||
|
||||
num_obj = 0
|
||||
for obj in pack.stream_iter():
|
||||
num_obj += 1
|
||||
info = pack.info(obj.pack_offset)
|
||||
stream = pack.stream(obj.pack_offset)
|
||||
|
||||
assert info.pack_offset == stream.pack_offset
|
||||
assert info.type_id == stream.type_id
|
||||
assert hasattr(stream, 'read')
|
||||
|
||||
# it should be possible to read from both streams
|
||||
assert obj.read() == stream.read()
|
||||
|
||||
streams = pack.collect_streams(obj.pack_offset)
|
||||
assert streams
|
||||
|
||||
# read the stream
|
||||
try:
|
||||
dstream = DeltaApplyReader.new(streams)
|
||||
except ValueError:
|
||||
# ignore these, old git versions use only ref deltas,
|
||||
# which we haven't resolved ( as we are without an index )
|
||||
# Also ignore non-delta streams
|
||||
continue
|
||||
# END get deltastream
|
||||
|
||||
# read all
|
||||
data = dstream.read()
|
||||
assert len(data) == dstream.size
|
||||
|
||||
# test seek
|
||||
dstream.seek(0)
|
||||
assert dstream.read() == data
|
||||
|
||||
# read chunks
|
||||
# NOTE: the current implementation is safe, it basically transfers
|
||||
# all calls to the underlying memory map
|
||||
|
||||
# END for each object
|
||||
assert num_obj == size
|
||||
|
||||
def test_pack_index(self):
|
||||
# check version 1 and 2
|
||||
for indexfile, version, size in (self.packindexfile_v1, self.packindexfile_v2):
|
||||
index = PackIndexFile(indexfile)
|
||||
self._assert_index_file(index, version, size)
|
||||
# END run tests
|
||||
|
||||
def test_pack(self):
|
||||
# there is this special version 3, but apparently its like 2 ...
|
||||
for packfile, version, size in (self.packfile_v2_3_ascii, self.packfile_v2_1, self.packfile_v2_2):
|
||||
pack = PackFile(packfile)
|
||||
self._assert_pack_file(pack, version, size)
|
||||
# END for each pack to test
|
||||
|
||||
@with_rw_directory
|
||||
def test_pack_entity(self, rw_dir):
|
||||
pack_objs = list()
|
||||
for packinfo, indexinfo in ((self.packfile_v2_1, self.packindexfile_v1),
|
||||
(self.packfile_v2_2, self.packindexfile_v2),
|
||||
(self.packfile_v2_3_ascii, self.packindexfile_v2_3_ascii)):
|
||||
packfile, version, size = packinfo
|
||||
indexfile, version, size = indexinfo
|
||||
entity = PackEntity(packfile)
|
||||
assert entity.pack().path() == packfile
|
||||
assert entity.index().path() == indexfile
|
||||
pack_objs.extend(entity.stream_iter())
|
||||
|
||||
count = 0
|
||||
for info, stream in zip(entity.info_iter(), entity.stream_iter()):
|
||||
count += 1
|
||||
assert info.binsha == stream.binsha
|
||||
assert len(info.binsha) == 20
|
||||
assert info.type_id == stream.type_id
|
||||
assert info.size == stream.size
|
||||
|
||||
# we return fully resolved items, which is implied by the sha centric access
|
||||
assert not info.type_id in delta_types
|
||||
|
||||
# try all calls
|
||||
assert len(entity.collect_streams(info.binsha))
|
||||
oinfo = entity.info(info.binsha)
|
||||
assert isinstance(oinfo, OInfo)
|
||||
assert oinfo.binsha is not None
|
||||
ostream = entity.stream(info.binsha)
|
||||
assert isinstance(ostream, OStream)
|
||||
assert ostream.binsha is not None
|
||||
|
||||
# verify the stream
|
||||
try:
|
||||
assert entity.is_valid_stream(info.binsha, use_crc=True)
|
||||
except UnsupportedOperation:
|
||||
pass
|
||||
# END ignore version issues
|
||||
assert entity.is_valid_stream(info.binsha, use_crc=False)
|
||||
# END for each info, stream tuple
|
||||
assert count == size
|
||||
|
||||
# END for each entity
|
||||
|
||||
# pack writing - write all packs into one
|
||||
# index path can be None
|
||||
pack_path1 = tempfile.mktemp('', "pack1", rw_dir)
|
||||
pack_path2 = tempfile.mktemp('', "pack2", rw_dir)
|
||||
index_path = tempfile.mktemp('', 'index', rw_dir)
|
||||
iteration = 0
|
||||
|
||||
def rewind_streams():
|
||||
for obj in pack_objs:
|
||||
obj.stream.seek(0)
|
||||
# END utility
|
||||
for ppath, ipath, num_obj in zip((pack_path1, pack_path2),
|
||||
(index_path, None),
|
||||
(len(pack_objs), None)):
|
||||
iwrite = None
|
||||
if ipath:
|
||||
ifile = open(ipath, 'wb')
|
||||
iwrite = ifile.write
|
||||
# END handle ip
|
||||
|
||||
# make sure we rewind the streams ... we work on the same objects over and over again
|
||||
if iteration > 0:
|
||||
rewind_streams()
|
||||
# END rewind streams
|
||||
iteration += 1
|
||||
|
||||
with open(ppath, 'wb') as pfile:
|
||||
pack_sha, index_sha = PackEntity.write_pack(pack_objs, pfile.write, iwrite, object_count=num_obj)
|
||||
assert os.path.getsize(ppath) > 100
|
||||
|
||||
# verify pack
|
||||
pf = PackFile(ppath)
|
||||
assert pf.size() == len(pack_objs)
|
||||
assert pf.version() == PackFile.pack_version_default
|
||||
assert pf.checksum() == pack_sha
|
||||
pf.close()
|
||||
|
||||
# verify index
|
||||
if ipath is not None:
|
||||
ifile.close()
|
||||
assert os.path.getsize(ipath) > 100
|
||||
idx = PackIndexFile(ipath)
|
||||
assert idx.version() == PackIndexFile.index_version_default
|
||||
assert idx.packfile_checksum() == pack_sha
|
||||
assert idx.indexfile_checksum() == index_sha
|
||||
assert idx.size() == len(pack_objs)
|
||||
idx.close()
|
||||
# END verify files exist
|
||||
# END for each packpath, indexpath pair
|
||||
|
||||
# verify the packs thoroughly
|
||||
rewind_streams()
|
||||
entity = PackEntity.create(pack_objs, rw_dir)
|
||||
count = 0
|
||||
for info in entity.info_iter():
|
||||
count += 1
|
||||
for use_crc in range(2):
|
||||
assert entity.is_valid_stream(info.binsha, use_crc)
|
||||
# END for each crc mode
|
||||
# END for each info
|
||||
assert count == len(pack_objs)
|
||||
entity.close()
|
||||
|
||||
def test_pack_64(self):
|
||||
# TODO: hex-edit a pack helping us to verify that we can handle 64 byte offsets
|
||||
# of course without really needing such a huge pack
|
||||
pytest.skip('not implemented')
|
||||
@@ -0,0 +1,164 @@
|
||||
# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors
|
||||
#
|
||||
# This module is part of GitDB and is released under
|
||||
# the New BSD License: http://www.opensource.org/licenses/bsd-license.php
|
||||
"""Test for object db"""
|
||||
|
||||
from gitdb.test.lib import (
|
||||
TestBase,
|
||||
DummyStream,
|
||||
make_bytes,
|
||||
make_object,
|
||||
fixture_path
|
||||
)
|
||||
|
||||
from gitdb import (
|
||||
DecompressMemMapReader,
|
||||
FDCompressedSha1Writer,
|
||||
LooseObjectDB,
|
||||
Sha1Writer,
|
||||
MemoryDB,
|
||||
IStream,
|
||||
)
|
||||
from gitdb.util import hex_to_bin
|
||||
|
||||
import zlib
|
||||
from gitdb.typ import (
|
||||
str_blob_type
|
||||
)
|
||||
|
||||
import tempfile
|
||||
import os
|
||||
from io import BytesIO
|
||||
|
||||
|
||||
class TestStream(TestBase):
|
||||
|
||||
"""Test stream classes"""
|
||||
|
||||
data_sizes = (15, 10000, 1000 * 1024 + 512)
|
||||
|
||||
def _assert_stream_reader(self, stream, cdata, rewind_stream=lambda s: None):
|
||||
"""Make stream tests - the orig_stream is seekable, allowing it to be
|
||||
rewound and reused
|
||||
:param cdata: the data we expect to read from stream, the contents
|
||||
:param rewind_stream: function called to rewind the stream to make it ready
|
||||
for reuse"""
|
||||
ns = 10
|
||||
assert len(cdata) > ns - 1, "Data must be larger than %i, was %i" % (ns, len(cdata))
|
||||
|
||||
# read in small steps
|
||||
ss = len(cdata) // ns
|
||||
for i in range(ns):
|
||||
data = stream.read(ss)
|
||||
chunk = cdata[i * ss:(i + 1) * ss]
|
||||
assert data == chunk
|
||||
# END for each step
|
||||
rest = stream.read()
|
||||
if rest:
|
||||
assert rest == cdata[-len(rest):]
|
||||
# END handle rest
|
||||
|
||||
if isinstance(stream, DecompressMemMapReader):
|
||||
assert len(stream.data()) == stream.compressed_bytes_read()
|
||||
# END handle special type
|
||||
|
||||
rewind_stream(stream)
|
||||
|
||||
# read everything
|
||||
rdata = stream.read()
|
||||
assert rdata == cdata
|
||||
|
||||
if isinstance(stream, DecompressMemMapReader):
|
||||
assert len(stream.data()) == stream.compressed_bytes_read()
|
||||
# END handle special type
|
||||
|
||||
def test_decompress_reader(self):
|
||||
for close_on_deletion in range(2):
|
||||
for with_size in range(2):
|
||||
for ds in self.data_sizes:
|
||||
cdata = make_bytes(ds, randomize=False)
|
||||
|
||||
# zdata = zipped actual data
|
||||
# cdata = original content data
|
||||
|
||||
# create reader
|
||||
if with_size:
|
||||
# need object data
|
||||
zdata = zlib.compress(make_object(str_blob_type, cdata))
|
||||
typ, size, reader = DecompressMemMapReader.new(zdata, close_on_deletion)
|
||||
assert size == len(cdata)
|
||||
assert typ == str_blob_type
|
||||
|
||||
# even if we don't set the size, it will be set automatically on first read
|
||||
test_reader = DecompressMemMapReader(zdata, close_on_deletion=False)
|
||||
assert test_reader._s == len(cdata)
|
||||
else:
|
||||
# here we need content data
|
||||
zdata = zlib.compress(cdata)
|
||||
reader = DecompressMemMapReader(zdata, close_on_deletion, len(cdata))
|
||||
assert reader._s == len(cdata)
|
||||
# END get reader
|
||||
|
||||
self._assert_stream_reader(reader, cdata, lambda r: r.seek(0))
|
||||
|
||||
# put in a dummy stream for closing
|
||||
dummy = DummyStream()
|
||||
reader._m = dummy
|
||||
|
||||
assert not dummy.closed
|
||||
del(reader)
|
||||
assert dummy.closed == close_on_deletion
|
||||
# END for each datasize
|
||||
# END whether size should be used
|
||||
# END whether stream should be closed when deleted
|
||||
|
||||
def test_sha_writer(self):
|
||||
writer = Sha1Writer()
|
||||
assert 2 == writer.write(b"hi")
|
||||
assert len(writer.sha(as_hex=1)) == 40
|
||||
assert len(writer.sha(as_hex=0)) == 20
|
||||
|
||||
# make sure it does something ;)
|
||||
prev_sha = writer.sha()
|
||||
writer.write(b"hi again")
|
||||
assert writer.sha() != prev_sha
|
||||
|
||||
def test_compressed_writer(self):
|
||||
for ds in self.data_sizes:
|
||||
fd, path = tempfile.mkstemp()
|
||||
ostream = FDCompressedSha1Writer(fd)
|
||||
data = make_bytes(ds, randomize=False)
|
||||
|
||||
# for now, just a single write, code doesn't care about chunking
|
||||
assert len(data) == ostream.write(data)
|
||||
ostream.close()
|
||||
|
||||
# its closed already
|
||||
self.assertRaises(OSError, os.close, fd)
|
||||
|
||||
# read everything back, compare to data we zip
|
||||
fd = os.open(path, os.O_RDONLY | getattr(os, 'O_BINARY', 0))
|
||||
written_data = os.read(fd, os.path.getsize(path))
|
||||
assert len(written_data) == os.path.getsize(path)
|
||||
os.close(fd)
|
||||
assert written_data == zlib.compress(data, 1) # best speed
|
||||
|
||||
os.remove(path)
|
||||
# END for each os
|
||||
|
||||
def test_decompress_reader_special_case(self):
|
||||
odb = LooseObjectDB(fixture_path('objects'))
|
||||
mdb = MemoryDB()
|
||||
for sha in (b'888401851f15db0eed60eb1bc29dec5ddcace911',
|
||||
b'7bb839852ed5e3a069966281bb08d50012fb309b',):
|
||||
ostream = odb.stream(hex_to_bin(sha))
|
||||
|
||||
# if there is a bug, we will be missing one byte exactly !
|
||||
data = ostream.read()
|
||||
assert len(data) == ostream.size
|
||||
|
||||
# Putting it back in should yield nothing new - after all, we have
|
||||
dump = mdb.store(IStream(ostream.type, ostream.size, BytesIO(data)))
|
||||
assert dump.hexsha == sha
|
||||
# end for each loose object sha to test
|
||||
@@ -0,0 +1,100 @@
|
||||
# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors
|
||||
#
|
||||
# This module is part of GitDB and is released under
|
||||
# the New BSD License: http://www.opensource.org/licenses/bsd-license.php
|
||||
"""Test for object db"""
|
||||
import tempfile
|
||||
import os
|
||||
|
||||
from gitdb.test.lib import TestBase
|
||||
from gitdb.util import (
|
||||
to_hex_sha,
|
||||
to_bin_sha,
|
||||
NULL_HEX_SHA,
|
||||
LockedFD
|
||||
)
|
||||
|
||||
|
||||
class TestUtils(TestBase):
|
||||
|
||||
def test_basics(self):
|
||||
assert to_hex_sha(NULL_HEX_SHA) == NULL_HEX_SHA
|
||||
assert len(to_bin_sha(NULL_HEX_SHA)) == 20
|
||||
assert to_hex_sha(to_bin_sha(NULL_HEX_SHA)) == NULL_HEX_SHA.encode("ascii")
|
||||
|
||||
def _cmp_contents(self, file_path, data):
|
||||
# raise if data from file at file_path
|
||||
# does not match data string
|
||||
with open(file_path, "rb") as fp:
|
||||
assert fp.read() == data.encode("ascii")
|
||||
|
||||
def test_lockedfd(self):
|
||||
my_file = tempfile.mktemp()
|
||||
orig_data = "hello"
|
||||
new_data = "world"
|
||||
with open(my_file, "wb") as my_file_fp:
|
||||
my_file_fp.write(orig_data.encode("ascii"))
|
||||
|
||||
try:
|
||||
lfd = LockedFD(my_file)
|
||||
lockfilepath = lfd._lockfilepath()
|
||||
|
||||
# cannot end before it was started
|
||||
self.assertRaises(AssertionError, lfd.rollback)
|
||||
self.assertRaises(AssertionError, lfd.commit)
|
||||
|
||||
# open for writing
|
||||
assert not os.path.isfile(lockfilepath)
|
||||
wfd = lfd.open(write=True)
|
||||
assert lfd._fd is wfd
|
||||
assert os.path.isfile(lockfilepath)
|
||||
|
||||
# write data and fail
|
||||
os.write(wfd, new_data.encode("ascii"))
|
||||
lfd.rollback()
|
||||
assert lfd._fd is None
|
||||
self._cmp_contents(my_file, orig_data)
|
||||
assert not os.path.isfile(lockfilepath)
|
||||
|
||||
# additional call doesn't fail
|
||||
lfd.commit()
|
||||
lfd.rollback()
|
||||
|
||||
# test reading
|
||||
lfd = LockedFD(my_file)
|
||||
rfd = lfd.open(write=False)
|
||||
assert os.read(rfd, len(orig_data)) == orig_data.encode("ascii")
|
||||
|
||||
assert os.path.isfile(lockfilepath)
|
||||
# deletion rolls back
|
||||
del(lfd)
|
||||
assert not os.path.isfile(lockfilepath)
|
||||
|
||||
# write data - concurrently
|
||||
lfd = LockedFD(my_file)
|
||||
olfd = LockedFD(my_file)
|
||||
assert not os.path.isfile(lockfilepath)
|
||||
wfdstream = lfd.open(write=True, stream=True) # this time as stream
|
||||
assert os.path.isfile(lockfilepath)
|
||||
# another one fails
|
||||
self.assertRaises(IOError, olfd.open)
|
||||
|
||||
wfdstream.write(new_data.encode("ascii"))
|
||||
lfd.commit()
|
||||
assert not os.path.isfile(lockfilepath)
|
||||
self._cmp_contents(my_file, new_data)
|
||||
|
||||
# could test automatic _end_writing on destruction
|
||||
finally:
|
||||
os.remove(my_file)
|
||||
# END final cleanup
|
||||
|
||||
# try non-existing file for reading
|
||||
lfd = LockedFD(tempfile.mktemp())
|
||||
try:
|
||||
lfd.open(write=False)
|
||||
except OSError:
|
||||
assert not os.path.exists(lfd._lockfilepath())
|
||||
else:
|
||||
self.fail("expected OSError")
|
||||
# END handle exceptions
|
||||
10
zero-cost-nas/.eggs/gitdb-4.0.10-py3.8.egg/gitdb/typ.py
Normal file
10
zero-cost-nas/.eggs/gitdb-4.0.10-py3.8.egg/gitdb/typ.py
Normal file
@@ -0,0 +1,10 @@
|
||||
# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors
|
||||
#
|
||||
# This module is part of GitDB and is released under
|
||||
# the New BSD License: http://www.opensource.org/licenses/bsd-license.php
|
||||
"""Module containing information about types known to the database"""
|
||||
|
||||
str_blob_type = b'blob'
|
||||
str_commit_type = b'commit'
|
||||
str_tree_type = b'tree'
|
||||
str_tag_type = b'tag'
|
||||
398
zero-cost-nas/.eggs/gitdb-4.0.10-py3.8.egg/gitdb/util.py
Normal file
398
zero-cost-nas/.eggs/gitdb-4.0.10-py3.8.egg/gitdb/util.py
Normal file
@@ -0,0 +1,398 @@
|
||||
# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors
|
||||
#
|
||||
# This module is part of GitDB and is released under
|
||||
# the New BSD License: http://www.opensource.org/licenses/bsd-license.php
|
||||
import binascii
|
||||
import os
|
||||
import mmap
|
||||
import sys
|
||||
import time
|
||||
import errno
|
||||
|
||||
from io import BytesIO
|
||||
|
||||
from smmap import (
|
||||
StaticWindowMapManager,
|
||||
SlidingWindowMapManager,
|
||||
SlidingWindowMapBuffer
|
||||
)
|
||||
|
||||
# initialize our global memory manager instance
|
||||
# Use it to free cached (and unused) resources.
|
||||
mman = SlidingWindowMapManager()
|
||||
# END handle mman
|
||||
|
||||
import hashlib
|
||||
|
||||
try:
|
||||
from struct import unpack_from
|
||||
except ImportError:
|
||||
from struct import unpack, calcsize
|
||||
__calcsize_cache = dict()
|
||||
|
||||
def unpack_from(fmt, data, offset=0):
|
||||
try:
|
||||
size = __calcsize_cache[fmt]
|
||||
except KeyError:
|
||||
size = calcsize(fmt)
|
||||
__calcsize_cache[fmt] = size
|
||||
# END exception handling
|
||||
return unpack(fmt, data[offset: offset + size])
|
||||
# END own unpack_from implementation
|
||||
|
||||
|
||||
#{ Aliases
|
||||
|
||||
hex_to_bin = binascii.a2b_hex
|
||||
bin_to_hex = binascii.b2a_hex
|
||||
|
||||
# errors
|
||||
ENOENT = errno.ENOENT
|
||||
|
||||
# os shortcuts
|
||||
exists = os.path.exists
|
||||
mkdir = os.mkdir
|
||||
chmod = os.chmod
|
||||
isdir = os.path.isdir
|
||||
isfile = os.path.isfile
|
||||
rename = os.rename
|
||||
dirname = os.path.dirname
|
||||
basename = os.path.basename
|
||||
join = os.path.join
|
||||
read = os.read
|
||||
write = os.write
|
||||
close = os.close
|
||||
fsync = os.fsync
|
||||
|
||||
|
||||
def _retry(func, *args, **kwargs):
|
||||
# Wrapper around functions, that are problematic on "Windows". Sometimes
|
||||
# the OS or someone else has still a handle to the file
|
||||
if sys.platform == "win32":
|
||||
for _ in range(10):
|
||||
try:
|
||||
return func(*args, **kwargs)
|
||||
except Exception:
|
||||
time.sleep(0.1)
|
||||
return func(*args, **kwargs)
|
||||
else:
|
||||
return func(*args, **kwargs)
|
||||
|
||||
|
||||
def remove(*args, **kwargs):
|
||||
return _retry(os.remove, *args, **kwargs)
|
||||
|
||||
|
||||
# Backwards compatibility imports
|
||||
from gitdb.const import (
|
||||
NULL_BIN_SHA,
|
||||
NULL_HEX_SHA
|
||||
)
|
||||
|
||||
#} END Aliases
|
||||
|
||||
#{ compatibility stuff ...
|
||||
|
||||
|
||||
class _RandomAccessBytesIO:
|
||||
|
||||
"""Wrapper to provide required functionality in case memory maps cannot or may
|
||||
not be used. This is only really required in python 2.4"""
|
||||
__slots__ = '_sio'
|
||||
|
||||
def __init__(self, buf=''):
|
||||
self._sio = BytesIO(buf)
|
||||
|
||||
def __getattr__(self, attr):
|
||||
return getattr(self._sio, attr)
|
||||
|
||||
def __len__(self):
|
||||
return len(self.getvalue())
|
||||
|
||||
def __getitem__(self, i):
|
||||
return self.getvalue()[i]
|
||||
|
||||
def __getslice__(self, start, end):
|
||||
return self.getvalue()[start:end]
|
||||
|
||||
|
||||
def byte_ord(b):
|
||||
"""
|
||||
Return the integer representation of the byte string. This supports Python
|
||||
3 byte arrays as well as standard strings.
|
||||
"""
|
||||
try:
|
||||
return ord(b)
|
||||
except TypeError:
|
||||
return b
|
||||
|
||||
#} END compatibility stuff ...
|
||||
|
||||
#{ Routines
|
||||
|
||||
|
||||
def make_sha(source=b''):
|
||||
"""A python2.4 workaround for the sha/hashlib module fiasco
|
||||
|
||||
**Note** From the dulwich project """
|
||||
try:
|
||||
return hashlib.sha1(source)
|
||||
except NameError:
|
||||
import sha
|
||||
sha1 = sha.sha(source)
|
||||
return sha1
|
||||
|
||||
|
||||
def allocate_memory(size):
|
||||
""":return: a file-protocol accessible memory block of the given size"""
|
||||
if size == 0:
|
||||
return _RandomAccessBytesIO(b'')
|
||||
# END handle empty chunks gracefully
|
||||
|
||||
try:
|
||||
return mmap.mmap(-1, size) # read-write by default
|
||||
except OSError:
|
||||
# setup real memory instead
|
||||
# this of course may fail if the amount of memory is not available in
|
||||
# one chunk - would only be the case in python 2.4, being more likely on
|
||||
# 32 bit systems.
|
||||
return _RandomAccessBytesIO(b"\0" * size)
|
||||
# END handle memory allocation
|
||||
|
||||
|
||||
def file_contents_ro(fd, stream=False, allow_mmap=True):
|
||||
""":return: read-only contents of the file represented by the file descriptor fd
|
||||
|
||||
:param fd: file descriptor opened for reading
|
||||
:param stream: if False, random access is provided, otherwise the stream interface
|
||||
is provided.
|
||||
:param allow_mmap: if True, its allowed to map the contents into memory, which
|
||||
allows large files to be handled and accessed efficiently. The file-descriptor
|
||||
will change its position if this is False"""
|
||||
try:
|
||||
if allow_mmap:
|
||||
# supports stream and random access
|
||||
try:
|
||||
return mmap.mmap(fd, 0, access=mmap.ACCESS_READ)
|
||||
except OSError:
|
||||
# python 2.4 issue, 0 wants to be the actual size
|
||||
return mmap.mmap(fd, os.fstat(fd).st_size, access=mmap.ACCESS_READ)
|
||||
# END handle python 2.4
|
||||
except OSError:
|
||||
pass
|
||||
# END exception handling
|
||||
|
||||
# read manually
|
||||
contents = os.read(fd, os.fstat(fd).st_size)
|
||||
if stream:
|
||||
return _RandomAccessBytesIO(contents)
|
||||
return contents
|
||||
|
||||
|
||||
def file_contents_ro_filepath(filepath, stream=False, allow_mmap=True, flags=0):
|
||||
"""Get the file contents at filepath as fast as possible
|
||||
|
||||
:return: random access compatible memory of the given filepath
|
||||
:param stream: see ``file_contents_ro``
|
||||
:param allow_mmap: see ``file_contents_ro``
|
||||
:param flags: additional flags to pass to os.open
|
||||
:raise OSError: If the file could not be opened
|
||||
|
||||
**Note** for now we don't try to use O_NOATIME directly as the right value needs to be
|
||||
shared per database in fact. It only makes a real difference for loose object
|
||||
databases anyway, and they use it with the help of the ``flags`` parameter"""
|
||||
fd = os.open(filepath, os.O_RDONLY | getattr(os, 'O_BINARY', 0) | flags)
|
||||
try:
|
||||
return file_contents_ro(fd, stream, allow_mmap)
|
||||
finally:
|
||||
close(fd)
|
||||
# END assure file is closed
|
||||
|
||||
|
||||
def sliding_ro_buffer(filepath, flags=0):
|
||||
"""
|
||||
:return: a buffer compatible object which uses our mapped memory manager internally
|
||||
ready to read the whole given filepath"""
|
||||
return SlidingWindowMapBuffer(mman.make_cursor(filepath), flags=flags)
|
||||
|
||||
|
||||
def to_hex_sha(sha):
|
||||
""":return: hexified version of sha"""
|
||||
if len(sha) == 40:
|
||||
return sha
|
||||
return bin_to_hex(sha)
|
||||
|
||||
|
||||
def to_bin_sha(sha):
|
||||
if len(sha) == 20:
|
||||
return sha
|
||||
return hex_to_bin(sha)
|
||||
|
||||
|
||||
#} END routines
|
||||
|
||||
|
||||
#{ Utilities
|
||||
|
||||
class LazyMixin:
|
||||
|
||||
"""
|
||||
Base class providing an interface to lazily retrieve attribute values upon
|
||||
first access. If slots are used, memory will only be reserved once the attribute
|
||||
is actually accessed and retrieved the first time. All future accesses will
|
||||
return the cached value as stored in the Instance's dict or slot.
|
||||
"""
|
||||
|
||||
__slots__ = tuple()
|
||||
|
||||
def __getattr__(self, attr):
|
||||
"""
|
||||
Whenever an attribute is requested that we do not know, we allow it
|
||||
to be created and set. Next time the same attribute is requested, it is simply
|
||||
returned from our dict/slots. """
|
||||
self._set_cache_(attr)
|
||||
# will raise in case the cache was not created
|
||||
return object.__getattribute__(self, attr)
|
||||
|
||||
def _set_cache_(self, attr):
|
||||
"""
|
||||
This method should be overridden in the derived class.
|
||||
It should check whether the attribute named by attr can be created
|
||||
and cached. Do nothing if you do not know the attribute or call your subclass
|
||||
|
||||
The derived class may create as many additional attributes as it deems
|
||||
necessary in case a git command returns more information than represented
|
||||
in the single attribute."""
|
||||
pass
|
||||
|
||||
|
||||
class LockedFD:
|
||||
|
||||
"""
|
||||
This class facilitates a safe read and write operation to a file on disk.
|
||||
If we write to 'file', we obtain a lock file at 'file.lock' and write to
|
||||
that instead. If we succeed, the lock file will be renamed to overwrite
|
||||
the original file.
|
||||
|
||||
When reading, we obtain a lock file, but to prevent other writers from
|
||||
succeeding while we are reading the file.
|
||||
|
||||
This type handles error correctly in that it will assure a consistent state
|
||||
on destruction.
|
||||
|
||||
**note** with this setup, parallel reading is not possible"""
|
||||
__slots__ = ("_filepath", '_fd', '_write')
|
||||
|
||||
def __init__(self, filepath):
|
||||
"""Initialize an instance with the givne filepath"""
|
||||
self._filepath = filepath
|
||||
self._fd = None
|
||||
self._write = None # if True, we write a file
|
||||
|
||||
def __del__(self):
|
||||
# will do nothing if the file descriptor is already closed
|
||||
if self._fd is not None:
|
||||
self.rollback()
|
||||
|
||||
def _lockfilepath(self):
|
||||
return "%s.lock" % self._filepath
|
||||
|
||||
def open(self, write=False, stream=False):
|
||||
"""
|
||||
Open the file descriptor for reading or writing, both in binary mode.
|
||||
|
||||
:param write: if True, the file descriptor will be opened for writing. Other
|
||||
wise it will be opened read-only.
|
||||
:param stream: if True, the file descriptor will be wrapped into a simple stream
|
||||
object which supports only reading or writing
|
||||
:return: fd to read from or write to. It is still maintained by this instance
|
||||
and must not be closed directly
|
||||
:raise IOError: if the lock could not be retrieved
|
||||
:raise OSError: If the actual file could not be opened for reading
|
||||
|
||||
**note** must only be called once"""
|
||||
if self._write is not None:
|
||||
raise AssertionError("Called %s multiple times" % self.open)
|
||||
|
||||
self._write = write
|
||||
|
||||
# try to open the lock file
|
||||
binary = getattr(os, 'O_BINARY', 0)
|
||||
lockmode = os.O_WRONLY | os.O_CREAT | os.O_EXCL | binary
|
||||
try:
|
||||
fd = os.open(self._lockfilepath(), lockmode, int("600", 8))
|
||||
if not write:
|
||||
os.close(fd)
|
||||
else:
|
||||
self._fd = fd
|
||||
# END handle file descriptor
|
||||
except OSError as e:
|
||||
raise OSError("Lock at %r could not be obtained" % self._lockfilepath()) from e
|
||||
# END handle lock retrieval
|
||||
|
||||
# open actual file if required
|
||||
if self._fd is None:
|
||||
# we could specify exclusive here, as we obtained the lock anyway
|
||||
try:
|
||||
self._fd = os.open(self._filepath, os.O_RDONLY | binary)
|
||||
except:
|
||||
# assure we release our lockfile
|
||||
remove(self._lockfilepath())
|
||||
raise
|
||||
# END handle lockfile
|
||||
# END open descriptor for reading
|
||||
|
||||
if stream:
|
||||
# need delayed import
|
||||
from gitdb.stream import FDStream
|
||||
return FDStream(self._fd)
|
||||
else:
|
||||
return self._fd
|
||||
# END handle stream
|
||||
|
||||
def commit(self):
|
||||
"""When done writing, call this function to commit your changes into the
|
||||
actual file.
|
||||
The file descriptor will be closed, and the lockfile handled.
|
||||
|
||||
**Note** can be called multiple times"""
|
||||
self._end_writing(successful=True)
|
||||
|
||||
def rollback(self):
|
||||
"""Abort your operation without any changes. The file descriptor will be
|
||||
closed, and the lock released.
|
||||
|
||||
**Note** can be called multiple times"""
|
||||
self._end_writing(successful=False)
|
||||
|
||||
def _end_writing(self, successful=True):
|
||||
"""Handle the lock according to the write mode """
|
||||
if self._write is None:
|
||||
raise AssertionError("Cannot end operation if it wasn't started yet")
|
||||
|
||||
if self._fd is None:
|
||||
return
|
||||
|
||||
os.close(self._fd)
|
||||
self._fd = None
|
||||
|
||||
lockfile = self._lockfilepath()
|
||||
if self._write and successful:
|
||||
# on windows, rename does not silently overwrite the existing one
|
||||
if sys.platform == "win32":
|
||||
if isfile(self._filepath):
|
||||
remove(self._filepath)
|
||||
# END remove if exists
|
||||
# END win32 special handling
|
||||
os.rename(lockfile, self._filepath)
|
||||
|
||||
# assure others can at least read the file - the tmpfile left it at rw--
|
||||
# We may also write that file, on windows that boils down to a remove-
|
||||
# protection as well
|
||||
chmod(self._filepath, int("644", 8))
|
||||
else:
|
||||
# just delete the file so far, we failed
|
||||
remove(lockfile)
|
||||
# END successful handling
|
||||
|
||||
#} END utilities
|
||||
@@ -0,0 +1,18 @@
|
||||
def force_bytes(data, encoding="utf-8"):
|
||||
if isinstance(data, bytes):
|
||||
return data
|
||||
|
||||
if isinstance(data, str):
|
||||
return data.encode(encoding)
|
||||
|
||||
return data
|
||||
|
||||
|
||||
def force_text(data, encoding="utf-8"):
|
||||
if isinstance(data, str):
|
||||
return data
|
||||
|
||||
if isinstance(data, bytes):
|
||||
return data.decode(encoding)
|
||||
|
||||
return str(data, encoding)
|
||||
30
zero-cost-nas/.eggs/smmap-5.0.0-py3.8.egg/EGG-INFO/LICENSE
Normal file
30
zero-cost-nas/.eggs/smmap-5.0.0-py3.8.egg/EGG-INFO/LICENSE
Normal file
@@ -0,0 +1,30 @@
|
||||
Copyright (C) 2010, 2011 Sebastian Thiel and contributors
|
||||
All rights reserved.
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions
|
||||
are met:
|
||||
|
||||
* Redistributions of source code must retain the above copyright
|
||||
notice, this list of conditions and the following disclaimer.
|
||||
|
||||
* Redistributions in binary form must reproduce the above copyright
|
||||
notice, this list of conditions and the following disclaimer in the
|
||||
documentation and/or other materials provided with the distribution.
|
||||
|
||||
* Neither the name of the async project nor the names of
|
||||
its contributors may be used to endorse or promote products derived
|
||||
from this software without specific prior written permission.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
|
||||
TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
|
||||
PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
|
||||
LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
|
||||
NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
|
||||
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
113
zero-cost-nas/.eggs/smmap-5.0.0-py3.8.egg/EGG-INFO/PKG-INFO
Normal file
113
zero-cost-nas/.eggs/smmap-5.0.0-py3.8.egg/EGG-INFO/PKG-INFO
Normal file
@@ -0,0 +1,113 @@
|
||||
Metadata-Version: 2.1
|
||||
Name: smmap
|
||||
Version: 5.0.0
|
||||
Summary: A pure Python implementation of a sliding window memory map manager
|
||||
Home-page: https://github.com/gitpython-developers/smmap
|
||||
Author: Sebastian Thiel
|
||||
Author-email: byronimo@gmail.com
|
||||
License: BSD
|
||||
Platform: any
|
||||
Classifier: Development Status :: 5 - Production/Stable
|
||||
Classifier: Environment :: Console
|
||||
Classifier: Intended Audience :: Developers
|
||||
Classifier: License :: OSI Approved :: BSD License
|
||||
Classifier: Operating System :: OS Independent
|
||||
Classifier: Operating System :: POSIX
|
||||
Classifier: Operating System :: Microsoft :: Windows
|
||||
Classifier: Operating System :: MacOS :: MacOS X
|
||||
Classifier: Programming Language :: Python
|
||||
Classifier: Programming Language :: Python :: 3
|
||||
Classifier: Programming Language :: Python :: 3.6
|
||||
Classifier: Programming Language :: Python :: 3.7
|
||||
Classifier: Programming Language :: Python :: 3.8
|
||||
Classifier: Programming Language :: Python :: 3.9
|
||||
Classifier: Programming Language :: Python :: 3.10
|
||||
Classifier: Programming Language :: Python :: 3 :: Only
|
||||
Requires-Python: >=3.6
|
||||
Description-Content-Type: text/markdown
|
||||
|
||||
## Motivation
|
||||
|
||||
When reading from many possibly large files in a fashion similar to random access, it is usually the fastest and most efficient to use memory maps.
|
||||
|
||||
Although memory maps have many advantages, they represent a very limited system resource as every map uses one file descriptor, whose amount is limited per process. On 32 bit systems, the amount of memory you can have mapped at a time is naturally limited to theoretical 4GB of memory, which may not be enough for some applications.
|
||||
|
||||
|
||||
## Limitations
|
||||
|
||||
* **System resources (file-handles) are likely to be leaked!** This is due to the library authors reliance on a deterministic `__del__()` destructor.
|
||||
* The memory access is read-only by design.
|
||||
|
||||
|
||||
## Overview
|
||||
|
||||
|
||||
|
||||
Smmap wraps an interface around mmap and tracks the mapped files as well as the amount of clients who use it. If the system runs out of resources, or if a memory limit is reached, it will automatically unload unused maps to allow continued operation.
|
||||
|
||||
To allow processing large files even on 32 bit systems, it allows only portions of the file to be mapped. Once the user reads beyond the mapped region, smmap will automatically map the next required region, unloading unused regions using a LRU algorithm.
|
||||
|
||||
Although the library can be used most efficiently with its native interface, a Buffer implementation is provided to hide these details behind a simple string-like interface.
|
||||
|
||||
For performance critical 64 bit applications, a simplified version of memory mapping is provided which always maps the whole file, but still provides the benefit of unloading unused mappings on demand.
|
||||
|
||||
|
||||
|
||||
## Prerequisites
|
||||
|
||||
* Python 3.6+
|
||||
* OSX, Windows or Linux
|
||||
|
||||
The package was tested on all of the previously mentioned configurations.
|
||||
|
||||
## Installing smmap
|
||||
|
||||
[](https://readthedocs.org/projects/smmap/?badge=latest)
|
||||
|
||||
Its easiest to install smmap using the [pip](http://www.pip-installer.org/en/latest) program:
|
||||
|
||||
```bash
|
||||
$ pip install smmap
|
||||
```
|
||||
|
||||
As the command will install smmap in your respective python distribution, you will most likely need root permissions to authorize the required changes.
|
||||
|
||||
If you have downloaded the source archive, the package can be installed by running the `setup.py` script:
|
||||
|
||||
```bash
|
||||
$ python setup.py install
|
||||
```
|
||||
|
||||
It is advised to have a look at the **Usage Guide** for a brief introduction on the different database implementations.
|
||||
|
||||
|
||||
|
||||
## Homepage and Links
|
||||
|
||||
The project is home on github at https://github.com/gitpython-developers/smmap .
|
||||
|
||||
The latest source can be cloned from github as well:
|
||||
|
||||
* git://github.com/gitpython-developers/smmap.git
|
||||
|
||||
|
||||
For support, please use the git-python mailing list:
|
||||
|
||||
* http://groups.google.com/group/git-python
|
||||
|
||||
|
||||
Issues can be filed on github:
|
||||
|
||||
* https://github.com/gitpython-developers/smmap/issues
|
||||
|
||||
A link to the pypi page related to this repository:
|
||||
|
||||
* https://pypi.org/project/smmap/
|
||||
|
||||
|
||||
## License Information
|
||||
|
||||
*smmap* is licensed under the New BSD License.
|
||||
|
||||
|
||||
|
||||
16
zero-cost-nas/.eggs/smmap-5.0.0-py3.8.egg/EGG-INFO/RECORD
Normal file
16
zero-cost-nas/.eggs/smmap-5.0.0-py3.8.egg/EGG-INFO/RECORD
Normal file
@@ -0,0 +1,16 @@
|
||||
smmap/__init__.py,sha256=PcnjXprv7SB7WONeQ0qk93xgfBPi-by7_j0stBChsrU,342
|
||||
smmap/buf.py,sha256=CzvLJ93vVKqNTt09XXMvagMb9PBE0qOYaJA83e86T_g,5765
|
||||
smmap/mman.py,sha256=q2VUnBzTw46OndAenaU-vgjoNlR1d3itWktxu-dpAUQ,24021
|
||||
smmap/util.py,sha256=x40DONHh3VljRqCvSsrUHATt6UC1gbIWzsNkAlDfy6c,7486
|
||||
smmap/test/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
||||
smmap/test/lib.py,sha256=fsegoTch5fFXXd5MmJuL3pkrtM4gFQ5w2v2SpQaxmhg,1460
|
||||
smmap/test/test_buf.py,sha256=YN3fNuORHrV2kzv9EIZ0UBnljuy0f-ZgUk1KQZoYRe8,5439
|
||||
smmap/test/test_mman.py,sha256=hOvjf3yuCqplOkToUNoSpGy20qzxCLq6ebf_46xM9LU,10818
|
||||
smmap/test/test_tutorial.py,sha256=ZwCphCbKAGYt_fn_CiOJ9lWwpWwpqE4-zBUp-v-t9eM,3174
|
||||
smmap/test/test_util.py,sha256=3hJyW9Km7k7XSgxxtDOkG8eVagk3lIzP4H2pR0S2ewg,3468
|
||||
smmap-5.0.0.dist-info/LICENSE,sha256=iOnZP3CNEQsyioNDAt0dXGr72lMOdyHRXYCzUR2G8jU,1519
|
||||
smmap-5.0.0.dist-info/METADATA,sha256=Fk4ARflM-i58flLvrjIPhsFUEnC8Soaml51ObDXX380,4225
|
||||
smmap-5.0.0.dist-info/WHEEL,sha256=U88EhGIw8Sj2_phqajeu_EAi3RAo8-C6zV3REsWbWbs,92
|
||||
smmap-5.0.0.dist-info/top_level.txt,sha256=h0Tp1UdaROCy2bjWNha1MFpwgVghxEUQsaLHbZs96Gw,6
|
||||
smmap-5.0.0.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
|
||||
smmap-5.0.0.dist-info/RECORD,,
|
||||
5
zero-cost-nas/.eggs/smmap-5.0.0-py3.8.egg/EGG-INFO/WHEEL
Normal file
5
zero-cost-nas/.eggs/smmap-5.0.0-py3.8.egg/EGG-INFO/WHEEL
Normal file
@@ -0,0 +1,5 @@
|
||||
Wheel-Version: 1.0
|
||||
Generator: bdist_wheel (0.33.1)
|
||||
Root-Is-Purelib: true
|
||||
Tag: py3-none-any
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
smmap
|
||||
@@ -0,0 +1 @@
|
||||
|
||||
11
zero-cost-nas/.eggs/smmap-5.0.0-py3.8.egg/smmap/__init__.py
Normal file
11
zero-cost-nas/.eggs/smmap-5.0.0-py3.8.egg/smmap/__init__.py
Normal file
@@ -0,0 +1,11 @@
|
||||
"""Intialize the smmap package"""
|
||||
|
||||
__author__ = "Sebastian Thiel"
|
||||
__contact__ = "byronimo@gmail.com"
|
||||
__homepage__ = "https://github.com/gitpython-developers/smmap"
|
||||
version_info = (5, 0, 0)
|
||||
__version__ = '.'.join(str(i) for i in version_info)
|
||||
|
||||
# make everything available in root package for convenience
|
||||
from .mman import *
|
||||
from .buf import *
|
||||
143
zero-cost-nas/.eggs/smmap-5.0.0-py3.8.egg/smmap/buf.py
Normal file
143
zero-cost-nas/.eggs/smmap-5.0.0-py3.8.egg/smmap/buf.py
Normal file
@@ -0,0 +1,143 @@
|
||||
"""Module with a simple buffer implementation using the memory manager"""
|
||||
import sys
|
||||
|
||||
__all__ = ["SlidingWindowMapBuffer"]
|
||||
|
||||
|
||||
class SlidingWindowMapBuffer:
|
||||
|
||||
"""A buffer like object which allows direct byte-wise object and slicing into
|
||||
memory of a mapped file. The mapping is controlled by the provided cursor.
|
||||
|
||||
The buffer is relative, that is if you map an offset, index 0 will map to the
|
||||
first byte at the offset you used during initialization or begin_access
|
||||
|
||||
**Note:** Although this type effectively hides the fact that there are mapped windows
|
||||
underneath, it can unfortunately not be used in any non-pure python method which
|
||||
needs a buffer or string"""
|
||||
__slots__ = (
|
||||
'_c', # our cursor
|
||||
'_size', # our supposed size
|
||||
)
|
||||
|
||||
def __init__(self, cursor=None, offset=0, size=sys.maxsize, flags=0):
|
||||
"""Initalize the instance to operate on the given cursor.
|
||||
:param cursor: if not None, the associated cursor to the file you want to access
|
||||
If None, you have call begin_access before using the buffer and provide a cursor
|
||||
:param offset: absolute offset in bytes
|
||||
:param size: the total size of the mapping. Defaults to the maximum possible size
|
||||
From that point on, the __len__ of the buffer will be the given size or the file size.
|
||||
If the size is larger than the mappable area, you can only access the actually available
|
||||
area, although the length of the buffer is reported to be your given size.
|
||||
Hence it is in your own interest to provide a proper size !
|
||||
:param flags: Additional flags to be passed to os.open
|
||||
:raise ValueError: if the buffer could not achieve a valid state"""
|
||||
self._c = cursor
|
||||
if cursor and not self.begin_access(cursor, offset, size, flags):
|
||||
raise ValueError("Failed to allocate the buffer - probably the given offset is out of bounds")
|
||||
# END handle offset
|
||||
|
||||
def __del__(self):
|
||||
self.end_access()
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc_value, traceback):
|
||||
self.end_access()
|
||||
|
||||
def __len__(self):
|
||||
return self._size
|
||||
|
||||
def __getitem__(self, i):
|
||||
if isinstance(i, slice):
|
||||
return self.__getslice__(i.start or 0, i.stop or self._size)
|
||||
c = self._c
|
||||
assert c.is_valid()
|
||||
if i < 0:
|
||||
i = self._size + i
|
||||
if not c.includes_ofs(i):
|
||||
c.use_region(i, 1)
|
||||
# END handle region usage
|
||||
return c.buffer()[i - c.ofs_begin()]
|
||||
|
||||
def __getslice__(self, i, j):
|
||||
c = self._c
|
||||
# fast path, slice fully included - safes a concatenate operation and
|
||||
# should be the default
|
||||
assert c.is_valid()
|
||||
if i < 0:
|
||||
i = self._size + i
|
||||
if j == sys.maxsize:
|
||||
j = self._size
|
||||
if j < 0:
|
||||
j = self._size + j
|
||||
if (c.ofs_begin() <= i) and (j < c.ofs_end()):
|
||||
b = c.ofs_begin()
|
||||
return c.buffer()[i - b:j - b]
|
||||
else:
|
||||
l = j - i # total length
|
||||
ofs = i
|
||||
# It's fastest to keep tokens and join later, especially in py3, which was 7 times slower
|
||||
# in the previous iteration of this code
|
||||
md = list()
|
||||
while l:
|
||||
c.use_region(ofs, l)
|
||||
assert c.is_valid()
|
||||
d = c.buffer()[:l]
|
||||
ofs += len(d)
|
||||
l -= len(d)
|
||||
# Make sure we don't keep references, as c.use_region() might attempt to free resources, but
|
||||
# can't unless we use pure bytes
|
||||
if hasattr(d, 'tobytes'):
|
||||
d = d.tobytes()
|
||||
md.append(d)
|
||||
# END while there are bytes to read
|
||||
return bytes().join(md)
|
||||
# END fast or slow path
|
||||
#{ Interface
|
||||
|
||||
def begin_access(self, cursor=None, offset=0, size=sys.maxsize, flags=0):
|
||||
"""Call this before the first use of this instance. The method was already
|
||||
called by the constructor in case sufficient information was provided.
|
||||
|
||||
For more information no the parameters, see the __init__ method
|
||||
:param path: if cursor is None the existing one will be used.
|
||||
:return: True if the buffer can be used"""
|
||||
if cursor:
|
||||
self._c = cursor
|
||||
# END update our cursor
|
||||
|
||||
# reuse existing cursors if possible
|
||||
if self._c is not None and self._c.is_associated():
|
||||
res = self._c.use_region(offset, size, flags).is_valid()
|
||||
if res:
|
||||
# if given size is too large or default, we computer a proper size
|
||||
# If its smaller, we assume the combination between offset and size
|
||||
# as chosen by the user is correct and use it !
|
||||
# If not, the user is in trouble.
|
||||
if size > self._c.file_size():
|
||||
size = self._c.file_size() - offset
|
||||
# END handle size
|
||||
self._size = size
|
||||
# END set size
|
||||
return res
|
||||
# END use our cursor
|
||||
return False
|
||||
|
||||
def end_access(self):
|
||||
"""Call this method once you are done using the instance. It is automatically
|
||||
called on destruction, and should be called just in time to allow system
|
||||
resources to be freed.
|
||||
|
||||
Once you called end_access, you must call begin access before reusing this instance!"""
|
||||
self._size = 0
|
||||
if self._c is not None:
|
||||
self._c.unuse_region()
|
||||
# END unuse region
|
||||
|
||||
def cursor(self):
|
||||
""":return: the currently set cursor which provides access to the data"""
|
||||
return self._c
|
||||
|
||||
#}END interface
|
||||
588
zero-cost-nas/.eggs/smmap-5.0.0-py3.8.egg/smmap/mman.py
Normal file
588
zero-cost-nas/.eggs/smmap-5.0.0-py3.8.egg/smmap/mman.py
Normal file
@@ -0,0 +1,588 @@
|
||||
"""Module containing a memory memory manager which provides a sliding window on a number of memory mapped files"""
|
||||
from .util import (
|
||||
MapWindow,
|
||||
MapRegion,
|
||||
MapRegionList,
|
||||
is_64_bit,
|
||||
)
|
||||
|
||||
import sys
|
||||
from functools import reduce
|
||||
|
||||
__all__ = ["StaticWindowMapManager", "SlidingWindowMapManager", "WindowCursor"]
|
||||
#{ Utilities
|
||||
|
||||
#}END utilities
|
||||
|
||||
|
||||
class WindowCursor:
|
||||
|
||||
"""
|
||||
Pointer into the mapped region of the memory manager, keeping the map
|
||||
alive until it is destroyed and no other client uses it.
|
||||
|
||||
Cursors should not be created manually, but are instead returned by the SlidingWindowMapManager
|
||||
|
||||
**Note:**: The current implementation is suited for static and sliding window managers, but it also means
|
||||
that it must be suited for the somewhat quite different sliding manager. It could be improved, but
|
||||
I see no real need to do so."""
|
||||
__slots__ = (
|
||||
'_manager', # the manger keeping all file regions
|
||||
'_rlist', # a regions list with regions for our file
|
||||
'_region', # our current class:`MapRegion` or None
|
||||
'_ofs', # relative offset from the actually mapped area to our start area
|
||||
'_size' # maximum size we should provide
|
||||
)
|
||||
|
||||
def __init__(self, manager=None, regions=None):
|
||||
self._manager = manager
|
||||
self._rlist = regions
|
||||
self._region = None
|
||||
self._ofs = 0
|
||||
self._size = 0
|
||||
|
||||
def __del__(self):
|
||||
self._destroy()
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc_value, traceback):
|
||||
self._destroy()
|
||||
|
||||
def _destroy(self):
|
||||
"""Destruction code to decrement counters"""
|
||||
self.unuse_region()
|
||||
|
||||
if self._rlist is not None:
|
||||
# Actual client count, which doesn't include the reference kept by the manager, nor ours
|
||||
# as we are about to be deleted
|
||||
try:
|
||||
if len(self._rlist) == 0:
|
||||
# Free all resources associated with the mapped file
|
||||
self._manager._fdict.pop(self._rlist.path_or_fd())
|
||||
# END remove regions list from manager
|
||||
except (TypeError, KeyError):
|
||||
# sometimes, during shutdown, getrefcount is None. Its possible
|
||||
# to re-import it, however, its probably better to just ignore
|
||||
# this python problem (for now).
|
||||
# The next step is to get rid of the error prone getrefcount alltogether.
|
||||
pass
|
||||
# END exception handling
|
||||
# END handle regions
|
||||
|
||||
def _copy_from(self, rhs):
|
||||
"""Copy all data from rhs into this instance, handles usage count"""
|
||||
self._manager = rhs._manager
|
||||
self._rlist = type(rhs._rlist)(rhs._rlist)
|
||||
self._region = rhs._region
|
||||
self._ofs = rhs._ofs
|
||||
self._size = rhs._size
|
||||
|
||||
for region in self._rlist:
|
||||
region.increment_client_count()
|
||||
|
||||
if self._region is not None:
|
||||
self._region.increment_client_count()
|
||||
# END handle regions
|
||||
|
||||
def __copy__(self):
|
||||
"""copy module interface"""
|
||||
cpy = type(self)()
|
||||
cpy._copy_from(self)
|
||||
return cpy
|
||||
|
||||
#{ Interface
|
||||
def assign(self, rhs):
|
||||
"""Assign rhs to this instance. This is required in order to get a real copy.
|
||||
Alternativly, you can copy an existing instance using the copy module"""
|
||||
self._destroy()
|
||||
self._copy_from(rhs)
|
||||
|
||||
def use_region(self, offset=0, size=0, flags=0):
|
||||
"""Assure we point to a window which allows access to the given offset into the file
|
||||
|
||||
:param offset: absolute offset in bytes into the file
|
||||
:param size: amount of bytes to map. If 0, all available bytes will be mapped
|
||||
:param flags: additional flags to be given to os.open in case a file handle is initially opened
|
||||
for mapping. Has no effect if a region can actually be reused.
|
||||
:return: this instance - it should be queried for whether it points to a valid memory region.
|
||||
This is not the case if the mapping failed because we reached the end of the file
|
||||
|
||||
**Note:**: The size actually mapped may be smaller than the given size. If that is the case,
|
||||
either the file has reached its end, or the map was created between two existing regions"""
|
||||
need_region = True
|
||||
man = self._manager
|
||||
fsize = self._rlist.file_size()
|
||||
size = min(size or fsize, man.window_size() or fsize) # clamp size to window size
|
||||
|
||||
if self._region is not None:
|
||||
if self._region.includes_ofs(offset):
|
||||
need_region = False
|
||||
else:
|
||||
self.unuse_region()
|
||||
# END handle existing region
|
||||
# END check existing region
|
||||
|
||||
# offset too large ?
|
||||
if offset >= fsize:
|
||||
return self
|
||||
# END handle offset
|
||||
|
||||
if need_region:
|
||||
self._region = man._obtain_region(self._rlist, offset, size, flags, False)
|
||||
self._region.increment_client_count()
|
||||
# END need region handling
|
||||
|
||||
self._ofs = offset - self._region._b
|
||||
self._size = min(size, self._region.ofs_end() - offset)
|
||||
|
||||
return self
|
||||
|
||||
def unuse_region(self):
|
||||
"""Unuse the current region. Does nothing if we have no current region
|
||||
|
||||
**Note:** the cursor unuses the region automatically upon destruction. It is recommended
|
||||
to un-use the region once you are done reading from it in persistent cursors as it
|
||||
helps to free up resource more quickly"""
|
||||
if self._region is not None:
|
||||
self._region.increment_client_count(-1)
|
||||
self._region = None
|
||||
# note: should reset ofs and size, but we spare that for performance. Its not
|
||||
# allowed to query information if we are not valid !
|
||||
|
||||
def buffer(self):
|
||||
"""Return a buffer object which allows access to our memory region from our offset
|
||||
to the window size. Please note that it might be smaller than you requested when calling use_region()
|
||||
|
||||
**Note:** You can only obtain a buffer if this instance is_valid() !
|
||||
|
||||
**Note:** buffers should not be cached passed the duration of your access as it will
|
||||
prevent resources from being freed even though they might not be accounted for anymore !"""
|
||||
return memoryview(self._region.buffer())[self._ofs:self._ofs+self._size]
|
||||
|
||||
def map(self):
|
||||
"""
|
||||
:return: the underlying raw memory map. Please not that the offset and size is likely to be different
|
||||
to what you set as offset and size. Use it only if you are sure about the region it maps, which is the whole
|
||||
file in case of StaticWindowMapManager"""
|
||||
return self._region.map()
|
||||
|
||||
def is_valid(self):
|
||||
""":return: True if we have a valid and usable region"""
|
||||
return self._region is not None
|
||||
|
||||
def is_associated(self):
|
||||
""":return: True if we are associated with a specific file already"""
|
||||
return self._rlist is not None
|
||||
|
||||
def ofs_begin(self):
|
||||
""":return: offset to the first byte pointed to by our cursor
|
||||
|
||||
**Note:** only if is_valid() is True"""
|
||||
return self._region._b + self._ofs
|
||||
|
||||
def ofs_end(self):
|
||||
""":return: offset to one past the last available byte"""
|
||||
# unroll method calls for performance !
|
||||
return self._region._b + self._ofs + self._size
|
||||
|
||||
def size(self):
|
||||
""":return: amount of bytes we point to"""
|
||||
return self._size
|
||||
|
||||
def region(self):
|
||||
""":return: our mapped region, or None if nothing is mapped yet
|
||||
:raise AssertionError: if we have no current region. This is only useful for debugging"""
|
||||
return self._region
|
||||
|
||||
def includes_ofs(self, ofs):
|
||||
""":return: True if the given absolute offset is contained in the cursors
|
||||
current region
|
||||
|
||||
**Note:** cursor must be valid for this to work"""
|
||||
# unroll methods
|
||||
return (self._region._b + self._ofs) <= ofs < (self._region._b + self._ofs + self._size)
|
||||
|
||||
def file_size(self):
|
||||
""":return: size of the underlying file"""
|
||||
return self._rlist.file_size()
|
||||
|
||||
def path_or_fd(self):
|
||||
""":return: path or file descriptor of the underlying mapped file"""
|
||||
return self._rlist.path_or_fd()
|
||||
|
||||
def path(self):
|
||||
""":return: path of the underlying mapped file
|
||||
:raise ValueError: if attached path is not a path"""
|
||||
if isinstance(self._rlist.path_or_fd(), int):
|
||||
raise ValueError("Path queried although mapping was applied to a file descriptor")
|
||||
# END handle type
|
||||
return self._rlist.path_or_fd()
|
||||
|
||||
def fd(self):
|
||||
""":return: file descriptor used to create the underlying mapping.
|
||||
|
||||
**Note:** it is not required to be valid anymore
|
||||
:raise ValueError: if the mapping was not created by a file descriptor"""
|
||||
if isinstance(self._rlist.path_or_fd(), str):
|
||||
raise ValueError("File descriptor queried although mapping was generated from path")
|
||||
# END handle type
|
||||
return self._rlist.path_or_fd()
|
||||
|
||||
#} END interface
|
||||
|
||||
|
||||
class StaticWindowMapManager:
|
||||
|
||||
"""Provides a manager which will produce single size cursors that are allowed
|
||||
to always map the whole file.
|
||||
|
||||
Clients must be written to specifically know that they are accessing their data
|
||||
through a StaticWindowMapManager, as they otherwise have to deal with their window size.
|
||||
|
||||
These clients would have to use a SlidingWindowMapBuffer to hide this fact.
|
||||
|
||||
This type will always use a maximum window size, and optimize certain methods to
|
||||
accommodate this fact"""
|
||||
|
||||
__slots__ = [
|
||||
'_fdict', # mapping of path -> StorageHelper (of some kind
|
||||
'_window_size', # maximum size of a window
|
||||
'_max_memory_size', # maximum amount of memory we may allocate
|
||||
'_max_handle_count', # maximum amount of handles to keep open
|
||||
'_memory_size', # currently allocated memory size
|
||||
'_handle_count', # amount of currently allocated file handles
|
||||
]
|
||||
|
||||
#{ Configuration
|
||||
MapRegionListCls = MapRegionList
|
||||
MapWindowCls = MapWindow
|
||||
MapRegionCls = MapRegion
|
||||
WindowCursorCls = WindowCursor
|
||||
#} END configuration
|
||||
|
||||
_MB_in_bytes = 1024 * 1024
|
||||
|
||||
def __init__(self, window_size=0, max_memory_size=0, max_open_handles=sys.maxsize):
|
||||
"""initialize the manager with the given parameters.
|
||||
:param window_size: if -1, a default window size will be chosen depending on
|
||||
the operating system's architecture. It will internally be quantified to a multiple of the page size
|
||||
If 0, the window may have any size, which basically results in mapping the whole file at one
|
||||
:param max_memory_size: maximum amount of memory we may map at once before releasing mapped regions.
|
||||
If 0, a viable default will be set depending on the system's architecture.
|
||||
It is a soft limit that is tried to be kept, but nothing bad happens if we have to over-allocate
|
||||
:param max_open_handles: if not maxint, limit the amount of open file handles to the given number.
|
||||
Otherwise the amount is only limited by the system itself. If a system or soft limit is hit,
|
||||
the manager will free as many handles as possible"""
|
||||
self._fdict = dict()
|
||||
self._window_size = window_size
|
||||
self._max_memory_size = max_memory_size
|
||||
self._max_handle_count = max_open_handles
|
||||
self._memory_size = 0
|
||||
self._handle_count = 0
|
||||
|
||||
if window_size < 0:
|
||||
coeff = 64
|
||||
if is_64_bit():
|
||||
coeff = 1024
|
||||
# END handle arch
|
||||
self._window_size = coeff * self._MB_in_bytes
|
||||
# END handle max window size
|
||||
|
||||
if max_memory_size == 0:
|
||||
coeff = 1024
|
||||
if is_64_bit():
|
||||
coeff = 8192
|
||||
# END handle arch
|
||||
self._max_memory_size = coeff * self._MB_in_bytes
|
||||
# END handle max memory size
|
||||
|
||||
#{ Internal Methods
|
||||
|
||||
def _collect_lru_region(self, size):
|
||||
"""Unmap the region which was least-recently used and has no client
|
||||
:param size: size of the region we want to map next (assuming its not already mapped partially or full
|
||||
if 0, we try to free any available region
|
||||
:return: Amount of freed regions
|
||||
|
||||
.. Note::
|
||||
We don't raise exceptions anymore, in order to keep the system working, allowing temporary overallocation.
|
||||
If the system runs out of memory, it will tell.
|
||||
|
||||
.. TODO::
|
||||
implement a case where all unusued regions are discarded efficiently.
|
||||
Currently its only brute force
|
||||
"""
|
||||
num_found = 0
|
||||
while (size == 0) or (self._memory_size + size > self._max_memory_size):
|
||||
lru_region = None
|
||||
lru_list = None
|
||||
for regions in self._fdict.values():
|
||||
for region in regions:
|
||||
# check client count - if it's 1, it's just us
|
||||
if (region.client_count() == 1 and
|
||||
(lru_region is None or region._uc < lru_region._uc)):
|
||||
lru_region = region
|
||||
lru_list = regions
|
||||
# END update lru_region
|
||||
# END for each region
|
||||
# END for each regions list
|
||||
|
||||
if lru_region is None:
|
||||
break
|
||||
# END handle region not found
|
||||
|
||||
num_found += 1
|
||||
del(lru_list[lru_list.index(lru_region)])
|
||||
lru_region.increment_client_count(-1)
|
||||
self._memory_size -= lru_region.size()
|
||||
self._handle_count -= 1
|
||||
# END while there is more memory to free
|
||||
return num_found
|
||||
|
||||
def _obtain_region(self, a, offset, size, flags, is_recursive):
|
||||
"""Utilty to create a new region - for more information on the parameters,
|
||||
see MapCursor.use_region.
|
||||
:param a: A regions (a)rray
|
||||
:return: The newly created region"""
|
||||
if self._memory_size + size > self._max_memory_size:
|
||||
self._collect_lru_region(size)
|
||||
# END handle collection
|
||||
|
||||
r = None
|
||||
if a:
|
||||
assert len(a) == 1
|
||||
r = a[0]
|
||||
else:
|
||||
try:
|
||||
r = self.MapRegionCls(a.path_or_fd(), 0, sys.maxsize, flags)
|
||||
except Exception:
|
||||
# apparently we are out of system resources or hit a limit
|
||||
# As many more operations are likely to fail in that condition (
|
||||
# like reading a file from disk, etc) we free up as much as possible
|
||||
# As this invalidates our insert position, we have to recurse here
|
||||
if is_recursive:
|
||||
# we already tried this, and still have no success in obtaining
|
||||
# a mapping. This is an exception, so we propagate it
|
||||
raise
|
||||
# END handle existing recursion
|
||||
self._collect_lru_region(0)
|
||||
return self._obtain_region(a, offset, size, flags, True)
|
||||
# END handle exceptions
|
||||
|
||||
self._handle_count += 1
|
||||
self._memory_size += r.size()
|
||||
a.append(r)
|
||||
# END handle array
|
||||
|
||||
assert r.includes_ofs(offset)
|
||||
return r
|
||||
|
||||
#}END internal methods
|
||||
|
||||
#{ Interface
|
||||
def make_cursor(self, path_or_fd):
|
||||
"""
|
||||
:return: a cursor pointing to the given path or file descriptor.
|
||||
It can be used to map new regions of the file into memory
|
||||
|
||||
**Note:** if a file descriptor is given, it is assumed to be open and valid,
|
||||
but may be closed afterwards. To refer to the same file, you may reuse
|
||||
your existing file descriptor, but keep in mind that new windows can only
|
||||
be mapped as long as it stays valid. This is why the using actual file paths
|
||||
are preferred unless you plan to keep the file descriptor open.
|
||||
|
||||
**Note:** file descriptors are problematic as they are not necessarily unique, as two
|
||||
different files opened and closed in succession might have the same file descriptor id.
|
||||
|
||||
**Note:** Using file descriptors directly is faster once new windows are mapped as it
|
||||
prevents the file to be opened again just for the purpose of mapping it."""
|
||||
regions = self._fdict.get(path_or_fd)
|
||||
if regions is None:
|
||||
regions = self.MapRegionListCls(path_or_fd)
|
||||
self._fdict[path_or_fd] = regions
|
||||
# END obtain region for path
|
||||
return self.WindowCursorCls(self, regions)
|
||||
|
||||
def collect(self):
|
||||
"""Collect all available free-to-collect mapped regions
|
||||
:return: Amount of freed handles"""
|
||||
return self._collect_lru_region(0)
|
||||
|
||||
def num_file_handles(self):
|
||||
""":return: amount of file handles in use. Each mapped region uses one file handle"""
|
||||
return self._handle_count
|
||||
|
||||
def num_open_files(self):
|
||||
"""Amount of opened files in the system"""
|
||||
return reduce(lambda x, y: x + y, (1 for rlist in self._fdict.values() if len(rlist) > 0), 0)
|
||||
|
||||
def window_size(self):
|
||||
""":return: size of each window when allocating new regions"""
|
||||
return self._window_size
|
||||
|
||||
def mapped_memory_size(self):
|
||||
""":return: amount of bytes currently mapped in total"""
|
||||
return self._memory_size
|
||||
|
||||
def max_file_handles(self):
|
||||
""":return: maximium amount of handles we may have opened"""
|
||||
return self._max_handle_count
|
||||
|
||||
def max_mapped_memory_size(self):
|
||||
""":return: maximum amount of memory we may allocate"""
|
||||
return self._max_memory_size
|
||||
|
||||
#} END interface
|
||||
|
||||
#{ Special Purpose Interface
|
||||
|
||||
def force_map_handle_removal_win(self, base_path):
|
||||
"""ONLY AVAILABLE ON WINDOWS
|
||||
On windows removing files is not allowed if anybody still has it opened.
|
||||
If this process is ourselves, and if the whole process uses this memory
|
||||
manager (as far as the parent framework is concerned) we can enforce
|
||||
closing all memory maps whose path matches the given base path to
|
||||
allow the respective operation after all.
|
||||
The respective system must NOT access the closed memory regions anymore !
|
||||
This really may only be used if you know that the items which keep
|
||||
the cursors alive will not be using it anymore. They need to be recreated !
|
||||
:return: Amount of closed handles
|
||||
|
||||
**Note:** does nothing on non-windows platforms"""
|
||||
if sys.platform != 'win32':
|
||||
return
|
||||
# END early bailout
|
||||
|
||||
num_closed = 0
|
||||
for path, rlist in self._fdict.items():
|
||||
if path.startswith(base_path):
|
||||
for region in rlist:
|
||||
region.release()
|
||||
num_closed += 1
|
||||
# END path matches
|
||||
# END for each path
|
||||
return num_closed
|
||||
#} END special purpose interface
|
||||
|
||||
|
||||
class SlidingWindowMapManager(StaticWindowMapManager):
|
||||
|
||||
"""Maintains a list of ranges of mapped memory regions in one or more files and allows to easily
|
||||
obtain additional regions assuring there is no overlap.
|
||||
Once a certain memory limit is reached globally, or if there cannot be more open file handles
|
||||
which result from each mmap call, the least recently used, and currently unused mapped regions
|
||||
are unloaded automatically.
|
||||
|
||||
**Note:** currently not thread-safe !
|
||||
|
||||
**Note:** in the current implementation, we will automatically unload windows if we either cannot
|
||||
create more memory maps (as the open file handles limit is hit) or if we have allocated more than
|
||||
a safe amount of memory already, which would possibly cause memory allocations to fail as our address
|
||||
space is full."""
|
||||
|
||||
__slots__ = tuple()
|
||||
|
||||
def __init__(self, window_size=-1, max_memory_size=0, max_open_handles=sys.maxsize):
|
||||
"""Adjusts the default window size to -1"""
|
||||
super().__init__(window_size, max_memory_size, max_open_handles)
|
||||
|
||||
def _obtain_region(self, a, offset, size, flags, is_recursive):
|
||||
# bisect to find an existing region. The c++ implementation cannot
|
||||
# do that as it uses a linked list for regions.
|
||||
r = None
|
||||
lo = 0
|
||||
hi = len(a)
|
||||
while lo < hi:
|
||||
mid = (lo + hi) // 2
|
||||
ofs = a[mid]._b
|
||||
if ofs <= offset:
|
||||
if a[mid].includes_ofs(offset):
|
||||
r = a[mid]
|
||||
break
|
||||
# END have region
|
||||
lo = mid + 1
|
||||
else:
|
||||
hi = mid
|
||||
# END handle position
|
||||
# END while bisecting
|
||||
|
||||
if r is None:
|
||||
window_size = self._window_size
|
||||
left = self.MapWindowCls(0, 0)
|
||||
mid = self.MapWindowCls(offset, size)
|
||||
right = self.MapWindowCls(a.file_size(), 0)
|
||||
|
||||
# we want to honor the max memory size, and assure we have anough
|
||||
# memory available
|
||||
# Save calls !
|
||||
if self._memory_size + window_size > self._max_memory_size:
|
||||
self._collect_lru_region(window_size)
|
||||
# END handle collection
|
||||
|
||||
# we assume the list remains sorted by offset
|
||||
insert_pos = 0
|
||||
len_regions = len(a)
|
||||
if len_regions == 1:
|
||||
if a[0]._b <= offset:
|
||||
insert_pos = 1
|
||||
# END maintain sort
|
||||
else:
|
||||
# find insert position
|
||||
insert_pos = len_regions
|
||||
for i, region in enumerate(a):
|
||||
if region._b > offset:
|
||||
insert_pos = i
|
||||
break
|
||||
# END if insert position is correct
|
||||
# END for each region
|
||||
# END obtain insert pos
|
||||
|
||||
# adjust the actual offset and size values to create the largest
|
||||
# possible mapping
|
||||
if insert_pos == 0:
|
||||
if len_regions:
|
||||
right = self.MapWindowCls.from_region(a[insert_pos])
|
||||
# END adjust right side
|
||||
else:
|
||||
if insert_pos != len_regions:
|
||||
right = self.MapWindowCls.from_region(a[insert_pos])
|
||||
# END adjust right window
|
||||
left = self.MapWindowCls.from_region(a[insert_pos - 1])
|
||||
# END adjust surrounding windows
|
||||
|
||||
mid.extend_left_to(left, window_size)
|
||||
mid.extend_right_to(right, window_size)
|
||||
mid.align()
|
||||
|
||||
# it can happen that we align beyond the end of the file
|
||||
if mid.ofs_end() > right.ofs:
|
||||
mid.size = right.ofs - mid.ofs
|
||||
# END readjust size
|
||||
|
||||
# insert new region at the right offset to keep the order
|
||||
try:
|
||||
if self._handle_count >= self._max_handle_count:
|
||||
raise Exception
|
||||
# END assert own imposed max file handles
|
||||
r = self.MapRegionCls(a.path_or_fd(), mid.ofs, mid.size, flags)
|
||||
except Exception:
|
||||
# apparently we are out of system resources or hit a limit
|
||||
# As many more operations are likely to fail in that condition (
|
||||
# like reading a file from disk, etc) we free up as much as possible
|
||||
# As this invalidates our insert position, we have to recurse here
|
||||
if is_recursive:
|
||||
# we already tried this, and still have no success in obtaining
|
||||
# a mapping. This is an exception, so we propagate it
|
||||
raise
|
||||
# END handle existing recursion
|
||||
self._collect_lru_region(0)
|
||||
return self._obtain_region(a, offset, size, flags, True)
|
||||
# END handle exceptions
|
||||
|
||||
self._handle_count += 1
|
||||
self._memory_size += r.size()
|
||||
a.insert(insert_pos, r)
|
||||
# END create new region
|
||||
return r
|
||||
72
zero-cost-nas/.eggs/smmap-5.0.0-py3.8.egg/smmap/test/lib.py
Normal file
72
zero-cost-nas/.eggs/smmap-5.0.0-py3.8.egg/smmap/test/lib.py
Normal file
@@ -0,0 +1,72 @@
|
||||
"""Provide base classes for the test system"""
|
||||
from unittest import TestCase
|
||||
import os
|
||||
import tempfile
|
||||
|
||||
__all__ = ['TestBase', 'FileCreator']
|
||||
|
||||
|
||||
#{ Utilities
|
||||
|
||||
class FileCreator:
|
||||
|
||||
"""A instance which creates a temporary file with a prefix and a given size
|
||||
and provides this info to the user.
|
||||
Once it gets deleted, it will remove the temporary file as well."""
|
||||
__slots__ = ("_size", "_path")
|
||||
|
||||
def __init__(self, size, prefix=''):
|
||||
assert size, "Require size to be larger 0"
|
||||
|
||||
self._path = tempfile.mktemp(prefix=prefix)
|
||||
self._size = size
|
||||
|
||||
with open(self._path, "wb") as fp:
|
||||
fp.seek(size - 1)
|
||||
fp.write(b'1')
|
||||
|
||||
assert os.path.getsize(self.path) == size
|
||||
|
||||
def __del__(self):
|
||||
try:
|
||||
os.remove(self.path)
|
||||
except OSError:
|
||||
pass
|
||||
# END exception handling
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc_value, traceback):
|
||||
self.__del__()
|
||||
|
||||
@property
|
||||
def path(self):
|
||||
return self._path
|
||||
|
||||
@property
|
||||
def size(self):
|
||||
return self._size
|
||||
|
||||
#} END utilities
|
||||
|
||||
|
||||
class TestBase(TestCase):
|
||||
|
||||
"""Foundation used by all tests"""
|
||||
|
||||
#{ Configuration
|
||||
k_window_test_size = 1000 * 1000 * 8 + 5195
|
||||
#} END configuration
|
||||
|
||||
#{ Overrides
|
||||
@classmethod
|
||||
def setUpAll(cls):
|
||||
# nothing for now
|
||||
pass
|
||||
|
||||
# END overrides
|
||||
|
||||
#{ Interface
|
||||
|
||||
#} END interface
|
||||
126
zero-cost-nas/.eggs/smmap-5.0.0-py3.8.egg/smmap/test/test_buf.py
Normal file
126
zero-cost-nas/.eggs/smmap-5.0.0-py3.8.egg/smmap/test/test_buf.py
Normal file
@@ -0,0 +1,126 @@
|
||||
from .lib import TestBase, FileCreator
|
||||
|
||||
from smmap.mman import (
|
||||
SlidingWindowMapManager,
|
||||
StaticWindowMapManager
|
||||
)
|
||||
from smmap.buf import SlidingWindowMapBuffer
|
||||
|
||||
from random import randint
|
||||
from time import time
|
||||
import sys
|
||||
import os
|
||||
|
||||
|
||||
man_optimal = SlidingWindowMapManager()
|
||||
man_worst_case = SlidingWindowMapManager(
|
||||
window_size=TestBase.k_window_test_size // 100,
|
||||
max_memory_size=TestBase.k_window_test_size // 3,
|
||||
max_open_handles=15)
|
||||
static_man = StaticWindowMapManager()
|
||||
|
||||
|
||||
class TestBuf(TestBase):
|
||||
|
||||
def test_basics(self):
|
||||
with FileCreator(self.k_window_test_size, "buffer_test") as fc:
|
||||
|
||||
# invalid paths fail upon construction
|
||||
c = man_optimal.make_cursor(fc.path)
|
||||
self.assertRaises(ValueError, SlidingWindowMapBuffer, type(c)()) # invalid cursor
|
||||
self.assertRaises(ValueError, SlidingWindowMapBuffer, c, fc.size) # offset too large
|
||||
|
||||
buf = SlidingWindowMapBuffer() # can create uninitailized buffers
|
||||
assert buf.cursor() is None
|
||||
|
||||
# can call end access any time
|
||||
buf.end_access()
|
||||
buf.end_access()
|
||||
assert len(buf) == 0
|
||||
|
||||
# begin access can revive it, if the offset is suitable
|
||||
offset = 100
|
||||
assert buf.begin_access(c, fc.size) == False
|
||||
assert buf.begin_access(c, offset) == True
|
||||
assert len(buf) == fc.size - offset
|
||||
assert buf.cursor().is_valid()
|
||||
|
||||
# empty begin access keeps it valid on the same path, but alters the offset
|
||||
assert buf.begin_access() == True
|
||||
assert len(buf) == fc.size
|
||||
assert buf.cursor().is_valid()
|
||||
|
||||
# simple access
|
||||
with open(fc.path, 'rb') as fp:
|
||||
data = fp.read()
|
||||
assert data[offset] == buf[0]
|
||||
assert data[offset:offset * 2] == buf[0:offset]
|
||||
|
||||
# negative indices, partial slices
|
||||
assert buf[-1] == buf[len(buf) - 1]
|
||||
assert buf[-10:] == buf[len(buf) - 10:len(buf)]
|
||||
|
||||
# end access makes its cursor invalid
|
||||
buf.end_access()
|
||||
assert not buf.cursor().is_valid()
|
||||
assert buf.cursor().is_associated() # but it remains associated
|
||||
|
||||
# an empty begin access fixes it up again
|
||||
assert buf.begin_access() == True and buf.cursor().is_valid()
|
||||
del(buf) # ends access automatically
|
||||
del(c)
|
||||
|
||||
assert man_optimal.num_file_handles() == 1
|
||||
|
||||
# PERFORMANCE
|
||||
# blast away with random access and a full mapping - we don't want to
|
||||
# exaggerate the manager's overhead, but measure the buffer overhead
|
||||
# We do it once with an optimal setting, and with a worse manager which
|
||||
# will produce small mappings only !
|
||||
max_num_accesses = 100
|
||||
fd = os.open(fc.path, os.O_RDONLY)
|
||||
for item in (fc.path, fd):
|
||||
for manager, man_id in ((man_optimal, 'optimal'),
|
||||
(man_worst_case, 'worst case'),
|
||||
(static_man, 'static optimal')):
|
||||
buf = SlidingWindowMapBuffer(manager.make_cursor(item))
|
||||
assert manager.num_file_handles() == 1
|
||||
for access_mode in range(2): # single, multi
|
||||
num_accesses_left = max_num_accesses
|
||||
num_bytes = 0
|
||||
fsize = fc.size
|
||||
|
||||
st = time()
|
||||
buf.begin_access()
|
||||
while num_accesses_left:
|
||||
num_accesses_left -= 1
|
||||
if access_mode: # multi
|
||||
ofs_start = randint(0, fsize)
|
||||
ofs_end = randint(ofs_start, fsize)
|
||||
d = buf[ofs_start:ofs_end]
|
||||
assert len(d) == ofs_end - ofs_start
|
||||
assert d == data[ofs_start:ofs_end]
|
||||
num_bytes += len(d)
|
||||
del d
|
||||
else:
|
||||
pos = randint(0, fsize)
|
||||
assert buf[pos] == data[pos]
|
||||
num_bytes += 1
|
||||
# END handle mode
|
||||
# END handle num accesses
|
||||
|
||||
buf.end_access()
|
||||
assert manager.num_file_handles()
|
||||
assert manager.collect()
|
||||
assert manager.num_file_handles() == 0
|
||||
elapsed = max(time() - st, 0.001) # prevent zero division errors on windows
|
||||
mb = float(1000 * 1000)
|
||||
mode_str = (access_mode and "slice") or "single byte"
|
||||
print("%s: Made %i random %s accesses to buffer created from %s reading a total of %f mb in %f s (%f mb/s)"
|
||||
% (man_id, max_num_accesses, mode_str, type(item), num_bytes / mb, elapsed, (num_bytes / mb) / elapsed),
|
||||
file=sys.stderr)
|
||||
# END handle access mode
|
||||
del buf
|
||||
# END for each manager
|
||||
# END for each input
|
||||
os.close(fd)
|
||||
@@ -0,0 +1,224 @@
|
||||
from .lib import TestBase, FileCreator
|
||||
|
||||
from smmap.mman import (
|
||||
WindowCursor,
|
||||
SlidingWindowMapManager,
|
||||
StaticWindowMapManager
|
||||
)
|
||||
from smmap.util import align_to_mmap
|
||||
|
||||
from random import randint
|
||||
from time import time
|
||||
import os
|
||||
import sys
|
||||
from copy import copy
|
||||
|
||||
|
||||
class TestMMan(TestBase):
|
||||
|
||||
def test_cursor(self):
|
||||
with FileCreator(self.k_window_test_size, "cursor_test") as fc:
|
||||
man = SlidingWindowMapManager()
|
||||
ci = WindowCursor(man) # invalid cursor
|
||||
assert not ci.is_valid()
|
||||
assert not ci.is_associated()
|
||||
assert ci.size() == 0 # this is cached, so we can query it in invalid state
|
||||
|
||||
cv = man.make_cursor(fc.path)
|
||||
assert not cv.is_valid() # no region mapped yet
|
||||
assert cv.is_associated() # but it know where to map it from
|
||||
assert cv.file_size() == fc.size
|
||||
assert cv.path() == fc.path
|
||||
|
||||
# copy module
|
||||
cio = copy(cv)
|
||||
assert not cio.is_valid() and cio.is_associated()
|
||||
|
||||
# assign method
|
||||
assert not ci.is_associated()
|
||||
ci.assign(cv)
|
||||
assert not ci.is_valid() and ci.is_associated()
|
||||
|
||||
# unuse non-existing region is fine
|
||||
cv.unuse_region()
|
||||
cv.unuse_region()
|
||||
|
||||
# destruction is fine (even multiple times)
|
||||
cv._destroy()
|
||||
WindowCursor(man)._destroy()
|
||||
|
||||
def test_memory_manager(self):
|
||||
slide_man = SlidingWindowMapManager()
|
||||
static_man = StaticWindowMapManager()
|
||||
|
||||
for man in (static_man, slide_man):
|
||||
assert man.num_file_handles() == 0
|
||||
assert man.num_open_files() == 0
|
||||
winsize_cmp_val = 0
|
||||
if isinstance(man, StaticWindowMapManager):
|
||||
winsize_cmp_val = -1
|
||||
# END handle window size
|
||||
assert man.window_size() > winsize_cmp_val
|
||||
assert man.mapped_memory_size() == 0
|
||||
assert man.max_mapped_memory_size() > 0
|
||||
|
||||
# collection doesn't raise in 'any' mode
|
||||
man._collect_lru_region(0)
|
||||
# doesn't raise if we are within the limit
|
||||
man._collect_lru_region(10)
|
||||
|
||||
# doesn't fail if we over-allocate
|
||||
assert man._collect_lru_region(sys.maxsize) == 0
|
||||
|
||||
# use a region, verify most basic functionality
|
||||
with FileCreator(self.k_window_test_size, "manager_test") as fc:
|
||||
fd = os.open(fc.path, os.O_RDONLY)
|
||||
try:
|
||||
for item in (fc.path, fd):
|
||||
c = man.make_cursor(item)
|
||||
assert c.path_or_fd() is item
|
||||
assert c.use_region(10, 10).is_valid()
|
||||
assert c.ofs_begin() == 10
|
||||
assert c.size() == 10
|
||||
with open(fc.path, 'rb') as fp:
|
||||
assert c.buffer()[:] == fp.read(20)[10:]
|
||||
|
||||
if isinstance(item, int):
|
||||
self.assertRaises(ValueError, c.path)
|
||||
else:
|
||||
self.assertRaises(ValueError, c.fd)
|
||||
# END handle value error
|
||||
# END for each input
|
||||
finally:
|
||||
os.close(fd)
|
||||
# END for each manasger type
|
||||
|
||||
def test_memman_operation(self):
|
||||
# test more access, force it to actually unmap regions
|
||||
with FileCreator(self.k_window_test_size, "manager_operation_test") as fc:
|
||||
with open(fc.path, 'rb') as fp:
|
||||
data = fp.read()
|
||||
fd = os.open(fc.path, os.O_RDONLY)
|
||||
try:
|
||||
max_num_handles = 15
|
||||
# small_size =
|
||||
for mtype, args in ((StaticWindowMapManager, (0, fc.size // 3, max_num_handles)),
|
||||
(SlidingWindowMapManager, (fc.size // 100, fc.size // 3, max_num_handles)),):
|
||||
for item in (fc.path, fd):
|
||||
assert len(data) == fc.size
|
||||
|
||||
# small windows, a reasonable max memory. Not too many regions at once
|
||||
man = mtype(window_size=args[0], max_memory_size=args[1], max_open_handles=args[2])
|
||||
c = man.make_cursor(item)
|
||||
|
||||
# still empty (more about that is tested in test_memory_manager()
|
||||
assert man.num_open_files() == 0
|
||||
assert man.mapped_memory_size() == 0
|
||||
|
||||
base_offset = 5000
|
||||
# window size is 0 for static managers, hence size will be 0. We take that into consideration
|
||||
size = man.window_size() // 2
|
||||
assert c.use_region(base_offset, size).is_valid()
|
||||
rr = c.region()
|
||||
assert rr.client_count() == 2 # the manager and the cursor and us
|
||||
|
||||
assert man.num_open_files() == 1
|
||||
assert man.num_file_handles() == 1
|
||||
assert man.mapped_memory_size() == rr.size()
|
||||
|
||||
# assert c.size() == size # the cursor may overallocate in its static version
|
||||
assert c.ofs_begin() == base_offset
|
||||
assert rr.ofs_begin() == 0 # it was aligned and expanded
|
||||
if man.window_size():
|
||||
# but isn't larger than the max window (aligned)
|
||||
assert rr.size() == align_to_mmap(man.window_size(), True)
|
||||
else:
|
||||
assert rr.size() == fc.size
|
||||
# END ignore static managers which dont use windows and are aligned to file boundaries
|
||||
|
||||
assert c.buffer()[:] == data[base_offset:base_offset + (size or c.size())]
|
||||
|
||||
# obtain second window, which spans the first part of the file - it is a still the same window
|
||||
nsize = (size or fc.size) - 10
|
||||
assert c.use_region(0, nsize).is_valid()
|
||||
assert c.region() == rr
|
||||
assert man.num_file_handles() == 1
|
||||
assert c.size() == nsize
|
||||
assert c.ofs_begin() == 0
|
||||
assert c.buffer()[:] == data[:nsize]
|
||||
|
||||
# map some part at the end, our requested size cannot be kept
|
||||
overshoot = 4000
|
||||
base_offset = fc.size - (size or c.size()) + overshoot
|
||||
assert c.use_region(base_offset, size).is_valid()
|
||||
if man.window_size():
|
||||
assert man.num_file_handles() == 2
|
||||
assert c.size() < size
|
||||
assert c.region() is not rr # old region is still available, but has not curser ref anymore
|
||||
assert rr.client_count() == 1 # only held by manager
|
||||
else:
|
||||
assert c.size() < fc.size
|
||||
# END ignore static managers which only have one handle per file
|
||||
rr = c.region()
|
||||
assert rr.client_count() == 2 # manager + cursor
|
||||
assert rr.ofs_begin() < c.ofs_begin() # it should have extended itself to the left
|
||||
assert rr.ofs_end() <= fc.size # it cannot be larger than the file
|
||||
assert c.buffer()[:] == data[base_offset:base_offset + (size or c.size())]
|
||||
|
||||
# unising a region makes the cursor invalid
|
||||
c.unuse_region()
|
||||
assert not c.is_valid()
|
||||
if man.window_size():
|
||||
# but doesn't change anything regarding the handle count - we cache it and only
|
||||
# remove mapped regions if we have to
|
||||
assert man.num_file_handles() == 2
|
||||
# END ignore this for static managers
|
||||
|
||||
# iterate through the windows, verify data contents
|
||||
# this will trigger map collection after a while
|
||||
max_random_accesses = 5000
|
||||
num_random_accesses = max_random_accesses
|
||||
memory_read = 0
|
||||
st = time()
|
||||
|
||||
# cache everything to get some more performance
|
||||
includes_ofs = c.includes_ofs
|
||||
max_mapped_memory_size = man.max_mapped_memory_size()
|
||||
max_file_handles = man.max_file_handles()
|
||||
mapped_memory_size = man.mapped_memory_size
|
||||
num_file_handles = man.num_file_handles
|
||||
while num_random_accesses:
|
||||
num_random_accesses -= 1
|
||||
base_offset = randint(0, fc.size - 1)
|
||||
|
||||
# precondition
|
||||
if man.window_size():
|
||||
assert max_mapped_memory_size >= mapped_memory_size()
|
||||
# END statics will overshoot, which is fine
|
||||
assert max_file_handles >= num_file_handles()
|
||||
assert c.use_region(base_offset, (size or c.size())).is_valid()
|
||||
csize = c.size()
|
||||
assert c.buffer()[:] == data[base_offset:base_offset + csize]
|
||||
memory_read += csize
|
||||
|
||||
assert includes_ofs(base_offset)
|
||||
assert includes_ofs(base_offset + csize - 1)
|
||||
assert not includes_ofs(base_offset + csize)
|
||||
# END while we should do an access
|
||||
elapsed = max(time() - st, 0.001) # prevent zero divison errors on windows
|
||||
mb = float(1000 * 1000)
|
||||
print("%s: Read %i mb of memory with %i random on cursor initialized with %s accesses in %fs (%f mb/s)\n"
|
||||
% (mtype, memory_read / mb, max_random_accesses, type(item), elapsed, (memory_read / mb) / elapsed),
|
||||
file=sys.stderr)
|
||||
|
||||
# an offset as large as the size doesn't work !
|
||||
assert not c.use_region(fc.size, size).is_valid()
|
||||
|
||||
# collection - it should be able to collect all
|
||||
assert man.num_file_handles()
|
||||
assert man.collect()
|
||||
assert man.num_file_handles() == 0
|
||||
# END for each item
|
||||
# END for each manager type
|
||||
finally:
|
||||
os.close(fd)
|
||||
@@ -0,0 +1,75 @@
|
||||
from .lib import TestBase
|
||||
|
||||
|
||||
class TestTutorial(TestBase):
|
||||
|
||||
def test_example(self):
|
||||
# Memory Managers
|
||||
##################
|
||||
import smmap
|
||||
# This instance should be globally available in your application
|
||||
# It is configured to be well suitable for 32-bit or 64 bit applications.
|
||||
mman = smmap.SlidingWindowMapManager()
|
||||
|
||||
# the manager provides much useful information about its current state
|
||||
# like the amount of open file handles or the amount of mapped memory
|
||||
assert mman.num_file_handles() == 0
|
||||
assert mman.mapped_memory_size() == 0
|
||||
# and many more ...
|
||||
|
||||
# Cursors
|
||||
##########
|
||||
import smmap.test.lib
|
||||
with smmap.test.lib.FileCreator(1024 * 1024 * 8, "test_file") as fc:
|
||||
# obtain a cursor to access some file.
|
||||
c = mman.make_cursor(fc.path)
|
||||
|
||||
# the cursor is now associated with the file, but not yet usable
|
||||
assert c.is_associated()
|
||||
assert not c.is_valid()
|
||||
|
||||
# before you can use the cursor, you have to specify a window you want to
|
||||
# access. The following just says you want as much data as possible starting
|
||||
# from offset 0.
|
||||
# To be sure your region could be mapped, query for validity
|
||||
assert c.use_region().is_valid() # use_region returns self
|
||||
|
||||
# once a region was mapped, you must query its dimension regularly
|
||||
# to assure you don't try to access its buffer out of its bounds
|
||||
assert c.size()
|
||||
c.buffer()[0] # first byte
|
||||
c.buffer()[1:10] # first 9 bytes
|
||||
c.buffer()[c.size() - 1] # last byte
|
||||
|
||||
# you can query absolute offsets, and check whether an offset is included
|
||||
# in the cursor's data.
|
||||
assert c.ofs_begin() < c.ofs_end()
|
||||
assert c.includes_ofs(100)
|
||||
|
||||
# If you are over out of bounds with one of your region requests, the
|
||||
# cursor will be come invalid. It cannot be used in that state
|
||||
assert not c.use_region(fc.size, 100).is_valid()
|
||||
# map as much as possible after skipping the first 100 bytes
|
||||
assert c.use_region(100).is_valid()
|
||||
|
||||
# You can explicitly free cursor resources by unusing the cursor's region
|
||||
c.unuse_region()
|
||||
assert not c.is_valid()
|
||||
|
||||
# Buffers
|
||||
#########
|
||||
# Create a default buffer which can operate on the whole file
|
||||
buf = smmap.SlidingWindowMapBuffer(mman.make_cursor(fc.path))
|
||||
|
||||
# you can use it right away
|
||||
assert buf.cursor().is_valid()
|
||||
|
||||
buf[0] # access the first byte
|
||||
buf[-1] # access the last ten bytes on the file
|
||||
buf[-10:] # access the last ten bytes
|
||||
|
||||
# If you want to keep the instance between different accesses, use the
|
||||
# dedicated methods
|
||||
buf.end_access()
|
||||
assert not buf.cursor().is_valid() # you cannot use the buffer anymore
|
||||
assert buf.begin_access(offset=10) # start using the buffer at an offset
|
||||
@@ -0,0 +1,105 @@
|
||||
from .lib import TestBase, FileCreator
|
||||
|
||||
from smmap.util import (
|
||||
MapWindow,
|
||||
MapRegion,
|
||||
MapRegionList,
|
||||
ALLOCATIONGRANULARITY,
|
||||
is_64_bit,
|
||||
align_to_mmap
|
||||
)
|
||||
|
||||
import os
|
||||
import sys
|
||||
|
||||
|
||||
class TestMMan(TestBase):
|
||||
|
||||
def test_window(self):
|
||||
wl = MapWindow(0, 1) # left
|
||||
wc = MapWindow(1, 1) # center
|
||||
wc2 = MapWindow(10, 5) # another center
|
||||
wr = MapWindow(8000, 50) # right
|
||||
|
||||
assert wl.ofs_end() == 1
|
||||
assert wc.ofs_end() == 2
|
||||
assert wr.ofs_end() == 8050
|
||||
|
||||
# extension does nothing if already in place
|
||||
maxsize = 100
|
||||
wc.extend_left_to(wl, maxsize)
|
||||
assert wc.ofs == 1 and wc.size == 1
|
||||
wl.extend_right_to(wc, maxsize)
|
||||
wl.extend_right_to(wc, maxsize)
|
||||
assert wl.ofs == 0 and wl.size == 1
|
||||
|
||||
# an actual left extension
|
||||
pofs_end = wc2.ofs_end()
|
||||
wc2.extend_left_to(wc, maxsize)
|
||||
assert wc2.ofs == wc.ofs_end() and pofs_end == wc2.ofs_end()
|
||||
|
||||
# respects maxsize
|
||||
wc.extend_right_to(wr, maxsize)
|
||||
assert wc.ofs == 1 and wc.size == maxsize
|
||||
wc.extend_right_to(wr, maxsize)
|
||||
assert wc.ofs == 1 and wc.size == maxsize
|
||||
|
||||
# without maxsize
|
||||
wc.extend_right_to(wr, sys.maxsize)
|
||||
assert wc.ofs_end() == wr.ofs and wc.ofs == 1
|
||||
|
||||
# extend left
|
||||
wr.extend_left_to(wc2, maxsize)
|
||||
wr.extend_left_to(wc2, maxsize)
|
||||
assert wr.size == maxsize
|
||||
|
||||
wr.extend_left_to(wc2, sys.maxsize)
|
||||
assert wr.ofs == wc2.ofs_end()
|
||||
|
||||
wc.align()
|
||||
assert wc.ofs == 0 and wc.size == align_to_mmap(wc.size, True)
|
||||
|
||||
def test_region(self):
|
||||
with FileCreator(self.k_window_test_size, "window_test") as fc:
|
||||
half_size = fc.size // 2
|
||||
rofs = align_to_mmap(4200, False)
|
||||
rfull = MapRegion(fc.path, 0, fc.size)
|
||||
rhalfofs = MapRegion(fc.path, rofs, fc.size)
|
||||
rhalfsize = MapRegion(fc.path, 0, half_size)
|
||||
|
||||
# offsets
|
||||
assert rfull.ofs_begin() == 0 and rfull.size() == fc.size
|
||||
assert rfull.ofs_end() == fc.size # if this method works, it works always
|
||||
|
||||
assert rhalfofs.ofs_begin() == rofs and rhalfofs.size() == fc.size - rofs
|
||||
assert rhalfsize.ofs_begin() == 0 and rhalfsize.size() == half_size
|
||||
|
||||
assert rfull.includes_ofs(0) and rfull.includes_ofs(fc.size - 1) and rfull.includes_ofs(half_size)
|
||||
assert not rfull.includes_ofs(-1) and not rfull.includes_ofs(sys.maxsize)
|
||||
|
||||
# auto-refcount
|
||||
assert rfull.client_count() == 1
|
||||
rfull2 = rfull
|
||||
assert rfull.client_count() == 1, "no auto-counting"
|
||||
|
||||
# window constructor
|
||||
w = MapWindow.from_region(rfull)
|
||||
assert w.ofs == rfull.ofs_begin() and w.ofs_end() == rfull.ofs_end()
|
||||
|
||||
def test_region_list(self):
|
||||
with FileCreator(100, "sample_file") as fc:
|
||||
fd = os.open(fc.path, os.O_RDONLY)
|
||||
try:
|
||||
for item in (fc.path, fd):
|
||||
ml = MapRegionList(item)
|
||||
|
||||
assert len(ml) == 0
|
||||
assert ml.path_or_fd() == item
|
||||
assert ml.file_size() == fc.size
|
||||
finally:
|
||||
os.close(fd)
|
||||
|
||||
def test_util(self):
|
||||
assert isinstance(is_64_bit(), bool) # just call it
|
||||
assert align_to_mmap(1, False) == 0
|
||||
assert align_to_mmap(1, True) == ALLOCATIONGRANULARITY
|
||||
222
zero-cost-nas/.eggs/smmap-5.0.0-py3.8.egg/smmap/util.py
Normal file
222
zero-cost-nas/.eggs/smmap-5.0.0-py3.8.egg/smmap/util.py
Normal file
@@ -0,0 +1,222 @@
|
||||
"""Module containing a memory memory manager which provides a sliding window on a number of memory mapped files"""
|
||||
import os
|
||||
import sys
|
||||
|
||||
from mmap import mmap, ACCESS_READ
|
||||
from mmap import ALLOCATIONGRANULARITY
|
||||
|
||||
__all__ = ["align_to_mmap", "is_64_bit",
|
||||
"MapWindow", "MapRegion", "MapRegionList", "ALLOCATIONGRANULARITY"]
|
||||
|
||||
#{ Utilities
|
||||
|
||||
|
||||
def align_to_mmap(num, round_up):
|
||||
"""
|
||||
Align the given integer number to the closest page offset, which usually is 4096 bytes.
|
||||
|
||||
:param round_up: if True, the next higher multiple of page size is used, otherwise
|
||||
the lower page_size will be used (i.e. if True, 1 becomes 4096, otherwise it becomes 0)
|
||||
:return: num rounded to closest page"""
|
||||
res = (num // ALLOCATIONGRANULARITY) * ALLOCATIONGRANULARITY
|
||||
if round_up and (res != num):
|
||||
res += ALLOCATIONGRANULARITY
|
||||
# END handle size
|
||||
return res
|
||||
|
||||
|
||||
def is_64_bit():
|
||||
""":return: True if the system is 64 bit. Otherwise it can be assumed to be 32 bit"""
|
||||
return sys.maxsize > (1 << 32) - 1
|
||||
|
||||
#}END utilities
|
||||
|
||||
|
||||
#{ Utility Classes
|
||||
|
||||
class MapWindow:
|
||||
|
||||
"""Utility type which is used to snap windows towards each other, and to adjust their size"""
|
||||
__slots__ = (
|
||||
'ofs', # offset into the file in bytes
|
||||
'size' # size of the window in bytes
|
||||
)
|
||||
|
||||
def __init__(self, offset, size):
|
||||
self.ofs = offset
|
||||
self.size = size
|
||||
|
||||
def __repr__(self):
|
||||
return "MapWindow(%i, %i)" % (self.ofs, self.size)
|
||||
|
||||
@classmethod
|
||||
def from_region(cls, region):
|
||||
""":return: new window from a region"""
|
||||
return cls(region._b, region.size())
|
||||
|
||||
def ofs_end(self):
|
||||
return self.ofs + self.size
|
||||
|
||||
def align(self):
|
||||
"""Assures the previous window area is contained in the new one"""
|
||||
nofs = align_to_mmap(self.ofs, 0)
|
||||
self.size += self.ofs - nofs # keep size constant
|
||||
self.ofs = nofs
|
||||
self.size = align_to_mmap(self.size, 1)
|
||||
|
||||
def extend_left_to(self, window, max_size):
|
||||
"""Adjust the offset to start where the given window on our left ends if possible,
|
||||
but don't make yourself larger than max_size.
|
||||
The resize will assure that the new window still contains the old window area"""
|
||||
rofs = self.ofs - window.ofs_end()
|
||||
nsize = rofs + self.size
|
||||
rofs -= nsize - min(nsize, max_size)
|
||||
self.ofs = self.ofs - rofs
|
||||
self.size += rofs
|
||||
|
||||
def extend_right_to(self, window, max_size):
|
||||
"""Adjust the size to make our window end where the right window begins, but don't
|
||||
get larger than max_size"""
|
||||
self.size = min(self.size + (window.ofs - self.ofs_end()), max_size)
|
||||
|
||||
|
||||
class MapRegion:
|
||||
|
||||
"""Defines a mapped region of memory, aligned to pagesizes
|
||||
|
||||
**Note:** deallocates used region automatically on destruction"""
|
||||
__slots__ = [
|
||||
'_b', # beginning of mapping
|
||||
'_mf', # mapped memory chunk (as returned by mmap)
|
||||
'_uc', # total amount of usages
|
||||
'_size', # cached size of our memory map
|
||||
'__weakref__'
|
||||
]
|
||||
|
||||
#{ Configuration
|
||||
#} END configuration
|
||||
|
||||
def __init__(self, path_or_fd, ofs, size, flags=0):
|
||||
"""Initialize a region, allocate the memory map
|
||||
:param path_or_fd: path to the file to map, or the opened file descriptor
|
||||
:param ofs: **aligned** offset into the file to be mapped
|
||||
:param size: if size is larger then the file on disk, the whole file will be
|
||||
allocated the the size automatically adjusted
|
||||
:param flags: additional flags to be given when opening the file.
|
||||
:raise Exception: if no memory can be allocated"""
|
||||
self._b = ofs
|
||||
self._size = 0
|
||||
self._uc = 0
|
||||
|
||||
if isinstance(path_or_fd, int):
|
||||
fd = path_or_fd
|
||||
else:
|
||||
fd = os.open(path_or_fd, os.O_RDONLY | getattr(os, 'O_BINARY', 0) | flags)
|
||||
# END handle fd
|
||||
|
||||
try:
|
||||
kwargs = dict(access=ACCESS_READ, offset=ofs)
|
||||
corrected_size = size
|
||||
sizeofs = ofs
|
||||
|
||||
# have to correct size, otherwise (instead of the c version) it will
|
||||
# bark that the size is too large ... many extra file accesses because
|
||||
# if this ... argh !
|
||||
actual_size = min(os.fstat(fd).st_size - sizeofs, corrected_size)
|
||||
self._mf = mmap(fd, actual_size, **kwargs)
|
||||
# END handle memory mode
|
||||
|
||||
self._size = len(self._mf)
|
||||
finally:
|
||||
if isinstance(path_or_fd, str):
|
||||
os.close(fd)
|
||||
# END only close it if we opened it
|
||||
# END close file handle
|
||||
# We assume the first one to use us keeps us around
|
||||
self.increment_client_count()
|
||||
|
||||
def __repr__(self):
|
||||
return "MapRegion<%i, %i>" % (self._b, self.size())
|
||||
|
||||
#{ Interface
|
||||
|
||||
def buffer(self):
|
||||
""":return: a buffer containing the memory"""
|
||||
return self._mf
|
||||
|
||||
def map(self):
|
||||
""":return: a memory map containing the memory"""
|
||||
return self._mf
|
||||
|
||||
def ofs_begin(self):
|
||||
""":return: absolute byte offset to the first byte of the mapping"""
|
||||
return self._b
|
||||
|
||||
def size(self):
|
||||
""":return: total size of the mapped region in bytes"""
|
||||
return self._size
|
||||
|
||||
def ofs_end(self):
|
||||
""":return: Absolute offset to one byte beyond the mapping into the file"""
|
||||
return self._b + self._size
|
||||
|
||||
def includes_ofs(self, ofs):
|
||||
""":return: True if the given offset can be read in our mapped region"""
|
||||
return self._b <= ofs < self._b + self._size
|
||||
|
||||
def client_count(self):
|
||||
""":return: number of clients currently using this region"""
|
||||
return self._uc
|
||||
|
||||
def increment_client_count(self, ofs = 1):
|
||||
"""Adjust the usage count by the given positive or negative offset.
|
||||
If usage count equals 0, we will auto-release our resources
|
||||
:return: True if we released resources, False otherwise. In the latter case, we can still be used"""
|
||||
self._uc += ofs
|
||||
assert self._uc > -1, "Increments must match decrements, usage counter negative: %i" % self._uc
|
||||
|
||||
if self.client_count() == 0:
|
||||
self.release()
|
||||
return True
|
||||
else:
|
||||
return False
|
||||
# end handle release
|
||||
|
||||
def release(self):
|
||||
"""Release all resources this instance might hold. Must only be called if there usage_count() is zero"""
|
||||
self._mf.close()
|
||||
|
||||
#} END interface
|
||||
|
||||
|
||||
class MapRegionList(list):
|
||||
|
||||
"""List of MapRegion instances associating a path with a list of regions."""
|
||||
__slots__ = (
|
||||
'_path_or_fd', # path or file descriptor which is mapped by all our regions
|
||||
'_file_size' # total size of the file we map
|
||||
)
|
||||
|
||||
def __new__(cls, path):
|
||||
return super().__new__(cls)
|
||||
|
||||
def __init__(self, path_or_fd):
|
||||
self._path_or_fd = path_or_fd
|
||||
self._file_size = None
|
||||
|
||||
def path_or_fd(self):
|
||||
""":return: path or file descriptor we are attached to"""
|
||||
return self._path_or_fd
|
||||
|
||||
def file_size(self):
|
||||
""":return: size of file we manager"""
|
||||
if self._file_size is None:
|
||||
if isinstance(self._path_or_fd, str):
|
||||
self._file_size = os.stat(self._path_or_fd).st_size
|
||||
else:
|
||||
self._file_size = os.fstat(self._path_or_fd).st_size
|
||||
# END handle path type
|
||||
# END update file size
|
||||
return self._file_size
|
||||
|
||||
#} END utility classes
|
||||
6
zero-cost-nas/.gitignore
vendored
Normal file
6
zero-cost-nas/.gitignore
vendored
Normal file
@@ -0,0 +1,6 @@
|
||||
.vscode
|
||||
__pycache__
|
||||
*.egg-info
|
||||
*.pyc
|
||||
dist/
|
||||
build/
|
||||
201
zero-cost-nas/LICENSE
Normal file
201
zero-cost-nas/LICENSE
Normal file
@@ -0,0 +1,201 @@
|
||||
Apache License
|
||||
Version 2.0, January 2004
|
||||
http://www.apache.org/licenses/
|
||||
|
||||
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
||||
|
||||
1. Definitions.
|
||||
|
||||
"License" shall mean the terms and conditions for use, reproduction,
|
||||
and distribution as defined by Sections 1 through 9 of this document.
|
||||
|
||||
"Licensor" shall mean the copyright owner or entity authorized by
|
||||
the copyright owner that is granting the License.
|
||||
|
||||
"Legal Entity" shall mean the union of the acting entity and all
|
||||
other entities that control, are controlled by, or are under common
|
||||
control with that entity. For the purposes of this definition,
|
||||
"control" means (i) the power, direct or indirect, to cause the
|
||||
direction or management of such entity, whether by contract or
|
||||
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
||||
outstanding shares, or (iii) beneficial ownership of such entity.
|
||||
|
||||
"You" (or "Your") shall mean an individual or Legal Entity
|
||||
exercising permissions granted by this License.
|
||||
|
||||
"Source" form shall mean the preferred form for making modifications,
|
||||
including but not limited to software source code, documentation
|
||||
source, and configuration files.
|
||||
|
||||
"Object" form shall mean any form resulting from mechanical
|
||||
transformation or translation of a Source form, including but
|
||||
not limited to compiled object code, generated documentation,
|
||||
and conversions to other media types.
|
||||
|
||||
"Work" shall mean the work of authorship, whether in Source or
|
||||
Object form, made available under the License, as indicated by a
|
||||
copyright notice that is included in or attached to the work
|
||||
(an example is provided in the Appendix below).
|
||||
|
||||
"Derivative Works" shall mean any work, whether in Source or Object
|
||||
form, that is based on (or derived from) the Work and for which the
|
||||
editorial revisions, annotations, elaborations, or other modifications
|
||||
represent, as a whole, an original work of authorship. For the purposes
|
||||
of this License, Derivative Works shall not include works that remain
|
||||
separable from, or merely link (or bind by name) to the interfaces of,
|
||||
the Work and Derivative Works thereof.
|
||||
|
||||
"Contribution" shall mean any work of authorship, including
|
||||
the original version of the Work and any modifications or additions
|
||||
to that Work or Derivative Works thereof, that is intentionally
|
||||
submitted to Licensor for inclusion in the Work by the copyright owner
|
||||
or by an individual or Legal Entity authorized to submit on behalf of
|
||||
the copyright owner. For the purposes of this definition, "submitted"
|
||||
means any form of electronic, verbal, or written communication sent
|
||||
to the Licensor or its representatives, including but not limited to
|
||||
communication on electronic mailing lists, source code control systems,
|
||||
and issue tracking systems that are managed by, or on behalf of, the
|
||||
Licensor for the purpose of discussing and improving the Work, but
|
||||
excluding communication that is conspicuously marked or otherwise
|
||||
designated in writing by the copyright owner as "Not a Contribution."
|
||||
|
||||
"Contributor" shall mean Licensor and any individual or Legal Entity
|
||||
on behalf of whom a Contribution has been received by Licensor and
|
||||
subsequently incorporated within the Work.
|
||||
|
||||
2. Grant of Copyright License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
copyright license to reproduce, prepare Derivative Works of,
|
||||
publicly display, publicly perform, sublicense, and distribute the
|
||||
Work and such Derivative Works in Source or Object form.
|
||||
|
||||
3. Grant of Patent License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
(except as stated in this section) patent license to make, have made,
|
||||
use, offer to sell, sell, import, and otherwise transfer the Work,
|
||||
where such license applies only to those patent claims licensable
|
||||
by such Contributor that are necessarily infringed by their
|
||||
Contribution(s) alone or by combination of their Contribution(s)
|
||||
with the Work to which such Contribution(s) was submitted. If You
|
||||
institute patent litigation against any entity (including a
|
||||
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
||||
or a Contribution incorporated within the Work constitutes direct
|
||||
or contributory patent infringement, then any patent licenses
|
||||
granted to You under this License for that Work shall terminate
|
||||
as of the date such litigation is filed.
|
||||
|
||||
4. Redistribution. You may reproduce and distribute copies of the
|
||||
Work or Derivative Works thereof in any medium, with or without
|
||||
modifications, and in Source or Object form, provided that You
|
||||
meet the following conditions:
|
||||
|
||||
(a) You must give any other recipients of the Work or
|
||||
Derivative Works a copy of this License; and
|
||||
|
||||
(b) You must cause any modified files to carry prominent notices
|
||||
stating that You changed the files; and
|
||||
|
||||
(c) You must retain, in the Source form of any Derivative Works
|
||||
that You distribute, all copyright, patent, trademark, and
|
||||
attribution notices from the Source form of the Work,
|
||||
excluding those notices that do not pertain to any part of
|
||||
the Derivative Works; and
|
||||
|
||||
(d) If the Work includes a "NOTICE" text file as part of its
|
||||
distribution, then any Derivative Works that You distribute must
|
||||
include a readable copy of the attribution notices contained
|
||||
within such NOTICE file, excluding those notices that do not
|
||||
pertain to any part of the Derivative Works, in at least one
|
||||
of the following places: within a NOTICE text file distributed
|
||||
as part of the Derivative Works; within the Source form or
|
||||
documentation, if provided along with the Derivative Works; or,
|
||||
within a display generated by the Derivative Works, if and
|
||||
wherever such third-party notices normally appear. The contents
|
||||
of the NOTICE file are for informational purposes only and
|
||||
do not modify the License. You may add Your own attribution
|
||||
notices within Derivative Works that You distribute, alongside
|
||||
or as an addendum to the NOTICE text from the Work, provided
|
||||
that such additional attribution notices cannot be construed
|
||||
as modifying the License.
|
||||
|
||||
You may add Your own copyright statement to Your modifications and
|
||||
may provide additional or different license terms and conditions
|
||||
for use, reproduction, or distribution of Your modifications, or
|
||||
for any such Derivative Works as a whole, provided Your use,
|
||||
reproduction, and distribution of the Work otherwise complies with
|
||||
the conditions stated in this License.
|
||||
|
||||
5. Submission of Contributions. Unless You explicitly state otherwise,
|
||||
any Contribution intentionally submitted for inclusion in the Work
|
||||
by You to the Licensor shall be under the terms and conditions of
|
||||
this License, without any additional terms or conditions.
|
||||
Notwithstanding the above, nothing herein shall supersede or modify
|
||||
the terms of any separate license agreement you may have executed
|
||||
with Licensor regarding such Contributions.
|
||||
|
||||
6. Trademarks. This License does not grant permission to use the trade
|
||||
names, trademarks, service marks, or product names of the Licensor,
|
||||
except as required for reasonable and customary use in describing the
|
||||
origin of the Work and reproducing the content of the NOTICE file.
|
||||
|
||||
7. Disclaimer of Warranty. Unless required by applicable law or
|
||||
agreed to in writing, Licensor provides the Work (and each
|
||||
Contributor provides its Contributions) on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
implied, including, without limitation, any warranties or conditions
|
||||
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
||||
PARTICULAR PURPOSE. You are solely responsible for determining the
|
||||
appropriateness of using or redistributing the Work and assume any
|
||||
risks associated with Your exercise of permissions under this License.
|
||||
|
||||
8. Limitation of Liability. In no event and under no legal theory,
|
||||
whether in tort (including negligence), contract, or otherwise,
|
||||
unless required by applicable law (such as deliberate and grossly
|
||||
negligent acts) or agreed to in writing, shall any Contributor be
|
||||
liable to You for damages, including any direct, indirect, special,
|
||||
incidental, or consequential damages of any character arising as a
|
||||
result of this License or out of the use or inability to use the
|
||||
Work (including but not limited to damages for loss of goodwill,
|
||||
work stoppage, computer failure or malfunction, or any and all
|
||||
other commercial damages or losses), even if such Contributor
|
||||
has been advised of the possibility of such damages.
|
||||
|
||||
9. Accepting Warranty or Additional Liability. While redistributing
|
||||
the Work or Derivative Works thereof, You may choose to offer,
|
||||
and charge a fee for, acceptance of support, warranty, indemnity,
|
||||
or other liability obligations and/or rights consistent with this
|
||||
License. However, in accepting such obligations, You may act only
|
||||
on Your own behalf and on Your sole responsibility, not on behalf
|
||||
of any other Contributor, and only if You agree to indemnify,
|
||||
defend, and hold each Contributor harmless for any liability
|
||||
incurred by, or claims asserted against, such Contributor by reason
|
||||
of your accepting any such warranty or additional liability.
|
||||
|
||||
END OF TERMS AND CONDITIONS
|
||||
|
||||
APPENDIX: How to apply the Apache License to your work.
|
||||
|
||||
To apply the Apache License to your work, attach the following
|
||||
boilerplate notice, with the fields enclosed by brackets "[]"
|
||||
replaced with your own identifying information. (Don't include
|
||||
the brackets!) The text should be enclosed in the appropriate
|
||||
comment syntax for the file format. We also recommend that a
|
||||
file or class name and description of purpose be included on the
|
||||
same "printed page" as the copyright notice for easier
|
||||
identification within third-party archives.
|
||||
|
||||
Copyright [yyyy] [name of copyright owner]
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
95
zero-cost-nas/README.md
Normal file
95
zero-cost-nas/README.md
Normal file
@@ -0,0 +1,95 @@
|
||||
# Zero-Cost-NAS
|
||||
Companion code for the ICLR2021 paper: [Zero-Cost Proxies for Lightweight NAS](https://openreview.net/forum?id=0cmMMy8J5q)
|
||||
**tl;dr A single minibatch of data is used to score neural networks for NAS instead of performing full training.**
|
||||
|
||||
In this README, we provide:
|
||||
- [Summary of our work](#Summary)
|
||||
- [How to run the code](#Running-the-Code)
|
||||
- [API](#API)
|
||||
- [Reproducing results on NAS benchmrks](#Reproducing-Results)
|
||||
- [Incorporate with NAS algorithms](#NAS-with-Zero-Cost-Proxies)
|
||||
|
||||
**If you have any questions, please open an issue or email us.** (last update: 02.02.2021)
|
||||
|
||||
## Summary
|
||||
|
||||
**Intro.** To perform neural architecture search (NAS), deep neural networks (DNNs) are typically trained until a final validation accuracy is computed and used to compare DNNs to each other and select the best one.
|
||||
However, this is time-consuming because training takes multiple GPU-hours/days/weeks.
|
||||
This is why a _proxy_ for final accuracy is often used to speed up NAS.
|
||||
Typically, this proxy is a reduced form of training (e.g. EcoNAS) where the number of epochs is reduced, a smaller model is used or the training data is subsampled.
|
||||
|
||||
**Proxies.** Instead, we propose a series of "zero-cost" proxies that use a single-minibatch of data to score a DNN.
|
||||
These metrics are inspired by recent pruning-at-initialization literature, but are adapted to score an entire DNN and work within a NAS setting.
|
||||
When compared against `econas` (see orange pentagon in plot below), our zero-cost metrics take ~1000X less time to run but are better-correlated with final validation accuracy (especially `synflow` and `jacob_cov`), making them better (and much cheaper!) proxies for use within NAS.
|
||||
Even when EcoNAS is tuned specifically for NAS-Bench-201 (see `econas+` purple circle in the plot), our `vote` zero-cost proxy is still better-correlated and is 3 orders of magnitude cheaper to compute.
|
||||
|
||||
_Figure 1: Correlation of validation accuracy to final accuracy during the first 12 epochs of training (blue line) for three CIFAR-10 on the NAS-Bench-201 search space. Zero-cost and EcoNAS proxies are also labeled for comparison._
|
||||
|
||||
<img src="images/nasbench201_comparison.JPG" width=350 alt="zero-cost vs econas">
|
||||
|
||||
**Zero-Cost NAS** We use the zero-cost metrics to enhance 4 existing NAS algorithms, and we test it out on 3 different NAS benchmarks. For all cases, we achieve a new SOTA (state of the art result) in terms of search speed. We incorporate zero-cost proxies in two ways: (1) warmup: Use proxies to initialize NAS algorithms, (2) move proposal: Use proxies to improve the selection of the next model for evaluation. As Figure 2 shows, there is a significant speedup to all evaluated NAS algorithms.
|
||||
|
||||
_Figure 2: Zero-Cost warmup and move proposal consistently improves speed and accuracy of 4 different NAS algorithms._
|
||||
|
||||
<img src="images/nasbench201_search_speedup.JPG" width=700 alt="Zero-Cost-NAS speedup">
|
||||
|
||||
For more details, please take a look at our [paper](https://openreview.net/pdf?id=0cmMMy8J5q)!
|
||||
|
||||
## Running the Code
|
||||
|
||||
- Install [PyTorch](https://pytorch.org/) for your system (v1.5.0 or later).
|
||||
- Install the package: `pip install .` (add `-e` for editable mode) -- note that all dependencies other than pytorch will be automatically installed.
|
||||
|
||||
### API
|
||||
|
||||
The main function is `find_measures` below. Given a neural net and some information about the input data (`dataloader`) and loss function (`loss_fn`) it returns an array of zero-cost proxy metrics.
|
||||
|
||||
```python
|
||||
def find_measures(net_orig, # neural network
|
||||
dataloader, # a data loader (typically for training data)
|
||||
dataload_info, # a tuple with (dataload_type = {random, grasp}, number_of_batches_for_random_or_images_per_class_for_grasp, number of classes)
|
||||
device, # GPU/CPU device used
|
||||
loss_fn=F.cross_entropy, # loss function to use within the zero-cost metrics
|
||||
measure_names=None, # an array of measure names to compute, if left blank, all measures are computed by default
|
||||
measures_arr=None): # [not used] if the measures are already computed but need to be summarized, pass them here
|
||||
```
|
||||
|
||||
The available zero-cost metrics are in the [measures](foresight/pruners/measures) directory. You can add new metrics by simply following one of the examples then registering the metric in the [load_all](https://github.sec.samsung.net/mohamed1-a/foresight-nas/blob/29ec5ad17496fb6bb24b27dbc782db1615214b0f/foresight/pruners/measures/__init__.py#L35) function. More examples of how to use this function can be found in the code to reproduce results (below). You can also modify data loading functions in [p_utils.py](foresight/pruners/p_utils.py)
|
||||
|
||||
### Reproducing Results
|
||||
|
||||
#### NAS-Bench-201
|
||||
|
||||
1. Download the [NAS-Bench-201 dataset](https://drive.google.com/open?id=1SKW0Cu0u8-gb18zDpaAGi0f74UdXeGKs) and put in the `data` directory in the root folder of this project.
|
||||
2. Run python `nasbench2_pred.py` with the appropriate cmd-line options -- a pickle file is produced with zero-cost metrics (see `notebooks` folder on how to use the pickle file.
|
||||
3. Note that you need to manually download [ImageNet16](https://drive.google.com/drive/folders/1NE63Vdo2Nia0V7LK1CdybRLjBFY72w40?usp=sharing) and put in `_datasets/ImageNet16` directory in the root folder. CIFAR-10/100 will be automatically downloaded.
|
||||
|
||||
#### NAS-Bench-101
|
||||
|
||||
1. Download the [`data` directory](https://drive.google.com/drive/folders/18Eia6YuTE5tn5Lis_43h30HYpnF9Ynqf?usp=sharing) and save it to the root folder of this repo. This contains pre-cached info from the NAS-Bench-101 repo.
|
||||
2. [Optional] Download the [NAS-Bench-101 dataset](https://storage.googleapis.com/nasbench/nasbench_only108.tfrecord) and put in the `data` directory in the root folder of this project and also clone the [NAS-Bench-101 repo](https://github.com/google-research/nasbench) and install the package.
|
||||
3. Run `python nasbench1_pred.py`. Note that this takes a long time to go through ~400k architectures, but precomputed results are in the `notebooks` folder (with a link to the [results](https://drive.google.com/drive/folders/1fUBaTd05OHrKIRs-x9Fx8Zsk5QqErks8?usp=sharing)).
|
||||
|
||||
#### PyTorchCV
|
||||
|
||||
1. Run python `ptcv_pred.py`
|
||||
|
||||
#### NAS-Bench-ASR
|
||||
|
||||
Coming soon...
|
||||
|
||||
### NAS with Zero-Cost Proxies
|
||||
|
||||
For the full list of NAS algorithms in our paper, we used a different NAS tool which is not publicly released. However, we included a notebook [`nas_examples.ipynb`](notebooks/nas_examples.ipynb) to show how to use zero-cost proxies to speed up aging evolution and random search methods using both warmup and move proposal.
|
||||
|
||||
## Citation
|
||||
|
||||
```
|
||||
@inproceedings{
|
||||
abdelfattah2021zerocost,
|
||||
title={{Zero-Cost Proxies for Lightweight NAS}},
|
||||
author={Mohamed S. Abdelfattah and Abhinav Mehrotra and {\L}ukasz Dudziak and Nicholas D. Lane},
|
||||
booktitle={International Conference on Learning Representations (ICLR)},
|
||||
year={2021}
|
||||
}
|
||||
```
|
||||
16
zero-cost-nas/foresight/__init__.py
Normal file
16
zero-cost-nas/foresight/__init__.py
Normal file
@@ -0,0 +1,16 @@
|
||||
# Copyright 2021 Samsung Electronics Co., Ltd.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# =============================================================================
|
||||
|
||||
from .version import *
|
||||
121
zero-cost-nas/foresight/dataset.py
Normal file
121
zero-cost-nas/foresight/dataset.py
Normal file
@@ -0,0 +1,121 @@
|
||||
|
||||
# Copyright 2021 Samsung Electronics Co., Ltd.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# =============================================================================
|
||||
|
||||
from torch.utils.data import DataLoader
|
||||
from torchvision.datasets import MNIST, CIFAR10, CIFAR100, SVHN
|
||||
from torchvision.transforms import Compose, ToTensor, Normalize
|
||||
from torchvision import transforms
|
||||
|
||||
from .imagenet16 import *
|
||||
|
||||
|
||||
def get_cifar_dataloaders(train_batch_size, test_batch_size, dataset, num_workers, resize=None, datadir='_dataset'):
|
||||
|
||||
if 'ImageNet16' in dataset:
|
||||
mean = [x / 255 for x in [122.68, 116.66, 104.01]]
|
||||
std = [x / 255 for x in [63.22, 61.26 , 65.09]]
|
||||
size, pad = 16, 2
|
||||
elif 'cifar' in dataset:
|
||||
mean = (0.4914, 0.4822, 0.4465)
|
||||
std = (0.2023, 0.1994, 0.2010)
|
||||
size, pad = 32, 4
|
||||
elif 'svhn' in dataset:
|
||||
mean = (0.5, 0.5, 0.5)
|
||||
std = (0.5, 0.5, 0.5)
|
||||
size, pad = 32, 0
|
||||
elif dataset == 'ImageNet1k':
|
||||
from .h5py_dataset import H5Dataset
|
||||
size,pad = 224,2
|
||||
mean = (0.485, 0.456, 0.406)
|
||||
std = (0.229, 0.224, 0.225)
|
||||
#resize = 256
|
||||
|
||||
if resize is None:
|
||||
resize = size
|
||||
|
||||
train_transform = transforms.Compose([
|
||||
transforms.RandomCrop(size, padding=pad),
|
||||
transforms.Resize(resize),
|
||||
transforms.RandomHorizontalFlip(),
|
||||
transforms.ToTensor(),
|
||||
transforms.Normalize(mean,std),
|
||||
])
|
||||
|
||||
test_transform = transforms.Compose([
|
||||
transforms.Resize(resize),
|
||||
transforms.ToTensor(),
|
||||
transforms.Normalize(mean,std),
|
||||
])
|
||||
|
||||
if dataset == 'cifar10':
|
||||
train_dataset = CIFAR10(datadir, True, train_transform, download=True)
|
||||
test_dataset = CIFAR10(datadir, False, test_transform, download=True)
|
||||
elif dataset == 'cifar100':
|
||||
train_dataset = CIFAR100(datadir, True, train_transform, download=True)
|
||||
test_dataset = CIFAR100(datadir, False, test_transform, download=True)
|
||||
elif dataset == 'svhn':
|
||||
train_dataset = SVHN(datadir, split='train', transform=train_transform, download=True)
|
||||
test_dataset = SVHN(datadir, split='test', transform=test_transform, download=True)
|
||||
elif dataset == 'ImageNet16-120':
|
||||
train_dataset = ImageNet16(os.path.join(datadir, 'ImageNet16'), True , train_transform, 120)
|
||||
test_dataset = ImageNet16(os.path.join(datadir, 'ImageNet16'), False, test_transform , 120)
|
||||
elif dataset == 'ImageNet1k':
|
||||
train_dataset = H5Dataset(os.path.join(datadir, 'imagenet-train-256.h5'), transform=train_transform)
|
||||
test_dataset = H5Dataset(os.path.join(datadir, 'imagenet-val-256.h5'), transform=test_transform)
|
||||
|
||||
else:
|
||||
raise ValueError('There are no more cifars or imagenets.')
|
||||
|
||||
train_loader = DataLoader(
|
||||
train_dataset,
|
||||
train_batch_size,
|
||||
shuffle=True,
|
||||
num_workers=num_workers,
|
||||
pin_memory=True)
|
||||
test_loader = DataLoader(
|
||||
test_dataset,
|
||||
test_batch_size,
|
||||
shuffle=False,
|
||||
num_workers=num_workers,
|
||||
pin_memory=True)
|
||||
|
||||
return train_loader, test_loader
|
||||
|
||||
|
||||
def get_mnist_dataloaders(train_batch_size, val_batch_size, num_workers):
|
||||
|
||||
data_transform = Compose([transforms.ToTensor()])
|
||||
|
||||
# Normalise? transforms.Normalize((0.1307,), (0.3081,))
|
||||
|
||||
train_dataset = MNIST("_dataset", True, data_transform, download=True)
|
||||
test_dataset = MNIST("_dataset", False, data_transform, download=True)
|
||||
|
||||
train_loader = DataLoader(
|
||||
train_dataset,
|
||||
train_batch_size,
|
||||
shuffle=True,
|
||||
num_workers=num_workers,
|
||||
pin_memory=True)
|
||||
test_loader = DataLoader(
|
||||
test_dataset,
|
||||
val_batch_size,
|
||||
shuffle=False,
|
||||
num_workers=num_workers,
|
||||
pin_memory=True)
|
||||
|
||||
return train_loader, test_loader
|
||||
|
||||
55
zero-cost-nas/foresight/h5py_dataset.py
Normal file
55
zero-cost-nas/foresight/h5py_dataset.py
Normal file
@@ -0,0 +1,55 @@
|
||||
# Copyright 2021 Samsung Electronics Co., Ltd.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# =============================================================================
|
||||
|
||||
import h5py
|
||||
import numpy as np
|
||||
from PIL import Image
|
||||
|
||||
|
||||
import torch
|
||||
from torch.utils.data import Dataset, DataLoader
|
||||
|
||||
class H5Dataset(Dataset):
|
||||
def __init__(self, h5_path, transform=None):
|
||||
self.h5_path = h5_path
|
||||
self.h5_file = None
|
||||
self.length = len(h5py.File(h5_path, 'r'))
|
||||
self.transform = transform
|
||||
|
||||
def __getitem__(self, index):
|
||||
|
||||
#loading in getitem allows us to use multiple processes for data loading
|
||||
#because hdf5 files aren't pickelable so can't transfer them across processes
|
||||
# https://discuss.pytorch.org/t/hdf5-a-data-format-for-pytorch/40379
|
||||
# https://discuss.pytorch.org/t/dataloader-when-num-worker-0-there-is-bug/25643/16
|
||||
# TODO possible look at __getstate__ and __setstate__ as a more elegant solution
|
||||
if self.h5_file is None:
|
||||
self.h5_file = h5py.File(self.h5_path, 'r')
|
||||
|
||||
record = self.h5_file[str(index)]
|
||||
|
||||
if self.transform:
|
||||
x = Image.fromarray(record['data'][()])
|
||||
x = self.transform(x)
|
||||
else:
|
||||
x = torch.from_numpy(record['data'][()])
|
||||
|
||||
y = record['target'][()]
|
||||
y = torch.from_numpy(np.asarray(y))
|
||||
|
||||
return (x,y)
|
||||
|
||||
def __len__(self):
|
||||
return self.length
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user