cloudpathlib.S3Path

Bases: CloudPath

Class for representing and operating on AWS S3 URIs, in the style of the Python standard library's pathlib module. Instances represent a path in S3 with filesystem path semantics, and convenient methods allow for basic operations like joining, reading, writing, iterating over contents, etc. This class almost entirely mimics the pathlib.Path interface, so most familiar properties and methods should be available and behave in the expected way.

The S3Client class handles authentication with AWS. If a client instance is not explicitly specified on S3Path instantiation, a default client is used. See S3Client's documentation for more details.

Source code in cloudpathlib/s3/s3path.py

@register_path_class("s3")
class S3Path(CloudPath):
    """Class for representing and operating on AWS S3 URIs, in the style of the Python standard
    library's [`pathlib` module](https://docs.python.org/3/library/pathlib.html). Instances
    represent a path in S3 with filesystem path semantics, and convenient methods allow for basic
    operations like joining, reading, writing, iterating over contents, etc. This class almost
    entirely mimics the [`pathlib.Path`](https://docs.python.org/3/library/pathlib.html#pathlib.Path)
    interface, so most familiar properties and methods should be available and behave in the
    expected way.

    The [`S3Client`](../s3client/) class handles authentication with AWS. If a client instance is
    not explicitly specified on `S3Path` instantiation, a default client is used. See `S3Client`'s
    documentation for more details.
    """

    cloud_prefix: str = "s3://"
    client: "S3Client"
    _bucket: str
    _local_path: Path

    @property
    def drive(self) -> str:
        return self.bucket

    def mkdir(
        self, parents=False, exist_ok=False, mode: Optional[Any] = None, *, parent_mode=None
    ):
        # not possible to make empty directory on s3
        pass

    def touch(self, exist_ok: bool = True, mode: Optional[Any] = None):
        if self.exists():
            if not exist_ok:
                raise FileExistsError(f"File exists: {self}")
            self.client._move_file(self, self)
        else:
            tf = TemporaryDirectory()
            p = Path(tf.name) / "empty"
            p.touch()

            self.client._upload_file(p, self)

            tf.cleanup()

    def stat(self, follow_symlinks=True):
        try:
            meta = self.client._get_metadata(self)
        except self.client.client.exceptions.ClientError as error:
            # head_object returns a 404 (not NoSuchKey) for a missing key; let other errors raise
            error_info = error.response.get("Error", {})
            status_code = error.response.get("ResponseMetadata", {}).get("HTTPStatusCode")
            if error_info.get("Code") in ("404", "NoSuchKey") or status_code == 404:
                raise NoStatError(
                    f"No stats available for {self}; it may be a directory or not exist."
                )
            raise

        return os.stat_result(
            (
                None,  # mode
                None,  # ino
                self.cloud_prefix,  # dev,
                None,  # nlink,
                None,  # uid,
                None,  # gid,
                meta.get("size", 0),  # size,
                None,  # atime,
                meta.get("last_modified", 0).timestamp(),  # mtime,
                None,  # ctime,
            )
        )

    @property
    def bucket(self) -> str:
        """The bucket name, or the full MRAP ARN for MRAP paths.

        :type: :class:`str`
        """
        if hasattr(self, "_bucket"):
            return self._bucket
        if match := _MRAP_PATTERN.match(str(self)):
            self._bucket = match.group("arn")
        else:
            self._bucket = self._no_prefix.split("/", 1)[0]
        return self._bucket

    @property
    def key(self) -> str:
        key = self._no_prefix_no_drive

        # key should never have starting slash for
        # use with boto, etc.
        if key.startswith("/"):
            key = key[1:]

        return key

    @property
    def etag(self):
        return self.client._get_metadata(self).get("etag")

    @property
    def _local(self) -> Path:
        if hasattr(self, "_local_path"):
            return self._local_path
        no_prefix = self._no_prefix
        # `:` is invalid in Windows paths; percent-encode it for MRAP ARNs
        if sys.platform == "win32":
            no_prefix = no_prefix.replace(":", "%3A")
        self._local_path = _ensure_local_path_within_base(
            self.client._local_cache_dir / no_prefix,
            self.client._local_cache_dir,
            self,
            resolve=False,
        )
        return self._local_path

Attributes

anchor: str property

bucket: str property

The bucket name, or the full MRAP ARN for MRAP paths.

:type: :class:str

client: S3Client instance-attribute

cloud_prefix: str = 's3://' class-attribute instance-attribute

drive: str property

etag property

fspath: str property

key: str property

name: str property

parent: Self property

parents: Tuple[Self, ...] property

parser: Self property

parts: Tuple[str, ...] property

stem: str property

suffix: str property

suffixes: List[str] property

Methods:

__init__(cloud_path: Union[str, Self, CloudPath], *parts: str, client: Optional[Client] = None) -> None

Source code in cloudpathlib/cloudpath.py

def __init__(
    self,
    cloud_path: Union[str, Self, "CloudPath"],
    *parts: str,
    client: Optional["Client"] = None,
) -> None:
    # handle if local file gets opened. must be set at the top of the method in case any code
    # below raises an exception, this prevents __del__ from raising an AttributeError
    self._handle: Optional[IO] = None
    self._client: Optional["Client"] = None

    if parts:
        # ensure first part ends in "/"; (sometimes it is just prefix, sometimes a longer path)
        if not str(cloud_path).endswith("/"):
            cloud_path = str(cloud_path) + "/"

        cloud_path = str(cloud_path) + "/".join(p.strip("/") for p in parts)

    self.is_valid_cloudpath(cloud_path, raise_on_error=True)
    self._cloud_meta.validate_completeness()

    # versions of the raw string that provide useful methods
    self._str = str(cloud_path)
    self._url = urlparse(self._str)
    self._path = PurePosixPath(f"/{self._no_prefix}")

    # setup client
    if client is None:
        if isinstance(cloud_path, CloudPath):
            self._client = cloud_path.client
    else:
        self._client = client

    if client is not None and not isinstance(client, self._cloud_meta.client_class):
        raise ClientMismatchError(
            f"Client of type [{client.__class__}] is not valid for cloud path of type "
            f"[{self.__class__}]; must be instance of [{self._cloud_meta.client_class}], or "
            f"None to use default client for this cloud path class."
        )

    # track if local has been written to, if so it may need to be uploaded
    self._dirty = False

absolute() -> Self

Source code in cloudpathlib/cloudpath.py

def absolute(self) -> Self:
    return self

as_uri() -> str

Source code in cloudpathlib/cloudpath.py

def as_uri(self) -> str:
    return str(self)

as_url(presign: bool = False, expire_seconds: int = 60 * 60) -> str

Source code in cloudpathlib/cloudpath.py

def as_url(self, presign: bool = False, expire_seconds: int = 60 * 60) -> str:
    if presign:
        url = self.client._generate_presigned_url(self, expire_seconds=expire_seconds)
    else:
        url = self.client._get_public_url(self)
    return url

clear_cache()

Removes cache if it exists

Source code in cloudpathlib/cloudpath.py

def clear_cache(self):
    """Removes cache if it exists"""
    try:
        local = self._local
    except CloudPathLocalPathTraversalError:
        # a key that cannot be safely mapped to a local path can never have been cached,
        # so there is nothing to clear (and __del__ must not raise on such paths)
        return
    if local.exists():
        if local.is_file():
            local.unlink()
        else:
            shutil.rmtree(local)

copy(target: Union[str, os.PathLike, Self], follow_symlinks: bool = True, preserve_metadata: bool = False, force_overwrite_to_cloud: Optional[bool] = None) -> Union[Path, Self]

copy(
    target: Self,
    follow_symlinks=True,
    preserve_metadata=False,
    force_overwrite_to_cloud: Optional[bool] = None,
) -> Self

copy(
    target: Path,
    follow_symlinks=True,
    preserve_metadata=False,
    force_overwrite_to_cloud: Optional[bool] = None,
) -> Path

copy(
    target: str,
    follow_symlinks=True,
    preserve_metadata=False,
    force_overwrite_to_cloud: Optional[bool] = None,
) -> Union[Path, CloudPath]

Copy self to target folder or file, if self is a file.

Source code in cloudpathlib/cloudpath.py

def copy(
    self,
    target: Union[str, os.PathLike, Self],
    follow_symlinks: bool = True,
    preserve_metadata: bool = False,
    force_overwrite_to_cloud: Optional[bool] = None,
) -> Union[Path, Self]:
    """Copy self to target folder or file, if self is a file."""
    return self._copy(
        target,
        follow_symlinks=follow_symlinks,
        preserve_metadata=preserve_metadata,
        force_overwrite_to_cloud=force_overwrite_to_cloud,
        remove_src=False,
    )

copy_into(target_dir: Union[str, os.PathLike, Self], follow_symlinks: bool = True, preserve_metadata: bool = False, force_overwrite_to_cloud: Optional[bool] = None) -> Union[Path, Self]

copy_into(
    target_dir: Self,
    follow_symlinks: bool = True,
    preserve_metadata: bool = False,
    force_overwrite_to_cloud: Optional[bool] = None,
) -> Self

copy_into(
    target_dir: Path,
    follow_symlinks: bool = True,
    preserve_metadata: bool = False,
    force_overwrite_to_cloud: Optional[bool] = None,
) -> Path

copy_into(
    target_dir: str,
    follow_symlinks: bool = True,
    preserve_metadata: bool = False,
    force_overwrite_to_cloud: Optional[bool] = None,
) -> Union[Path, CloudPath]

Copy self into target directory, preserving the filename.

Source code in cloudpathlib/cloudpath.py

def copy_into(
    self,
    target_dir: Union[str, os.PathLike, Self],
    follow_symlinks: bool = True,
    preserve_metadata: bool = False,
    force_overwrite_to_cloud: Optional[bool] = None,
) -> Union[Path, Self]:
    """Copy self into target directory, preserving the filename."""
    target_path = anypath.to_anypath(target_dir) / self.name

    result = self._copy(
        target_path,
        follow_symlinks=follow_symlinks,
        preserve_metadata=preserve_metadata,
        force_overwrite_to_cloud=force_overwrite_to_cloud,
        remove_src=False,
    )
    return cast(Union[Path, Self], result)

copytree(destination, force_overwrite_to_cloud=None, ignore=None)

copytree(
    destination: Self,
    force_overwrite_to_cloud: Optional[bool] = None,
    ignore: Optional[
        Callable[[str, Iterable[str]], Container[str]]
    ] = None,
) -> Self

copytree(
    destination: Path,
    force_overwrite_to_cloud: Optional[bool] = None,
    ignore: Optional[
        Callable[[str, Iterable[str]], Container[str]]
    ] = None,
) -> Path

copytree(
    destination: Union[str, os.PathLike, Self],
    force_overwrite_to_cloud: Optional[bool] = None,
    ignore: Optional[
        Callable[[str, Iterable[str]], Container[str]]
    ] = None,
) -> Union[Path, CloudPath]

Copy self to a directory, if self is a directory.

Source code in cloudpathlib/cloudpath.py

def copytree(self, destination, force_overwrite_to_cloud=None, ignore=None):
    """Copy self to a directory, if self is a directory."""
    if not self.is_dir():
        raise CloudPathNotADirectoryError(
            f"Origin path {self} must be a directory. To copy a single file use the method copy."
        )

    destination = anypath.to_anypath(destination)

    if destination.exists() and destination.is_file():
        raise CloudPathFileExistsError(
            f"Destination path {destination} of copytree must be a directory."
        )

    contents = list(self.iterdir())

    if ignore is not None:
        ignored_names = ignore(self._no_prefix_no_drive, [x.name for x in contents])
    else:
        ignored_names = set()

    destination.mkdir(parents=True, exist_ok=True)

    # when copying to a local destination, a listing may surface a key's ".." segment as a
    # directory entry named "..", so each child destination must be confirmed within the
    # destination before recursing (otherwise the recursion walks outside it); cloud-to-cloud
    # copytree is not a local-filesystem traversal concern
    destination_is_local = isinstance(destination, Path)

    for subpath in contents:
        if subpath.name in ignored_names:
            continue
        if subpath.is_file():
            file_destination = destination / subpath.name
            if destination_is_local:
                _ensure_local_path_within_base(file_destination, destination, subpath)
            subpath.copy(file_destination, force_overwrite_to_cloud=force_overwrite_to_cloud)
        elif subpath.is_dir():
            dir_destination = destination / (
                subpath.name + ("" if subpath.name.endswith("/") else "/")
            )
            if destination_is_local:
                _ensure_local_path_within_base(dir_destination, destination, subpath)
            subpath.copytree(
                dir_destination,
                force_overwrite_to_cloud=force_overwrite_to_cloud,
                ignore=ignore,
            )

    return destination

download_to(destination: Union[str, os.PathLike]) -> Path

Source code in cloudpathlib/cloudpath.py

def download_to(self, destination: Union[str, os.PathLike]) -> Path:
    destination = Path(destination)

    if not self.exists():
        raise CloudPathNotExistsError(f"Cannot download because path does not exist: {self}")

    if self.is_file():
        if destination.is_dir():
            destination = _ensure_local_path_within_base(
                destination / self.name, destination, self
            )
        return self.client._download_file(self, destination)
    else:
        destination.mkdir(exist_ok=True)
        for f in self.iterdir():
            rel = str(self)
            if not rel.endswith("/"):
                rel = rel + "/"

            rel_dest = str(f)[len(rel) :]
            target = _ensure_local_path_within_base(destination / rel_dest, destination, f)
            f.download_to(target)

        return destination

exists(follow_symlinks=True) -> bool

Source code in cloudpathlib/cloudpath.py

def exists(self, follow_symlinks=True) -> bool:
    return self.client._exists(self)

from_uri(uri: str) -> Self classmethod

Source code in cloudpathlib/cloudpath.py

@classmethod
def from_uri(cls, uri: str) -> Self:
    return cls(uri)

full_match(pattern: str, case_sensitive: Optional[bool] = None) -> bool

Source code in cloudpathlib/cloudpath.py

def full_match(self, pattern: str, case_sensitive: Optional[bool] = None) -> bool:
    if sys.version_info < (3, 13):
        raise NotImplementedError("full_match requires Python 3.13 or higher")

    # strip scheme from start of pattern before testing
    if pattern.startswith(self.anchor + self.drive):
        pattern = pattern[len(self.anchor + self.drive) :]
    elif pattern.startswith(self.anchor):
        # for http paths, keep leading slash
        pattern = pattern[len(self.anchor) - 1 :]

    # remove drive, which is kept on normal dispatch to pathlib
    return PurePosixPath(self._no_prefix_no_drive).full_match(  # type: ignore[attr-defined]
        pattern, case_sensitive=case_sensitive
    )

glob(pattern: Union[str, os.PathLike], case_sensitive: Optional[bool] = None, recurse_symlinks: bool = True) -> Generator[Self, None, None]

Source code in cloudpathlib/cloudpath.py

def glob(
    self,
    pattern: Union[str, os.PathLike],
    case_sensitive: Optional[bool] = None,
    recurse_symlinks: bool = True,
) -> Generator[Self, None, None]:
    pattern = self._glob_checks(pattern)

    pattern_parts = PurePosixPath(pattern).parts
    selector = _make_selector(
        tuple(pattern_parts), _posix_flavour, case_sensitive=case_sensitive
    )

    yield from self._glob(
        selector,
        "/" in pattern
        or "**"
        in pattern,  # recursive listing needed if explicit ** or any sub folder in pattern
    )

info() -> CloudPathInfo

Return a CloudPathInfo object for this path.

Source code in cloudpathlib/cloudpath.py

def info(self) -> "CloudPathInfo":
    """Return a CloudPathInfo object for this path."""
    return CloudPathInfo(self)

is_absolute() -> bool

Source code in cloudpathlib/cloudpath.py

def is_absolute(self) -> bool:
    return True

is_dir(follow_symlinks=True) -> bool

Source code in cloudpathlib/cloudpath.py

def is_dir(self, follow_symlinks=True) -> bool:
    return self.client._is_file_or_dir(self) == "dir"

is_file(follow_symlinks=True) -> bool

Source code in cloudpathlib/cloudpath.py

def is_file(self, follow_symlinks=True) -> bool:
    return self.client._is_file_or_dir(self) == "file"

is_junction()

Source code in cloudpathlib/cloudpath.py

def is_junction(self):
    return False  # only windows paths can be junctions, not cloudpaths

is_relative_to(other: Self) -> bool

Source code in cloudpathlib/cloudpath.py

def is_relative_to(self, other: Self) -> bool:
    try:
        self.relative_to(other)
        return True
    except ValueError:
        return False

is_valid_cloudpath(path: Union[str, CloudPath], raise_on_error: bool = False) -> Union[bool, TypeGuard[Self]] classmethod

is_valid_cloudpath(
    path: CloudPath, raise_on_error: bool = ...
) -> TypeGuard[Self]

is_valid_cloudpath(
    path: str, raise_on_error: bool = ...
) -> bool

Source code in cloudpathlib/cloudpath.py

@classmethod
def is_valid_cloudpath(
    cls, path: Union[str, "CloudPath"], raise_on_error: bool = False
) -> Union[bool, TypeGuard[Self]]:
    valid = str(path).lower().startswith(cls.cloud_prefix.lower())

    if raise_on_error and not valid:
        raise InvalidPrefixError(
            f"'{path}' is not a valid path since it does not start with '{cls.cloud_prefix}'"
        )

    return valid

iterdir() -> Generator[Self, None, None]

Source code in cloudpathlib/cloudpath.py

def iterdir(self) -> Generator[Self, None, None]:
    for f, _ in self.client._list_dir(self, recursive=False):
        if f != self:  # iterdir does not include itself in pathlib
            yield f

joinpath(*pathsegments: Union[str, os.PathLike]) -> Self

Source code in cloudpathlib/cloudpath.py

def joinpath(self, *pathsegments: Union[str, os.PathLike]) -> Self:
    return self._dispatch_to_path("joinpath", *pathsegments)

match(path_pattern: str, case_sensitive: Optional[bool] = None) -> bool

Source code in cloudpathlib/cloudpath.py

def match(self, path_pattern: str, case_sensitive: Optional[bool] = None) -> bool:
    # strip scheme from start of pattern before testing
    if path_pattern.startswith(self.anchor + self.drive + "/"):
        path_pattern = path_pattern[len(self.anchor + self.drive + "/") :]

    kwargs = dict(case_sensitive=case_sensitive)

    if sys.version_info < (3, 12):
        kwargs.pop("case_sensitive")

    return self._dispatch_to_path("match", path_pattern, **kwargs)

mkdir(parents=False, exist_ok=False, mode: Optional[Any] = None, *, parent_mode=None)

Source code in cloudpathlib/s3/s3path.py

def mkdir(
    self, parents=False, exist_ok=False, mode: Optional[Any] = None, *, parent_mode=None
):
    # not possible to make empty directory on s3
    pass

move(target: Union[str, os.PathLike, Self], follow_symlinks: bool = True, preserve_metadata: bool = False, force_overwrite_to_cloud: Optional[bool] = None) -> Union[Path, Self]

move(
    target: Self,
    follow_symlinks: bool = True,
    preserve_metadata: bool = False,
    force_overwrite_to_cloud: Optional[bool] = None,
) -> Self

move(
    target: Path,
    follow_symlinks: bool = True,
    preserve_metadata: bool = False,
    force_overwrite_to_cloud: Optional[bool] = None,
) -> Path

move(
    target: str,
    follow_symlinks: bool = True,
    preserve_metadata: bool = False,
    force_overwrite_to_cloud: Optional[bool] = None,
) -> Union[Path, CloudPath]

Move self to target location, removing the source.

Source code in cloudpathlib/cloudpath.py

def move(
    self,
    target: Union[str, os.PathLike, Self],
    follow_symlinks: bool = True,
    preserve_metadata: bool = False,
    force_overwrite_to_cloud: Optional[bool] = None,
) -> Union[Path, Self]:
    """Move self to target location, removing the source."""
    return self._copy(
        target,
        follow_symlinks=follow_symlinks,
        preserve_metadata=preserve_metadata,
        force_overwrite_to_cloud=force_overwrite_to_cloud,
        remove_src=True,
    )

move_into(target_dir: Union[str, os.PathLike, Self], follow_symlinks: bool = True, preserve_metadata: bool = False, force_overwrite_to_cloud: Optional[bool] = None) -> Union[Path, Self]

move_into(
    target_dir: Self,
    follow_symlinks: bool = True,
    preserve_metadata: bool = False,
    force_overwrite_to_cloud: Optional[bool] = None,
) -> Self

move_into(
    target_dir: Path,
    follow_symlinks: bool = True,
    preserve_metadata: bool = False,
    force_overwrite_to_cloud: Optional[bool] = None,
) -> Path

move_into(
    target_dir: str,
    follow_symlinks: bool = True,
    preserve_metadata: bool = False,
    force_overwrite_to_cloud: Optional[bool] = None,
) -> Union[Path, CloudPath]

Move self into target directory, preserving the filename and removing the source.

Source code in cloudpathlib/cloudpath.py

def move_into(
    self,
    target_dir: Union[str, os.PathLike, Self],
    follow_symlinks: bool = True,
    preserve_metadata: bool = False,
    force_overwrite_to_cloud: Optional[bool] = None,
) -> Union[Path, Self]:
    """Move self into target directory, preserving the filename and removing the source."""
    target_path = anypath.to_anypath(target_dir) / self.name

    result = self._copy(
        target_path,
        follow_symlinks=follow_symlinks,
        preserve_metadata=preserve_metadata,
        force_overwrite_to_cloud=force_overwrite_to_cloud,
        remove_src=True,
    )
    return cast(Union[Path, Self], result)

open(mode: str = 'r', buffering: int = -1, encoding: Optional[str] = None, errors: Optional[str] = None, newline: Optional[str] = None, force_overwrite_from_cloud: Optional[bool] = None, force_overwrite_to_cloud: Optional[bool] = None) -> IO[Any]

open(
    mode: OpenTextMode = "r",
    buffering: int = -1,
    encoding: Optional[str] = None,
    errors: Optional[str] = None,
    newline: Optional[str] = None,
    force_overwrite_from_cloud: Optional[bool] = None,
    force_overwrite_to_cloud: Optional[bool] = None,
) -> TextIOWrapper

open(
    mode: OpenBinaryMode,
    buffering: Literal[0],
    encoding: None = None,
    errors: None = None,
    newline: None = None,
    force_overwrite_from_cloud: Optional[bool] = None,
    force_overwrite_to_cloud: Optional[bool] = None,
) -> FileIO

open(
    mode: OpenBinaryModeUpdating,
    buffering: Literal[-1, 1] = -1,
    encoding: None = None,
    errors: None = None,
    newline: None = None,
    force_overwrite_from_cloud: Optional[bool] = None,
    force_overwrite_to_cloud: Optional[bool] = None,
) -> BufferedRandom

open(
    mode: OpenBinaryModeWriting,
    buffering: Literal[-1, 1] = -1,
    encoding: None = None,
    errors: None = None,
    newline: None = None,
    force_overwrite_from_cloud: Optional[bool] = None,
    force_overwrite_to_cloud: Optional[bool] = None,
) -> BufferedWriter

open(
    mode: OpenBinaryModeReading,
    buffering: Literal[-1, 1] = -1,
    encoding: None = None,
    errors: None = None,
    newline: None = None,
    force_overwrite_from_cloud: Optional[bool] = None,
    force_overwrite_to_cloud: Optional[bool] = None,
) -> BufferedReader

open(
    mode: OpenBinaryMode,
    buffering: int = -1,
    encoding: None = None,
    errors: None = None,
    newline: None = None,
    force_overwrite_from_cloud: Optional[bool] = None,
    force_overwrite_to_cloud: Optional[bool] = None,
) -> BinaryIO

open(
    mode: str,
    buffering: int = -1,
    encoding: Optional[str] = None,
    errors: Optional[str] = None,
    newline: Optional[str] = None,
    force_overwrite_from_cloud: Optional[bool] = None,
    force_overwrite_to_cloud: Optional[bool] = None,
) -> IO[Any]

Source code in cloudpathlib/cloudpath.py

def open(
    self,
    mode: str = "r",
    buffering: int = -1,
    encoding: Optional[str] = None,
    errors: Optional[str] = None,
    newline: Optional[str] = None,
    force_overwrite_from_cloud: Optional[bool] = None,  # extra kwarg not in pathlib
    force_overwrite_to_cloud: Optional[bool] = None,  # extra kwarg not in pathlib
) -> "IO[Any]":
    from .cloud_io import _validate_file_mode

    _validate_file_mode(mode)
    binary_mode = "b" in mode
    if binary_mode and encoding is not None:
        raise ValueError("binary mode doesn't take an encoding argument")
    if binary_mode and errors is not None:
        raise ValueError("binary mode doesn't take an errors argument")
    if binary_mode and newline is not None:
        raise ValueError("binary mode doesn't take a newline argument")
    if not binary_mode and buffering == 0:
        raise ValueError("can't have unbuffered text I/O")

    # if trying to call open on a directory that exists
    exists_on_cloud = self.exists()

    if exists_on_cloud and not self.is_file():
        raise CloudPathIsADirectoryError(
            f"Cannot open directory, only files. Tried to open ({self})"
        )

    if not exists_on_cloud and "r" in mode:
        raise CloudPathFileNotFoundError(
            f"File opened for read, but it does not exist on cloud: {self}"
        )

    if "x" in mode and exists_on_cloud:
        raise CloudPathFileExistsError(f"Cannot open existing file ({self}) for creation.")

    if self.client.file_cache_mode == FileCacheMode.streaming:
        # Append (a) and update (+) modes cannot be done as pure streaming over object
        # storage; they fall through to the cached path for correct semantics.
        if any(m in mode for m in ("a", "+")):
            warn(
                f"Opening {self} with mode {mode!r} in streaming mode downloads the whole "
                f"object to the local cache, modifies it there, and re-uploads it on close "
                f"(object stores cannot append to or update an object in place).",
                stacklevel=2,
            )
        else:
            return self._open_streaming(
                mode,
                buffering,
                encoding,
                errors,
                newline,
                exists_on_cloud=exists_on_cloud,
                force_overwrite_to_cloud=force_overwrite_to_cloud,
            )

    # Standard cached mode
    self._refresh_cache(force_overwrite_from_cloud=force_overwrite_from_cloud)

    # create any directories that may be needed if the file is new
    if not self._local.exists():
        self._local.parent.mkdir(parents=True, exist_ok=True)
        original_mtime = 0.0
    else:
        original_mtime = self._local.stat().st_mtime

    buffer = self._local.open(
        mode=mode,
        buffering=buffering,
        encoding=encoding,
        errors=errors,
        newline=newline,
    )

    # write modes need special on closing the buffer
    if any(m in mode for m in ("w", "+", "x", "a")):
        # dirty, handle, patch close
        wrapped_close = buffer.close

        # since we are pretending this is a cloud file, upload it to the cloud
        # when the buffer is closed
        def _patched_close_upload(*args, **kwargs) -> None:
            wrapped_close(*args, **kwargs)

            # we should be idempotent and not upload again if
            # we already ran our close method patch
            if not self._dirty:
                return

            # original mtime should match what was in the cloud; because of system clocks or rounding
            # by the cloud provider, the new version in our cache is not newer than the original
            # version; explicitly set the new modified time to be after the original modified
            # time. Equal mtimes need the same treatment as older ones: we know this file was
            # written, and the upload requires the local file to be strictly newer.
            if self._local.stat().st_mtime <= original_mtime:
                new_mtime = original_mtime + 1
                os.utime(self._local, times=(new_mtime, new_mtime))

            self._upload_local_to_cloud(force_overwrite_to_cloud=force_overwrite_to_cloud)
            self._dirty = False

        buffer.close = _patched_close_upload  # type: ignore

        # keep reference in case we need to close when __del__ is called on this object
        self._handle = buffer

        # opened for write, so mark dirty
        self._dirty = True

    # if we don't want any cache around, remove the cache as soon as the file is
    # closed; streaming mode only reaches the cache through the append/update
    # fallback and should not leave anything behind either
    if self.client.file_cache_mode in (FileCacheMode.close_file, FileCacheMode.streaming):
        # this may be _patched_close_upload, in which case we need to
        # make sure to call that first so the file gets uploaded
        wrapped_close_for_cache = buffer.close

        def _patched_close_empty_cache(*args, **kwargs):
            wrapped_close_for_cache(*args, **kwargs)

            # remove local file as last step on closing
            self.clear_cache()

        buffer.close = _patched_close_empty_cache  # type: ignore

    return buffer

read_bytes() -> bytes

Source code in cloudpathlib/cloudpath.py

def read_bytes(self) -> bytes:
    with self.open(mode="rb") as f:
        return f.read()

read_text(encoding: Optional[str] = None, errors: Optional[str] = None, newline: Optional[str] = None) -> str

Source code in cloudpathlib/cloudpath.py

def read_text(
    self,
    encoding: Optional[str] = None,
    errors: Optional[str] = None,
    newline: Optional[str] = None,
) -> str:
    with self.open(mode="r", encoding=encoding, errors=errors, newline=newline) as f:
        return f.read()

relative_to(other: Self, walk_up: bool = False) -> PurePosixPath

Source code in cloudpathlib/cloudpath.py

def relative_to(self, other: Self, walk_up: bool = False) -> PurePosixPath:
    # We don't dispatch regularly since this never returns a cloud path (since it is relative, and cloud paths are
    # absolute)
    if not isinstance(other, CloudPath):
        raise ValueError(f"{self} is a cloud path, but {other} is not")
    if self.anchor != other.anchor:
        raise ValueError(
            f"{self} is a {self.anchor} path, but {other} is a {other.anchor} path"
        )

    kwargs = dict(walk_up=walk_up)

    if sys.version_info < (3, 12):
        kwargs.pop("walk_up")

    return self._path.relative_to(other._path, **kwargs)  # type: ignore[call-arg]

rename(target: Self) -> Self

Source code in cloudpathlib/cloudpath.py

def rename(self, target: Self) -> Self:
    # for cloud services replace == rename since we don't just rename,
    # we actually move files
    return self.replace(target)

replace(target: Self) -> Self

Source code in cloudpathlib/cloudpath.py

def replace(self, target: Self) -> Self:
    if type(self) is not type(target):
        raise TypeError(
            f"The target based to rename must be an instantiated class of type: {type(self)}"
        )

    if self.is_dir():
        raise CloudPathIsADirectoryError(
            f"Path {self} is a directory; rename/replace the files recursively."
        )

    if target == self:
        # Request is to replace/rename this with the same path - nothing to do
        return self

    if target.exists():
        target.unlink()

    self.client._move_file(self, target)
    return target

resolve(strict: bool = False) -> Self

Source code in cloudpathlib/cloudpath.py

def resolve(self, strict: bool = False) -> Self:
    return self

rglob(pattern: Union[str, os.PathLike], case_sensitive: Optional[bool] = None, recurse_symlinks: bool = True) -> Generator[Self, None, None]

Source code in cloudpathlib/cloudpath.py

def rglob(
    self,
    pattern: Union[str, os.PathLike],
    case_sensitive: Optional[bool] = None,
    recurse_symlinks: bool = True,
) -> Generator[Self, None, None]:
    pattern = self._glob_checks(pattern)

    pattern_parts = PurePosixPath(pattern).parts
    selector = _make_selector(
        ("**",) + tuple(pattern_parts), _posix_flavour, case_sensitive=case_sensitive
    )

    yield from self._glob(selector, True)

rmdir() -> None

Source code in cloudpathlib/cloudpath.py

def rmdir(self) -> None:
    if self.is_file():
        raise CloudPathNotADirectoryError(
            f"Path {self} is a file; call unlink instead of rmdir."
        )
    try:
        next(self.iterdir())
        raise DirectoryNotEmptyError(
            f"Directory not empty: '{self}'. Use rmtree to delete recursively."
        )
    except StopIteration:
        pass
    self.client._remove(self)

rmtree(ignore_errors: bool = False, onerror: Optional[Callable] = None, *, onexc: Optional[Callable] = None) -> None

Delete an entire directory tree.

By default, exceptions from checking the path or removing cloud objects are raised. Set ignore_errors=True to suppress these exceptions. This takes precedence over either callback. Control-flow exceptions such as KeyboardInterrupt are not suppressed.

onexc(function, path, exception) handles a failure with the exception object; onerror(function, path, exc_info) receives the tuple returned by sys.exc_info(). If both callbacks are supplied, onexc takes precedence, matching shutil.rmtree. Exceptions raised by either callback propagate to the caller.

Callbacks receive the unbound rmtree method and this cloud path, so function(path) retries the entire operation. Callbacks run once per failed operation, rather than once per object in the tree; a backend may already have removed some objects.

Source code in cloudpathlib/cloudpath.py

def rmtree(
    self,
    ignore_errors: bool = False,
    onerror: Optional[Callable] = None,
    *,
    onexc: Optional[Callable] = None,
) -> None:
    """Delete an entire directory tree.

    By default, exceptions from checking the path or removing cloud objects are raised.
    Set ``ignore_errors=True`` to suppress these exceptions. This takes precedence over
    either callback. Control-flow exceptions such as ``KeyboardInterrupt`` are not suppressed.

    ``onexc(function, path, exception)`` handles a failure with the exception object;
    ``onerror(function, path, exc_info)`` receives the tuple returned by ``sys.exc_info()``.
    If both callbacks are supplied, ``onexc`` takes precedence, matching ``shutil.rmtree``.
    Exceptions raised by either callback propagate to the caller.

    Callbacks receive the unbound ``rmtree`` method and this cloud path, so
    ``function(path)`` retries the entire operation. Callbacks run once per failed operation,
    rather than once per object in the tree; a backend may already have removed some objects.
    """
    try:
        if self.is_file():
            raise CloudPathNotADirectoryError(
                f"Path {self} is a file; call unlink instead of rmtree."
            )
        self.client._remove(self)
    except Exception as error:
        if ignore_errors:
            return
        if onexc is not None:
            onexc(type(self).rmtree, self, error)
        elif onerror is not None:
            onerror(type(self).rmtree, self, sys.exc_info())
        else:
            raise

samefile(other_path: Union[str, os.PathLike]) -> bool

Source code in cloudpathlib/cloudpath.py

def samefile(self, other_path: Union[str, os.PathLike]) -> bool:
    # all cloud paths are absolute and the paths are used for hash
    return self == other_path

stat(follow_symlinks=True)

Source code in cloudpathlib/s3/s3path.py

def stat(self, follow_symlinks=True):
    try:
        meta = self.client._get_metadata(self)
    except self.client.client.exceptions.ClientError as error:
        # head_object returns a 404 (not NoSuchKey) for a missing key; let other errors raise
        error_info = error.response.get("Error", {})
        status_code = error.response.get("ResponseMetadata", {}).get("HTTPStatusCode")
        if error_info.get("Code") in ("404", "NoSuchKey") or status_code == 404:
            raise NoStatError(
                f"No stats available for {self}; it may be a directory or not exist."
            )
        raise

    return os.stat_result(
        (
            None,  # mode
            None,  # ino
            self.cloud_prefix,  # dev,
            None,  # nlink,
            None,  # uid,
            None,  # gid,
            meta.get("size", 0),  # size,
            None,  # atime,
            meta.get("last_modified", 0).timestamp(),  # mtime,
            None,  # ctime,
        )
    )

touch(exist_ok: bool = True, mode: Optional[Any] = None)

Source code in cloudpathlib/s3/s3path.py

def touch(self, exist_ok: bool = True, mode: Optional[Any] = None):
    if self.exists():
        if not exist_ok:
            raise FileExistsError(f"File exists: {self}")
        self.client._move_file(self, self)
    else:
        tf = TemporaryDirectory()
        p = Path(tf.name) / "empty"
        p.touch()

        self.client._upload_file(p, self)

        tf.cleanup()

unlink(missing_ok: bool = True) -> None

Source code in cloudpathlib/cloudpath.py

def unlink(self, missing_ok: bool = True) -> None:
    # Note: missing_ok defaults to False in pathlib, but changing the default now would be a breaking change.
    if self.is_dir():
        raise CloudPathIsADirectoryError(
            f"Path {self} is a directory; call rmdir instead of unlink."
        )
    self.client._remove(self, missing_ok)

upload_from(source: Union[str, os.PathLike], force_overwrite_to_cloud: Optional[bool] = None) -> Self

Upload a file or directory to the cloud path.

Source code in cloudpathlib/cloudpath.py

def upload_from(
    self,
    source: Union[str, os.PathLike],
    force_overwrite_to_cloud: Optional[bool] = None,
) -> Self:
    """Upload a file or directory to the cloud path."""
    source = Path(source)

    if source.is_dir():
        for p in source.iterdir():
            (self / p.name).upload_from(p, force_overwrite_to_cloud=force_overwrite_to_cloud)

        return self

    else:
        if self.exists() and self.is_dir():
            dst = self / source.name
        else:
            dst = self

        dst._upload_file_to_cloud(source, force_overwrite_to_cloud=force_overwrite_to_cloud)

        return dst

validate(v: str) -> Self classmethod

Used as a Pydantic validator. See https://docs.pydantic.dev/2.0/usage/types/custom/

Source code in cloudpathlib/cloudpath.py

@classmethod
def validate(cls, v: str) -> Self:
    """Used as a Pydantic validator. See
    https://docs.pydantic.dev/2.0/usage/types/custom/"""
    return cls(v)

walk(top_down: bool = True, on_error: Optional[Callable] = None, follow_symlinks: bool = False, lazy: bool = False) -> Generator[Tuple[Self, List[str], List[str]], None, None]

Walk the directory tree rooted at this path, similar to :meth:pathlib.Path.walk and :func:os.walk.

By default (lazy=False) the entire subtree is listed up front with a single recursive listing, which is the fastest option for a full walk on cloud backends.

Pass lazy=True to defer listing until each directory is visited (one non-recursive listing per directory). This is slower for a full walk, but lets callers prune dirnames in place (when top_down=True) to avoid fetching the contents of skipped subtrees entirely -- which can be dramatically faster and cheaper for sparse traversals.

follow_symlinks is accepted for signature parity with pathlib but has no effect on cloud backends, which do not have symlinks.

Source code in cloudpathlib/cloudpath.py

def walk(
    self,
    top_down: bool = True,
    on_error: Optional[Callable] = None,
    follow_symlinks: bool = False,
    lazy: bool = False,
) -> Generator[Tuple[Self, List[str], List[str]], None, None]:
    """Walk the directory tree rooted at this path, similar to
    :meth:`pathlib.Path.walk` and :func:`os.walk`.

    By default (``lazy=False``) the entire subtree is listed up front with a
    single recursive listing, which is the fastest option for a full walk on
    cloud backends.

    Pass ``lazy=True`` to defer listing until each directory is visited (one
    non-recursive listing per directory). This is slower for a full walk,
    but lets callers prune ``dirnames`` in place (when ``top_down=True``) to
    avoid fetching the contents of skipped subtrees entirely -- which can be
    dramatically faster and cheaper for sparse traversals.

    ``follow_symlinks`` is accepted for signature parity with ``pathlib``
    but has no effect on cloud backends, which do not have symlinks.
    """
    if lazy:
        yield from self._walk_lazy(top_down=top_down, on_error=on_error)
        return

    try:
        file_tree = self._build_subtree(recursive=True)  # walking is always recursive
        yield from self._walk_results_from_tree(self, file_tree, top_down=top_down)

    except Exception as e:
        if on_error is not None:
            on_error(e)
        else:
            raise

with_name(name: str) -> Self

Source code in cloudpathlib/cloudpath.py

def with_name(self, name: str) -> Self:
    return self._dispatch_to_path("with_name", name)

with_segments(*pathsegments) -> Self

Create a new CloudPath with the same client out of the given segments. The first segment will be interpreted as the bucket/container name.

Source code in cloudpathlib/cloudpath.py

def with_segments(self, *pathsegments) -> Self:
    """Create a new CloudPath with the same client out of the given segments.
    The first segment will be interpreted as the bucket/container name.
    """
    return self._new_cloudpath("/".join(pathsegments))

with_stem(stem: str) -> Self

Source code in cloudpathlib/cloudpath.py

def with_stem(self, stem: str) -> Self:
    return self._dispatch_to_path("with_stem", stem)

with_suffix(suffix: str) -> Self

Source code in cloudpathlib/cloudpath.py

def with_suffix(self, suffix: str) -> Self:
    return self._dispatch_to_path("with_suffix", suffix)

write_bytes(data: bytes) -> int

Open the file in bytes mode, write to it, and close the file.

NOTE: vendored from pathlib since we override open https://github.com/python/cpython/blob/3.8/Lib/pathlib.py#L1235-L1242

Source code in cloudpathlib/cloudpath.py

def write_bytes(self, data: bytes) -> int:
    """Open the file in bytes mode, write to it, and close the file.

    NOTE: vendored from pathlib since we override open
    https://github.com/python/cpython/blob/3.8/Lib/pathlib.py#L1235-L1242
    """
    # type-check for the buffer interface before truncating the file
    view = memoryview(data)
    with self.open(mode="wb") as f:
        return f.write(view)

write_text(data: str, encoding: Optional[str] = None, errors: Optional[str] = None, newline: Optional[str] = None) -> int

Open the file in text mode, write to it, and close the file.

NOTE: vendored from pathlib since we override open https://github.com/python/cpython/blob/3.10/Lib/pathlib.py#L1146-L1155

Source code in cloudpathlib/cloudpath.py

def write_text(
    self,
    data: str,
    encoding: Optional[str] = None,
    errors: Optional[str] = None,
    newline: Optional[str] = None,
) -> int:
    """Open the file in text mode, write to it, and close the file.

    NOTE: vendored from pathlib since we override open
    https://github.com/python/cpython/blob/3.10/Lib/pathlib.py#L1146-L1155
    """
    if not isinstance(data, str):
        raise TypeError("data must be str, not %s" % data.__class__.__name__)

    with self.open(mode="w", encoding=encoding, errors=errors, newline=newline) as f:
        return f.write(data)