29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187 | class Client(abc.ABC, Generic[BoundedCloudPath]):
_cloud_meta: CloudImplementation
_default_client = None
def __init__(
self,
file_cache_mode: Optional[Union[str, FileCacheMode]] = None,
local_cache_dir: Optional[Union[str, os.PathLike]] = None,
content_type_method: Optional[Callable] = mimetypes.guess_type,
):
self.file_cache_mode = None
self._cache_tmp_dir = None
self._cloud_meta.validate_completeness()
# convert strings passed to enum
if isinstance(file_cache_mode, str):
file_cache_mode = FileCacheMode(file_cache_mode)
# if not explicitly passed to client, get from env var
if file_cache_mode is None:
file_cache_mode = FileCacheMode.from_environment()
if local_cache_dir is None:
local_cache_dir = os.environ.get("CLOUDPATHLIB_LOCAL_CACHE_DIR", None)
# treat empty string as None to avoid writing cache in cwd; set to "." for cwd
if local_cache_dir == "":
local_cache_dir = None
# explicitly passing a cache dir, so we set to persistent
# unless user explicitly passes a different file cache mode
if local_cache_dir and file_cache_mode is None:
file_cache_mode = FileCacheMode.persistent
if file_cache_mode == FileCacheMode.persistent and local_cache_dir is None:
raise InvalidConfigurationException(
f"If you use the '{FileCacheMode.persistent}' cache mode, you must pass a `local_cache_dir` when you instantiate the client."
)
# if no explicit local dir, setup caching in temporary dir
if local_cache_dir is None:
self._cache_tmp_dir = TemporaryDirectory()
local_cache_dir = self._cache_tmp_dir.name
if file_cache_mode is None:
file_cache_mode = FileCacheMode.tmp_dir
self._local_cache_dir = Path(local_cache_dir)
self.content_type_method = content_type_method
# Fallback: if not set anywhere, default to tmp_dir (for backwards compatibility)
if file_cache_mode is None:
file_cache_mode = FileCacheMode.tmp_dir
self.file_cache_mode = file_cache_mode
def __del__(self) -> None:
# remove containing dir, even if a more aggressive strategy
# removed the actual files
if getattr(self, "file_cache_mode", None) in [
FileCacheMode.tmp_dir,
FileCacheMode.close_file,
FileCacheMode.cloudpath_object,
]:
self.clear_cache()
if self._local_cache_dir.exists():
self._local_cache_dir.rmdir()
@classmethod
def get_default_client(cls) -> "Client":
"""Get the default client, which the one that is used when instantiating a cloud path
instance for this cloud without a client specified.
"""
if cls._default_client is None:
cls._default_client = cls()
return cls._default_client
def set_as_default_client(self) -> None:
"""Set this client instance as the default one used when instantiating cloud path
instances for this cloud without a client specified."""
self.__class__._default_client = self
def CloudPath(self, cloud_path: Union[str, BoundedCloudPath]) -> BoundedCloudPath:
return self._cloud_meta.path_class(cloud_path=cloud_path, client=self) # type: ignore
def clear_cache(self):
"""Clears the contents of the cache folder.
Does not remove folder so it can keep being written to.
"""
if self._local_cache_dir.exists():
for p in self._local_cache_dir.iterdir():
if p.is_file():
p.unlink()
else:
shutil.rmtree(p)
@abc.abstractmethod
def _download_file(
self, cloud_path: BoundedCloudPath, local_path: Union[str, os.PathLike]
) -> Path:
pass
@abc.abstractmethod
def _exists(self, cloud_path: BoundedCloudPath) -> bool:
pass
@abc.abstractmethod
def _list_dir(
self, cloud_path: BoundedCloudPath, recursive: bool
) -> Iterable[Tuple[BoundedCloudPath, bool]]:
"""List all the files and folders in a directory.
Parameters
----------
cloud_path : CloudPath
The folder to start from.
recursive : bool
Whether or not to list recursively.
Returns
-------
contents : Iterable[Tuple]
Of the form [(CloudPath, is_dir), ...] for every child of the dir.
"""
pass
@abc.abstractmethod
def _move_file(
self, src: BoundedCloudPath, dst: BoundedCloudPath, remove_src: bool = True
) -> BoundedCloudPath:
pass
@abc.abstractmethod
def _remove(self, path: BoundedCloudPath, missing_ok: bool = True) -> None:
"""Remove a file or folder from the server.
Parameters
----------
path : CloudPath
The file or folder to remove.
"""
pass
@abc.abstractmethod
def _upload_file(
self, local_path: Union[str, os.PathLike], cloud_path: BoundedCloudPath
) -> BoundedCloudPath:
pass
@abc.abstractmethod
def _get_public_url(self, cloud_path: BoundedCloudPath) -> str:
pass
@abc.abstractmethod
def _generate_presigned_url(
self, cloud_path: BoundedCloudPath, expire_seconds: int = 60 * 60
) -> str:
pass
|