Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
csst-pipeline
csst_fs
Commits
df3091b7
Commit
df3091b7
authored
Aug 14, 2024
by
WeidenthalerMatthias
Browse files
Merge pull request !11 from WeidenthalerMatthias/feature.delete
parents
4d626ab1
6d1f928d
Changes
4
Hide whitespace changes
Inline
Side-by-side
README.md
View file @
df3091b7
...
...
@@ -71,6 +71,17 @@ fs.isfile('s3://csst-prod/test.txt')
fs
.
isfile
(
's3://csst-prod/test.txt'
,
s3_options
=
s3_options
)
```
### Delete a file from local or s3
```
python
from
csst_fs
import
fs
# local or on s3, depending on the given path
fs
.
delete
(
'requirements.txt'
)
# uses os.remove
fs
.
delete
(
'test'
,
dir_fd
=
1
)
fs
.
delete
(
's3://csst-prod/test.txt'
)
# uses fsspec.delete
fs
.
delete
(
's3://csst-prod/test.txt'
,
recursive
=
True
,
maxdepth
=
3
)
fs
.
delete
(
's3://csst-prod/test.txt'
,
s3_options
=
s3_options
)
```
## astropy直接读写s3的写法适配
### fits.open
#### 老写法
...
...
csst_fs/fs.py
View file @
df3091b7
from
.s3_fs
import
is_s3_path
from
.s3_fs
import
isfile
as
s3_isfile
from
.s3_fs
import
delete
as
s3_delete
from
.local_fs
import
delete
as
local_delete
from
.local_fs
import
isfile
as
local_isfile
def
isfile
(
path
:
str
,
*
args
,
**
kwargs
)
->
bool
:
...
...
@@ -7,3 +9,9 @@ def isfile(path: str, *args, **kwargs) -> bool:
return
s3_isfile
(
path
,
*
args
,
**
kwargs
)
else
:
return
local_isfile
(
path
,
*
args
,
**
kwargs
)
def
delete
(
path
,
**
kwargs
):
if
is_s3_path
(
path
):
s3_delete
(
path
,
**
kwargs
)
else
:
local_delete
(
path
,
**
kwargs
)
csst_fs/local_fs.py
View file @
df3091b7
...
...
@@ -2,4 +2,7 @@ import os
def
isfile
(
path
:
str
)
->
bool
:
return
os
.
path
.
isfile
(
path
)
def
delete
(
path
:
str
,
*
,
dir_fd
=
None
,
**
kwargs
):
os
.
remove
(
path
,
dir_fd
=
dir_fd
)
\ No newline at end of file
csst_fs/s3_fs.py
View file @
df3091b7
...
...
@@ -20,6 +20,10 @@ def open(file: str, mode: str='r', s3_options=load_s3_options()):
def
isfile
(
path
:
str
,
s3_options
:
dict
=
load_s3_options
())
->
bool
:
s3_fs
=
fsspec
.
filesystem
(
's3'
,
**
s3_options
)
return
s3_fs
.
isfile
(
path
)
def
delete
(
path
:
str
,
*
,
recursive
=
False
,
maxdepth
=
None
,
s3_options
:
dict
=
load_s3_options
(),
**
kwargs
):
s3_fs
=
fsspec
.
filesystem
(
's3'
,
**
s3_options
)
s3_fs
.
delete
(
path
,
recursive
,
maxdepth
)
def
is_s3_path
(
path
:
str
):
if
(
path
.
startswith
(
"s3"
)):
...
...
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment