Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
csst-pipeline
csst_fs
Commits
8f228830
Commit
8f228830
authored
Jun 27, 2024
by
WeidenthalerMatthias
Committed by
戚攀
Jun 27, 2024
Browse files
Merge pull request !4 from WeidenthalerMatthias/feature.open
parents
4e27be18
f50c745d
Changes
3
Hide whitespace changes
Inline
Side-by-side
README.md
View file @
8f228830
...
...
@@ -52,6 +52,16 @@ s3_fs.info('s3://csst-prod/gaia/data')
s3_fs
.
info
(
's3://csst-prod/gaia/test/requirements.txt'
,
s3_options
=
s3_options
)
```
### Open for read/write
```
python
from
csst_fs
import
s3_fs
# open single file (s3 or local)
with
s3_fs
.
open
(
's3://csst-prod/gaia/data'
)
as
file
:
file
.
read
()
with
s3_fs
.
open
(
's3://csst-prod/gaia/test/requirements.txt'
,
s3_options
=
s3_options
,
mode
=
'w'
)
as
file
:
file
.
write
(
"CSST"
)
```
## astropy直接读写s3的写法适配
### fits.open
#### 老写法
...
...
csst_fs/fsspec_fileobj.py
View file @
8f228830
...
...
@@ -5,5 +5,5 @@ def open_fileobj(path: str, s3_options, mode='w'):
s3_fs
=
fsspec
.
filesystem
(
's3'
,
**
s3_options
)
fileobj
=
s3_fs
.
open
(
path
,
mode
=
mode
).
__enter__
()
else
:
fileobj
=
path
fileobj
=
open
(
path
,
mode
=
mode
).
__enter__
()
return
fileobj
\ No newline at end of file
csst_fs/s3_fs.py
View file @
8f228830
import
fsspec
from
.s3_config
import
load_s3_options
from
.fsspec_fileobj
import
open_fileobj
def
put
(
lpath
,
rpath
,
recursive
=
False
,
callback
=
fsspec
.
callbacks
.
DEFAULT_CALLBACK
,
maxdepth
=
None
,
s3_options
=
load_s3_options
(),
**
kwargs
,):
s3_fs
=
fsspec
.
filesystem
(
's3'
,
**
s3_options
)
...
...
@@ -12,3 +13,6 @@ def get(rpath, lpath, recursive=False, callback=fsspec.callbacks.DEFAULT_CALLBAC
def
info
(
path
,
bucket
=
None
,
key
=
None
,
refresh
=
False
,
version_id
=
None
,
s3_options
=
load_s3_options
())
->
dict
:
s3_fs
=
fsspec
.
filesystem
(
's3'
,
**
s3_options
)
return
s3_fs
.
info
(
path
,
bucket
,
key
,
refresh
,
version_id
)
def
open
(
file
:
str
,
mode
:
str
=
'r'
,
s3_options
=
load_s3_options
()):
return
open_fileobj
(
path
=
file
,
s3_options
=
s3_options
,
mode
=
mode
)
\ No newline at end of file
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment