Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
csst-pipeline
csst_fs
Commits
f50c745d
Commit
f50c745d
authored
Jun 27, 2024
by
Matthias Weidenthaler
Browse files
Changed the behavior of 'open_fileobj' for local files, made 'open' function use 'open_fileobj'
parent
d63100fe
Changes
3
Hide whitespace changes
Inline
Side-by-side
README.md
View file @
f50c745d
...
...
@@ -55,7 +55,7 @@ s3_fs.info('s3://csst-prod/gaia/test/requirements.txt', s3_options=s3_options)
### Open for read/write
```
python
from
csst_fs
import
s3_fs
# open single file
# open single file
(s3 or local)
with
s3_fs
.
open
(
's3://csst-prod/gaia/data'
)
as
file
:
file
.
read
()
with
s3_fs
.
open
(
's3://csst-prod/gaia/test/requirements.txt'
,
s3_options
=
s3_options
,
mode
=
'w'
)
as
file
:
...
...
csst_fs/fsspec_fileobj.py
View file @
f50c745d
...
...
@@ -5,5 +5,5 @@ def open_fileobj(path: str, s3_options, mode='w'):
s3_fs
=
fsspec
.
filesystem
(
's3'
,
**
s3_options
)
fileobj
=
s3_fs
.
open
(
path
,
mode
=
mode
).
__enter__
()
else
:
fileobj
=
path
fileobj
=
open
(
path
,
mode
=
mode
).
__enter__
()
return
fileobj
\ No newline at end of file
csst_fs/s3_fs.py
View file @
f50c745d
import
fsspec
from
.s3_config
import
load_s3_options
from
.fsspec_fileobj
import
open_fileobj
def
put
(
lpath
,
rpath
,
recursive
=
False
,
callback
=
fsspec
.
callbacks
.
DEFAULT_CALLBACK
,
maxdepth
=
None
,
s3_options
=
load_s3_options
(),
**
kwargs
,):
s3_fs
=
fsspec
.
filesystem
(
's3'
,
**
s3_options
)
...
...
@@ -13,6 +14,5 @@ def info(path, bucket=None, key=None, refresh=False, version_id=None, s3_options
s3_fs
=
fsspec
.
filesystem
(
's3'
,
**
s3_options
)
return
s3_fs
.
info
(
path
,
bucket
,
key
,
refresh
,
version_id
)
def
open
(
path
,
s3_options
=
load_s3_options
(),
**
kwargs
):
s3_fs
=
fsspec
.
filesystem
(
's3'
,
**
s3_options
)
return
s3_fs
.
open
(
path
,
**
kwargs
)
\ No newline at end of file
def
open
(
file
:
str
,
mode
:
str
=
'r'
,
s3_options
=
load_s3_options
()):
return
open_fileobj
(
path
=
file
,
s3_options
=
s3_options
,
mode
=
mode
)
\ No newline at end of file
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment