Commit f50c745d authored by Matthias Weidenthaler's avatar Matthias Weidenthaler
Browse files

Changed the behavior of 'open_fileobj' for local files, made 'open' function use 'open_fileobj'

parent d63100fe
...@@ -55,7 +55,7 @@ s3_fs.info('s3://csst-prod/gaia/test/requirements.txt', s3_options=s3_options) ...@@ -55,7 +55,7 @@ s3_fs.info('s3://csst-prod/gaia/test/requirements.txt', s3_options=s3_options)
### Open for read/write ### Open for read/write
```python ```python
from csst_fs import s3_fs from csst_fs import s3_fs
# open single file # open single file (s3 or local)
with s3_fs.open('s3://csst-prod/gaia/data') as file: with s3_fs.open('s3://csst-prod/gaia/data') as file:
file.read() file.read()
with s3_fs.open('s3://csst-prod/gaia/test/requirements.txt', s3_options=s3_options, mode='w') as file: with s3_fs.open('s3://csst-prod/gaia/test/requirements.txt', s3_options=s3_options, mode='w') as file:
......
...@@ -5,5 +5,5 @@ def open_fileobj(path: str, s3_options, mode='w'): ...@@ -5,5 +5,5 @@ def open_fileobj(path: str, s3_options, mode='w'):
s3_fs = fsspec.filesystem('s3', **s3_options) s3_fs = fsspec.filesystem('s3', **s3_options)
fileobj = s3_fs.open(path, mode=mode).__enter__() fileobj = s3_fs.open(path, mode=mode).__enter__()
else: else:
fileobj = path fileobj = open(path, mode=mode).__enter__()
return fileobj return fileobj
\ No newline at end of file
import fsspec import fsspec
from .s3_config import load_s3_options from .s3_config import load_s3_options
from .fsspec_fileobj import open_fileobj
def put(lpath, rpath, recursive=False, callback=fsspec.callbacks.DEFAULT_CALLBACK, maxdepth=None, s3_options=load_s3_options(), **kwargs,): def put(lpath, rpath, recursive=False, callback=fsspec.callbacks.DEFAULT_CALLBACK, maxdepth=None, s3_options=load_s3_options(), **kwargs,):
s3_fs = fsspec.filesystem('s3', **s3_options) s3_fs = fsspec.filesystem('s3', **s3_options)
...@@ -13,6 +14,5 @@ def info(path, bucket=None, key=None, refresh=False, version_id=None, s3_options ...@@ -13,6 +14,5 @@ def info(path, bucket=None, key=None, refresh=False, version_id=None, s3_options
s3_fs = fsspec.filesystem('s3', **s3_options) s3_fs = fsspec.filesystem('s3', **s3_options)
return s3_fs.info(path, bucket, key, refresh, version_id) return s3_fs.info(path, bucket, key, refresh, version_id)
def open(path, s3_options=load_s3_options(), **kwargs): def open(file: str, mode: str='r', s3_options=load_s3_options()):
s3_fs = fsspec.filesystem('s3', **s3_options) return open_fileobj(path=file, s3_options=s3_options, mode=mode)
return s3_fs.open(path, **kwargs) \ No newline at end of file
\ No newline at end of file
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment