Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
csst-pipeline
csst_common
Commits
5f786af9
Commit
5f786af9
authored
Dec 12, 2022
by
BO ZHANG
🏀
Browse files
added dm.dfs_root
parent
d07f6fa2
Changes
1
Show whitespace changes
Inline
Side-by-side
csst_common/data_manager.py
View file @
5f786af9
...
...
@@ -81,6 +81,8 @@ class CsstMsDataManager:
dfs_node : str
The environment in which the pipeline will run.
Use "pml" for Purple Mountain Lab cluster and "local" for others.
dfs_root : str
The DFS root path.
obs_id : int
The exposure ID.
exp_start : int
...
...
@@ -137,6 +139,7 @@ class CsstMsDataManager:
path_aux
:
str
=
""
,
# bias dark flat
use_dfs
:
bool
=
False
,
dfs_node
:
str
=
"kmust"
,
dfs_root
:
str
=
"/share/dfs"
,
obs_id
:
str
=
"100000100"
,
exp_start
:
int
=
"20270810081950"
,
exp_stop
:
int
=
"20270810082220"
,
...
...
@@ -198,6 +201,7 @@ class CsstMsDataManager:
# DFS configuration
self
.
use_dfs
=
use_dfs
self
.
dfs_node
=
dfs_node
self
.
dfs_root
=
dfs_root
# data directory
self
.
dir_l0
=
dir_l0
...
...
@@ -615,6 +619,19 @@ class CsstMsDataManager:
""" Push SLS spectra to DFS. """
pass
@
staticmethod
def
from_l1id
(
l1_id
=
"1000000001"
,
datatype
=
"sls"
,
dir_l0
=
"/L1Pipeline/L0"
,
dir_l1
=
"/L1Pipeline/L1"
,
use_dfs
=
True
,
dfs_node
=
"pml"
,
clear_l1
=
False
,
dfs_root
=
"/share/dfs"
):
pass
@
staticmethod
def
from_dfs
(
obs_id
=
"100000100"
,
...
...
@@ -624,6 +641,7 @@ class CsstMsDataManager:
use_dfs
=
True
,
dfs_node
=
"pml"
,
clear_l1
=
False
,
dfs_root
=
"/share/dfs"
):
""" Initialize CsstMsDataManager from DFS. """
# (clear and) make directories
...
...
@@ -648,11 +666,6 @@ class CsstMsDataManager:
tbl
=
Table
([
_
.
__dict__
for
_
in
records
[
"data"
]])
tbl
.
sort
([
"detector_no"
,
"obs_type"
])
if
os
.
uname
()[
1
]
==
"ubuntu"
:
# use pml test node
dfs_root
=
"/share/dfs"
else
:
# in docker
dfs_root
=
"/dfsroot"
print
(
"Making symbolic links ..."
)
for
i_rec
in
range
(
len
(
tbl
)):
os
.
symlink
(
...
...
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment