Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
csst-pipeline
csst_common
Commits
5f786af9
Commit
5f786af9
authored
Dec 12, 2022
by
BO ZHANG
🏀
Browse files
added dm.dfs_root
parent
d07f6fa2
Changes
1
Hide whitespace changes
Inline
Side-by-side
csst_common/data_manager.py
View file @
5f786af9
...
@@ -81,6 +81,8 @@ class CsstMsDataManager:
...
@@ -81,6 +81,8 @@ class CsstMsDataManager:
dfs_node : str
dfs_node : str
The environment in which the pipeline will run.
The environment in which the pipeline will run.
Use "pml" for Purple Mountain Lab cluster and "local" for others.
Use "pml" for Purple Mountain Lab cluster and "local" for others.
dfs_root : str
The DFS root path.
obs_id : int
obs_id : int
The exposure ID.
The exposure ID.
exp_start : int
exp_start : int
...
@@ -137,6 +139,7 @@ class CsstMsDataManager:
...
@@ -137,6 +139,7 @@ class CsstMsDataManager:
path_aux
:
str
=
""
,
# bias dark flat
path_aux
:
str
=
""
,
# bias dark flat
use_dfs
:
bool
=
False
,
use_dfs
:
bool
=
False
,
dfs_node
:
str
=
"kmust"
,
dfs_node
:
str
=
"kmust"
,
dfs_root
:
str
=
"/share/dfs"
,
obs_id
:
str
=
"100000100"
,
obs_id
:
str
=
"100000100"
,
exp_start
:
int
=
"20270810081950"
,
exp_start
:
int
=
"20270810081950"
,
exp_stop
:
int
=
"20270810082220"
,
exp_stop
:
int
=
"20270810082220"
,
...
@@ -198,6 +201,7 @@ class CsstMsDataManager:
...
@@ -198,6 +201,7 @@ class CsstMsDataManager:
# DFS configuration
# DFS configuration
self
.
use_dfs
=
use_dfs
self
.
use_dfs
=
use_dfs
self
.
dfs_node
=
dfs_node
self
.
dfs_node
=
dfs_node
self
.
dfs_root
=
dfs_root
# data directory
# data directory
self
.
dir_l0
=
dir_l0
self
.
dir_l0
=
dir_l0
...
@@ -615,6 +619,19 @@ class CsstMsDataManager:
...
@@ -615,6 +619,19 @@ class CsstMsDataManager:
""" Push SLS spectra to DFS. """
""" Push SLS spectra to DFS. """
pass
pass
@
staticmethod
def
from_l1id
(
l1_id
=
"1000000001"
,
datatype
=
"sls"
,
dir_l0
=
"/L1Pipeline/L0"
,
dir_l1
=
"/L1Pipeline/L1"
,
use_dfs
=
True
,
dfs_node
=
"pml"
,
clear_l1
=
False
,
dfs_root
=
"/share/dfs"
):
pass
@
staticmethod
@
staticmethod
def
from_dfs
(
def
from_dfs
(
obs_id
=
"100000100"
,
obs_id
=
"100000100"
,
...
@@ -624,6 +641,7 @@ class CsstMsDataManager:
...
@@ -624,6 +641,7 @@ class CsstMsDataManager:
use_dfs
=
True
,
use_dfs
=
True
,
dfs_node
=
"pml"
,
dfs_node
=
"pml"
,
clear_l1
=
False
,
clear_l1
=
False
,
dfs_root
=
"/share/dfs"
):
):
""" Initialize CsstMsDataManager from DFS. """
""" Initialize CsstMsDataManager from DFS. """
# (clear and) make directories
# (clear and) make directories
...
@@ -648,11 +666,6 @@ class CsstMsDataManager:
...
@@ -648,11 +666,6 @@ class CsstMsDataManager:
tbl
=
Table
([
_
.
__dict__
for
_
in
records
[
"data"
]])
tbl
=
Table
([
_
.
__dict__
for
_
in
records
[
"data"
]])
tbl
.
sort
([
"detector_no"
,
"obs_type"
])
tbl
.
sort
([
"detector_no"
,
"obs_type"
])
if
os
.
uname
()[
1
]
==
"ubuntu"
:
# use pml test node
dfs_root
=
"/share/dfs"
else
:
# in docker
dfs_root
=
"/dfsroot"
print
(
"Making symbolic links ..."
)
print
(
"Making symbolic links ..."
)
for
i_rec
in
range
(
len
(
tbl
)):
for
i_rec
in
range
(
len
(
tbl
)):
os
.
symlink
(
os
.
symlink
(
...
...
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment