Skip to content
GitLab
Explore
Projects
Groups
Snippets
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
csst-pipeline
csst_common
Commits
ccbe4807
Commit
ccbe4807
authored
2 years ago
by
BO ZHANG
Browse files
Options
Download
Email Patches
Plain Diff
implement dm.from_dfs
parent
ce63aa9e
dev
main
wangxia-main-patch-87026
zhaobowei-main-patch-61353
No related merge requests found
Changes
1
Hide whitespace changes
Inline
Side-by-side
Showing
1 changed file
csst_common/data_manager.py
+41
-4
csst_common/data_manager.py
with
41 additions
and
4 deletions
+41
-4
csst_common/data_manager.py
+
41
-
4
View file @
ccbe4807
...
@@ -252,10 +252,6 @@ class CsstMsDataManager:
...
@@ -252,10 +252,6 @@ class CsstMsDataManager:
def
set_detectors
(
self
,
detectors
=
None
):
def
set_detectors
(
self
,
detectors
=
None
):
raise
DeprecationWarning
(
"This method is deprecated, please directly use dm.target_detectors = detectors!"
)
raise
DeprecationWarning
(
"This method is deprecated, please directly use dm.target_detectors = detectors!"
)
@
staticmethod
def
from_dfs
(
expid
,
ver_sim
=
"C5.2"
,
datatype
=
"mbi"
,
dir_l1
=
"."
):
raise
NotImplementedError
(
"from_dfs is currently not available!"
)
@
staticmethod
@
staticmethod
def
from_dir
(
ver_sim
=
"C5.2"
,
datatype
=
"mbi"
,
dir_l0
=
"."
,
dir_l1
=
"."
,
path_aux
=
""
,
dfs_mode
=
False
,
dfs_node
=
"kmust"
):
def
from_dir
(
ver_sim
=
"C5.2"
,
datatype
=
"mbi"
,
dir_l0
=
"."
,
dir_l1
=
"."
,
path_aux
=
""
,
dfs_mode
=
False
,
dfs_node
=
"kmust"
):
""" initialize the multi-band imaging data manager """
""" initialize the multi-band imaging data manager """
...
@@ -571,6 +567,47 @@ class CsstMsDataManager:
...
@@ -571,6 +567,47 @@ class CsstMsDataManager:
""" Push SLS spectra to DFS. """
""" Push SLS spectra to DFS. """
pass
pass
@
staticmethod
def
from_dfs
(
obs_id
=
"100000100"
,
datatype
=
"mbi"
,
dir_l0
=
"/L1Pipeline/L0"
,
dir_l1
=
"."
,
dfs_mode
=
True
,
dfs_node
=
"pml"
):
""" Initialize CsstMsDataManager from DFS. """
records
=
Level0DataApi
().
find
(
obs_id
=
obs_id
)
# query DFS
tbl
=
Table
([
_
.
__dict__
for
_
in
records
[
"data"
]])
tbl
.
sort
([
"detector_no"
,
"obs_type"
])
if
os
.
uname
()[
1
]
==
"ubuntu"
:
# use pml test node
dfs_root
=
"/share/dfs"
else
:
# in docker
dfs_root
=
"/dfsroot"
# (clear and) make directories
if
os
.
path
.
exists
(
dir_l0
):
os
.
system
(
f
"rm -rf
{
dir_l0
}
"
)
if
os
.
path
.
exists
(
dir_l1
):
os
.
system
(
f
"rm -rf
{
dir_l1
}
"
)
os
.
mkdir
(
dir_l0
)
os
.
mkdir
(
dir_l1
)
for
i_rec
in
range
(
len
(
tbl
)):
# make symbolic links
os
.
symlink
(
src
=
os
.
path
.
join
(
dfs_root
,
tbl
[
"file_path"
][
i_rec
]),
dst
=
os
.
path
.
join
(
dir_l0
,
tbl
[
"file_path"
][
i_rec
])
)
if
datatype
==
"mbi"
:
path_aux
=
"/share/L1Pipeline/aux/C5.2_ref_mbi/MSC_{}_*_{:02d}_combine.fits"
elif
datatype
==
"sls"
:
path_aux
=
"/share/L1Pipeline/aux/C5.2_ref_sls/csst_{:02d}{}.fits"
else
:
path_aux
=
""
# initialize dm
dm
=
CsstMsDataManager
.
from_dir
(
ver_sim
=
"C5.2"
,
datatype
=
datatype
,
dir_l0
=
dir_l0
,
dir_l1
=
dir_l1
,
path_aux
=
path_aux
,
dfs_mode
=
dfs_mode
,
dfs_node
=
dfs_node
)
return
dm
def
dfs_l0_query
(
self
,
obs_id
:
str
=
"100000100"
):
def
dfs_l0_query
(
self
,
obs_id
:
str
=
"100000100"
):
""" Query L0 data from DFS. """
""" Query L0 data from DFS. """
result
=
self
.
dfs_L0DataApi
.
find
(
obs_id
=
str
(
obs_id
))
result
=
self
.
dfs_L0DataApi
.
find
(
obs_id
=
str
(
obs_id
))
...
...
This diff is collapsed.
Click to expand it.
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment
Menu
Explore
Projects
Groups
Snippets