Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
csst-pipeline
csst_common
Commits
51b32253
Commit
51b32253
authored
Jan 10, 2023
by
BO ZHANG
🏀
Browse files
support C6.1 and C5.2 only
parent
9920c659
Changes
1
Show whitespace changes
Inline
Side-by-side
csst_common/data_manager.py
View file @
51b32253
...
@@ -305,12 +305,14 @@ class CsstMsDataManager:
...
@@ -305,12 +305,14 @@ class CsstMsDataManager:
path_aux
=
""
,
path_aux
=
""
,
use_dfs
=
False
,
use_dfs
=
False
,
dfs_node
=
"kmust"
,
dfs_node
=
"kmust"
,
log_ppl
=
"csst-l1ppl.log"
,
log_mod
=
"csst-l1mod.log"
,
n_jobs
=
18
,
n_jobs
=
18
,
backend
=
"multiprocessing"
backend
=
"multiprocessing"
):
):
""" initialize the multi-band imaging data manager """
""" initialize the multi-band imaging data manager """
assert
ver_sim
in
[
"C5.2"
,
]
assert
ver_sim
in
CP
[
"sim"
][
"versions"
]
# glob files
# glob files
fps_img
=
CsstMsDataManager
.
glob_image
(
dir_l0
,
ver_sim
=
ver_sim
)
fps_img
=
CsstMsDataManager
.
glob_image
(
dir_l0
,
ver_sim
=
ver_sim
)
...
@@ -348,6 +350,8 @@ class CsstMsDataManager:
...
@@ -348,6 +350,8 @@ class CsstMsDataManager:
_survey
=
_survey
,
_survey
=
_survey
,
obs_type
=
obs_type
,
obs_type
=
obs_type
,
l0_post
=
"_"
.
join
(
l0_post
),
l0_post
=
"_"
.
join
(
l0_post
),
log_ppl
=
log_ppl
,
log_mod
=
log_mod
,
n_jobs
=
n_jobs
,
n_jobs
=
n_jobs
,
backend
=
backend
backend
=
backend
)
)
...
@@ -355,10 +359,7 @@ class CsstMsDataManager:
...
@@ -355,10 +359,7 @@ class CsstMsDataManager:
@
staticmethod
@
staticmethod
def
glob_image
(
dir_l0
,
ver_sim
=
"C5.2"
):
def
glob_image
(
dir_l0
,
ver_sim
=
"C5.2"
):
""" glob files in L0 data directory """
""" glob files in L0 data directory """
if
ver_sim
==
"C3"
:
assert
ver_sim
in
CP
[
"sim"
][
"versions"
]
pattern
=
os
.
path
.
join
(
dir_l0
,
"MSC_MS_*_raw.fits"
)
else
:
assert
ver_sim
in
[
"C5.1"
,
"C5.2"
]
pattern
=
os
.
path
.
join
(
dir_l0
,
"CSST_MSC_MS_SCI_*.fits"
)
pattern
=
os
.
path
.
join
(
dir_l0
,
"CSST_MSC_MS_SCI_*.fits"
)
fps
=
glob
.
glob
(
pattern
)
fps
=
glob
.
glob
(
pattern
)
fps
=
[
os
.
path
.
basename
(
fp
)
for
fp
in
fps
]
fps
=
[
os
.
path
.
basename
(
fp
)
for
fp
in
fps
]
...
@@ -370,7 +371,7 @@ class CsstMsDataManager:
...
@@ -370,7 +371,7 @@ class CsstMsDataManager:
@
staticmethod
@
staticmethod
def
glob_cat
(
dir_l0
,
ver_sim
=
"C5"
):
def
glob_cat
(
dir_l0
,
ver_sim
=
"C5"
):
""" glob input catalogs in L0 data directory """
""" glob input catalogs in L0 data directory """
assert
ver_sim
in
[
"C5.1"
,
"C5.2
"
]
assert
ver_sim
in
CP
[
"sim"
][
"versions
"
]
pattern
=
os
.
path
.
join
(
dir_l0
,
"MSC_*.cat"
)
pattern
=
os
.
path
.
join
(
dir_l0
,
"MSC_*.cat"
)
fps
=
glob
.
glob
(
pattern
)
fps
=
glob
.
glob
(
pattern
)
fps
=
[
os
.
path
.
basename
(
fp
)
for
fp
in
fps
]
fps
=
[
os
.
path
.
basename
(
fp
)
for
fp
in
fps
]
...
@@ -385,21 +386,21 @@ class CsstMsDataManager:
...
@@ -385,21 +386,21 @@ class CsstMsDataManager:
def
l0_cat
(
self
,
detector
=
6
):
def
l0_cat
(
self
,
detector
=
6
):
""" the L0 cat file path"""
""" the L0 cat file path"""
assert
self
.
ver_sim
==
"C5.2"
assert
self
.
ver_sim
in
CP
[
"sim"
][
"versions"
]
fn
=
"{}_{}_chip_{:02d}_filt_{}.cat"
.
format
(
fn
=
"{}_{}_chip_{:02d}_filt_{}.cat"
.
format
(
self
.
_instrument
,
self
.
obs_id
,
detector
,
self
.
detector2filter
[
detector
])
self
.
_instrument
,
self
.
obs_id
,
detector
,
self
.
detector2filter
[
detector
])
return
os
.
path
.
join
(
self
.
dir_l0
,
fn
)
return
os
.
path
.
join
(
self
.
dir_l0
,
fn
)
def
l0_log
(
self
,
detector
=
6
):
def
l0_log
(
self
,
detector
=
6
):
""" L0 log file path """
""" L0 log file path """
assert
self
.
ver_sim
==
"C5.2"
assert
self
.
ver_sim
in
CP
[
"sim"
][
"versions"
]
fn
=
"{}_{}_chip_{:02d}_filt_{}.log"
.
format
(
fn
=
"{}_{}_chip_{:02d}_filt_{}.log"
.
format
(
self
.
_instrument
,
self
.
obs_id
,
detector
,
self
.
detector2filter
[
detector
])
self
.
_instrument
,
self
.
obs_id
,
detector
,
self
.
detector2filter
[
detector
])
return
os
.
path
.
join
(
self
.
dir_l0
,
fn
)
return
os
.
path
.
join
(
self
.
dir_l0
,
fn
)
def
l0_detector
(
self
,
detector
=
6
):
def
l0_detector
(
self
,
detector
=
6
):
""" L0 detector-specific image file path """
""" L0 detector-specific image file path """
assert
self
.
ver_sim
in
[
"C5.1"
,
"C5.2
"
]
assert
self
.
ver_sim
in
CP
[
"sim"
][
"versions
"
]
fn
=
"{}_{}_{}_SCI_{}_{}_{}_{:02d}_L0_1.fits"
.
format
(
fn
=
"{}_{}_{}_SCI_{}_{}_{}_{:02d}_L0_1.fits"
.
format
(
self
.
_telescope
,
self
.
_instrument
,
self
.
_survey
,
self
.
_telescope
,
self
.
_instrument
,
self
.
_survey
,
self
.
exp_start
,
self
.
exp_stop
,
self
.
obs_id
,
detector
)
self
.
exp_start
,
self
.
exp_stop
,
self
.
obs_id
,
detector
)
...
@@ -407,7 +408,7 @@ class CsstMsDataManager:
...
@@ -407,7 +408,7 @@ class CsstMsDataManager:
def
l0_crs
(
self
,
detector
=
6
):
def
l0_crs
(
self
,
detector
=
6
):
""" L0 cosmic ray file path """
""" L0 cosmic ray file path """
assert
self
.
ver_sim
in
[
"C5.1"
,
"C5.2
"
]
assert
self
.
ver_sim
in
CP
[
"sim"
][
"versions
"
]
fn
=
"{}_{}_{}_CRS_{}_{}_{}_{:02d}_L0_1.fits"
.
format
(
fn
=
"{}_{}_{}_CRS_{}_{}_{}_{:02d}_L0_1.fits"
.
format
(
self
.
_telescope
,
self
.
_instrument
,
self
.
_survey
,
self
.
_telescope
,
self
.
_instrument
,
self
.
_survey
,
self
.
exp_start
,
self
.
exp_stop
,
self
.
obs_id
,
detector
)
self
.
exp_start
,
self
.
exp_stop
,
self
.
obs_id
,
detector
)
...
@@ -429,7 +430,7 @@ class CsstMsDataManager:
...
@@ -429,7 +430,7 @@ class CsstMsDataManager:
L1 file path
L1 file path
"""
"""
assert
self
.
ver_sim
in
[
"C5.1"
,
"C5.2
"
]
assert
self
.
ver_sim
in
CP
[
"sim"
][
"versions
"
]
fn
=
"{}_{}_{}_SCI_{}_{}_{}_{:02d}_{}"
.
format
(
fn
=
"{}_{}_{}_SCI_{}_{}_{}_{:02d}_{}"
.
format
(
self
.
_telescope
,
self
.
_instrument
,
self
.
_survey
,
self
.
_telescope
,
self
.
_instrument
,
self
.
_survey
,
self
.
exp_start
,
self
.
exp_stop
,
self
.
obs_id
,
detector
,
post
)
self
.
exp_start
,
self
.
exp_stop
,
self
.
obs_id
,
detector
,
post
)
...
@@ -752,12 +753,17 @@ class CsstMsDataManager:
...
@@ -752,12 +753,17 @@ class CsstMsDataManager:
dst
=
os
.
path
.
join
(
dir_l0
,
os
.
path
.
basename
(
tbl
[
"file_path"
][
i_rec
])),
dst
=
os
.
path
.
join
(
dir_l0
,
os
.
path
.
basename
(
tbl
[
"file_path"
][
i_rec
])),
)
)
# initialize dm
# initialize dm
dm
=
CsstMsDataManager
.
from_dir
(
dm
=
CsstMsDataManager
.
from_dir
(
ver_sim
=
"C5.2"
,
datatype
=
datatype
,
dir_l0
=
dir_l0
,
dir_l1
=
dir_l1
,
ver_sim
=
"C5.2"
,
path_aux
=
path_aux
,
use_dfs
=
use_dfs
,
dfs_node
=
dfs_node
,
datatype
=
datatype
,
n_jobs
=
n_jobs
,
backend
=
backend
dir_l0
=
dir_l0
,
dir_l1
=
dir_l1
,
path_aux
=
path_aux
,
use_dfs
=
use_dfs
,
dfs_node
=
dfs_node
,
n_jobs
=
n_jobs
,
backend
=
backend
)
)
assert
dm
.
obs_id
==
obs_id
assert
dm
.
obs_id
==
obs_id
...
...
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment