Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
csst-pipeline
csst_common
Commits
a6340265
Commit
a6340265
authored
Nov 21, 2022
by
BO ZHANG
🏀
Browse files
chdir to dir_l1 before the first initialization
parent
9ca00eb1
Changes
1
Show whitespace changes
Inline
Side-by-side
csst_common/data_manager.py
View file @
a6340265
...
@@ -314,7 +314,7 @@ class CsstMsDataManager:
...
@@ -314,7 +314,7 @@ class CsstMsDataManager:
fps
=
[
os
.
path
.
basename
(
fp
)
for
fp
in
fps
]
fps
=
[
os
.
path
.
basename
(
fp
)
for
fp
in
fps
]
fps
.
sort
()
fps
.
sort
()
print
(
"
@DM.glob_dir:
{} files found with pattern: {}"
.
format
(
len
(
fps
),
pattern
))
print
(
"{} files found with pattern: {}"
.
format
(
len
(
fps
),
pattern
))
return
fps
return
fps
@
staticmethod
@
staticmethod
...
@@ -582,6 +582,17 @@ class CsstMsDataManager:
...
@@ -582,6 +582,17 @@ class CsstMsDataManager:
def
from_dfs
(
obs_id
=
"100000100"
,
datatype
=
"mbi"
,
dir_l0
=
"/L1Pipeline/L0"
,
dir_l1
=
"/L1Pipeline/L1"
,
def
from_dfs
(
obs_id
=
"100000100"
,
datatype
=
"mbi"
,
dir_l0
=
"/L1Pipeline/L0"
,
dir_l1
=
"/L1Pipeline/L1"
,
dfs_mode
=
True
,
dfs_node
=
"pml"
):
dfs_mode
=
True
,
dfs_node
=
"pml"
):
""" Initialize CsstMsDataManager from DFS. """
""" Initialize CsstMsDataManager from DFS. """
# (clear and) make directories
if
os
.
path
.
exists
(
dir_l0
):
os
.
system
(
f
"rm -rf
{
dir_l0
}
/*"
)
else
:
os
.
mkdir
(
dir_l0
)
if
os
.
path
.
exists
(
dir_l1
):
os
.
system
(
f
"rm -rf
{
dir_l1
}
/*"
)
else
:
os
.
mkdir
(
dir_l1
)
os
.
chdir
(
dir_l1
)
print
(
f
"Query obs_id=
{
obs_id
}
..."
,
end
=
""
)
print
(
f
"Query obs_id=
{
obs_id
}
..."
,
end
=
""
)
records
=
CsstMsDataManager
(
dfs_node
=
dfs_node
,
verbose
=
False
).
dfs_L0DataApi
.
find
(
obs_id
=
obs_id
)
records
=
CsstMsDataManager
(
dfs_node
=
dfs_node
,
verbose
=
False
).
dfs_L0DataApi
.
find
(
obs_id
=
obs_id
)
print
(
f
"
{
records
[
'totalCount'
]
}
records obtained!"
)
print
(
f
"
{
records
[
'totalCount'
]
}
records obtained!"
)
...
@@ -593,16 +604,6 @@ class CsstMsDataManager:
...
@@ -593,16 +604,6 @@ class CsstMsDataManager:
else
:
# in docker
else
:
# in docker
dfs_root
=
"/dfsroot"
dfs_root
=
"/dfsroot"
# (clear and) make directories
if
os
.
path
.
exists
(
dir_l0
):
os
.
system
(
f
"rm -rf
{
dir_l0
}
/*"
)
else
:
os
.
mkdir
(
dir_l0
)
if
os
.
path
.
exists
(
dir_l1
):
os
.
system
(
f
"rm -rf
{
dir_l1
}
/*"
)
else
:
os
.
mkdir
(
dir_l1
)
print
(
"Making symbolic links ..."
)
print
(
"Making symbolic links ..."
)
for
i_rec
in
range
(
len
(
tbl
)):
for
i_rec
in
range
(
len
(
tbl
)):
os
.
symlink
(
os
.
symlink
(
...
@@ -610,12 +611,14 @@ class CsstMsDataManager:
...
@@ -610,12 +611,14 @@ class CsstMsDataManager:
dst
=
os
.
path
.
join
(
dir_l0
,
os
.
path
.
basename
(
tbl
[
"file_path"
][
i_rec
])),
dst
=
os
.
path
.
join
(
dir_l0
,
os
.
path
.
basename
(
tbl
[
"file_path"
][
i_rec
])),
)
)
# as from_dfs only works in docker mode
if
datatype
==
"mbi"
:
if
datatype
==
"mbi"
:
path_aux
=
"/
share/
L1Pipeline/aux/C5.2_ref_mbi/MSC_{}_*_{:02d}_combine.fits"
path_aux
=
"/L1Pipeline/aux/C5.2_ref_mbi/MSC_{}_*_{:02d}_combine.fits"
elif
datatype
==
"sls"
:
elif
datatype
==
"sls"
:
path_aux
=
"/
share/
L1Pipeline/aux/C5.2_ref_sls/csst_{:02d}{}.fits"
path_aux
=
"/L1Pipeline/aux/C5.2_ref_sls/csst_{:02d}{}.fits"
else
:
else
:
path_aux
=
""
raise
ValueError
(
f
"Bad datatype:
{
datatype
}
"
)
# initialize dm
# initialize dm
dm
=
CsstMsDataManager
.
from_dir
(
dm
=
CsstMsDataManager
.
from_dir
(
ver_sim
=
"C5.2"
,
datatype
=
datatype
,
dir_l0
=
dir_l0
,
dir_l1
=
dir_l1
,
ver_sim
=
"C5.2"
,
datatype
=
datatype
,
dir_l0
=
dir_l0
,
dir_l1
=
dir_l1
,
...
...
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment