Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
csst-pipeline
csst_common
Commits
62e27171
Commit
62e27171
authored
Dec 09, 2023
by
BO ZHANG
🏀
Browse files
change CRDS to CCDS
parent
25296703
Pipeline
#2153
failed with stage
in 0 seconds
Changes
2
Pipelines
1
Hide whitespace changes
Inline
Side-by-side
csst_common/crds/__init__.py
View file @
62e27171
from
.c
r
ds
import
C
R
DS
from
.c
c
ds
import
C
C
DS
csst_common/crds/c
r
ds.py
→
csst_common/crds/c
c
ds.py
View file @
62e27171
import
functools
import
os
from
c
r
ds
import
client
from
c
c
ds
import
client
class
CRDS
:
class
CCDS
:
def
__init__
(
self
,
ccds_root
=
"/ccds_root"
,
ccds_cache
=
"/pipeline/ccds_cache"
):
print
(
"Setting CCDS root path ... "
,
end
=
""
)
self
.
ccds_root
=
ccds_root
print
(
self
.
ccds_root
)
def
__init__
(
self
,
crdsroot
=
"/crdsroot"
,
crdscache
=
"/pipeline/crds_cache"
):
print
(
"Setting CRDS root path ... "
,
end
=
""
)
self
.
crdsroot
=
crdsroot
print
(
self
.
crdsroot
)
print
(
"Setting CCDS cache path ... "
,
end
=
""
)
self
.
ccds_cache
=
ccds_cache
print
(
self
.
ccds_cache
)
print
(
"Setting CRDS cache path ... "
,
end
=
""
)
self
.
crdscache
=
crdscache
print
(
self
.
crdscache
)
print
(
"Setting CRDS environment variables ... "
,
end
=
""
)
os
.
environ
[
'CRDS_MODE'
]
=
'remote'
os
.
environ
[
'CRDS_PATH'
]
=
crdscache
os
.
environ
[
'CRDS_OBSERVATORY'
]
=
'csst'
print
(
"Setting CCDS environment variables ... "
,
end
=
""
)
os
.
environ
[
"CRDS_MODE"
]
=
"remote"
os
.
environ
[
"CRDS_PATH"
]
=
ccds_cache
os
.
environ
[
"CRDS_OBSERVATORY"
]
=
"csst"
print
(
"Done"
)
print
(
"Query for observatory ... "
,
end
=
""
)
self
.
observatory
=
client
.
get_default_observatory
()
print
(
self
.
observatory
)
assert
self
.
observatory
==
"csst"
,
f
"observatory [
{
self
.
observatory
}
] is not csst!"
assert
(
self
.
observatory
==
"csst"
),
f
"observatory [
{
self
.
observatory
}
] is not csst!"
print
(
"Query for operational_context ... "
,
end
=
""
)
try
:
...
...
@@ -33,24 +34,30 @@ class CRDS:
client
.
dump_mappings
(
self
.
operational_context
)
self
.
is_available
=
True
except
BaseException
as
e
:
print
(
"C
R
DS access failed! "
,
e
.
args
)
print
(
"C
C
DS access failed! "
,
e
.
args
)
self
.
operational_context
=
""
self
.
is_available
=
False
print
(
self
.
operational_context
)
# assert self.operational_context != "", f"operational_context [{self.operational_context}] is empty!"
def
__repr__
(
self
):
return
f
"< CRDS url='
{
os
.
getenv
(
'CRDS_SERVER_URL'
,
default
=
''
)
}
' >
\n
"
\
f
" - is_available=
{
self
.
is_available
}
\n
"
\
f
" - observatory='
{
self
.
observatory
}
'
\n
"
\
f
" - operational_context='
{
self
.
operational_context
}
'
\n
"
\
f
" - crdsroot='
{
self
.
crdsroot
}
'
\n
"
\
f
" - crdscache='
{
self
.
crdscache
}
'
\n
"
def
get_refs
(
self
,
file_path
=
"/dfsroot/L0/MSC/SCIE/62030/10160000105/MS/CSST_MSC_MS_SCIE_20280916072059_20280916072329_10160000105_10_L0_V01.fits"
):
return
(
f
"< CRDS url='
{
os
.
getenv
(
'CRDS_SERVER_URL'
,
default
=
''
)
}
' >
\n
"
f
" - is_available=
{
self
.
is_available
}
\n
"
f
" - observatory='
{
self
.
observatory
}
'
\n
"
f
" - operational_context='
{
self
.
operational_context
}
'
\n
"
f
" - ccds_root='
{
self
.
ccds_root
}
'
\n
"
f
" - crdscache='
{
self
.
ccds_cache
}
'
\n
"
)
def
get_refs
(
self
,
file_path
=
"/dfsroot/L0/MSC/SCIE/62030/10160000105/MS/CSST_MSC_MS_SCIE_20280916072059_20280916072329_10160000105_10_L0_V01.fits"
,
):
# print("get min header")
hdrs
=
client
.
api
.
get_minimum_header
(
self
.
operational_context
,
file_path
,
ignore_cache
=
False
)
hdrs
=
client
.
api
.
get_minimum_header
(
self
.
operational_context
,
file_path
,
ignore_cache
=
False
)
# {'CAMERA': 'MS',
# 'CHIPID': '10',
# 'DATE-OBS': '2028-09-16T07:20:59',
...
...
@@ -70,14 +77,14 @@ class CRDS:
for
ref_type
in
refs
.
keys
():
d
=
client
.
api
.
get_file_info
(
self
.
operational_context
,
refs
[
ref_type
])
if
d
:
fp
=
os
.
path
.
join
(
self
.
c
r
dsroot
,
"references"
,
d
[
"file_path"
])
fp
=
os
.
path
.
join
(
self
.
c
c
ds
_
root
,
"references"
,
d
[
"file_path"
])
assert
os
.
path
.
exists
(
fp
),
f
"File path [
{
fp
}
] does not exist!"
refs_fp
[
ref_type
]
=
fp
else
:
print
(
f
"Failed to get file info for [
{
ref_type
}
:
{
refs
[
ref_type
]
}
]"
)
# {'bias': '/c
r
dsroot/references/msc/csst_msc_ms_bias_10_000001.fits',
# 'dark': '/c
r
dsroot/references/msc/csst_msc_ms_dark_10_000001.fits',
# 'ledflat': '/c
r
dsroot/references/msc/csst_msc_ms_ledflat_10_000001.fits'}
# {'bias': '/c
c
ds
_
root/references/msc/csst_msc_ms_bias_10_000001.fits',
# 'dark': '/c
c
ds
_
root/references/msc/csst_msc_ms_dark_10_000001.fits',
# 'ledflat': '/c
c
ds
_
root/references/msc/csst_msc_ms_ledflat_10_000001.fits'}
return
refs_fp
@
staticmethod
...
...
@@ -89,7 +96,7 @@ class CRDS:
return
res
except
BaseException
as
e
:
print
(
f
"Error occurs:
{
e
.
__repr__
()
}
"
)
raise
RuntimeError
(
"All attempts failed."
)
raise
RuntimeError
(
f
"All
{
n
}
attempts failed."
)
def
retry
(
func
,
n
=
3
):
...
...
@@ -113,23 +120,24 @@ refs = crds.getreferences(hdrs, reftypes=['bias'], context=c.operational_context
docker run --rm -it
\
-v /share/crdsdata/data:/c
r
dsroot
\
-v /share/crdsdata/data:/c
c
ds
_
root
\
-e CRDS_SERVER_URL=http://172.24.27.2:29000
\
-v /share/dfs:/dfsroot
\
csst/csst-msc-l1-mbi bash
"""
if
__name__
==
"__main__"
:
c
=
C
R
DS
()
c
=
C
C
DS
()
refs
=
c
.
get_refs
()
print
(
refs
)
refs
=
c
.
retry
(
c
.
get_refs
,
3
,
file_path
=
"/dfsroot/L0/MSC/SCIE/62030/10160000105/MS/CSST_MSC_MS_SCIE_20280916072059_20280916072329_10160000105_11_L0_V01.fits"
c
.
get_refs
,
3
,
file_path
=
"/dfsroot/L0/MSC/SCIE/62030/10160000105/MS/CSST_MSC_MS_SCIE_20280916072059_20280916072329_10160000105_11_L0_V01.fits"
,
)
print
(
refs
)
# {'bias': '/c
r
dsroot/references/msc/csst_msc_ms_bias_11_000001.fits',
# 'dark': '/c
r
dsroot/references/msc/csst_msc_ms_dark_11_000001.fits',
# 'ledflat': '/c
r
dsroot/references/msc/csst_msc_ms_ledflat_11_000001.fits',
# 'shutter': '/c
r
dsroot/references/msc/csst_msc_ms_shutter_11_000001.fits'}
# {'bias': '/c
c
ds
_
root/references/msc/csst_msc_ms_bias_11_000001.fits',
# 'dark': '/c
c
ds
_
root/references/msc/csst_msc_ms_dark_11_000001.fits',
# 'ledflat': '/c
c
ds
_
root/references/msc/csst_msc_ms_ledflat_11_000001.fits',
# 'shutter': '/c
c
ds
_
root/references/msc/csst_msc_ms_shutter_11_000001.fits'}
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment