Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
csst-pipeline
csst_common
Commits
3570fb46
Commit
3570fb46
authored
Dec 19, 2023
by
BO ZHANG
🏀
Browse files
configurable log paths
parent
a3521701
Pipeline
#2564
failed with stage
in 0 seconds
Changes
1
Pipelines
1
Show whitespace changes
Inline
Side-by-side
csst_common/pipeline.py
View file @
3570fb46
...
@@ -12,6 +12,7 @@ Modified-History:
...
@@ -12,6 +12,7 @@ Modified-History:
"""
"""
import
json
import
json
import
os
import
os
import
shutil
import
subprocess
import
subprocess
import
warnings
import
warnings
from
typing
import
Any
from
typing
import
Any
...
@@ -24,6 +25,7 @@ from .file import File
...
@@ -24,6 +25,7 @@ from .file import File
from
.logger
import
get_logger
from
.logger
import
get_logger
# TODO: unit test in need
class
Pipeline
:
class
Pipeline
:
"""
"""
CSST pipeline configuration class.
CSST pipeline configuration class.
...
@@ -62,6 +64,8 @@ class Pipeline:
...
@@ -62,6 +64,8 @@ class Pipeline:
dfs_root
:
str
=
"/dfs_root"
,
dfs_root
:
str
=
"/dfs_root"
,
ccds_root
:
str
=
"/ccds_root"
,
ccds_root
:
str
=
"/ccds_root"
,
ccds_cache
:
str
=
"/pipeline/ccds_cache"
,
ccds_cache
:
str
=
"/pipeline/ccds_cache"
,
pipeline_log
:
str
=
"pipeline.log"
,
module_log
:
str
=
"module.log"
,
filter_warnings
:
bool
=
False
,
filter_warnings
:
bool
=
False
,
dfs
:
bool
=
True
,
dfs
:
bool
=
True
,
ccds
:
bool
=
False
,
ccds
:
bool
=
False
,
...
@@ -86,11 +90,11 @@ class Pipeline:
...
@@ -86,11 +90,11 @@ class Pipeline:
# set logger
# set logger
self
.
pipeline_logger
=
get_logger
(
self
.
pipeline_logger
=
get_logger
(
name
=
"pipeline logger"
,
name
=
"pipeline logger"
,
filename
=
os
.
path
.
join
(
self
.
dir_output
,
"
pipeline
.
log
"
),
filename
=
os
.
path
.
join
(
self
.
dir_output
,
pipeline
_
log
),
)
)
self
.
module_logger
=
get_logger
(
self
.
module_logger
=
get_logger
(
name
=
"module logger"
,
name
=
"module logger"
,
filename
=
os
.
path
.
join
(
self
.
dir_output
,
"
module
.
log
"
),
filename
=
os
.
path
.
join
(
self
.
dir_output
,
module
_
log
),
)
)
# change working directory
# change working directory
...
@@ -108,10 +112,7 @@ class Pipeline:
...
@@ -108,10 +112,7 @@ class Pipeline:
else
:
else
:
self
.
dfs
=
None
self
.
dfs
=
None
if
ccds
:
if
ccds
:
self
.
ccds
=
CCDS
(
self
.
ccds
=
CCDS
(
ccds_root
=
ccds_root
,
ccds_cache
=
ccds_cache
)
ccds_root
=
ccds_root
,
ccds_cache
=
ccds_cache
,
)
else
:
else
:
self
.
ccds
=
None
self
.
ccds
=
None
...
@@ -144,15 +145,27 @@ class Pipeline:
...
@@ -144,15 +145,27 @@ class Pipeline:
@
staticmethod
@
staticmethod
def
reset_warnings
(
self
):
def
reset_warnings
(
self
):
#
Reset warning filters
"""
Reset warning filters
."""
warnings
.
resetwarnings
()
warnings
.
resetwarnings
()
def
file
(
self
,
file_path
):
def
file
(
self
,
file_path
):
"""Initialize File object."""
return
File
(
file_path
,
new_dir
=
self
.
dir_output
)
return
File
(
file_path
,
new_dir
=
self
.
dir_output
)
def
new
(
self
,
file_name
=
"test.fits"
):
def
new
(
self
,
file_name
=
"test.fits"
):
"""Create new file in output directory."""
return
os
.
path
.
join
(
self
.
dir_output
,
file_name
)
return
os
.
path
.
join
(
self
.
dir_output
,
file_name
)
@
staticmethod
def
move
(
file_src
:
str
,
file_dst
:
str
):
"""Move file `file_src` to `file_dist`."""
shutil
.
move
(
file_src
,
file_dst
)
@
staticmethod
def
copy
(
file_src
:
str
,
file_dst
:
str
):
"""Move file `file_src` to `file_dist`."""
shutil
.
copy
(
file_src
,
file_dst
)
# class ErrorTrace:
# class ErrorTrace:
# """Write error trace to file."""
# """Write error trace to file."""
...
...
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment