Skip to content
GitLab
Commit 48949a0c authored by Xie Zhou's avatar Xie Zhou
Browse files

update

parent 29c32645
Hide whitespace changes
Inline Side-by-side
.gitignore
View file @ 48949a0c
......@@ -37,4 +37,7 @@ test_reports.xml
**/*.egg-info
**/*.rdb
**/.eggs
**/_version.py
\ No newline at end of file
**/_version.py
# tmp test
.tmp/*
csst_dfs_api_local/common/db.py
View file @ 48949a0c
import os
import datetime
import sqlite3
from DBUtils.PersistentDB import PersistentDB
from dbutils.persistent_db import PersistentDB
from .utils import singleton
import logging
......
csst_dfs_api_local/common/db.sql
View file @ 48949a0c
create table if not exists "t_rawfits" (
"id" TEXT,
"filename" TEXT,
"obs_time" NUMERIC,
"ccd_num" NUMERIC,
"type" TEXT,
"path" TEXT,
PRIMARY KEY("id")
);
csst_dfs_api_local/ifs/fits.py
View file @ 48949a0c
import os
import logging
from ..common.utils import get_parameter
import os
import shutil
from glob import glob
from astropy.io import fits
from ..common.db import DBClient
from ..common.utils import get_parameter
log = logging.getLogger('csst')
class FitsApi(object):
def __init__(self):
self.root_dir = os.getenv("CSST_LOCAL_FILE_ROOT", "/opt/temp/csst")
......@@ -26,15 +32,29 @@ class FitsApi(object):
def find(self, **kwargs):
'''
parameter kwargs:
obs_time = [str]
obs_time = [int]
type = [str]
fits_id = [str]
return list of fits_id
return list of paths
'''
paths = {}
for (path, _, file_names) in os.walk(os.path.join(self.root_dir, "fits")):
for filename in file_names:
if filename.find(".fits") > 0:
paths[filename] = os.path.join(path, filename)
paths = []
obs_time = get_parameter(kwargs, "obs_time")
type = get_parameter(kwargs, "type")
fits_id = get_parameter(kwargs, "fits_id")
if (obs_time is None or type is None) and fits_id is None:
raise Exception('obs_time and type need to be defind')
if fits_id is None:
c, r = self.db.select_many(
'select * from t_rawfits where obs_time=? and type=?',
(obs_time, type)
)
if len(r) < 1:
raise Exception('not found')
for items in r:
paths.append(items[4])
return paths
def read(self, **kwargs):
......@@ -42,7 +62,8 @@ class FitsApi(object):
parameter kwargs:
fits_id = [str]
file_path = [str]
chunk_size = [int]
yield bytes of fits file
'''
fits_id = get_parameter(kwargs, "fits_id")
......@@ -52,33 +73,67 @@ class FitsApi(object):
raise Exception("fits_id or file_path need to be defined")
if fits_id is not None:
c, r = self.db.select_one("select * from t_rawfits where id=?",(fits_id))
c, r = self.db.select_one(
"select * from t_rawfits where id=?", (fits_id))
if c == 1:
file_path = r["path"]
if file_path is not None:
chunk_size = get_parameter(kwargs, "chunk_size", 1024)
with open(file_path,'r') as f:
with open(file_path, 'r') as f:
while True:
data = f.read(chunk_size)
if not data:
break
yield data
def update_status(self, **kwargs):
pass
def upload(self,**kwargs):
pass
def upload(self, **kwargs):
'''
parameter kwargs:
file_path = [str]
upload to database and copy to csstpath
'''
file_path = get_parameter(kwargs, "file_path")
if file_path is None:
raise Exception("file_path need to be defined")
basename = os.path.basename(file_path)
name = basename.split('.fits')[0]
c, r = self.db.select_many(
"select * from t_rawfits where id=?",
(name,)
)
if len(r) >= 1:
print('already upload', name)
return
hu = fits.getheader(file_path)
obs_time = hu['obst']
ccd_num = hu['ccd_num']
# print(obs_time, ccd_num)
type = 'obs'
save_path = os.path.join(self.root_dir, 'fits')
save_path = os.path.join(save_path, basename)
self.db.execute(
'INSERT INTO t_rawfits VALUES(?,?,?,?,?)',
(name, obs_time, ccd_num, type, save_path)
)
self.db._conn.commit()
if file_path != save_path:
shutil.copyfile(file_path, save_path)
log.info("%s imported.", save_path)
def scan2db(self):
paths = {}
for (path, _, file_names) in os.walk(os.path.join(self.root_dir, "fits")):
for filename in file_names:
if filename.find(".fits") > 0:
obs_time = ""
self.db.execute("insert into t_rawfits values(?,?,?)", param=(filename, obs_time, os.path.join(path, filename)))
log.info("%s imported.",os.path.join(path, filename))
return paths
\ No newline at end of file
self.upload(file_path=os.path.join(path, filename))
return paths
csst_dfs_api_local/ifs/reffits.py
View file @ 48949a0c
import logging
import os
import shutil
from astropy.io import fits
from ..common.db import DBClient
from ..common.utils import get_parameter
from . import FitsApi
log = logging.getLogger('csst')
class RefFitsApi(object):
def __init__(self, sub_system = "ifs"):
self.sub_system = sub_system
def fetch(self, **kwargs):
pass
class RefFitsApi(FitsApi):
def upload(self, **kwargs):
'''
parameter kwargs:
file_path = [str]
upload to database and copy to csstpath
'''
file_path = get_parameter(kwargs, "file_path")
if file_path is None:
raise Exception("file_path need to be defined")
basename = os.path.basename(file_path)
name = basename.split('.fits')[0]
c, r = self.db.select_many(
"select * from t_rawfits where id=?",
(name,)
)
if len(r) >= 1:
print('already upload', name)
return
hu = fits.getheader(file_path)
obs_time = hu['obst']
ccd_num = hu['ccd_num']
type = name.split('_')[1]
save_path = os.path.join(self.root_dir, 'refs')
save_path = os.path.join(save_path, basename)
self.db.execute(
'INSERT INTO t_rawfits VALUES(?,?,?,?,?)',
(name, obs_time, ccd_num, type, save_path)
)
self.db._conn.commit()
if file_path != save_path:
shutil.copyfile(file_path, save_path)
log.info("%s imported.", save_path)
def scan2db(self):
paths = {}
for (path, _, file_names) in os.walk(os.path.join(self.root_dir, "refs")):
for filename in file_names:
if filename.find(".fits") > 0:
self.upload(file_path=os.path.join(path, filename))
return paths
csst_dfs_api_local/ifs/result0.py
View file @ 48949a0c
......@@ -6,4 +6,21 @@ class Result0Api(object):
self.sub_system = sub_system
def upload(self, **kwargs):
'''
parameter kwargs:
fits_id = [str]
file_path = [str]
chunk_size = [int]
yield bytes of fits file
'''
pass
def find(self, **kwargs):
pass
def read(self, **kwargs):
pass
def wirte(self, **kwargs):
pass
\ No newline at end of file
tests/test_ifs_fits.py
View file @ 48949a0c
import logging
import unittest
from csst_dfs_api_local.ifs import FitsApi
log = logging.getLogger('csst')
class IFSFitsTestCase(unittest.TestCase):
def setUp(self):
self.api = FitsApi()
self.api.scan2db()
def test_find(self):
files = self.api.find()
print("find ", files)
\ No newline at end of file
path = self.api.find(obs_time=900, type='obs')
log.info('find', path)
path = self.api.find(fits_id='CCD2_ObsTime_600_ObsNum_8')
log.info('find', path)
def test_read(self):
file = self.api.read(fits_id='CCD2_ObsTime_600_ObsNum_8')
log.info('read', str(type(file)))
path = self.api.find(obs_time=900, type='obs')
file = self.api.read(file_path=path)
log.info('read', str(type(file)))
tests/test_ifs_refs.py 0 → 100644
View file @ 48949a0c
import logging
import unittest
from csst_dfs_api_local.ifs import RefFitsApi
log = logging.getLogger('csst')
class IFSFitsTestCase(unittest.TestCase):
def setUp(self):
self.api = RefFitsApi()
self.api.scan2db()
def test_find(self):
path = self.api.find(obs_time=300, type='Flat')
log.info('find', path)
path = self.api.find(fits_id='CCD1_Flat_img')
log.info('find', path)
def test_read(self):
file = self.api.read(fits_id='CCD1_Flat_img')
log.info('read', str(type(file)))
path = self.api.find(obs_time=300, type='Flat')
file = self.api.read(file_path=path)
log.info('read', str(type(file)))
\ No newline at end of file
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment