Initial commit

requirements.txt

+requests==2.31.0
+boto3==1.35.24
\ No newline at end of file

setup.cfg

+[bumpver]
+current_version = "20241119"
+version_pattern = "YYYYMMDD[.PATCH]"
+commit_message = "Release {new_version}"
+commit = True
+tag = True
+push = True
+
+[bumpver:file_patterns]
+setup.cfg =
+    current_version = "{version}"
+csst_dfs_client/version.py =
+    __version__ = "{version}"
+

setup.py

+import setuptools
+import os
+
+with open("README.md", "r") as f:
+    long_description = f.read()
+
+with open("requirements.txt", "r") as f:
+    requirements = [
+        req.strip()
+        for req in f.readlines()
+        if not req.startswith("#") and req.__contains__("==")
+    ]
+
+def version():
+    __version = {}
+    version_path = os.path.join(os.path.dirname(__file__), "csst_dfs_client", "version.py")
+    with open(version_path, "r") as file:
+        exec(file.read(), __version)
+    return __version["__version__"]
+
+setuptools.setup(
+    name="csst-dfs-client",
+    version=version(),
+    author="Shoulin Wei",
+    author_email="weishoulin@kust.edu.cn",
+    description="CSST DFS Project",
+    long_description=long_description,
+    long_description_content_type="text/markdown",
+    url="https://gitlab.asos2.ynao.ac.cn/csst_dfs_client",
+    packages=setuptools.find_packages(where="." , exclude=['tests']), 
+    license="MIT",
+    classifiers=[
+        # How mature is this project?
+        #   3 - Alpha
+        #   4 - Beta
+        #   5 - Production/Stable
+        "Development Status :: 3 - Alpha",
+        "Intended Audience :: Science/Research",
+        "License :: OSI Approved :: MIT License",
+        "Operating System :: OS Independent",
+        "Programming Language :: Python :: 3",
+        "Framework :: FastAPI",
+        "Topic :: Scientific/Engineering :: Physics",
+        "Topic :: Scientific/Engineering :: Astronomy",
+    ],
+    include_package_data=True,
+    package_data={
+        "csst_dfs_client": ["data/*"],
+    },
+    python_requires=">=3.9",
+    install_requires=requirements
+)

tests/data/ingest.data

+ingest test data
\ No newline at end of file

tests/level0.py

+
+from csst_dfs_client import level0
+result = level0.find(file_name = 'CSST_MSC_MS_SCI_20221126233300_20221126233530_10609300035818_12_L0_V01.cat')
+print(result)

tests/test_catalog.py

+import unittest
+
+from csst_dfs_client.catalog import search
+
+class Level0TestCase(unittest.TestCase):
+
+    def setUp(self):
+        pass
+
+    def test_search(self):
+        result = search(
+            ra=90, 
+            dec=24.5, 
+            radius=0.21, 
+            columns = ('ref_epoch','ra','ra_error','dec','dec_error','parallax','parallax_error','pmra','pmra_error','pmdec','pmdec_error','phot_g_mean_mag','source_id'),            
+            catalog_name='gaia3', 
+            min_mag=-1, 
+            max_mag=-1, 
+            obstime = -1, 
+            limit = 10
+        )
+        print(result)
+        self.assertEqual(result.code, 200, "error code: " + str(result.code) + ", message: " + result.message)
+        self.assertIsNotNone(result.data, "error message: " + result.message)

tests/test_common.py

+import unittest
+import os
+from csst_dfs_client import download_file, read_file
+
+class CommonTestCase(unittest.TestCase):
+    def setUp(self):
+        pass
+
+    # def test_download_level0_file(self):
+    #     bytes_data = download_file(file_path= "L0/MSC/SCI/59900/10609400034529/MS/CSST_MSC_MS_SCI_20221117100218_20221117100448_10609400034529_25_L0_V01.fits")
+    #     # save the downloaded file to a local directory
+    #     tmp_file_dir = os.environ.get("UNIT_TEST_DATA_ROOT", "/tmp")
+    #     tmp_file_path = os.path.join(tmp_file_dir, "test_downloaded_file.fits")
+    #     if os.path.exists(tmp_file_path):
+    #         os.remove(tmp_file_path)
+
+    #     with open(tmp_file_path, "wb") as f:
+    #         f.write(bytes_data)
+        
+    #     assert os.path.exists(tmp_file_path)
+    #     os.remove(tmp_file_path)
+
+    def test_read_file(self):
+        bytes_io = read_file(file_path= "L0/MSC/SCI/60470/10109100157991/MS/CSST_MSC_MS_SCI_20240609181116_20240609181347_10109100157991_27_L0_V01.fits")
+        # save the downloaded file to a local directory
+        tmp_file_dir = os.environ.get("UNIT_TEST_DATA_ROOT", "/tmp")
+        tmp_file_path = os.path.join(tmp_file_dir, "test_downloaded_file.fits")
+        if os.path.exists(tmp_file_path):
+            os.remove(tmp_file_path)
+
+        with open(tmp_file_path, "wb") as f:
+            f.write(bytes_io.getvalue())
+        
+        assert os.path.exists(tmp_file_path)
+        os.remove(tmp_file_path)
+        
\ No newline at end of file

tests/test_level0.py

+import unittest
+import time
+from csst_dfs_client import level0
+
+class Level0TestCase(unittest.TestCase):
+
+    def setUp(self):
+        pass
+
+    def test_find(self):
+        start_time = time.time()
+        result = level0.find(obs_id = "10609400034529",
+                            create_time = ("2021-08-30 00:00:00", "2024-12-30 23:59:59"))
+        start_time = time.time()
+        result = level0.find(ra_obj = 170,
+                            dec_obj = -24,
+                            radius = 2)
+        print(f"操作执行时间: {time.time() - start_time} 秒")
+        result = level0.find(file_name="CSST_MSC_MS_SCI_20240609181116_20240609181347_101091001")
+        print(f"操作执行时间: {time.time() - start_time} 秒, 数据量：{result['total_count']}")        
+        self.assertEqual(result.code, 200, "error code: " + str(result.code))
+        self.assertIsNotNone(result.data, "error message: " + result.message)
+
+    # def test_find_by_level0_id(self):
+    #     result = level0.find_by_level0_id(level0_id = "1060940003452925")
+    #     print(result)
+    #     self.assertEqual(result.code, 200, "error code: " + str(result.code) + ", message: " + result.message)
+    #     self.assertIsNotNone(result.data, "error message: " + result.message)
+
+    # def test_update_qc0_status(self):
+    #     result = level0.update_qc0_status(level0_id = "1060940003452925", file_type='SCI', qc0_status=1)
+    #     print(result)
+    #     self.assertEqual(result.code, 200, "error code: " + str(result.code) + ", message: " + result.message)
+
+    # def test_update_prc_status(self):
+    #     result = level0.update_prc_status(level0_id = "1060940003452925", file_type='SCI', prc_status=3)
+    #     print(result)
+    #     self.assertEqual(result.code, 200, "error code: " + str(result.code) + ", message: " + result.message)
+
+    def test_write(self):
+        file_path = "/Users/wsl/temp/csst/import/CSST_MSC_MS_SCI_20240609181116_20240609181347_10109100157991_27_L0_V01.fits"
+        result = level0.write(local_file = file_path)
+        print(result)
+        self.assertEqual(result.code, 200, "error code: " + str(result.code) + ", message: " + result.message)
+
+    # def test_generate_prc_message(self):
+    #     result = level0.generate_prc_msg(
+    #         module_id='MSC',
+    #         obs_id="10609400034529",
+    #         detector_no="30")
+    #     print(result)
+    #     self.assertEqual(result.code, 200, "error code: " + str(result.code) + ", message: " + result.message)
+
+    # def test_process_list(self):
+    #     result = level0.process_list(level0_id="1060940003452925")
+    #     print(result)
+    #     self.assertEqual(result.code, 200, "error code: " + str(result.code) + ", message: " + result.message)
+    # def test_add_process(self):
+    #     result = level0.add_process(level0_id="1060940003452925", pipeline_id="csst-msc-l1-mbi", run_id="202411071002481234", prc_time="2024-11-07 10:24:12", prc_status=1, prc_module="MSC", message="")
+    #     print(result)
+    #     self.assertEqual(result.code, 200, "error code: " + str(result.code) + ", message: " + result.message)           
\ No newline at end of file

tests/test_level1.py

+import unittest
+
+from csst_dfs_client import level1
+
+class Level1TestCase(unittest.TestCase):
+
+    def setUp(self):
+        pass
+
+    def test_find(self):
+        result = level1.find()
+        print(result)
+        self.assertEqual(result.code, 200, "error code: " + str(result.code))
+        self.assertIsNotNone(result.data, "error message: " + result.message)
+    # def test_find_by_brick_id(self):
+    #     result = level1.find_by_brick_id(brick_id = 401847)
+    #     print(result)
+    #     self.assertEqual(result.code, 200, "error code: " + str(result.code) + ", message: " + result.message)
+    #     self.assertIsNotNone(result.data, "error message: " + result.message)
+    # def test_find_by_level1_id(self):
+    #     result = level1.find_by_level1_id(level1_id = "1060940003452925")
+    #     print(result)
+    #     self.assertEqual(result.code, 200, "error code: " + str(result.code) + ", message: " + result.message)
+    #     self.assertIsNotNone(result.data, "error message: " + result.message)
+
+    # def test_update_qc1_status(self):
+    #     result = level1.update_qc1_status(level1_id = "1060940003452925", file_type='SCI', qc1_status=1)
+    #     print(result)
+    #     self.assertEqual(result.code, 200, "error code: " + str(result.code) + ", message: " + result.message)
+
+    # def test_update_prc_status(self):
+    #     result = level1.update_prc_status(level1_id = "1060940003452925", file_type='SCI', prc_status=3)
+    #     print(result)
+    #     self.assertEqual(result.code, 200, "error code: " + str(result.code) + ", message: " + result.message)
+
+    # def test_write(self):
+    #     file_path = "/Users/wsl/temp/csst/import/CSST_MSC_MS_SCI_20240609181116_20240609181347_10109100157991_27_L1_V01.fits"
+    #     result = level1.write(local_file = file_path, 
+    #         module_id = "MSC",
+    #         level0_id="1010910015799127",
+    #         level1_id = "1010910015799127", 
+    #         pipeline_id = "csst-msc-l1-mbi",
+    #         pmapname="csst_000128.pmap",
+    #         file_type='SCI', 
+    #         build=1,
+    #         prc_status=1)
+    #     print(result)
+    #     self.assertEqual(result.code, 200, "error code: " + str(result.code) + ", message: " + result.message)
+
+    # def test_process_list(self):
+    #     result = level1.process_list(level1_id="1060940003452925")
+    #     print(result)
+    #     self.assertEqual(result.code, 200, "error code: " + str(result.code) + ", message: " + result.message)
+    # def test_add_process(self):
+    #     result = level1.add_process(level1_id="1060940003452925", pipeline_id="csst-msc-l1-mbi", run_id="202411071002481234", prc_time="2024-11-07 10:24:12", prc_status=1, prc_module="MSC", message="")
+    #     print(result)
+    #     self.assertEqual(result.code, 200, "error code: " + str(result.code) + ", message: " + result.message)   
+
+    # def test_sls_find_by_qc1_status(self):
+    #     result = level1.sls_find_by_qc1_status(qc1_status=0, limit=1)
+    #     print(result)
+    #     self.assertEqual(result.code, 200, "error code: " + str(result.code) + ", message: " + result.message)
+    
\ No newline at end of file

tests/test_level2.py

+import unittest
+
+from csst_dfs_client import level2
+
+class Level2TestCase(unittest.TestCase):
+
+    def setUp(self):
+        pass
+
+    def test_find(self):
+        result = level2.find(obs_id = "10609400034529")
+        print(result)
+        self.assertEqual(result.code, 200, "error code: " + str(result.code))
+        self.assertIsNotNone(result.data, "error message: " + result.message)
+
+    # def test_find_by_level2_id(self):
+    #     result = level2.find_by_level2_id(level2_id = "1060940003452925")
+    #     print(result)
+    #     self.assertEqual(result.code, 200, "error code: " + str(result.code) + ", message: " + result.message)
+    #     self.assertIsNotNone(result.data, "error message: " + result.message)
+
+    # def test_update_qc2_status(self):
+    #     result = level2.update_qc2_status(level2_id = "1060940003452925", data_type='csst-msc-l1-mbi-cat', qc2_status=1)
+    #     print(result)
+    #     self.assertEqual(result.code, 200, "error code: " + str(result.code) + ", message: " + result.message)
+
+    # def test_update_prc_status(self):
+    #     result = level2.update_prc_status(level2_id = "1060940003452925", data_type='csst-msc-l1-mbi-cat', prc_status=3)
+    #     print(result)
+    #     self.assertEqual(result.code, 200, "error code: " + str(result.code) + ", message: " + result.message)
+
+    # def test_update_qc2_status_by_file_name(self):
+    #     result = level2.update_qc2_status_by_file_name(file_name = "CSST_MSC_MS_SCIE_20290713003437_20290713003707_10160000136_24_L1_V01_CAT.fits", qc2_status=1)
+    #     print(result)
+    #     self.assertEqual(result.code, 200, "error code: " + str(result.code) + ", message: " + result.message)
+
+    # def test_update_prc_status_by_file_name(self):
+    #     result = level2.update_prc_status_by_file_name(file_name = "CSST_MSC_MS_SCIE_20290713003437_20290713003707_10160000136_24_L1_V01_CAT.fits", prc_status=3)
+    #     print(result)
+    #     self.assertEqual(result.code, 200, "error code: " + str(result.code) + ", message: " + result.message)
+
+    # def test_write(self):
+    #     file_path = "/Users/wsl/temp/csst/import/CSST_MSC_MS_SCIE_20290713003437_20290713003707_10160000136_24_L1_V01_CAT.fits"
+    #     result = level2.write(local_file = file_path, 
+    #         module_id = "MSC",
+    #         level0_id="1060940003452925",
+    #         level1_id = "1060940003452925", 
+    #         level2_id = "1060940003452925", 
+    #         data_type='csst-msc-l1-mbi-cat', 
+    #         brick_id = 254,
+    #         pipeline_id = "csst-msc-l1-mbi",
+    #         build=1,
+    #         run_id="20241107100248",
+    #         version="093",
+    #         prc_status=1)
+    #     print(result)
+    #     self.assertEqual(result.code, 200, "error code: " + str(result.code) + ", message: " + result.message)
+
+    def test_catalog_query(self):
+        result = level2.catalog_query(sql="select * from tb_level2")
+        print(result)
+        self.assertEqual(result.code, 200, "error code: " + str(result.code) + ", message: " + result.message)