Initial Commit

.gitignore

+# Python temp files
+**/__pycache__/
+**/*.py~
+**/*.pyc
+
+# Python virtualenv folder
+**/venv/
+
+# IDE Project files
+**/.idea/
+**/.vscode/
+**/*.swp
+
+# MacOS folder attributes files
+**/*.DS_Store
+
+# Files created during testing
+/_build/
+test_reports.xml
+
+**/.cache
+**/.pytest_cache/
+
+# Compiled python modules.
+**/.pyc
+
+# Setuptools distribution folder.
+**/build/
+**/dist/
+
+# Code coverage
+**/.coverage
+
+# log
+*.log
+# Python egg metadata, regenerated from source files by setuptools.
+**/*.egg-info
+**/*.rdb

MANIFEST.in

+include csst/dfs/api/local/common/db.sql
\ No newline at end of file

README.md

+# CSST local APIs library
+
+## Introduction
+
+This package provides APIs to access csst's files and databases in your localized environment.
+
+## Installation
+
+This library can be installed with the following command: 
+
+```bash
+python setup.py install
+```
+
+## Configuration
+set enviroment variable
+    CSST_LOCAL_FILE_ROOT = [a local file directory] #default: /opt/temp/csst

csst/dfs/api/local/__init__.py

+__version_info__ = (1, 0, 0)
+__version__ = '.'.join(map(str, __version_info__))
\ No newline at end of file

csst/dfs/api/local/common/db.py

+import os
+import datetime
+import sqlite3
+from DBUtils.PersistentDB import PersistentDB
+from .utils import singleton
+
+import logging
+
+log = logging.getLogger('csst')
+
+@singleton
+class DBClient(object):
+    def __init__(self):
+        db_path = os.path.join(os.getenv("CSST_LOCAL_FILE_ROOT", "/opt/temp/csst"), "csst.sqlite")
+
+        self.inited = os.path.exists(db_path)
+        self.pool = PersistentDB(sqlite3, maxusage=2, database=db_path)
+        
+        log.info("Creating connection pool with host = [%s]", db_path)
+
+        self._conn = None
+        self._cursor = None
+        self.__get_conn()
+
+        if not self.inited:
+            self.__init_db()
+    
+    def __del__(self):
+        self.close()
+
+    def __get_conn(self):
+        self._conn = self.pool.connection()
+        self._cursor = self._conn.cursor()
+        
+    def __execute(self, sql, param=()):
+        count = self._cursor.execute(sql, param)
+        return count
+    
+    def __init_db(self):
+        with open(os.path.join(os.path.abspath(os.path.dirname(__file__)), "db.sql")) as f:
+            self.execute(f.read())
+
+    def select_one(self, sql, param=()):
+        """查询单个结果"""
+        count = self.__execute(sql, param)
+        result = self._cursor.fetchone()
+        """:type result:dict"""
+        result = self.__dict_datetime_obj_to_str(result)
+        return count, result
+
+    def select_many(self, sql, param=()):
+        """
+        查询多个结果
+        :param sql: qsl语句
+        :param param: sql参数
+        :return: 结果数量和查询结果集
+        """
+        count = self.__execute(sql, param)
+        result = self._cursor.fetchall()
+        """:type result:list"""
+        [self.__dict_datetime_obj_to_str(row_dict) for row_dict in result]
+        return count, result
+
+    def execute(self, sql, param=()):
+        count = self.__execute(sql, param)
+        return count
+
+    def begin(self):
+        """开启事务"""
+        self._conn.autocommit(0)
+
+    def end(self, option='commit'):
+        """结束事务"""
+        if option == 'commit':
+            self._conn.autocommit()
+        else:
+            self._conn.rollback()    
+    
+    def __dict_datetime_obj_to_str(self, result_dict):
+        """把字典里面的datatime对象转成字符串，使json转换不出错"""
+        if result_dict and isinstance(result_dict, tuple):
+            result_replace = [v.__str__() for  v in result_dict if isinstance(v, datetime.datetime)]
+            return result_replace
+
+        if result_dict and isinstance(result_dict, dict):
+            result_replace = {k: v.__str__() for k, v in result_dict.items() if isinstance(v, datetime.datetime)}
+            result_dict.update(result_replace)            
+        return result_dict
+
+    def close(self):
+        try:
+            self._cursor.close()
+            self._conn.close()
+        except Exception as e:
+            log.error(e)

csst/dfs/api/local/common/db.sql

+create table if not exists "t_rawfits" (
+	"id"	TEXT,
+	"filename"	TEXT,
+	"obs_time"	NUMERIC,
+	"path"	TEXT,
+	PRIMARY KEY("id")
+);

csst/dfs/api/local/common/logging.py

+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Copyright (c) 2020 Shoulin Wei
+#
+# This file is part of CSST.
+
+# coding: utf-8
+import logging
+import logging.handlers
+
+def setup_logging():
+    """ Setup logging configuration """
+
+    # Console formatter, mention name
+    cfmt = logging.Formatter(('%(name)s - %(levelname)s - %(message)s'))
+
+    # File formatter, mention time
+    ffmt = logging.Formatter(('%(asctime)s - %(levelname)s - %(message)s'))
+
+    # Console handler
+    ch = logging.StreamHandler()
+    ch.setLevel(logging.INFO)
+    ch.setFormatter(cfmt)
+
+    # File handler
+    fh = logging.handlers.RotatingFileHandler('csst.log',
+        maxBytes=10*1024*1024, backupCount=10)
+    fh.setLevel(logging.INFO)
+    fh.setFormatter(ffmt)
+
+    # Create the logger,
+    # adding the console and file handler
+    csst_logger = logging.getLogger('csst')
+    csst_logger.handlers = []
+    csst_logger.setLevel(logging.DEBUG)
+    csst_logger.addHandler(ch)
+    csst_logger.addHandler(fh)
+
+    # Set up the concurrent.futures logger
+    cf_logger = logging.getLogger('concurrent.futures')
+    cf_logger.setLevel(logging.DEBUG)
+    cf_logger.addHandler(ch)
+    cf_logger.addHandler(fh)
+
+    return csst_logger
+
+def setup_test_logging():
+    # Console formatter, mention name
+    cfmt = logging.Formatter(('%(name)s - %(levelname)s - %(message)s'))
+
+    # File formatter, mention time
+    ffmt = logging.Formatter(('%(asctime)s - %(levelname)s - %(message)s'))
+
+    # Only warnings and more serious stuff on the console
+    ch = logging.StreamHandler()
+    ch.setLevel(logging.WARN)
+    ch.setFormatter(cfmt)
+
+    # Outputs DEBUG level logging to file
+    fh = logging.FileHandler('csst-test.log')
+    fh.setLevel(logging.DEBUG)
+    fh.setFormatter(ffmt)
+
+    # Set up the montblanc logger
+    csst_logger = logging.getLogger('csst')
+    csst_logger.handlers = []
+    csst_logger.setLevel(logging.DEBUG)
+    csst_logger.addHandler(ch)
+    csst_logger.addHandler(fh)
+
+    # Set up the concurrent.futures logger
+    cf_logger = logging.getLogger('concurrent.futures')
+    cf_logger.setLevel(logging.DEBUG)
+    cf_logger.addHandler(ch)
+    cf_logger.addHandler(fh)
+
+    return csst_logger
\ No newline at end of file

csst/dfs/api/local/common/utils.py

+from datetime import datetime
+import time
+
+def format_datetime(dt):
+    return dt.strftime('%Y-%m-%d %H:%M:%S')
+
+def format_date(dt):
+    return dt.strftime('%Y-%m-%d')
+
+def format_time_ms(float_time):
+    local_time = time.localtime(float_time)
+    data_head = time.strftime("%Y-%m-%d %H:%M:%S", local_time)
+    data_secs = (float_time - int(float_time)) * 1000
+    return "%s.%03d" % (data_head, data_secs)
+
+
+def get_parameter(kwargs, key, default=None):
+    """ Get a specified named value for this (calling) function
+
+    The parameter is searched for in kwargs
+
+    :param kwargs: Parameter dictionary
+    :param key: Key e.g. 'max_workers'
+    :param default: Default value
+    :return: result
+    """
+
+    if kwargs is None:
+        return default
+
+    value = default
+    if key in kwargs.keys():
+        value = kwargs[key]
+    return value
+    
+def to_int(s, default_value = 0):
+    try:
+        return int(s)
+    except:
+        return default_value
+
+
+def singleton(cls):
+    _instance = {}
+
+    def inner():
+        if cls not in _instance:
+            _instance[cls] = cls()
+        return _instance[cls]
+    return inner
\ No newline at end of file

csst/dfs/api/local/ifs/__init__.py

+__version_info__ = (1, 0, 0)
+__version__ = '.'.join(map(str, __version_info__))
\ No newline at end of file

requirements.txt

+DBUtils
\ No newline at end of file

setup.py

+# coding: utf-8
+"""Setup config file to package the configuration database."""
+from os import walk, listdir
+from setuptools import find_packages
+from os.path import join
+from setuptools import setup
+import csst_client
+
+
+def package_files(directory):
+    """Get list of data files to add to the package."""
+    paths = []
+    for (path, _, file_names) in walk(directory):
+        for filename in file_names:
+            paths.append(join('..', path, filename))
+    return paths
+
+
+with open('README.md', 'r') as file:
+    LONG_DESCRIPTION = file.read()
+
+setup(name='csst_dfs_api_local',
+      version=csst_client.__version__,
+      author='CSST DFS Team.',
+      description='CSST DFS Local APIs Library.',
+      long_description=LONG_DESCRIPTION,
+      long_description_content_type='text/markdown',
+      url='',
+      include_package_data=True,
+      packages=['csst/dfs/api/local/ifs','csst/dfs/api/local/common'],
+      install_requires=[
+      ],
+      zip_safe=False,
+      classifiers=[
+          "Programming Language :: Python :: 3 :: Only"
+      ]
+)

tests/test_common_db.py

+import unittest
+
+from csst.dfs.api.local.common.db import DBClient
+
+class DBClientTestCase(unittest.TestCase):
+    def setUp(self):
+        self.db_path = "/opt/temp/csst"
+
+    def test_db_init(self):
+        db = DBClient()
+        db.select_one("select * from t_rawfits")
+