From 0d2a7a782d54dd184e3ba5d54d6404a4e47d6b90 Mon Sep 17 00:00:00 2001 From: shoulinwei Date: Mon, 12 Apr 2021 13:55:15 +0800 Subject: [PATCH] update gaia3 --- README.md | 7 +- csst_dfs_api_cluster/common/catalog.py | 19 +- csst_dfs_api_cluster/common/client_config.py | 17 -- .../common/config/__init__.py | 0 .../common/config/changeset.py | 48 --- .../common/config/configurator.py | 34 --- .../common/config/grpc_loader.py | 35 --- .../common/config/grpc_options.py | 6 - csst_dfs_api_cluster/common/config/model.py | 19 -- csst_dfs_api_cluster/common/config/options.py | 4 - .../common/config/proto/__init__.py | 0 .../common/config/proto/grpc.proto | 31 -- .../common/config/proto/grpc_pb2.py | 289 ------------------ .../common/config/proto/grpc_pb2_grpc.py | 62 ---- csst_dfs_api_cluster/common/service.py | 82 +---- csst_dfs_api_cluster/ifs/fits.py | 3 +- csst_dfs_api_cluster/ifs/reffits.py | 1 - csst_dfs_api_cluster/ifs/result0.py | 1 - csst_dfs_api_cluster/ifs/result1.py | 1 - requirements.txt | 9 +- tests/test_common_catalog.py | 2 +- tests/test_ifs_fits.py | 20 +- 22 files changed, 43 insertions(+), 647 deletions(-) delete mode 100644 csst_dfs_api_cluster/common/client_config.py delete mode 100644 csst_dfs_api_cluster/common/config/__init__.py delete mode 100644 csst_dfs_api_cluster/common/config/changeset.py delete mode 100644 csst_dfs_api_cluster/common/config/configurator.py delete mode 100644 csst_dfs_api_cluster/common/config/grpc_loader.py delete mode 100644 csst_dfs_api_cluster/common/config/grpc_options.py delete mode 100644 csst_dfs_api_cluster/common/config/model.py delete mode 100644 csst_dfs_api_cluster/common/config/options.py delete mode 100644 csst_dfs_api_cluster/common/config/proto/__init__.py delete mode 100644 csst_dfs_api_cluster/common/config/proto/grpc.proto delete mode 100644 csst_dfs_api_cluster/common/config/proto/grpc_pb2.py delete mode 100644 csst_dfs_api_cluster/common/config/proto/grpc_pb2_grpc.py diff --git a/README.md b/README.md index cc32c33..ebb4c14 100644 --- a/README.md +++ b/README.md @@ -9,10 +9,13 @@ This package provides APIs to access csst's files and databases. This library can be installed with the following command: ```bash -pip install git+https://github.com/astronomical-data-processing/csst-dfs-api-cluster.git +git clone https://github.com/astronomical-data-processing/csst-dfs-api-cluster.git +cd csst-dfs-api-cluster +pip install -r requirements.txt +python setup.py install ``` ## Configuration set enviroment variable - CSST_DFS_CONFIG_SERVER = ip:port + CSST_DFS_GATEWAY = ip:port diff --git a/csst_dfs_api_cluster/common/catalog.py b/csst_dfs_api_cluster/common/catalog.py index 307356c..0122aa6 100644 --- a/csst_dfs_api_cluster/common/catalog.py +++ b/csst_dfs_api_cluster/common/catalog.py @@ -1,13 +1,12 @@ -from ..common.client_config import ClientConfigurator -from ..common.service import ServiceProxy - +import grpc +from csst_dfs_commons.models import Result from csst_dfs_proto.common.ephem import ephem_pb2, ephem_pb2_grpc +from ..common.service import ServiceProxy from ..common.constants import * -import grpc class CatalogApi(object): def __init__(self): - self.proxy = ServiceProxy(ClientConfigurator().gatewayCfg) + self.proxy = ServiceProxy() self.stub = self.proxy.insecure(ephem_pb2_grpc.EphemSearchSrvStub) def gaia3_query(self, ra: float, dec: float, radius: float, min_mag: float, max_mag: float, obstime: int, limit: int): @@ -32,8 +31,12 @@ class CatalogApi(object): obstime = obstime, limit = limit )) - return resp - except grpc.RpcError as identifier: - raise Exception("Rpc Error") + if resp.success: + return Result.ok_data(data=resp.records).append("totalCount", resp.totalCount) + else: + return Result.error(message = resp.message) + + except grpc.RpcError as e: + return Result.error(message="%s:%s" % (e.code().value, e.details)) diff --git a/csst_dfs_api_cluster/common/client_config.py b/csst_dfs_api_cluster/common/client_config.py deleted file mode 100644 index 0385426..0000000 --- a/csst_dfs_api_cluster/common/client_config.py +++ /dev/null @@ -1,17 +0,0 @@ -import os - -from .config.grpc_loader import GrpcLoader -from .config.configurator import Configurator -from .utils import singleton - -def configOptions(opts): - opts.address = os.getenv('CSST_DFS_CONFIG_SERVER', "localhost:9600") - -@singleton -class ClientConfigurator(object): - def __init__(self) -> None: - self.configurator = Configurator(GrpcLoader(configOptions)) - self.globalCfg = self.configurator.Global() - self.gatewayCfg = self.configurator.Gateway() - -globalConfig = ClientConfigurator() diff --git a/csst_dfs_api_cluster/common/config/__init__.py b/csst_dfs_api_cluster/common/config/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/csst_dfs_api_cluster/common/config/changeset.py b/csst_dfs_api_cluster/common/config/changeset.py deleted file mode 100644 index fb6335e..0000000 --- a/csst_dfs_api_cluster/common/config/changeset.py +++ /dev/null @@ -1,48 +0,0 @@ -import time - -class ChangeSet(object): - def __init__(self): - self._data = bytes([]) - self._checksum = "" - self._format = "" - self._source = "" - self._timestamp = time.time() - - @property - def data(self): - return self._data - - @data.setter - def data(self, value): - self._data = value - - @property - def checksum(self): - return self._checksum - - @checksum.setter - def checksum(self, value): - self._checksum = value - - @property - def format(self): - return self._format - - @format.setter - def format(self, value): - self._format = value - - @property - def source(self): - return self._source - @source.setter - def source(self, value): - self._source = value - - @property - def timestamp(self): - return self._timestamp - - @timestamp.setter - def timestamp(self, value): - self._timestamp = value diff --git a/csst_dfs_api_cluster/common/config/configurator.py b/csst_dfs_api_cluster/common/config/configurator.py deleted file mode 100644 index c5cff1c..0000000 --- a/csst_dfs_api_cluster/common/config/configurator.py +++ /dev/null @@ -1,34 +0,0 @@ -import orjson -from .model import Server as ServerCfg, Global as GlobalCfg, Gateway as GatewayCfg - -class Configurator(object): - def __init__(self, loader): - self.loader = loader - change_set = self.loader.Read() - self.config = orjson.loads(change_set.data) - - def Server(self, srv_name): - return self.Any(srv_name, ServerCfg) - - def Gateway(self, path = "gateway"): - return self.Any(path, GatewayCfg) - - def Global(self, path = "global"): - return self.Any(path, GlobalCfg) - - def Any(self, path, ref_object): - paths = path.split(".") - cfg = self.config - for p in paths: - cfg = cfg[p] - if isinstance(cfg, dict): - return ref_object(**cfg) - else: - return ref_object(cfg) - - - - - - - \ No newline at end of file diff --git a/csst_dfs_api_cluster/common/config/grpc_loader.py b/csst_dfs_api_cluster/common/config/grpc_loader.py deleted file mode 100644 index f7246f5..0000000 --- a/csst_dfs_api_cluster/common/config/grpc_loader.py +++ /dev/null @@ -1,35 +0,0 @@ - -import grpc - -from .grpc_options import GrpcOptions -from pymicro.config.grpc.proto import grpc_pb2, grpc_pb2_grpc -from .changeset import ChangeSet - -class GrpcLoader(object): - - def __init__(self, *args, **kwargs): - self.options = GrpcOptions() - for o in args: - o(self.options) - - self._connect() - - def Read(self): - request = grpc_pb2.ReadRequest(path = self.options.path) - response = self.stub.Read(request) - return self._to_changset(response.change_set) - - def _connect(self): - self.channel = grpc.insecure_channel(self.options.address) - self.stub = grpc_pb2_grpc.SourceStub(self.channel) - - def _to_changset(self, cs): - changeset = ChangeSet() - changeset.data = cs.data - changeset.checksum = cs.checksum - changeset.source = cs.source - changeset.format = cs.format - changeset.timestamp = cs.timestamp - return changeset - - diff --git a/csst_dfs_api_cluster/common/config/grpc_options.py b/csst_dfs_api_cluster/common/config/grpc_options.py deleted file mode 100644 index a386dc4..0000000 --- a/csst_dfs_api_cluster/common/config/grpc_options.py +++ /dev/null @@ -1,6 +0,0 @@ -from pymicro.config.options import Options - -class GrpcOptions(Options): - def __init__(self): - self.address = "localhost:9600" - self.path = "micro" \ No newline at end of file diff --git a/csst_dfs_api_cluster/common/config/model.py b/csst_dfs_api_cluster/common/config/model.py deleted file mode 100644 index 73c48ec..0000000 --- a/csst_dfs_api_cluster/common/config/model.py +++ /dev/null @@ -1,19 +0,0 @@ -import dataclasses - -@dataclasses.dataclass -class Server: - name: str="" - version: str="" - address: str="" - port: int=-1 - -@dataclasses.dataclass -class Global: - fitsFileRootDir: str="" - fileExternalPrefix: str="" - -@dataclasses.dataclass -class Gateway: - enabled: bool=True - url: str="" - diff --git a/csst_dfs_api_cluster/common/config/options.py b/csst_dfs_api_cluster/common/config/options.py deleted file mode 100644 index 14919f6..0000000 --- a/csst_dfs_api_cluster/common/config/options.py +++ /dev/null @@ -1,4 +0,0 @@ - -class Options(object): - def __init__(self): - pass \ No newline at end of file diff --git a/csst_dfs_api_cluster/common/config/proto/__init__.py b/csst_dfs_api_cluster/common/config/proto/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/csst_dfs_api_cluster/common/config/proto/grpc.proto b/csst_dfs_api_cluster/common/config/proto/grpc.proto deleted file mode 100644 index 95217f3..0000000 --- a/csst_dfs_api_cluster/common/config/proto/grpc.proto +++ /dev/null @@ -1,31 +0,0 @@ -syntax = "proto3"; - -service Source { - rpc Read(ReadRequest) returns (ReadResponse) {}; - rpc Watch(WatchRequest) returns (stream WatchResponse) {}; -} - -message ChangeSet { - bytes data = 1; - string checksum = 2; - string format = 3; - string source = 4; - int64 timestamp = 5; -} - -message ReadRequest { - string path = 1; -} - -message ReadResponse { - ChangeSet change_set = 1; -} - -message WatchRequest { - string path = 1; -} - -message WatchResponse { - ChangeSet change_set = 1; -} - diff --git a/csst_dfs_api_cluster/common/config/proto/grpc_pb2.py b/csst_dfs_api_cluster/common/config/proto/grpc_pb2.py deleted file mode 100644 index 14c7c20..0000000 --- a/csst_dfs_api_cluster/common/config/proto/grpc_pb2.py +++ /dev/null @@ -1,289 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by the protocol buffer compiler. DO NOT EDIT! -# source: grpc.proto - -import sys -_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) -from google.protobuf import descriptor as _descriptor -from google.protobuf import message as _message -from google.protobuf import reflection as _reflection -from google.protobuf import symbol_database as _symbol_database -# @@protoc_insertion_point(imports) - -_sym_db = _symbol_database.Default() - - - - -DESCRIPTOR = _descriptor.FileDescriptor( - name='grpc.proto', - package='', - syntax='proto3', - serialized_options=None, - serialized_pb=_b('\n\ngrpc.proto\"^\n\tChangeSet\x12\x0c\n\x04\x64\x61ta\x18\x01 \x01(\x0c\x12\x10\n\x08\x63hecksum\x18\x02 \x01(\t\x12\x0e\n\x06\x66ormat\x18\x03 \x01(\t\x12\x0e\n\x06source\x18\x04 \x01(\t\x12\x11\n\ttimestamp\x18\x05 \x01(\x03\"\x1b\n\x0bReadRequest\x12\x0c\n\x04path\x18\x01 \x01(\t\".\n\x0cReadResponse\x12\x1e\n\nchange_set\x18\x01 \x01(\x0b\x32\n.ChangeSet\"\x1c\n\x0cWatchRequest\x12\x0c\n\x04path\x18\x01 \x01(\t\"/\n\rWatchResponse\x12\x1e\n\nchange_set\x18\x01 \x01(\x0b\x32\n.ChangeSet2[\n\x06Source\x12%\n\x04Read\x12\x0c.ReadRequest\x1a\r.ReadResponse\"\x00\x12*\n\x05Watch\x12\r.WatchRequest\x1a\x0e.WatchResponse\"\x00\x30\x01\x62\x06proto3') -) - - - - -_CHANGESET = _descriptor.Descriptor( - name='ChangeSet', - full_name='ChangeSet', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='data', full_name='ChangeSet.data', index=0, - number=1, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=_b(""), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='checksum', full_name='ChangeSet.checksum', index=1, - number=2, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='format', full_name='ChangeSet.format', index=2, - number=3, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='source', full_name='ChangeSet.source', index=3, - number=4, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='timestamp', full_name='ChangeSet.timestamp', index=4, - number=5, type=3, cpp_type=2, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=14, - serialized_end=108, -) - - -_READREQUEST = _descriptor.Descriptor( - name='ReadRequest', - full_name='ReadRequest', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='path', full_name='ReadRequest.path', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=110, - serialized_end=137, -) - - -_READRESPONSE = _descriptor.Descriptor( - name='ReadResponse', - full_name='ReadResponse', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='change_set', full_name='ReadResponse.change_set', index=0, - number=1, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=139, - serialized_end=185, -) - - -_WATCHREQUEST = _descriptor.Descriptor( - name='WatchRequest', - full_name='WatchRequest', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='path', full_name='WatchRequest.path', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=187, - serialized_end=215, -) - - -_WATCHRESPONSE = _descriptor.Descriptor( - name='WatchResponse', - full_name='WatchResponse', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='change_set', full_name='WatchResponse.change_set', index=0, - number=1, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=217, - serialized_end=264, -) - -_READRESPONSE.fields_by_name['change_set'].message_type = _CHANGESET -_WATCHRESPONSE.fields_by_name['change_set'].message_type = _CHANGESET -DESCRIPTOR.message_types_by_name['ChangeSet'] = _CHANGESET -DESCRIPTOR.message_types_by_name['ReadRequest'] = _READREQUEST -DESCRIPTOR.message_types_by_name['ReadResponse'] = _READRESPONSE -DESCRIPTOR.message_types_by_name['WatchRequest'] = _WATCHREQUEST -DESCRIPTOR.message_types_by_name['WatchResponse'] = _WATCHRESPONSE -_sym_db.RegisterFileDescriptor(DESCRIPTOR) - -ChangeSet = _reflection.GeneratedProtocolMessageType('ChangeSet', (_message.Message,), { - 'DESCRIPTOR' : _CHANGESET, - '__module__' : 'grpc_pb2' - # @@protoc_insertion_point(class_scope:ChangeSet) - }) -_sym_db.RegisterMessage(ChangeSet) - -ReadRequest = _reflection.GeneratedProtocolMessageType('ReadRequest', (_message.Message,), { - 'DESCRIPTOR' : _READREQUEST, - '__module__' : 'grpc_pb2' - # @@protoc_insertion_point(class_scope:ReadRequest) - }) -_sym_db.RegisterMessage(ReadRequest) - -ReadResponse = _reflection.GeneratedProtocolMessageType('ReadResponse', (_message.Message,), { - 'DESCRIPTOR' : _READRESPONSE, - '__module__' : 'grpc_pb2' - # @@protoc_insertion_point(class_scope:ReadResponse) - }) -_sym_db.RegisterMessage(ReadResponse) - -WatchRequest = _reflection.GeneratedProtocolMessageType('WatchRequest', (_message.Message,), { - 'DESCRIPTOR' : _WATCHREQUEST, - '__module__' : 'grpc_pb2' - # @@protoc_insertion_point(class_scope:WatchRequest) - }) -_sym_db.RegisterMessage(WatchRequest) - -WatchResponse = _reflection.GeneratedProtocolMessageType('WatchResponse', (_message.Message,), { - 'DESCRIPTOR' : _WATCHRESPONSE, - '__module__' : 'grpc_pb2' - # @@protoc_insertion_point(class_scope:WatchResponse) - }) -_sym_db.RegisterMessage(WatchResponse) - - - -_SOURCE = _descriptor.ServiceDescriptor( - name='Source', - full_name='Source', - file=DESCRIPTOR, - index=0, - serialized_options=None, - serialized_start=266, - serialized_end=357, - methods=[ - _descriptor.MethodDescriptor( - name='Read', - full_name='Source.Read', - index=0, - containing_service=None, - input_type=_READREQUEST, - output_type=_READRESPONSE, - serialized_options=None, - ), - _descriptor.MethodDescriptor( - name='Watch', - full_name='Source.Watch', - index=1, - containing_service=None, - input_type=_WATCHREQUEST, - output_type=_WATCHRESPONSE, - serialized_options=None, - ), -]) -_sym_db.RegisterServiceDescriptor(_SOURCE) - -DESCRIPTOR.services_by_name['Source'] = _SOURCE - -# @@protoc_insertion_point(module_scope) diff --git a/csst_dfs_api_cluster/common/config/proto/grpc_pb2_grpc.py b/csst_dfs_api_cluster/common/config/proto/grpc_pb2_grpc.py deleted file mode 100644 index e7dddf7..0000000 --- a/csst_dfs_api_cluster/common/config/proto/grpc_pb2_grpc.py +++ /dev/null @@ -1,62 +0,0 @@ -# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! -import grpc - -from . import grpc_pb2 as grpc__pb2 - -class SourceStub(object): - # missing associated documentation comment in .proto file - pass - - def __init__(self, channel): - """Constructor. - - Args: - channel: A grpc.Channel. - """ - self.Read = channel.unary_unary( - '/Source/Read', - request_serializer=grpc__pb2.ReadRequest.SerializeToString, - response_deserializer=grpc__pb2.ReadResponse.FromString, - ) - self.Watch = channel.unary_stream( - '/Source/Watch', - request_serializer=grpc__pb2.WatchRequest.SerializeToString, - response_deserializer=grpc__pb2.WatchResponse.FromString, - ) - - -class SourceServicer(object): - # missing associated documentation comment in .proto file - pass - - def Read(self, request, context): - # missing associated documentation comment in .proto file - pass - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') - - def Watch(self, request, context): - # missing associated documentation comment in .proto file - pass - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') - - -def add_SourceServicer_to_server(servicer, server): - rpc_method_handlers = { - 'Read': grpc.unary_unary_rpc_method_handler( - servicer.Read, - request_deserializer=grpc__pb2.ReadRequest.FromString, - response_serializer=grpc__pb2.ReadResponse.SerializeToString, - ), - 'Watch': grpc.unary_stream_rpc_method_handler( - servicer.Watch, - request_deserializer=grpc__pb2.WatchRequest.FromString, - response_serializer=grpc__pb2.WatchResponse.SerializeToString, - ), - } - generic_handler = grpc.method_handlers_generic_handler( - 'Source', rpc_method_handlers) - server.add_generic_rpc_handlers((generic_handler,)) diff --git a/csst_dfs_api_cluster/common/service.py b/csst_dfs_api_cluster/common/service.py index 0974c2e..99fa945 100644 --- a/csst_dfs_api_cluster/common/service.py +++ b/csst_dfs_api_cluster/common/service.py @@ -1,79 +1,13 @@ +import os import grpc -import random - -class _Service: - def __init__(self, data): - self._data = data - - @property - def name(self): - raise NotImplementedError() - - @property - def version(self): - raise NotImplementedError() - - @property - def metadata(self): - raise NotImplementedError() - - @property - def id(self): - raise NotImplementedError() - - @property - def address(self): - raise NotImplementedError() - - @property - def port(self): - raise NotImplementedError() - - @property - def node_metadata(self): - raise NotImplementedError() - - def __str__(self): - return '{}:{}'.format(self.name, self.address) - -class Service(_Service): - - def __init__(self, data): - self._data = data - - @property - def name(self): - return self._data['name'] - - @property - def version(self): - return self._data['version'] - - @property - def metadata(self): - return self._data['metadata'] - - @property - def _node(self): - return random.choice(self._data['nodes']) - - @property - def id(self): - return self._node['id'] - - @property - def address(self): - return self._node['address'] - - @property - def node_metadata(self): - return self._node['metadata'] - - class ServiceProxy: - def __init__(self, gatewayCfg = None): - self.gatewayCfg = gatewayCfg + def __init__(self): + self.gateway = os.getenv("CSST_DFS_GATEWAY",'172.31.248.218:30880') def insecure(self, stubCls): - channel = grpc.insecure_channel(self.gatewayCfg.url) + options = [('grpc.max_send_message_length', 1000 * 1024 * 1024), + ('grpc.max_receive_message_length', 1000 * 1024 * 1024), + ('grpc.enable_retries', 1), + ('grpc.service_config', '{ "retryPolicy":{ "maxAttempts": 4, "initialBackoff": "0.1s", "maxBackoff": "1s", "backoffMutiplier": 2, "retryableStatusCodes": [ "UNAVAILABLE" ] } }')] + channel = grpc.insecure_channel(self.gateway, options=options) return stubCls(channel) \ No newline at end of file diff --git a/csst_dfs_api_cluster/ifs/fits.py b/csst_dfs_api_cluster/ifs/fits.py index 662d392..4761581 100644 --- a/csst_dfs_api_cluster/ifs/fits.py +++ b/csst_dfs_api_cluster/ifs/fits.py @@ -3,7 +3,6 @@ import grpc from csst_dfs_proto.ifs.fits import fits_pb2, fits_pb2_grpc -from ..common.client_config import ClientConfigurator from ..common.service import ServiceProxy from ..common.utils import * from ..common.constants import UPLOAD_CHUNK_SIZE @@ -11,7 +10,7 @@ from ..common.constants import UPLOAD_CHUNK_SIZE class FitsApi(object): def __init__(self, sub_system = "ifs"): self.sub_system = sub_system - self.proxy = ServiceProxy(ClientConfigurator().gatewayCfg) + self.proxy = ServiceProxy() self.stub = self.proxy.insecure(fits_pb2_grpc.FitsSrvStub) def find(self, **kwargs): diff --git a/csst_dfs_api_cluster/ifs/reffits.py b/csst_dfs_api_cluster/ifs/reffits.py index 07dbe77..1b4c794 100644 --- a/csst_dfs_api_cluster/ifs/reffits.py +++ b/csst_dfs_api_cluster/ifs/reffits.py @@ -2,7 +2,6 @@ import os import grpc -from ..common.client_config import ClientConfigurator from ..common.service import ServiceProxy from ..common.utils import * from ..common.constants import * diff --git a/csst_dfs_api_cluster/ifs/result0.py b/csst_dfs_api_cluster/ifs/result0.py index 0c2b6ab..b79413d 100644 --- a/csst_dfs_api_cluster/ifs/result0.py +++ b/csst_dfs_api_cluster/ifs/result0.py @@ -2,7 +2,6 @@ import os import grpc -from ..common.client_config import ClientConfigurator from ..common.service import ServiceProxy from ..common.utils import * from ..common.constants import * diff --git a/csst_dfs_api_cluster/ifs/result1.py b/csst_dfs_api_cluster/ifs/result1.py index c80486b..b004608 100644 --- a/csst_dfs_api_cluster/ifs/result1.py +++ b/csst_dfs_api_cluster/ifs/result1.py @@ -2,7 +2,6 @@ import os import grpc -from ..common.client_config import ClientConfigurator from ..common.service import ServiceProxy from ..common.utils import * from ..common.constants import * diff --git a/requirements.txt b/requirements.txt index 694c8ea..8a217cf 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ -astropy -grpc -orjson -git+https://github.com/astronomical-data-processing/csst-dfs-proto-py.git \ No newline at end of file +astropy>=4.0 +grpcio>=1.28.1 +orjson>=3.0.2 +git+https://github.com/astronomical-data-processing/csst-dfs-proto-py.git +git+https://github.com/astronomical-data-processing/csst-dfs-commons.git \ No newline at end of file diff --git a/tests/test_common_catalog.py b/tests/test_common_catalog.py index 93f62c0..b18bc93 100644 --- a/tests/test_common_catalog.py +++ b/tests/test_common_catalog.py @@ -10,5 +10,5 @@ class CommonEphemTestCase(unittest.TestCase): self.api = CatalogApi() def test_gaia3_query(self): - recs = self.api.gaia3_query(ra=56.234039029108935, dec=14.466534827703473, radius=4, min_mag=-1, max_mag=-1, obstime = -1, limit = 2) + recs = self.api.gaia3_query(ra=56.234039029108935, dec=14.466534827703473, radius=0.1, min_mag=4, max_mag=12, obstime = -1, limit = 2) print('find:', recs) diff --git a/tests/test_ifs_fits.py b/tests/test_ifs_fits.py index fa60db8..41a0276 100644 --- a/tests/test_ifs_fits.py +++ b/tests/test_ifs_fits.py @@ -9,14 +9,18 @@ class IFSFitsTestCase(unittest.TestCase): def setUp(self): self.api = FitsApi() - def test_find(self): - recs = self.api.find(file_name='CCD1_ObsTime_300_ObsNum_7.fits') - print('find:', recs) - assert len(recs) == 1 - - recs = self.api.find() - print('find:', recs) - assert len(recs) > 1 + # def test_find(self): + # recs = self.api.find(file_name='CCD1_ObsTime_300_ObsNum_7.fits') + # print('find:', recs) + # assert len(recs) == 1 + + # recs = self.api.find() + # print('find:', recs) + # assert len(recs) > 1 + + def test_get(self): + rec = self.api.get(fits_id=0) + print('get:', rec) # def test_read(self): # recs = self.api.find(file_name='CCD1_ObsTime_300_ObsNum_7.fits') -- GitLab