remove redundant files

batch/msc-ooc/trigger-msc-ooc-v20241219-rdx-naoc-v1.py

-import numpy as np
-from csst_dag import CSST_DAG_LIST, Redis, dump_message_list
-from csst_dag.constants import MSC_MBI_CHIPID
-from csst_dfs_client import level0
-
-# set BATCH_ID and DATASET
-BATCH_ID = "msc-ooc-v20241219-rdx-naoc-v1"
-DATASET = "msc-ooc-v20241219"
-LOCATION = "naoc"
-
-# query for dlist in DFS
-result = level0.find(
-    dataset=DATASET,
-    file_type="SCI",
-)
-print(result)
-
-for _ in result.data:
-    print(_.obs_id, _.detector_no)
-
-
-# get DAG
-dag = CSST_DAG_LIST.get("csst-msc-l1-mbi")
-
-# generate messages
-message_list = []
-for _ in result.data:
-    this_obsid = _.obs_id
-    this_chipid = _.detector_no
-    this_message = dag.gen_message(
-        batch_id=BATCH_ID,
-        dataset=DATASET,
-        obsid=this_obsid,
-        chipid=this_chipid,
-    )
-    message_list.append(this_message)
-
-
-# setup redis
-r = Redis(location=LOCATION)
-
-# push messages to redis
-for msg in message_list:
-    r.push(msg)
-
-print(f" N_message = {len(message_list)}")
-print(f" N_in_redis = {len(r.get_all())}")
-
-# ===============================================================
-# dump messages
-dump_message_list(message_list, path=f"batch/msc-v093/msc-v093.{BATCH_ID}")
-
-# re-process?
-# read messages
-with open(f"batch/msc-v093/C9_W1_Phot.{BATCH_ID}.messages", "r") as f:
-    message_list = [_.strip() for _ in f.readlines()]
-for msg in message_list[20:30]:
-    r.push(msg)
-
-print(f" N_message = {len(message_list)}")
-print(f" N_in_redis = {len(r.get_all())}")

batch/msc-v093/C9_W1_Phot.obsid

-10109200100412
-10109200100413
-10109200100414
-10109200107587
-10109200155157
-10109200159781
-10109200204333
-10109200206112
-10109200244032
-10109200244125
-10109200244146
-10109200244172
-10109200244757
-10109200245216
-10109200249013
-10109200250794
-10109200261370
-10109200261371
-10109200261437
-10109200261457
-10109200261477
-10109200261495
-10109200261496
-10109200261514
-10109200261515
-10109200261625
-10109200261626
-10109200261644
-10109200261645
-10109200261663
-10109200261702
-10109200261757
-10109200261800
-10109200261859
-10109200261860
-10109200261938
-10109200261992
-10109200262712
-10109200262991
-10109200263242
-10109200263310
-10109200263688
-10109200264030
-10109200264264
-10109200264310
-10109200264311
-10109200264352
-10109200264394
-10109200264395
-10109200264493
-10109200264516
-10109200264517
-10109200264562
-10109200264563
-10109200264607
-10109200264653
-10109200264671
-10109200264815
-10109200264883
-10109200264906
-10109200265055
-10109200265266
-10109200265289
-10109200265472
-10109200265495
-10109200265540
-10109200265563
-10109200265609
-10109200267605
-10109200267786
-10109200268067
-10109200268103
-10109200268396
-10109200273030
-10109200273318
-10109200273355
-10109200273375
-10109200273417
-10109200273439
-10109200273482
-10109200273549
-10109200273642
-10109200273664
-10109200273683
-10109200273726
-10109200273794
-10109200273815
-10109200273860
-10109200273980
-10109200273999
-10109200274059
-10109200274083
-10109200274104
-10109200274149
-10109200274173
-10109200274194
-10109200274294
-10109200274315
-10109200274370
-10109200274391
-10109200274414
-10109200274782
-10109200274828
-10109200274943
-10109200274982
-10109200275023
-10109200275112
-10109200275274
-10109200275646
-10109200277563
-10109200277608
-10109200305571
-10109200305596
-10109200305646
-10109200305800
-10109200311461
-10109200322269
-10109200341927
-10109200342216
-10109200343719
-10109200349710
-10109200349732
-10109200349871
-10109200350531
-10109200350532
-10109200351315
-10109200351452
-10109200351515
-10109200352179
-10109200352619
-10109200352620
-10109200352641
-10109200352664
-10109200352754
-10109200352777
-10109200352800
-10109200352820
-10109200352962
-10109200353051
-10109200353074
-10109200353097
-10109200353138
-10109200353388
-10109200353411
-10109200353434
-10109200353475
-10109200353515
-10109200353620
-10109200353665
-10109200353704
-10109200353770
-10109200353993
-10109200354446
-10109200354467
-10109200355570
-10109200355881
-10109200355936
-10109200356137
-10109200356294
-10109200356509
-10109200356749
-10109200356796
-10109200360964
-10109200360985
-10109200361007
-10109200361072
-10109200361170
-10109200361246
-10109200361314
-10109200361336
-10109200361381
-10109200361624
-10109200361814
-10109200361853
-10109200361873
-10109200362000
-10109200362021
-10109200362045
-10109200362164
-10109200362203
-10109200362291
-10109200362313
-10109200363418
-10109200363514
-10109200363554
-10109200363577
-10109200363599
-10109200363942

batch/msc-v093/msc-v093-all.obsid

-10109200100412
-10109200100413
-10109200100414
-10109200107587
-10109200155157
-10109200159781
-10109200204333
-10109200206112
-10109200244032
-10109200244125
-10109200244146
-10109200244172
-10109200244757
-10109200245216
-10109200249013
-10109200250794
-10109200261370
-10109200261371
-10109200261437
-10109200261457
-10109200261477
-10109200261495
-10109200261496
-10109200261514
-10109200261515
-10109200261625
-10109200261626
-10109200261644
-10109200261645
-10109200261663
-10109200261702
-10109200261757
-10109200261800
-10109200261859
-10109200261860
-10109200261938
-10109200261992
-10109200262712
-10109200262991
-10109200263242
-10109200263310
-10109200263688
-10109200264030
-10109200264264
-10109200264310
-10109200264311
-10109200264352
-10109200264394
-10109200264395
-10109200264493
-10109200264516
-10109200264517
-10109200264562
-10109200264563
-10109200264607
-10109200264653
-10109200264671
-10109200264815
-10109200264883
-10109200264906
-10109200265055
-10109200265266
-10109200265289
-10109200265472
-10109200265495
-10109200265540
-10109200265563
-10109200265609
-10109200267605
-10109200267786
-10109200268067
-10109200268103
-10109200268396
-10109200273030
-10109200273318
-10109200273355
-10109200273375
-10109200273417
-10109200273439
-10109200273482
-10109200273549
-10109200273642
-10109200273664
-10109200273683
-10109200273726
-10109200273794
-10109200273815
-10109200273860
-10109200273980
-10109200273999
-10109200274059
-10109200274083
-10109200274104
-10109200274149
-10109200274173
-10109200274194
-10109200274294
-10109200274315
-10109200274370
-10109200274391
-10109200274414
-10109200274782
-10109200274828
-10109200274943
-10109200274982
-10109200275023
-10109200275112
-10109200275274
-10109200275646
-10109200277563
-10109200277608
-10109200305571
-10109200305596
-10109200305646
-10109200305800
-10109200311461
-10109200322269
-10109200341927
-10109200342216
-10109200343719
-10109200349710
-10109200349732
-10109200349871
-10109200350531
-10109200350532
-10109200351315
-10109200351452
-10109200351515
-10109200352179
-10109200352619
-10109200352620
-10109200352641
-10109200352664
-10109200352754
-10109200352777
-10109200352800
-10109200352820
-10109200352962
-10109200353051
-10109200353074
-10109200353097
-10109200353138
-10109200353388
-10109200353411
-10109200353434
-10109200353475
-10109200353515
-10109200353620
-10109200353665
-10109200353704
-10109200353770
-10109200353993
-10109200354446
-10109200354467
-10109200355570
-10109200355881
-10109200355936
-10109200356137
-10109200356294
-10109200356509
-10109200356749
-10109200356796
-10109200360964
-10109200360985
-10109200361007
-10109200361072
-10109200361170
-10109200361246
-10109200361314
-10109200361336
-10109200361381
-10109200361624
-10109200361814
-10109200361853
-10109200361873
-10109200362000
-10109200362021
-10109200362045
-10109200362164
-10109200362203
-10109200362291
-10109200362313
-10109200363418
-10109200363514
-10109200363554
-10109200363577
-10109200363599
-10109200363942
-10109200364126
-10109200364147
-10109200364256
-10109200393160
-10109200393430
-10109200393567
-10109200395378
-10109200402715
-10109200408285
-10109200408305
-10109200424926
-10109200424983
-10109200426603
-10109200451331
-10109200451577
-10109200452508
-10109200455242
-10109200455359
-10109200481067
-10109200481341
-10109200481626
-10109200481645
-10109200481903
-10109200481922
-10109200482064
-10109200482251
-10109200482491
-10109200482508
-10109200483570
-10109200483864
-10109200487291
-10109200487371
-10109200489993
-10109200489994
-10109200490202
-10109200490994
-10109200491097
-10109200491329
-10109200074165
-10109200086246
-10109200107803
-10109200108384
-10109200108576
-10109200108743
-10109200109419
-10109200109542
-10109200109563
-10109200109878
-10109200110537
-10109200119281
-10109200119322
-10109200119536
-10109200124015
-10109200125784
-10109200125806
-10109200125827
-10109200141330
-10109200142315
-10109200151017
-10109200152968
-10109200153361
-10109200154602
-10109200154623
-10109200154764
-10109200154805
-10109200155055
-10109200155945
-10109200155986
-10109200156065
-10109200157759
-10109200157760
-10109200157761
-10109200157762
-10109200157782
-10109200157890
-10109200157929
-10109200157930
-10109200157970
-10109200157971
-10109200157990
-10109200157991
-10109200158010
-10109200158011
-10109200158126
-10109200158275
-10109200158295
-10109200158315
-10109200165443
-10109200165444
-10109200165445
-10109200165446
-10109200165469
-10109200165528
-10109200165529
-10109200165530
-10109200165531
-10109200165669
-10109200167523
-10109200170181
-10109200170493
-10109200172250
-10109200172251
-10109200172296
-10109200172297
-10109200172298
-10109200198030
-10109200198330
-10109200200104
-10109200200185
-10109200200409
-10109200200428
-10109200200447
-10109200200594
-10109200201312
-10109200201355
-10109200208295
-10109200213871
-10109200216117
-10109200216309
-10109200216344
-10109200232320
-10109200232366
-10109200232387
-10109200232388
-10109200232407
-10109200238075
-10109200240904
-10109200241428
-10109200241971
-10109200245124
-10109200245145
-10109200245362
-10109200245384
-10109200245406
-10109200245641
-10109200245660
-10109200245681
-10109200245720
-10109200245721
-10109200245741
-10109200245807
-10109200245922
-10109200245943
-10109200245988
-10109200247776
-10109200247777
-10109200247796
-10109200254793
-10109200255125
-10109200256185
-10109200256186
-10109200256512
-10109200262831
-10109200263268
-10109200263336
-10109200263358
-10109200263694
-10109200263733
-10109200263770
-10109200264016
-10109200264094
-10609200000493
-10609200000494
-10609200000495
-10609200000496
-10609200000510
-10609200000511
-10609200000512
-10609200000513
-10609200000514
-10609200000571
-10609200000572
-10609200000573
-10609200000574
-10609200000575
-10609200000593
-10609200000594
-10609200000595
-10609200000596
-10609200000597
-10609200000598
-10609200000616
-10609200000617
-10609200000618
-10609200000619
-10609200000620
-10609200000638
-10609200000639
-10609200000640
-10609200000641
-10609200000642
-10609200000659
-10609200000660
-10609200000661
-10609200000662
-10609200000663
-10609200000664
-10609200000682
-10609200000683
-10609200000684
-10609200000685
-10609200000703
-10609200000704
-10609200000718
-10609200000736
-10609200003869
-10609200004548
-10609200034529
-10609200035818
-10609200035899
-10609200036340
-10609200036407
-10609200036429
-10609200036451
-10609200036473
-10609200036495
-10609200036535
-10609200037202
-10609200037374
-10609200037468
-10609200038744
-10609200038810
-10609200038908
-10609200039133
-10609200039274
-10609200039365
-10609200040128
-10609200040129
-10609200040130
-10609200040131
-10609200040132
-10609200040133
-10609200040134
-10609200040760
-10609200041415
-10609200041493
-10609200043196
-10609200043197
-10609200043198
-10609200043218
-10609200043783
-10609200048306
-10609200048307
-10609200048913
-10609200048932
-10609200049027
-10609200049065
-10609200049172
-10609200051022
-10609200051141
-10609200051756
-10609200057589
-10609200057739
-10609200059825
-10609200059826
-10609200059827
-10609200059828
-10609200064450
-10609200064451
-10609200064452
-10609200064603
-10609200064604
-10609200064680
-10609200064702
-10609200065832
-10609200065896
-10109200070276
-10109200070335
-10109200070445
-10109200098076
-10109200098926
-10109200099076
-10109200111264
-10109200111332
-10109200142766
-10109200142855
-10109200143107
-10109200143174
-10109200143214
-10109200192150
-10109200192389
-10109200233356
-10109200233376
-10109200233496
-10109200233733
-10109200233753
-10109200233832
-10109200343027
-10109200354245
-10109200384472
-10109200384905
-10109200398711
-10109200443314
-10109200451794
-10109200453877
-10109200455442
-10109200455506
-10109200455978
-10109200456149
-10109200456163
-10109200456179
-10109200456211
-10109200456226
-10109200456255
-10109200456270
-10109200456287
-10109200456303
-10109200490320
-10109200493568
-10109200494179
-10109200494329
-10109200494381
-10109200520132
-10109200521119
-10109200521467
-10109200521777
-10109200522431
-10109200526843
-10109200531792
-10109200531811
-10109200531902
-10109200531920
-10109200531938
-10109200532000
-10109200532013
-10109200532026
-10109200532040
-10109200532068
-10109200532106
-10109200532125
-10109200532143
-10109200532201
-10109200532218
-10109200532219
-10109200532253
-10109200532254
-10109200532287
-10109200532288
-10109200532306
-10109200532670
-10109200532671
-10109200532753
-10109200532754
-10109200532755
-10109200532756
-10109200559315
-10109200559715
-10109200569963
-10109200570053
-10109200570465
-10109200570480
-10109200570495
-10109200570985
-10109200571000
-10109200571015
-10109200571757
-10109200571826
-10109200571842
-10109200571893
-10109200571966
-10109200572162
-10109200572196
-10109200576766
-10109200578874
-10109200578875
-10109200578938
-10109200578939
-10109200579340
-10109200579341
-10109200579342
-10109200579414
-10109200579477
-10109200579478
-10109200579507
-10109200579508
-10109200579509
-10109200579510
-10109200579527
-10109200579528
-10109200579545
-10109200579546
-10109200579547
-10109200579620
-10109200579621
-10109200579622
-10109200579623
-10109200579642
-10109200579658
-10109200579659
-10109200579660
-10109200579661
-10109200579680
-10109200579694
-10109200579695
-10109200579696
-10109200579765
-10109200579766
-10109200579767
-10109200579768
-10109200579935
-10109200579936
-10109200579937
-10109200580197
-10109200580198
-10109200580199
-10109200580248
-10109200580249
-10109200580296
-10109200580297
-10109200580298
-10109200580299
-10109200580333
-10109200580334
-10109200580335
-10109200580354
-10109200580418
-10109200580419
-10109200580420
-10109200580434
-10109200580435
-10109200580436
-10109200580437
-10109200580451
-10109200580452
-10109200580453
-10109200580489
-10109200580617
-10109200580638
-10109200580639
-10109200580666
-10109200580667
-10109200580681
-10109200580695
-10109200580713
-10109200580714
-10109200580715
-10109200580746
-10109200580747
-10109200580868
-10109200580869
-10109200580889
-10109200580890
-10109200580906
-10109200580919
-10109200580920
-10109200580921
-10109200580935
-10109200580936
-10109200580937
-10109200581055
-10109200581056
-10109200581092
-10109200581093
-10109200581094
-10109200581217
-10109200581218
-10109200581283
-10109200581301
-10109200581302
-10109200581321
-10109200581322
-10109200581397
-10109200581398
-10109200581399
-10109200581466
-10109200581467
-10109200581468
-10109200581482
-10109200581683
-10109200581721
-10109200610170
-10109200610171
-10109200610172
-10109200610173
-10109200610174
-10109200610545
-10109200610686
-10109200610687
-10109200610716
-10109200610717
-10109200610718
-10109200612813
-10109200626444
-10109200626508
-10109200631974
-10109200631991
-10109200637952
-10109200637953
-10109200637954
-10109200638003
-10109200638004
-10109200638059
-10109200638060
-10109200638071
-10109200638072
-10109200638099
-10109200638100
-10109200638127
-10109200638128
-10109200638129
-10109200638138
-10109200638139
-10109200638140
-10109200638169
-10109200638183
-10109200638184
-10109200638185
-10109200638195
-10109200638208
-10109200638223
-10109200638224
-10109200638238
-10109200638239
-10109200638252
-10109200638267
-10109200638281
-10109200638324
-10109200638325
-10109200638434
-10109200638716
-10109200638726
-10109200638759
-10109200638760
-10109200638761
-10109200638770
-10109200638866
-10109200638867
-10109200013310
-10109200275075
-10109200275554
-10109200327076
-10109200327077
-10109200327078
-10109200327079
-10109200327080
-10109200327081
-10109200327114
-10109200327115
-10109200327116
-10109200327174
-10109200327187
-10109200327188
-10109200327189
-10109200327190
-10109200327191
-10109200327192
-10109200327456
-10109200327469
-10109200327470
-10109200327471
-10109200327472
-10109200327514
-10109200327534
-10109200327700
-10109200327701
-10109200328029
-10109200328052
-10109200334752
-10109200334753
-10109200334769
-10109200334869
-10109200334888
-10109200334947
-10109200335832
-10109200335969
-10109200335988
-10109200336064
-10109200336110
-10109200336127
-10109200336252
-10109200363095
-10109200364369
-10109200364513
-10109200376024
-10109200376907
-10109200376908
-10109200404311
-10109200405982
-10109200413340
-10109200413341
-10109200413815
-10109200420649
-10109200420674
-10109200420686
-10109200426961
-10109200460725
-10109200461409
-10109200461690
-10109200461742
-10109200462210
-10109200475885
-10109200475921
-10109200476385
-10109200476403
-10109200476421
-10109200500512
-10109200507031
-10109200507067
-10109200507240
-10109200507345
-10109200535566
-10109200535747
-10109200536831
-10109200537496
-10109200537533
-10109200537621
-10109200537695
-10109200537732
-10109200537749
-10109200537765
-10109200537946
-10109200538522
-10109200538555
-10109200539011
-10109200544511
-10109200548955
-10109200549047
-10109200549759
-10109200549966
-10109200550487
-10109200550544
-10109200550618
-10109200550723
-10109200550887
-10109200550924
-10109200550940
-10109200551080
-10109200551470
-10109200551502
-10109200557663
-10109200557724
-10109200557778
-10109200571988
-10109200573346
-10109200581083
-10109200582889
-10109200583318
-10109200583483
-10109200583517
-10109200583534
-10109200583584
-10109200583717
-10109200583735
-10109200583768
-10109200583822
-10109200583839
-10109200583918
-10109200583933
-10109200583979
-10109200584015
-10109200584051
-10109200584072
-10109200584107
-10109200584609
-10109200584647
-10109200584674
-10109200584742
-10109200584815
-10109200584933
-10109200584934
-10109200585018
-10109200585032
-10109200585069
-10109200585070
-10109200585078
-10109200585087
-10109200585097
-10109200585106
-10109200585115
-10109200585124
-10109200585133
-10109200585134
-10109200585143
-10109200585152
-10109200585153
-10109200585175
-10109200585188
-10109200585317
-10109200585319
-10109200585343
-10109200585588
-10109200585645
-10109200585728
-10109200585759
-10109200585787
-10109200585992
-10109200586011
-10109200586016
-10109200586030
-10109200586090
-10109200586109
-10109200586128
-10109200586325
-10109200586347
-10109200586400
-10109200586435
-10109200586831
-10109200586929
-10109200587033
-10109200587105
-10109200587122
-10109200587354
-10109200588997
-10109200592182
-10109200592281
-10109200592407
-10109200592500
-10109200592513
-10109200592527
-10109200592540
-10109200592552
-10109200592570
-10109200592607
-10109200592622
-10109200592635
-10109200593560
-10109200593745
-10109200593758
-10109200593952
-10109200595824
-10109200595843
-10109200595870
-10109200596080
-10109200596116
-10109200596152
-10109200596171
-10109200596190
-10109200596227
-10109200596247
-10109200596266
-10109200596284
-10109200596298
-10109200596314
-10109200596315
-10109200596347
-10109200596381
-10109200596436
-10109200596492
-10109200596567
-10109200596600
-10109200596711
-10109200596729
-10109200596748
-10109200596786
-10109200596804
-10109200596823
-10109200596876
-10109200596999
-10109200597422
-10109200597456
-10109200597626
-10109200597628
-10109200597661
-10109200597677
-10109200597727
-10109200597838
-10109200597856
-10109200597875
-10109200597913
-10109200597930
-10109200597996
-10109200598084
-10109200598087
-10109200598103
-10109200598104
-10109200598106
-10109200598143
-10109200598160
-10109200598161
-10109200598163
-10109200598179
-10109200598180
-10109200598182
-10109200598197
-10109200598200
-10109200598213
-10109200598216
-10109200598229
-10109200598232
-10109200598249
-10109200598263
-10109200598264
-10109200598266
-10109200598506
-10109200598509
-10109200598523
-10109200598524
-10109200598541
-10109200598542
-10109200598594
-10109200598631
-10109200598650
-10109200598688
-10109200058504
-10109200059069
-10109200059495
-10109200059918
-10109200131914
-10109200132508
-10109200132550
-10109200138604
-10109200146228
-10109200146895
-10109200179479
-10109200180617
-10109200193552
-10109200194482
-10109200222785
-10109200222943
-10109200223232
-10109200224411
-10109200224533
-10109200225068
-10109200225146
-10109200225164
-10109200241883
-10109200285339
-10109200285453
-10109200285711
-10109200285839
-10109200286519
-10109200286725
-10109200286822
-10109200286979
-10109200290571
-10109200290665
-10109200304970
-10109200328199
-10109200344799
-10109200344820
-10109200344821
-10109200344886
-10109200345765
-10109200345787
-10109200374240
-10109200392386
-10109200392430
-10109200417565
-10109200417587
-10109200421566
-10109200421621
-10109200421641
-10109200421677
-10109200421711
-10109200421962
-10109200423696
-10109200423717
-10109200423781
-10109200423803
-10109200423825
-10109200423939
-10109200423958
-10109200432953
-10109200441454
-10109200441455
-10109200441456
-10109200445014
-10109200449386
-10109200449388
-10109200449390
-10109200449423
-10109200449425
-10109200449437
-10109200449455
-10109200449457
-10109200449459
-10109200449470
-10109200449474
-10109200449490
-10109200449822
-10109200449920
-10109200450058
-10109200471410
-10109200471552
-10109200471568
-10109200471632
-10109200471646
-10109200471751
-10109200472163
-10109200472718
-10109200477254
-10109200477269
-10109200477286
-10109200477350
-10109200477383
-10109200477466
-10109200477657
-10109200480308
-10109200480329
-10109200480350
-10109200480644
-10109200484573
-10109200484574
-10109200484696
-10109200484774
-10109200484775
-10109200484832
-10109200484833
-10109200484850
-10109200484851
-10109200484990
-10109200484991
-10109200484992
-10109200485262
-10109200485807
-10109200509536
-10109200509537
-10109200509560
-10109200509561
-10109200509599
-10109200509615
-10109200509817
-10109200509897
-10109200509932
-10109200525516
-10109200525545
-10109200525601
-10109200525602
-10109200525613
-10109200525614
-10109200525627
-10109200525683
-10109200534269
-10109200534536
-10109200536896
-10109200536914
-10109200537075
-10109200537104
-10109200537159
-10109200537205
-10109200542254
-10109200542270
-10109200542419
-10109200542555
-10109200542652
-10109200542668
-10109200542699
-10109200542734
-10109200542804
-10109200542912
-10109200542928
-10109200542944
-10109200542959
-10109200542976
-10109200543062
-10109200543182
-10109200543198
-10109200543232
-10109200543249
-10109200543266
-10109200543332
-10109200543363
-10109200543422
-10109200543437
-10109200543470
-10109200543486
-10109200543502
-10109200543518
-10109200543550
-10109200543580
-10109200543609
-10109200543655
-10109200543685
-10109200543740
-10109200543757
-10109200543773
-10109200543790
-10109200543807
-10109200543822
-10109200543911
-10109200543926
-10109200544000
-10109200544034
-10109200544051
-10109200544066
-10109200544222
-10109200544240
-10109200544340
-10109200544355
-10109200544530
-10109200544559
-10109200544572
-10109200544584
-10109200544809
-10109200545036
-10109200545284
-10109200546180
-10109200546192
-10109200546204
-10109200546216
-10109200546388
-10109200546400
-10109200546566
-10109200546589
-10109200546601
-10109200546604
-10109200546614
-10109200546647
-10109200546789
-10109200546805
-10109200547045
-10109200547103
-10109200547139
-10109200547339
-10109200547373
-10109200547375
-10109200547392
-10109200547410
-10109200547428
-10109200548469
-10109200548470
-10109200548471
-10109200548535
-10109200548536
-10109200551578
-10109200551610
-10109200551627
-10109200551645
-10109200551663
-10109200551699
-10109200551717
-10109200551735
-10109200551753
-10109200551843
-10109200551859
-10109200551875
-10109200551910
-10109200551928
-10109200551946
-10109200551982
-10109200552230
-10109200552371
-10109200552387
\ No newline at end of file

batch/msc-v093/trigger-msc-v093-rdx-zjlab-v3.py

-import numpy as np
-from csst_dag import CSST_DAG_LIST, Redis, dump_message_list
-from csst_dag.constants import MSC_MBI_CHIPID
-
-# set BATCH_ID and DATASET
-BATCH_ID = "msc-v093-rdx-zjlab-v3"
-DATASET = "msc-v093"
-
-OBSID_LIST_PATH = "batch/msc-v093/C9_W1_Phot.obsid"
-
-# get OBSID
-with open(OBSID_LIST_PATH, "r") as f:
-    OBSID_LIST = [_.strip() for _ in f.readlines()]
-OBSID_LIST_CORRECTED = [_[:3] + "093" + _[6:] for _ in OBSID_LIST]
-
-# get DAG
-dag = CSST_DAG_LIST.get("csst-msc-l1-mbi")
-
-# generate messages
-message_list = []
-for this_obsid in OBSID_LIST_CORRECTED:
-    for this_chipid in MSC_MBI_CHIPID:
-        this_message = dag.gen_message(
-            batch_id=BATCH_ID,
-            dataset=DATASET,
-            obsid=this_obsid,
-            chipid=this_chipid,
-        )
-        message_list.append(this_message)
-
-
-# setup redis
-r = Redis(location="zjlab")
-# push messages to redis
-for msg in message_list:
-    r.push(msg)
-
-print(f" N_message = {len(message_list)}")
-print(f" N_in_redis = {len(r.get_all())}")
-
-# dump messages
-dump_message_list(message_list, path=f"batch/msc-v093/C9_W1_Phot.{BATCH_ID}")
-
-
-# read messages
-with open(f"batch/msc-v093/C9_W1_Phot.{BATCH_ID}.messages", "r") as f:
-    message_list = [_.strip() for _ in f.readlines()]
-for msg in message_list[20:30]:
-    r.push(msg)
-
-print(f" N_message = {len(message_list)}")
-print(f" N_in_redis = {len(r.get_all())}")

batch/msc-v093/trigger-msc-v093-rdx-zjlab-v4.py

-import numpy as np
-from csst_dag import CSST_DAG_LIST, Redis, dump_message_list
-from csst_dag.constants import MSC_MBI_CHIPID
-
-# set BATCH_ID and DATASET
-BATCH_ID = "msc-v093-rdx-zjlab-v4"
-DATASET = "msc-v093-r1"
-
-OBSID_LIST_PATH = "batch/msc-v093/msc-v093-all.obsid"
-
-# get OBSID
-with open(OBSID_LIST_PATH, "r") as f:
-    OBSID_LIST = [_.strip() for _ in f.readlines()]
-OBSID_LIST_CORRECTED = [_[:3] + "093" + _[6:] for _ in OBSID_LIST]
-
-# get DAG
-dag = CSST_DAG_LIST.get("csst-msc-l1-mbi")
-
-# generate messages
-message_list = []
-for this_obsid in OBSID_LIST_CORRECTED:
-    for this_chipid in MSC_MBI_CHIPID:
-        this_message = dag.gen_message(
-            batch_id=BATCH_ID,
-            dataset=DATASET,
-            obsid=this_obsid,
-            chipid=this_chipid,
-        )
-        message_list.append(this_message)
-
-
-# setup redis
-r = Redis(location="zjlab")
-# push messages to redis
-for msg in message_list:
-    r.push(msg)
-
-print(f" N_message = {len(message_list)}")
-print(f" N_in_redis = {len(r.get_all())}")
-
-# dump messages
-dump_message_list(message_list, path=f"batch/msc-v093/msc-v093.{BATCH_ID}")
-
-# re-process?
-# read messages
-with open(f"batch/msc-v093/C9_W1_Phot.{BATCH_ID}.messages", "r") as f:
-    message_list = [_.strip() for _ in f.readlines()]
-for msg in message_list[20:30]:
-    r.push(msg)
-
-print(f" N_message = {len(message_list)}")
-print(f" N_in_redis = {len(r.get_all())}")

batch/msc-v093/trigger-msc-v093-rdx-zjlab-v5.py

-import numpy as np
-from csst_dag import CSST_DAG_LIST, Redis, dump_message_list
-from csst_dag.constants import MSC_MBI_CHIPID
-
-# set BATCH_ID and DATASET
-BATCH_ID = "msc-v093-rdx-zjlab-v5"
-DATASET = "msc-v093-r1"
-
-OBSID_LIST_PATH = "batch/msc-v093/msc-v093-all.obsid"
-
-# get OBSID
-with open(OBSID_LIST_PATH, "r") as f:
-    OBSID_LIST = [_.strip() for _ in f.readlines()]
-OBSID_LIST_CORRECTED = [_[:3] + "093" + _[6:] for _ in OBSID_LIST]
-
-# get DAG
-dag = CSST_DAG_LIST.get("csst-msc-l1-mbi")
-
-# generate messages
-message_list = []
-for this_obsid in OBSID_LIST_CORRECTED:
-    for this_chipid in MSC_MBI_CHIPID:
-        this_message = dag.gen_message(
-            batch_id=BATCH_ID,
-            dataset=DATASET,
-            obsid=this_obsid,
-            chipid=this_chipid,
-        )
-        message_list.append(this_message)
-
-
-# setup redis
-r = Redis(location="zjlab")
-# push messages to redis
-for msg in message_list:
-    r.push(msg)
-
-print(f" N_message = {len(message_list)}")
-print(f" N_in_redis = {len(r.get_all())}")
-
-# dump messages
-dump_message_list(message_list, path=f"batch/msc-v093/msc-v093.{BATCH_ID}")
-
-# re-process?
-# read messages
-with open(f"batch/msc-v093/C9_W1_Phot.{BATCH_ID}.messages", "r") as f:
-    message_list = [_.strip() for _ in f.readlines()]
-for msg in message_list[20:30]:
-    r.push(msg)
-
-print(f" N_message = {len(message_list)}")
-print(f" N_in_redis = {len(r.get_all())}")

batch/msc-v096/obsid.txt

-10109600580920
-10109600192150
-10109600580937
-10109600580919
-10109600571893
-10109600581683
-10109600456149
-10109600626444
-10109600580333
-10109600143214
-10109600638208
-10109600638099
-10109600581092
-10109600456270
-10109600494329
-10109600638281
-10109600579509
-10109600581094
-10109600532756
-10109600070335
-10109600580746
-10109600579510
-10109600581466
-10109600580921
-10109600559315
-10109600580936
-10109600638238
-10109600581302
-10109600233753
-10109600579508
-10109600638252
-10109600579696
-10109600579342
-10109600111264
-10109600455442
-10109600456303
-10109600581093
-10109600581056
-10109600532013
-10109600579621
-10109600580419
-10109600638184
-10109600579622
-10109600456226
-10109600580437
-10109600638223
-10109600580638
-10109600579695
-10109600579341
-10109600099076
-10109600581397
-10109600581301
-10109600579766
-10109600070445
-10109600570495
-10109600580869
-10109600532026
-10109600637953
-10109600532218
-10109600443314
-10109600638059
-10109600579547
-10109600581398
-10109600571826
-10109600532755
-10109600532253
-10109600579340
-10109600384472
-10109600580639
-10109600456255
-10109600456287
-10109600384905
-10109600494179
-10109600456163
-10109600580436
-10109600580418
-10109600580198
-10109600579623
-10109600343027
-10109600579768
-10109600581321
-10109600111332
-10109600580297
-10109600579546
-10109600521777
-10109600638195
-10109600532754
-10109600532306
-10109600580868
-10109600571842
-10109600098926
-10109600579767
-10109600456211
-10109600637952
-10109600532219
\ No newline at end of file

batch/msc-v096/trigger.py

-"""
-COSMOS
-"""
-
-# cd /nfsdata/share/simulation-collection/csst-dag_cfg
-# http://159.226.170.52:3000/scalebox/TaskList?app=csst-msc-l1-mbi.apps2
-
-from csst_dag import get_redis, gen_level1_dag_message
-from csst_dag.msc import MBI_CHIPID, SLS_CHIPID
-import numpy as np
-
-r = get_redis(location="naoc")
-obsid_list = np.loadtxt("batch/msc_096/obsid.txt", dtype=str)
-for obsid in obsid_list:
-    for chipid in MBI_CHIPID:
-        msg = gen_level1_dag_message(obsid=obsid, chipid=chipid)
-        print(msg)
-        r.push(msg)
-    break
-
-r.get_all()

batch/msc.md

-
-### 1000 平方度宽场测试天区 50个指向
-
-#### `csst-msc-c11-1000sqdeg-wide-test-v1`
-batch list:
-- `1000sqdeg-test-b1` : 第一版，流量定标没更新
-- `1000sqdeg-test-b2` : 第二版，流量定标更新, 测光解决了PSF无法读取的问题
-
-```bash
-python -m csst_dag.cli.csst_msc_l1 \
-    --dags csst-msc-l1-mbi \
-    --dataset=csst-msc-c11-1000sqdeg-wide-test-v1 \
-    --instrument=MSC \
-    --batch-id=1000sqdeg-test-b2 \
-    --ref-cat=trilegal_1000_w1 \
-    --priority=2 \
-    --pmapname=csst_000094.pmap \
-    --submit
-    
-    --obs-id=10600004269 \    
-```
-
-#### `csst-msc-c11-1000sqdeg-wide-test-v2`
-batch list:
-- `1000sqdeg-v2-b1` : 2025-07-14
-
-```bash
-python -m csst_dag.cli.csst_msc_l1 \
-    --dags csst-msc-l1-mbi \
-    --dataset=csst-msc-c11-1000sqdeg-wide-test-v2 \
-    --instrument=MSC \
-    --obs-id=10600003327 \
-    --batch-id=1000sqdeg-v2-b1 \
-    --ref-cat=trilegal_1000_w1 \
-    --priority=2 \
-    --pmapname=csst_000094.pmap \
-    --submit
-    
-    --obs-id=10600003327 \
-    
-    --obs-id=10600004269 \    
-```
-
-### 25 平方度宽场
-
-- `25sqdeg-test-b1` : 测试版，只跑了W1一部分
-- `25sqdeg-test-b2` : 第二版，跑了W1-5，但是没跑第一批已经处理的
-- `25sqdeg-test-b3` : 第三版，跑了W5所有
-- `25sqdeg-full-b1` : 测试版
-- `25sqdeg-full-b2` : 修复了20号探测器wcs问题，重新做了0级入库
-- `25sqdeg-full-b3` : 确认没问题，跑了W1-W5
-
-成像+光谱
-```bash
-python -m csst_dag.cli.run \
-    --dags csst-msc-l1-mbi csst-msc-l1-sls\
-    --dataset=csst-msc-c9-25sqdeg-v3 \
-    --instrument=MSC \
-    --obs-group=W5 \
-    --batch-id=25sqdeg-full-b3 \
-    --pmapname=csst_000094.pmap \
-    --ref-cat=trilegal_093 \
-    --verbose \
-    --submit
-
-python -m csst_dag.cli.inspect \
-    --level=1 \
-    --dataset=csst-msc-c9-25sqdeg-v3 \
-    --instrument=MSC \
-    --data-model=csst-msc-l1-mbi \
-    --batch-id=25sqdeg-full-b3
-```
-
-光谱
-```bash
-python -m csst_dag.cli.run \
-    --dags csst-msc-l1-sls \
-    --instrument=MSC \
-    --dataset=csst-msc-c9-25sqdeg-v3 \
-    --obs-group=W5 \
-    --batch-id=25sqdeg-sls-test-b2 \
-    --ref-cat=trilegal_093 \
-    --submit # 没反应
-    
-    --obs-group=W5 \    
-```
-
-
-### cosmos无缝光谱
-
-```bash
-python -m csst_dag.cli.inspect \
-    --level=0 \
-    --dataset=csst-msc-c11-sls-v1 \
-    --instrument=MSC \
-    --obs-id=10100070335
-```
-
-```bash
-python -m csst_dag.cli.run \
-    --dags csst-msc-l1-sls csst-msc-l1-mbi \
-    --dataset=csst-msc-c11-sls-v1 \
-    --instrument=MSC \
-    --batch-id=sls-cosmos-v1-b6 \
-    --ref-cat=trilegal_093 \
-    --pmapname=csst_000118.pmap \
-    --obs-group=W2 \
-    --obs-id=10100070335 \
-    --verbose \
-    --submit
-
-# 第二版, mbi修复了wcs，obs_group更新成了W2
-python -m csst_dag.cli.run \
-    --dags csst-msc-l1-mbi csst-msc-l1-sls \
-    --dataset=csst-msc-c11-sls-v1 \
-    --instrument=MSC \
-    --batch-id=sls-cosmos-v1-b4 \
-    --ref-cat=trilegal_093 \
-    --verbose \
-    --submit
-    
-
-python -m csst_dag.cli.run \
-    --dags csst-msc-l1-mbi \
-    --dataset=csst-msc-c11-sls-v1 \
-    --instrument=MSC \
-    --obs-id=10100070335 \
-    --batch-id=sls-cosmos-v1-b1 \
-    --ref-cat=trilegal_093 \
-    --priority=2 \
-    --verbose \
-    --submit    
-
-python -m csst_dag.cli.inspect \
-    --level=1 \
-    --dataset=csst-msc-c11-sls-v1 \
-    --instrument=MSC \
-    --data-model=csst-msc-l1-mbi \
-    --batch-id=sls-cosmos-v1-b1 \
-    
-    --obs-id=10100070335
-
-
-python -m csst_dag.cli.inspect \
-    --level=0 \
-    --dataset=csst-msc-c9-25sqdeg-v3 \
-    --instrument=MSC \
-    --obs-group=W5 \
-    --data-model=csst-msc-l1-mbi \
-    --batch-id=25sqdeg-full-b1 \
-        
-python -m csst_dag.cli.inspect \
-    --level=1 \
-    --dataset=csst-msc-c9-25sqdeg-v3 \
-    --instrument=MSC \
-    --obs-group=W5 \
-    --data-model=csst-msc-l1-mbi \
-    --batch-id=25sqdeg-full-b1 \
-    
-    --obs-id=10100070335    
-
-# 之江实验室试跑    
-python -m csst_dag.cli.run \
-    --dags csst-msc-l1-mbi \
-    --dataset=csst-msc-c11-sls-v1 \
-    --instrument=MSC \
-    --obs-id=10100070335 \
-    --batch-id=zjlab-test-b1 \
-    --ref-cat=trilegal_093 \
-    --priority=2 \
-    --verbose \
-    --submit     
-
-```
\ No newline at end of file

csst_dag/cli/_run.py

-import argparse
-import os
-import json
-from typing import Optional
-import joblib
-
-from csst_dag import CSST_DAGS, Dispatcher, BaseDAG, dfs
-
-
-def run_l1_pipeline(
-    # data parameters
-    dataset: str,
-    instrument: Optional[str] = None,
-    obs_type: Optional[str] = None,
-    obs_group: Optional[str] = None,
-    obs_id: Optional[str] = None,
-    detector: Optional[str] = None,
-    prc_status: Optional[int] = None,
-    qc_status: Optional[int] = None,
-    # task parameters
-    batch_id: Optional[str] = "test-batch",
-    priority: Optional[str] = "1",
-    # DAG parameters
-    pmapname: Optional[str] = "",
-    ref_cat: Optional[str] = "trilegal_093",
-    # submit
-    verbose: Optional[bool] = True,
-    submit: Optional[bool] = False,
-    final_prc_status: Optional[int] = -2,
-    force: Optional[bool] = False,
-    top_n: Optional[int] = -1,
-    # select DAGs
-    dags: Optional[list[str]] = None,
-    dag_group: Optional[str] = "csst-l1-pipeline",
-):
-    """Run a DAG.
-
-    Parameters
-    ----------
-    dataset : str
-        Dataset name
-    instrument : Optional[str], optional
-        Instrument name, by default None
-    obs_type : Optional[str], optional
-        Observation type, by default None
-    obs_group : Optional[str], optional
-        Observation group, by default None
-    obs_id : Optional[str], optional
-        Observation ID, by default None
-    detector : Optional[str], optional
-        Detector name, by default None
-    prc_status : Optional[int], optional
-        Initial processing status, by default None
-    qc_status : Optional[int], optional
-        Initial QC status, by default None
-    batch_id : Optional[str], optional
-        Batch ID, by default "test-batch"
-    priority : Optional[str], optional
-        Task priority, by default "1"
-    pmapname : Optional[str], optional
-        CCDS pmapname, by default ""
-    ref_cat : Optional[str], optional
-        Reference catalog, by default "trilegal_093"
-    submit : Optional[bool], optional
-        Push results, by default False
-    final_prc_status : Optional[int], optional
-        Final processing status, by default -2
-    force : Optional[bool], optional
-        Force success, by default False
-    verbose : Optional[bool], optional
-        Force success, by default False
-    top_n : Optional[int], optional
-        Submit top N tasks, by default -1
-    dags : Optional[list[str]], optional
-        DAGs to select, by default None
-    dag_group: Optional[str]="csst-l1-pipeline",
-        DAG group.
-    """
-
-    plan_basis, data_basis = Dispatcher.find_plan_level0_basis(
-        dataset=dataset,
-        instrument=instrument,
-        obs_type=obs_type,
-        obs_group=obs_group,
-        obs_id=obs_id,
-        detector=detector,
-        prc_status=prc_status,
-        qc_status=qc_status,
-    )
-    print(f"{len(plan_basis)} plan basis, {len(data_basis)} data basis found")
-
-    # generate DAG group run
-    dag_group_run = BaseDAG.generate_dag_group_run(
-        dag_group=dag_group,
-        batch_id=batch_id,
-        priority=priority,
-    )
-
-    # generate DAG run list
-    print("\n")
-    print(">>> Matching DAGs ...")
-    dag_run_list = []
-    data_id_list = []
-    n_dag_run_all = 0
-    n_dag_run_success = 0
-
-    DEFAULT_DAGS = {
-        "csst-msc-l1-mbi",
-        "csst-msc-l1-ast",
-        "csst-msc-l1-sls",
-        "csst-msc-l1-qc0",
-        "csst-msc-l1-ooc",
-        "csst-mci-l1-qc0",
-        "csst-mci-l1",
-        "csst-ifs-l1",
-        "csst-cpic-l1-qc0",
-        "csst-cpic-l1",
-        "csst-hstdm-l1",
-    }
-    assert dags is None or set(dags).issubset(DEFAULT_DAGS), f"Selected DAGs: {dags}"
-    SELECTED_DAGS = (
-        DEFAULT_DAGS.intersection(dags) if dags is not None else DEFAULT_DAGS
-    )
-    print("Selected DAGs: ", SELECTED_DAGS)
-
-    for dag in SELECTED_DAGS:
-        this_task_list = CSST_DAGS[dag].schedule(
-            dag_group_run=dag_group_run,
-            plan_basis=plan_basis,
-            data_basis=data_basis,
-            pmapname=pmapname,
-            ref_cat=ref_cat,
-            force_success=force,
-        )
-        this_dag_run_list = [
-            this_task["dag_run"]
-            for this_task in this_task_list
-            if this_task["dag_run"] is not None
-        ]
-        this_data_id_list = [
-            this_task["relevant_data_id_list"]
-            for this_task in this_task_list
-            if this_task["dag_run"] is not None
-        ]
-
-        dag_run_list.extend(this_dag_run_list)
-        data_id_list.extend(this_data_id_list)
-        this_n_dag_run_all = len(this_task_list)
-        this_n_dag_run_success = len(this_dag_run_list)
-        n_dag_run_all += this_n_dag_run_all
-        n_dag_run_success += this_n_dag_run_success
-        print(
-            f"- `{dag}` : [ {this_n_dag_run_success} / {this_n_dag_run_all} ] dag_runs"
-        )
-
-    # print dag_group_run and dag_run_list
-    if verbose:
-        print("\n")
-        print(">>> `dag_group_run` :")
-        print(f"\t- {json.dumps(dag_group_run, separators=(',', ':'))}")
-        print(f">>> `dag_run_list` : [ {n_dag_run_success} / {n_dag_run_all} ]")
-        if len(dag_run_list) > 0:
-            for dag_run in dag_run_list:
-                try:
-                    print(f"\t- {json.dumps(dag_run, separators=(',', ':'))}")
-                except:
-                    print(f"\t- {dag_run}")
-
-    # dump dag_group_run
-    # joblib.dump(
-    #     dict(
-    #         plan_basis=plan_basis,
-    #         data_basis=data_basis,
-    #         dag_group_run=dag_group_run,
-    #         dag_run_list=dag_run_list,
-    #     ),
-    #     os.path.join(
-    #         os.getenv("HOME"),
-    #         "csst_dag",
-    #         f"{dag_group_run['dag_group_run']}.joblib",
-    #     ),
-    # )
-    # submit DAG group run
-    if submit:
-        res = dfs.dag.new_dag_group_run(
-            dag_group_run=dag_group_run,
-            dag_run_list=dag_run_list,
-        )
-        print(res)

csst_dag/cli/inspect_dag_group_run.py

-import os
-import sys
-import joblib
-
-# get dag_group_run hash
-dag_group_run_hash = sys.argv[1]
-assert len(dag_group_run_hash) == 40
-
-# get file path
-file_path = os.path.join(
-    os.getenv("HOME"),
-    "csst_dag",
-    f"{dag_group_run_hash}.joblib",
-)
-
-# load DAG group run
-data = joblib.load(file_path)
-dag_group_run = data["dag_group_run"]
-dag_run_list = data["dag_run_list"]
-
-# print dag_group_run and dag_run_list
-print("`dag_group_run`:")
-print(f"  - {dag_group_run}")
-for dag_run in dag_run_list:
-    print(f"  - {dag_run}")

csst_dag/cli/inspect_data.py

-"""
-Aim
----
-Inspect a dataset.
-
-Example
--------
-python -m csst_dag.cli.inspect -h
-
-python -m csst_dag.cli.inspect \
-    --dataset=csst-msc-c9-25sqdeg-v3 \
-    --instrument=MSC \
-    --obs-type=WIDE \
-    --obs-group=W2
-
-25平方度宽场
-python -m csst_dag.cli.inspect \
-    --level=0 \
-    --dataset=csst-msc-c9-25sqdeg-v3 \
-    --instrument=MSC \
-    --obs-type=WIDE
-
-python -m csst_dag.cli.inspect \
-    --level=0 \
-    --dataset=csst-msc-c11-1000sqdeg-wide-test-v1 \
-    --instrument=MSC \
-    --obs-type=WIDE
-
-python -m csst_dag.cli.inspect \
-    --level=1 \
-    --dataset=csst-msc-c11-1000sqdeg-wide-test-v1 \
-    --instrument=MSC \
-    --obs-type=WIDE \
-    --data-model=csst-msc-l1-mbi \
-    --batch-id=1000sqdeg-test-b1
-
-CPIC:
-python -m csst_dag.cli.inspect \
-    --level=0 \
-    --dataset=csst-cpic-c11-hip71681-v1 \
-    --instrument=CPIC \
-    --obs-type=SCI \
-    --obs-group=hip71681
-
-"""
-
-import argparse
-
-import numpy as np
-from astropy import table
-
-from csst_dag import Dispatcher, dfs
-
-parser = argparse.ArgumentParser(
-    description="Inspector for CSST datasets.",
-    formatter_class=argparse.ArgumentDefaultsHelpFormatter,
-)
-
-# data level
-parser.add_argument("--level", type=int, help="Data level", default=0)
-# level0 data parameters
-parser.add_argument("--dataset", type=str, help="Dataset name", default="csst-msc-c9-25sqdeg-v3")
-parser.add_argument("--instrument", type=str, help="Instrument name", default="MSC")
-parser.add_argument("--obs-type", type=str, help="Observation type", default=None)
-parser.add_argument("--obs-group", type=str, help="Observation group", default=None)
-parser.add_argument("--obs-id", type=str, help="Observation ID", default=None)
-parser.add_argument("--detector", type=str, help="Detector name", default=None)
-parser.add_argument("--prc-status", type=int, help="Prc status", default=None)
-# level1 data parameters
-parser.add_argument("--data-model", type=str, help="Data model", default=None)
-parser.add_argument("--batch-id", type=str, help="Batch ID", default=None)
-parser.add_argument("--build", type=str, help="Build number", default=None)
-parser.add_argument("--pmapname", type=str, help="CCDS pmap name", default=None)
-# reset prc_status
-parser.add_argument("--reset", type=int, help="Reset prc_status", default=None)
-# additional options
-parser.add_argument("--verbose", action="store_true", help="Verbose mode", default=False)
-args = parser.parse_args()
-
-# from csst_dag import DotDict
-#
-# args = DotDict(
-#     dataset="csst-msc-c9-25sqdeg-v3",
-#     instrument="MSC",
-#     obs_type="WIDE",
-#     obs_group="W2",
-#     obs_id=None,
-#     detector=None,
-#     prc_status=None,
-# )
-
-print("CLI parameters: ", args)
-
-if args.level == 0:
-    print("Inspecting level0 data...")
-    data_basis = Dispatcher.find_level0_basis(
-        dataset=args.dataset,
-        instrument=args.instrument,
-        obs_type=args.obs_type,
-        obs_group=args.obs_group,
-        obs_id=args.obs_id,
-        detector=args.detector,
-        prc_status=args.prc_status,
-    )
-else:
-    print("Inspecting level1 data...")
-    data_basis = Dispatcher.find_level1_basis(
-        dataset=args.dataset,
-        instrument=args.instrument,
-        obs_type=args.obs_type,
-        obs_group=args.obs_group,
-        obs_id=args.obs_id,
-        detector=args.detector,
-        prc_status=args.prc_status,
-        data_model=args.data_model, # level1 specific
-        batch_id=args.batch_id,
-        build=args.build,
-        pmapname=args.pmapname,
-    )
-print(f">>> {len(data_basis)} data basis found")
-# data_basis.remove_columns(["file_name", "_id"])
-
-stats_config = dict(
-    prc_status=["dataset", "prc_status", "qc_status"],
-    obs_type=["dataset", "instrument", "obs_type", "prc_status", "qc_status"],
-    obs_group=["dataset", "instrument", "obs_type", "obs_group", "prc_status", "qc_status"],
-    detector=["dataset", "instrument", "obs_type", "obs_group", "detector", "prc_status", "qc_status"],
-    obs_id=["dataset", "instrument", "obs_type", "obs_group", "obs_id", "prc_status", "qc_status"],
-)
-if args.level == 1:
-    for k in stats_config.keys():
-        stats_config[k].extend(["data_model", "batch_id", "build"])
-
-for stats_type, stats_keys in stats_config.items():
-    if args.verbose:
-        data_basis.pprint_all()
-    print("\n>>> STATS DIM: ", stats_type)
-    u_data, c_data = np.unique(data_basis[stats_keys], return_counts=True)
-    u_table = table.Table(u_data)
-    u_table.add_column(table.Column(c_data, name="count"))
-    u_table.pprint_all()
-
-if args.reset is not None:
-    print("Resetting prc_status to ", args.reset)
-    dfs.update_prc_status_by_ids(
-        list(data_basis["_id"]),
-        args.reset,
-    )

csst_dag/cli/run.py

-"""
-Aim
----
-Process an MSC dataset, given a set of parameters.
-
-Example
--------
-python -m csst_dag.cli.run -h
-
-python -m csst_dag.cli.run \
-    --dags csst-msc-l1-mbi \
-    --dataset=csst-msc-c9-25sqdeg-v3 \
-    --instrument=MSC \
-    --obs-type=WIDE \
-    --obs-group=W2 \
-    --obs-id=10100232366 \
-    --detector=09 \
-    --batch-id=test-b1 \
-    --priority=1 \
-    --pmapname=csst_000070.pmap \
-    --ref-cat=trilegal_093
-
-# 25平方度宽场
-python -m csst_dag.cli.run \
-    --dags csst-msc-l1-mbi \
-    --dataset=csst-msc-c9-25sqdeg-v3 \
-    --instrument=MSC \
-    --obs-type=WIDE \
-    --prc-status=-2 \
-    --batch-id=25sqdeg-test-b2 \
-    --priority=1 \
-    --pmapname=csst_000070.pmap \
-    --ref-cat=trilegal_093 \
-    --submit
-
-"""
-
-import argparse
-import os
-import json
-
-import joblib
-
-from csst_dag import CSST_DAGS, Dispatcher, BaseDAG, dfs
-
-parser = argparse.ArgumentParser(
-    description="Scheduler for CSST L1 pipeline.",
-    formatter_class=argparse.ArgumentDefaultsHelpFormatter,
-)
-
-# data parameters
-parser.add_argument("--dataset", type=str, help="Dataset name")
-parser.add_argument("--instrument", type=str, help="Instrument name", default=None)
-parser.add_argument("--obs-type", type=str, help="Observation type", default=None)
-parser.add_argument("--obs-group", type=str, help="Observation group", default=None)
-parser.add_argument("--obs-id", type=str, help="Observation ID", default=None)
-parser.add_argument("--detector", type=str, help="Detector name", default=None)
-parser.add_argument(
-    "--prc-status", type=int, help="Initial processing status", default=None
-)
-parser.add_argument("--qc-status", type=int, help="Initial QC status", default=None)
-# task parameters
-parser.add_argument("--batch-id", type=str, help="Batch ID", default="test-batch")
-parser.add_argument("--priority", type=str, help="Task priority", default="1")
-# DAG parameters
-parser.add_argument("--pmapname", type=str, help="CCDS pmapname", default="")
-parser.add_argument(
-    "--ref-cat", type=str, help="Reference catalog", default="trilegal_093"
-)
-# submit
-parser.add_argument("--submit", action="store_true", help="Push results", default=False)
-# post-processing parameters
-parser.add_argument(
-    "--final-prc-status", type=int, help="Final processing status", default=-2
-)
-# additional options
-parser.add_argument("--force", action="store_true", help="Force success", default=False)
-parser.add_argument(
-    "--verbose", action="store_true", help="Force success", default=False
-)
-# submit top N
-parser.add_argument("--top-n", type=int, help="Submit top N tasks", default=-1)
-# select DAGs
-parser.add_argument("--dags", nargs="+", type=str, help="DAGs to select", default=None)
-
-
-args = parser.parse_args()
-
-
-# from csst_dag import DotDict
-#
-# args = DotDict(
-#     dataset="csst-msc-c9-25sqdeg-v3",
-#     instrument="MSC",
-#     obs_type="WIDE",
-#     obs_group="W2",
-#     obs_id="10100232366",
-#     detector=None,
-#     prc_status=None,
-#     batch_id="test-batch",
-#     priority=1,
-#     pmapname="csst_000070.pmap",
-#     ref_cat="trilegal_093",
-#     submit=False,
-# )
-
-print("CLI parameters: ", args)
-
-plan_basis, data_basis = Dispatcher.find_plan_level0_basis(
-    dataset=args.dataset,
-    instrument=args.instrument,
-    obs_type=args.obs_type,
-    obs_group=args.obs_group,
-    obs_id=args.obs_id,
-    detector=args.detector,
-    prc_status=args.prc_status,
-    qc_status=args.qc_status,
-)
-print(f"{len(plan_basis)} plan basis, {len(data_basis)} data basis found")
-
-# generate DAG group run
-dag_group_run = BaseDAG.generate_dag_group_run(
-    dag_group="csst-l1-pipeline",
-    batch_id=args.batch_id,
-    priority=args.priority,
-)
-
-# generate DAG run list
-print("\n")
-print(">>> Matching DAGs ...")
-dag_run_list = []
-data_id_list = []
-n_dag_run_all = 0
-n_dag_run_success = 0
-
-DEFAULT_DAGS = {
-    "csst-msc-l1-mbi",
-    "csst-msc-l1-ast",
-    "csst-msc-l1-sls",
-    "csst-msc-l1-qc0",
-    "csst-msc-l1-ooc",
-    "csst-mci-l1-qc0",
-    "csst-mci-l1",
-    "csst-ifs-l1",
-    "csst-cpic-l1-qc0",
-    "csst-cpic-l1",
-    "csst-hstdm-l1",
-}
-assert args.dags is None or set(args.dags).issubset(
-    DEFAULT_DAGS
-), f"Selected DAGs: {args.dags}"
-SELECTED_DAGS = (
-    DEFAULT_DAGS.intersection(args.dags) if args.dags is not None else DEFAULT_DAGS
-)
-print("Selected DAGs: ", SELECTED_DAGS)
-
-for dag in SELECTED_DAGS:
-    this_task_list = CSST_DAGS[dag].schedule(
-        dag_group_run=dag_group_run,
-        plan_basis=plan_basis,
-        data_basis=data_basis,
-        pmapname=args.pmapname,
-        ref_cat=args.ref_cat,
-        force_success=args.force,
-    )
-    this_dag_run_list = [
-        this_task["dag_run"]
-        for this_task in this_task_list
-        if this_task["dag_run"] is not None
-    ]
-    this_data_id_list = [
-        this_task["relevant_data_id_list"]
-        for this_task in this_task_list
-        if this_task["dag_run"] is not None
-    ]
-
-    dag_run_list.extend(this_dag_run_list)
-    data_id_list.extend(this_data_id_list)
-    this_n_dag_run_all = len(this_task_list)
-    this_n_dag_run_success = len(this_dag_run_list)
-    n_dag_run_all += this_n_dag_run_all
-    n_dag_run_success += this_n_dag_run_success
-    print(f"- `{dag}` : [ {this_n_dag_run_success} / {this_n_dag_run_all} ] dag_runs")
-
-# print dag_group_run and dag_run_list
-if args.verbose:
-    print("\n")
-    print(">>> `dag_group_run` :")
-    print(f"\t- {json.dumps(dag_group_run, separators=(',', ':'))}")
-    print(f">>> `dag_run_list` : [ {n_dag_run_success} / {n_dag_run_all} ]")
-    if len(dag_run_list) > 0:
-        for dag_run in dag_run_list:
-            try:
-                print(f"\t- {json.dumps(dag_run, separators=(',', ':'))}")
-            except:
-                print(f"\t- {dag_run}")
-
-# dump dag_group_run
-joblib.dump(
-    dict(
-        plan_basis=plan_basis,
-        data_basis=data_basis,
-        dag_group_run=dag_group_run,
-        dag_run_list=dag_run_list,
-    ),
-    os.path.join(
-        os.getenv("HOME"),
-        "csst_dag",
-        f"{dag_group_run['dag_group_run']}.joblib",
-    ),
-)
-# submit DAG group run
-if args.submit:
-    res = dfs.dag.new_dag_group_run(
-        dag_group_run=dag_group_run,
-        dag_run_list=dag_run_list,
-    )
-    print(res)

csst_dag/cli/test.py

-import  csst_dag
-print("csst_dag version: ", csst_dag.__version__)

csst_dag/dfs.py

@@ -10,7 +10,7 @@ from astropy import table
 from csst_dfs_client import plan, level0, level1, catalog
 
 from ._csst import csst
-from .utils import override_common_keys
+from .dag._dag_utils import override_common_keys
 
 
 # -----------------

csst_dag/utils.py

-def override_common_keys(d1: dict, d2: dict) -> dict:
-    """
-    Construct a new dictionary by updating the values of basis_keys that exists in the first dictionary
-    with the values of the second dictionary.
-
-    Parameters
-    ----------
-    d1 : dict
-        The first dictionary.
-    d2 : dict
-        The second dictionary.
-
-    Returns
-    -------
-    dict:
-        The updated dictionary.
-    """
-    return {k: d2[k] if k in d2.keys() else d1[k] for k in d1.keys()}