Skip to content

References

args = parser.parse_args() module-attribute

params = yaml.safe_load(file) module-attribute

parser = argparse.ArgumentParser(description=f'muvis-align') module-attribute

pipeline = Pipeline(params) module-attribute

muvis_align

__all__ = ('MainWidget',) module-attribute

__getattr__(name)

Source code in src\muvis_align\__init__.py
 6
 7
 8
 9
10
11
def __getattr__(name):
    # imported on use: worker processes import the package for its registration code, not napari's UI
    if name == "MainWidget":
        from ._widget import MainWidget
        return MainWidget
    raise AttributeError(f"module {__name__!r} has no attribute {name!r}")

GlobalOptProgress

Progress for multiview_stitcher's global optimization, read out of its own debug log.

groupwise_resolution() is one long blocking call with nothing to hook (no dask tasks, no tqdm), but it logs each inner-loop iteration. The bar counts outer-loop passes, each removing one edge, so the edge count bounds them; a pass also advances with its iterations. Headless, the progress and residual are logged every heartbeat_seconds, a falling residual telling slow from stuck.

DEFAULT_MAX_ITER = 500 module-attribute

FINISHED_MESSAGE = 'Finished glob opt. Max and mean residuals: %s \t %s' module-attribute

GLOBAL_OPT_LOGGER = 'multiview_stitcher.param_resolution.global_optimization' module-attribute

ITERATION_MESSAGE = 'Glob opt iter %s, node %s, mean residual %s, max residual %s' module-attribute

MAX_ITER_MESSAGE = 'Global optimization: setting max_iter to %s' module-attribute

GlobalOptProgress

Reports multiview_stitcher's global optimization into progress_factory's bar, or only the log without one.

Source code in src\muvis_align\GlobalOptProgress.py
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
class GlobalOptProgress:
    """Reports multiview_stitcher's global optimization into `progress_factory`'s bar, or only the log without one."""

    heartbeat_seconds = 30

    def __init__(self, progress_factory=None, desc='Global registration', max_passes=None,
                 max_iter=None, weight=1, heartbeat_seconds=None):
        self.progress_factory = progress_factory
        self.desc = desc
        self.max_passes = max(int(max_passes), 1) if max_passes else None
        self.max_iter = max_iter or DEFAULT_MAX_ITER
        self.weight = weight
        if heartbeat_seconds is not None:
            self.heartbeat_seconds = heartbeat_seconds
        self._logger = None
        self._handler = None
        self._prior_level = None
        self._forward_level = logging.NOTSET
        self._prior_propagate = None
        self._phase = None
        self._pass = 0
        self._iteration = None
        self._reported = 0.0
        self._reported_at = None
        self._saw_iterations = False

    def __enter__(self):
        self._logger = logging.getLogger(GLOBAL_OPT_LOGGER)
        # debug level is needed for the iteration lines, so propagation is off and forward() passes on
        # only what the level in force would have shown
        self._prior_level = self._logger.level
        self._forward_level = self._logger.getEffectiveLevel()
        self._prior_propagate = self._logger.propagate
        self._handler = _RecordListener(self)
        self._logger.addHandler(self._handler)
        self._logger.setLevel(logging.DEBUG)
        self._logger.propagate = False
        self._pass = 1
        self._saw_iterations = False
        return self

    def __exit__(self, exc_type, exc_value, traceback):
        if self._logger is not None:
            self._logger.removeHandler(self._handler)
            self._logger.setLevel(self._prior_level)
            self._logger.propagate = self._prior_propagate
        self._handler = None
        if not self._saw_iterations and exc_type is None:
            # nothing recognised: still cross one phase
            self._open_phase()
        self._end_phase(exc_type)
        return False

    def forward(self, record):
        if record.levelno < self._forward_level:
            return
        parent = self._logger.parent if self._logger is not None else None
        if parent is not None:
            parent.handle(record)

    def note_max_iter(self, max_iter):
        try:
            self.max_iter = int(max_iter)
        except (TypeError, ValueError):
            pass

    def note_iteration(self, iteration, max_residual=None):
        try:
            iteration = int(iteration)
        except (TypeError, ValueError):
            return
        if self._iteration is not None and iteration <= self._iteration:
            # the count restarting means the outer loop dropped an edge and began again
            self._pass += 1
        self._iteration = iteration
        self._saw_iterations = True
        self._open_phase()
        self._advance(self._pass - 1 + min((iteration + 1) / self.max_iter, 1.0))
        self._log_heartbeat(max_residual)

    def note_finished(self):
        if self._saw_iterations:
            self._advance(self._pass)

    def _open_phase(self):
        # opened at the first iteration: max_iter is only logged after __enter__
        if self._phase is not None or self.progress_factory is None:
            return
        self._phase = self.progress_factory(total=self.max_passes, desc=self.desc,
                                            weight=self.weight)
        self._phase.__enter__()

    def _end_phase(self, exc_type):
        if self._phase is not None:
            self._phase.__exit__(exc_type, None, None)
            self._phase = None

    def _advance(self, progress):
        """Move the phase to `progress` passes done, which only ever grows."""
        progress = max(progress, self._reported)
        if self.max_passes is not None:
            progress = min(progress, self.max_passes)
        step = progress - self._reported
        self._reported = progress
        if self._phase is not None and step > 0:
            self._phase.update(step)

    def _log_heartbeat(self, max_residual):
        if not self.heartbeat_seconds:
            return
        now = time.monotonic()
        if self._reported_at is not None and now - self._reported_at < self.heartbeat_seconds:
            return
        self._reported_at = now
        residual = ''
        if max_residual is not None:
            try:
                residual = f', max residual {float(max_residual):.4g}'
            except (TypeError, ValueError):
                pass
        of_passes = f'/{self.max_passes}' if self.max_passes else ''
        logging.info(f'{self.desc}: optimisation pass {self._pass}{of_passes},'
                     f' iteration {self._iteration + 1}/{self.max_iter}{residual}')
desc = desc instance-attribute
heartbeat_seconds = 30 class-attribute instance-attribute
max_iter = max_iter or DEFAULT_MAX_ITER instance-attribute
max_passes = max(int(max_passes), 1) if max_passes else None instance-attribute
progress_factory = progress_factory instance-attribute
weight = weight instance-attribute
__enter__()
Source code in src\muvis_align\GlobalOptProgress.py
49
50
51
52
53
54
55
56
57
58
59
60
61
62
def __enter__(self):
    self._logger = logging.getLogger(GLOBAL_OPT_LOGGER)
    # debug level is needed for the iteration lines, so propagation is off and forward() passes on
    # only what the level in force would have shown
    self._prior_level = self._logger.level
    self._forward_level = self._logger.getEffectiveLevel()
    self._prior_propagate = self._logger.propagate
    self._handler = _RecordListener(self)
    self._logger.addHandler(self._handler)
    self._logger.setLevel(logging.DEBUG)
    self._logger.propagate = False
    self._pass = 1
    self._saw_iterations = False
    return self
__exit__(exc_type, exc_value, traceback)
Source code in src\muvis_align\GlobalOptProgress.py
64
65
66
67
68
69
70
71
72
73
74
def __exit__(self, exc_type, exc_value, traceback):
    if self._logger is not None:
        self._logger.removeHandler(self._handler)
        self._logger.setLevel(self._prior_level)
        self._logger.propagate = self._prior_propagate
    self._handler = None
    if not self._saw_iterations and exc_type is None:
        # nothing recognised: still cross one phase
        self._open_phase()
    self._end_phase(exc_type)
    return False
__init__(progress_factory=None, desc='Global registration', max_passes=None, max_iter=None, weight=1, heartbeat_seconds=None)
Source code in src\muvis_align\GlobalOptProgress.py
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
def __init__(self, progress_factory=None, desc='Global registration', max_passes=None,
             max_iter=None, weight=1, heartbeat_seconds=None):
    self.progress_factory = progress_factory
    self.desc = desc
    self.max_passes = max(int(max_passes), 1) if max_passes else None
    self.max_iter = max_iter or DEFAULT_MAX_ITER
    self.weight = weight
    if heartbeat_seconds is not None:
        self.heartbeat_seconds = heartbeat_seconds
    self._logger = None
    self._handler = None
    self._prior_level = None
    self._forward_level = logging.NOTSET
    self._prior_propagate = None
    self._phase = None
    self._pass = 0
    self._iteration = None
    self._reported = 0.0
    self._reported_at = None
    self._saw_iterations = False
forward(record)
Source code in src\muvis_align\GlobalOptProgress.py
76
77
78
79
80
81
def forward(self, record):
    if record.levelno < self._forward_level:
        return
    parent = self._logger.parent if self._logger is not None else None
    if parent is not None:
        parent.handle(record)
note_finished()
Source code in src\muvis_align\GlobalOptProgress.py
103
104
105
def note_finished(self):
    if self._saw_iterations:
        self._advance(self._pass)
note_iteration(iteration, max_residual=None)
Source code in src\muvis_align\GlobalOptProgress.py
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
def note_iteration(self, iteration, max_residual=None):
    try:
        iteration = int(iteration)
    except (TypeError, ValueError):
        return
    if self._iteration is not None and iteration <= self._iteration:
        # the count restarting means the outer loop dropped an edge and began again
        self._pass += 1
    self._iteration = iteration
    self._saw_iterations = True
    self._open_phase()
    self._advance(self._pass - 1 + min((iteration + 1) / self.max_iter, 1.0))
    self._log_heartbeat(max_residual)
note_max_iter(max_iter)
Source code in src\muvis_align\GlobalOptProgress.py
83
84
85
86
87
def note_max_iter(self, max_iter):
    try:
        self.max_iter = int(max_iter)
    except (TypeError, ValueError):
        pass

MVSRegistration

MVSRegistration

Source code in src\muvis_align\MVSRegistration.py
 192
 193
 194
 195
 196
 197
 198
 199
 200
 201
 202
 203
 204
 205
 206
 207
 208
 209
 210
 211
 212
 213
 214
 215
 216
 217
 218
 219
 220
 221
 222
 223
 224
 225
 226
 227
 228
 229
 230
 231
 232
 233
 234
 235
 236
 237
 238
 239
 240
 241
 242
 243
 244
 245
 246
 247
 248
 249
 250
 251
 252
 253
 254
 255
 256
 257
 258
 259
 260
 261
 262
 263
 264
 265
 266
 267
 268
 269
 270
 271
 272
 273
 274
 275
 276
 277
 278
 279
 280
 281
 282
 283
 284
 285
 286
 287
 288
 289
 290
 291
 292
 293
 294
 295
 296
 297
 298
 299
 300
 301
 302
 303
 304
 305
 306
 307
 308
 309
 310
 311
 312
 313
 314
 315
 316
 317
 318
 319
 320
 321
 322
 323
 324
 325
 326
 327
 328
 329
 330
 331
 332
 333
 334
 335
 336
 337
 338
 339
 340
 341
 342
 343
 344
 345
 346
 347
 348
 349
 350
 351
 352
 353
 354
 355
 356
 357
 358
 359
 360
 361
 362
 363
 364
 365
 366
 367
 368
 369
 370
 371
 372
 373
 374
 375
 376
 377
 378
 379
 380
 381
 382
 383
 384
 385
 386
 387
 388
 389
 390
 391
 392
 393
 394
 395
 396
 397
 398
 399
 400
 401
 402
 403
 404
 405
 406
 407
 408
 409
 410
 411
 412
 413
 414
 415
 416
 417
 418
 419
 420
 421
 422
 423
 424
 425
 426
 427
 428
 429
 430
 431
 432
 433
 434
 435
 436
 437
 438
 439
 440
 441
 442
 443
 444
 445
 446
 447
 448
 449
 450
 451
 452
 453
 454
 455
 456
 457
 458
 459
 460
 461
 462
 463
 464
 465
 466
 467
 468
 469
 470
 471
 472
 473
 474
 475
 476
 477
 478
 479
 480
 481
 482
 483
 484
 485
 486
 487
 488
 489
 490
 491
 492
 493
 494
 495
 496
 497
 498
 499
 500
 501
 502
 503
 504
 505
 506
 507
 508
 509
 510
 511
 512
 513
 514
 515
 516
 517
 518
 519
 520
 521
 522
 523
 524
 525
 526
 527
 528
 529
 530
 531
 532
 533
 534
 535
 536
 537
 538
 539
 540
 541
 542
 543
 544
 545
 546
 547
 548
 549
 550
 551
 552
 553
 554
 555
 556
 557
 558
 559
 560
 561
 562
 563
 564
 565
 566
 567
 568
 569
 570
 571
 572
 573
 574
 575
 576
 577
 578
 579
 580
 581
 582
 583
 584
 585
 586
 587
 588
 589
 590
 591
 592
 593
 594
 595
 596
 597
 598
 599
 600
 601
 602
 603
 604
 605
 606
 607
 608
 609
 610
 611
 612
 613
 614
 615
 616
 617
 618
 619
 620
 621
 622
 623
 624
 625
 626
 627
 628
 629
 630
 631
 632
 633
 634
 635
 636
 637
 638
 639
 640
 641
 642
 643
 644
 645
 646
 647
 648
 649
 650
 651
 652
 653
 654
 655
 656
 657
 658
 659
 660
 661
 662
 663
 664
 665
 666
 667
 668
 669
 670
 671
 672
 673
 674
 675
 676
 677
 678
 679
 680
 681
 682
 683
 684
 685
 686
 687
 688
 689
 690
 691
 692
 693
 694
 695
 696
 697
 698
 699
 700
 701
 702
 703
 704
 705
 706
 707
 708
 709
 710
 711
 712
 713
 714
 715
 716
 717
 718
 719
 720
 721
 722
 723
 724
 725
 726
 727
 728
 729
 730
 731
 732
 733
 734
 735
 736
 737
 738
 739
 740
 741
 742
 743
 744
 745
 746
 747
 748
 749
 750
 751
 752
 753
 754
 755
 756
 757
 758
 759
 760
 761
 762
 763
 764
 765
 766
 767
 768
 769
 770
 771
 772
 773
 774
 775
 776
 777
 778
 779
 780
 781
 782
 783
 784
 785
 786
 787
 788
 789
 790
 791
 792
 793
 794
 795
 796
 797
 798
 799
 800
 801
 802
 803
 804
 805
 806
 807
 808
 809
 810
 811
 812
 813
 814
 815
 816
 817
 818
 819
 820
 821
 822
 823
 824
 825
 826
 827
 828
 829
 830
 831
 832
 833
 834
 835
 836
 837
 838
 839
 840
 841
 842
 843
 844
 845
 846
 847
 848
 849
 850
 851
 852
 853
 854
 855
 856
 857
 858
 859
 860
 861
 862
 863
 864
 865
 866
 867
 868
 869
 870
 871
 872
 873
 874
 875
 876
 877
 878
 879
 880
 881
 882
 883
 884
 885
 886
 887
 888
 889
 890
 891
 892
 893
 894
 895
 896
 897
 898
 899
 900
 901
 902
 903
 904
 905
 906
 907
 908
 909
 910
 911
 912
 913
 914
 915
 916
 917
 918
 919
 920
 921
 922
 923
 924
 925
 926
 927
 928
 929
 930
 931
 932
 933
 934
 935
 936
 937
 938
 939
 940
 941
 942
 943
 944
 945
 946
 947
 948
 949
 950
 951
 952
 953
 954
 955
 956
 957
 958
 959
 960
 961
 962
 963
 964
 965
 966
 967
 968
 969
 970
 971
 972
 973
 974
 975
 976
 977
 978
 979
 980
 981
 982
 983
 984
 985
 986
 987
 988
 989
 990
 991
 992
 993
 994
 995
 996
 997
 998
 999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
2474
2475
2476
2477
2478
2479
2480
2481
2482
2483
2484
2485
2486
2487
2488
2489
2490
2491
2492
2493
2494
2495
2496
2497
2498
2499
2500
2501
2502
2503
2504
2505
2506
2507
2508
2509
2510
2511
2512
2513
2514
2515
2516
2517
2518
2519
2520
2521
2522
2523
2524
2525
2526
2527
2528
2529
2530
2531
2532
2533
2534
2535
2536
2537
2538
2539
2540
2541
2542
2543
2544
2545
2546
2547
2548
2549
2550
2551
2552
2553
2554
2555
2556
2557
2558
2559
2560
2561
2562
2563
2564
2565
2566
2567
2568
2569
2570
class MVSRegistration:
    def __init__(self, operation='register', label='', input_path=None, output_path=None,
                 source_metadata={}, extra_metadata={},
                 global_rotation=None, global_center=None,
                 overwrite=True, clear=False, ui='', verbose=False, debug=False):
        # set here as well as in init(): an instance may never have init() called on it
        self.verbose = verbose
        self.logging_dask = self.verbose
        self.logging_time = self.verbose
        self.reset()

        if input_path is not None:
            self.init(operation=operation, label=label, input_path=input_path, output_path=output_path,
                      source_metadata=source_metadata, extra_metadata=extra_metadata,
                      global_rotation=global_rotation, global_center=global_center,
                      overwrite=overwrite, clear=clear, ui=ui, verbose=verbose, debug=debug)

    @property
    def msims(self):
        # built lazily from the geometry init_data() resolved, only once pixel data is needed
        return self.ensure_msims()

    @msims.setter
    def msims(self, value):
        self._msims = value

    @staticmethod
    def has_eager_pre_processing(params):
        """Whether pre-processing will compute over the image data, rather than only rebuild the graph.

        Scaling and Gaussian filtering are lazy; flat-field, normalisation and foreground filtering read every source.
        """
        return bool(params.get('flatfield_quantiles') or params.get('filter_foreground')
                    or normalisation_enabled(params.get('normalisation')))

    def ensure_msims(self, progress_factory=None, target_scale=None, weight=1, ends_only=False):
        """The lazy build the msims property triggers, callable ahead of time to report its progress.

        `target_scale` builds each source's pyramid from the level that scale needs, cached per scale, never as
        self.msims, which stays the full-resolution pyramid. `ends_only` builds just each source's finest level at
        that scale and its coarsest (see source_levels): all pre-processing's consumers read.
        """
        # a factor as text ('2') or a pixel size ('10um'): one cache entry however it was written
        target_scale = parse_scale(target_scale, default=None)
        key = self._scaled_msims_key(target_scale, ends_only)
        if (target_scale or ends_only) and key not in self._scaled_msims:
            from_levels = [get_level_from_scale(source, target_scale)[0] if target_scale else 0
                           for source in self.sources]
            # nothing to skip: the shared full-resolution build serves, unless only the ends are wanted,
            # which the shared build must not be cut down to
            if any(from_levels) or ends_only:
                self._scaled_msims[key] = self._build_msims(progress_factory=progress_factory,
                                                            from_levels=from_levels, store=False,
                                                            weight=weight, ends_only=ends_only)
        if key in self._scaled_msims:
            return self._scaled_msims[key]
        if self._msims is None:
            self._build_msims(progress_factory=progress_factory, weight=weight)
        return self._msims

    @staticmethod
    def _scaled_msims_key(target_scale, ends_only=False):
        return f'{target_scale}:ends' if ends_only else str(target_scale)

    def msims_build_pending(self, target_scale=None, ends_only=False):
        """Whether ensure_msims() would actually build, and so report a progress phase of its own."""
        target_scale = parse_scale(target_scale, default=None)
        key = self._scaled_msims_key(target_scale, ends_only)
        if key in self._scaled_msims:
            return False
        if ends_only:
            return True
        if target_scale:
            from_levels = [get_level_from_scale(source, target_scale)[0] for source in self.sources]
            if any(from_levels):
                return True
        return self._msims is None

    def _build_source_msim(self, index, from_level=0, ends_only=False):
        return build_source_msim(self.sources[index], self._msim_output_order, self.positions[index],
                                 self._msim_transforms[index], self.source_transform_key,
                                 z_scale=self._msim_z_scale, from_level=from_level, ends_only=ends_only)

    def _build_msims(self, progress_factory=None, from_levels=None, store=True, weight=1, ends_only=False):
        # where each source's pixels are first opened: threads overlap network reads, each source's caches its own
        def build_msim(index):
            return self._build_source_msim(index, from_levels[index] if from_levels else 0, ends_only)

        msims = pooled_map(build_msim, len(self.sources), default_msim_build_workers, progress_factory,
                           desc='Building sources', weight=weight,
                           timing_label='Build msims' if self.logging_time else None)
        if store:
            self._apply_mappings(msims)
            self._msims = msims
        return msims

    def scaled_source_msims(self, indices, scale):
        """Sources `indices` as pre-processing starts from them, without building the others: from the level `scale`
        needs, binned down to it where the pyramid stops short."""
        built = self._scaled_msims.get(self._scaled_msims_key(scale, ends_only=True))
        if built is not None:
            msims = [built[index] for index in indices]
        else:
            def build_msim(index):
                from_level = get_level_from_scale(self.sources[index], scale)[0] if scale else 0
                return self._build_source_msim(index, from_level, ends_only=True)
            # kept per scale: flat-field models and normalisation stats each read a group's sources
            source_msims = self._scaled_source_msims.setdefault(scale, {})
            missing = [index for index in indices if index not in source_msims]
            source_msims.update(zip(missing, thread_map(build_msim, missing, default_msim_build_workers)))
            msims = [source_msims[index] for index in indices]
        if scale and scale != 1:
            msims = select_msim_subpyramid_at_scale(msims, [self.sources[index] for index in indices], scale,
                                                    bin_shortfall=True)
        return msims

    def _apply_mappings(self, msims):
        """The registration's corrections (self.mappings) chained onto each msim's source transform, as
        register_global writes them: for msims built after a saved registration was loaded."""
        for index, mapping in (self.mappings or {}).items():
            msi_utils.set_affine_transform(msims[index], mapping, transform_key=self.reg_transform_key,
                                           base_transform_key=self.source_transform_key)

    def source_geometry(self, transform_key, promote_z=False, z_scale=None, progress=None):
        """Each source's translation and level-0 stack properties under transform_key, from its metadata and
        self.mappings alone, building no msim: the registered transform is the mapping chained onto the source
        transform, as _apply_mappings writes it, identity for a source without one. `progress` is called a source."""
        translations, stack_props = [], []
        for index, (source, position, transform) in enumerate(zip(self.sources, self.positions,
                                                                   self._msim_transforms)):
            translation = dict(position)
            if promote_z and 'z' not in translation:
                translation['z'] = index * (z_scale or 1)
            props = build_source_stack_props(source, self._msim_output_order, translation, transform,
                                             self.source_transform_key, z_scale=self._msim_z_scale,
                                             promote_z=promote_z)
            if transform_key == self.reg_transform_key:
                matrix = np.asarray(props['transform'], dtype=float)
                mapping = (self.mappings or {}).get(index)
                if mapping is None:
                    matrix = np.eye(len(matrix))
                else:
                    mapping_matrix = widened_affine_matrix(mapping) if len(matrix) == 4 else np.asarray(
                        mapping.sel(t=0) if 't' in mapping.dims else mapping, dtype=float)
                    matrix = mapping_matrix @ matrix
                props['transform'] = props['transform'].copy(data=matrix)
            translations.append(translation)
            stack_props.append(props)
            if progress is not None:
                progress()
        return translations, stack_props

    def reset(self):
        self.state = RegState.UNINIT
        self.source_transform_key = 'source_metadata'
        self.reg_transform_key = 'registered'
        self.transition_transform_key = 'transition'
        self.source_metadata = {}
        self.extra_metadata = {}
        self.msims = []
        self.register_msims = None
        self.pair_msims = None
        self.pairs = []
        self.pairs_graph = None
        self._scaled_msims = {}
        self._scaled_source_msims = {}
        self.sources = []
        # set on each source read, cleared once shown
        self.missing_metadata_summary = ''
        self.metrics = {}
        self.group_pairs = {}
        self.register_indices = None
        # per source index, for fusion and the view as well as registration
        self._flatfield_key = None
        self._flatfield_models = {}
        self._flatfield_lock = threading.Lock()
        # what register_msims were made with, and global normalisation's stats per (settings, source group)
        self.preprocessed_settings = None
        self._global_normalise_stats = {}
        # per source index: the registration's correction on top of the source transform; absent means identity
        self.mappings = None
        self.input_output_params = {}

    def is_initialised(self):
        return self.state.value >= RegState.INIT.value

    def is_pairs_registered(self):
        return self.state.value >= RegState.PAIRS_REG.value

    def is_global_registered(self):
        return self.state.value >= RegState.GLOBAL_REG.value

    def is_fused(self):
        return self.state.value >= RegState.FUSED.value

    def init_params(self, params_general, params, label='', input_path=None, global_rotation=None, global_center=None):
        """A CLI operation: its settings in the project template's sections (input_output, pre_processing,
        registration, fusion), as a project file has them, with the run's general settings."""
        check_operation_layout(params)
        self.params_general = params_general
        self.params = params
        self.input_output_params = params.get('input_output', {})
        self.preprocess_params = params.get('pre_processing', {})
        self.register_params = params.get('registration', {})
        self.fusion_params = params.get('fusion', {})
        input_output = self.input_output_params
        if input_path is None:
            input_path = input_output.get('input_path')

        return self.init(operation=self.register_params.get('operation', ''), label=label, input_path=input_path,
                         pairing=self.register_params.get('pairing', ''),
                         input_labels=input_output.get('labels'),
                         output_path=input_output.get('output_path'),
                         source_metadata=source_metadata_from_params(input_output),
                         extra_metadata=extra_metadata_from_params(input_output),
                         global_rotation=global_rotation, global_center=global_center,
                         overwrite=input_output.get('overwrite', False), clear=input_output.get('clear', False),
                         ui=params_general.get('ui', ''),
                         verbose=params_general.get('verbose', False), debug=params_general.get('debug', False))

    @property
    def is_stack(self):
        """Stack registration: pair consecutive views, and treat the set as a z-stack rather than a tiled plane.
        Selected by `pairing: stack` only, as the plugin never sets an operation."""
        return 'stack' in self.pairing

    def init(self, operation='', label='', input_path=None, input_labels=None, output_path=None,
             source_metadata={}, extra_metadata={}, global_rotation=None, global_center=None,
             overwrite=True, clear=False, ui='', verbose=False, debug=False, pairing=''):
        self.overwrite = overwrite
        self.clear = clear
        self.ui = ui
        self.verbose = verbose
        self.debug = debug
        self.logging_dask = self.verbose
        self.logging_time = self.verbose
        self.mpl_ui = ('mpl' in self.ui or 'plot' in self.ui)
        self.operation = operation
        self.pairing = (pairing or '').lower()
        self.fileset_label = label
        self.global_rotation = global_rotation
        self.global_center = global_center
        self.source_transform_key = 'source_metadata'
        self.reg_transform_key = 'registered'
        self.transition_transform_key = 'transition'
        self.msims = []
        self.sources = []
        self.state = RegState.INIT
        self.start_time = datetime.now()

        self.input_path = input_path
        # a folder stands for its files, but a zarr folder is one image
        patterns = [os.path.join(path, '*') if os.path.isdir(path) and '.zar' not in path.lower() else path
                    for path in ensure_list(input_path)]
        self.input_dir = pattern_base_dir(patterns[0])
        self.filenames = [Path(path).as_posix() for path in dir_regex(patterns)]
        if not self.filenames:
            return False
        item_files = [{Path(path).as_posix() for path in dir_regex(pattern)} for pattern in patterns]
        # the input path item each file came from: a channel each, in a channel registration
        self.input_groups = [next(item for item, files in enumerate(item_files) if filename in files)
                             for filename in self.filenames]

        if input_labels:
            self.file_labels = input_labels
        else:
            self.file_labels = get_unique_file_labels(self.filenames)

        self.source_metadata = source_metadata
        self.extra_metadata = extra_metadata

        try:
            output_path = output_path.format_map(split_numeric_dict(self.filenames[0]))
            self.output = os.path.join(self.input_dir, output_path)    # preserve trailing slash: do not use os.path.normpath()
            output_dir = os.path.dirname(self.output)
            if self.clear:
                shutil.rmtree(output_dir, ignore_errors=True)
            if not os.path.exists(output_dir):
                os.makedirs(output_dir, exist_ok=True)
        except Exception as e:
            logging.error(f"Error initializing output directory: {e}")
            return False

        return True

    def run(self):
        with ProgressBar(minimum=60, dt=1) if self.logging_dask else nullcontext():
            return self._run()

    def _run(self):
        filenames = self.filenames
        file_labels = self.file_labels

        output = self.output
        operation = self.operation
        source_metadata = self.source_metadata
        extra_metadata = self.extra_metadata
        z_scale = get_metadata_z_scale(extra_metadata)
        channels = extra_metadata.get('channels', []) if isinstance(extra_metadata, dict) else []
        normalise_orientation = 'norm' in source_metadata
        fusion_params = self.fusion_params
        overlap_threshold = self.register_params.get('overlap_threshold', 0.5)
        save_images = fusion_params.get('save_images', True)

        output_format = fusion_params.get('format', zarr_extension)
        output_block_size = parse_block_size(fusion_params.get('block_size'))
        output_compression = fusion_params.get('compression')
        output_pyramid_downsample = fusion_params.get('pyramid_downsample', 2)
        output_npyramid_add = fusion_params.get('npyramid_add', 0)
        output_ome_version = fusion_params.get('ome_version') or default_ome_zarr_version

        if len(filenames) == 0:
            logging.warning('Skipping (no images)')
            return False

        output_filename = operation_to_past_participle(operation)

        self.check_progress(output_filename, output_format)

        if self.is_fused() and not self.overwrite:
            logging.warning(f'Skipping existing output {output_filename}')
            return False

        with Timer('init sims', verbose=self.logging_time):
            self.init_data()
            msims = self.msims

        is_3d = (self.sources[0].get_size().get('z', 0) > 1)
        is_stack = self.is_stack
        is_simple_stack = is_stack and not is_3d
        is_transition = ('transition' in operation)
        is_channel_overlay = (len(channels) > 1)
        if not z_scale:
            z_scale = self.scales[0].get('z', 1)

        with Timer('pre-process', verbose=self.logging_time):
            self.preprocess(msims, **self.preprocess_params)
            register_msims, register_indices = self.register_msims, self.register_indices

        self.init_progress(output_filename, output_format)

        data = []
        mappings_header = ['id', 'filename', 'x_pixels', 'y_pixels', 'z_pixels', 'x', 'y', 'z', 'rotation']
        for label, filename, position, rotation, scale in zip(file_labels, self.filenames, self.positions, self.rotations, self.scales):
            position_pixels = {dim: position[dim] / float(scale.get(dim, 1)) for dim in position.keys()}
            row = [label] + [filename] + dict_to_xyz(position_pixels, add_zeros=True) + dict_to_xyz(position, add_zeros=True) + [rotation]
            data.append(row)
        export_csv(output + prereg_mappings_name, data, header=mappings_header)

        if len(filenames) == 1 and save_images and not 'register' in operation:
            logging.warning('Skipping operation (single image)')
            sim = msi_utils.get_sim_from_msim(msims[0], scale='scale0')
            self.save(output_filename, sim, translations0=self.positions,
                      format=output_format,
                      block_size=output_block_size,
                      pyramid_downsample=output_pyramid_downsample,
                      npyramid_add=output_npyramid_add,
                      ome_version=output_ome_version)
            return False

        _, has_overlaps = self.validate_overlap(msims, file_labels, is_stack=is_simple_stack,
                                                expect_large_overlap=is_simple_stack or is_channel_overlay)
        overall_overlap = np.mean(has_overlaps)
        if overall_overlap < overlap_threshold:
            raise ValueError(f'Not enough overlap: {overall_overlap * 100:.1f}%')

        if not self.is_global_registered() or self.overwrite:
            if 'register' in operation:
                with Timer('register', verbose=self.logging_time):
                    results = self.register(register_msims, register_indices, self.register_params)
                reg_result = results['reg_result']
                mappings = results['mappings']
                metrics = results['metrics']

            if 'register' in operation:
                logging.info(metrics['summary'])
                self.save_mappings_csv(mappings, normalise_orientation=normalise_orientation)
                self.save_residuals(results['residual_errors'])
                self.write_ro_crates([], {'general': self.params_general, **self.params})

                for reg_label, reg_item in reg_result.items():
                    if isinstance(reg_item, dict):
                        summary_plot = reg_item.get('summary_plot')
                        if summary_plot is not None:
                            figure, axes = summary_plot
                            summary_plot_filename = output + f'{reg_label}.pdf'
                            figure.savefig(summary_plot_filename)

        self.msims = msims
        registered_positions_filename = output + registered_positions_name
        if self.reg_transform_key in get_msim_transform_keys(msims[0]):
            transform_key = self.reg_transform_key
            with Timer('plot positions', verbose=self.logging_time):
                # plot_positions needs sims; a stack is stored 2D, so promoted for this plot to space its slices in z
                plot_msims = make_msims_3d(msims, z_scale, self.positions) if is_stack else msims
                plot_sims = [msi_utils.get_sim_from_msim(msim, scale='scale0') for msim in plot_msims]
                vis_utils.plot_positions(plot_sims, transform_key=transform_key,
                                         use_positional_colors=False, view_labels=file_labels, view_labels_size=3,
                                         show_plot=self.mpl_ui, output_filename=registered_positions_filename)
                plt_close()
        else:
            transform_key = self.source_transform_key

        logging.info('Exporting...')

        image_paths = []
        if save_images:
            if fusion_params.get('preview'):
                with Timer('create preview', verbose=self.logging_time):
                    self.create_preview('preview_' + output_filename,
                                        nom_msims=msims,
                                        transform_key=transform_key)

            if ('register' in operation or 'fuse' in operation or 'merge' in operation):
                with Timer('fuse image', verbose=self.logging_time):
                    fusion_method, output_spacing = self.fusion_settings()
                    zarr_output_filename = output_filename if 'zar' in output_format else None
                    # msims as stored (2D for a stack): fuse() promotes what it needs to
                    fuse_msims = self.flatfield_corrected_msims(msims, self.preprocess_params)
                    fused_msim, is_saved = self.fuse(fuse_msims, fusion_method=fusion_method, output_spacing=output_spacing,
                                                     transform_key=transform_key, output_filename=zarr_output_filename,
                                                     block_size=output_block_size, ome_version=output_ome_version)
                    self.state = RegState.FUSED
            else:
                fused_msim = msims
                is_saved = False

            # save_image() and save_video() only accept sims
            fused_sims = extract_sims_from_fused(fused_msim)

            if not is_saved or 'tif' in output_format:
                extra_output_format = output_format
                if is_saved:
                    extra_output_format = extra_output_format.replace('ome.zarr', '').replace('zar', '')
                logging.info('Saving fused image...')
                with Timer('save fused image', verbose=self.logging_time):
                    self.save(output_filename, fused_sims,
                              transform_key=transform_key, translations0=self.positions,
                              format = extra_output_format,
                              block_size = output_block_size,
                              compression = output_compression,
                              pyramid_downsample = output_pyramid_downsample,
                              npyramid_add = output_npyramid_add,
                              ome_version = output_ome_version)

            if 'tif' in output_format:
                filename = output_filename + tiff_extension
                image_paths.append(filename)
            if 'zar' in output_format:
                filename = output_filename + zarr_extension
                image_paths.append(filename)

        self.write_ro_crates(image_paths, {'general': self.params_general, **self.params})

        if is_transition:
            self.save_video(output, msims, fused_msim)

        return True

    def write_ro_crates(self, image_filenames, params):
        """A crate in each fused zarr for where it came from, and a workflow run crate over all output with the
        run's parameter values. Written once the registration is saved, and again over the fused images."""
        output_dir = os.path.dirname(self.output)
        end_time = datetime.now()
        try:
            software_version = importlib.metadata.version('muvis-align')
        except importlib.metadata.PackageNotFoundError:
            software_version = None
        software = {'name': 'muvis-align', 'url': MUVIS_ALIGN_URL, 'version': software_version}
        measurements = {name: {'value': value, 'description': f'Registration {name}, over the overlaps of all '
                                                              f'registered pairs'}
                        for name, value in self.summary_metrics().items()}
        sources = {'name': 'Source images', 'paths': self.filenames, 'list_file': 'source-images.csv',
                   'description': f'The {len(self.filenames)} source images registered by muvis-align'}
        # empty: fair-metadata-integration's default
        license = params.get('fusion', {}).get('license') or None
        for filename in image_filenames:
            if filename.endswith(zarr_extension):
                # the first and last sources stand for the acquisition: reading every source's metadata is too slow
                create_zarr_ro_crate(self.output + filename, source_paths=[sources],
                                     acquisition_metadata=[self.sources[0].metadata, self.sources[-1].metadata],
                                     measurements=measurements,
                                     description=f'OME-Zarr image fused by muvis-align from {len(self.filenames)} '
                                                 f'sources',
                                     derivation={'name': 'Fusion to OME-Zarr', 'software': software,
                                                 'end_time': end_time},
                                     license=license)
        result_paths = [self.output + filename for filename in image_filenames] + sorted(self.saved_progress_paths())
        template = importlib.resources.files('muvis_align').joinpath(NAPARI_PROJECT_TEMPLATE)
        with importlib.resources.as_file(template) as template_path:
            create_ro_crate(output_dir, str(template_path), [sources], result_paths, parameters=params,
                            name=f'muvis-align run: {get_filetitle(os.path.normpath(output_dir))}',
                            description=f'Registration and fusion of {len(self.filenames)} source images by '
                                        f'muvis-align, with its settings and results',
                            run_name='muvis-align run', start_time=self.start_time, end_time=end_time,
                            software=software, license=license)

    def middle_section_indices(self):
        """The files of the middle section, by section number (a label as S000 or SBEMimage's s00538), else by folder;
        None if neither tells sections apart."""
        # tile and slice indices look alike in a name: only the label tells them apart
        labelled = [find_labelled_numbers(Path(filename).name) for filename in self.filenames]
        candidates = [[numbers.get(label) for numbers in labelled] for label in section_number_labels]
        candidates.append([Path(filename).parent for filename in self.filenames])
        for keys in candidates:
            sections = sorted(set(keys)) if None not in keys else []
            if len(sections) > 1:
                # the step napari's dims start on: int((nsteps - 1) / 2)
                middle = sections[(len(sections) - 1) // 2]
                return [index for index, key in enumerate(keys) if key == middle]
        return None

    def section_registration(self, indices):
        """A registration of only the files at `indices`, labelled as here: to show them before the rest are read."""
        section = MVSRegistration()
        section.init(operation=self.operation, label=self.fileset_label,
                     input_path=[self.filenames[index] for index in indices],
                     input_labels=[self.file_labels[index] for index in indices], output_path=self.output,
                     overwrite=False, pairing=self.pairing, verbose=self.verbose)
        return section

    def init_sources(self, progress_factory=None):
        def build_source(index):
            return create_image_source(self.filenames[index], self.source_metadata, file_label=self.file_labels[index],
                                       index=index, transform_key=self.source_transform_key)

        self.sources = pooled_map(build_source, len(self.filenames), default_source_init_workers, progress_factory,
                                  desc='Initialising sources', timing_label='Init sources' if self.logging_time else None)
        self.missing_metadata_summary = get_missing_metadata_summary(self.sources)
        if self.missing_metadata_summary:
            logging.warning(self.missing_metadata_summary)

    def init_data(self, source_metadata={}, extra_metadata={}, target_scale=None, store=True,
                  progress_factory=None):
        if not source_metadata:
            source_metadata = self.source_metadata
        if not extra_metadata:
            extra_metadata = self.extra_metadata
        source_metadata = import_metadata(source_metadata, input_path=self.input_path)
        extra_metadata = import_metadata(extra_metadata, input_path=self.input_path)
        source_metadata_changed = (source_metadata != self.source_metadata)
        self.source_metadata = source_metadata
        self.extra_metadata = extra_metadata
        z_scale = get_metadata_z_scale(source_metadata) or get_metadata_z_scale(extra_metadata)

        if len(self.filenames) == 0:
            raise ValueError('No input files')

        logging.info('Initialising sources...')
        if not self.sources or source_metadata_changed:
            self.init_sources(progress_factory=progress_factory)
        sources = self.sources
        source0 = sources[0]
        scales = []
        translations = []
        rotations = []
        levels = []
        rescales = []

        is_3d = (source0.get_size().get('z', 0) > 1)
        is_stack = self.is_stack
        output_order = 'zyx' if is_3d else 'yx'

        ndims = len(output_order)
        if source0.get_nchannels() > 1:
            if source0.is_rgb:
                output_order = output_order + 'c'
            else:
                output_order = 'c' + output_order

        last_z_position = None
        delta_zs = []
        # timed apart from init_sources(): these serial per-source passes are not in its pooled timing
        geometry_timer = Timer('init_data: resolve per-source geometry', verbose=self.logging_time)
        geometry_timer.start()
        for filename, source in zip(self.filenames, sources):
            # metadata ImageSource already resolved (fix_metadata); no pixel data is read
            # copies: the z and x/y filled in below are this run's, not the source's
            scale = dict(source.get_pixel_size())
            translation = dict(source.get_position())
            rotation = source.get_rotation()

            level = 0
            rescale = {}
            if target_scale:
                # Only downscaling
                level, rescale, scale = get_level_from_scale(source, target_scale)
            if 'z' in translation:
                z_position = translation['z']
            else:
                z_position = 0
            if last_z_position is not None and z_position != last_z_position:
                delta_zs.append(z_position - last_z_position)
            if self.global_rotation is not None:
                rotation = self.global_rotation

            scales.append(scale)
            translations.append(translation)
            rotations.append(rotation)
            levels.append(level)
            rescales.append(rescale)
            last_z_position = z_position
        geometry_timer.record()
        geometry_timer.get_total_time()

        dimension = getattr(self, 'registration_dimension', None) or self.input_output_params.get('registration_dimension')
        if not is_3d and self.is_channel_overlay(dimension, extra_metadata):
            # a z per channel is a focus offset, not stack planes: all channels on the first one's plane
            channel_zs = {}
            for group, translation in zip(self.channel_groups(), translations):
                channel_zs.setdefault(group, set()).add(translation.get('z'))
            if all(len(zs) == 1 for zs in channel_zs.values()) and 'z' in translations[0]:
                for translation in translations:
                    translation['z'] = translations[0]['z']

        if 'z' in output_order and z_scale is None:
            if len(delta_zs) > 0:
                z_scale = np.min(delta_zs)
            else:
                z_scale = 1

        if 'norm' in source_metadata:
            sizes = [source.get_physical_size() for source in sources]
            center = {dim: 0 for dim in output_order}
            if 'center' in source_metadata:
                if 'global' in source_metadata:
                    center = self.global_center
                else:
                    center = {dim: float(np.mean([translation[dim] for translation in translations])) for dim in translations[0]}
            translations, rotations = normalise_rotated_positions(translations, rotations, sizes, center, len(output_order))

        #translations = [np.array(translation) * 1.25 for translation in translations]

        # applies only to 3D sources: make_msims_3d() spaces 2D slices but skips a source already 3D
        increase_z_positions = is_stack

        z_position = 0
        final_scales = []
        final_translations = []
        transforms = []
        msims = [] if not store else None
        transform_timer = Timer(f'init_data: build per-source transforms'
                                f'{"" if store else " and msims"}', verbose=self.logging_time)
        transform_timer.start()
        for source, level, rescale, scale, translation, rotation, file_label in zip(
                sources, levels, rescales, scales, translations, rotations, self.file_labels):
            # transform #dimensions need to match
            if 'z' in output_order:
                if len(scale) > 0 and 'z' not in scale:
                    scale['z'] = abs(z_scale)
                if (len(translation) > 0 and 'z' not in translation) or increase_z_positions:
                    translation['z'] = z_position
                if increase_z_positions:
                    z_position += z_scale
            if rotation is None or 'norm' in source_metadata:
                # if positions are normalised, don't use rotation
                transform = None
            else:
                transform = param_utils.invert_coordinate_order(
                    create_transform(translation, rotation, matrix_size=ndims + 1)
                )
            if file_label in extra_metadata:
                transform2 = extra_metadata[file_label]
                if transform is None:
                    transform = np.array(transform2)
                else:
                    transform = np.array(combine_transforms([transform, transform2]))

            translation = complete_translation(translation)
            transforms.append(transform)
            if not store:
                # from the source's cached msim, adding only the run-level deltas no single source knows
                msim = build_source_msim(source, output_order, translation, transform, self.source_transform_key,
                                         z_scale=z_scale)
                msims.append(msim)
            final_scales.append(scale)
            final_translations.append(translation)
        transform_timer.record()
        transform_timer.get_total_time()

        if store:
            self.scales = final_scales
            self.positions = final_translations
            self.rotations = rotations
            self.state = RegState.SIMS_INIT
            # the msim build itself waits until self.msims is read
            self._msim_output_order = output_order
            self._msim_z_scale = z_scale
            self._msim_transforms = transforms
            self._msims = None
            # the geometry was re-resolved, so the per-scale builds are stale too
            self._scaled_msims = {}
            self._scaled_source_msims = {}
            return None

        return msims

    def has_saved_progress(self, output_filename=None, output_format=None):
        """Whether there is a saved registration to resume: the files check_progress() reads, without changing
        the state. Without an output filename only the mappings are checked."""
        pair_mappings_filename = self.output + self.input_output_params.get('pair_mappings', default_pair_mappings_name)
        mappings_filename = self.output + self.input_output_params.get('mappings', default_mappings_name)
        return bool((output_filename and self.output_exists(output_filename, output_format))
                    or os.path.exists(mappings_filename)
                    or os.path.exists(pair_mappings_filename))

    def saved_progress_paths(self, output=None):
        """The saved registration's existing files, in this run's output or another (e.g. before init() has run)."""
        output = os.path.join(output, '') if output else self.output
        paths = [output + self.input_output_params.get('pair_mappings', default_pair_mappings_name),
                 output + self.input_output_params.get('mappings', default_mappings_name),
                 output + self.input_output_params.get('mappings', default_mappings_tabular_name),
                 output + metrics_name,
                 output + residuals_name]
        return [path for path in set(paths) if os.path.exists(path)]

    def discard_saved_progress(self, output=None):
        """Removes the saved registration, which no longer matches the sources."""
        for path in self.saved_progress_paths(output):
            os.remove(path)
        logging.info(f'Saved registration in {output or self.output} discarded')

    def check_progress(self, output_filename, output_format):
        pair_mappings_filename = self.output + self.input_output_params.get('pair_mappings', default_pair_mappings_name)
        mappings_filename = self.output + self.input_output_params.get('mappings', default_mappings_name)
        if self.output_exists(output_filename, output_format):
            self.state = RegState.FUSED
        elif os.path.exists(mappings_filename):
            self.state = RegState.GLOBAL_REG
        elif os.path.exists(pair_mappings_filename):
            self.state = RegState.PAIRS_REG

    def init_progress(self, output_filename, output_format, progress_factory=None):
        pair_mappings_filename = self.output + self.input_output_params.get('pair_mappings', default_pair_mappings_name)
        mappings_filename = self.output + self.input_output_params.get('mappings', default_mappings_name)
        metrics_filename = self.output + metrics_name
        self.check_progress(output_filename, output_format)

        load_pairs = self.is_pairs_registered() and os.path.exists(pair_mappings_filename)
        self.group_pairs = {}
        # builds no msims: the graph comes from source geometry, mappings are applied as msims are built
        file_index = file_list_index_finder(self.filenames)

        if load_pairs:
            logging.info(f'Loading pair mapping from {pair_mappings_filename}')
            pairs = import_json(pair_mappings_filename)
            indexed_pair_transforms = {}
            indexed_qualities = {}
            indexed_bboxes = {}
            for key, value in pairs.items():
                key1, key2 = json.loads(key)
                if value.get('kind') == split_group_pair_kind:
                    # a label such as 'S000' would also match a file under S000/ as a tile pair
                    self.group_pairs[(key1, key2)] = {item: data for item, data in value.items() if item != 'kind'}
                else:
                    index1, index2 = file_index(key1), file_index(key2)
                    if index1 is not None and index2 is not None:
                        indexed_key = index1, index2
                        indexed_pair_transforms[indexed_key] = (
                            xaffine_from_matrix(value['mapping'], with_t=True))
                        indexed_qualities[indexed_key] = np.array(value.get(default_quality_key, 0))
                        if 'bbox' in value:
                            indexed_bboxes[indexed_key] = data_array_from_values(value['bbox'])
            # not per-source-divisible, but long enough to report as two steps
            progress_context = progress_phase(progress_factory, total=2, desc='Loading pair registration')
            with progress_context as pbar:
                self.pair_msims = self._msims
                self.pairs = list(indexed_pair_transforms.keys())
                self.metrics = {
                    'summary': {default_transform_key: {default_quality_key: np.mean(list(indexed_qualities.values()))}},
                    'pairs': {key: {default_transform_key: {default_quality_key: value.item()}}
                              for key, value in indexed_qualities.items()}
                }
                if pbar is not None:
                    pbar.update(1)
                # built directly from the saved pairs, rather than multiview_stitcher rediscovering each overlap;
                # the saved bboxes give the overlap weight
                overlaps = {key: float(np.prod([abs(high - low) for low, high
                                                in zip(*np.array(value).reshape(2, -1))]))
                            for key, value in indexed_bboxes.items()}
                self.pairs_graph = build_pairs_graph(self.source_geometry(self.source_transform_key)[1], self.pairs,
                                                     overlaps=overlaps)
                if pbar is not None:
                    pbar.update(1)
            nx.set_edge_attributes(self.pairs_graph, indexed_pair_transforms, default_transform_key)
            nx.set_edge_attributes(self.pairs_graph, indexed_qualities, default_quality_key)
            nx.set_edge_attributes(self.pairs_graph, indexed_bboxes, 'bbox')

        if self.is_global_registered():
            logging.info(f'Loading global mapping from {mappings_filename}')

            mappings = read_transforms(mappings_filename)
            missing = [filename for filename in self.filenames if find_file_dict_item(mappings, filename) is None]
            if missing:
                # mappings for a different fileset (e.g. a shared output dir) can't be trusted as this one's
                logging.warning(f'{mappings_filename} has no mapping for {len(missing)} of '
                                f'{len(self.filenames)} files (e.g. {missing[0]}) - ignoring it')
                self.state = RegState.INIT
            else:
                progress_context = progress_phase(progress_factory, total=len(self.filenames),
                                                  desc='Loading global registration')
                self.mappings = {}
                with progress_context as pbar:
                    for index, filename in enumerate(self.filenames):
                        # stored as registration produced it - 2D for a stack, whose msims are stored 2D too
                        self.mappings[index] = xaffine_from_matrix(find_file_dict_item(mappings, filename))
                        if pbar is not None:
                            pbar.update(1)
                if self._msims is not None:
                    self._apply_mappings(self._msims)
                self.pair_msims = self._msims
                # a project saved before metrics.csv reopens without metrics
                rows = import_csv(metrics_filename) if os.path.exists(metrics_filename) else []
                indexed_metrics = {}
                for row in rows:
                    # an empty cell: a metric this pair has no value for
                    values = {metric: float(value) for metric, value in row.items()
                              if metric not in pair_table_columns and value not in ('', None)}
                    # a split group pair's row has no filenames, and an empty name would match any file
                    if not row.get('filename1'):
                        labels = (row['id1'], row['id2'])
                        if labels in self.group_pairs:
                            # its quality is the pair mappings file's
                            self.group_pairs[labels]['metrics'] = {metric: value for metric, value in values.items()
                                                                   if metric != default_quality_key}
                    else:
                        index1, index2 = file_index(row['filename1']), file_index(row['filename2'])
                        if index1 is not None and index2 is not None:
                            indexed_metrics[index1, index2] = values
                # each metric's mean under the registered key, as a global registration's own summary
                metric_keys = list(dict.fromkeys(metric for value in indexed_metrics.values() for metric in value))
                summary = {metric: float(np.mean([value[metric] for value in indexed_metrics.values()
                                                  if value.get(metric) is not None]))
                           for metric in metric_keys}
                self.metrics = {
                    'summary': {self.reg_transform_key: summary},
                    'pairs': {key: {self.reg_transform_key: value} for key, value in indexed_metrics.items()}
                }
        if self.group_pairs and self.metrics:
            self.metrics['group_pairs'] = self.group_pair_metrics()

    def validate_overlap(self, items, labels, is_stack=False, expect_large_overlap=False):
        # sims or msims: only position/size metadata is read
        sims = sims_from_sims_or_msims(items)
        min_dists = []
        has_overlaps = []
        n = len(sims)
        positions = [get_sim_position_final(sim, get_center=True) for sim in sims]
        sizes = [float(np.linalg.norm(list(get_sim_physical_size(sim).values()))) for sim in sims]
        for i in range(n):
            norm_dists = []
            # check if only single z slices
            if is_stack:
                if i + 1 < n:
                    compare_indices = [i + 1]
                else:
                    compare_indices = []
            else:
                compare_indices = range(n)
            for j in compare_indices:
                if not j == i:
                    distance = math.dist(positions[i].values(), positions[j].values())
                    norm_dist = distance / np.mean([sizes[i], sizes[j]])
                    norm_dists.append(norm_dist)
            if len(norm_dists) > 0:
                norm_dist = min(norm_dists)
                min_dists.append(float(norm_dist))
                if norm_dist >= 1:
                    logging.warning(f'{labels[i]} has no overlap')
                    has_overlaps.append(False)
                elif expect_large_overlap and norm_dist > 0.5:
                    logging.warning(f'{labels[i]} has small overlap')
                    has_overlaps.append(False)
                else:
                    has_overlaps.append(True)
        return min_dists, has_overlaps

    def preprocess(self, msims, scale=None,
                   flatfield_quantiles=None, normalisation=None, gaussian_sigma=None, filter_foreground=False,
                   progress_factory=None,
                   **kwargs):
        if kwargs:
            # not fatal, but visible: a mistyped option otherwise silently leaves pre-processing at its defaults
            logging.warning('Ignoring unknown pre-processing option(s):'
                            f' {", ".join(sorted(str(key) for key in kwargs))}')

        do_normalisation = normalisation_enabled(normalisation)
        # a factor, as a number or the text of a text field, or a pixel size with its unit ('10um')
        scale = parse_scale(scale, default=None)
        params = {'scale': scale, 'flatfield_quantiles': flatfield_quantiles, 'normalisation': normalisation,
                  'gaussian_sigma': gaussian_sigma, 'filter_foreground': filter_foreground}
        settings = self.preprocess_settings(params)

        def count_progress_steps():
            n_steps = 0
            if scale and scale != 1:
                n_steps += 1
            if filter_foreground:
                # foreground map + final foreground filtering
                n_steps += 2
            if flatfield_quantiles:
                n_steps += 1
            if gaussian_sigma:
                n_steps += 1
            if do_normalisation:
                n_steps += 1
            return max(n_steps, 1)

        def update_progress():
            if pbar is not None:
                pbar.update(1)

        progress_context = progress_phase(progress_factory, total=count_progress_steps(), desc='Pre-processing')

        with progress_context as pbar:
            modified = False
            source_indices = range(len(msims))

            if scale and scale != 1:
                # native levels where the pyramid reaches the scale, binned down to it where it does not
                msims = select_msim_subpyramid_at_scale(msims, self.sources, scale, bin_shortfall=True)
                modified = True
                update_progress()

            if filter_foreground:
                foreground_map = calc_foreground_map(msims)
                modified = True
                update_progress()
            else:
                foreground_map = None

            if flatfield_quantiles:
                logging.info('Flat-field correction...')
                flatfield_params = {'flatfield_quantiles': flatfield_quantiles, 'scale': scale,
                                    'filter_foreground': filter_foreground}
                self.flatfield_models(flatfield_params, msims=msims, foreground_map=foreground_map)
                msims = self.flatfield_corrected_msims(msims, flatfield_params)
                modified = True
                update_progress()

            if gaussian_sigma:
                logging.info('Applying Gaussian filtering...')
                msims = self._gaussian_filtered_msims(msims, gaussian_sigma)
                modified = True
                update_progress()

            if do_normalisation:
                use_global = ('global' in str(normalisation).lower())
                if use_global:
                    logging.info('Normalising (global)...')
                else:
                    logging.info('Normalising (individual)...')
                if use_global:
                    groups = self.source_groups()
                    msims_of = dict(zip(source_indices, msims))
                    group_stats = {group: self._global_normalise_stat(group, params, msims_of)
                                   for group in set(groups)}
                    norm_stats, norm_dtypes = zip(*(group_stats[groups[index]] for index in source_indices))
                else:
                    norm_stats, norm_dtype = calc_normalise_stats(msims, use_global=False)
                    norm_dtypes = [norm_dtype] * len(msims)
                msims = self._normalised_msims(msims, norm_stats, norm_dtypes)
                modified = True
                update_progress()

            if filter_foreground:
                logging.info('Filtering foreground images...')
                # tile_vars = np.array([np.asarray(np.std(sim)).item() for sim in sims])
                # threshold1 = np.mean(tile_vars)
                # threshold2 = np.median(tile_vars)
                # threshold3, _ = cv.threshold(np.array(tile_vars).astype(np.uint16), 0, 1, cv.THRESH_OTSU)
                # threshold = min(threshold1, threshold2, threshold3)
                # foregrounds = (tile_vars >= threshold)
                new_msims = [msim for msim, is_foreground in zip(msims, foreground_map) if is_foreground]
                logging.info(f'Foreground images: {len(new_msims)} / {len(msims)}')
                indices = np.where(foreground_map)[0]
                msims = new_msims
                modified = True
                update_progress()
            else:
                indices = range(len(msims))

        self.register_msims = msims
        self.register_indices = indices
        self.preprocessed_settings = settings
        return msims, indices, modified

    @staticmethod
    def preprocess_settings(params):
        """Pre-processing params, parsed: equal for settings that give the same pixels, however they were written."""
        quantiles, scale, filter_foreground = MVSRegistration.flatfield_settings(params)
        normalisation = params.get('normalisation')
        if normalisation_enabled(normalisation):
            normalisation = 'global' if 'global' in str(normalisation).lower() else 'individual'
        else:
            normalisation = None
        gaussian_sigma = params.get('gaussian_sigma')
        return (quantiles, scale if scale != 1 else None, normalisation,
                float(gaussian_sigma) if gaussian_sigma else None, filter_foreground)

    def register_msims_current(self, params):
        """Whether register_msims were made with these pre-processing settings."""
        return self.register_msims is not None and self.preprocessed_settings == self.preprocess_settings(params)

    def needs_full_preprocess(self, params):
        """Whether preprocess_sources() cannot give these settings' result: it needs every source."""
        return self.preprocess_settings(params)[4]

    def _global_normalise_stat(self, group, params, msims_of=None):
        """Group `group`'s global normalisation (mean, stddev) and dtype at these settings, over its sources as
        pre-processed up to normalisation: from `msims_of` ({source index: msim}) or pre-processed for it."""
        key = (self.preprocess_settings(params), group)
        if key not in self._global_normalise_stats:
            members = [index for index, member_group in enumerate(self.source_groups()) if member_group == group]
            if msims_of is not None:
                group_msims = [msims_of[index] for index in members]
            else:
                group_msims = self.preprocess_sources(members, {**params, 'normalisation': 'none'})
            stats, dtype = calc_normalise_stats(group_msims, use_global=True)
            self._global_normalise_stats[key] = (stats[0], dtype)
        return self._global_normalise_stats[key]

    def preprocess_sources(self, indices, params):
        """Sources `indices` pre-processed as preprocess() makes them, reading no other source's pixels; only where
        needs_full_preprocess() is False."""
        settings = self.preprocess_settings(params)
        quantiles, scale, normalisation, gaussian_sigma, _ = settings
        msims = self.scaled_source_msims(indices, scale)
        if quantiles:
            models = [self.flatfield_model(index, params) for index in indices]
            msims = [apply_msim_flatfield_model(msim, model) if model is not None else msim
                     for msim, model in zip(msims, models)]
        if gaussian_sigma:
            msims = self._gaussian_filtered_msims(msims, gaussian_sigma)
        if normalisation == 'global':
            groups = self.source_groups()
            norm_stats, norm_dtypes = zip(*(self._global_normalise_stat(groups[index], params) for index in indices))
            msims = self._normalised_msims(msims, norm_stats, norm_dtypes)
        elif normalisation:
            norm_stats, norm_dtype = calc_normalise_stats(msims, use_global=False)
            msims = self._normalised_msims(msims, norm_stats, [norm_dtype] * len(msims))
        return msims

    def _gaussian_filtered_msims(self, msims, gaussian_sigma):
        # sigma is in pixels at the finest native pixel size: every image and level gets the same physical blur
        finest = {dim: min(scale[dim] for scale in self.scales if dim in scale)
                  for dim in 'xy' if any(dim in scale for scale in self.scales)}

        def level_func(level_sim, scale_key):
            level_scale = si_utils.get_spacing_from_sim(level_sim)
            ratios = [finest[dim] / level_scale[dim] for dim in finest if dim in level_scale]
            return gaussian_filter_sim(level_sim, self.source_transform_key,
                                       gaussian_sigma * (np.mean(ratios) if ratios else 1))
        return [map_msim_levels(msim, level_func) for msim in msims]

    def _normalised_msims(self, msims, norm_stats, norm_dtypes):
        normalised = []
        for msim, (norm_min, norm_range), norm_dtype in zip(msims, norm_stats, norm_dtypes):
            # the scale0 min/range at every level, so the whole pyramid is normalised alike
            def level_func(level_sim, scale_key, norm_min=norm_min, norm_range=norm_range, norm_dtype=norm_dtype):
                return normalise_sim(level_sim, self.source_transform_key, norm_min, norm_range, norm_dtype)
            normalised.append(map_msim_levels(msim, level_func))
        return normalised

    @staticmethod
    def flatfield_settings(params):
        """(quantiles, scale, filter_foreground) of pre-processing params, parsed: what a flat-field model depends on."""
        quantiles = params.get('flatfield_quantiles')
        if isinstance(quantiles, str):
            quantiles = [quantile for quantile in quantiles.split(',') if quantile.strip()]
        if not isinstance(quantiles, (list, tuple)):
            quantiles = [quantiles] if quantiles else []
        return (tuple(float(quantile) for quantile in quantiles), parse_scale(params.get('scale'), default=None),
                bool(params.get('filter_foreground')))

    def source_groups(self):
        """Each source's pre-processing group: the input path item it came from (one item: a single group)."""
        input_groups = getattr(self, 'input_groups', None)
        return list(input_groups) if input_groups is not None else [0] * len(self.sources)

    def flatfield_groups(self):
        """Source indices per flat-field model: a section's sources of one group that cover the same field (pixel size
        x image size), as images of a different field share no shading."""
        section_of = {index: section for section, indices in enumerate(group_sims_by_z(self.positions, self.positions))
                      for index in indices}
        groups = {}
        for index, (group, source, scale) in enumerate(zip(self.source_groups(), self.sources, self.scales)):
            size = source.get_size()
            field = tuple(float(f'{size[dim] * scale[dim]:.4g}') for dim in 'yx' if dim in size and dim in scale)
            groups.setdefault((section_of.get(index), group, field), []).append(index)
        return list(groups.values())

    def _flatfield_group_of(self, settings):
        """{source index: flat-field group} the models are kept by, emptied when the settings or sources change."""
        key = (settings, id(self.positions))
        if self._flatfield_key != key:
            self._flatfield_key, self._flatfield_models = key, {}
            self._flatfield_groups = self.flatfield_groups()
            self._flatfield_group_index = {index: group for group, indices in enumerate(self._flatfield_groups)
                                           for index in indices}
        return self._flatfield_group_index

    def _calc_flatfield_groups(self, groups, settings, msims=None, foreground_map=None):
        """Models of flat-field `groups`, from `msims` (every source at the pre-processing scale) or their own tiles
        built at that scale as pre-processing builds them; None for a group of too few images."""
        quantiles, scale, filter_foreground = settings
        indices = [index for group in groups for index in self._flatfield_groups[group]]
        if msims is None:
            msims = dict(zip(indices, self.scaled_source_msims(indices, scale)))
        if filter_foreground and foreground_map is None:
            # the tiles kept out of the quantiles are chosen over every source: groups are then computed all at once
            foreground_map = calc_foreground_map([msims[index] for index in range(len(self.sources))])
        for group in groups:
            group_indices = self._flatfield_groups[group]
            if len(group_indices) < flatfield_min_images:
                logging.warning(f'No flat-field correction for {len(group_indices)} image(s) of one field'
                                f' (at least {flatfield_min_images} needed):'
                                f' {", ".join(self.file_labels[index] for index in group_indices[:5])}')
                self._flatfield_models[group] = None
            else:
                self._flatfield_models[group] = calc_msims_flatfield_model(
                    [msims[index] for index in group_indices], quantiles,
                    foreground_map=[foreground_map[index] for index in group_indices] if foreground_map is not None
                    else None)

    def flatfield_model(self, index, params):
        """Source `index`'s flat-field model, its group's computed from that group's tiles alone when first asked:
        None without quantiles or for too few images, and with foreground filtering until flatfield_models() has
        computed every group."""
        settings = self.flatfield_settings(params)
        if not settings[0]:
            return None
        with self._flatfield_lock:
            group = self._flatfield_group_of(settings)[index]
            if group not in self._flatfield_models and not settings[2]:
                self._calc_flatfield_groups([group], settings)
            return self._flatfield_models.get(group)

    def flatfield_models(self, params, msims=None, foreground_map=None, progress_factory=None):
        """Every source's flat-field model (None without quantiles), the sections not computed yet computed together,
        as a progress phase: from `msims` (every source at the pre-processing scale) if given, else from their tiles."""
        settings = self.flatfield_settings(params)
        if not settings[0]:
            return None
        with self._flatfield_lock:
            group_of = self._flatfield_group_of(settings)
            missing = self._missing_flatfield_groups(settings)
            if missing:
                logging.info(f'Flat-field model of {len(missing)} group(s)...')
                if msims is not None:
                    msims = dict(enumerate(msims))
                with progress_phase(progress_factory, total=1, desc='Flat-field model') as pbar:
                    self._calc_flatfield_groups(missing, settings, msims, foreground_map)
                    if pbar is not None:
                        pbar.update(1)
            return [self._flatfield_models[group_of[index]] for index in range(len(self.sources))]

    def flatfield_pending(self, params):
        """Whether flatfield_models() would compute a model, and so report a progress phase of its own."""
        settings = self.flatfield_settings(params)
        if not settings[0]:
            return False
        with self._flatfield_lock:
            self._flatfield_group_of(settings)
            return bool(self._missing_flatfield_groups(settings))

    def _missing_flatfield_groups(self, settings):
        missing = [group for group in range(len(self._flatfield_groups)) if group not in self._flatfield_models]
        # foreground filtering chooses its tiles over every source, so all groups are computed together
        return list(range(len(self._flatfield_groups))) if missing and settings[2] else missing

    def flatfield_corrected_msims(self, msims, params, progress_factory=None):
        """msims (every source, in source order) flat-field corrected by flatfield_models(); unchanged without one."""
        models = self.flatfield_models(params, progress_factory=progress_factory)
        if models is None:
            return msims
        return [apply_msim_flatfield_model(msim, model) if model is not None else msim
                for msim, model in zip(msims, models)]

    def create_registration_method(self, sim0, params={}, method=''):
        registration_method = None
        pairwise_reg_func_kwargs = None

        if not method:
            method = params.get('method',
                                params.get('name', ''))
        method = method.lower()
        method_kind = registration_method_kind(method)

        if method_kind == 'cpd':
            try:
                from muvis_align.registration_methods.RegistrationMethodCPD import RegistrationMethodCPD
            except ImportError:
                raise ImportError('probreg is required for CPD registration.')
            registration_method = RegistrationMethodCPD(sim0, params, self.debug)
            pairwise_reg_func = registration_method.registration
        elif method_kind == 'feature':
            if 'cv' in method:
                from muvis_align.registration_methods.RegistrationMethodCvFeatures import RegistrationMethodCvFeatures
                registration_method = RegistrationMethodCvFeatures(sim0, params, self.debug)
            else:
                from muvis_align.registration_methods.RegistrationMethodSkFeatures import RegistrationMethodSkFeatures
                registration_method = RegistrationMethodSkFeatures(sim0, params, self.debug)
            pairwise_reg_func = registration_method.registration
        elif method_kind == 'elastix':
            try:
                from itk import ElastixRegistrationMethod
            except ImportError:
                raise ImportError('ITK-Elastix is required for ITK-Elastix registration.')
            pairwise_reg_func = elastix_registration_or_identity
            # a translation to get close, then the project's transform type (multiview_stitcher's default ends rigid)
            transform_type = params.get('transform_type', 'rigid')
            pairwise_reg_func_kwargs = {'transform_types': ['translation'] if transform_type == 'translation'
                                        else ['translation', transform_type]}
        elif method_kind == 'ants':
            try:
                import ants
            except ImportError:
                raise ImportError('ANTsPy is required for ANTsPy registration.')
            pairwise_reg_func = registration.registration_ANTsPy
            pairwise_reg_func_kwargs = {
                'transform_types': ['Rigid'],
                "aff_random_sampling_rate": 0.5,
                "aff_iterations": (2000, 2000, 1000, 1000),
                "aff_smoothing_sigmas": (4, 2, 1, 0),
                "aff_shrink_factors": (16, 8, 2, 1),
            }
        else:
            pairwise_reg_func = registration.phase_correlation_registration

        self.registration_method = registration_method

        return method, pairwise_reg_func, pairwise_reg_func_kwargs

    def select_pair_overlap(self, msim1, msim2, params=None):
        """The pair's overlap images at the resolution multiview_stitcher would auto-select for it.

        Depends only on the source data and the registration channel, not the method or its parameters, so it can
        be cached across parameter changes. See register_overlap() to register on it.
        """
        params = params or {}
        # the binning and overlap helpers below do not handle a 'c' dim themselves
        if 'c' in get_msim_dims(msim1):
            chosen = {}
            msim1 = msi_utils.multiscale_sel_coords(
                msim1, {'c': resolve_registration_channel(msim1, params.get('channel', 0), chosen)})
            msim2 = msi_utils.multiscale_sel_coords(
                msim2, {'c': resolve_registration_channel(msim2, params.get('channel', 0), chosen)})

        sim1_0 = msi_utils.get_sim_from_msim(msim1, scale='scale0')
        sim2_0 = msi_utils.get_sim_from_msim(msim2, scale='scale0')
        registration_binning = registration.get_optimal_registration_binning(sim1_0, sim2_0)
        scale_key, remaining_binning = msi_utils.get_res_level_from_binning_factors(msim1, registration_binning)
        sim1 = msi_utils.get_sim_from_msim(msim1, scale=scale_key)
        sim2 = msi_utils.get_sim_from_msim(msim2, scale=scale_key)
        if max(remaining_binning.values()) > 1:
            binned = []
            for sim in (sim1, sim2):
                sim_b = sim.coarsen(remaining_binning, boundary='trim').mean().astype(sim.dtype)
                sim_b.attrs.update(copy.deepcopy(sim.attrs))
                binned.append(sim_b)
            sim1, sim2 = binned

        return get_overlap_images(sim1, sim2, self.source_transform_key)

    def register_overlap(self, overlap1, overlap2, sims_pixel_space, params=None):
        """Register an overlap crop from select_pair_overlap().

        Returns the physical-space transform, quality, and pairwise_reg_func's raw result (e.g. feature matches).
        """
        params = params or {}
        _, pairwise_reg_func, pairwise_reg_func_kwargs = self.create_registration_method(overlap1, params=params)
        pairwise_reg_func_kwargs = dict(pairwise_reg_func_kwargs or {})

        fixed_data = overlap1.compute() if hasattr(overlap1, 'compute') else overlap1
        moving_data = overlap2.compute() if hasattr(overlap2, 'compute') else overlap2
        result = pairwise_reg_func(fixed_data, moving_data, **pairwise_reg_func_kwargs)

        try:
            transform = affine_from_intrinsic_affine(result['affine_matrix'], sims_pixel_space, self.source_transform_key)
        except NotImplementedError:
            transform = result['affine_matrix']

        return transform, result.get('quality'), result

    def fusion_settings(self):
        """(method, output spacing) from the fusion section, the spacing defaulting to native as in the plugin."""
        return self.fusion_params.get('method', ''), self.fusion_params.get('spacing') or 'native'

    def create_fusion_method(self, fusion_method, sim0):
        if fusion_method is None:
            fusion_method = ''
        if 'compos' in fusion_method:
            fusion_method = None
            fuse_func = None
        elif 'exclus' in fusion_method:
            from muvis_align.fusion_methods.FusionMethodExclusive import FusionMethodExclusive
            fusion_method = FusionMethodExclusive(sim0, self.debug)
            fuse_func = fusion_method.fusion
        elif 'add' in fusion_method:
            from muvis_align.fusion_methods.FusionMethodAdditive import FusionMethodAdditive
            fusion_method = FusionMethodAdditive(sim0, self.debug)
            fuse_func = fusion_method.fusion
        else:
            from muvis_align.fusion_methods.FusionMethodAverage import FusionMethodAverage
            fusion_method = FusionMethodAverage(sim0, self.debug)
            fuse_func = fusion_method.fusion

        self.fusion_method = fusion_method

        return fuse_func

    def register(self, register_msims=None, register_indices=None, params=None):
        pair_results = self.register_pairs(register_msims=register_msims, register_indices=register_indices, params=params)
        self.save_pair_results(pair_results['pair_mappings'], metrics=pair_results['metrics'])
        results = self.register_global(self.pair_msims, register_indices=register_indices, params=params)
        self.save_group_pair_mappings()
        self.save_mappings(results['mappings'])
        self.save_metrics(results['metrics'])
        return results

    def register_pairs(self, register_msims=None, register_indices=None, params=None,
                       progress_factory=None):
        logging.info('Pair registration...')
        if register_indices is None:
            if self.register_indices is not None:
                register_indices = self.register_indices
            else:
                register_indices = range(len(self.msims))

        pairing = params.get('pairing', '').lower()
        n_parallel_pairwise_regs = params.get('n_parallel_pairwise_regs')
        # None or 0 is automatic; a yaml value may be a float
        n_parallel_pairwise_regs = int(n_parallel_pairwise_regs) if n_parallel_pairwise_regs else None
        logging.info(f"Pair registration settings: pairing {pairing or 'default'}, method {params.get('method')},"
                     f" transform {params.get('transform_type')}, channel {params.get('channel')},"
                     f" workers {n_parallel_pairwise_regs or f'automatic ({default_pair_workers})'},"
                     f" registration dimension {getattr(self, 'registration_dimension', None) or 'space'}")

        # the caller's params win over init()'s: the plugin's dropdown can change in between
        self.pairing = pairing
        is_3d = (self.sources[0].get_size().get('z', 0) > 1)
        is_stack = self.is_stack

        reg_channel = params.get('channel', 0)
        if isinstance(reg_channel, int):
            reg_channel_index = reg_channel
            reg_channel = None
        else:
            reg_channel_index = None

        if register_msims is None:
            register_msims = self.register_msims

        if is_stack and not is_3d:
            # register in 2D, pairing consecutive views: max-project every level that has z
            def level_func(level_sim, scale_key):
                return si_utils.max_project_sim(level_sim, dim='z') if 'z' in level_sim.dims else level_sim
            register_msims = [map_msim_levels(msim, level_func) for msim in register_msims]
            pairs = [(index, index + 1) for index in range(len(register_msims) - 1)]
        elif 'ortho' in pairing or SPLIT in pairing:
            # every source, never the filtered register_msims, so it matches self.positions; from metadata,
            # as building self.msims just to read their geometry is expensive
            with Timer('pair registration: source geometry', verbose=self.logging_time):
                shape_sims = [build_source_shape_sim(source, self._msim_output_order, position, transform,
                                                     self.source_transform_key, z_scale=self._msim_z_scale)
                              for source, position, transform in zip(self.sources, self.positions,
                                                                     self._msim_transforms)]
                origins = np.array([get_sim_position_final(sim, position, get_center=True)
                                    for sim, position in zip(shape_sims, self.positions)])
                sizes = [get_sim_physical_size(sim) for sim in shape_sims]
            with Timer('pair registration: get_pairs', verbose=self.logging_time):
                pairs, _ = get_pairs(origins, sizes)
            if SPLIT in pairing:
                # stage 1: each z-plane's (or channel's) tiles only; the groups are registered in register_global
                pairs = within_group_pairs(pairs, self.split_groups())
            # found over every source, but registered on the retained ones, numbered from 0
            retained_index = {int(index): position for position, index in enumerate(register_indices)}
            pairs = [(retained_index[index1], retained_index[index2]) for index1, index2 in pairs
                     if index1 in retained_index and index2 in retained_index]
            logging.info(f'#pairs: {len(pairs)}')
            #for pair in pairs:
            #    print(f'{self.file_labels[pair[0]]} - {self.file_labels[pair[1]]}')
        else:
            pairs = None

        # feature-based/CPD methods read pixel data from one concrete sim
        reg_method, pairwise_reg_func, pairwise_reg_func_kwargs = self.create_registration_method(
            msi_utils.get_sim_from_msim(register_msims[0], scale='scale0'), params=params)
        logging.info(f'Registration method: {reg_method}')
        logging.info('Registering pairs...')
        # a full pyramid per source lets compute_pairwise_registrations select each pair's resolution

        overlap_tolerance = 0

        # ******* start MVS registration functions

        with Timer('pair registration: select channel', verbose=self.logging_time):
            has_channel = ["c" in get_msim_dims(msim) for msim in register_msims]
            if has_channel[0]:
                if reg_channel is None and reg_channel_index is None:
                    raise Exception("Please choose a registration channel.")
                requested = reg_channel_index if reg_channel is None else reg_channel
                # each source by its own labels: one project's files need not all name it the same
                chosen = {}
                msims_reg = [
                    msi_utils.multiscale_sel_coords(
                        msim, {"c": resolve_registration_channel(msim, requested, chosen)})
                    if has_channel[imsim]
                    else msim
                    for imsim, msim in enumerate(register_msims)
                ]
            else:
                msims_reg = register_msims

        # match each level's transform to its image dimensions: a multiscale image can mix 3D/4D transforms
        for msim in msims_reg:
            for scale_node in msim.ds.values():
                if 'source_metadata' in scale_node.ds.data_vars:
                    img_data = scale_node.ds['image']
                    if hasattr(img_data, 'data'):
                        img_data = img_data.data
                    spatial_dims = [d for d in img_data.dims if d not in ('t', 'c')]
                    current_transform = scale_node.ds.data_vars[self.source_transform_key]
                    transform_spatial_dims = [d for d in current_transform.coords['x_in'].values if d != '1']
                    if len(transform_spatial_dims) != len(spatial_dims):
                        relevant_dim_names = spatial_dims + ['1']
                        adapted = current_transform.sel(x_in=relevant_dim_names, x_out=relevant_dim_names)
                        scale_node.ds[self.source_transform_key] = adapted

        try:
            # fusion off: dask's fused key names drop the unique token, so per-pair crop chains can collide
            # and a pair silently registers another pair's crop
            with dask.config.set({'scheduler': 'threads', 'optimization.fuse.active': False}):
                # multiview_stitcher would test every source within the largest one's diameter: with overview
                # images among tiles, nearly every pair
                graph_pairs = (pairs if pairs is not None
                               else find_candidate_overlap_pairs(msims_reg, self.source_transform_key))
                if pairs is None:
                    logging.info(f'#candidate pairs: {len(graph_pairs)}')
                raise_if_cancelled()
                with Timer('pair registration: view adjacency graph', verbose=self.logging_time):
                    g_reg = build_view_adjacency_graph(msims_reg, self.source_transform_key, graph_pairs,
                                                       overlap_tolerance=overlap_tolerance)
                raise_if_cancelled()

                g_reg_computed = g_reg.copy()
                workers = n_parallel_pairwise_regs or default_pair_workers
                log_every = 2 * workers
                pair_times, pair_cpu_times = [], []
                timed_reg_func = timed_calls(resolve_deferred_quality(pairwise_reg_func), pair_times,
                                             pair_cpu_times)

                pair_kwargs = dict(transform_key=self.source_transform_key, overlap_tolerance=overlap_tolerance,
                                   pairwise_reg_func_kwargs=pairwise_reg_func_kwargs, n_parallel_pairwise_regs=1)
                # largest first, so workers are not left idle at the end and the bar's estimate holds
                edges = sorted(g_reg.edges, key=lambda edge: estimated_pair_cost(g_reg, edge), reverse=True)
                pool = None
                # worker processes when a pair can be sent to one (its sources' arrays, the method)
                if (workers > 1 and len(edges) > workers
                        and picklable(([msims_reg[node] for node in edges[0]], pairwise_reg_func))):
                    pool = worker_process_pool(workers, max_tasks_per_child=default_pair_worker_tasks)

                def register_edge(edge):
                    graph = g_reg.edge_subgraph([edge]).copy()
                    if pool is None:
                        return compute_pairwise_registrations(msims_reg, graph, pairwise_reg_func=timed_reg_func,
                                                              **pair_kwargs)
                    registered, wall_time, cpu_time = result_unless_cancelled(pool.submit(
                        register_pair_in_worker, {node: msims_reg[node] for node in graph.nodes}, graph,
                        pairwise_reg_func, pair_kwargs))
                    pair_times.append(wall_time)
                    pair_cpu_times.append(cpu_time)
                    return registered

                phase_start, phase_cpu_start = time.time(), time.process_time()
                # one pair per thread, each its own compute, so a slow pair holds up no batch; a single worker
                # keeps the threads scheduler so one (e.g. 3D) pair still runs its tasks in parallel
                with progress_phase(progress_factory, total=g_reg.number_of_edges(),
                                         desc='Registering pairs') as pbar, \
                        Timer(f'register {g_reg.number_of_edges()} pairs on {workers}'
                              f' {"processes" if pool is not None else "threads"}', verbose=self.logging_time), \
                        (pool if pool is not None else nullcontext()), \
                        deferred_link_quality(pairwise_reg_func), \
                        dask.config.set(scheduler='synchronous' if workers > 1 else 'threads'):
                    log_start = time.time()
                    for count, (edge, pair_computed) in enumerate(
                            rolling_map(register_edge, edges, workers), start=1):
                        g_reg_computed.edges[edge].update(pair_computed.edges[edge])
                        del pair_computed
                        if count % log_every == 0 or count == g_reg.number_of_edges():
                            release_memory(generation=1)
                            if self.logging_time:
                                # cpu per pair rising means harder pairs; wall alone rising means idle workers or I/O
                                batch_wall = time.time() - log_start
                                batch_times, batch_cpu_times = pair_times[-log_every:], pair_cpu_times[-log_every:]
                                pair_timing = ''
                                if batch_times and batch_cpu_times:
                                    pair_timing = (f', a pair {np.mean(batch_times):.2f}s wall'
                                                   f' {np.mean(batch_cpu_times):.2f}s cpu,'
                                                   f' {sum(batch_times) / max(batch_wall, 1e-9):.1f} busy')
                                logging.info(f'Pairs {count}/{g_reg.number_of_edges()}:'
                                             f' last {log_every} in {batch_wall:.1f}s{pair_timing}'
                                             f'{print_memory_usage()}')
                            log_start = time.time()
                        if pbar is not None:
                            pbar.update(1)
                if self.logging_time and pair_times:
                    logging.info(f'Register pairs: {len(pair_times)} pairs'
                                 f' {format_phase_timing(time.time() - phase_start, pair_times, pair_cpu_times,
                                                         workers, time.process_time() - phase_cpu_start,
                                                         processes=pool is not None)}')

                # ******* end MVS registration functions

        except NotEnoughOverlapError:
            g_reg_computed = g_reg

        release_memory('Pair registration')
        mappings = nx.get_edge_attributes(g_reg_computed, default_transform_key)
        mappings_dict = {(register_indices[indices[0]], register_indices[indices[1]]): mapping
                         for indices, mapping in mappings.items()}

        with Timer(f'pair metrics ({g_reg_computed.number_of_edges()} pairs)', verbose=self.logging_time):
            metrics = calc_pair_metrics(msims_reg, g_reg_computed, params.get('metrics', []),
                                        self.source_transform_key, reg_channel=reg_channel_index,
                                        n_parallel_pairs=n_parallel_pairwise_regs,
                                        progress_factory=progress_factory)
            remap_pair_metrics(metrics, register_indices)
        release_memory('Pair metrics')

        self.pairs_graph = g_reg_computed
        self.pair_msims = msims_reg
        self.pairs = pairs
        self.metrics = metrics
        self.state = RegState.PAIRS_REG
        return {
            'pairs_graph': self.pairs_graph,
            'msims': msims_reg,
            'pairs': pairs,
            'pair_mappings': mappings_dict,
            'metrics': metrics
        }

    def split_groups(self, indices=None):
        """Group index per source (of `indices`, renumbered from 0): z-planes, or channels for dimension 'c'."""
        dimension = (getattr(self, 'registration_dimension', None)
                     or getattr(self, 'register_params', {}).get('registration_dimension'))
        groups = split_groups(self.positions, self.sources, dimension, getattr(self, 'input_groups', None))
        if indices is not None:
            groups = [groups[index] for index in indices]
            order = sorted(set(groups))
            groups = [order.index(group) for group in groups]
        return groups

    def channel_groups(self):
        """Each source's output channel in a channel overlay: see split_registration.channel_groups."""
        return channel_groups(self.sources, getattr(self, 'input_groups', None))

    def channel_labels(self, channels=None):
        """A label per channel of channel_groups(): as set in `channels` (extra_metadata's), else its sources' shared
        metadata channel label if no other channel has it, else their common label prefix, else 'channel <index>'."""
        groups = self.channel_groups()
        members = [[index for index, member_group in enumerate(groups) if member_group == group]
                   for group in range(max(groups, default=-1) + 1)]
        metadata_labels = []
        for group_members in members:
            group_labels = {(self.sources[index].get_channels() or [{}])[0].get('label') or ''
                            for index in group_members}
            metadata_labels.append(group_labels.pop() if len(group_labels) == 1 else '')
        labels = []
        for group, group_members in enumerate(members):
            label = metadata_labels[group] if metadata_labels.count(metadata_labels[group]) == 1 else ''
            if channels and group < len(channels):
                label = channels[group].get('label') or label
            labels.append(label or group_label([self.file_labels[index] for index in group_members])
                          or f'channel {group}')
        return labels

    def split_group_labels(self, indices):
        """A label per group of split_groups(indices): its sources' common label prefix (e.g. 'S000'), else its
        z-position or channel."""
        groups = self.split_groups(indices)
        labels = []
        for group in range(max(groups) + 1):
            members = [index for index, member_group in zip(indices, groups) if member_group == group]
            label = group_label([self.file_labels[index] for index in members])
            if not label:
                channel = (self.sources[members[0]].get_channels() or [{}])[0].get('label')
                is_channel = (getattr(self, 'registration_dimension', None) == 'c')
                label = str(channel) if is_channel else f"z {self.positions[members[0]].get('z', 0)}"
            labels.append(label)
        return labels

    def _register_split_groups(self, pair_msims, transforms, register_indices, params, progress_factory=None):
        method, pairwise_reg_func, pairwise_reg_func_kwargs = self.create_registration_method(
            msi_utils.get_sim_from_msim(pair_msims[0], scale='scale0'), params=params)
        transform_type = params.get('transform_type', 'rigid')
        resolution_method = ROBUST_LINEAR if transform_type in ('translation', 'rigid') else 'global_optimization'
        binning = params.get('split_binning')
        binning = max(int(float(binning)), 1) if is_valid_value(binning) else default_split_binning
        with progress_phase(progress_factory, total=1, desc='Registering z-planes / channels'), \
                Timer('split: register groups', verbose=self.logging_time):
            transforms, group_graph = register_groups(
                pair_msims, transforms, self.split_groups(list(register_indices)), self.source_transform_key,
                pairwise_reg_func, pairwise_reg_func_kwargs, resolution_method=resolution_method,
                resolution_kwargs={'transform': transform_type}, binning=binning,
                metric_funcs=create_metric_methods(params['metrics'], pair_msims[0]) if params.get('metrics') else None)
        labels = self.split_group_labels(list(register_indices))
        self.group_pairs = {}
        for group1, group2 in group_graph.edges:
            data = group_graph.edges[group1, group2]
            entry = {'mapping': np.array(data[default_transform_key].sel(t=0)).tolist(),
                     default_quality_key: float(quality_to_scalar(data[default_quality_key]))}
            if 'bbox' in data:
                entry['bbox'] = np.array(data['bbox'].sel(t=0)).tolist()
            if data.get('metrics'):
                entry['metrics'] = data['metrics']
            self.group_pairs[(labels[group1], labels[group2])] = entry
        return transforms

    def register_global(self, pair_msims, register_indices=None, params=None,
                        pairs_graph=None, progress_factory=None):
        logging.info('Global registration...')
        if register_indices is None:
            if self.register_indices is not None:
                register_indices = self.register_indices
            else:
                register_indices = range(len(pair_msims))

        if pairs_graph is not None:
            g_reg_computed = pairs_graph
        else:
            g_reg_computed = self.pairs_graph

        ndims = si_utils.get_ndim_from_sim(get_msim_image0(pair_msims[0]))

        groupwise_resolution_method = params.get('groupwise_resolution_method', 'global_optimization')
        transform_type = params.get('transform_type', 'affine')
        if groupwise_resolution_method in (ROBUST_LINEAR, 'linear_two_pass') and transform_type not in ('translation', 'rigid'):
            logging.warning(f'{groupwise_resolution_method} resolves translation or rigid only,'
                            f' not {transform_type}: using global_optimization')
            groupwise_resolution_method = 'global_optimization'
        split_settings = ''
        if SPLIT in params.get('pairing', '').lower():
            split_settings = f", split binning {params.get('split_binning') or default_split_binning}"
        logging.info(f'Global registration settings: resolution {groupwise_resolution_method},'
                     f' transform {transform_type}{split_settings}')
        groupwise_resolution_kwargs = {}
        if groupwise_resolution_method in ('global_optimization', ROBUST_LINEAR, 'linear_two_pass'):
            # transform_type options include 'translation', 'rigid', 'affine', 'similarity'
            groupwise_resolution_kwargs['transform'] = transform_type

        quality_threshold = params.get('post_registration_quality_threshold')
        quality_threshold = float(quality_threshold) if is_valid_value(quality_threshold) else 0.0

        n_parallel_pairwise_regs = params.get('n_parallel_pairwise_regs')
        # None or 0 is automatic; a yaml value may be a float
        n_parallel_pairwise_regs = int(n_parallel_pairwise_regs) if n_parallel_pairwise_regs else None

        plot_summary = self.mpl_ui

        # ******* start MVS registration functions

        # a pair scored at the threshold or below (a failed method gives the identity with 0) is left out of the solve
        # only, so a tile without other pairs keeps its metadata position rather than an unconstrained one's
        failed_pairs = [edge for edge, quality in nx.get_edge_attributes(g_reg_computed, 'quality').items()
                        if not np.nan_to_num(np.asarray(quality, dtype=float), nan=0.0).max() > quality_threshold]
        resolve_graph = g_reg_computed
        if failed_pairs:
            resolve_graph = g_reg_computed.copy()
            resolve_graph.remove_edges_from(failed_pairs)
            logging.warning(f'Global registration: {len(failed_pairs)} of {g_reg_computed.number_of_edges()} pairs'
                            f' left out, scored {quality_threshold:g} or below')
            unregistered = sorted(node for node in resolve_graph.nodes
                                  if resolve_graph.degree(node) == 0 and g_reg_computed.degree(node) > 0)
            if unregistered:
                logging.warning('Global registration: no registered pair, so kept at their metadata position: '
                                + ', '.join(str(self.file_labels[register_indices[node]]) for node in unregistered))

        # multiview_stitcher's own search is O(nodes x edges); a reference outside a component would send that
        # component back to it, so only a connected graph gets one
        if resolve_graph.number_of_nodes() > 2 and nx.is_connected(resolve_graph):
            groupwise_resolution_kwargs['reference_view'] = find_reference_view(resolve_graph)

        if groupwise_resolution_method == ROBUST_LINEAR:
            resolution_progress = progress_phase(progress_factory, total=default_robust_rounds,
                                                      desc='Global registration', weight=4)
        else:
            # global_optimization has nothing to report into a phase: GlobalOptProgress follows its log instead
            resolution_progress = GlobalOptProgress(progress_factory, desc='Global registration',
                                                    max_passes=g_reg_computed.number_of_edges(), weight=4)
        with resolution_progress as pbar, dask.config.set(scheduler='threads'):
            if groupwise_resolution_method == ROBUST_LINEAR and pbar is not None:
                groupwise_resolution_kwargs['progress'] = lambda: pbar.update(1)
            transforms_dict, groupwise_resolution_info_dict = groupwise_resolution(
                resolve_graph,
                method=groupwise_resolution_method,
                **groupwise_resolution_kwargs,
            )

        transforms = [
            transforms_dict[iview] for iview in sorted(g_reg_computed.nodes())
        ]
        if SPLIT in params.get('pairing', '').lower():
            transforms = self._register_split_groups(pair_msims, transforms, register_indices, params,
                                                     progress_factory=progress_factory)

        # the stages below take long enough on a large project to need progress of their own
        with progress_phase(progress_factory, total=len(pair_msims),
                                 desc='Applying transforms') as pbar, \
                Timer('apply registered transforms', verbose=self.logging_time):
            for imsim, msim in enumerate(pair_msims):
                set_msim_affine(
                    msim,
                    transforms[imsim],
                    transform_key=self.reg_transform_key,
                    base_transform_key=self.source_transform_key,
                )
                if pbar is not None:
                    pbar.update(1)

        if plot_summary:
            with progress_phase(progress_factory, total=1, desc='Registration summary'), \
                    Timer('plot registration summaries', verbose=self.logging_time):
                plot_info = _plot_registration_summaries(
                    pair_msims,
                    self.source_transform_key,
                    self.reg_transform_key,
                    resolve_graph,
                    groupwise_resolution_info_dict,
                    show_plot=plot_summary,
                )
        else:
            plot_info = {}

        reg_result = {
            "params": transforms,
            "pairwise_registration": {
                "graph": g_reg_computed,
                "metrics": {
                    "qualities": nx.get_edge_attributes(
                        g_reg_computed, "quality"
                    )
                },
                "summary_plot": None if plot_summary is False
                else (
                    plot_info['fig_pair_reg'],
                    plot_info['ax_pair_reg']
                )
            },
            "groupwise_resolution": {
                "metrics": groupwise_resolution_info_dict,
                "summary_plot": None if plot_summary is False
                else (
                    plot_info['fig_group_res'],
                    plot_info['ax_group_res']
                )
            },
        }

        # ******* end MVS registration functions

        # onto self.msims, the persistent full per-source pyramid
        with progress_phase(progress_factory, total=len(pair_msims),
                                 desc='Storing transforms') as pbar, \
                Timer('store transforms on sources', verbose=self.logging_time):
            for reg_msim, index in zip(pair_msims, register_indices):
                reg_transform = msi_utils.get_transform_from_msim(reg_msim, transform_key=self.reg_transform_key)
                set_msim_affine(self.msims[index], reg_transform, transform_key=self.reg_transform_key)
                if pbar is not None:
                    pbar.update(1)

        # set missing transforms - sources that never took part in registration (e.g. filtered out)
        for msim in self.msims:
            if self.reg_transform_key not in get_msim_transform_keys(msim):
                set_msim_affine(
                    msim,
                    param_utils.identity_transform(ndim=ndims, t_coords=[0]),
                    transform_key=self.reg_transform_key)

        mappings = reg_result['params']
        # re-index from subset of sims
        # per timepoint index; a registration resolves its first timepoint
        edge_residuals = (groupwise_resolution_info_dict.get('edge_residuals') or {}).get(0, {})
        residual_error_dict = {(int(register_indices[key[0]]), int(register_indices[key[1]])): float(value)
                               for key, value in edge_residuals.items()}
        self.log_residuals(residual_error_dict)
        registration_qualities_dict = reg_result.get('pairwise_registration', {}).get('metrics', {}).get('qualities', {})
        registration_qualities_dict = {(register_indices[key[0]], register_indices[key[1]]): value
                                       for key, value in registration_qualities_dict.items()}

        # re-index from subset of sims
        mappings_dict = {index: mapping for index, mapping in zip(register_indices, mappings)}

        with Timer('global registration metrics', verbose=self.logging_time):
            metrics = calc_global_metrics(pair_msims, self.reg_transform_key, params.get('metrics', []), reg_result,
                                          n_parallel_pairs=n_parallel_pairwise_regs,
                                          progress_factory=progress_factory)
            remap_pair_metrics(metrics, register_indices)

        self.mappings = mappings_dict
        if self.group_pairs:
            metrics['group_pairs'] = self.group_pair_metrics()
        self.metrics = metrics
        self.state = RegState.GLOBAL_REG
        return {'reg_result': reg_result,
                'mappings': mappings_dict,
                'residual_errors': residual_error_dict,
                'registration_qualities': registration_qualities_dict,
                'metrics': metrics}

    @staticmethod
    def _fusion_batch_options(saving_zarr, max_workers=None):
        """Options to fuse a zarr export's blocks on every worker, as multiview_stitcher otherwise fuses them
        sequentially: each block writes its own chunk, and reserves its memory from the shared pool first."""
        if not saving_zarr:
            return None
        return {'workers': max(1, max_workers or default_fusion_workers), 'memory_pool': default_fusion_memory_pool}

    def is_channel_overlay(self, dimension=None, extra_metadata=None):
        if extra_metadata is None:
            extra_metadata = self.extra_metadata
        channels = extra_metadata.get('channels', []) if isinstance(extra_metadata, dict) else []
        return bool((dimension and dimension == 'c') or len(channels) > 1)

    def fuses_to_zarr(self, fusion_method=None, dimension=None, extra_metadata=None):
        """Whether fuse() given an output filename writes the export itself; otherwise its caller saves the lazy
        result it returns, which is where the work is then done."""
        return not self.is_channel_overlay(dimension, extra_metadata) and 'compos' not in (fusion_method or '')

    def _fusion_z_geometry(self, extra_metadata):
        """(z scale, number of distinct z positions, whether 2D sources become a 3D stack) as fusion takes them: a
        stack's slices have no z position of their own, and make_msims_3d() spaces them by index."""
        z_scale = get_metadata_z_scale(self.source_metadata) or get_metadata_z_scale(extra_metadata)
        if z_scale is None:
            z_scale = extract_z_scale(self.positions, self.scales)
        num_z_positions = len({position.get('z') for position in self.positions if 'z' in position})
        return z_scale, num_z_positions, num_z_positions > 1 or self.is_stack

    @staticmethod
    def _output_spacing_method(output_spacing, exports):
        """(native, the spacing method for calc_output_properties): 'native' only for an export of fused sources."""
        is_native = (output_spacing is None or str(output_spacing).lower() == 'native') and exports
        if is_native:
            return True, 'min'
        if output_spacing is None or str(output_spacing).lower() == 'native':
            return False, 'mean'
        return False, output_spacing

    @staticmethod
    def _export_chunksize(dtype, output_stack_properties, sources, num_z_positions, block_size=None):
        """The block an export fuses and writes: a configured block_size, else the memory budget's, for `sources`'
        (msims' or stack properties') extents."""
        if not block_size:
            return get_export_chunk_sizes(dtype, output_stack_properties, sources, num_z_positions=num_z_positions)
        if not isinstance(block_size, (list, tuple)):
            block_size = [block_size] * 2
        chunksize = xyz_to_dict(block_size)
        if 'z' in output_stack_properties['shape'] and 'z' not in chunksize:
            # block_size says nothing about z, so the budget decides it
            chunksize['z'] = get_chunk_sizes(dtype, list(output_stack_properties['shape']), num_sources=len(sources),
                                             num_z_positions=num_z_positions).get('z', 1)
        return chunksize

    def estimate_fusion_size(self, fusion_method=None, output_spacing=None, transform_key=None, dimension=None,
                             block_size=None, extra_metadata=None):
        """Uncompressed bytes per level of the OME-Zarr fuse() would write, from source geometry alone (no msims):
        {'levels': [{'spacing', 'shape', 'bytes'}], 'bytes', 'native'}.
        Only the blocks some source reaches are written - exactly these; a channel overlay is saved whole."""
        if extra_metadata is None:
            extra_metadata = self.extra_metadata
        is_channel_overlay = self.is_channel_overlay(dimension, extra_metadata)
        if not self.sources:
            return None
        if 'compos' in (fusion_method or '') and not is_channel_overlay:
            return self._composed_size()
        if transform_key is None:
            transform_key = self.reg_transform_key
        z_scale, num_z_positions, promote_z = self._fusion_z_geometry(extra_metadata)
        is_native, spacing_method = self._output_spacing_method(output_spacing, not is_channel_overlay)
        dtype = np.dtype(self.sources[0].dtype)
        nonspatial = int(np.prod([size for dim, size in self.sources[0].get_size().items() if dim not in 'zyx']))
        voxel_bytes = nonspatial * dtype.itemsize

        translations, source_props = self.source_geometry(transform_key, promote_z=promote_z, z_scale=z_scale)
        matrices = [np.asarray(props['transform'], dtype=float) for props in source_props]

        stacks = {}

        def stack(index, level):
            if (index, level) not in stacks:
                stacks[index, level] = build_source_stack_props(
                    self.sources[index], self._msim_output_order, translations[index], None, None,
                    z_scale=self._msim_z_scale, level=level, promote_z=promote_z)
            return stacks[index, level]

        views0 = [(stack(index, 0), matrix) for index, matrix in enumerate(matrices)]
        level0 = calc_output_properties_from_views(views0, output_spacing_method=spacing_method, z_scale=z_scale)
        if is_channel_overlay:
            return self._channel_overlay_size(level0, voxel_bytes * len(set(self.channel_groups())))
        levels = []
        source_spacings = [min(view[0]['spacing'][dim] for dim in 'yx') for view in views0]
        scaled_dims = ['y', 'x'] + (['z'] if views0[0][0]['shape'].get('z', 1) > 1 else [])
        chunksize = self._export_chunksize(dtype, level0, [view[0] for view in views0], num_z_positions, block_size)
        for level_spacing, properties, selected in export_levels(source_spacings, level0, scaled_dims,
                                                                 native=is_native):
            output_properties = {key: properties[key] for key in ('spacing', 'origin', 'shape')}
            views = []
            for index in selected:
                # as fusion picks it (fuse_levels_to_ome_zarr)
                level = coarsest_level_within([stack(index, level)['spacing'] for level in range(len(
                    self.sources[index].shapes))], output_properties['spacing'],
                    dims=list(output_properties['spacing']))
                views.append((stack(index, level), matrices[index]))
            chunk, groups = level_blocks(views, output_properties, chunksize)
            levels.append({'spacing': float(level_spacing), 'shape': output_properties['shape'],
                           'bytes': written_voxels(groups, output_properties, chunk) * voxel_bytes})
        return {'levels': levels, 'bytes': sum(level['bytes'] for level in levels), 'native': is_native}

    def _composed_size(self, min_length=128):
        """estimate_fusion_size()'s for save_composed(): each source's own levels, padded as
        make_ome_multiscale_levels() pads them; a level's shape and spacing are its finest source's."""
        levels = []
        for source in self.sources:
            dtype_bytes = np.dtype(source.dtype).itemsize
            source_levels = [(source.get_size(level), source.get_pixel_size(level))
                             for level in range(len(source.shapes))]
            coarsest_size, coarsest_spacing = source_levels[-1]
            spatial_size = {dim: size for dim, size in coarsest_size.items() if dim in 'xyz'}
            for factors in calc_pyramid_level_factors(spatial_size, min_size=min_length):
                source_levels.append(({dim: ceil(size / factors.get(dim, 1)) for dim, size in coarsest_size.items()},
                                      {dim: spacing * factors.get(dim, 1) for dim, spacing in coarsest_spacing.items()}))
            for index, (size, pixel_size) in enumerate(source_levels):
                spacing = float(min(pixel_size[dim] for dim in 'yx'))
                level_bytes = int(np.prod(list(size.values()))) * dtype_bytes
                if index == len(levels):
                    levels.append({'spacing': spacing, 'shape': {dim: size[dim] for dim in size if dim in 'zyx'},
                                   'bytes': 0})
                elif spacing < levels[index]['spacing']:
                    levels[index].update(spacing=spacing, shape={dim: size[dim] for dim in size if dim in 'zyx'})
                levels[index]['bytes'] += level_bytes
        return {'levels': levels, 'bytes': sum(level['bytes'] for level in levels), 'native': True}

    @staticmethod
    def _channel_overlay_size(level0, voxel_bytes):
        """estimate_fusion_size()'s for the bounding box that save() writes whole, with its yx pyramid."""
        shape = dict(level0['shape'])
        spacing = min(level0['spacing'][dim] for dim in 'yx')
        levels = [{'spacing': float(spacing), 'shape': dict(shape)}]
        # ngff_zarr's rule: a level while a dim spans two chunks, halving each that keeps a chunk and is not
        # already at most half the longest
        while any(shape[dim] >= 2 * default_chunk_size for dim in 'yx'):
            longest = max(shape[dim] for dim in 'yx')
            shape = {dim: int(size / 2) if dim in 'yx' and default_chunk_size <= size / 2 and size > longest / 2
                     else size for dim, size in shape.items()}
            spacing *= 2
            levels.append({'spacing': float(spacing), 'shape': dict(shape)})
        for level in levels:
            level['bytes'] = int(np.prod(list(level['shape'].values()))) * voxel_bytes
        return {'levels': levels, 'bytes': sum(level['bytes'] for level in levels), 'native': False}

    def fuse(self, msims, fusion_method=None, output_spacing=None, transform_key=None,
             dimension=None, output_filename=None,
             block_size=None, ome_version=default_ome_zarr_version, extra_metadata=None,
             output_chunksize=None, workers=None, level_bytes=None, source_indices=None):
        """Fuse the sources' msims into one multiscale output msim; returns (fused, whether it was written).
        Sims are wrapped first with util.wrap_sims_as_msims. `source_indices`: each msim's source, when not all are
        given in order, for its channel in a channel overlay.

        output_chunksize, a chunk size per spatial dim, is used as-is; otherwise it is sized against memory.
        Written to a file, every level is fused from the sources (fusion_blocks.export_levels): with output_spacing
        None or 'native' each at its own pixel size from the sources at least that fine, else doubling from the set
        spacing. In memory, 'native' stays 'mean'. `level_bytes` (estimate_fusion_size's, a level) shares the progress
        bar out over the levels.
        """
        logging.info('Fusion...')
        if output_filename is not None:
            output_filename = self.output + output_filename

        # only its dtype is read, which needs no full sim
        sim0 = get_msim_image0(msims[0])

        if extra_metadata is None:
            extra_metadata = self.extra_metadata

        channels = extra_metadata.get('channels', []) if isinstance(extra_metadata, dict) else []
        is_channel_overlay = self.is_channel_overlay(dimension, extra_metadata)

        if transform_key is None:
            transform_key = self.reg_transform_key

        z_scale, num_z_positions, promote_z = self._fusion_z_geometry(extra_metadata)
        # the one place a stack becomes 3D: msims are stored 2D, and each consumer that needs the stacked geometry
        # asks for it, rather than every level of every source being rebuilt up front
        if promote_z:
            with Timer('fusion: make_msims_3d', verbose=self.logging_time):
                msims = make_msims_3d(msims, z_scale=z_scale, positions=self.positions)
        if not is_channel_overlay:
            with Timer('fusion: unify channels', verbose=self.logging_time):
                msims = unify_msim_channels(msims)

        is_native, output_spacing = self._output_spacing_method(
            output_spacing, output_filename is not None and not is_channel_overlay)
        with Timer('fusion: output properties', verbose=self.logging_time):
            output_stack_properties = calc_output_properties(msims, transform_key,
                                                             output_spacing_method=output_spacing, z_scale=z_scale)

        if self.verbose:
            logging.info(f'Output stack: {numpy_to_native(output_stack_properties)}')
        data_size = np.prod(list(output_stack_properties['shape'].values())) * sim0.dtype.itemsize
        # the bounding box, not what is written: see estimate_fusion_size()
        logging.info(f'Fusion output bounding box at full resolution: {print_hbytes(data_size)}')

        # peak memory follows the sources in one output chunk, each transformed to a full float32 chunk;
        # the input's own chunk grid would put every source in one chunk of a coarse output
        default_output_chunksize = get_chunk_sizes(sim0.dtype, list(output_stack_properties['shape']),
                                                   num_sources=len(msims),
                                                   num_z_positions=num_z_positions)
        saving_zarr = False
        if is_channel_overlay:
            # each channel's sources fused together, onto the same grid
            source_groups = self.channel_groups()
            groups = [source_groups[index] for index in (source_indices or range(len(msims)))]
            labels = self.channel_labels(channels)
            labels = [labels[group] for group in sorted(set(groups))]
            fuse_func = self.create_fusion_method(fusion_method, sim0)
            channel_results = []
            for group in sorted(set(groups)):
                group_msims = unify_msim_channels([msim for msim, msim_group in zip(msims, groups)
                                                   if msim_group == group])
                channel_results.append(fusion.fuse(
                    group_msims,
                    # compose (no fusion function) still fuses, as a channel needs one image
                    fusion_func=fuse_func or fusion.weighted_average_fusion,
                    transform_key=transform_key,
                    output_stack_properties=output_stack_properties,
                    output_chunksize=output_chunksize or get_chunk_sizes(
                        sim0.dtype, list(output_stack_properties['shape']), num_sources=len(group_msims),
                        num_z_positions=num_z_positions)))
            fused_image = combine_msims_as_channels(channel_results, labels)
        else:
            if fusion_method:
                logging.info(f'Fusion method: {fusion_method}')
            else:
                logging.info('Fusion method: [default method]')
            fuse_func = self.create_fusion_method(fusion_method, sim0)
            if fuse_func:
                saving_zarr = output_filename is not None
                if output_chunksize is None and saving_zarr:
                    with Timer('fusion: export chunk sizes', verbose=self.logging_time):
                        output_chunksize = self._export_chunksize(sim0.dtype, output_stack_properties, msims,
                                                                  num_z_positions, block_size)
                if output_chunksize is None:
                    output_chunksize = dict(default_output_chunksize)
                # a chunk past the output's extent also fails dask's zarr write-alignment check, with a warning
                output_chunksize = {dim: min(int(size), output_stack_properties['shape'].get(dim, int(size)))
                                    for dim, size in output_chunksize.items()}
                if self.verbose:
                    # logged once settled by the branches above
                    logging.info(f'Fusion output_chunksize: {numpy_to_native(output_chunksize)}'
                                 f' ({len(msims)} sources over'
                                 f' {max(1, num_z_positions)} z position(s))')
                if saving_zarr:
                    if not output_filename.lower().endswith('.zarr'):
                        output_filename += zarr_extension
                    zarr_options = {'ngff_version': ome_version}
                else:
                    zarr_options = None
                with dask.config.set(scheduler='threads'), warnings.catch_warnings():
                    # fused blocks are whole zarr chunks, one writer each: dask flags every edge block, whose chunk
                    # the array ends in. Set here, as a napari worker puts 'always' ahead of any module-level filter
                    warnings.filterwarnings('ignore', message='The input Dask array will be rechunked',
                                            category=PerformanceWarning)
                    if saving_zarr:
                        source_spacings = [min(si_utils.get_spacing_from_sim(get_msim_image0(msim))[dim]
                                               for dim in 'yx') for msim in msims]
                        # a section stack keeps every section at every level; a volume is scaled in z too
                        scaled_dims = ['y', 'x'] + (['z'] if get_msim_image0(msims[0]).sizes.get('z', 1) > 1 else [])
                        levels = export_levels(source_spacings, output_stack_properties, scaled_dims, native=is_native)
                        # native: tiles over the overview, where a finer source covers a pixel the coarser one under it
                        # is left out, each level from the sources it has; a set spacing averages them as one, and
                        # every level has all of them, so averaging the level above gives it
                        fused_image = fuse_levels_to_ome_zarr(
                            msims, levels, output_filename, transform_key, output_stack_properties, output_chunksize,
                            fusion_func=fuse_func, zarr_options=zarr_options,
                            batch_options=self._fusion_batch_options(saving_zarr, max_workers=workers),
                            ranks=source_spacings if is_native else None, average_levels=not is_native,
                            level_weights=level_bytes)
                    else:
                        fused_image = fusion.fuse(msims, fusion_func=fuse_func, transform_key=transform_key,
                                                  output_stack_properties=output_stack_properties,
                                                  output_chunksize=output_chunksize)
            else:
                # 'compose' mode: no fusion, the sources as they are
                fused_image = msims
                if output_filename is not None:
                    if str(ome_version) != '0.6':
                        logging.warning(f'Compose writes an OME-Zarr scene, which only 0.6 has: written as 0.6, '
                                        f'not {ome_version}')
                    if not output_filename.lower().endswith('.zarr'):
                        output_filename += zarr_extension
                    self.save_composed(output_filename, msims, channels)
                    saving_zarr = True
        return fused_image, saving_zarr

    def save_pair_results(self, pair_mappings, metrics=None, qualities=None):
        """The pair mappings file, each pair with its quality (from `metrics` unless given) and its pairs graph bbox."""
        if qualities is None:
            qualities = {key: metric[default_transform_key][default_quality_key]
                         for key, metric in metrics['pairs'].items() if default_quality_key in metric[default_transform_key]}
        bboxes = {key: np.array(value.sel(t=0) if 't' in value.dims else value).tolist()
                  for key, value in nx.get_edge_attributes(self.pairs_graph, 'bbox').items()}
        self.save_pair_mappings(pair_mappings, qualities, bboxes)

    def save_pair_mappings(self, mappings, qualities, bboxes):
        pair_mappings_filename = self.output + self.input_output_params.get('pair_mappings', default_pair_mappings_name)
        output_mappings = {}
        for keys, mapping in mappings.items():
            label_key = json.dumps([self.filenames[keys[0]], self.filenames[keys[1]]])
            output_mappings[label_key] = {'mapping': np.array(mapping.sel(t=0)).tolist()}
            if keys in qualities:
                quality = quality_to_scalar(qualities[keys])
                output_mappings[label_key][default_quality_key] = float(quality)
            if keys in bboxes:
                output_mappings[label_key]['bbox'] = bboxes[keys]
        export_json(pair_mappings_filename, output_mappings)

    def group_pair_metrics(self):
        return {labels: {self.reg_transform_key: entry.get('metrics', {})
                         | {default_quality_key: entry[default_quality_key]}}
                for labels, entry in self.group_pairs.items()}

    def save_group_pair_mappings(self):
        """Split pairing's group (e.g. section) pairs added to the pair mappings file, keyed by their labels and marked
        by kind, replacing those of an earlier run."""
        pair_mappings_filename = self.output + self.input_output_params.get('pair_mappings', default_pair_mappings_name)
        output_mappings = import_json(pair_mappings_filename) if os.path.exists(pair_mappings_filename) else {}
        output_mappings = {key: value for key, value in output_mappings.items()
                           if value.get('kind') != split_group_pair_kind}
        for labels, entry in self.group_pairs.items():
            # its image metrics go in the metrics file, as every pair's do
            output_mappings[json.dumps(list(labels))] = {'kind': split_group_pair_kind} | {
                key: value for key, value in entry.items() if key != 'metrics'}
        export_json(pair_mappings_filename, output_mappings)

    def save_mappings(self, mappings):
        mappings_filename = self.output + self.input_output_params.get('mappings', default_mappings_name)
        output_mappings = {self.filenames[int(key)]: np.array(mapping.sel(t=0)).tolist()
                           for key, mapping in mappings.items()}
        write_transforms(mappings_filename, output_mappings, self.source_transform_key, self.reg_transform_key)

    def save_mappings_csv(self, mappings, normalise_orientation=False):
        data = []
        mappings_header = ['id',' filename', 'x_pixels', 'y_pixels', 'z_pixels', 'x', 'y', 'z', 'rotation']
        mappings_filename = self.output + self.input_output_params.get('mappings', default_mappings_tabular_name)
        for label, filename, msim, mapping, scale, position, rotation \
                in zip(self.file_labels, self.filenames, self.msims, mappings.values(), self.scales, self.positions, self.rotations):
            if not normalise_orientation:
                # rotation already in msim affine transform
                rotation = None
            position, rotation = get_data_mapping(msim,
                                                  transform_key=self.reg_transform_key,
                                                  transform=mapping,
                                                  translation0=position,
                                                  rotation=rotation)
            position_pixels = {dim: position[dim] / float(scale.get(dim, 1)) for dim in position.keys()}
            row = ([label] + [filename] + dict_to_xyz(position_pixels, add_zeros=True)
                   + dict_to_xyz(position, add_zeros=True) + [rotation])
            data.append(row)
        export_csv(mappings_filename, data, header=mappings_header)

    def save_metrics(self, metrics):
        """The registered pairs' metrics, then split pairing's group pairs' (without filenames), also read back on
        resuming."""
        pair_metrics = {keys: {metric: float(value) for metric, value in metric_dict[self.reg_transform_key].items()}
                        for keys, metric_dict in metrics['pairs'].items() if metric_dict[self.reg_transform_key]}
        group_metrics = {labels: metric_dict[self.reg_transform_key]
                         for labels, metric_dict in metrics.get('group_pairs', {}).items()}
        metric_names = list(dict.fromkeys(metric for values in list(pair_metrics.values()) + list(group_metrics.values())
                                          for metric in values))
        rows = [[self.file_labels[keys[0]], self.file_labels[keys[1]], self.filenames[keys[0]], self.filenames[keys[1]]]
                + [values.get(metric) for metric in metric_names]
                for keys, values in sorted(pair_metrics.items())]
        rows += [list(labels) + ['', ''] + [values.get(metric) for metric in metric_names]
                 for labels, values in group_metrics.items()]
        export_csv(self.output + metrics_name, rows, header=pair_table_columns + metric_names)

    def log_residuals(self, residuals):
        """Each pair's residual from the global solution (RMS distance, physical units), the largest first."""
        if residuals:
            values = list(residuals.values())
            pair_lines = [f'  {self.file_labels[keys[0]]} - {self.file_labels[keys[1]]}: {value:.4g}'
                          for keys, value in sorted(residuals.items(), key=lambda item: -item[1])]
            logging.info(f'Global registration residuals over {len(values)} pairs: mean {np.mean(values):.4g},'
                         f' median {np.median(values):.4g}, max {np.max(values):.4g}\n' + '\n'.join(pair_lines))

    def save_residuals(self, residuals):
        rows = [[self.file_labels[keys[0]], self.file_labels[keys[1]], self.filenames[keys[0]], self.filenames[keys[1]],
                 value] for keys, value in sorted(residuals.items())]
        export_csv(self.output + residuals_name, rows, header=pair_table_columns + ['residual'])

    def create_preview(self, output_filename=None, nom_msims=None, transform_key=None):
        preview_scale = parse_scale(self.input_output_params.get('preview_scale'), default=16)
        is_stack = self.is_stack
        z_scale = get_metadata_z_scale(self.extra_metadata)

        msims = select_msim_subpyramid_at_scale(self.msims, self.sources, preview_scale)
        msims = self.flatfield_corrected_msims(msims, self.preprocess_params)
        if is_stack:
            msims = make_msims_3d(msims, z_scale, self.positions)

        if nom_msims is not None:
            preview_image0 = get_msim_image0(msims[0])
            nom_image0 = get_msim_image0(nom_msims[0])
            if preview_image0.sizes['x'] >= nom_image0.sizes['x']:
                logging.warning('Unable to generate scaled down preview due to lack of source pyramid sizes')
                return None

            if transform_key is not None and transform_key != self.source_transform_key:
                copy_transforms_to_msims(nom_msims, msims, transform_key)
        fusion_method, _ = self.fusion_settings()
        fused_msim, is_saved = self.fuse(msims, fusion_method=fusion_method, transform_key=transform_key,
                                         output_spacing='max', output_filename=output_filename)
        fused_sim = extract_sims_from_fused(fused_msim)
        preview_format = self.fusion_params.get('preview')
        if output_filename and (not is_saved or 'tif' in preview_format):
            self.save(output_filename, fused_sim.squeeze(), transform_key=transform_key, format=preview_format,
                      ome_version=self.fusion_params.get('ome_version') or default_ome_zarr_version)
        return fused_sim

    def save(self, output_filename, data, format=zarr_extension, transform_key=None, translations0=None, channels=[],
             block_size=None, compression=None, pyramid_downsample=2, npyramid_add=4, ome_version=default_ome_zarr_version,
             progress_factory=None):
        if output_filename is not None:
            output_filename = self.output + output_filename
        if isinstance(self.extra_metadata, dict) and self.extra_metadata:
            channels = self.extra_metadata.get('channels', [])
        save_image(output_filename, data, format,
                   transform_key=transform_key, channels=channels, translations0=translations0,
                   block_size=block_size, compression=compression,
                   pyramid_downsample=pyramid_downsample, npyramid_add=npyramid_add,
                   ome_version=ome_version,
                   verbose=self.verbose, progress_factory=progress_factory)

    def save_native_levels(self, output_filename, msim, position=None, channels=None, min_length=128,
                           compression=None, ome_version=default_ome_zarr_version):
        """Write msim's own native pyramid levels as-is, without resampling (see save_ome_multiscale_levels()).

        position: this source's position; a source without a 'z' dim is promoted to keep its z position.
        """
        if output_filename is not None:
            output_filename = self.output + output_filename
        if channels is None:
            channels = self.extra_metadata.get('channels', []) if isinstance(self.extra_metadata, dict) else []

        if position is not None and 'z' not in get_msim_dims(msim):
            msim = make_msims_3d([msim], z_scale=self._msim_z_scale, positions=[position])[0]

        levels, dim_order, translation = msim_native_levels(msim)
        save_ome_multiscale_levels(str(output_filename) + zarr_extension, levels, dim_order, channels,
                                   translation, min_length=min_length,
                                   compression=compression, ome_version=ome_version)

    def save_composed(self, output_filename, msims, channels, min_length=128, compression=None):
        """Each source as its own image at its native levels, in one OME-Zarr 0.6 scene with a coordinate system per
        transform key; the registered one sorts first, so it is the default view."""
        transform_keys = [key for key in sorted([self.reg_transform_key, self.source_transform_key])
                          if key in get_msim_transform_keys(msims[0])]
        labels = self.file_labels if len(self.file_labels) == len(msims) else [str(index) for index in range(len(msims))]
        sources = {}
        for label, msim in zip(labels, msims):
            levels, dim_order, translation = msim_native_levels(msim)
            affines = {}
            for transform_key in transform_keys:
                affine = msi_utils.get_transform_from_msim(msim, transform_key)
                affines[transform_key] = affine.sel(t=0) if 't' in affine.dims else affine
            spatial_dims = [str(dim) for dim in affines[transform_keys[0]].coords['x_in'].values[:-1]]
            sources[label] = (levels, dim_order, translation, spatial_dims, affines)
        save_ome_zarr_scene(output_filename, sources, transform_keys, channels, min_length=min_length,
                            compression=compression)

    def save_video(self, output, msims, fused_msim):
        logging.info('Creating transition video...')
        sims = [msi_utils.get_sim_from_msim(msim, scale='scale0') for msim in msims]
        fused_image = extract_sims_from_fused(fused_msim)
        pixel_size = [si_utils.get_spacing_from_sim(sims[0]).get(dim, 1) for dim in 'xy']
        params = self.fusion_params.get('transition', {})
        nframes = params.get('frames', 1)
        spacing = params.get('spacing', [1.1, 1])
        scale = params.get('scale', 1)
        transition_filename = output + 'transition'
        video = Video(transition_filename + '.mp4', fps=params.get('fps', 1))
        positions0 = np.array([si_utils.get_origin_from_sim(sim, asarray=True) for sim in sims])
        center = np.mean(positions0, 0)
        window = get_image_window(fused_image)

        max_size = None
        acum = 0
        for framei in range(nframes):
            c = (1 - np.cos(framei / (nframes - 1) * 2 * math.pi)) / 2
            acum += c / (nframes / 2)
            spacing1 = spacing[0] + (spacing[1] - spacing[0]) * acum
            for sim, position0 in zip(sims, positions0):
                transform = param_utils.identity_transform(ndim=2, t_coords=[0])
                transform[0][:2, 2] += (position0 - center) * spacing1
                si_utils.set_sim_affine(sim, transform, transform_key=self.transition_transform_key)
            frame = fusion.fuse(sims, transform_key=self.transition_transform_key).squeeze()
            frame = float2int_image(normalise_values(frame, window[0], window[1]))
            frame = cv.resize(np.asarray(frame), None, fx=scale, fy=scale)
            if max_size is None:
                max_size = frame.shape[1], frame.shape[0]
                video.size = max_size
            frame = image_reshape(frame, max_size)
            save_tiff(transition_filename + f'{framei:04d}.tiff', frame, None, pixel_size)
            video.write(frame)

        video.close()

    def summary_metrics(self):
        """The registration's metrics over all its pairs, by name; empty without a registration."""
        summary = self.metrics.get('summary', {})
        if self.reg_transform_key in summary:
            # the pairs' registration quality is kept apart from the metrics of their registered overlaps
            values = summary[self.reg_transform_key] | summary.get(default_transform_key, {})
        else:
            # loaded from the metrics file, without the overlap areas the summary weighs its pairs by
            pair_metrics = [pair.get(self.reg_transform_key, {}) for pair in self.metrics.get('pairs', {}).values()]
            names = {name for metrics in pair_metrics for name in metrics}
            values = {name: np.mean([metrics[name] for metrics in pair_metrics if name in metrics]) for name in names}
        return {name: float(value) for name, value in values.items() if value is not None and np.isfinite(value)}

    def get_metrics(self, metric_key=None, pair=None):
        metrics = self.metrics
        if pair is not None:
            if isinstance(pair, np.ndarray):
                pair = pair.tolist()
                pair = tuple(pair)
            metrics = metrics.get('pairs', {}).get(pair, {})
        else:
            if 'summary' in metrics:
                metrics = metrics['summary']
        reg_transform_key = self.reg_transform_key
        if reg_transform_key in metrics:
            transform_key = reg_transform_key
        elif default_transform_key in metrics:
            transform_key = default_transform_key
        elif metrics:
            transform_key = list(metrics)[-1]
        else:
            transform_key = None
        metrics = metrics.get(transform_key, {})
        if metric_key is not None:
            return metrics.get(metric_key)
        else:
            return metrics

    def current_fused_output(self, output_filename):
        """The fused OME-Zarr's path if it was written after every saved registration file (it shows that
        registration), else None."""
        path = self.output + output_filename + zarr_extension
        metadata = [os.path.join(path, name) for name in ('zarr.json', '.zattrs')
                    if os.path.exists(os.path.join(path, name))]
        if not metadata:
            return None
        registration_times = [os.path.getmtime(saved) for saved in self.saved_progress_paths()]
        if registration_times and max(registration_times) > os.path.getmtime(metadata[0]):
            return None
        return path

    def output_exists(self, output_filename, output_format):
        if not output_format.startswith('.'):
            output_format = '.' + output_format
        output_filename = self.output + output_filename + output_format
        if output_format == zarr_extension:
            return (os.path.exists(os.path.join(output_filename, '.zattrs')) or
                    os.path.exists(os.path.join(output_filename, 'zarr.json')))
        else:
            return os.path.exists(output_filename)
is_stack property

Stack registration: pair consecutive views, and treat the set as a z-stack rather than a tiled plane. Selected by pairing: stack only, as the plugin never sets an operation.

logging_dask = self.verbose instance-attribute
logging_time = self.verbose instance-attribute
msims property writable
verbose = verbose instance-attribute
__init__(operation='register', label='', input_path=None, output_path=None, source_metadata={}, extra_metadata={}, global_rotation=None, global_center=None, overwrite=True, clear=False, ui='', verbose=False, debug=False)
Source code in src\muvis_align\MVSRegistration.py
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
def __init__(self, operation='register', label='', input_path=None, output_path=None,
             source_metadata={}, extra_metadata={},
             global_rotation=None, global_center=None,
             overwrite=True, clear=False, ui='', verbose=False, debug=False):
    # set here as well as in init(): an instance may never have init() called on it
    self.verbose = verbose
    self.logging_dask = self.verbose
    self.logging_time = self.verbose
    self.reset()

    if input_path is not None:
        self.init(operation=operation, label=label, input_path=input_path, output_path=output_path,
                  source_metadata=source_metadata, extra_metadata=extra_metadata,
                  global_rotation=global_rotation, global_center=global_center,
                  overwrite=overwrite, clear=clear, ui=ui, verbose=verbose, debug=debug)
channel_groups()

Each source's output channel in a channel overlay: see split_registration.channel_groups.

Source code in src\muvis_align\MVSRegistration.py
1747
1748
1749
def channel_groups(self):
    """Each source's output channel in a channel overlay: see split_registration.channel_groups."""
    return channel_groups(self.sources, getattr(self, 'input_groups', None))
channel_labels(channels=None)

A label per channel of channel_groups(): as set in channels (extra_metadata's), else its sources' shared metadata channel label if no other channel has it, else their common label prefix, else 'channel '.

Source code in src\muvis_align\MVSRegistration.py
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
def channel_labels(self, channels=None):
    """A label per channel of channel_groups(): as set in `channels` (extra_metadata's), else its sources' shared
    metadata channel label if no other channel has it, else their common label prefix, else 'channel <index>'."""
    groups = self.channel_groups()
    members = [[index for index, member_group in enumerate(groups) if member_group == group]
               for group in range(max(groups, default=-1) + 1)]
    metadata_labels = []
    for group_members in members:
        group_labels = {(self.sources[index].get_channels() or [{}])[0].get('label') or ''
                        for index in group_members}
        metadata_labels.append(group_labels.pop() if len(group_labels) == 1 else '')
    labels = []
    for group, group_members in enumerate(members):
        label = metadata_labels[group] if metadata_labels.count(metadata_labels[group]) == 1 else ''
        if channels and group < len(channels):
            label = channels[group].get('label') or label
        labels.append(label or group_label([self.file_labels[index] for index in group_members])
                      or f'channel {group}')
    return labels
check_progress(output_filename, output_format)
Source code in src\muvis_align\MVSRegistration.py
914
915
916
917
918
919
920
921
922
def check_progress(self, output_filename, output_format):
    pair_mappings_filename = self.output + self.input_output_params.get('pair_mappings', default_pair_mappings_name)
    mappings_filename = self.output + self.input_output_params.get('mappings', default_mappings_name)
    if self.output_exists(output_filename, output_format):
        self.state = RegState.FUSED
    elif os.path.exists(mappings_filename):
        self.state = RegState.GLOBAL_REG
    elif os.path.exists(pair_mappings_filename):
        self.state = RegState.PAIRS_REG
create_fusion_method(fusion_method, sim0)
Source code in src\muvis_align\MVSRegistration.py
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
def create_fusion_method(self, fusion_method, sim0):
    if fusion_method is None:
        fusion_method = ''
    if 'compos' in fusion_method:
        fusion_method = None
        fuse_func = None
    elif 'exclus' in fusion_method:
        from muvis_align.fusion_methods.FusionMethodExclusive import FusionMethodExclusive
        fusion_method = FusionMethodExclusive(sim0, self.debug)
        fuse_func = fusion_method.fusion
    elif 'add' in fusion_method:
        from muvis_align.fusion_methods.FusionMethodAdditive import FusionMethodAdditive
        fusion_method = FusionMethodAdditive(sim0, self.debug)
        fuse_func = fusion_method.fusion
    else:
        from muvis_align.fusion_methods.FusionMethodAverage import FusionMethodAverage
        fusion_method = FusionMethodAverage(sim0, self.debug)
        fuse_func = fusion_method.fusion

    self.fusion_method = fusion_method

    return fuse_func
create_preview(output_filename=None, nom_msims=None, transform_key=None)
Source code in src\muvis_align\MVSRegistration.py
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
def create_preview(self, output_filename=None, nom_msims=None, transform_key=None):
    preview_scale = parse_scale(self.input_output_params.get('preview_scale'), default=16)
    is_stack = self.is_stack
    z_scale = get_metadata_z_scale(self.extra_metadata)

    msims = select_msim_subpyramid_at_scale(self.msims, self.sources, preview_scale)
    msims = self.flatfield_corrected_msims(msims, self.preprocess_params)
    if is_stack:
        msims = make_msims_3d(msims, z_scale, self.positions)

    if nom_msims is not None:
        preview_image0 = get_msim_image0(msims[0])
        nom_image0 = get_msim_image0(nom_msims[0])
        if preview_image0.sizes['x'] >= nom_image0.sizes['x']:
            logging.warning('Unable to generate scaled down preview due to lack of source pyramid sizes')
            return None

        if transform_key is not None and transform_key != self.source_transform_key:
            copy_transforms_to_msims(nom_msims, msims, transform_key)
    fusion_method, _ = self.fusion_settings()
    fused_msim, is_saved = self.fuse(msims, fusion_method=fusion_method, transform_key=transform_key,
                                     output_spacing='max', output_filename=output_filename)
    fused_sim = extract_sims_from_fused(fused_msim)
    preview_format = self.fusion_params.get('preview')
    if output_filename and (not is_saved or 'tif' in preview_format):
        self.save(output_filename, fused_sim.squeeze(), transform_key=transform_key, format=preview_format,
                  ome_version=self.fusion_params.get('ome_version') or default_ome_zarr_version)
    return fused_sim
create_registration_method(sim0, params={}, method='')
Source code in src\muvis_align\MVSRegistration.py
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
def create_registration_method(self, sim0, params={}, method=''):
    registration_method = None
    pairwise_reg_func_kwargs = None

    if not method:
        method = params.get('method',
                            params.get('name', ''))
    method = method.lower()
    method_kind = registration_method_kind(method)

    if method_kind == 'cpd':
        try:
            from muvis_align.registration_methods.RegistrationMethodCPD import RegistrationMethodCPD
        except ImportError:
            raise ImportError('probreg is required for CPD registration.')
        registration_method = RegistrationMethodCPD(sim0, params, self.debug)
        pairwise_reg_func = registration_method.registration
    elif method_kind == 'feature':
        if 'cv' in method:
            from muvis_align.registration_methods.RegistrationMethodCvFeatures import RegistrationMethodCvFeatures
            registration_method = RegistrationMethodCvFeatures(sim0, params, self.debug)
        else:
            from muvis_align.registration_methods.RegistrationMethodSkFeatures import RegistrationMethodSkFeatures
            registration_method = RegistrationMethodSkFeatures(sim0, params, self.debug)
        pairwise_reg_func = registration_method.registration
    elif method_kind == 'elastix':
        try:
            from itk import ElastixRegistrationMethod
        except ImportError:
            raise ImportError('ITK-Elastix is required for ITK-Elastix registration.')
        pairwise_reg_func = elastix_registration_or_identity
        # a translation to get close, then the project's transform type (multiview_stitcher's default ends rigid)
        transform_type = params.get('transform_type', 'rigid')
        pairwise_reg_func_kwargs = {'transform_types': ['translation'] if transform_type == 'translation'
                                    else ['translation', transform_type]}
    elif method_kind == 'ants':
        try:
            import ants
        except ImportError:
            raise ImportError('ANTsPy is required for ANTsPy registration.')
        pairwise_reg_func = registration.registration_ANTsPy
        pairwise_reg_func_kwargs = {
            'transform_types': ['Rigid'],
            "aff_random_sampling_rate": 0.5,
            "aff_iterations": (2000, 2000, 1000, 1000),
            "aff_smoothing_sigmas": (4, 2, 1, 0),
            "aff_shrink_factors": (16, 8, 2, 1),
        }
    else:
        pairwise_reg_func = registration.phase_correlation_registration

    self.registration_method = registration_method

    return method, pairwise_reg_func, pairwise_reg_func_kwargs
current_fused_output(output_filename)

The fused OME-Zarr's path if it was written after every saved registration file (it shows that registration), else None.

Source code in src\muvis_align\MVSRegistration.py
2549
2550
2551
2552
2553
2554
2555
2556
2557
2558
2559
2560
def current_fused_output(self, output_filename):
    """The fused OME-Zarr's path if it was written after every saved registration file (it shows that
    registration), else None."""
    path = self.output + output_filename + zarr_extension
    metadata = [os.path.join(path, name) for name in ('zarr.json', '.zattrs')
                if os.path.exists(os.path.join(path, name))]
    if not metadata:
        return None
    registration_times = [os.path.getmtime(saved) for saved in self.saved_progress_paths()]
    if registration_times and max(registration_times) > os.path.getmtime(metadata[0]):
        return None
    return path
discard_saved_progress(output=None)

Removes the saved registration, which no longer matches the sources.

Source code in src\muvis_align\MVSRegistration.py
908
909
910
911
912
def discard_saved_progress(self, output=None):
    """Removes the saved registration, which no longer matches the sources."""
    for path in self.saved_progress_paths(output):
        os.remove(path)
    logging.info(f'Saved registration in {output or self.output} discarded')
ensure_msims(progress_factory=None, target_scale=None, weight=1, ends_only=False)

The lazy build the msims property triggers, callable ahead of time to report its progress.

target_scale builds each source's pyramid from the level that scale needs, cached per scale, never as self.msims, which stays the full-resolution pyramid. ends_only builds just each source's finest level at that scale and its coarsest (see source_levels): all pre-processing's consumers read.

Source code in src\muvis_align\MVSRegistration.py
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
def ensure_msims(self, progress_factory=None, target_scale=None, weight=1, ends_only=False):
    """The lazy build the msims property triggers, callable ahead of time to report its progress.

    `target_scale` builds each source's pyramid from the level that scale needs, cached per scale, never as
    self.msims, which stays the full-resolution pyramid. `ends_only` builds just each source's finest level at
    that scale and its coarsest (see source_levels): all pre-processing's consumers read.
    """
    # a factor as text ('2') or a pixel size ('10um'): one cache entry however it was written
    target_scale = parse_scale(target_scale, default=None)
    key = self._scaled_msims_key(target_scale, ends_only)
    if (target_scale or ends_only) and key not in self._scaled_msims:
        from_levels = [get_level_from_scale(source, target_scale)[0] if target_scale else 0
                       for source in self.sources]
        # nothing to skip: the shared full-resolution build serves, unless only the ends are wanted,
        # which the shared build must not be cut down to
        if any(from_levels) or ends_only:
            self._scaled_msims[key] = self._build_msims(progress_factory=progress_factory,
                                                        from_levels=from_levels, store=False,
                                                        weight=weight, ends_only=ends_only)
    if key in self._scaled_msims:
        return self._scaled_msims[key]
    if self._msims is None:
        self._build_msims(progress_factory=progress_factory, weight=weight)
    return self._msims
estimate_fusion_size(fusion_method=None, output_spacing=None, transform_key=None, dimension=None, block_size=None, extra_metadata=None)

Uncompressed bytes per level of the OME-Zarr fuse() would write, from source geometry alone (no msims): {'levels': [{'spacing', 'shape', 'bytes'}], 'bytes', 'native'}. Only the blocks some source reaches are written - exactly these; a channel overlay is saved whole.

Source code in src\muvis_align\MVSRegistration.py
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
def estimate_fusion_size(self, fusion_method=None, output_spacing=None, transform_key=None, dimension=None,
                         block_size=None, extra_metadata=None):
    """Uncompressed bytes per level of the OME-Zarr fuse() would write, from source geometry alone (no msims):
    {'levels': [{'spacing', 'shape', 'bytes'}], 'bytes', 'native'}.
    Only the blocks some source reaches are written - exactly these; a channel overlay is saved whole."""
    if extra_metadata is None:
        extra_metadata = self.extra_metadata
    is_channel_overlay = self.is_channel_overlay(dimension, extra_metadata)
    if not self.sources:
        return None
    if 'compos' in (fusion_method or '') and not is_channel_overlay:
        return self._composed_size()
    if transform_key is None:
        transform_key = self.reg_transform_key
    z_scale, num_z_positions, promote_z = self._fusion_z_geometry(extra_metadata)
    is_native, spacing_method = self._output_spacing_method(output_spacing, not is_channel_overlay)
    dtype = np.dtype(self.sources[0].dtype)
    nonspatial = int(np.prod([size for dim, size in self.sources[0].get_size().items() if dim not in 'zyx']))
    voxel_bytes = nonspatial * dtype.itemsize

    translations, source_props = self.source_geometry(transform_key, promote_z=promote_z, z_scale=z_scale)
    matrices = [np.asarray(props['transform'], dtype=float) for props in source_props]

    stacks = {}

    def stack(index, level):
        if (index, level) not in stacks:
            stacks[index, level] = build_source_stack_props(
                self.sources[index], self._msim_output_order, translations[index], None, None,
                z_scale=self._msim_z_scale, level=level, promote_z=promote_z)
        return stacks[index, level]

    views0 = [(stack(index, 0), matrix) for index, matrix in enumerate(matrices)]
    level0 = calc_output_properties_from_views(views0, output_spacing_method=spacing_method, z_scale=z_scale)
    if is_channel_overlay:
        return self._channel_overlay_size(level0, voxel_bytes * len(set(self.channel_groups())))
    levels = []
    source_spacings = [min(view[0]['spacing'][dim] for dim in 'yx') for view in views0]
    scaled_dims = ['y', 'x'] + (['z'] if views0[0][0]['shape'].get('z', 1) > 1 else [])
    chunksize = self._export_chunksize(dtype, level0, [view[0] for view in views0], num_z_positions, block_size)
    for level_spacing, properties, selected in export_levels(source_spacings, level0, scaled_dims,
                                                             native=is_native):
        output_properties = {key: properties[key] for key in ('spacing', 'origin', 'shape')}
        views = []
        for index in selected:
            # as fusion picks it (fuse_levels_to_ome_zarr)
            level = coarsest_level_within([stack(index, level)['spacing'] for level in range(len(
                self.sources[index].shapes))], output_properties['spacing'],
                dims=list(output_properties['spacing']))
            views.append((stack(index, level), matrices[index]))
        chunk, groups = level_blocks(views, output_properties, chunksize)
        levels.append({'spacing': float(level_spacing), 'shape': output_properties['shape'],
                       'bytes': written_voxels(groups, output_properties, chunk) * voxel_bytes})
    return {'levels': levels, 'bytes': sum(level['bytes'] for level in levels), 'native': is_native}
flatfield_corrected_msims(msims, params, progress_factory=None)

msims (every source, in source order) flat-field corrected by flatfield_models(); unchanged without one.

Source code in src\muvis_align\MVSRegistration.py
1363
1364
1365
1366
1367
1368
1369
def flatfield_corrected_msims(self, msims, params, progress_factory=None):
    """msims (every source, in source order) flat-field corrected by flatfield_models(); unchanged without one."""
    models = self.flatfield_models(params, progress_factory=progress_factory)
    if models is None:
        return msims
    return [apply_msim_flatfield_model(msim, model) if model is not None else msim
            for msim, model in zip(msims, models)]
flatfield_groups()

Source indices per flat-field model: a section's sources of one group that cover the same field (pixel size x image size), as images of a different field share no shading.

Source code in src\muvis_align\MVSRegistration.py
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
def flatfield_groups(self):
    """Source indices per flat-field model: a section's sources of one group that cover the same field (pixel size
    x image size), as images of a different field share no shading."""
    section_of = {index: section for section, indices in enumerate(group_sims_by_z(self.positions, self.positions))
                  for index in indices}
    groups = {}
    for index, (group, source, scale) in enumerate(zip(self.source_groups(), self.sources, self.scales)):
        size = source.get_size()
        field = tuple(float(f'{size[dim] * scale[dim]:.4g}') for dim in 'yx' if dim in size and dim in scale)
        groups.setdefault((section_of.get(index), group, field), []).append(index)
    return list(groups.values())
flatfield_model(index, params)

Source index's flat-field model, its group's computed from that group's tiles alone when first asked: None without quantiles or for too few images, and with foreground filtering until flatfield_models() has computed every group.

Source code in src\muvis_align\MVSRegistration.py
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
def flatfield_model(self, index, params):
    """Source `index`'s flat-field model, its group's computed from that group's tiles alone when first asked:
    None without quantiles or for too few images, and with foreground filtering until flatfield_models() has
    computed every group."""
    settings = self.flatfield_settings(params)
    if not settings[0]:
        return None
    with self._flatfield_lock:
        group = self._flatfield_group_of(settings)[index]
        if group not in self._flatfield_models and not settings[2]:
            self._calc_flatfield_groups([group], settings)
        return self._flatfield_models.get(group)
flatfield_models(params, msims=None, foreground_map=None, progress_factory=None)

Every source's flat-field model (None without quantiles), the sections not computed yet computed together, as a progress phase: from msims (every source at the pre-processing scale) if given, else from their tiles.

Source code in src\muvis_align\MVSRegistration.py
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
def flatfield_models(self, params, msims=None, foreground_map=None, progress_factory=None):
    """Every source's flat-field model (None without quantiles), the sections not computed yet computed together,
    as a progress phase: from `msims` (every source at the pre-processing scale) if given, else from their tiles."""
    settings = self.flatfield_settings(params)
    if not settings[0]:
        return None
    with self._flatfield_lock:
        group_of = self._flatfield_group_of(settings)
        missing = self._missing_flatfield_groups(settings)
        if missing:
            logging.info(f'Flat-field model of {len(missing)} group(s)...')
            if msims is not None:
                msims = dict(enumerate(msims))
            with progress_phase(progress_factory, total=1, desc='Flat-field model') as pbar:
                self._calc_flatfield_groups(missing, settings, msims, foreground_map)
                if pbar is not None:
                    pbar.update(1)
        return [self._flatfield_models[group_of[index]] for index in range(len(self.sources))]
flatfield_pending(params)

Whether flatfield_models() would compute a model, and so report a progress phase of its own.

Source code in src\muvis_align\MVSRegistration.py
1349
1350
1351
1352
1353
1354
1355
1356
def flatfield_pending(self, params):
    """Whether flatfield_models() would compute a model, and so report a progress phase of its own."""
    settings = self.flatfield_settings(params)
    if not settings[0]:
        return False
    with self._flatfield_lock:
        self._flatfield_group_of(settings)
        return bool(self._missing_flatfield_groups(settings))
flatfield_settings(params) staticmethod

(quantiles, scale, filter_foreground) of pre-processing params, parsed: what a flat-field model depends on.

Source code in src\muvis_align\MVSRegistration.py
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
@staticmethod
def flatfield_settings(params):
    """(quantiles, scale, filter_foreground) of pre-processing params, parsed: what a flat-field model depends on."""
    quantiles = params.get('flatfield_quantiles')
    if isinstance(quantiles, str):
        quantiles = [quantile for quantile in quantiles.split(',') if quantile.strip()]
    if not isinstance(quantiles, (list, tuple)):
        quantiles = [quantiles] if quantiles else []
    return (tuple(float(quantile) for quantile in quantiles), parse_scale(params.get('scale'), default=None),
            bool(params.get('filter_foreground')))
fuse(msims, fusion_method=None, output_spacing=None, transform_key=None, dimension=None, output_filename=None, block_size=None, ome_version=default_ome_zarr_version, extra_metadata=None, output_chunksize=None, workers=None, level_bytes=None, source_indices=None)

Fuse the sources' msims into one multiscale output msim; returns (fused, whether it was written). Sims are wrapped first with util.wrap_sims_as_msims. source_indices: each msim's source, when not all are given in order, for its channel in a channel overlay.

output_chunksize, a chunk size per spatial dim, is used as-is; otherwise it is sized against memory. Written to a file, every level is fused from the sources (fusion_blocks.export_levels): with output_spacing None or 'native' each at its own pixel size from the sources at least that fine, else doubling from the set spacing. In memory, 'native' stays 'mean'. level_bytes (estimate_fusion_size's, a level) shares the progress bar out over the levels.

Source code in src\muvis_align\MVSRegistration.py
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
def fuse(self, msims, fusion_method=None, output_spacing=None, transform_key=None,
         dimension=None, output_filename=None,
         block_size=None, ome_version=default_ome_zarr_version, extra_metadata=None,
         output_chunksize=None, workers=None, level_bytes=None, source_indices=None):
    """Fuse the sources' msims into one multiscale output msim; returns (fused, whether it was written).
    Sims are wrapped first with util.wrap_sims_as_msims. `source_indices`: each msim's source, when not all are
    given in order, for its channel in a channel overlay.

    output_chunksize, a chunk size per spatial dim, is used as-is; otherwise it is sized against memory.
    Written to a file, every level is fused from the sources (fusion_blocks.export_levels): with output_spacing
    None or 'native' each at its own pixel size from the sources at least that fine, else doubling from the set
    spacing. In memory, 'native' stays 'mean'. `level_bytes` (estimate_fusion_size's, a level) shares the progress
    bar out over the levels.
    """
    logging.info('Fusion...')
    if output_filename is not None:
        output_filename = self.output + output_filename

    # only its dtype is read, which needs no full sim
    sim0 = get_msim_image0(msims[0])

    if extra_metadata is None:
        extra_metadata = self.extra_metadata

    channels = extra_metadata.get('channels', []) if isinstance(extra_metadata, dict) else []
    is_channel_overlay = self.is_channel_overlay(dimension, extra_metadata)

    if transform_key is None:
        transform_key = self.reg_transform_key

    z_scale, num_z_positions, promote_z = self._fusion_z_geometry(extra_metadata)
    # the one place a stack becomes 3D: msims are stored 2D, and each consumer that needs the stacked geometry
    # asks for it, rather than every level of every source being rebuilt up front
    if promote_z:
        with Timer('fusion: make_msims_3d', verbose=self.logging_time):
            msims = make_msims_3d(msims, z_scale=z_scale, positions=self.positions)
    if not is_channel_overlay:
        with Timer('fusion: unify channels', verbose=self.logging_time):
            msims = unify_msim_channels(msims)

    is_native, output_spacing = self._output_spacing_method(
        output_spacing, output_filename is not None and not is_channel_overlay)
    with Timer('fusion: output properties', verbose=self.logging_time):
        output_stack_properties = calc_output_properties(msims, transform_key,
                                                         output_spacing_method=output_spacing, z_scale=z_scale)

    if self.verbose:
        logging.info(f'Output stack: {numpy_to_native(output_stack_properties)}')
    data_size = np.prod(list(output_stack_properties['shape'].values())) * sim0.dtype.itemsize
    # the bounding box, not what is written: see estimate_fusion_size()
    logging.info(f'Fusion output bounding box at full resolution: {print_hbytes(data_size)}')

    # peak memory follows the sources in one output chunk, each transformed to a full float32 chunk;
    # the input's own chunk grid would put every source in one chunk of a coarse output
    default_output_chunksize = get_chunk_sizes(sim0.dtype, list(output_stack_properties['shape']),
                                               num_sources=len(msims),
                                               num_z_positions=num_z_positions)
    saving_zarr = False
    if is_channel_overlay:
        # each channel's sources fused together, onto the same grid
        source_groups = self.channel_groups()
        groups = [source_groups[index] for index in (source_indices or range(len(msims)))]
        labels = self.channel_labels(channels)
        labels = [labels[group] for group in sorted(set(groups))]
        fuse_func = self.create_fusion_method(fusion_method, sim0)
        channel_results = []
        for group in sorted(set(groups)):
            group_msims = unify_msim_channels([msim for msim, msim_group in zip(msims, groups)
                                               if msim_group == group])
            channel_results.append(fusion.fuse(
                group_msims,
                # compose (no fusion function) still fuses, as a channel needs one image
                fusion_func=fuse_func or fusion.weighted_average_fusion,
                transform_key=transform_key,
                output_stack_properties=output_stack_properties,
                output_chunksize=output_chunksize or get_chunk_sizes(
                    sim0.dtype, list(output_stack_properties['shape']), num_sources=len(group_msims),
                    num_z_positions=num_z_positions)))
        fused_image = combine_msims_as_channels(channel_results, labels)
    else:
        if fusion_method:
            logging.info(f'Fusion method: {fusion_method}')
        else:
            logging.info('Fusion method: [default method]')
        fuse_func = self.create_fusion_method(fusion_method, sim0)
        if fuse_func:
            saving_zarr = output_filename is not None
            if output_chunksize is None and saving_zarr:
                with Timer('fusion: export chunk sizes', verbose=self.logging_time):
                    output_chunksize = self._export_chunksize(sim0.dtype, output_stack_properties, msims,
                                                              num_z_positions, block_size)
            if output_chunksize is None:
                output_chunksize = dict(default_output_chunksize)
            # a chunk past the output's extent also fails dask's zarr write-alignment check, with a warning
            output_chunksize = {dim: min(int(size), output_stack_properties['shape'].get(dim, int(size)))
                                for dim, size in output_chunksize.items()}
            if self.verbose:
                # logged once settled by the branches above
                logging.info(f'Fusion output_chunksize: {numpy_to_native(output_chunksize)}'
                             f' ({len(msims)} sources over'
                             f' {max(1, num_z_positions)} z position(s))')
            if saving_zarr:
                if not output_filename.lower().endswith('.zarr'):
                    output_filename += zarr_extension
                zarr_options = {'ngff_version': ome_version}
            else:
                zarr_options = None
            with dask.config.set(scheduler='threads'), warnings.catch_warnings():
                # fused blocks are whole zarr chunks, one writer each: dask flags every edge block, whose chunk
                # the array ends in. Set here, as a napari worker puts 'always' ahead of any module-level filter
                warnings.filterwarnings('ignore', message='The input Dask array will be rechunked',
                                        category=PerformanceWarning)
                if saving_zarr:
                    source_spacings = [min(si_utils.get_spacing_from_sim(get_msim_image0(msim))[dim]
                                           for dim in 'yx') for msim in msims]
                    # a section stack keeps every section at every level; a volume is scaled in z too
                    scaled_dims = ['y', 'x'] + (['z'] if get_msim_image0(msims[0]).sizes.get('z', 1) > 1 else [])
                    levels = export_levels(source_spacings, output_stack_properties, scaled_dims, native=is_native)
                    # native: tiles over the overview, where a finer source covers a pixel the coarser one under it
                    # is left out, each level from the sources it has; a set spacing averages them as one, and
                    # every level has all of them, so averaging the level above gives it
                    fused_image = fuse_levels_to_ome_zarr(
                        msims, levels, output_filename, transform_key, output_stack_properties, output_chunksize,
                        fusion_func=fuse_func, zarr_options=zarr_options,
                        batch_options=self._fusion_batch_options(saving_zarr, max_workers=workers),
                        ranks=source_spacings if is_native else None, average_levels=not is_native,
                        level_weights=level_bytes)
                else:
                    fused_image = fusion.fuse(msims, fusion_func=fuse_func, transform_key=transform_key,
                                              output_stack_properties=output_stack_properties,
                                              output_chunksize=output_chunksize)
        else:
            # 'compose' mode: no fusion, the sources as they are
            fused_image = msims
            if output_filename is not None:
                if str(ome_version) != '0.6':
                    logging.warning(f'Compose writes an OME-Zarr scene, which only 0.6 has: written as 0.6, '
                                    f'not {ome_version}')
                if not output_filename.lower().endswith('.zarr'):
                    output_filename += zarr_extension
                self.save_composed(output_filename, msims, channels)
                saving_zarr = True
    return fused_image, saving_zarr
fuses_to_zarr(fusion_method=None, dimension=None, extra_metadata=None)

Whether fuse() given an output filename writes the export itself; otherwise its caller saves the lazy result it returns, which is where the work is then done.

Source code in src\muvis_align\MVSRegistration.py
2018
2019
2020
2021
def fuses_to_zarr(self, fusion_method=None, dimension=None, extra_metadata=None):
    """Whether fuse() given an output filename writes the export itself; otherwise its caller saves the lazy
    result it returns, which is where the work is then done."""
    return not self.is_channel_overlay(dimension, extra_metadata) and 'compos' not in (fusion_method or '')
fusion_settings()

(method, output spacing) from the fusion section, the spacing defaulting to native as in the plugin.

Source code in src\muvis_align\MVSRegistration.py
1477
1478
1479
def fusion_settings(self):
    """(method, output spacing) from the fusion section, the spacing defaulting to native as in the plugin."""
    return self.fusion_params.get('method', ''), self.fusion_params.get('spacing') or 'native'
get_metrics(metric_key=None, pair=None)
Source code in src\muvis_align\MVSRegistration.py
2524
2525
2526
2527
2528
2529
2530
2531
2532
2533
2534
2535
2536
2537
2538
2539
2540
2541
2542
2543
2544
2545
2546
2547
def get_metrics(self, metric_key=None, pair=None):
    metrics = self.metrics
    if pair is not None:
        if isinstance(pair, np.ndarray):
            pair = pair.tolist()
            pair = tuple(pair)
        metrics = metrics.get('pairs', {}).get(pair, {})
    else:
        if 'summary' in metrics:
            metrics = metrics['summary']
    reg_transform_key = self.reg_transform_key
    if reg_transform_key in metrics:
        transform_key = reg_transform_key
    elif default_transform_key in metrics:
        transform_key = default_transform_key
    elif metrics:
        transform_key = list(metrics)[-1]
    else:
        transform_key = None
    metrics = metrics.get(transform_key, {})
    if metric_key is not None:
        return metrics.get(metric_key)
    else:
        return metrics
group_pair_metrics()
Source code in src\muvis_align\MVSRegistration.py
2320
2321
2322
2323
def group_pair_metrics(self):
    return {labels: {self.reg_transform_key: entry.get('metrics', {})
                     | {default_quality_key: entry[default_quality_key]}}
            for labels, entry in self.group_pairs.items()}
has_eager_pre_processing(params) staticmethod

Whether pre-processing will compute over the image data, rather than only rebuild the graph.

Scaling and Gaussian filtering are lazy; flat-field, normalisation and foreground filtering read every source.

Source code in src\muvis_align\MVSRegistration.py
218
219
220
221
222
223
224
225
@staticmethod
def has_eager_pre_processing(params):
    """Whether pre-processing will compute over the image data, rather than only rebuild the graph.

    Scaling and Gaussian filtering are lazy; flat-field, normalisation and foreground filtering read every source.
    """
    return bool(params.get('flatfield_quantiles') or params.get('filter_foreground')
                or normalisation_enabled(params.get('normalisation')))
has_saved_progress(output_filename=None, output_format=None)

Whether there is a saved registration to resume: the files check_progress() reads, without changing the state. Without an output filename only the mappings are checked.

Source code in src\muvis_align\MVSRegistration.py
889
890
891
892
893
894
895
896
def has_saved_progress(self, output_filename=None, output_format=None):
    """Whether there is a saved registration to resume: the files check_progress() reads, without changing
    the state. Without an output filename only the mappings are checked."""
    pair_mappings_filename = self.output + self.input_output_params.get('pair_mappings', default_pair_mappings_name)
    mappings_filename = self.output + self.input_output_params.get('mappings', default_mappings_name)
    return bool((output_filename and self.output_exists(output_filename, output_format))
                or os.path.exists(mappings_filename)
                or os.path.exists(pair_mappings_filename))
init(operation='', label='', input_path=None, input_labels=None, output_path=None, source_metadata={}, extra_metadata={}, global_rotation=None, global_center=None, overwrite=True, clear=False, ui='', verbose=False, debug=False, pairing='')
Source code in src\muvis_align\MVSRegistration.py
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
def init(self, operation='', label='', input_path=None, input_labels=None, output_path=None,
         source_metadata={}, extra_metadata={}, global_rotation=None, global_center=None,
         overwrite=True, clear=False, ui='', verbose=False, debug=False, pairing=''):
    self.overwrite = overwrite
    self.clear = clear
    self.ui = ui
    self.verbose = verbose
    self.debug = debug
    self.logging_dask = self.verbose
    self.logging_time = self.verbose
    self.mpl_ui = ('mpl' in self.ui or 'plot' in self.ui)
    self.operation = operation
    self.pairing = (pairing or '').lower()
    self.fileset_label = label
    self.global_rotation = global_rotation
    self.global_center = global_center
    self.source_transform_key = 'source_metadata'
    self.reg_transform_key = 'registered'
    self.transition_transform_key = 'transition'
    self.msims = []
    self.sources = []
    self.state = RegState.INIT
    self.start_time = datetime.now()

    self.input_path = input_path
    # a folder stands for its files, but a zarr folder is one image
    patterns = [os.path.join(path, '*') if os.path.isdir(path) and '.zar' not in path.lower() else path
                for path in ensure_list(input_path)]
    self.input_dir = pattern_base_dir(patterns[0])
    self.filenames = [Path(path).as_posix() for path in dir_regex(patterns)]
    if not self.filenames:
        return False
    item_files = [{Path(path).as_posix() for path in dir_regex(pattern)} for pattern in patterns]
    # the input path item each file came from: a channel each, in a channel registration
    self.input_groups = [next(item for item, files in enumerate(item_files) if filename in files)
                         for filename in self.filenames]

    if input_labels:
        self.file_labels = input_labels
    else:
        self.file_labels = get_unique_file_labels(self.filenames)

    self.source_metadata = source_metadata
    self.extra_metadata = extra_metadata

    try:
        output_path = output_path.format_map(split_numeric_dict(self.filenames[0]))
        self.output = os.path.join(self.input_dir, output_path)    # preserve trailing slash: do not use os.path.normpath()
        output_dir = os.path.dirname(self.output)
        if self.clear:
            shutil.rmtree(output_dir, ignore_errors=True)
        if not os.path.exists(output_dir):
            os.makedirs(output_dir, exist_ok=True)
    except Exception as e:
        logging.error(f"Error initializing output directory: {e}")
        return False

    return True
init_data(source_metadata={}, extra_metadata={}, target_scale=None, store=True, progress_factory=None)
Source code in src\muvis_align\MVSRegistration.py
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
def init_data(self, source_metadata={}, extra_metadata={}, target_scale=None, store=True,
              progress_factory=None):
    if not source_metadata:
        source_metadata = self.source_metadata
    if not extra_metadata:
        extra_metadata = self.extra_metadata
    source_metadata = import_metadata(source_metadata, input_path=self.input_path)
    extra_metadata = import_metadata(extra_metadata, input_path=self.input_path)
    source_metadata_changed = (source_metadata != self.source_metadata)
    self.source_metadata = source_metadata
    self.extra_metadata = extra_metadata
    z_scale = get_metadata_z_scale(source_metadata) or get_metadata_z_scale(extra_metadata)

    if len(self.filenames) == 0:
        raise ValueError('No input files')

    logging.info('Initialising sources...')
    if not self.sources or source_metadata_changed:
        self.init_sources(progress_factory=progress_factory)
    sources = self.sources
    source0 = sources[0]
    scales = []
    translations = []
    rotations = []
    levels = []
    rescales = []

    is_3d = (source0.get_size().get('z', 0) > 1)
    is_stack = self.is_stack
    output_order = 'zyx' if is_3d else 'yx'

    ndims = len(output_order)
    if source0.get_nchannels() > 1:
        if source0.is_rgb:
            output_order = output_order + 'c'
        else:
            output_order = 'c' + output_order

    last_z_position = None
    delta_zs = []
    # timed apart from init_sources(): these serial per-source passes are not in its pooled timing
    geometry_timer = Timer('init_data: resolve per-source geometry', verbose=self.logging_time)
    geometry_timer.start()
    for filename, source in zip(self.filenames, sources):
        # metadata ImageSource already resolved (fix_metadata); no pixel data is read
        # copies: the z and x/y filled in below are this run's, not the source's
        scale = dict(source.get_pixel_size())
        translation = dict(source.get_position())
        rotation = source.get_rotation()

        level = 0
        rescale = {}
        if target_scale:
            # Only downscaling
            level, rescale, scale = get_level_from_scale(source, target_scale)
        if 'z' in translation:
            z_position = translation['z']
        else:
            z_position = 0
        if last_z_position is not None and z_position != last_z_position:
            delta_zs.append(z_position - last_z_position)
        if self.global_rotation is not None:
            rotation = self.global_rotation

        scales.append(scale)
        translations.append(translation)
        rotations.append(rotation)
        levels.append(level)
        rescales.append(rescale)
        last_z_position = z_position
    geometry_timer.record()
    geometry_timer.get_total_time()

    dimension = getattr(self, 'registration_dimension', None) or self.input_output_params.get('registration_dimension')
    if not is_3d and self.is_channel_overlay(dimension, extra_metadata):
        # a z per channel is a focus offset, not stack planes: all channels on the first one's plane
        channel_zs = {}
        for group, translation in zip(self.channel_groups(), translations):
            channel_zs.setdefault(group, set()).add(translation.get('z'))
        if all(len(zs) == 1 for zs in channel_zs.values()) and 'z' in translations[0]:
            for translation in translations:
                translation['z'] = translations[0]['z']

    if 'z' in output_order and z_scale is None:
        if len(delta_zs) > 0:
            z_scale = np.min(delta_zs)
        else:
            z_scale = 1

    if 'norm' in source_metadata:
        sizes = [source.get_physical_size() for source in sources]
        center = {dim: 0 for dim in output_order}
        if 'center' in source_metadata:
            if 'global' in source_metadata:
                center = self.global_center
            else:
                center = {dim: float(np.mean([translation[dim] for translation in translations])) for dim in translations[0]}
        translations, rotations = normalise_rotated_positions(translations, rotations, sizes, center, len(output_order))

    #translations = [np.array(translation) * 1.25 for translation in translations]

    # applies only to 3D sources: make_msims_3d() spaces 2D slices but skips a source already 3D
    increase_z_positions = is_stack

    z_position = 0
    final_scales = []
    final_translations = []
    transforms = []
    msims = [] if not store else None
    transform_timer = Timer(f'init_data: build per-source transforms'
                            f'{"" if store else " and msims"}', verbose=self.logging_time)
    transform_timer.start()
    for source, level, rescale, scale, translation, rotation, file_label in zip(
            sources, levels, rescales, scales, translations, rotations, self.file_labels):
        # transform #dimensions need to match
        if 'z' in output_order:
            if len(scale) > 0 and 'z' not in scale:
                scale['z'] = abs(z_scale)
            if (len(translation) > 0 and 'z' not in translation) or increase_z_positions:
                translation['z'] = z_position
            if increase_z_positions:
                z_position += z_scale
        if rotation is None or 'norm' in source_metadata:
            # if positions are normalised, don't use rotation
            transform = None
        else:
            transform = param_utils.invert_coordinate_order(
                create_transform(translation, rotation, matrix_size=ndims + 1)
            )
        if file_label in extra_metadata:
            transform2 = extra_metadata[file_label]
            if transform is None:
                transform = np.array(transform2)
            else:
                transform = np.array(combine_transforms([transform, transform2]))

        translation = complete_translation(translation)
        transforms.append(transform)
        if not store:
            # from the source's cached msim, adding only the run-level deltas no single source knows
            msim = build_source_msim(source, output_order, translation, transform, self.source_transform_key,
                                     z_scale=z_scale)
            msims.append(msim)
        final_scales.append(scale)
        final_translations.append(translation)
    transform_timer.record()
    transform_timer.get_total_time()

    if store:
        self.scales = final_scales
        self.positions = final_translations
        self.rotations = rotations
        self.state = RegState.SIMS_INIT
        # the msim build itself waits until self.msims is read
        self._msim_output_order = output_order
        self._msim_z_scale = z_scale
        self._msim_transforms = transforms
        self._msims = None
        # the geometry was re-resolved, so the per-scale builds are stale too
        self._scaled_msims = {}
        self._scaled_source_msims = {}
        return None

    return msims
init_params(params_general, params, label='', input_path=None, global_rotation=None, global_center=None)

A CLI operation: its settings in the project template's sections (input_output, pre_processing, registration, fusion), as a project file has them, with the run's general settings.

Source code in src\muvis_align\MVSRegistration.py
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
def init_params(self, params_general, params, label='', input_path=None, global_rotation=None, global_center=None):
    """A CLI operation: its settings in the project template's sections (input_output, pre_processing,
    registration, fusion), as a project file has them, with the run's general settings."""
    check_operation_layout(params)
    self.params_general = params_general
    self.params = params
    self.input_output_params = params.get('input_output', {})
    self.preprocess_params = params.get('pre_processing', {})
    self.register_params = params.get('registration', {})
    self.fusion_params = params.get('fusion', {})
    input_output = self.input_output_params
    if input_path is None:
        input_path = input_output.get('input_path')

    return self.init(operation=self.register_params.get('operation', ''), label=label, input_path=input_path,
                     pairing=self.register_params.get('pairing', ''),
                     input_labels=input_output.get('labels'),
                     output_path=input_output.get('output_path'),
                     source_metadata=source_metadata_from_params(input_output),
                     extra_metadata=extra_metadata_from_params(input_output),
                     global_rotation=global_rotation, global_center=global_center,
                     overwrite=input_output.get('overwrite', False), clear=input_output.get('clear', False),
                     ui=params_general.get('ui', ''),
                     verbose=params_general.get('verbose', False), debug=params_general.get('debug', False))
init_progress(output_filename, output_format, progress_factory=None)
Source code in src\muvis_align\MVSRegistration.py
 924
 925
 926
 927
 928
 929
 930
 931
 932
 933
 934
 935
 936
 937
 938
 939
 940
 941
 942
 943
 944
 945
 946
 947
 948
 949
 950
 951
 952
 953
 954
 955
 956
 957
 958
 959
 960
 961
 962
 963
 964
 965
 966
 967
 968
 969
 970
 971
 972
 973
 974
 975
 976
 977
 978
 979
 980
 981
 982
 983
 984
 985
 986
 987
 988
 989
 990
 991
 992
 993
 994
 995
 996
 997
 998
 999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
def init_progress(self, output_filename, output_format, progress_factory=None):
    pair_mappings_filename = self.output + self.input_output_params.get('pair_mappings', default_pair_mappings_name)
    mappings_filename = self.output + self.input_output_params.get('mappings', default_mappings_name)
    metrics_filename = self.output + metrics_name
    self.check_progress(output_filename, output_format)

    load_pairs = self.is_pairs_registered() and os.path.exists(pair_mappings_filename)
    self.group_pairs = {}
    # builds no msims: the graph comes from source geometry, mappings are applied as msims are built
    file_index = file_list_index_finder(self.filenames)

    if load_pairs:
        logging.info(f'Loading pair mapping from {pair_mappings_filename}')
        pairs = import_json(pair_mappings_filename)
        indexed_pair_transforms = {}
        indexed_qualities = {}
        indexed_bboxes = {}
        for key, value in pairs.items():
            key1, key2 = json.loads(key)
            if value.get('kind') == split_group_pair_kind:
                # a label such as 'S000' would also match a file under S000/ as a tile pair
                self.group_pairs[(key1, key2)] = {item: data for item, data in value.items() if item != 'kind'}
            else:
                index1, index2 = file_index(key1), file_index(key2)
                if index1 is not None and index2 is not None:
                    indexed_key = index1, index2
                    indexed_pair_transforms[indexed_key] = (
                        xaffine_from_matrix(value['mapping'], with_t=True))
                    indexed_qualities[indexed_key] = np.array(value.get(default_quality_key, 0))
                    if 'bbox' in value:
                        indexed_bboxes[indexed_key] = data_array_from_values(value['bbox'])
        # not per-source-divisible, but long enough to report as two steps
        progress_context = progress_phase(progress_factory, total=2, desc='Loading pair registration')
        with progress_context as pbar:
            self.pair_msims = self._msims
            self.pairs = list(indexed_pair_transforms.keys())
            self.metrics = {
                'summary': {default_transform_key: {default_quality_key: np.mean(list(indexed_qualities.values()))}},
                'pairs': {key: {default_transform_key: {default_quality_key: value.item()}}
                          for key, value in indexed_qualities.items()}
            }
            if pbar is not None:
                pbar.update(1)
            # built directly from the saved pairs, rather than multiview_stitcher rediscovering each overlap;
            # the saved bboxes give the overlap weight
            overlaps = {key: float(np.prod([abs(high - low) for low, high
                                            in zip(*np.array(value).reshape(2, -1))]))
                        for key, value in indexed_bboxes.items()}
            self.pairs_graph = build_pairs_graph(self.source_geometry(self.source_transform_key)[1], self.pairs,
                                                 overlaps=overlaps)
            if pbar is not None:
                pbar.update(1)
        nx.set_edge_attributes(self.pairs_graph, indexed_pair_transforms, default_transform_key)
        nx.set_edge_attributes(self.pairs_graph, indexed_qualities, default_quality_key)
        nx.set_edge_attributes(self.pairs_graph, indexed_bboxes, 'bbox')

    if self.is_global_registered():
        logging.info(f'Loading global mapping from {mappings_filename}')

        mappings = read_transforms(mappings_filename)
        missing = [filename for filename in self.filenames if find_file_dict_item(mappings, filename) is None]
        if missing:
            # mappings for a different fileset (e.g. a shared output dir) can't be trusted as this one's
            logging.warning(f'{mappings_filename} has no mapping for {len(missing)} of '
                            f'{len(self.filenames)} files (e.g. {missing[0]}) - ignoring it')
            self.state = RegState.INIT
        else:
            progress_context = progress_phase(progress_factory, total=len(self.filenames),
                                              desc='Loading global registration')
            self.mappings = {}
            with progress_context as pbar:
                for index, filename in enumerate(self.filenames):
                    # stored as registration produced it - 2D for a stack, whose msims are stored 2D too
                    self.mappings[index] = xaffine_from_matrix(find_file_dict_item(mappings, filename))
                    if pbar is not None:
                        pbar.update(1)
            if self._msims is not None:
                self._apply_mappings(self._msims)
            self.pair_msims = self._msims
            # a project saved before metrics.csv reopens without metrics
            rows = import_csv(metrics_filename) if os.path.exists(metrics_filename) else []
            indexed_metrics = {}
            for row in rows:
                # an empty cell: a metric this pair has no value for
                values = {metric: float(value) for metric, value in row.items()
                          if metric not in pair_table_columns and value not in ('', None)}
                # a split group pair's row has no filenames, and an empty name would match any file
                if not row.get('filename1'):
                    labels = (row['id1'], row['id2'])
                    if labels in self.group_pairs:
                        # its quality is the pair mappings file's
                        self.group_pairs[labels]['metrics'] = {metric: value for metric, value in values.items()
                                                               if metric != default_quality_key}
                else:
                    index1, index2 = file_index(row['filename1']), file_index(row['filename2'])
                    if index1 is not None and index2 is not None:
                        indexed_metrics[index1, index2] = values
            # each metric's mean under the registered key, as a global registration's own summary
            metric_keys = list(dict.fromkeys(metric for value in indexed_metrics.values() for metric in value))
            summary = {metric: float(np.mean([value[metric] for value in indexed_metrics.values()
                                              if value.get(metric) is not None]))
                       for metric in metric_keys}
            self.metrics = {
                'summary': {self.reg_transform_key: summary},
                'pairs': {key: {self.reg_transform_key: value} for key, value in indexed_metrics.items()}
            }
    if self.group_pairs and self.metrics:
        self.metrics['group_pairs'] = self.group_pair_metrics()
init_sources(progress_factory=None)
Source code in src\muvis_align\MVSRegistration.py
713
714
715
716
717
718
719
720
721
722
def init_sources(self, progress_factory=None):
    def build_source(index):
        return create_image_source(self.filenames[index], self.source_metadata, file_label=self.file_labels[index],
                                   index=index, transform_key=self.source_transform_key)

    self.sources = pooled_map(build_source, len(self.filenames), default_source_init_workers, progress_factory,
                              desc='Initialising sources', timing_label='Init sources' if self.logging_time else None)
    self.missing_metadata_summary = get_missing_metadata_summary(self.sources)
    if self.missing_metadata_summary:
        logging.warning(self.missing_metadata_summary)
is_channel_overlay(dimension=None, extra_metadata=None)
Source code in src\muvis_align\MVSRegistration.py
2012
2013
2014
2015
2016
def is_channel_overlay(self, dimension=None, extra_metadata=None):
    if extra_metadata is None:
        extra_metadata = self.extra_metadata
    channels = extra_metadata.get('channels', []) if isinstance(extra_metadata, dict) else []
    return bool((dimension and dimension == 'c') or len(channels) > 1)
is_fused()
Source code in src\muvis_align\MVSRegistration.py
384
385
def is_fused(self):
    return self.state.value >= RegState.FUSED.value
is_global_registered()
Source code in src\muvis_align\MVSRegistration.py
381
382
def is_global_registered(self):
    return self.state.value >= RegState.GLOBAL_REG.value
is_initialised()
Source code in src\muvis_align\MVSRegistration.py
375
376
def is_initialised(self):
    return self.state.value >= RegState.INIT.value
is_pairs_registered()
Source code in src\muvis_align\MVSRegistration.py
378
379
def is_pairs_registered(self):
    return self.state.value >= RegState.PAIRS_REG.value
log_residuals(residuals)

Each pair's residual from the global solution (RMS distance, physical units), the largest first.

Source code in src\muvis_align\MVSRegistration.py
2380
2381
2382
2383
2384
2385
2386
2387
def log_residuals(self, residuals):
    """Each pair's residual from the global solution (RMS distance, physical units), the largest first."""
    if residuals:
        values = list(residuals.values())
        pair_lines = [f'  {self.file_labels[keys[0]]} - {self.file_labels[keys[1]]}: {value:.4g}'
                      for keys, value in sorted(residuals.items(), key=lambda item: -item[1])]
        logging.info(f'Global registration residuals over {len(values)} pairs: mean {np.mean(values):.4g},'
                     f' median {np.median(values):.4g}, max {np.max(values):.4g}\n' + '\n'.join(pair_lines))
middle_section_indices()

The files of the middle section, by section number (a label as S000 or SBEMimage's s00538), else by folder; None if neither tells sections apart.

Source code in src\muvis_align\MVSRegistration.py
689
690
691
692
693
694
695
696
697
698
699
700
701
702
def middle_section_indices(self):
    """The files of the middle section, by section number (a label as S000 or SBEMimage's s00538), else by folder;
    None if neither tells sections apart."""
    # tile and slice indices look alike in a name: only the label tells them apart
    labelled = [find_labelled_numbers(Path(filename).name) for filename in self.filenames]
    candidates = [[numbers.get(label) for numbers in labelled] for label in section_number_labels]
    candidates.append([Path(filename).parent for filename in self.filenames])
    for keys in candidates:
        sections = sorted(set(keys)) if None not in keys else []
        if len(sections) > 1:
            # the step napari's dims start on: int((nsteps - 1) / 2)
            middle = sections[(len(sections) - 1) // 2]
            return [index for index, key in enumerate(keys) if key == middle]
    return None
msims_build_pending(target_scale=None, ends_only=False)

Whether ensure_msims() would actually build, and so report a progress phase of its own.

Source code in src\muvis_align\MVSRegistration.py
256
257
258
259
260
261
262
263
264
265
266
267
268
def msims_build_pending(self, target_scale=None, ends_only=False):
    """Whether ensure_msims() would actually build, and so report a progress phase of its own."""
    target_scale = parse_scale(target_scale, default=None)
    key = self._scaled_msims_key(target_scale, ends_only)
    if key in self._scaled_msims:
        return False
    if ends_only:
        return True
    if target_scale:
        from_levels = [get_level_from_scale(source, target_scale)[0] for source in self.sources]
        if any(from_levels):
            return True
    return self._msims is None
needs_full_preprocess(params)

Whether preprocess_sources() cannot give these settings' result: it needs every source.

Source code in src\muvis_align\MVSRegistration.py
1196
1197
1198
def needs_full_preprocess(self, params):
    """Whether preprocess_sources() cannot give these settings' result: it needs every source."""
    return self.preprocess_settings(params)[4]
output_exists(output_filename, output_format)
Source code in src\muvis_align\MVSRegistration.py
2562
2563
2564
2565
2566
2567
2568
2569
2570
def output_exists(self, output_filename, output_format):
    if not output_format.startswith('.'):
        output_format = '.' + output_format
    output_filename = self.output + output_filename + output_format
    if output_format == zarr_extension:
        return (os.path.exists(os.path.join(output_filename, '.zattrs')) or
                os.path.exists(os.path.join(output_filename, 'zarr.json')))
    else:
        return os.path.exists(output_filename)
preprocess(msims, scale=None, flatfield_quantiles=None, normalisation=None, gaussian_sigma=None, filter_foreground=False, progress_factory=None, **kwargs)
Source code in src\muvis_align\MVSRegistration.py
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
def preprocess(self, msims, scale=None,
               flatfield_quantiles=None, normalisation=None, gaussian_sigma=None, filter_foreground=False,
               progress_factory=None,
               **kwargs):
    if kwargs:
        # not fatal, but visible: a mistyped option otherwise silently leaves pre-processing at its defaults
        logging.warning('Ignoring unknown pre-processing option(s):'
                        f' {", ".join(sorted(str(key) for key in kwargs))}')

    do_normalisation = normalisation_enabled(normalisation)
    # a factor, as a number or the text of a text field, or a pixel size with its unit ('10um')
    scale = parse_scale(scale, default=None)
    params = {'scale': scale, 'flatfield_quantiles': flatfield_quantiles, 'normalisation': normalisation,
              'gaussian_sigma': gaussian_sigma, 'filter_foreground': filter_foreground}
    settings = self.preprocess_settings(params)

    def count_progress_steps():
        n_steps = 0
        if scale and scale != 1:
            n_steps += 1
        if filter_foreground:
            # foreground map + final foreground filtering
            n_steps += 2
        if flatfield_quantiles:
            n_steps += 1
        if gaussian_sigma:
            n_steps += 1
        if do_normalisation:
            n_steps += 1
        return max(n_steps, 1)

    def update_progress():
        if pbar is not None:
            pbar.update(1)

    progress_context = progress_phase(progress_factory, total=count_progress_steps(), desc='Pre-processing')

    with progress_context as pbar:
        modified = False
        source_indices = range(len(msims))

        if scale and scale != 1:
            # native levels where the pyramid reaches the scale, binned down to it where it does not
            msims = select_msim_subpyramid_at_scale(msims, self.sources, scale, bin_shortfall=True)
            modified = True
            update_progress()

        if filter_foreground:
            foreground_map = calc_foreground_map(msims)
            modified = True
            update_progress()
        else:
            foreground_map = None

        if flatfield_quantiles:
            logging.info('Flat-field correction...')
            flatfield_params = {'flatfield_quantiles': flatfield_quantiles, 'scale': scale,
                                'filter_foreground': filter_foreground}
            self.flatfield_models(flatfield_params, msims=msims, foreground_map=foreground_map)
            msims = self.flatfield_corrected_msims(msims, flatfield_params)
            modified = True
            update_progress()

        if gaussian_sigma:
            logging.info('Applying Gaussian filtering...')
            msims = self._gaussian_filtered_msims(msims, gaussian_sigma)
            modified = True
            update_progress()

        if do_normalisation:
            use_global = ('global' in str(normalisation).lower())
            if use_global:
                logging.info('Normalising (global)...')
            else:
                logging.info('Normalising (individual)...')
            if use_global:
                groups = self.source_groups()
                msims_of = dict(zip(source_indices, msims))
                group_stats = {group: self._global_normalise_stat(group, params, msims_of)
                               for group in set(groups)}
                norm_stats, norm_dtypes = zip(*(group_stats[groups[index]] for index in source_indices))
            else:
                norm_stats, norm_dtype = calc_normalise_stats(msims, use_global=False)
                norm_dtypes = [norm_dtype] * len(msims)
            msims = self._normalised_msims(msims, norm_stats, norm_dtypes)
            modified = True
            update_progress()

        if filter_foreground:
            logging.info('Filtering foreground images...')
            # tile_vars = np.array([np.asarray(np.std(sim)).item() for sim in sims])
            # threshold1 = np.mean(tile_vars)
            # threshold2 = np.median(tile_vars)
            # threshold3, _ = cv.threshold(np.array(tile_vars).astype(np.uint16), 0, 1, cv.THRESH_OTSU)
            # threshold = min(threshold1, threshold2, threshold3)
            # foregrounds = (tile_vars >= threshold)
            new_msims = [msim for msim, is_foreground in zip(msims, foreground_map) if is_foreground]
            logging.info(f'Foreground images: {len(new_msims)} / {len(msims)}')
            indices = np.where(foreground_map)[0]
            msims = new_msims
            modified = True
            update_progress()
        else:
            indices = range(len(msims))

    self.register_msims = msims
    self.register_indices = indices
    self.preprocessed_settings = settings
    return msims, indices, modified
preprocess_settings(params) staticmethod

Pre-processing params, parsed: equal for settings that give the same pixels, however they were written.

Source code in src\muvis_align\MVSRegistration.py
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
@staticmethod
def preprocess_settings(params):
    """Pre-processing params, parsed: equal for settings that give the same pixels, however they were written."""
    quantiles, scale, filter_foreground = MVSRegistration.flatfield_settings(params)
    normalisation = params.get('normalisation')
    if normalisation_enabled(normalisation):
        normalisation = 'global' if 'global' in str(normalisation).lower() else 'individual'
    else:
        normalisation = None
    gaussian_sigma = params.get('gaussian_sigma')
    return (quantiles, scale if scale != 1 else None, normalisation,
            float(gaussian_sigma) if gaussian_sigma else None, filter_foreground)
preprocess_sources(indices, params)

Sources indices pre-processed as preprocess() makes them, reading no other source's pixels; only where needs_full_preprocess() is False.

Source code in src\muvis_align\MVSRegistration.py
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
def preprocess_sources(self, indices, params):
    """Sources `indices` pre-processed as preprocess() makes them, reading no other source's pixels; only where
    needs_full_preprocess() is False."""
    settings = self.preprocess_settings(params)
    quantiles, scale, normalisation, gaussian_sigma, _ = settings
    msims = self.scaled_source_msims(indices, scale)
    if quantiles:
        models = [self.flatfield_model(index, params) for index in indices]
        msims = [apply_msim_flatfield_model(msim, model) if model is not None else msim
                 for msim, model in zip(msims, models)]
    if gaussian_sigma:
        msims = self._gaussian_filtered_msims(msims, gaussian_sigma)
    if normalisation == 'global':
        groups = self.source_groups()
        norm_stats, norm_dtypes = zip(*(self._global_normalise_stat(groups[index], params) for index in indices))
        msims = self._normalised_msims(msims, norm_stats, norm_dtypes)
    elif normalisation:
        norm_stats, norm_dtype = calc_normalise_stats(msims, use_global=False)
        msims = self._normalised_msims(msims, norm_stats, [norm_dtype] * len(msims))
    return msims
register(register_msims=None, register_indices=None, params=None)
Source code in src\muvis_align\MVSRegistration.py
1504
1505
1506
1507
1508
1509
1510
1511
def register(self, register_msims=None, register_indices=None, params=None):
    pair_results = self.register_pairs(register_msims=register_msims, register_indices=register_indices, params=params)
    self.save_pair_results(pair_results['pair_mappings'], metrics=pair_results['metrics'])
    results = self.register_global(self.pair_msims, register_indices=register_indices, params=params)
    self.save_group_pair_mappings()
    self.save_mappings(results['mappings'])
    self.save_metrics(results['metrics'])
    return results
register_global(pair_msims, register_indices=None, params=None, pairs_graph=None, progress_factory=None)
Source code in src\muvis_align\MVSRegistration.py
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
def register_global(self, pair_msims, register_indices=None, params=None,
                    pairs_graph=None, progress_factory=None):
    logging.info('Global registration...')
    if register_indices is None:
        if self.register_indices is not None:
            register_indices = self.register_indices
        else:
            register_indices = range(len(pair_msims))

    if pairs_graph is not None:
        g_reg_computed = pairs_graph
    else:
        g_reg_computed = self.pairs_graph

    ndims = si_utils.get_ndim_from_sim(get_msim_image0(pair_msims[0]))

    groupwise_resolution_method = params.get('groupwise_resolution_method', 'global_optimization')
    transform_type = params.get('transform_type', 'affine')
    if groupwise_resolution_method in (ROBUST_LINEAR, 'linear_two_pass') and transform_type not in ('translation', 'rigid'):
        logging.warning(f'{groupwise_resolution_method} resolves translation or rigid only,'
                        f' not {transform_type}: using global_optimization')
        groupwise_resolution_method = 'global_optimization'
    split_settings = ''
    if SPLIT in params.get('pairing', '').lower():
        split_settings = f", split binning {params.get('split_binning') or default_split_binning}"
    logging.info(f'Global registration settings: resolution {groupwise_resolution_method},'
                 f' transform {transform_type}{split_settings}')
    groupwise_resolution_kwargs = {}
    if groupwise_resolution_method in ('global_optimization', ROBUST_LINEAR, 'linear_two_pass'):
        # transform_type options include 'translation', 'rigid', 'affine', 'similarity'
        groupwise_resolution_kwargs['transform'] = transform_type

    quality_threshold = params.get('post_registration_quality_threshold')
    quality_threshold = float(quality_threshold) if is_valid_value(quality_threshold) else 0.0

    n_parallel_pairwise_regs = params.get('n_parallel_pairwise_regs')
    # None or 0 is automatic; a yaml value may be a float
    n_parallel_pairwise_regs = int(n_parallel_pairwise_regs) if n_parallel_pairwise_regs else None

    plot_summary = self.mpl_ui

    # ******* start MVS registration functions

    # a pair scored at the threshold or below (a failed method gives the identity with 0) is left out of the solve
    # only, so a tile without other pairs keeps its metadata position rather than an unconstrained one's
    failed_pairs = [edge for edge, quality in nx.get_edge_attributes(g_reg_computed, 'quality').items()
                    if not np.nan_to_num(np.asarray(quality, dtype=float), nan=0.0).max() > quality_threshold]
    resolve_graph = g_reg_computed
    if failed_pairs:
        resolve_graph = g_reg_computed.copy()
        resolve_graph.remove_edges_from(failed_pairs)
        logging.warning(f'Global registration: {len(failed_pairs)} of {g_reg_computed.number_of_edges()} pairs'
                        f' left out, scored {quality_threshold:g} or below')
        unregistered = sorted(node for node in resolve_graph.nodes
                              if resolve_graph.degree(node) == 0 and g_reg_computed.degree(node) > 0)
        if unregistered:
            logging.warning('Global registration: no registered pair, so kept at their metadata position: '
                            + ', '.join(str(self.file_labels[register_indices[node]]) for node in unregistered))

    # multiview_stitcher's own search is O(nodes x edges); a reference outside a component would send that
    # component back to it, so only a connected graph gets one
    if resolve_graph.number_of_nodes() > 2 and nx.is_connected(resolve_graph):
        groupwise_resolution_kwargs['reference_view'] = find_reference_view(resolve_graph)

    if groupwise_resolution_method == ROBUST_LINEAR:
        resolution_progress = progress_phase(progress_factory, total=default_robust_rounds,
                                                  desc='Global registration', weight=4)
    else:
        # global_optimization has nothing to report into a phase: GlobalOptProgress follows its log instead
        resolution_progress = GlobalOptProgress(progress_factory, desc='Global registration',
                                                max_passes=g_reg_computed.number_of_edges(), weight=4)
    with resolution_progress as pbar, dask.config.set(scheduler='threads'):
        if groupwise_resolution_method == ROBUST_LINEAR and pbar is not None:
            groupwise_resolution_kwargs['progress'] = lambda: pbar.update(1)
        transforms_dict, groupwise_resolution_info_dict = groupwise_resolution(
            resolve_graph,
            method=groupwise_resolution_method,
            **groupwise_resolution_kwargs,
        )

    transforms = [
        transforms_dict[iview] for iview in sorted(g_reg_computed.nodes())
    ]
    if SPLIT in params.get('pairing', '').lower():
        transforms = self._register_split_groups(pair_msims, transforms, register_indices, params,
                                                 progress_factory=progress_factory)

    # the stages below take long enough on a large project to need progress of their own
    with progress_phase(progress_factory, total=len(pair_msims),
                             desc='Applying transforms') as pbar, \
            Timer('apply registered transforms', verbose=self.logging_time):
        for imsim, msim in enumerate(pair_msims):
            set_msim_affine(
                msim,
                transforms[imsim],
                transform_key=self.reg_transform_key,
                base_transform_key=self.source_transform_key,
            )
            if pbar is not None:
                pbar.update(1)

    if plot_summary:
        with progress_phase(progress_factory, total=1, desc='Registration summary'), \
                Timer('plot registration summaries', verbose=self.logging_time):
            plot_info = _plot_registration_summaries(
                pair_msims,
                self.source_transform_key,
                self.reg_transform_key,
                resolve_graph,
                groupwise_resolution_info_dict,
                show_plot=plot_summary,
            )
    else:
        plot_info = {}

    reg_result = {
        "params": transforms,
        "pairwise_registration": {
            "graph": g_reg_computed,
            "metrics": {
                "qualities": nx.get_edge_attributes(
                    g_reg_computed, "quality"
                )
            },
            "summary_plot": None if plot_summary is False
            else (
                plot_info['fig_pair_reg'],
                plot_info['ax_pair_reg']
            )
        },
        "groupwise_resolution": {
            "metrics": groupwise_resolution_info_dict,
            "summary_plot": None if plot_summary is False
            else (
                plot_info['fig_group_res'],
                plot_info['ax_group_res']
            )
        },
    }

    # ******* end MVS registration functions

    # onto self.msims, the persistent full per-source pyramid
    with progress_phase(progress_factory, total=len(pair_msims),
                             desc='Storing transforms') as pbar, \
            Timer('store transforms on sources', verbose=self.logging_time):
        for reg_msim, index in zip(pair_msims, register_indices):
            reg_transform = msi_utils.get_transform_from_msim(reg_msim, transform_key=self.reg_transform_key)
            set_msim_affine(self.msims[index], reg_transform, transform_key=self.reg_transform_key)
            if pbar is not None:
                pbar.update(1)

    # set missing transforms - sources that never took part in registration (e.g. filtered out)
    for msim in self.msims:
        if self.reg_transform_key not in get_msim_transform_keys(msim):
            set_msim_affine(
                msim,
                param_utils.identity_transform(ndim=ndims, t_coords=[0]),
                transform_key=self.reg_transform_key)

    mappings = reg_result['params']
    # re-index from subset of sims
    # per timepoint index; a registration resolves its first timepoint
    edge_residuals = (groupwise_resolution_info_dict.get('edge_residuals') or {}).get(0, {})
    residual_error_dict = {(int(register_indices[key[0]]), int(register_indices[key[1]])): float(value)
                           for key, value in edge_residuals.items()}
    self.log_residuals(residual_error_dict)
    registration_qualities_dict = reg_result.get('pairwise_registration', {}).get('metrics', {}).get('qualities', {})
    registration_qualities_dict = {(register_indices[key[0]], register_indices[key[1]]): value
                                   for key, value in registration_qualities_dict.items()}

    # re-index from subset of sims
    mappings_dict = {index: mapping for index, mapping in zip(register_indices, mappings)}

    with Timer('global registration metrics', verbose=self.logging_time):
        metrics = calc_global_metrics(pair_msims, self.reg_transform_key, params.get('metrics', []), reg_result,
                                      n_parallel_pairs=n_parallel_pairwise_regs,
                                      progress_factory=progress_factory)
        remap_pair_metrics(metrics, register_indices)

    self.mappings = mappings_dict
    if self.group_pairs:
        metrics['group_pairs'] = self.group_pair_metrics()
    self.metrics = metrics
    self.state = RegState.GLOBAL_REG
    return {'reg_result': reg_result,
            'mappings': mappings_dict,
            'residual_errors': residual_error_dict,
            'registration_qualities': registration_qualities_dict,
            'metrics': metrics}
register_msims_current(params)

Whether register_msims were made with these pre-processing settings.

Source code in src\muvis_align\MVSRegistration.py
1192
1193
1194
def register_msims_current(self, params):
    """Whether register_msims were made with these pre-processing settings."""
    return self.register_msims is not None and self.preprocessed_settings == self.preprocess_settings(params)
register_overlap(overlap1, overlap2, sims_pixel_space, params=None)

Register an overlap crop from select_pair_overlap().

Returns the physical-space transform, quality, and pairwise_reg_func's raw result (e.g. feature matches).

Source code in src\muvis_align\MVSRegistration.py
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
def register_overlap(self, overlap1, overlap2, sims_pixel_space, params=None):
    """Register an overlap crop from select_pair_overlap().

    Returns the physical-space transform, quality, and pairwise_reg_func's raw result (e.g. feature matches).
    """
    params = params or {}
    _, pairwise_reg_func, pairwise_reg_func_kwargs = self.create_registration_method(overlap1, params=params)
    pairwise_reg_func_kwargs = dict(pairwise_reg_func_kwargs or {})

    fixed_data = overlap1.compute() if hasattr(overlap1, 'compute') else overlap1
    moving_data = overlap2.compute() if hasattr(overlap2, 'compute') else overlap2
    result = pairwise_reg_func(fixed_data, moving_data, **pairwise_reg_func_kwargs)

    try:
        transform = affine_from_intrinsic_affine(result['affine_matrix'], sims_pixel_space, self.source_transform_key)
    except NotImplementedError:
        transform = result['affine_matrix']

    return transform, result.get('quality'), result
register_pairs(register_msims=None, register_indices=None, params=None, progress_factory=None)
Source code in src\muvis_align\MVSRegistration.py
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
def register_pairs(self, register_msims=None, register_indices=None, params=None,
                   progress_factory=None):
    logging.info('Pair registration...')
    if register_indices is None:
        if self.register_indices is not None:
            register_indices = self.register_indices
        else:
            register_indices = range(len(self.msims))

    pairing = params.get('pairing', '').lower()
    n_parallel_pairwise_regs = params.get('n_parallel_pairwise_regs')
    # None or 0 is automatic; a yaml value may be a float
    n_parallel_pairwise_regs = int(n_parallel_pairwise_regs) if n_parallel_pairwise_regs else None
    logging.info(f"Pair registration settings: pairing {pairing or 'default'}, method {params.get('method')},"
                 f" transform {params.get('transform_type')}, channel {params.get('channel')},"
                 f" workers {n_parallel_pairwise_regs or f'automatic ({default_pair_workers})'},"
                 f" registration dimension {getattr(self, 'registration_dimension', None) or 'space'}")

    # the caller's params win over init()'s: the plugin's dropdown can change in between
    self.pairing = pairing
    is_3d = (self.sources[0].get_size().get('z', 0) > 1)
    is_stack = self.is_stack

    reg_channel = params.get('channel', 0)
    if isinstance(reg_channel, int):
        reg_channel_index = reg_channel
        reg_channel = None
    else:
        reg_channel_index = None

    if register_msims is None:
        register_msims = self.register_msims

    if is_stack and not is_3d:
        # register in 2D, pairing consecutive views: max-project every level that has z
        def level_func(level_sim, scale_key):
            return si_utils.max_project_sim(level_sim, dim='z') if 'z' in level_sim.dims else level_sim
        register_msims = [map_msim_levels(msim, level_func) for msim in register_msims]
        pairs = [(index, index + 1) for index in range(len(register_msims) - 1)]
    elif 'ortho' in pairing or SPLIT in pairing:
        # every source, never the filtered register_msims, so it matches self.positions; from metadata,
        # as building self.msims just to read their geometry is expensive
        with Timer('pair registration: source geometry', verbose=self.logging_time):
            shape_sims = [build_source_shape_sim(source, self._msim_output_order, position, transform,
                                                 self.source_transform_key, z_scale=self._msim_z_scale)
                          for source, position, transform in zip(self.sources, self.positions,
                                                                 self._msim_transforms)]
            origins = np.array([get_sim_position_final(sim, position, get_center=True)
                                for sim, position in zip(shape_sims, self.positions)])
            sizes = [get_sim_physical_size(sim) for sim in shape_sims]
        with Timer('pair registration: get_pairs', verbose=self.logging_time):
            pairs, _ = get_pairs(origins, sizes)
        if SPLIT in pairing:
            # stage 1: each z-plane's (or channel's) tiles only; the groups are registered in register_global
            pairs = within_group_pairs(pairs, self.split_groups())
        # found over every source, but registered on the retained ones, numbered from 0
        retained_index = {int(index): position for position, index in enumerate(register_indices)}
        pairs = [(retained_index[index1], retained_index[index2]) for index1, index2 in pairs
                 if index1 in retained_index and index2 in retained_index]
        logging.info(f'#pairs: {len(pairs)}')
        #for pair in pairs:
        #    print(f'{self.file_labels[pair[0]]} - {self.file_labels[pair[1]]}')
    else:
        pairs = None

    # feature-based/CPD methods read pixel data from one concrete sim
    reg_method, pairwise_reg_func, pairwise_reg_func_kwargs = self.create_registration_method(
        msi_utils.get_sim_from_msim(register_msims[0], scale='scale0'), params=params)
    logging.info(f'Registration method: {reg_method}')
    logging.info('Registering pairs...')
    # a full pyramid per source lets compute_pairwise_registrations select each pair's resolution

    overlap_tolerance = 0

    # ******* start MVS registration functions

    with Timer('pair registration: select channel', verbose=self.logging_time):
        has_channel = ["c" in get_msim_dims(msim) for msim in register_msims]
        if has_channel[0]:
            if reg_channel is None and reg_channel_index is None:
                raise Exception("Please choose a registration channel.")
            requested = reg_channel_index if reg_channel is None else reg_channel
            # each source by its own labels: one project's files need not all name it the same
            chosen = {}
            msims_reg = [
                msi_utils.multiscale_sel_coords(
                    msim, {"c": resolve_registration_channel(msim, requested, chosen)})
                if has_channel[imsim]
                else msim
                for imsim, msim in enumerate(register_msims)
            ]
        else:
            msims_reg = register_msims

    # match each level's transform to its image dimensions: a multiscale image can mix 3D/4D transforms
    for msim in msims_reg:
        for scale_node in msim.ds.values():
            if 'source_metadata' in scale_node.ds.data_vars:
                img_data = scale_node.ds['image']
                if hasattr(img_data, 'data'):
                    img_data = img_data.data
                spatial_dims = [d for d in img_data.dims if d not in ('t', 'c')]
                current_transform = scale_node.ds.data_vars[self.source_transform_key]
                transform_spatial_dims = [d for d in current_transform.coords['x_in'].values if d != '1']
                if len(transform_spatial_dims) != len(spatial_dims):
                    relevant_dim_names = spatial_dims + ['1']
                    adapted = current_transform.sel(x_in=relevant_dim_names, x_out=relevant_dim_names)
                    scale_node.ds[self.source_transform_key] = adapted

    try:
        # fusion off: dask's fused key names drop the unique token, so per-pair crop chains can collide
        # and a pair silently registers another pair's crop
        with dask.config.set({'scheduler': 'threads', 'optimization.fuse.active': False}):
            # multiview_stitcher would test every source within the largest one's diameter: with overview
            # images among tiles, nearly every pair
            graph_pairs = (pairs if pairs is not None
                           else find_candidate_overlap_pairs(msims_reg, self.source_transform_key))
            if pairs is None:
                logging.info(f'#candidate pairs: {len(graph_pairs)}')
            raise_if_cancelled()
            with Timer('pair registration: view adjacency graph', verbose=self.logging_time):
                g_reg = build_view_adjacency_graph(msims_reg, self.source_transform_key, graph_pairs,
                                                   overlap_tolerance=overlap_tolerance)
            raise_if_cancelled()

            g_reg_computed = g_reg.copy()
            workers = n_parallel_pairwise_regs or default_pair_workers
            log_every = 2 * workers
            pair_times, pair_cpu_times = [], []
            timed_reg_func = timed_calls(resolve_deferred_quality(pairwise_reg_func), pair_times,
                                         pair_cpu_times)

            pair_kwargs = dict(transform_key=self.source_transform_key, overlap_tolerance=overlap_tolerance,
                               pairwise_reg_func_kwargs=pairwise_reg_func_kwargs, n_parallel_pairwise_regs=1)
            # largest first, so workers are not left idle at the end and the bar's estimate holds
            edges = sorted(g_reg.edges, key=lambda edge: estimated_pair_cost(g_reg, edge), reverse=True)
            pool = None
            # worker processes when a pair can be sent to one (its sources' arrays, the method)
            if (workers > 1 and len(edges) > workers
                    and picklable(([msims_reg[node] for node in edges[0]], pairwise_reg_func))):
                pool = worker_process_pool(workers, max_tasks_per_child=default_pair_worker_tasks)

            def register_edge(edge):
                graph = g_reg.edge_subgraph([edge]).copy()
                if pool is None:
                    return compute_pairwise_registrations(msims_reg, graph, pairwise_reg_func=timed_reg_func,
                                                          **pair_kwargs)
                registered, wall_time, cpu_time = result_unless_cancelled(pool.submit(
                    register_pair_in_worker, {node: msims_reg[node] for node in graph.nodes}, graph,
                    pairwise_reg_func, pair_kwargs))
                pair_times.append(wall_time)
                pair_cpu_times.append(cpu_time)
                return registered

            phase_start, phase_cpu_start = time.time(), time.process_time()
            # one pair per thread, each its own compute, so a slow pair holds up no batch; a single worker
            # keeps the threads scheduler so one (e.g. 3D) pair still runs its tasks in parallel
            with progress_phase(progress_factory, total=g_reg.number_of_edges(),
                                     desc='Registering pairs') as pbar, \
                    Timer(f'register {g_reg.number_of_edges()} pairs on {workers}'
                          f' {"processes" if pool is not None else "threads"}', verbose=self.logging_time), \
                    (pool if pool is not None else nullcontext()), \
                    deferred_link_quality(pairwise_reg_func), \
                    dask.config.set(scheduler='synchronous' if workers > 1 else 'threads'):
                log_start = time.time()
                for count, (edge, pair_computed) in enumerate(
                        rolling_map(register_edge, edges, workers), start=1):
                    g_reg_computed.edges[edge].update(pair_computed.edges[edge])
                    del pair_computed
                    if count % log_every == 0 or count == g_reg.number_of_edges():
                        release_memory(generation=1)
                        if self.logging_time:
                            # cpu per pair rising means harder pairs; wall alone rising means idle workers or I/O
                            batch_wall = time.time() - log_start
                            batch_times, batch_cpu_times = pair_times[-log_every:], pair_cpu_times[-log_every:]
                            pair_timing = ''
                            if batch_times and batch_cpu_times:
                                pair_timing = (f', a pair {np.mean(batch_times):.2f}s wall'
                                               f' {np.mean(batch_cpu_times):.2f}s cpu,'
                                               f' {sum(batch_times) / max(batch_wall, 1e-9):.1f} busy')
                            logging.info(f'Pairs {count}/{g_reg.number_of_edges()}:'
                                         f' last {log_every} in {batch_wall:.1f}s{pair_timing}'
                                         f'{print_memory_usage()}')
                        log_start = time.time()
                    if pbar is not None:
                        pbar.update(1)
            if self.logging_time and pair_times:
                logging.info(f'Register pairs: {len(pair_times)} pairs'
                             f' {format_phase_timing(time.time() - phase_start, pair_times, pair_cpu_times,
                                                     workers, time.process_time() - phase_cpu_start,
                                                     processes=pool is not None)}')

            # ******* end MVS registration functions

    except NotEnoughOverlapError:
        g_reg_computed = g_reg

    release_memory('Pair registration')
    mappings = nx.get_edge_attributes(g_reg_computed, default_transform_key)
    mappings_dict = {(register_indices[indices[0]], register_indices[indices[1]]): mapping
                     for indices, mapping in mappings.items()}

    with Timer(f'pair metrics ({g_reg_computed.number_of_edges()} pairs)', verbose=self.logging_time):
        metrics = calc_pair_metrics(msims_reg, g_reg_computed, params.get('metrics', []),
                                    self.source_transform_key, reg_channel=reg_channel_index,
                                    n_parallel_pairs=n_parallel_pairwise_regs,
                                    progress_factory=progress_factory)
        remap_pair_metrics(metrics, register_indices)
    release_memory('Pair metrics')

    self.pairs_graph = g_reg_computed
    self.pair_msims = msims_reg
    self.pairs = pairs
    self.metrics = metrics
    self.state = RegState.PAIRS_REG
    return {
        'pairs_graph': self.pairs_graph,
        'msims': msims_reg,
        'pairs': pairs,
        'pair_mappings': mappings_dict,
        'metrics': metrics
    }
reset()
Source code in src\muvis_align\MVSRegistration.py
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
def reset(self):
    self.state = RegState.UNINIT
    self.source_transform_key = 'source_metadata'
    self.reg_transform_key = 'registered'
    self.transition_transform_key = 'transition'
    self.source_metadata = {}
    self.extra_metadata = {}
    self.msims = []
    self.register_msims = None
    self.pair_msims = None
    self.pairs = []
    self.pairs_graph = None
    self._scaled_msims = {}
    self._scaled_source_msims = {}
    self.sources = []
    # set on each source read, cleared once shown
    self.missing_metadata_summary = ''
    self.metrics = {}
    self.group_pairs = {}
    self.register_indices = None
    # per source index, for fusion and the view as well as registration
    self._flatfield_key = None
    self._flatfield_models = {}
    self._flatfield_lock = threading.Lock()
    # what register_msims were made with, and global normalisation's stats per (settings, source group)
    self.preprocessed_settings = None
    self._global_normalise_stats = {}
    # per source index: the registration's correction on top of the source transform; absent means identity
    self.mappings = None
    self.input_output_params = {}
run()
Source code in src\muvis_align\MVSRegistration.py
477
478
479
def run(self):
    with ProgressBar(minimum=60, dt=1) if self.logging_dask else nullcontext():
        return self._run()
save(output_filename, data, format=zarr_extension, transform_key=None, translations0=None, channels=[], block_size=None, compression=None, pyramid_downsample=2, npyramid_add=4, ome_version=default_ome_zarr_version, progress_factory=None)
Source code in src\muvis_align\MVSRegistration.py
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
def save(self, output_filename, data, format=zarr_extension, transform_key=None, translations0=None, channels=[],
         block_size=None, compression=None, pyramid_downsample=2, npyramid_add=4, ome_version=default_ome_zarr_version,
         progress_factory=None):
    if output_filename is not None:
        output_filename = self.output + output_filename
    if isinstance(self.extra_metadata, dict) and self.extra_metadata:
        channels = self.extra_metadata.get('channels', [])
    save_image(output_filename, data, format,
               transform_key=transform_key, channels=channels, translations0=translations0,
               block_size=block_size, compression=compression,
               pyramid_downsample=pyramid_downsample, npyramid_add=npyramid_add,
               ome_version=ome_version,
               verbose=self.verbose, progress_factory=progress_factory)
save_composed(output_filename, msims, channels, min_length=128, compression=None)

Each source as its own image at its native levels, in one OME-Zarr 0.6 scene with a coordinate system per transform key; the registered one sorts first, so it is the default view.

Source code in src\muvis_align\MVSRegistration.py
2456
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471
2472
def save_composed(self, output_filename, msims, channels, min_length=128, compression=None):
    """Each source as its own image at its native levels, in one OME-Zarr 0.6 scene with a coordinate system per
    transform key; the registered one sorts first, so it is the default view."""
    transform_keys = [key for key in sorted([self.reg_transform_key, self.source_transform_key])
                      if key in get_msim_transform_keys(msims[0])]
    labels = self.file_labels if len(self.file_labels) == len(msims) else [str(index) for index in range(len(msims))]
    sources = {}
    for label, msim in zip(labels, msims):
        levels, dim_order, translation = msim_native_levels(msim)
        affines = {}
        for transform_key in transform_keys:
            affine = msi_utils.get_transform_from_msim(msim, transform_key)
            affines[transform_key] = affine.sel(t=0) if 't' in affine.dims else affine
        spatial_dims = [str(dim) for dim in affines[transform_keys[0]].coords['x_in'].values[:-1]]
        sources[label] = (levels, dim_order, translation, spatial_dims, affines)
    save_ome_zarr_scene(output_filename, sources, transform_keys, channels, min_length=min_length,
                        compression=compression)
save_group_pair_mappings()

Split pairing's group (e.g. section) pairs added to the pair mappings file, keyed by their labels and marked by kind, replacing those of an earlier run.

Source code in src\muvis_align\MVSRegistration.py
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
def save_group_pair_mappings(self):
    """Split pairing's group (e.g. section) pairs added to the pair mappings file, keyed by their labels and marked
    by kind, replacing those of an earlier run."""
    pair_mappings_filename = self.output + self.input_output_params.get('pair_mappings', default_pair_mappings_name)
    output_mappings = import_json(pair_mappings_filename) if os.path.exists(pair_mappings_filename) else {}
    output_mappings = {key: value for key, value in output_mappings.items()
                       if value.get('kind') != split_group_pair_kind}
    for labels, entry in self.group_pairs.items():
        # its image metrics go in the metrics file, as every pair's do
        output_mappings[json.dumps(list(labels))] = {'kind': split_group_pair_kind} | {
            key: value for key, value in entry.items() if key != 'metrics'}
    export_json(pair_mappings_filename, output_mappings)
save_mappings(mappings)
Source code in src\muvis_align\MVSRegistration.py
2338
2339
2340
2341
2342
def save_mappings(self, mappings):
    mappings_filename = self.output + self.input_output_params.get('mappings', default_mappings_name)
    output_mappings = {self.filenames[int(key)]: np.array(mapping.sel(t=0)).tolist()
                       for key, mapping in mappings.items()}
    write_transforms(mappings_filename, output_mappings, self.source_transform_key, self.reg_transform_key)
save_mappings_csv(mappings, normalise_orientation=False)
Source code in src\muvis_align\MVSRegistration.py
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
def save_mappings_csv(self, mappings, normalise_orientation=False):
    data = []
    mappings_header = ['id',' filename', 'x_pixels', 'y_pixels', 'z_pixels', 'x', 'y', 'z', 'rotation']
    mappings_filename = self.output + self.input_output_params.get('mappings', default_mappings_tabular_name)
    for label, filename, msim, mapping, scale, position, rotation \
            in zip(self.file_labels, self.filenames, self.msims, mappings.values(), self.scales, self.positions, self.rotations):
        if not normalise_orientation:
            # rotation already in msim affine transform
            rotation = None
        position, rotation = get_data_mapping(msim,
                                              transform_key=self.reg_transform_key,
                                              transform=mapping,
                                              translation0=position,
                                              rotation=rotation)
        position_pixels = {dim: position[dim] / float(scale.get(dim, 1)) for dim in position.keys()}
        row = ([label] + [filename] + dict_to_xyz(position_pixels, add_zeros=True)
               + dict_to_xyz(position, add_zeros=True) + [rotation])
        data.append(row)
    export_csv(mappings_filename, data, header=mappings_header)
save_metrics(metrics)

The registered pairs' metrics, then split pairing's group pairs' (without filenames), also read back on resuming.

Source code in src\muvis_align\MVSRegistration.py
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
def save_metrics(self, metrics):
    """The registered pairs' metrics, then split pairing's group pairs' (without filenames), also read back on
    resuming."""
    pair_metrics = {keys: {metric: float(value) for metric, value in metric_dict[self.reg_transform_key].items()}
                    for keys, metric_dict in metrics['pairs'].items() if metric_dict[self.reg_transform_key]}
    group_metrics = {labels: metric_dict[self.reg_transform_key]
                     for labels, metric_dict in metrics.get('group_pairs', {}).items()}
    metric_names = list(dict.fromkeys(metric for values in list(pair_metrics.values()) + list(group_metrics.values())
                                      for metric in values))
    rows = [[self.file_labels[keys[0]], self.file_labels[keys[1]], self.filenames[keys[0]], self.filenames[keys[1]]]
            + [values.get(metric) for metric in metric_names]
            for keys, values in sorted(pair_metrics.items())]
    rows += [list(labels) + ['', ''] + [values.get(metric) for metric in metric_names]
             for labels, values in group_metrics.items()]
    export_csv(self.output + metrics_name, rows, header=pair_table_columns + metric_names)
save_native_levels(output_filename, msim, position=None, channels=None, min_length=128, compression=None, ome_version=default_ome_zarr_version)

Write msim's own native pyramid levels as-is, without resampling (see save_ome_multiscale_levels()).

position: this source's position; a source without a 'z' dim is promoted to keep its z position.

Source code in src\muvis_align\MVSRegistration.py
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
def save_native_levels(self, output_filename, msim, position=None, channels=None, min_length=128,
                       compression=None, ome_version=default_ome_zarr_version):
    """Write msim's own native pyramid levels as-is, without resampling (see save_ome_multiscale_levels()).

    position: this source's position; a source without a 'z' dim is promoted to keep its z position.
    """
    if output_filename is not None:
        output_filename = self.output + output_filename
    if channels is None:
        channels = self.extra_metadata.get('channels', []) if isinstance(self.extra_metadata, dict) else []

    if position is not None and 'z' not in get_msim_dims(msim):
        msim = make_msims_3d([msim], z_scale=self._msim_z_scale, positions=[position])[0]

    levels, dim_order, translation = msim_native_levels(msim)
    save_ome_multiscale_levels(str(output_filename) + zarr_extension, levels, dim_order, channels,
                               translation, min_length=min_length,
                               compression=compression, ome_version=ome_version)
save_pair_mappings(mappings, qualities, bboxes)
Source code in src\muvis_align\MVSRegistration.py
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
def save_pair_mappings(self, mappings, qualities, bboxes):
    pair_mappings_filename = self.output + self.input_output_params.get('pair_mappings', default_pair_mappings_name)
    output_mappings = {}
    for keys, mapping in mappings.items():
        label_key = json.dumps([self.filenames[keys[0]], self.filenames[keys[1]]])
        output_mappings[label_key] = {'mapping': np.array(mapping.sel(t=0)).tolist()}
        if keys in qualities:
            quality = quality_to_scalar(qualities[keys])
            output_mappings[label_key][default_quality_key] = float(quality)
        if keys in bboxes:
            output_mappings[label_key]['bbox'] = bboxes[keys]
    export_json(pair_mappings_filename, output_mappings)
save_pair_results(pair_mappings, metrics=None, qualities=None)

The pair mappings file, each pair with its quality (from metrics unless given) and its pairs graph bbox.

Source code in src\muvis_align\MVSRegistration.py
2298
2299
2300
2301
2302
2303
2304
2305
def save_pair_results(self, pair_mappings, metrics=None, qualities=None):
    """The pair mappings file, each pair with its quality (from `metrics` unless given) and its pairs graph bbox."""
    if qualities is None:
        qualities = {key: metric[default_transform_key][default_quality_key]
                     for key, metric in metrics['pairs'].items() if default_quality_key in metric[default_transform_key]}
    bboxes = {key: np.array(value.sel(t=0) if 't' in value.dims else value).tolist()
              for key, value in nx.get_edge_attributes(self.pairs_graph, 'bbox').items()}
    self.save_pair_mappings(pair_mappings, qualities, bboxes)
save_residuals(residuals)
Source code in src\muvis_align\MVSRegistration.py
2389
2390
2391
2392
def save_residuals(self, residuals):
    rows = [[self.file_labels[keys[0]], self.file_labels[keys[1]], self.filenames[keys[0]], self.filenames[keys[1]],
             value] for keys, value in sorted(residuals.items())]
    export_csv(self.output + residuals_name, rows, header=pair_table_columns + ['residual'])
save_video(output, msims, fused_msim)
Source code in src\muvis_align\MVSRegistration.py
2474
2475
2476
2477
2478
2479
2480
2481
2482
2483
2484
2485
2486
2487
2488
2489
2490
2491
2492
2493
2494
2495
2496
2497
2498
2499
2500
2501
2502
2503
2504
2505
2506
2507
2508
2509
def save_video(self, output, msims, fused_msim):
    logging.info('Creating transition video...')
    sims = [msi_utils.get_sim_from_msim(msim, scale='scale0') for msim in msims]
    fused_image = extract_sims_from_fused(fused_msim)
    pixel_size = [si_utils.get_spacing_from_sim(sims[0]).get(dim, 1) for dim in 'xy']
    params = self.fusion_params.get('transition', {})
    nframes = params.get('frames', 1)
    spacing = params.get('spacing', [1.1, 1])
    scale = params.get('scale', 1)
    transition_filename = output + 'transition'
    video = Video(transition_filename + '.mp4', fps=params.get('fps', 1))
    positions0 = np.array([si_utils.get_origin_from_sim(sim, asarray=True) for sim in sims])
    center = np.mean(positions0, 0)
    window = get_image_window(fused_image)

    max_size = None
    acum = 0
    for framei in range(nframes):
        c = (1 - np.cos(framei / (nframes - 1) * 2 * math.pi)) / 2
        acum += c / (nframes / 2)
        spacing1 = spacing[0] + (spacing[1] - spacing[0]) * acum
        for sim, position0 in zip(sims, positions0):
            transform = param_utils.identity_transform(ndim=2, t_coords=[0])
            transform[0][:2, 2] += (position0 - center) * spacing1
            si_utils.set_sim_affine(sim, transform, transform_key=self.transition_transform_key)
        frame = fusion.fuse(sims, transform_key=self.transition_transform_key).squeeze()
        frame = float2int_image(normalise_values(frame, window[0], window[1]))
        frame = cv.resize(np.asarray(frame), None, fx=scale, fy=scale)
        if max_size is None:
            max_size = frame.shape[1], frame.shape[0]
            video.size = max_size
        frame = image_reshape(frame, max_size)
        save_tiff(transition_filename + f'{framei:04d}.tiff', frame, None, pixel_size)
        video.write(frame)

    video.close()
saved_progress_paths(output=None)

The saved registration's existing files, in this run's output or another (e.g. before init() has run).

Source code in src\muvis_align\MVSRegistration.py
898
899
900
901
902
903
904
905
906
def saved_progress_paths(self, output=None):
    """The saved registration's existing files, in this run's output or another (e.g. before init() has run)."""
    output = os.path.join(output, '') if output else self.output
    paths = [output + self.input_output_params.get('pair_mappings', default_pair_mappings_name),
             output + self.input_output_params.get('mappings', default_mappings_name),
             output + self.input_output_params.get('mappings', default_mappings_tabular_name),
             output + metrics_name,
             output + residuals_name]
    return [path for path in set(paths) if os.path.exists(path)]
scaled_source_msims(indices, scale)

Sources indices as pre-processing starts from them, without building the others: from the level scale needs, binned down to it where the pyramid stops short.

Source code in src\muvis_align\MVSRegistration.py
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
def scaled_source_msims(self, indices, scale):
    """Sources `indices` as pre-processing starts from them, without building the others: from the level `scale`
    needs, binned down to it where the pyramid stops short."""
    built = self._scaled_msims.get(self._scaled_msims_key(scale, ends_only=True))
    if built is not None:
        msims = [built[index] for index in indices]
    else:
        def build_msim(index):
            from_level = get_level_from_scale(self.sources[index], scale)[0] if scale else 0
            return self._build_source_msim(index, from_level, ends_only=True)
        # kept per scale: flat-field models and normalisation stats each read a group's sources
        source_msims = self._scaled_source_msims.setdefault(scale, {})
        missing = [index for index in indices if index not in source_msims]
        source_msims.update(zip(missing, thread_map(build_msim, missing, default_msim_build_workers)))
        msims = [source_msims[index] for index in indices]
    if scale and scale != 1:
        msims = select_msim_subpyramid_at_scale(msims, [self.sources[index] for index in indices], scale,
                                                bin_shortfall=True)
    return msims
section_registration(indices)

A registration of only the files at indices, labelled as here: to show them before the rest are read.

Source code in src\muvis_align\MVSRegistration.py
704
705
706
707
708
709
710
711
def section_registration(self, indices):
    """A registration of only the files at `indices`, labelled as here: to show them before the rest are read."""
    section = MVSRegistration()
    section.init(operation=self.operation, label=self.fileset_label,
                 input_path=[self.filenames[index] for index in indices],
                 input_labels=[self.file_labels[index] for index in indices], output_path=self.output,
                 overwrite=False, pairing=self.pairing, verbose=self.verbose)
    return section
select_pair_overlap(msim1, msim2, params=None)

The pair's overlap images at the resolution multiview_stitcher would auto-select for it.

Depends only on the source data and the registration channel, not the method or its parameters, so it can be cached across parameter changes. See register_overlap() to register on it.

Source code in src\muvis_align\MVSRegistration.py
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
def select_pair_overlap(self, msim1, msim2, params=None):
    """The pair's overlap images at the resolution multiview_stitcher would auto-select for it.

    Depends only on the source data and the registration channel, not the method or its parameters, so it can
    be cached across parameter changes. See register_overlap() to register on it.
    """
    params = params or {}
    # the binning and overlap helpers below do not handle a 'c' dim themselves
    if 'c' in get_msim_dims(msim1):
        chosen = {}
        msim1 = msi_utils.multiscale_sel_coords(
            msim1, {'c': resolve_registration_channel(msim1, params.get('channel', 0), chosen)})
        msim2 = msi_utils.multiscale_sel_coords(
            msim2, {'c': resolve_registration_channel(msim2, params.get('channel', 0), chosen)})

    sim1_0 = msi_utils.get_sim_from_msim(msim1, scale='scale0')
    sim2_0 = msi_utils.get_sim_from_msim(msim2, scale='scale0')
    registration_binning = registration.get_optimal_registration_binning(sim1_0, sim2_0)
    scale_key, remaining_binning = msi_utils.get_res_level_from_binning_factors(msim1, registration_binning)
    sim1 = msi_utils.get_sim_from_msim(msim1, scale=scale_key)
    sim2 = msi_utils.get_sim_from_msim(msim2, scale=scale_key)
    if max(remaining_binning.values()) > 1:
        binned = []
        for sim in (sim1, sim2):
            sim_b = sim.coarsen(remaining_binning, boundary='trim').mean().astype(sim.dtype)
            sim_b.attrs.update(copy.deepcopy(sim.attrs))
            binned.append(sim_b)
        sim1, sim2 = binned

    return get_overlap_images(sim1, sim2, self.source_transform_key)
source_geometry(transform_key, promote_z=False, z_scale=None, progress=None)

Each source's translation and level-0 stack properties under transform_key, from its metadata and self.mappings alone, building no msim: the registered transform is the mapping chained onto the source transform, as _apply_mappings writes it, identity for a source without one. progress is called a source.

Source code in src\muvis_align\MVSRegistration.py
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
def source_geometry(self, transform_key, promote_z=False, z_scale=None, progress=None):
    """Each source's translation and level-0 stack properties under transform_key, from its metadata and
    self.mappings alone, building no msim: the registered transform is the mapping chained onto the source
    transform, as _apply_mappings writes it, identity for a source without one. `progress` is called a source."""
    translations, stack_props = [], []
    for index, (source, position, transform) in enumerate(zip(self.sources, self.positions,
                                                               self._msim_transforms)):
        translation = dict(position)
        if promote_z and 'z' not in translation:
            translation['z'] = index * (z_scale or 1)
        props = build_source_stack_props(source, self._msim_output_order, translation, transform,
                                         self.source_transform_key, z_scale=self._msim_z_scale,
                                         promote_z=promote_z)
        if transform_key == self.reg_transform_key:
            matrix = np.asarray(props['transform'], dtype=float)
            mapping = (self.mappings or {}).get(index)
            if mapping is None:
                matrix = np.eye(len(matrix))
            else:
                mapping_matrix = widened_affine_matrix(mapping) if len(matrix) == 4 else np.asarray(
                    mapping.sel(t=0) if 't' in mapping.dims else mapping, dtype=float)
                matrix = mapping_matrix @ matrix
            props['transform'] = props['transform'].copy(data=matrix)
        translations.append(translation)
        stack_props.append(props)
        if progress is not None:
            progress()
    return translations, stack_props
source_groups()

Each source's pre-processing group: the input path item it came from (one item: a single group).

Source code in src\muvis_align\MVSRegistration.py
1267
1268
1269
1270
def source_groups(self):
    """Each source's pre-processing group: the input path item it came from (one item: a single group)."""
    input_groups = getattr(self, 'input_groups', None)
    return list(input_groups) if input_groups is not None else [0] * len(self.sources)
split_group_labels(indices)

A label per group of split_groups(indices): its sources' common label prefix (e.g. 'S000'), else its z-position or channel.

Source code in src\muvis_align\MVSRegistration.py
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
def split_group_labels(self, indices):
    """A label per group of split_groups(indices): its sources' common label prefix (e.g. 'S000'), else its
    z-position or channel."""
    groups = self.split_groups(indices)
    labels = []
    for group in range(max(groups) + 1):
        members = [index for index, member_group in zip(indices, groups) if member_group == group]
        label = group_label([self.file_labels[index] for index in members])
        if not label:
            channel = (self.sources[members[0]].get_channels() or [{}])[0].get('label')
            is_channel = (getattr(self, 'registration_dimension', None) == 'c')
            label = str(channel) if is_channel else f"z {self.positions[members[0]].get('z', 0)}"
        labels.append(label)
    return labels
split_groups(indices=None)

Group index per source (of indices, renumbered from 0): z-planes, or channels for dimension 'c'.

Source code in src\muvis_align\MVSRegistration.py
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
def split_groups(self, indices=None):
    """Group index per source (of `indices`, renumbered from 0): z-planes, or channels for dimension 'c'."""
    dimension = (getattr(self, 'registration_dimension', None)
                 or getattr(self, 'register_params', {}).get('registration_dimension'))
    groups = split_groups(self.positions, self.sources, dimension, getattr(self, 'input_groups', None))
    if indices is not None:
        groups = [groups[index] for index in indices]
        order = sorted(set(groups))
        groups = [order.index(group) for group in groups]
    return groups
summary_metrics()

The registration's metrics over all its pairs, by name; empty without a registration.

Source code in src\muvis_align\MVSRegistration.py
2511
2512
2513
2514
2515
2516
2517
2518
2519
2520
2521
2522
def summary_metrics(self):
    """The registration's metrics over all its pairs, by name; empty without a registration."""
    summary = self.metrics.get('summary', {})
    if self.reg_transform_key in summary:
        # the pairs' registration quality is kept apart from the metrics of their registered overlaps
        values = summary[self.reg_transform_key] | summary.get(default_transform_key, {})
    else:
        # loaded from the metrics file, without the overlap areas the summary weighs its pairs by
        pair_metrics = [pair.get(self.reg_transform_key, {}) for pair in self.metrics.get('pairs', {}).values()]
        names = {name for metrics in pair_metrics for name in metrics}
        values = {name: np.mean([metrics[name] for metrics in pair_metrics if name in metrics]) for name in names}
    return {name: float(value) for name, value in values.items() if value is not None and np.isfinite(value)}
validate_overlap(items, labels, is_stack=False, expect_large_overlap=False)
Source code in src\muvis_align\MVSRegistration.py
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
def validate_overlap(self, items, labels, is_stack=False, expect_large_overlap=False):
    # sims or msims: only position/size metadata is read
    sims = sims_from_sims_or_msims(items)
    min_dists = []
    has_overlaps = []
    n = len(sims)
    positions = [get_sim_position_final(sim, get_center=True) for sim in sims]
    sizes = [float(np.linalg.norm(list(get_sim_physical_size(sim).values()))) for sim in sims]
    for i in range(n):
        norm_dists = []
        # check if only single z slices
        if is_stack:
            if i + 1 < n:
                compare_indices = [i + 1]
            else:
                compare_indices = []
        else:
            compare_indices = range(n)
        for j in compare_indices:
            if not j == i:
                distance = math.dist(positions[i].values(), positions[j].values())
                norm_dist = distance / np.mean([sizes[i], sizes[j]])
                norm_dists.append(norm_dist)
        if len(norm_dists) > 0:
            norm_dist = min(norm_dists)
            min_dists.append(float(norm_dist))
            if norm_dist >= 1:
                logging.warning(f'{labels[i]} has no overlap')
                has_overlaps.append(False)
            elif expect_large_overlap and norm_dist > 0.5:
                logging.warning(f'{labels[i]} has small overlap')
                has_overlaps.append(False)
            else:
                has_overlaps.append(True)
    return min_dists, has_overlaps
write_ro_crates(image_filenames, params)

A crate in each fused zarr for where it came from, and a workflow run crate over all output with the run's parameter values. Written once the registration is saved, and again over the fused images.

Source code in src\muvis_align\MVSRegistration.py
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
def write_ro_crates(self, image_filenames, params):
    """A crate in each fused zarr for where it came from, and a workflow run crate over all output with the
    run's parameter values. Written once the registration is saved, and again over the fused images."""
    output_dir = os.path.dirname(self.output)
    end_time = datetime.now()
    try:
        software_version = importlib.metadata.version('muvis-align')
    except importlib.metadata.PackageNotFoundError:
        software_version = None
    software = {'name': 'muvis-align', 'url': MUVIS_ALIGN_URL, 'version': software_version}
    measurements = {name: {'value': value, 'description': f'Registration {name}, over the overlaps of all '
                                                          f'registered pairs'}
                    for name, value in self.summary_metrics().items()}
    sources = {'name': 'Source images', 'paths': self.filenames, 'list_file': 'source-images.csv',
               'description': f'The {len(self.filenames)} source images registered by muvis-align'}
    # empty: fair-metadata-integration's default
    license = params.get('fusion', {}).get('license') or None
    for filename in image_filenames:
        if filename.endswith(zarr_extension):
            # the first and last sources stand for the acquisition: reading every source's metadata is too slow
            create_zarr_ro_crate(self.output + filename, source_paths=[sources],
                                 acquisition_metadata=[self.sources[0].metadata, self.sources[-1].metadata],
                                 measurements=measurements,
                                 description=f'OME-Zarr image fused by muvis-align from {len(self.filenames)} '
                                             f'sources',
                                 derivation={'name': 'Fusion to OME-Zarr', 'software': software,
                                             'end_time': end_time},
                                 license=license)
    result_paths = [self.output + filename for filename in image_filenames] + sorted(self.saved_progress_paths())
    template = importlib.resources.files('muvis_align').joinpath(NAPARI_PROJECT_TEMPLATE)
    with importlib.resources.as_file(template) as template_path:
        create_ro_crate(output_dir, str(template_path), [sources], result_paths, parameters=params,
                        name=f'muvis-align run: {get_filetitle(os.path.normpath(output_dir))}',
                        description=f'Registration and fusion of {len(self.filenames)} source images by '
                                    f'muvis-align, with its settings and results',
                        run_name='muvis-align run', start_time=self.start_time, end_time=end_time,
                        software=software, license=license)

RegState

Bases: Enum

Source code in src\muvis_align\MVSRegistration.py
52
53
54
55
56
57
58
class RegState(Enum):
    UNINIT = auto()
    INIT = auto()
    SIMS_INIT = auto()
    PAIRS_REG = auto()
    GLOBAL_REG = auto()
    FUSED = auto()
FUSED = auto() class-attribute instance-attribute
GLOBAL_REG = auto() class-attribute instance-attribute
INIT = auto() class-attribute instance-attribute
PAIRS_REG = auto() class-attribute instance-attribute
SIMS_INIT = auto() class-attribute instance-attribute
UNINIT = auto() class-attribute instance-attribute

check_operation_layout(params)

Raise for an operation in the CLI's former layout, which the project template's sections replaced.

Source code in src\muvis_align\MVSRegistration.py
184
185
186
187
188
189
def check_operation_layout(params):
    """Raise for an operation in the CLI's former layout, which the project template's sections replaced."""
    old_keys = sorted({'operation', 'input', 'output', 'preprocessing', 'source_metadata'} & set(params))
    if old_keys:
        raise ValueError(f'Operation settings in the former layout ({", ".join(old_keys)}): give them in the '
                         f'sections input_output, pre_processing, registration and fusion, as a project file has them')

Defer link qualities for phase correlation, which scores every candidate shift but keeps only the chosen one's, which resolve_deferred_quality() then computes. Other methods score at once: their images may be views of buffers freed once they return (Elastix's resampled image).

Source code in src\muvis_align\MVSRegistration.py
103
104
105
106
107
108
109
110
111
112
113
114
115
116
@contextmanager
def deferred_link_quality(pairwise_reg_func):
    """Defer link qualities for phase correlation, which scores every candidate shift but keeps only the chosen
    one's, which resolve_deferred_quality() then computes. Other methods score at once: their images may be views of
    buffers freed once they return (Elastix's resampled image)."""
    if pairwise_reg_func is not registration.phase_correlation_registration:
        yield
        return
    original = registration.link_quality_metric_func
    registration.link_quality_metric_func = lambda *args: _DeferredQuality(original, *args)
    try:
        yield
    finally:
        registration.link_quality_metric_func = original

elastix_registration_or_identity(fixed_data, moving_data, *, fixed_origin, moving_origin, fixed_spacing, moving_spacing, initial_affine, transform_types=None, **kwargs)

multiview_stitcher's Elastix registration, or where Elastix fails on a pair (e.g. an overlap too thin to sample) the identity with quality 0, as the feature methods give for a pair with too few points.

Source code in src\muvis_align\MVSRegistration.py
130
131
132
133
134
135
136
137
138
139
140
141
def elastix_registration_or_identity(fixed_data, moving_data, *, fixed_origin, moving_origin, fixed_spacing,
                                     moving_spacing, initial_affine, transform_types=None, **kwargs):
    """multiview_stitcher's Elastix registration, or where Elastix fails on a pair (e.g. an overlap too thin to
    sample) the identity with quality 0, as the feature methods give for a pair with too few points."""
    try:
        return registration.registration_ITKElastix(
            fixed_data, moving_data, fixed_origin=fixed_origin, moving_origin=moving_origin,
            fixed_spacing=fixed_spacing, moving_spacing=moving_spacing, initial_affine=initial_affine,
            transform_types=transform_types, **kwargs)
    except RuntimeError as error:
        logging.warning(f'Elastix: unable to register a pair: {str(error).strip().splitlines()[-1]}')
        return {'affine_matrix': param_utils.identity_transform(len(fixed_spacing)), 'quality': 0}

estimated_pair_cost(graph, edge)

A pair's overlap in pixels at the coarser of its sources' pixel sizes: a proxy for its registration time.

Source code in src\muvis_align\MVSRegistration.py
154
155
156
157
158
def estimated_pair_cost(graph, edge):
    """A pair's overlap in pixels at the coarser of its sources' pixel sizes: a proxy for its registration time."""
    spacings = [graph.nodes[node]['stack_props']['spacing'] for node in edge]
    pixel_size = np.prod([max(spacing[dim] for spacing in spacings) for dim in spacings[0]])
    return graph.edges[edge].get('overlap', 0) / pixel_size

metric_value_range(metric, method=None)

A metric's (low, high) value range. Quality is the registration method's own: multiview_stitcher scores phase correlation, Elastix and ANTs by Spearman correlation, the feature methods and CPD by a 0..1 match rate.

Source code in src\muvis_align\MVSRegistration.py
82
83
84
85
86
87
88
def metric_value_range(metric, method=None):
    """A metric's (low, high) value range. Quality is the registration method's own: multiview_stitcher scores phase
    correlation, Elastix and ANTs by Spearman correlation, the feature methods and CPD by a 0..1 match rate."""
    if metric == default_quality_key:
        is_correlation = registration_method_kind(method) in ('phase_correlation', 'elastix', 'ants')
        return (-1, 1) if is_correlation else (0, 1)
    return metric_value_ranges.get(metric, (0, 1))

normalisation_enabled(value)

Source code in src\muvis_align\MVSRegistration.py
61
62
63
64
65
def normalisation_enabled(value):
    # a project file carries this as text, where 'none'/'no'/'false'/empty means off
    if isinstance(value, str) and value.lower() in ['false', 'no', 'none', '']:
        return False
    return bool(value)

register_pair_in_worker(msims, graph, pairwise_reg_func, kwargs)

compute_pairwise_registrations for one pair in a worker process, with its wall and CPU time.

Source code in src\muvis_align\MVSRegistration.py
144
145
146
147
148
149
150
151
def register_pair_in_worker(msims, graph, pairwise_reg_func, kwargs):
    """compute_pairwise_registrations for one pair in a worker process, with its wall and CPU time."""
    start, cpu_start = time.perf_counter(), time.thread_time()
    with deferred_link_quality(pairwise_reg_func), \
            dask.config.set({'scheduler': 'synchronous', 'optimization.fuse.active': False}):
        registered = compute_pairwise_registrations(
            msims, graph, pairwise_reg_func=resolve_deferred_quality(pairwise_reg_func), **kwargs)
    return registered, time.perf_counter() - start, time.thread_time() - cpu_start

registration_method_kind(method)

The pairwise registration a method name selects: 'cpd', 'feature', 'elastix', 'ants' or 'phase_correlation'.

Source code in src\muvis_align\MVSRegistration.py
68
69
70
71
72
73
74
75
76
77
78
79
def registration_method_kind(method):
    """The pairwise registration a method name selects: 'cpd', 'feature', 'elastix', 'ants' or 'phase_correlation'."""
    method = (method or '').lower()
    if 'cpd' in method:
        return 'cpd'
    if 'feature' in method or 'orb' in method or 'sift' in method:
        return 'feature'
    if 'elastix' in method:
        return 'elastix'
    if 'ant' in method:
        return 'ants'
    return 'phase_correlation'

resolve_deferred_quality(pairwise_reg_func)

pairwise_reg_func with a deferred quality in its result computed. Keeps its signature.

Source code in src\muvis_align\MVSRegistration.py
119
120
121
122
123
124
125
126
127
def resolve_deferred_quality(pairwise_reg_func):
    """`pairwise_reg_func` with a deferred quality in its result computed. Keeps its signature."""
    @functools.wraps(pairwise_reg_func)
    def resolved(*args, **kwargs):
        result = pairwise_reg_func(*args, **kwargs)
        if isinstance(result.get('quality'), _DeferredQuality):
            result['quality'] = result['quality'].value()
        return result
    return resolved

resolve_registration_channel(msim, channel, chosen=None)

The 'c' label to register on in this msim, from channel as an index or a label. A label msim lacks - files that name their channel differently, even within one project - falls back to its only channel if it has one, with a warning; with several it is an error naming them. chosen, a dict shared across a project's msims, keeps one answer (and warning) per set of labels.

Source code in src\muvis_align\MVSRegistration.py
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
def resolve_registration_channel(msim, channel, chosen=None):
    """The 'c' label to register on in this msim, from `channel` as an index or a label. A label
    msim lacks - files that name their channel differently, even within one project - falls back
    to its only channel if it has one, with a warning; with several it is an error naming them.
    `chosen`, a dict shared across a project's msims, keeps one answer (and warning) per set of labels."""
    labels = tuple(str(label) for label in get_msim_image0(msim).coords['c'].values)
    if chosen is not None and labels in chosen:
        return chosen[labels]
    if isinstance(channel, (int, np.integer)):
        label = labels[channel]
    elif channel in labels:
        label = channel
    elif len(labels) == 1:
        logging.warning(f'Registration channel {channel!r} not found - using the only channel, {labels[0]!r}')
        label = labels[0]
    else:
        raise ValueError(f'Registration channel {channel!r} not found; the channels are '
                         + ', '.join(repr(label) for label in labels))
    if chosen is not None:
        chosen[labels] = label
    return label

Pipeline

Pipeline

Bases: Thread

Source code in src\muvis_align\Pipeline.py
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
class Pipeline(Thread):
    def __init__(self, params):
        super().__init__()
        self.params = params

        self.params_general = params['general']
        params_logging = self.params_general.get('logging', {})
        log_filename = params_logging.get('filename', 'log/muvis-align.log')
        log_format = params_logging.get('format', '%(asctime)s %(levelname)s: %(message)s')
        self.verbose = params_logging.get('verbose', False)
        self.debug = params_logging.get('debug', False)
        init_logging(log_filename, log_format, verbose=self.verbose, debug=self.debug)

    def run(self):
        """Run every operation; True if none raised an error (an operation skipped on purpose is no error)."""
        break_on_error = self.params_general.get('break_on_error', False)
        errors = 0
        for operation_params in tqdm(self.params['operations']):
            error = False
            input_path = operation_params.get('input_output', {}).get('input_path')
            logging.info(f'Input: {input_path}')
            try:
                self.run_operation(operation_params)
            except Exception as e:
                logging.exception(f'Error processing: {input_path}')
                print(f'Error processing: {input_path}: {e}')
                error = True
                errors += 1

            if error and break_on_error:
                break

        if errors:
            logging.error(f'Done: {errors} of {len(self.params["operations"])} operations failed')
        else:
            logging.info('Done!')
        return errors == 0

    def run_operation(self, params):
        check_operation_layout(params)
        input_output = params.get('input_output', {})
        operation = params.get('registration', {}).get('operation', '')
        source_metadata = source_metadata_from_params(input_output)
        use_global_metadata = isinstance(source_metadata, dict) and 'global' in source_metadata
        metadata_summary = self.params_general.get('metadata_summary', False)

        path = input_output.get('input_path')
        filenames = sorted(dir_regex(path), key=lambda file: list(find_all_numbers(file)))    # sort first key first
        if len(filenames) == 0:
            logging.warning(f'Skipping operation {operation} (no files)')
            return False
        elif self.verbose:
            logging.info(f'# total files: {len(filenames)}')

        operation_parts = operation.split()
        if 'match' in operation_parts:
            index = operation_parts.index('match') + 1
            if index < len(operation_parts):
                match_label = operation_parts[index]
            else:
                match_label = None
            matches = {}
            for filename in filenames:
                match_value = find_target_numeric(filename, match_label)
                if match_value is not None:
                    if match_value not in matches:
                        matches[match_value] = []
                    matches[match_value].append(filename)
            if len(matches) == 0:
                matches[0] = filenames
            filesets = []
            fileset_labels = []
            for label in sorted(matches):
                filesets.append(matches[label])
                fileset_labels.append(f'{match_label}:{label}')
            logging.info(f'# matched file sets: {len(filesets)}')
        else:
            filesets = [filenames]
            fileset_labels = [get_filetitle(filename) for filename in filenames]

        metadatas = []
        rotations = []
        global_center = None
        if metadata_summary or use_global_metadata:
            for fileset, fileset_label in zip(filesets, fileset_labels):
                metadata = get_images_metadata(fileset, source_metadata)
                if metadata_summary:
                    logging.info(f'File set: {fileset_label} metadata:\n' + metadata['summary'])
                metadatas.append(metadata)
            if use_global_metadata:
                global_center = {dim: np.mean([metadata['center'][dim] for metadata in metadatas]) for dim in metadatas[0]['center']}
                rotations = [metadata['rotation'] for metadata in metadatas]
                rotations = pd.Series(rotations).interpolate(limit_direction='both').to_numpy()

        n_set_workers = input_output.get('n_set_workers', 1)
        if n_set_workers > 1:
            value_sets = []
            for index, (fileset, fileset_label) in enumerate(zip(filesets, fileset_labels)):
                center = global_center if use_global_metadata else None
                rotation = rotations[index] if use_global_metadata else None
                value_sets.append({'fileset_label': fileset_label,
                                   'fileset': fileset,
                                   'params': params,
                                   'center': center,
                                   'rotation': rotation})

            with ThreadPoolExecutor(max_workers=n_set_workers) as executor:
                oks = executor.map(lambda kwargs: self.run_operation_thread(**kwargs), value_sets)
            return np.all([ok for ok in oks])
        else:
            ok = False
            for index, (fileset, fileset_label) in enumerate(zip(filesets, fileset_labels)):
                center = global_center if use_global_metadata else None
                rotation = rotations[index] if use_global_metadata else None
                ok |= self.run_operation_thread(fileset_label=fileset_label, fileset=fileset, params=params,
                                                 center=center, rotation=rotation)
            return ok

    def run_operation_thread(self, fileset_label, fileset, params, center, rotation):
        if fileset_label:
            logging.info(f'File set: {fileset_label}')

        mvs_registration = MVSRegistration()
        mvs_registration.init_params(params_general=self.params_general, params=params,
                                     label=fileset_label, input_path=fileset,
                                     global_center=center, global_rotation=rotation)
        return mvs_registration.run()
debug = params_logging.get('debug', False) instance-attribute
params = params instance-attribute
params_general = params['general'] instance-attribute
verbose = params_logging.get('verbose', False) instance-attribute
__init__(params)
Source code in src\muvis_align\Pipeline.py
16
17
18
19
20
21
22
23
24
25
26
def __init__(self, params):
    super().__init__()
    self.params = params

    self.params_general = params['general']
    params_logging = self.params_general.get('logging', {})
    log_filename = params_logging.get('filename', 'log/muvis-align.log')
    log_format = params_logging.get('format', '%(asctime)s %(levelname)s: %(message)s')
    self.verbose = params_logging.get('verbose', False)
    self.debug = params_logging.get('debug', False)
    init_logging(log_filename, log_format, verbose=self.verbose, debug=self.debug)
run()

Run every operation; True if none raised an error (an operation skipped on purpose is no error).

Source code in src\muvis_align\Pipeline.py
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
def run(self):
    """Run every operation; True if none raised an error (an operation skipped on purpose is no error)."""
    break_on_error = self.params_general.get('break_on_error', False)
    errors = 0
    for operation_params in tqdm(self.params['operations']):
        error = False
        input_path = operation_params.get('input_output', {}).get('input_path')
        logging.info(f'Input: {input_path}')
        try:
            self.run_operation(operation_params)
        except Exception as e:
            logging.exception(f'Error processing: {input_path}')
            print(f'Error processing: {input_path}: {e}')
            error = True
            errors += 1

        if error and break_on_error:
            break

    if errors:
        logging.error(f'Done: {errors} of {len(self.params["operations"])} operations failed')
    else:
        logging.info('Done!')
    return errors == 0
run_operation(params)
Source code in src\muvis_align\Pipeline.py
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
def run_operation(self, params):
    check_operation_layout(params)
    input_output = params.get('input_output', {})
    operation = params.get('registration', {}).get('operation', '')
    source_metadata = source_metadata_from_params(input_output)
    use_global_metadata = isinstance(source_metadata, dict) and 'global' in source_metadata
    metadata_summary = self.params_general.get('metadata_summary', False)

    path = input_output.get('input_path')
    filenames = sorted(dir_regex(path), key=lambda file: list(find_all_numbers(file)))    # sort first key first
    if len(filenames) == 0:
        logging.warning(f'Skipping operation {operation} (no files)')
        return False
    elif self.verbose:
        logging.info(f'# total files: {len(filenames)}')

    operation_parts = operation.split()
    if 'match' in operation_parts:
        index = operation_parts.index('match') + 1
        if index < len(operation_parts):
            match_label = operation_parts[index]
        else:
            match_label = None
        matches = {}
        for filename in filenames:
            match_value = find_target_numeric(filename, match_label)
            if match_value is not None:
                if match_value not in matches:
                    matches[match_value] = []
                matches[match_value].append(filename)
        if len(matches) == 0:
            matches[0] = filenames
        filesets = []
        fileset_labels = []
        for label in sorted(matches):
            filesets.append(matches[label])
            fileset_labels.append(f'{match_label}:{label}')
        logging.info(f'# matched file sets: {len(filesets)}')
    else:
        filesets = [filenames]
        fileset_labels = [get_filetitle(filename) for filename in filenames]

    metadatas = []
    rotations = []
    global_center = None
    if metadata_summary or use_global_metadata:
        for fileset, fileset_label in zip(filesets, fileset_labels):
            metadata = get_images_metadata(fileset, source_metadata)
            if metadata_summary:
                logging.info(f'File set: {fileset_label} metadata:\n' + metadata['summary'])
            metadatas.append(metadata)
        if use_global_metadata:
            global_center = {dim: np.mean([metadata['center'][dim] for metadata in metadatas]) for dim in metadatas[0]['center']}
            rotations = [metadata['rotation'] for metadata in metadatas]
            rotations = pd.Series(rotations).interpolate(limit_direction='both').to_numpy()

    n_set_workers = input_output.get('n_set_workers', 1)
    if n_set_workers > 1:
        value_sets = []
        for index, (fileset, fileset_label) in enumerate(zip(filesets, fileset_labels)):
            center = global_center if use_global_metadata else None
            rotation = rotations[index] if use_global_metadata else None
            value_sets.append({'fileset_label': fileset_label,
                               'fileset': fileset,
                               'params': params,
                               'center': center,
                               'rotation': rotation})

        with ThreadPoolExecutor(max_workers=n_set_workers) as executor:
            oks = executor.map(lambda kwargs: self.run_operation_thread(**kwargs), value_sets)
        return np.all([ok for ok in oks])
    else:
        ok = False
        for index, (fileset, fileset_label) in enumerate(zip(filesets, fileset_labels)):
            center = global_center if use_global_metadata else None
            rotation = rotations[index] if use_global_metadata else None
            ok |= self.run_operation_thread(fileset_label=fileset_label, fileset=fileset, params=params,
                                             center=center, rotation=rotation)
        return ok
run_operation_thread(fileset_label, fileset, params, center, rotation)
Source code in src\muvis_align\Pipeline.py
133
134
135
136
137
138
139
140
141
def run_operation_thread(self, fileset_label, fileset, params, center, rotation):
    if fileset_label:
        logging.info(f'File set: {fileset_label}')

    mvs_registration = MVSRegistration()
    mvs_registration.init_params(params_general=self.params_general, params=params,
                                 label=fileset_label, input_path=fileset,
                                 global_center=center, global_rotation=rotation)
    return mvs_registration.run()

Timer

Timer

Bases: object

Source code in src\muvis_align\Timer.py
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
class Timer(object):
    def __init__(self, title, auto_unit=True, verbose=True):
        self.title = title
        self.auto_unit = auto_unit
        self.verbose = verbose
        self.reset()

    def __enter__(self):
        self.ptime_start = time.process_time()
        self.time_start = time.time()

    def __exit__(self, type, value, traceback):
        if self.verbose:
            ptime_end = time.process_time()
            time_end = time.time()
            pelapsed = ptime_end - self.ptime_start
            elapsed = time_end - self.time_start
            unit = 'seconds'
            if self.auto_unit and elapsed >= 60:
                pelapsed /= 60
                elapsed /= 60
                unit = 'minutes'
                if elapsed >= 60:
                    pelapsed /= 60
                    elapsed /= 60
                    unit = 'hours'
            logging.info(f'Time {self.title}: {elapsed:.1f} ({pelapsed:.1f}) {unit}'
                         f'{print_memory_usage()}')

    def reset(self):
        self.total_ptime = 0
        self.total_time = 0
        self.ntotal = 0

    def start(self):
        self.ptime_start = time.process_time()
        self.time_start = time.time()

    def record(self):
        ptime_end = time.process_time()
        time_end = time.time()
        pelapsed = ptime_end - self.ptime_start
        elapsed = time_end - self.time_start
        self.total_ptime += pelapsed
        self.total_time += elapsed
        self.ntotal += 1

    def get_total_time(self):
        if self.verbose:
            logging.info(f'Time {self.title}: {self.total_time:.1f} ({self.total_ptime:.1f}) seconds #{self.ntotal}')
        return self.total_time, self.total_ptime, self.ntotal
auto_unit = auto_unit instance-attribute
title = title instance-attribute
verbose = verbose instance-attribute
__enter__()
Source code in src\muvis_align\Timer.py
16
17
18
def __enter__(self):
    self.ptime_start = time.process_time()
    self.time_start = time.time()
__exit__(type, value, traceback)
Source code in src\muvis_align\Timer.py
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
def __exit__(self, type, value, traceback):
    if self.verbose:
        ptime_end = time.process_time()
        time_end = time.time()
        pelapsed = ptime_end - self.ptime_start
        elapsed = time_end - self.time_start
        unit = 'seconds'
        if self.auto_unit and elapsed >= 60:
            pelapsed /= 60
            elapsed /= 60
            unit = 'minutes'
            if elapsed >= 60:
                pelapsed /= 60
                elapsed /= 60
                unit = 'hours'
        logging.info(f'Time {self.title}: {elapsed:.1f} ({pelapsed:.1f}) {unit}'
                     f'{print_memory_usage()}')
__init__(title, auto_unit=True, verbose=True)
Source code in src\muvis_align\Timer.py
10
11
12
13
14
def __init__(self, title, auto_unit=True, verbose=True):
    self.title = title
    self.auto_unit = auto_unit
    self.verbose = verbose
    self.reset()
get_total_time()
Source code in src\muvis_align\Timer.py
56
57
58
59
def get_total_time(self):
    if self.verbose:
        logging.info(f'Time {self.title}: {self.total_time:.1f} ({self.total_ptime:.1f}) seconds #{self.ntotal}')
    return self.total_time, self.total_ptime, self.ntotal
record()
Source code in src\muvis_align\Timer.py
47
48
49
50
51
52
53
54
def record(self):
    ptime_end = time.process_time()
    time_end = time.time()
    pelapsed = ptime_end - self.ptime_start
    elapsed = time_end - self.time_start
    self.total_ptime += pelapsed
    self.total_time += elapsed
    self.ntotal += 1
reset()
Source code in src\muvis_align\Timer.py
38
39
40
41
def reset(self):
    self.total_ptime = 0
    self.total_time = 0
    self.ntotal = 0
start()
Source code in src\muvis_align\Timer.py
43
44
45
def start(self):
    self.ptime_start = time.process_time()
    self.time_start = time.time()

constants

MUVIS_ALIGN_URL = 'https://github.com/ccp-volume-em/muvis-align' module-attribute

NAPARI_PROJECT_TEMPLATE = 'ui/project_template.yaml' module-attribute

default_chunk_size = 1024 module-attribute

default_contrast_limits_max_tasks = 4096 module-attribute

default_export_chunk_size = 4096 module-attribute

default_fusion_block_bytes = default_fusion_memory_pool // min(max(1, default_fusion_workers), fusion_blocks_at_once) module-attribute

default_fusion_chunk_bytes = min(4 * 1024 ** 3, max(64 * 1024 ** 2, int((_available_memory_bytes or 16 * 1024 ** 3) * 0.25 / max(1, _available_cpus)))) module-attribute

default_fusion_memory_pool = max(64 * 1024 ** 2, int((_available_memory_bytes or 16 * 1024 ** 3) * 0.25)) module-attribute

default_fusion_workers = _available_cpus module-attribute

default_interactive_preview_scale = 16 module-attribute

default_mappings_name = 'mappings.json' module-attribute

default_mappings_tabular_name = 'mappings.csv' module-attribute

default_msim_build_workers = min(default_source_init_workers, 32) module-attribute

default_ome_zarr_version = '0.5' module-attribute

default_pair_mappings_name = 'pair_mappings.json' module-attribute

default_pair_worker_tasks = 1000 module-attribute

default_pair_workers = _available_cpus module-attribute

default_preview_max_bytes = 2048 * 2048 * 2 module-attribute

default_preview_max_volume_bytes = 4 * 1024 ** 3 module-attribute

default_quality_key = 'quality' module-attribute

default_source_init_workers = min(_source_init_worker_ceiling(), _available_cpus * 8) module-attribute

default_split_binning = 8 module-attribute

default_transform_key = 'transform' module-attribute

flatfield_min_images = 5 module-attribute

fusion_blocks_at_once = 16 module-attribute

fusion_stack_arrays = 3 module-attribute

metric_value_ranges = {'ncc': (-1, 1), 'ssim': (-1, 1), 'onmi': (0, 1)} module-attribute

metrics_name = 'metrics.csv' module-attribute

napari_dask_cache_share = 0.25 module-attribute

original_positions_name = 'positions_original.pdf' module-attribute

pair_table_columns = ['id1', 'id2', 'filename1', 'filename2'] module-attribute

prereg_mappings_name = 'prereg_mappings.csv' module-attribute

registered_positions_name = 'positions_registered.pdf' module-attribute

registration_method_params = {'normalisation': ('feature', 'cpd'), 'gaussian_sigma': ('feature', 'cpd'), 'max_keypoints': ('feature', 'cpd'), 'inlier_threshold_factor': ('feature',), 'max_trials': ('feature',), 'ransac_iterations': ('feature',)} module-attribute

residuals_name = 'residuals.csv' module-attribute

section_number_labels = ('section', 'slice', 's', 'z') module-attribute

spatial_unit = 'micrometer' module-attribute

split_grid_margin = 0.05 module-attribute

split_group_pair_kind = 'split_group' module-attribute

split_smoothing = 1 / 25 module-attribute

tiff_extension = '.ome.tiff' module-attribute

zarr_extension = '.ome.zarr' module-attribute

fusion_blocks

Fusion to OME-Zarr level by level and block by block, only the blocks some source reaches, each from only the sources reaching it: multiview_stitcher runs Python over every source it gives a block.

average_into_zarr_array(parent_url, store_url, factors, chunk, spatial_blocks, creation_kwargs=None, batch_options=None, progress=None, share=0)

A zarr array averaging the one at parent_url by whole factors per trailing spatial dim, writing only the chunk blocks in spatial_blocks: a pyramid level without z averaged across sections or empty blocks computed.

Source code in src\muvis_align\fusion_blocks.py
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
def average_into_zarr_array(parent_url, store_url, factors, chunk, spatial_blocks, creation_kwargs=None,
                            batch_options=None, progress=None, share=0):
    """A zarr array averaging the one at parent_url by whole `factors` per trailing spatial dim, writing only the
    `chunk` blocks in spatial_blocks: a pyramid level without z averaged across sections or empty blocks computed."""
    batch_options = batch_options or {}
    parent = zarr.open_array(parent_url, mode='r')
    sdims = [dim for dim in 'zyx' if dim in chunk]
    nonspatial = parent.ndim - len(sdims)
    axis_factors = [1] * nonspatial + [int(factors[dim]) for dim in sdims]
    shape = [size // factor for size, factor in zip(parent.shape, axis_factors)]
    chunks = [1] * nonspatial + [int(chunk[dim]) for dim in sdims]
    output = zarr.create(shape=shape, chunks=chunks, dtype=parent.dtype, store=store_url, overwrite=True,
                         **(creation_kwargs or {}))

    def average_block(block_id):
        target = tuple(slice(block * size, min((block + 1) * size, total))
                       for block, size, total in zip(block_id, chunks, shape))
        data = parent[tuple(slice(piece.start * factor, piece.stop * factor)
                            for piece, factor in zip(target, axis_factors))]
        grouped = data.reshape([size for length, factor in zip(data.shape, axis_factors)
                                for size in (length // factor, factor)])
        output[target] = grouped.mean(axis=tuple(range(1, grouped.ndim, 2))).astype(parent.dtype)

    block_ids = sorted(tuple(nonspatial_block) + tuple(spatial_block)
                       for nonspatial_block in np.ndindex(*shape[:nonspatial]) for spatial_block in spatial_blocks)
    # the parent region as read, and its mean taken in float64
    block_bytes = int(np.prod(axis_factors) * np.prod(chunks)) * (parent.dtype.itemsize + 8)
    _run_blocks(average_block, block_ids, batch_options, progress, share, block_bytes=block_bytes)

block_sources(bounds, output_stack_properties, output_chunksize, sdims)

The blocks some source reaches, grouped by exactly which: {source indices: [spatial block indices]}.

Source code in src\muvis_align\fusion_blocks.py
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
def block_sources(bounds, output_stack_properties, output_chunksize, sdims):
    """The blocks some source reaches, grouped by exactly which: {source indices: [spatial block indices]}."""
    overlaps = [_block_overlaps(bounds[:, axis], output_stack_properties['origin'][dim],
                                output_stack_properties['spacing'][dim], int(output_chunksize[dim]),
                                int(np.ceil(output_stack_properties['shape'][dim] / int(output_chunksize[dim]))))
                for axis, dim in enumerate(sdims)]
    groups = {}
    # the first dim's blocks one at a time keeps the remaining product small
    for first_block, first_overlap in enumerate(overlaps[0]):
        candidates = np.flatnonzero(first_overlap)
        if len(candidates):
            rest = [overlap[:, candidates] for overlap in overlaps[1:]]
            for rest_blocks in np.ndindex(*[len(overlap) for overlap in rest]):
                reach = np.ones(len(candidates), dtype=bool)
                for overlap, block in zip(rest, rest_blocks):
                    reach &= overlap[block]
                if reach.any():
                    groups.setdefault(tuple(candidates[reach].tolist()), []).append((first_block,) + rest_blocks)
    return groups

budget_chunksize(bounds, output_stack_properties, output_chunksize, sdims, budget_bytes, bytes_per_voxel, min_xy_chunk=256)

The largest yx chunk, shrinking from output_chunksize, whose busiest block fits budget_bytes: a block holds every source it meets transformed to its full size, counted from the sources' bounds.

Source code in src\muvis_align\fusion_blocks.py
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
def budget_chunksize(bounds, output_stack_properties, output_chunksize, sdims, budget_bytes, bytes_per_voxel,
                     min_xy_chunk=256):
    """The largest yx chunk, shrinking from output_chunksize, whose busiest block fits budget_bytes: a block holds
    every source it meets transformed to its full size, counted from the sources' bounds."""
    chunk = {dim: int(size) for dim, size in output_chunksize.items()}
    xy_dims = [dim for dim in ('y', 'x') if dim in chunk]
    while True:
        overlaps = {}
        for axis, dim in enumerate(sdims):
            nblocks = int(np.ceil(output_stack_properties['shape'][dim] / chunk[dim]))
            overlaps[dim] = _block_overlaps(bounds[:, axis], output_stack_properties['origin'][dim],
                                            output_stack_properties['spacing'][dim], chunk[dim], nblocks)
        busiest = 0
        for z_overlap in (overlaps['z'] if 'z' in overlaps else [np.ones(len(bounds), dtype=bool)]):
            sources = np.flatnonzero(z_overlap)
            if len(sources) and len(xy_dims) == 2:
                counts = overlaps['y'][:, sources].astype(np.int32) @ overlaps['x'][:, sources].T.astype(np.int32)
                busiest = max(busiest, int(counts.max()))
            elif len(sources):
                busiest = max(busiest, len(sources))
        voxels = int(np.prod([chunk[dim] for dim in sdims]))
        needed = busiest * voxels * bytes_per_voxel
        if needed <= budget_bytes or all(chunk[dim] <= min_xy_chunk for dim in xy_dims):
            return chunk
        # shrink by just the factor needed, then recheck: every extra block has a fixed overhead
        factor = min(0.95, np.sqrt(budget_bytes / needed))
        for dim in xy_dims:
            chunk[dim] = max(min_xy_chunk, int(chunk[dim] * factor) // 64 * 64)

export_levels(source_spacings, level0_properties, scaled_dims, native=True, tolerance=0.05)

(pixel size, its native_level_stack_properties, indices of the sources fused into it) per level of an export. Native: the levels of native_level_spacings, each from the sources whose group's size (source_level_sizes) is at least that fine. Otherwise level 0's own pixel size doubling, every level from every source.

Source code in src\muvis_align\fusion_blocks.py
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
def export_levels(source_spacings, level0_properties, scaled_dims, native=True, tolerance=0.05):
    """(pixel size, its native_level_stack_properties, indices of the sources fused into it) per level of an export.
    Native: the levels of native_level_spacings, each from the sources whose group's size (source_level_sizes) is at
    least that fine. Otherwise level 0's own pixel size doubling, every level from every source."""
    finest_shape = [level0_properties['shape'][dim] for dim in scaled_dims]
    if native:
        level_spacings = native_level_spacings(source_spacings, finest_shape, tolerance=tolerance)
    else:
        level_spacings = native_level_spacings([min(level0_properties['spacing'][dim] for dim in 'yx')], finest_shape)
    levels = native_level_stack_properties(level0_properties, level_spacings, scaled_dims)
    level_sizes = source_level_sizes(source_spacings)
    return [(level_spacing, properties,
             [index for index, spacing in enumerate(source_spacings)
              if not native or level_sizes[spacing] <= level_spacing * (1 + tolerance)])
            for level_spacing, properties in zip(level_spacings, levels)]

fuse_into_zarr_array(sims, store_url, transform_key, output_stack_properties, output_chunksize, fusion_func=None, creation_kwargs=None, batch_options=None, interpolation_order=1, progress=None, share=0, ranks=None, groups=None)

Fuse sims into a new zarr array at store_url: only the blocks some source reaches, each from only the sources reaching it. With ranks (lower first, e.g. pixel sizes), only the lowest-ranked source at each pixel is fused. The blocks move progress by share in all. Returns dims and properties.

Source code in src\muvis_align\fusion_blocks.py
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
def fuse_into_zarr_array(sims, store_url, transform_key, output_stack_properties, output_chunksize, fusion_func=None,
                         creation_kwargs=None, batch_options=None, interpolation_order=1, progress=None, share=0,
                         ranks=None, groups=None):
    """Fuse `sims` into a new zarr array at store_url: only the blocks some source reaches, each from only the sources
    reaching it. With `ranks` (lower first, e.g. pixel sizes), only the lowest-ranked source at each pixel is fused.
    The blocks move `progress` by `share` in all. Returns dims and properties."""
    batch_options = batch_options or {}
    dims = list(sims[0].dims)
    sdims = list(si_utils.get_spatial_dims_from_sim(sims[0]))
    block_axes = [dims.index(dim) for dim in sdims]
    if groups is None:
        bounds = source_bounds(sims, transform_key, output_stack_properties, interpolation_order)
        groups = block_sources(bounds, output_stack_properties, output_chunksize, sdims)

    def prepare(sources, create_output):
        fuse_kwargs = {'images': [sims[index] for index in sources], 'transform_key': transform_key,
                       'output_chunksize': output_chunksize,
                       'output_stack_properties': copy.deepcopy(output_stack_properties),
                       'interpolation_order': interpolation_order}
        block_func = fusion_func
        if ranks is not None and len({ranks[index] for index in sources}) > 1:
            group_params = [np.asarray(si_utils.get_affine_from_sim(sims[index], transform_key=transform_key)).squeeze()
                            for index in sources]
            block_func = prioritise_finer_views(fusion_func or fusion_core.weighted_average_fusion, group_params,
                                                [ranks[index] for index in sources])
        # None would replace multiview_stitcher's own default
        if block_func is not None:
            fuse_kwargs['fusion_func'] = block_func
        return fusion_core.prepare_block_fusion(store_url, fuse_kwargs=fuse_kwargs,
                                                zarr_array_creation_kwargs=creation_kwargs,
                                                create_output=create_output, verbose=False)

    # the store is created first, from any source: a block none reaches is never written
    info = prepare((0,), create_output=True)
    nblocks = info['nblocks']
    nonspatial_axes = [axis for axis in range(len(dims)) if axis not in block_axes]
    funcs = {}
    block_stats = {}
    for sources, spatial_blocks in groups.items():
        func = prepare(sources, create_output=False)['func']
        stats = (len(sources), sum(sims[index].data.nbytes for index in sources))
        for nonspatial_block in np.ndindex(*[nblocks[axis] for axis in nonspatial_axes]):
            for spatial_block in spatial_blocks:
                block_id = [0] * len(dims)
                for axis, block in zip(nonspatial_axes + block_axes, nonspatial_block + spatial_block):
                    block_id[axis] = block
                funcs[tuple(block_id)] = func
                block_stats[tuple(block_id)] = stats

    def fuse_block(block_id):
        return funcs[block_id](block_id)

    # what the memory budget counts for a source in a block (budget_chunksize)
    source_block_bytes = int(np.prod([int(output_chunksize[dim]) for dim in sdims])) * 4 * fusion_stack_arrays
    _run_blocks(fuse_block, sorted(funcs), batch_options, progress, share,
                block_stats=block_stats, source_block_bytes=source_block_bytes)
    return dims, info['output_stack_properties']

fuse_levels_to_ome_zarr(msims, levels, output_zarr_url, transform_key, level0_properties, output_chunksize, fusion_func=None, zarr_options=None, batch_options=None, interpolation_order=1, ranks=None, average_levels=False, level_weights=None)

An OME-Zarr multiscale image of levels (export_levels), each fused at its own pixel size from its sources at their nearest level - or, where its sources are the level above's (every level, with average_levels), averaged from that level: a source's own coarse levels end before the output's, and reading a finer one aliases. With ranks (one a source, lower first), a higher-ranked source under a lower-ranked one is left out. One progress bar runs over all levels, each its share of the square root of level_weights (the bytes it writes), else of its voxels.

Source code in src\muvis_align\fusion_blocks.py
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
def fuse_levels_to_ome_zarr(msims, levels, output_zarr_url, transform_key, level0_properties, output_chunksize,
                            fusion_func=None, zarr_options=None, batch_options=None, interpolation_order=1,
                            ranks=None, average_levels=False, level_weights=None):
    """An OME-Zarr multiscale image of `levels` (export_levels), each fused at its own pixel size from its sources at
    their nearest level - or, where its sources are the level above's (every level, with average_levels), averaged
    from that level: a source's own coarse levels end before the output's, and reading a finer one aliases. With `ranks`
    (one a source, lower first), a higher-ranked source under a lower-ranked one is left out. One progress bar runs
    over all levels, each its share of the square root of `level_weights` (the bytes it writes), else of its voxels."""
    ngff_version, creation_kwargs = _zarr_options(zarr_options)
    _remove_existing(output_zarr_url, zarr_options)
    logging.info(f'Fusion levels {[float(round(level[0], 4)) for level in levels]}')
    paths = level_paths(len(levels))
    if level_weights is None or len(level_weights) != len(levels) or not sum(level_weights) > 0:
        level_weights = [float(np.prod(list(properties['shape'].values()))) for _, properties, _ in levels]
    # a level's time grows slower than its size, each block's cost being partly per source
    level_weights = [float(np.sqrt(weight)) for weight in level_weights]
    progress = fusion_core.tqdm(total=float(sum(level_weights)))
    with _numpy_fusion_plan():
        for index, (level_spacing, properties, selected) in enumerate(levels):
            output_properties = {key: properties[key] for key in ('spacing', 'origin', 'shape')}
            # the source level at the output's pixel size: spacings read off coordinates differ in the last digits
            sims = [msi_utils.get_sim_from_msim(msims[source], scale='scale%d' % coarsest_level_within(
                msim_level_spacings(msims[source]), output_properties['spacing'],
                dims=list(output_properties['spacing']))) for source in selected]
            level_chunksize, groups = level_blocks(sim_views(sims, transform_key), output_properties,
                                                   output_chunksize, interpolation_order)
            progress.set_description(f'{index + 1}/{len(levels)} @ {print_significants(level_spacing, 3)} µm')
            if index > 0 and (average_levels or selected == levels[index - 1][2]):
                factors = {dim: round(properties['factors'][dim] / levels[index - 1][1]['factors'][dim])
                           for dim in output_properties['shape']}
                average_into_zarr_array(os.path.join(output_zarr_url, paths[index - 1]),
                                        os.path.join(output_zarr_url, paths[index]), factors, level_chunksize,
                                        [block for blocks in groups.values() for block in blocks],
                                        creation_kwargs=creation_kwargs, batch_options=batch_options,
                                        progress=progress, share=level_weights[index])
            else:
                busiest = max(groups, key=len, default=())
                logging.info(f'Fusion level {index + 1} fused from {len(sims)} sources: chunk {level_chunksize},'
                             f' {sum(len(blocks) for blocks in groups.values())} blocks, budget'
                             f' {print_hbytes(default_fusion_block_bytes)} a block of a'
                             f' {print_hbytes(default_fusion_memory_pool)} pool, busiest block'
                             f' {len(busiest)} sources'
                             f' ({print_hbytes(sum(sims[source].data.nbytes for source in busiest))} of source levels)')
                fuse_into_zarr_array(sims, os.path.join(output_zarr_url, paths[index]), transform_key,
                                     output_properties, level_chunksize, fusion_func=fusion_func,
                                     creation_kwargs=creation_kwargs, batch_options=batch_options,
                                     interpolation_order=interpolation_order, progress=progress,
                                     share=level_weights[index],
                                     ranks=[ranks[source] for source in selected] if ranks is not None else None,
                                     groups=groups)
        progress.close()
    sim0 = msi_utils.get_sim_from_msim(msims[0], scale='scale0')
    coordtfs, axes = ngff_utils.calc_ngff_coordinate_transformations_and_axes(
        level0_properties, [properties['factors'] for _, properties, _ in levels],
        nsdims=list(si_utils.get_nonspatial_dims_from_sim(sim0)), time_transform=ngff_utils.get_ngff_time_transform(sim0))
    group = zarr.open_group(output_zarr_url, mode='a',
                            **ngff_utils.zarr_group_creation_kwargs_for_ngff_version(storage_ngff_version(ngff_version)))
    write_multiscales_metadata(
        group, axes=axes, ngff_version=ngff_version,
        datasets=[{'path': paths[index], 'coordinateTransformations': coordtfs[index]} for index in range(len(levels))])
    return ngff_utils.read_msim_from_ome_zarr(output_zarr_url, transform_key=transform_key, array_backend='dask')

level_blocks(views, output_properties, output_chunksize, interpolation_order=1)

(chunk size, {source group: blocks}) of one fused level, its views - (stack properties, affine matrix) - at the source levels read for it: the chunk clipped to the level, then shrunk to the memory budget.

Source code in src\muvis_align\fusion_blocks.py
442
443
444
445
446
447
448
449
450
def level_blocks(views, output_properties, output_chunksize, interpolation_order=1):
    """(chunk size, {source group: blocks}) of one fused level, its views - (stack properties, affine matrix) - at
    the source levels read for it: the chunk clipped to the level, then shrunk to the memory budget."""
    sdims = list(views[0][0]['shape'])
    bounds = view_bounds(views, output_properties, interpolation_order)
    chunk = {dim: min(int(output_chunksize[dim]), output_properties['shape'][dim]) for dim in output_chunksize}
    chunk = budget_chunksize(bounds, output_properties, chunk, sdims, default_fusion_block_bytes,
                             4 * fusion_stack_arrays)
    return chunk, block_sources(bounds, output_properties, chunk, sdims)

level_paths(count)

Level path names, zero-padded from 11 levels on: readers listing arrays by name (napari) sort them as text.

Source code in src\muvis_align\fusion_blocks.py
462
463
464
465
def level_paths(count):
    """Level path names, zero-padded from 11 levels on: readers listing arrays by name (napari) sort them as text."""
    width = len(str(count - 1)) if count > 10 else 1
    return [str(index).zfill(width) for index in range(count)]

native_level_spacings(source_spacings, finest_shape, min_shape=100, tolerance=0.05)

Output level pixel sizes: the finest source's, doubling, with a level at each coarser group of source pixel sizes (source_level_sizes), until the largest dim would fall to min_shape - every group gets a level, or what only it covers would be in none.

Source code in src\muvis_align\fusion_blocks.py
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
def native_level_spacings(source_spacings, finest_shape, min_shape=100, tolerance=0.05):
    """Output level pixel sizes: the finest source's, doubling, with a level at each coarser group of source pixel
    sizes (source_level_sizes), until the largest dim would fall to min_shape - every group gets a level, or what
    only it covers would be in none."""
    sizes = sorted(set(source_level_sizes(source_spacings).values()))
    levels, pending = [sizes[0]], sizes[1:]
    largest = max(finest_shape)
    while pending or largest * levels[0] / (levels[-1] * 2) > min_shape:
        doubled = levels[-1] * 2
        if pending and doubled >= pending[0] * (1 - tolerance):
            # a step under sqrt(2) to a source size replaces the doubled level before it, never another source size
            if len(levels) > 1 and levels[-1] not in sizes and pending[0] < levels[-1] * np.sqrt(2):
                levels[-1] = pending.pop(0)
            else:
                levels.append(pending.pop(0))
        else:
            levels.append(doubled)
    return levels

native_level_stack_properties(level0_properties, level_spacings, scaled_dims)

Each level's output stack: level 0's extent at the level's pixel size in scaled_dims (other dims kept), its origin by OME-Zarr's pixel-centre convention.

Source code in src\muvis_align\fusion_blocks.py
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
def native_level_stack_properties(level0_properties, level_spacings, scaled_dims):
    """Each level's output stack: level 0's extent at the level's pixel size in `scaled_dims` (other dims kept), its
    origin by OME-Zarr's pixel-centre convention."""
    spacing0 = level0_properties['spacing']
    levels = []
    for level_spacing in level_spacings:
        factors = {dim: level_spacing / level_spacings[0] if dim in scaled_dims else 1.0 for dim in spacing0}
        levels.append({
            'spacing': {dim: spacing0[dim] * factors[dim] for dim in spacing0},
            'origin': {dim: level0_properties['origin'][dim] + (factors[dim] - 1) * spacing0[dim] / 2
                       for dim in spacing0},
            'shape': {dim: max(1, int(level0_properties['shape'][dim] // factors[dim])) for dim in spacing0},
            'factors': factors,
        })
    return levels

prioritise_finer_views(fusion_func, group_params, group_ranks)

fusion_func over only the finest views at each pixel: where a tile covers it, the overview under it is left out rather than averaged in. Views of equal rank are fused as fusion_func would.

Source code in src\muvis_align\fusion_blocks.py
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
def prioritise_finer_views(fusion_func, group_params, group_ranks):
    """fusion_func over only the finest views at each pixel: where a tile covers it, the overview under it is left
    out rather than averaged in. Views of equal rank are fused as fusion_func would."""
    wanted = inspect.signature(fusion_func).parameters

    def fusion(**kwargs):
        views = kwargs['transformed_views']
        ranks = _view_ranks(kwargs.pop('params'), group_params, group_ranks)
        ranks = ranks.reshape((-1,) + (1,) * (views.ndim - 1))
        valid = ~np.isnan(views)
        finest = np.min(np.where(valid, ranks, np.inf), axis=0)
        keep = valid & (ranks == finest)
        kwargs['transformed_views'] = np.where(keep, views, np.nan)
        if kwargs.get('blending_weights') is not None:
            weights = np.where(keep, kwargs['blending_weights'], 0)
            total = weights.sum(axis=0)
            kwargs['blending_weights'] = np.divide(weights, total, out=np.zeros_like(weights), where=total > 0)
        return fusion_func(**{name: value for name, value in kwargs.items() if name in wanted})

    # multiview_stitcher passes what the signature names: fusion_func's own arguments, and the views' affines
    parameters = [inspect.Parameter(name, inspect.Parameter.KEYWORD_ONLY) for name in wanted]
    if 'params' not in wanted:
        parameters.append(inspect.Parameter('params', inspect.Parameter.KEYWORD_ONLY))
    fusion.__signature__ = inspect.Signature(parameters)
    return fusion

source_bounds(sims, transform_key, output_stack_properties, interpolation_order=1)

view_bounds() of sims, each at its own geometry and transform.

Source code in src\muvis_align\fusion_blocks.py
25
26
27
def source_bounds(sims, transform_key, output_stack_properties, interpolation_order=1):
    """view_bounds() of sims, each at its own geometry and transform."""
    return view_bounds(sim_views(sims, transform_key), output_stack_properties, interpolation_order)

source_level_sizes(source_spacings)

{source pixel size: the pixel size of the level it is first fused into}: a size under sqrt(2) above the finest of its group joins that group's level, upsampled by less than that, rather than a few odd sources (overview sections imaged coarser) adding a level of their own.

Source code in src\muvis_align\fusion_blocks.py
375
376
377
378
379
380
381
382
383
384
385
def source_level_sizes(source_spacings):
    """{source pixel size: the pixel size of the level it is first fused into}: a size under sqrt(2) above the finest
    of its group joins that group's level, upsampled by less than that, rather than a few odd sources (overview
    sections imaged coarser) adding a level of their own."""
    level_sizes = {}
    start = None
    for size in sorted(set(source_spacings)):
        if start is None or size >= start * np.sqrt(2):
            start = size
        level_sizes[size] = start
    return level_sizes

storage_ngff_version(ngff_version)

Source code in src\muvis_align\fusion_blocks.py
328
329
330
def storage_ngff_version(ngff_version):
    # 0.6 changed the metadata, not the arrays: they are stored as 0.5's, which multiview_stitcher can create
    return '0.5' if str(ngff_version).startswith('0.6') else ngff_version

view_bounds(views, output_stack_properties, interpolation_order=1)

Per view (stack properties, affine matrix) and spatial dim, the (low, high) physical bounds it reaches in the output, padded for interpolation as multiview_stitcher does, except in grid-aligned or single-plane dims.

Source code in src\muvis_align\fusion_blocks.py
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
def view_bounds(views, output_stack_properties, interpolation_order=1):
    """Per view (stack properties, affine matrix) and spatial dim, the (low, high) physical bounds it reaches in the
    output, padded for interpolation as multiview_stitcher does, except in grid-aligned or single-plane dims."""
    stacks = [stack for stack, _ in views]
    sdims = list(stacks[0]['shape'])
    ndim = len(sdims)
    matrices = np.stack([matrix for _, matrix in views]).astype(float)
    shape = np.array([[stack['shape'][dim] for dim in sdims] for stack in stacks], dtype=float)
    spacing = np.array([[stack['spacing'][dim] for dim in sdims] for stack in stacks], dtype=float)
    origin = np.array([[stack['origin'][dim] for dim in sdims] for stack in stacks], dtype=float)
    corners = np.array(list(np.ndindex(*[2] * ndim)), dtype=float)
    local = corners[None] * ((shape - 1) * spacing)[:, None] + origin[:, None]
    vertices = np.einsum('nij,nkj->nki', matrices[:, :ndim, :ndim], local) + matrices[:, None, :ndim, ndim]
    aligned = _grid_aligned_dims(stacks, matrices, output_stack_properties, sdims)
    # a single plane's spacing is a placeholder 1.0: padded by it, a section reaches its neighbours' blocks
    padding = np.where((shape == 1) | np.isin(sdims, aligned)[None], 0.0, interpolation_order * spacing)
    return np.stack([vertices.min(axis=1) - padding, vertices.max(axis=1) + padding], axis=-1)

write_multiscales_metadata(group, axes, datasets, ngff_version)

multiview_stitcher's write_multiscales_metadata, plus OME-Zarr 0.6 (which it lacks) converted by ngff-zarr.

Source code in src\muvis_align\fusion_blocks.py
341
342
343
344
345
346
347
348
349
350
351
352
353
def write_multiscales_metadata(group, axes, datasets, ngff_version):
    """multiview_stitcher's write_multiscales_metadata, plus OME-Zarr 0.6 (which it lacks) converted by ngff-zarr."""
    if not str(ngff_version).startswith('0.6'):
        ngff_utils.write_multiscales_metadata(group, axes=axes, datasets=datasets, ngff_version=ngff_version)
        return
    metadata = ngff_zarr.Metadata(
        axes=[ngff_zarr.Axis(**dict(axis)) for axis in axes],
        datasets=[ngff_zarr.Dataset(path=dataset['path'],
                                    coordinateTransformations=[_ngff_transform(transform)
                                                               for transform in dataset['coordinateTransformations']])
                  for dataset in datasets],
        coordinateTransformations=None, name=group.name)
    group.attrs['ome'] = {'version': '0.6', 'multiscales': [_without_none(asdict(metadata.to_version('0.6')))]}

written_voxels(groups, output_properties, chunk)

Spatial voxels in the blocks some source reaches, edge blocks clipped to the output.

Source code in src\muvis_align\fusion_blocks.py
453
454
455
456
457
458
459
def written_voxels(groups, output_properties, chunk):
    """Spatial voxels in the blocks some source reaches, edge blocks clipped to the output."""
    sdims = list(output_properties['shape'])
    blocks = np.array([block for group_blocks in groups.values() for block in group_blocks]).reshape(-1, len(sdims))
    chunks = np.array([chunk[dim] for dim in sdims])
    shape = np.array([output_properties['shape'][dim] for dim in sdims])
    return int(np.prod(np.minimum(chunks, shape - blocks * chunks), axis=1).sum())

logging

close_fault_log()

Stop writing native crash stacks to the log, and release its file (Windows cannot delete an open file).

Source code in src\muvis_align\logging.py
60
61
62
63
64
65
66
def close_fault_log():
    """Stop writing native crash stacks to the log, and release its file (Windows cannot delete an open file)."""
    global _fault_file
    if _fault_file is not None:
        faulthandler.disable()
        _fault_file.close()
        _fault_file = None

enable_fault_log(log_filename)

Every thread's stack appended to the log on a fatal signal: a native crash otherwise leaves no trace at all.

Source code in src\muvis_align\logging.py
46
47
48
49
50
51
52
53
54
55
56
57
def enable_fault_log(log_filename):
    """Every thread's stack appended to the log on a fatal signal: a native crash otherwise leaves no trace at all."""
    global _fault_file
    if sys.platform == 'win32':
        # Windows also reports access violations its drivers raise and handle themselves: dumps that look like crashes
        return
    # faulthandler holds only the descriptor: the file must stay open for as long as it is enabled
    fault_file = open(log_filename, 'a', encoding='utf-8')
    faulthandler.enable(file=fault_file, all_threads=True)
    if _fault_file is not None:
        _fault_file.close()
    _fault_file = fault_file

init_logging(log_filename='log/muvis-align.log', log_format='%(asctime)s %(levelname)s: %(message)s', verbose=False, debug=False)

verbose: muvis-align's own logging (echoed to console, not just the log file). debug: external libraries (multiview_stitcher, zarr) at DEBUG level - off by default, since unlisted loggers also inherit root's level and get noisy otherwise.

Source code in src\muvis_align\logging.py
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
def init_logging(log_filename='log/muvis-align.log', log_format='%(asctime)s %(levelname)s: %(message)s',
                 verbose=False, debug=False):
    """verbose: muvis-align's own logging (echoed to console, not just the log file).
    debug: external libraries (multiview_stitcher, zarr) at DEBUG level - off by default,
    since unlisted loggers also inherit root's level and get noisy otherwise.
    """
    basepath = os.path.dirname(log_filename)
    if basepath and not os.path.exists(basepath):
        os.makedirs(basepath, exist_ok=True)

    handlers = [logging.FileHandler(log_filename, encoding='utf-8')]
    if verbose:
        handlers += [logging.StreamHandler()]
    logging.basicConfig(level=logging.DEBUG if debug else logging.INFO,
                        format=log_format, handlers=handlers, encoding='utf-8', force=True)

    if debug:
        mvsr_logger = logging.getLogger('multiview_stitcher.registration')
        mvsr_logger.setLevel(logging.DEBUG)
        if len(mvsr_logger.handlers) == 0:
            mvsr_logger.addHandler(logging.StreamHandler())
        for module in ['multiview_stitcher', 'multiview_stitcher.fusion', 'ome_zarr', 'zarr']:
            logging.getLogger(module).setLevel(logging.DEBUG)
    else:
        for module in ['multiview_stitcher', 'multiview_stitcher.registration', 'multiview_stitcher.fusion',
                       'ome_zarr', 'zarr']:
            logging.getLogger(module).setLevel(logging.WARNING)

    enable_fault_log(log_filename)

    logging.info(f'muvis-align version {version}')
    logging.info(f'Multiview-stitcher version: {mvs_version}')

metrics

calc_global_metrics(msims, reg_transform_key, metric_methods, reg_results, n_parallel_pairs=None, progress_factory=None)

Each registered pair's metrics over its overlap under the registered transforms, with its pair quality.

Source code in src\muvis_align\metrics.py
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
def calc_global_metrics(msims, reg_transform_key, metric_methods, reg_results, n_parallel_pairs=None,
                        progress_factory=None):
    """Each registered pair's metrics over its overlap under the registered transforms, with its pair quality."""
    query_transform_keys = [reg_transform_key]
    # only the registered pairs: over all msims the overlap mode would measure every overlapping pair
    workers = n_parallel_pairs or default_pair_workers
    edges = sorted({tuple(sorted(edge)) for edge in reg_results['pairwise_registration']['graph'].edges()})

    def pair_arguments(edge):
        # the registered key as base too: the overlap measured is where the pair lies once registered
        return [msims[node] for node in edge], edge, metric_methods, reg_transform_key, query_transform_keys

    if metric_methods:
        metric_results = merge_metric_results(
            map_pair_metrics(msims, edges, pair_arguments, workers, progress_factory, desc='Global metrics',
                             threaded=False))
    else:
        metric_results = unmeasured_pair_results(edges, query_transform_keys)

    # the summary as the plain mean over the registered pairs
    for candidate_key, metric_values in metric_results['summary'].items():
        for metric_key in list(metric_values):
            values = [value[candidate_key][metric_key] for value in metric_results['pairs'].values()
                     if candidate_key in value and value[candidate_key].get(metric_key) is not None
                     and not np.isnan(value[candidate_key][metric_key])]
            metric_values[metric_key] = float(np.mean(values)) if values else None

    qualities = reg_results['pairwise_registration']['metrics']['qualities']
    # a pair_key may be reversed, or missing from metric_results['pairs']: match by unordered identity
    pairs_by_unordered_key = {frozenset(key): key for key in metric_results['pairs']}

    quality_values = []
    for pair_key, value in qualities.items():
        value = quality_to_scalar(value)
        if value:
            actual_key = pairs_by_unordered_key.get(frozenset(pair_key), pair_key)
            metric_results['pairs'].setdefault(actual_key, {})
            metric_results['pairs'][actual_key].setdefault(reg_transform_key, {})[default_quality_key] = value
            quality_values.append(value)

    metric_results['summary'][reg_transform_key][default_quality_key] = float(np.nanmean(quality_values))

    return metric_results

calc_match_metrics(points1, points2, transform, threshold, lowe_ratio=None)

Source code in src\muvis_align\metrics.py
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
def calc_match_metrics(points1, points2, transform, threshold, lowe_ratio=None):
    metrics = {}
    final_matches = []
    inliers = []
    transformed_points1 = apply_transform(points1, transform)
    npoints1, npoints2 = len(points1), len(points2)
    npoints = min(npoints1, npoints2)
    if npoints1 == 0 or npoints2 == 0:
        return metrics

    swapped = (npoints1 > npoints2)
    if swapped:
        points1, points2 = points2, points1

    distance_matrix = euclidean_distances(transformed_points1, points2)
    matching_distances = np.diag(distance_matrix)
    if npoints1 == npoints2 and np.mean(matching_distances < threshold) > 0.5:
        # already matching points lists
        nmatches = np.sum(matching_distances < threshold)
    else:
        matches = []
        distances0 = []
        for rowi, row in enumerate(distance_matrix):
            sorted_indices = np.argsort(row)
            index0 = sorted_indices[0]
            distance0 = row[index0]
            matches.append((rowi, sorted_indices))
            distances0.append(distance0)
        sorted_matches = np.argsort(distances0)

        done = []
        nmatches = 0
        matching_distances = []
        for sorted_match in sorted_matches:
            i, match = matches[sorted_match]
            for j_index, j in enumerate(match):
                if j not in done:
                    # found best, available match
                    distance0 = distance_matrix[i, j]
                    # second best match distance
                    distance1 = distance_matrix[i, match[j_index + 1]] if j_index + 1 < len(match) else np.inf
                    matching_distances.append(distance0)    # use all distances to also weigh in the non-matches
                    final_matches.append((int(i), int(j)))
                    if distance0 < threshold and (lowe_ratio is None or distance0 < lowe_ratio * distance1):
                        done.append(j)
                        nmatches += 1
                        inliers.append(True)
                    else:
                        inliers.append(False)
                    break

    metrics['matches'] = final_matches
    metrics['inliers'] = inliers
    metrics['nmatches'] = nmatches
    metrics['match_rate'] = nmatches / npoints if npoints > 0 else 0
    distance = np.mean(matching_distances) if nmatches > 0 else np.inf
    metrics['distance'] = float(distance)
    metrics['norm_distance'] = float(distance / threshold)
    return metrics

calc_msims_metrics(msims, pair_transforms, qualities=None, base_transform_key=None, metric_methods='all', reg_channel=None, n_parallel_pairs=None)

Source code in src\muvis_align\metrics.py
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
def calc_msims_metrics(msims, pair_transforms, qualities=None, base_transform_key=None, metric_methods='all',
                       reg_channel=None, n_parallel_pairs=None):
    if base_transform_key is None:
        base_transform_key = next(iter(get_msim_transform_keys(msims[0])))
    with dask.config.set(scheduler=_scheduler('single-threaded')):
        pairs_graph = mv_graph.build_view_adjacency_graph_from_msims(
            msims,
            transform_key=base_transform_key,
            pairs=list(pair_transforms.keys())
        )
    nx.set_edge_attributes(pairs_graph, pair_transforms, default_transform_key)
    if qualities:
        nx.set_edge_attributes(pairs_graph, qualities, default_quality_key)
    return calc_pair_metrics(msims=msims, pairs_graph=pairs_graph, base_transform_key=base_transform_key,
                             metric_methods=metric_methods, reg_channel=reg_channel, n_parallel_pairs=n_parallel_pairs)

calc_ncc(image1, image2)

Source code in src\muvis_align\metrics.py
349
350
351
352
353
354
355
356
357
def calc_ncc(image1, image2):
    max_size = np.flip(np.max([image1.shape, image2.shape], 0))
    image1 = image_reshape(image1, max_size)
    image2 = image_reshape(image2, max_size)

    normimage1 = np.array(image1 - np.mean(image1))
    normimage2 = np.array(image2 - np.mean(image2))
    ncc = np.sum(normimage1 * normimage2) / (np.linalg.norm(normimage1) * np.linalg.norm(normimage2))
    return float(ncc)

calc_pair_metrics(msims, pairs_graph, metric_methods, base_transform_key, reg_channel=None, n_parallel_pairs=None, progress_factory=None)

Source code in src\muvis_align\metrics.py
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
def calc_pair_metrics(msims, pairs_graph, metric_methods, base_transform_key, reg_channel=None,
                      n_parallel_pairs=None, progress_factory=None):
    workers = n_parallel_pairs or default_pair_workers

    def pair_arguments(edge):
        # renumbered in order, so the lower index stays the fixed one
        nodes = tuple(sorted(edge))
        graph = nx.relabel_nodes(pairs_graph.edge_subgraph([edge]).copy(),
                                 {node: index for index, node in enumerate(nodes)})
        return [msims[node] for node in nodes], nodes, metric_methods, base_transform_key, None, graph

    if metric_methods:
        metric_results = merge_metric_results(
            map_pair_metrics(msims, list(pairs_graph.edges), pair_arguments, workers, progress_factory))
    else:
        metric_results = unmeasured_pair_results(pairs_graph.edges, [default_transform_key])

    qualities = nx.get_edge_attributes(pairs_graph, default_quality_key)

    quality_values = []
    for pair_key, value in qualities.items():
        value = quality_to_scalar(value)
        if value:
            metric_results['pairs'][pair_key][default_transform_key][default_quality_key] = value
            quality_values.append(value)

    value = float(np.nanmean(quality_values)) if quality_values else None
    metric_results['summary'][default_transform_key][default_quality_key] = value

    return metric_results

calc_ssim(image1, image2)

Source code in src\muvis_align\metrics.py
361
362
363
364
365
366
367
368
369
370
371
def calc_ssim(image1, image2):
    dtype = image1.dtype
    maxval = 2 ** (8 * dtype.itemsize) - 1
    max_size = np.flip(np.max([image1.shape, image2.shape], 0))
    image1 = image_reshape(image1, max_size)
    image2 = image_reshape(image2, max_size)
    try:
        ssim = structural_similarity(np.array(image1), np.array(image2), data_range=maxval)
    except ValueError:
        ssim = np.nan
    return float(ssim)

create_metric_methods(metric_methods, msim)

Source code in src\muvis_align\metrics.py
37
38
39
40
41
42
43
44
45
46
47
48
49
def create_metric_methods(metric_methods, msim):
    data_range = np.iinfo(msim["scale0/image"].dtype).max
    all_metric_funcs = {
        'ncc': multiview_stitcher.metrics.normalized_cross_correlation,
        'ssim': partial(ssim_metric, data_range=data_range),
        'onmi': lambda im1, im2: normalized_mutual_information(np.nan_to_num(im1), np.nan_to_num(im2)) - 1,
        'mse': lambda im1, im2: 1 / mean_squared_error(im1, im2),
    }
    if metric_methods == 'all':
        metric_funcs = all_metric_funcs
    else:
        metric_funcs = {metric_method: all_metric_funcs[metric_method] for metric_method in metric_methods}
    return metric_funcs

map_pair_metrics(msims, edges, pair_arguments, workers, progress_factory=None, desc='Pair metrics', threaded=True)

pair_image_metrics for each edge (a sorted node pair), in worker processes when the arguments pickle, else threads; results in completion order. threaded False runs without processes in the calling thread, as linprog (HiGHS) fails when called from changing threads.

Source code in src\muvis_align\metrics.py
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
def map_pair_metrics(msims, edges, pair_arguments, workers, progress_factory=None, desc='Pair metrics',
                     threaded=True):
    """pair_image_metrics for each edge (a sorted node pair), in worker processes when the arguments pickle, else
    threads; results in completion order. `threaded` False runs without processes in the calling thread, as
    linprog (HiGHS) fails when called from changing threads."""
    pool = None
    if workers > 1 and len(edges) > workers and picklable(pair_arguments(edges[0])):
        pool = worker_process_pool(workers, max_tasks_per_child=default_pair_worker_tasks)
    elif not threaded:
        workers = 1

    def pair_metrics(edge):
        arguments = pair_arguments(edge)
        if pool is None:
            return pair_image_metrics(*arguments)
        return result_unless_cancelled(pool.submit(pair_image_metrics, *arguments, scheduler='synchronous'))

    results = []
    progress = progress_phase(progress_factory, total=len(edges), desc=desc)
    # set once: BLAS thread limits changed from many threads at once crash the process
    compute = (_pair_metrics_compute('synchronous' if workers > 1 else 'threads') if pool is None
               else nullcontext())
    completed = (rolling_map(pair_metrics, edges, workers) if pool is not None or threaded
                 else ((edge, pair_metrics(edge)) for edge in edges))
    with (pool if pool is not None else nullcontext()), compute, progress as pbar:
        for count, (edge, result) in enumerate(completed, start=1):
            results.append(result)
            if count % (2 * workers) == 0:
                release_memory(generation=1)
            if pbar is not None:
                pbar.update(1)
    return results

merge_metric_results(results)

Merge tile_pair_image_metrics() results over disjoint pairs; the summary weights pairs by bbox area, standing in for the unreturned overlap area.

Source code in src\muvis_align\metrics.py
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
def merge_metric_results(results):
    """Merge tile_pair_image_metrics() results over disjoint pairs; the summary weights pairs by bbox area,
    standing in for the unreturned overlap area."""
    merged = {'pairs': {}, 'bboxes': {}, 'summary': {}}
    for result in results:
        merged['pairs'].update(result.get('pairs', {}))
        merged['bboxes'].update(result.get('bboxes', {}))
        for candidate_key, metric_values in result.get('summary', {}).items():
            # kept even with no metrics asked for: the caller adds the quality under it
            summary = merged['summary'].setdefault(candidate_key, {})
            for metric_key in metric_values:
                summary[metric_key] = np.nan
    for candidate_key, metric_values in merged['summary'].items():
        for metric_key in metric_values:
            values_and_weights = [(float(pair[candidate_key][metric_key]), _bbox_area(merged['bboxes'].get(pair_key)))
                                  for pair_key, pair in merged['pairs'].items()
                                  if metric_key in pair.get(candidate_key, {})]
            valid = [(value, weight) for value, weight in values_and_weights if not np.isnan(value) and weight > 0]
            if valid:
                metric_values[metric_key] = float(sum(value * weight for value, weight in valid)
                                                  / sum(weight for _, weight in valid))
    return merged

pair_image_metrics(msims, nodes, metric_methods, base_transform_key, query_transform_keys=None, pairs_graph=None, scheduler=None)

tile_pair_image_metrics for one pair's two msims, keys renumbered back to nodes; module-level for worker processes. scheduler None: the caller holds _pair_metrics_compute.

Source code in src\muvis_align\metrics.py
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
def pair_image_metrics(msims, nodes, metric_methods, base_transform_key, query_transform_keys=None,
                       pairs_graph=None, scheduler=None):
    """tile_pair_image_metrics for one pair's two msims, keys renumbered back to `nodes`; module-level for worker
    processes. `scheduler` None: the caller holds _pair_metrics_compute."""
    metric_funcs = create_metric_methods(metric_methods, msims[0])
    with (_pair_metrics_compute(scheduler) if scheduler is not None else nullcontext()):
        if pairs_graph is not None:
            result = multiview_stitcher.metrics.tile_pair_image_metrics(
                msims, base_transform_key=base_transform_key, pairs_graph=pairs_graph, metric_funcs=metric_funcs)
        else:
            result = multiview_stitcher.metrics.tile_pair_image_metrics(
                msims, base_transform_key=base_transform_key, query_transform_keys=query_transform_keys,
                metric_funcs=metric_funcs)
    for key in ('pairs', 'bboxes'):
        result[key] = {(nodes[fixed], nodes[moving]): value for (fixed, moving), value in result.get(key, {}).items()}
    return result

quality_to_scalar(value)

Source code in src\muvis_align\metrics.py
52
53
54
55
56
57
58
def quality_to_scalar(value):
    # quality values are often an xr.DataArray with a 't' dim
    if isinstance(value, DataArray):
        if 't' in value.dims:
            value = value.sel(t=0)
        value = value.item()
    return value

remap_pair_metrics(metrics, indices)

Remap metrics['pairs']' local (0, 1, ...) keys to the real, global source indices.

Source code in src\muvis_align\metrics.py
159
160
161
162
163
def remap_pair_metrics(metrics, indices):
    """Remap metrics['pairs']' local (0, 1, ...) keys to the real, global source indices."""
    pairs = metrics.get('pairs')
    if pairs:
        metrics['pairs'] = {tuple(int(indices[index]) for index in key): value for key, value in pairs.items()}

ssim_metric(image1, image2, data_range)

SSIM of two single-channel crops, its window narrowed to a thin overlap; NaN (left out of summaries) below 3 pixels.

Source code in src\muvis_align\metrics.py
26
27
28
29
30
31
32
33
34
def ssim_metric(image1, image2, data_range):
    """SSIM of two single-channel crops, its window narrowed to a thin overlap; NaN (left out of summaries) below 3
    pixels."""
    # the largest odd window up to skimage's default 7
    win_size = min(7, (min(np.shape(image1)) - 1) // 2 * 2 + 1)
    if win_size < 3:
        return np.nan
    return structural_similarity(np.nan_to_num(image1), np.nan_to_num(image2), data_range=data_range,
                                 win_size=win_size)

unmeasured_pair_results(edges, candidate_keys)

merge_metric_results()' result when no image metric is asked for, without computing any pair's overlap images: an empty entry per candidate key for each pair and the summary, for the quality to be added to.

Source code in src\muvis_align\metrics.py
152
153
154
155
156
def unmeasured_pair_results(edges, candidate_keys):
    """merge_metric_results()' result when no image metric is asked for, without computing any pair's overlap images:
    an empty entry per candidate key for each pair and the summary, for the quality to be added to."""
    return {'pairs': {tuple(sorted(edge)): {key: {} for key in candidate_keys} for edge in edges},
            'bboxes': {}, 'summary': {key: {} for key in candidate_keys}}

robust_resolution

A robust groupwise resolution method: multiview_stitcher's linear_two_pass as the solver of an iteratively reweighted least squares. See notes/multiview_stitcher.md.

ROBUST_LINEAR = 'robust_linear' module-attribute

default_robust_change_quantile = 0.99 module-attribute

default_robust_rounds = 10 module-attribute

default_robust_tolerance = 0.01 module-attribute

find_reference_view(graph, weight_key='quality')

The node with the largest summed edge weight, as mv_graph.get_node_with_maximal_edge_weight_sum_from_graph picks it, in one pass over the edges rather than one per node.

Source code in src\muvis_align\robust_resolution.py
33
34
35
36
37
38
39
40
41
42
def find_reference_view(graph, weight_key='quality'):
    """The node with the largest summed edge weight, as mv_graph.get_node_with_maximal_edge_weight_sum_from_graph
    picks it, in one pass over the edges rather than one per node."""
    totals = dict.fromkeys(graph.nodes, 0.0)
    for node1, node2, data in graph.edges(data=True):
        weight = data[weight_key]
        weight = float(np.sum(weight.data if isinstance(weight, xr.DataArray) else weight))
        totals[node1] += weight
        totals[node2] += weight
    return max(totals, key=totals.get)

groupwise_resolution_robust_linear(g_reg_component_tp, reference_view=None, transform='rigid', scale=None, rounds=default_robust_rounds, tolerance=default_robust_tolerance, change_quantile=default_robust_change_quantile, progress=None, **kwargs)

linear_two_pass re-solved up to rounds times, each edge weighted by its pair quality times a Cauchy weight of its last residual, 1 / (1 + (residual / scale)^2): bad pairs stop pulling the fit without being cut. Stops early once the change_quantile of the residuals' changes in a round is within tolerance * scale, as the weights then hardly change. scale defaults to half the voxel diagonal. Returns the round with the lowest median residual. progress is called after each round. Per connected component and timepoint, as any resolver.

Source code in src\muvis_align\robust_resolution.py
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
def groupwise_resolution_robust_linear(g_reg_component_tp, reference_view=None, transform='rigid', scale=None,
                                       rounds=default_robust_rounds, tolerance=default_robust_tolerance,
                                       change_quantile=default_robust_change_quantile, progress=None, **kwargs):
    """linear_two_pass re-solved up to `rounds` times, each edge weighted by its pair quality times a Cauchy weight
    of its last residual, 1 / (1 + (residual / scale)^2): bad pairs stop pulling the fit without being cut.
    Stops early once the `change_quantile` of the residuals' changes in a round is within `tolerance * scale`, as
    the weights then hardly change. `scale` defaults to half the voxel diagonal. Returns the round with the lowest
    median residual. `progress` is called after each round. Per connected component and timepoint, as any resolver."""
    graph = g_reg_component_tp.copy()
    if reference_view is None or reference_view not in graph:
        reference_view = find_reference_view(graph)
    if scale is None:
        scale = 0.5 * voxel_diagonal(graph)
    edges = [tuple(sorted(edge)) for edge in graph.edges]
    qualities = {edge: scalar(graph.edges[edge].get('quality')) for edge in edges}
    qualities = {edge: quality if np.isfinite(quality) else 0.0 for edge, quality in qualities.items()}
    weights = dict.fromkeys(edges, 1.0)
    params, info, last_residuals = None, None, None
    # the reweighting can swing between a good and a bad fit, so the last round is not necessarily the best
    best_params, best_info, best_residual = None, None, np.inf
    for round_index in range(rounds):
        raise_if_cancelled()
        for edge in edges:
            graph.edges[edge]['quality'] = qualities[edge] * weights[edge]
        round_start, round_cpu = time.perf_counter(), time.process_time()
        # pruning off: every edge stays in, down-weighted instead. One BLAS thread: a round is many tiny numpy calls,
        # which a pool spread over every core only slows, its threads spinning
        with _numpy_internals(), threadpool_limits(limits=1):
            params, info = groupwise_resolution_linear_two_pass(
                graph, reference_view=reference_view, transform=transform, weight_mode='quality',
                residual_threshold=np.inf, keep_mst=False, **kwargs)
        metrics = info['metrics']
        if metrics is None:
            break
        residuals = {tuple(sorted((node1, node2))): residual
                     for node1, node2, residual in zip(metrics['u'], metrics['v'], metrics['residual'])}
        weights = {edge: 1.0 / (1.0 + (residuals[edge] / scale) ** 2) if np.isfinite(residuals[edge]) else 0.0
                   for edge in edges}
        changes = residual_changes(last_residuals, residuals)
        last_residuals = residuals
        change = np.quantile(changes, change_quantile) if len(changes) else np.inf
        largest_change = max(changes, default=np.inf)
        median_residual = float(np.median(list(residuals.values())))
        if median_residual < best_residual:
            best_params, best_info, best_residual = params, info, median_residual
        logging.info(f'Robust linear resolution: round {round_index + 1}/{rounds}, {len(edges)} edges,'
                     f' median residual {median_residual:.3g},'
                     f' change {change_quantile:.0%} {change:.3g} (largest {largest_change:.3g},'
                     f' tolerance {tolerance * scale:.3g}) in {time.perf_counter() - round_start:.0f}s,'
                     f' {time.process_time() - round_cpu:.0f}s cpu')
        if progress is not None:
            progress()
        if change <= tolerance * scale:
            logging.info(f'Robust linear resolution: converged after {round_index + 1} rounds')
            break
    if best_params is None:
        return params, info
    return best_params, best_info

residual_changes(last_residuals, residuals)

Source code in src\muvis_align\robust_resolution.py
172
173
174
175
176
def residual_changes(last_residuals, residuals):
    if last_residuals is None:
        return []
    return [abs(residuals[edge] - last_residuals[edge]) for edge in residuals
            if np.isfinite(residuals[edge]) and np.isfinite(last_residuals[edge])]

scalar(value, default=1.0)

Source code in src\muvis_align\robust_resolution.py
25
26
27
28
29
30
def scalar(value, default=1.0):
    if value is None:
        return default
    if isinstance(value, xr.DataArray):
        value = value.data
    return float(np.mean(value))

voxel_diagonal(graph)

global_optimization's default abs_tol: the largest voxel diagonal among the views.

Source code in src\muvis_align\robust_resolution.py
45
46
47
48
def voxel_diagonal(graph):
    """global_optimization's default abs_tol: the largest voxel diagonal among the views."""
    return max(float(np.sqrt(sum(value ** 2 for value in graph.nodes[node]['stack_props']['spacing'].values())))
               for node in graph.nodes)

split_registration

Split registration ('split' pairing): each z-plane's (or channel's) tiles registered in x/y first - the normal pair and global registration, with pairs only within a group - then the stitched groups against each other as whole images, consecutive groups paired, as registration_dimension says: stacked planes or overlaid channels.

GROUP_KEY = 'split_group' module-attribute

SPLIT = 'split' module-attribute

STAGE1_KEY = 'split_stage1' module-attribute

channel_groups(sources, input_groups=None)

Each source's channel index: the input path item it came from when there are several, else its metadata channel label in order of appearance, else (all labels alike) its own index.

Source code in src\muvis_align\split_registration.py
37
38
39
40
41
42
43
44
45
46
47
48
49
def channel_groups(sources, input_groups=None):
    """Each source's channel index: the input path item it came from when there are several, else its metadata
    channel label in order of appearance, else (all labels alike) its own index."""
    if input_groups is not None and len(set(input_groups)) > 1:
        # in the items' order, not their files'
        keys = list(input_groups)
        order = sorted(set(keys))
    else:
        keys = [(source.get_channels() or [{}])[0].get('label') or '' for source in sources]
        if len(set(keys)) <= 1:
            keys = list(range(len(sources)))
        order = list(dict.fromkeys(keys))
    return [order.index(key) for key in keys]

fuse_group(msims, transform_key, grid, tile_size, metrics_binning=None)

One group's tiles fused under transform_key onto grid and smoothed, as an msim placed by GROUP_KEY, and given metrics_binning its metrics_group_msim (else None).

Source code in src\muvis_align\split_registration.py
130
131
132
133
134
135
136
137
138
def fuse_group(msims, transform_key, grid, tile_size, metrics_binning=None):
    """One group's tiles fused under `transform_key` onto `grid` and smoothed, as an msim placed by GROUP_KEY, and
    given `metrics_binning` its metrics_group_msim (else None)."""
    spacing = grid['spacing']['y']
    sims = [_level_sim(msim, spacing) for msim in msims]
    with dask.config.set(scheduler='threads'):
        fused = np.asarray(fusion.fuse(sims, transform_key=transform_key, output_stack_properties=grid).data).squeeze()
    metrics_msim = metrics_group_msim(fused, grid, metrics_binning) if metrics_binning else None
    return group_msim(smooth_group(fused, spacing, tile_size), spacing, grid['origin']), metrics_msim

group_grid(msims, transform_key)

The one y/x grid every group is fused onto - the union of all tiles at their finest (pre-processed) pixel size, with a margin - and the median tile size. On one grid two groups overlap in the whole frame, not a crop cut at an outline.

Source code in src\muvis_align\split_registration.py
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
def group_grid(msims, transform_key):
    """The one y/x grid every group is fused onto - the union of all tiles at their finest (pre-processed) pixel
    size, with a margin - and the median tile size. On one grid two groups overlap in the whole frame, not a crop
    cut at an outline."""
    finest = [msi_utils.get_sim_from_msim(msim, scale='scale0') for msim in msims]
    stack_props = [si_utils.get_stack_properties_from_sim(sim, transform_key=transform_key) for sim in finest]
    lower = np.min([[props['origin'][dim] for dim in 'yx'] for props in stack_props], axis=0)
    upper = np.max([[props['origin'][dim] + props['shape'][dim] * props['spacing'][dim] for dim in 'yx']
                    for props in stack_props], axis=0)
    spacing = min(min(props['spacing'][dim] for dim in 'yx') for props in stack_props)
    shape = np.ceil((upper - lower) / spacing).astype(int)
    margin = np.ceil(shape * split_grid_margin).astype(int)
    lower, shape = lower - margin * spacing, shape + 2 * margin
    grid = {'origin': {dim: float(value) for dim, value in zip('yx', lower)},
            'spacing': {dim: spacing for dim in 'yx'},
            'shape': {dim: int(value) for dim, value in zip('yx', shape)}}
    tile_size = float(np.median([max(props['shape'][dim] * props['spacing'][dim] for dim in 'yx')
                                 for props in stack_props]))
    return grid, tile_size

group_label(labels, separators='_- ')

The labels' common prefix up to a separator: 'S000' for 'S000_000_000' ... 'S000_007_007' (not 'S000_00').

Source code in src\muvis_align\split_registration.py
52
53
54
55
56
57
58
def group_label(labels, separators='_- '):
    """The labels' common prefix up to a separator: 'S000' for 'S000_000_000' ... 'S000_007_007' (not 'S000_00')."""
    prefix = os.path.commonprefix(list(labels))
    if any(len(label) > len(prefix) and label[len(prefix)] not in separators for label in labels):
        cut = max(prefix.rfind(separator) for separator in separators)
        prefix = prefix[:cut + 1] if cut >= 0 else ''
    return prefix.rstrip(separators)

group_msim(data, spacing, origin)

A fused group's y/x image as a single-level msim placed by GROUP_KEY.

Source code in src\muvis_align\split_registration.py
103
104
105
106
107
108
109
def group_msim(data, spacing, origin):
    """A fused group's y/x image as a single-level msim placed by GROUP_KEY."""
    sim = si_utils.get_sim_from_array(data, dims=['y', 'x'], scale={dim: spacing for dim in 'yx'},
                                      translation=origin, transform_key=GROUP_KEY)
    msim = msi_utils.get_msim_from_sim(sim, scale_factors=[])
    # the only channel selected, so registration sees spatial dims alone
    return msi_utils.multiscale_sel_coords(msim, {'c': sim.coords['c'].values[0]})

group_pair_image_metrics(metrics_msims, pair_graph, index1, metric_funcs)

The image metrics of groups index1 and index1 + 1 under their registered pair transform.

Source code in src\muvis_align\split_registration.py
141
142
143
144
145
146
147
148
149
150
def group_pair_image_metrics(metrics_msims, pair_graph, index1, metric_funcs):
    """The image metrics of groups `index1` and `index1 + 1` under their registered pair transform."""
    if not metric_funcs or any(msim is None for msim in metrics_msims) or not pair_graph.number_of_edges():
        return {}
    graph = nx.relabel_nodes(pair_graph, {index1: 0, index1 + 1: 1})
    with dask.config.set({'scheduler': 'threads', 'optimization.fuse.active': False}):
        result = tile_pair_image_metrics(metrics_msims, base_transform_key=GROUP_KEY, pairs_graph=graph,
                                         metric_funcs=metric_funcs)
    values = result.get('pairs', {}).get((0, 1), {}).get(default_transform_key, {})
    return {metric: float(value) for metric, value in values.items()}

metrics_group_msim(data, grid, binning)

The unsmoothed fused group binned and cropped to its tiles, NaN between them as outside a tile's image, for the pair metrics to measure only where both groups have tiles; None if it has none.

Source code in src\muvis_align\split_registration.py
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
def metrics_group_msim(data, grid, binning):
    """The unsmoothed fused group binned and cropped to its tiles, NaN between them as outside a tile's image, for
    the pair metrics to measure only where both groups have tiles; None if it has none."""
    shape = np.array(data.shape) // binning
    binned = (np.asarray(data[:shape[0] * binning, :shape[1] * binning], dtype=np.float32)
              .reshape(shape[0], binning, shape[1], binning).mean(axis=(1, 3)))
    mask = binned > 0
    if not np.any(mask):
        return None
    rows, cols = np.flatnonzero(mask.any(axis=1)), np.flatnonzero(mask.any(axis=0))
    binned = np.where(mask, binned, np.nan)[rows[0]:rows[-1] + 1, cols[0]:cols[-1] + 1]
    spacing = grid['spacing']['y']
    # a binned pixel's centre is the mean of its pixels' centres
    origin = {dim: grid['origin'][dim] + (start * binning + (binning - 1) / 2) * spacing
              for dim, start in zip('yx', (rows[0], cols[0]))}
    return group_msim(binned, spacing * binning, origin)

register_group_pair(msim1, msim2, index1, pairwise_reg_func, pairwise_reg_func_kwargs=None, binning=default_split_binning)

Groups index1 and index1 + 1 as a two-node pair graph, their edge registered by pairwise_reg_func with their images binned by binning.

Source code in src\muvis_align\split_registration.py
153
154
155
156
157
158
159
160
161
162
163
164
165
def register_group_pair(msim1, msim2, index1, pairwise_reg_func, pairwise_reg_func_kwargs=None,
                        binning=default_split_binning):
    """Groups `index1` and `index1 + 1` as a two-node pair graph, their edge registered by `pairwise_reg_func` with
    their images binned by `binning`."""
    with dask.config.set({'scheduler': 'threads', 'optimization.fuse.active': False}):
        graph = build_view_adjacency_graph([msim1, msim2], GROUP_KEY, [(0, 1)], overlap_tolerance=0)
        if graph.number_of_edges():
            graph = compute_pairwise_registrations([msim1, msim2], graph, transform_key=GROUP_KEY,
                                                   pairwise_reg_func=pairwise_reg_func,
                                                   pairwise_reg_func_kwargs=pairwise_reg_func_kwargs,
                                                   registration_binning={dim: binning for dim in 'yx'},
                                                   n_parallel_pairwise_regs=1)
    return nx.relabel_nodes(graph, {0: index1, 1: index1 + 1})

register_groups(msims, transforms, groups, base_transform_key, pairwise_reg_func, pairwise_reg_func_kwargs=None, resolution_method='robust_linear', resolution_kwargs=None, binning=default_split_binning, metric_funcs=None)

Stage 2: transforms (stage 1's, relative to base_transform_key) with each group's own correction composed on - from its fused image registered against the next group's and resolved over all groups - and the graph of those group pairs, each edge with its metric_funcs values under 'metrics'.

Source code in src\muvis_align\split_registration.py
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
def register_groups(msims, transforms, groups, base_transform_key, pairwise_reg_func, pairwise_reg_func_kwargs=None,
                    resolution_method='robust_linear', resolution_kwargs=None, binning=default_split_binning,
                    metric_funcs=None):
    """Stage 2: `transforms` (stage 1's, relative to `base_transform_key`) with each group's own correction
    composed on - from its fused image registered against the next group's and resolved over all groups - and the
    graph of those group pairs, each edge with its `metric_funcs` values under 'metrics'."""
    ngroups = max(groups) + 1
    if ngroups < 2:
        return transforms, nx.Graph()
    snapshot = snapshot_msims_transform(msims, STAGE1_KEY)
    graph = nx.Graph()
    try:
        for msim, transform in zip(msims, transforms):
            set_msim_affine(msim, transform, transform_key=STAGE1_KEY, base_transform_key=base_transform_key)
        grid, tile_size = group_grid(msims, STAGE1_KEY)
        # two groups held at a time: all of them may not fit in memory
        previous, previous_metrics = None, None
        for index in range(ngroups):
            raise_if_cancelled()
            current, current_metrics = fuse_group([msim for msim, group in zip(msims, groups) if group == index],
                                                  STAGE1_KEY, grid, tile_size,
                                                  metrics_binning=binning if metric_funcs else None)
            if previous is not None:
                pair_graph = register_group_pair(previous, current, index - 1, pairwise_reg_func,
                                                 pairwise_reg_func_kwargs, binning)
                if pair_graph.number_of_edges():
                    pair_graph.edges[index - 1, index]['metrics'] = group_pair_image_metrics(
                        [previous_metrics, current_metrics], pair_graph, index - 1, metric_funcs)
                graph = nx.compose(graph, pair_graph)
            previous, previous_metrics = current, current_metrics
    finally:
        restore_msims_transform(msims, STAGE1_KEY, snapshot)

    logging.info(f'Split registration: {ngroups} groups, {graph.number_of_edges()} group pairs registered')
    group_params, _ = groupwise_resolution(graph, method=resolution_method, **(resolution_kwargs or {}))
    # each group's correction acts in world space, after the stage-1 transform
    return ([param_utils.matmul_xparams(group_params[group], transform) for transform, group in zip(transforms, groups)],
            graph)

smooth_group(data, spacing, tile_size)

The fused group smoothed past what every tile repeats, its background (outside its tiles) filled with its mean: an empty background, the same in every group, would match best at zero shift.

Source code in src\muvis_align\split_registration.py
71
72
73
74
75
76
77
78
79
def smooth_group(data, spacing, tile_size):
    """The fused group smoothed past what every tile repeats, its background (outside its tiles) filled with its
    mean: an empty background, the same in every group, would match best at zero shift."""
    data = np.asarray(data, dtype=np.float32)
    mask = data > 0
    if not np.any(mask):
        return data
    filled = np.where(mask, data, data[mask].mean())
    return gaussian_filter(filled, tile_size * split_smoothing / spacing)

split_groups(positions, sources, dimension=None, input_groups=None)

Each source's group index: its channel (see channel_groups) for registration_dimension 'c', else its z-plane.

Source code in src\muvis_align\split_registration.py
28
29
30
31
32
33
34
def split_groups(positions, sources, dimension=None, input_groups=None):
    """Each source's group index: its channel (see channel_groups) for registration_dimension 'c', else its z-plane."""
    if dimension == 'c':
        return channel_groups(sources, input_groups)
    keys = [round(float(position.get('z', 0.0)), 9) for position in positions]
    order = sorted(set(keys))
    return [order.index(key) for key in keys]

within_group_pairs(pairs, groups)

Source code in src\muvis_align\split_registration.py
61
62
def within_group_pairs(pairs, groups):
    return [pair for pair in pairs if groups[pair[0]] == groups[pair[1]]]

ui

Interface

source_metadata_params = ['source_position_z', 'source_position_y', 'source_position_x', 'source_scale_z', 'source_scale_y', 'source_scale_x', 'source_rotation', 'normalise_rotated_positions'] module-attribute
Interface
Source code in src\muvis_align\ui\Interface.py
 100
 101
 102
 103
 104
 105
 106
 107
 108
 109
 110
 111
 112
 113
 114
 115
 116
 117
 118
 119
 120
 121
 122
 123
 124
 125
 126
 127
 128
 129
 130
 131
 132
 133
 134
 135
 136
 137
 138
 139
 140
 141
 142
 143
 144
 145
 146
 147
 148
 149
 150
 151
 152
 153
 154
 155
 156
 157
 158
 159
 160
 161
 162
 163
 164
 165
 166
 167
 168
 169
 170
 171
 172
 173
 174
 175
 176
 177
 178
 179
 180
 181
 182
 183
 184
 185
 186
 187
 188
 189
 190
 191
 192
 193
 194
 195
 196
 197
 198
 199
 200
 201
 202
 203
 204
 205
 206
 207
 208
 209
 210
 211
 212
 213
 214
 215
 216
 217
 218
 219
 220
 221
 222
 223
 224
 225
 226
 227
 228
 229
 230
 231
 232
 233
 234
 235
 236
 237
 238
 239
 240
 241
 242
 243
 244
 245
 246
 247
 248
 249
 250
 251
 252
 253
 254
 255
 256
 257
 258
 259
 260
 261
 262
 263
 264
 265
 266
 267
 268
 269
 270
 271
 272
 273
 274
 275
 276
 277
 278
 279
 280
 281
 282
 283
 284
 285
 286
 287
 288
 289
 290
 291
 292
 293
 294
 295
 296
 297
 298
 299
 300
 301
 302
 303
 304
 305
 306
 307
 308
 309
 310
 311
 312
 313
 314
 315
 316
 317
 318
 319
 320
 321
 322
 323
 324
 325
 326
 327
 328
 329
 330
 331
 332
 333
 334
 335
 336
 337
 338
 339
 340
 341
 342
 343
 344
 345
 346
 347
 348
 349
 350
 351
 352
 353
 354
 355
 356
 357
 358
 359
 360
 361
 362
 363
 364
 365
 366
 367
 368
 369
 370
 371
 372
 373
 374
 375
 376
 377
 378
 379
 380
 381
 382
 383
 384
 385
 386
 387
 388
 389
 390
 391
 392
 393
 394
 395
 396
 397
 398
 399
 400
 401
 402
 403
 404
 405
 406
 407
 408
 409
 410
 411
 412
 413
 414
 415
 416
 417
 418
 419
 420
 421
 422
 423
 424
 425
 426
 427
 428
 429
 430
 431
 432
 433
 434
 435
 436
 437
 438
 439
 440
 441
 442
 443
 444
 445
 446
 447
 448
 449
 450
 451
 452
 453
 454
 455
 456
 457
 458
 459
 460
 461
 462
 463
 464
 465
 466
 467
 468
 469
 470
 471
 472
 473
 474
 475
 476
 477
 478
 479
 480
 481
 482
 483
 484
 485
 486
 487
 488
 489
 490
 491
 492
 493
 494
 495
 496
 497
 498
 499
 500
 501
 502
 503
 504
 505
 506
 507
 508
 509
 510
 511
 512
 513
 514
 515
 516
 517
 518
 519
 520
 521
 522
 523
 524
 525
 526
 527
 528
 529
 530
 531
 532
 533
 534
 535
 536
 537
 538
 539
 540
 541
 542
 543
 544
 545
 546
 547
 548
 549
 550
 551
 552
 553
 554
 555
 556
 557
 558
 559
 560
 561
 562
 563
 564
 565
 566
 567
 568
 569
 570
 571
 572
 573
 574
 575
 576
 577
 578
 579
 580
 581
 582
 583
 584
 585
 586
 587
 588
 589
 590
 591
 592
 593
 594
 595
 596
 597
 598
 599
 600
 601
 602
 603
 604
 605
 606
 607
 608
 609
 610
 611
 612
 613
 614
 615
 616
 617
 618
 619
 620
 621
 622
 623
 624
 625
 626
 627
 628
 629
 630
 631
 632
 633
 634
 635
 636
 637
 638
 639
 640
 641
 642
 643
 644
 645
 646
 647
 648
 649
 650
 651
 652
 653
 654
 655
 656
 657
 658
 659
 660
 661
 662
 663
 664
 665
 666
 667
 668
 669
 670
 671
 672
 673
 674
 675
 676
 677
 678
 679
 680
 681
 682
 683
 684
 685
 686
 687
 688
 689
 690
 691
 692
 693
 694
 695
 696
 697
 698
 699
 700
 701
 702
 703
 704
 705
 706
 707
 708
 709
 710
 711
 712
 713
 714
 715
 716
 717
 718
 719
 720
 721
 722
 723
 724
 725
 726
 727
 728
 729
 730
 731
 732
 733
 734
 735
 736
 737
 738
 739
 740
 741
 742
 743
 744
 745
 746
 747
 748
 749
 750
 751
 752
 753
 754
 755
 756
 757
 758
 759
 760
 761
 762
 763
 764
 765
 766
 767
 768
 769
 770
 771
 772
 773
 774
 775
 776
 777
 778
 779
 780
 781
 782
 783
 784
 785
 786
 787
 788
 789
 790
 791
 792
 793
 794
 795
 796
 797
 798
 799
 800
 801
 802
 803
 804
 805
 806
 807
 808
 809
 810
 811
 812
 813
 814
 815
 816
 817
 818
 819
 820
 821
 822
 823
 824
 825
 826
 827
 828
 829
 830
 831
 832
 833
 834
 835
 836
 837
 838
 839
 840
 841
 842
 843
 844
 845
 846
 847
 848
 849
 850
 851
 852
 853
 854
 855
 856
 857
 858
 859
 860
 861
 862
 863
 864
 865
 866
 867
 868
 869
 870
 871
 872
 873
 874
 875
 876
 877
 878
 879
 880
 881
 882
 883
 884
 885
 886
 887
 888
 889
 890
 891
 892
 893
 894
 895
 896
 897
 898
 899
 900
 901
 902
 903
 904
 905
 906
 907
 908
 909
 910
 911
 912
 913
 914
 915
 916
 917
 918
 919
 920
 921
 922
 923
 924
 925
 926
 927
 928
 929
 930
 931
 932
 933
 934
 935
 936
 937
 938
 939
 940
 941
 942
 943
 944
 945
 946
 947
 948
 949
 950
 951
 952
 953
 954
 955
 956
 957
 958
 959
 960
 961
 962
 963
 964
 965
 966
 967
 968
 969
 970
 971
 972
 973
 974
 975
 976
 977
 978
 979
 980
 981
 982
 983
 984
 985
 986
 987
 988
 989
 990
 991
 992
 993
 994
 995
 996
 997
 998
 999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
class Interface:
    # a shape set's share of a refresh bar: it grows with the source count, and reports per source
    shapes_weight = 3

    def __init__(self, viewer, overview, enable_tabs=None, select_tab=None, is_tab_enabled=None,
                 enable_tab=None, enable_plugin_widget=None, verbose=False, initialize=True):
        self.viewer = viewer
        self.overview = overview
        self.enable_tabs = enable_tabs
        self.select_tab = select_tab
        self.is_tab_enabled = is_tab_enabled
        self.enable_tab = enable_tab
        self.enable_plugin_widget = enable_plugin_widget
        self.verbose = verbose
        if _available_memory_bytes:
            # napari sizes its dask cache from the machine's memory: on a cluster node, not the job's
            resize_dask_cache(nbytes=int(_available_memory_bytes * napari_dask_cache_share))
        self.raw_template = get_project_template()
        if not self.raw_template:
            raise FileNotFoundError('Project template not found')
        self.template = get_section_dict(self.raw_template, ['inputs', 'parameters', 'display_only', 'outputs'])
        self.param_widgets = {}
        self.params = {}
        self.pre_processing_performed = False
        self.metrics_methods = ['ncc', 'ssim', 'onmi']
        self.transform_key = 'source_metadata'
        self.need_source_reinit = False

        self.pair_metrics_timer = QTimer()
        self.pair_metrics_timer.setSingleShot(True)
        self.pair_metrics_timer.setInterval(1000)
        self.pair_metrics_timer.timeout.connect(self.update_pair_metrics)
        self._pair_metrics_worker = None
        self._pair_metrics_pending = False
        self._fusion_estimate = None
        # per viewer (by id): the shape layer drawn and the signature of what it was drawn from
        self._shape_layers = {}
        self._detached_view = None
        # on the viewers, not the shape layers: a layer's mouse callbacks fire only while it is the active layer
        on_click = _weak_view_click(weakref.ref(self))
        for view in (viewer, overview):
            view_model = view._qtwidget._viewer_model if isinstance(view, ViewerWidget) else view
            if view_model is not None:
                view_model.mouse_drag_callbacks.append(on_click)

        self.reg = MVSRegistration()
        if initialize:
            self.reset()

    def reset(self):
        self.source_metadata = {}
        self.extra_metadata = {}
        self.output_channels = []
        self.view_mode = None
        self._preview_overlap_cache = None
        self._view_msims = None
        self.reg.reset()
        self._shape_layers = {}
        self._detached_view = None
        self._clear_napari_view(self.overview)
        self._clear_napari_view(self.viewer)
        if self.enable_tabs:
            self.enable_tabs(False, 2)
        if self.select_tab:
            self.select_tab(1)
        self.update_fusion_preview_enabled()

    def get_all_widgets(self):
        # excludes a disabled tab's widgets: their .enabled reads False from the tab, so restoring
        # that state would keep them disabled once the tab is enabled again
        return {name: param_widget.widget for name, param_widget in self.param_widgets.items()
               if self.is_tab_enabled is None or self.is_tab_enabled(name.split('.', 1)[0])}

    def get_function(self, function_label):
        if hasattr(self, function_label):
            return eval(f'self.{function_label}')
        else:
            return None

    def tab_changed(self, tab_label):
        self.current_tab = tab_label
        if tab_label != 'registration' and self.view_mode == ViewMode.FEATURES and not self._restore_view():
            self._clear_napari_view(self.viewer)
            self.view_mode = None
        self.pair_metrics_timer.stop()

    def project_path(self, path):
        self.reset()
        self.params_path = path
        self.params = get_template_params(self.template)
        if os.path.exists(path):
            project_params = read_params(path)
            fusion_params = project_params.get('fusion') or {}
            if 'tile_size' in fusion_params:
                # the block size's name in projects saved before; written back as block_size
                fusion_params.setdefault('block_size', fusion_params.pop('tile_size'))
            # left out of the plugin: it drops the wrong tiles, and passed on it would filter unseen
            if (project_params.get('pre_processing') or {}).pop('filter_foreground', False):
                logging.warning('Foreground filtering is not available in the plugin: left out')
            unknown_params = get_unknown_params(self.template, project_params)
            if unknown_params:
                # still written back unchanged, as scripted runs may read them (e.g. extra_metadata)
                logging.warning(f'Project settings not used by the plugin: {", ".join(unknown_params)}')
            self.params = update_params(self.params, project_params)
            self.update_widgets()
        else:
            self.write_params()
        self.update_input_output_path()
        self.resolve_output_settings()

    def get_project_dir(self):
        # path params are stored relative to this directory, so a project moves with its data
        params_path = getattr(self, 'params_path', None)
        return os.path.dirname(os.path.abspath(params_path)) if params_path else None

    def update_widgets(self):
        # a dropdown filled later (channels, preview images) is empty until then: only fixed ones map old values
        fixed_choices = {f'{section_id}.{item["name"]}' for section_id, items in self.template.items()
                         for item in items if item.get('options')}
        for param_widget in self.param_widgets.values():
            # a mirrored widget's key is its tab's; param_name is where its value is stored
            param_name = param_widget.param_name
            # path widgets are set by update_input_output_path(), relative to the project directory
            if param_name not in ('input_output.input_path', 'input_output.output_path'):
                keys = param_name.split('.')
                value = self.params.get(keys[0], {}).get(keys[1])
                if value is not None and param_name in fixed_choices and isinstance(param_widget.widget, ComboBox) \
                        and value not in param_widget.widget.choices:
                    value = self._listed_choice(param_name, value, param_widget.widget)
                if value is not None:
                    param_widget.set_value(value)
                    if param_name == 'input_output.channels_table':
                        # set_value() blocks the table's signals, so channels_table() never sees the loaded value
                        if value:
                            self.set_channels(value)
                        # a loaded project skips update_output_channels(), so attach the color pickers here
                        self.populate_channels_table_color_pickers()
        # a loaded method equal to the widget's default changes nothing, so change_param() does not see it
        self.update_method_params_enabled()

    def _listed_choice(self, param_name, value, widget):
        """The dropdown's choice for a value saved before its setting became a list (e.g. a block size of 4096, or
        4096,4096 with every dim alike), else its current one; kept in the project's params."""
        parts = value if isinstance(value, (list, tuple)) else str(value).split(',')
        parts = {str(part).strip() for part in parts}
        # an empty list is no value, as an empty text is
        choice = parts.pop() if len(parts) == 1 else '' if not parts else None
        if choice not in widget.choices:
            logging.warning(f'{param_name}: {value!r} is not one of {list(widget.choices)}; using {widget.value!r}')
            choice = widget.value
        section, name = param_name.split('.')
        self.params[section][name] = choice
        return choice

    def write_params(self):
        write_params(self.params_path, self.params)

    def get_output_dir(self):
        return resolve_to_project_dir(path_param_to_text(self.params['input_output'].get('output_path', '')),
                                      self.get_project_dir())

    def get_output_params_path(self):
        """The copy of the project file next to its output, or None for a project kept in its own output."""
        params_path = getattr(self, 'params_path', None)
        if not params_path or not os.path.exists(params_path):
            return None
        target = os.path.join(self.get_output_dir(), os.path.basename(params_path))
        return target if os.path.abspath(target) != os.path.abspath(params_path) else None

    def prepare_action(self):
        """As an action starts: settings resolved against its output, sources read again where their settings
        changed, and the project file copied next to the output. Returns whether the sources were read again."""
        self.resolve_output_settings()
        read_again = self.need_source_reinit and self.reg.is_initialised()
        if read_again:
            self._input_output_process()
        self.copy_params_to_output()
        return read_again

    def copy_params_to_output(self):
        """The project file next to its output: the settings the output was made with."""
        target = self.get_output_params_path()
        if target:
            os.makedirs(os.path.dirname(target), exist_ok=True)
            shutil.copyfile(self.params_path, target)
            logging.info(f'Project settings copied to {target}')

    def resolve_output_settings(self):
        """A registration saved with other source metadata settings either gets them back or is discarded."""
        output_params_path = self.get_output_params_path()
        if not output_params_path or not os.path.exists(output_params_path):
            return
        output = self.get_output_dir()
        operation = self.params['registration'].get('operation', '')
        fused_name = operation_to_past_participle(operation) if operation else None
        fused_paths = self._output_paths(fused_name, output) if fused_name else []
        saved = self.reg.saved_progress_paths(output) + [path for path in fused_paths if os.path.exists(path)]
        previous = read_params(output_params_path).get('input_output', {})
        current = read_params(self.params_path).get('input_output', {})
        changed = [name for name in source_metadata_params if previous.get(name) != current.get(name)]
        if saved and changed:
            if self._ask_discard_output(f'Source metadata settings differ from the registration saved in {output}:'
                                        f' {", ".join(changed)}.\nReload the previous settings?'
                                        ' Or continue and discard all previous output?'):
                self.reg.discard_saved_progress(output)
                if fused_name:
                    self._remove_output(fused_name, output)
                self.need_source_reinit = True
            else:
                for name in changed:
                    self.params['input_output'][name] = previous.get(name)
                self.write_params()
                self.update_widgets()

    def _ask_discard_output(self, text):
        # no cancel: closing the box reloads, the answer that deletes nothing
        box = QMessageBox(QMessageBox.Icon.Question, 'muvis-align', text,
                          QMessageBox.StandardButton.Yes | QMessageBox.StandardButton.Discard)
        box.button(QMessageBox.StandardButton.Yes).setText('Reload')
        discard = box.button(QMessageBox.StandardButton.Discard)
        discard.setText('Discard')
        box.setDefaultButton(QMessageBox.StandardButton.Yes)
        box.exec()
        return box.clickedButton() is discard

    def change_param(self, param_name, value):
        keys = param_name.split('.')
        if keys[0] not in self.params:
            self.params[keys[0]] = {}
        if isinstance(value, list):
            # stored relative, as a single string for one path to keep simple projects unchanged
            value = [relativize_to_project_dir(item, self.get_project_dir()) for item in value]
            value = value[0] if len(value) == 1 else value
        elif isinstance(value, str):
            value = value.replace('\\', '/')
            if param_name in ('input_output.input_path', 'input_output.output_path'):
                # the file dialog reports an absolute path: store it relative, keeping the project portable
                value = relativize_to_project_dir(value, self.get_project_dir())
        self.params[keys[0]][keys[1]] = value
        self.write_params()
        same_setting = [param_widget for param_widget in self.param_widgets.values()
                        if param_widget.param_name == param_name]
        if len(same_setting) > 1:
            # the same setting shown on another tab follows, without reporting the change again
            for param_widget in same_setting:
                if param_widget.get_value() != value:
                    with param_widget.widget.changed.blocked():
                        param_widget.set_value(value)
        if param_name == 'registration.operation':
            self.update_fusion_preview_enabled()
        if param_name == 'registration.method':
            self.update_method_params_enabled()

    def update_input_output_path(self):
        # shows paths as stored: FileEdit.set_value() would force them absolute, so the line edit is set directly
        params = self.params['input_output']
        widget = self.param_widgets.get('input_output.input_path')
        # several comma-separated globs are one path value, shown as stored
        input_path = path_param_to_text(params.get('input_path', ''))
        if widget is not None:
            self._set_path_widget_text(widget, input_path)
        widget = self.param_widgets.get('input_output.output_path')
        output_path = path_param_to_text(params.get('output_path', ''))
        if widget is not None:
            self._set_path_widget_text(widget, output_path)
        resolved_output_path = resolve_to_project_dir(output_path, self.get_project_dir())
        init_logging(log_filename=os.path.join(resolved_output_path, 'muvis-align.log'), verbose=self.verbose)

    def _set_path_widget_text(self, param_widget, value):
        line_edit = getattr(param_widget.widget, 'line_edit', None)
        if line_edit is not None:
            line_edit.value = value
        else:
            param_widget.set_value(value)

    def input_path(self, value):
        self.need_source_reinit = True

    def source_position_z(self, value):
        self._set_source_metadata_value('source_position_z', value)

    def source_position_y(self, value):
        self._set_source_metadata_value('source_position_y', value)

    def source_position_x(self, value):
        self._set_source_metadata_value('source_position_x', value)

    def source_scale_z(self, value):
        self._set_source_metadata_value('source_scale_z', value)

    def source_scale_y(self, value):
        self._set_source_metadata_value('source_scale_y', value)

    def source_scale_x(self, value):
        self._set_source_metadata_value('source_scale_x', value)

    def source_rotation(self, value):
        self._set_source_metadata_value('source_rotation', value)

    def _set_source_metadata_value(self, name, value):
        set_source_metadata_value(self.source_metadata, SOURCE_METADATA_FIELDS[name], value)
        self.need_source_reinit = True

    def normalise_rotated_positions(self, value):
        if value:
            self.source_metadata['norm'] = True
        else:
            self.source_metadata.pop('norm', None)
        self.need_source_reinit = True

    def registration_dimension(self, value):
        # channels depend on the registration dimension, so they are built again, as are the source positions
        self.extra_metadata.pop('channels', None)
        self.reg.registration_dimension = value
        self.need_source_reinit = True

    def channels_table(self, value):
        old_value = self.param_widgets.get('input_output.channels_table').get_value()
        self.set_channels(update_dict_value(old_value, value))

    def set_channels(self, channels_dict):
        self.extra_metadata['channels'] = channels_from_table(channels_dict)

    def input_output_process(self):
        try:
            if not self.prepare_action():
                self._input_output_process()
        except OperationCancelled:
            logging.info('input_output_process cancelled')
            show_info('Cancelled')

    def _input_output_process(self):
        # paths are shown normalised on Process, not live while typing
        self.update_input_output_path()
        params = self.params['input_output']
        project_dir = self.get_project_dir()
        output = resolve_to_project_dir(path_param_to_text(params['output_path']), project_dir)
        if not self.reg.is_initialised() or self.need_source_reinit:
            self.need_source_reinit = False
            # reg.init() keeps the previous registration's mappings, metrics and pre-processed msims
            self.reg.reset()
            if not output.endswith('/'):
                output += '/'
            input_path = resolve_to_project_dir(path_param_to_text(params['input_path']), project_dir)
            # one bar for reading the sources and any saved registration; the view gets its own bar after
            try:
                ok = self._init_sources(input_path, output, params)
            except OperationCancelled:
                # sources read part-way are not kept: the next Process reads them again
                self.need_source_reinit = True
                self.reg.state = RegState.UNINIT
                raise
            if ok:
                self._show_loaded_project()
                self._go_to_shown_section()
            else:
                show_warning('Invalid input or output')
                self.reg.state = RegState.UNINIT
        elif self.reg.is_global_registered():
            self.update_registered(view_transform_key=self.reg.reg_transform_key)
        elif self.reg.is_pairs_registered():
            self.update_registered(view_transform_key=self.reg.source_transform_key)
        else:
            self.update_metadata_source()
        self._show_missing_metadata()

    def _show_missing_metadata(self):
        # one dialog per source read, whatever the number of files
        summary = self.reg.missing_metadata_summary
        if summary:
            self.reg.missing_metadata_summary = ''
            QMessageBox.warning(None, 'muvis-align', 'Missing or unusable source metadata:\n\n' + summary)

    def _init_sources(self, input_path, output, params):
        with self._operation_progress('Initialising sources', phases=1) as factory:
            # init_data() places channels by it
            self.reg.registration_dimension = self.params['input_output'].get('registration_dimension')
            ok = self.reg.init(input_path=eval_path(input_path),
                               output_path=output,
                               overwrite=params['overwrite'],
                               pairing=self.params['registration'].get('pairing', ''),
                               verbose=self.verbose)
            if ok:
                # resuming does far more than a fresh open: reserve its phases before the source read sizes
                # itself against the whole bar
                operation = self.params['registration'].get('operation', '')
                fused_name = operation_to_past_participle(operation) if operation else None
                if self.reg.has_saved_progress(fused_name, zarr_extension):
                    factory.ensure_phases(4)
                self._show_middle_section()
                # _show_loaded_project() draws the view, so drawing here would only show the unregistered transform
                ok = self.update_metadata_source(skip_view_update=True, progress_factory=factory)
                if ok:
                    self.populate_image_selection()
                    self._load_saved_progress(factory)
        return ok

    def _show_middle_section(self):
        """The middle section's sources read and drawn before the rest, so the viewer is usable while reading many
        sources takes long. The full view replaces it once they are all read."""
        indices = self.reg.middle_section_indices()
        self._shown_section_indices = indices
        if indices is None:
            return
        section = self.reg.section_registration(indices)
        # left off the bar: sized for reading every source, its phases would take a large share
        silent = SilentProgress()
        self._run_off_thread(
            lambda worker_factory: section.init_data(source_metadata=self.source_metadata,
                                                     progress_factory=worker_factory),
            silent)
        full, self.reg = self.reg, section
        try:
            self.update_views(show_images=False, progress_factory=silent)
        finally:
            self.reg = full

    def _go_to_shown_section(self):
        """Keep the section shown while the rest were read: napari's own start, the middle step of the z range, is a
        different section unless they are evenly spaced."""
        indices = getattr(self, '_shown_section_indices', None)
        dims = self.viewer.dims
        if indices and dims.ndim >= 3 and self.reg.sources[0].get_size().get('z', 0) <= 1:
            z_position = self.reg.positions[indices[0]].get('z')
            if z_position is not None:
                dims.set_point(dims.ndim - 3, z_position)

    def _run_off_thread(self, work, progress_factory):
        """Run work(progress_factory) on a worker thread and wait for it in a nested event loop, keeping the window
        responsive.

        `work` gets a headless progress factory whose positions reach the bar as a queued signal, as a worker must
        never touch a Qt widget. Patches a phase needs (dask callbacks, tqdm) belong inside `work`.
        Without a Qt application the work runs in the calling thread.
        """
        app = QApplication.instance() if QApplication is not None else None
        if app is None or progress_factory is None:
            return work(progress_factory)

        bridge = _ProgressBridge()
        bridge.moved.connect(progress_factory.set_position, Qt.QueuedConnection)
        bridge.stepped.connect(progress_factory.set_step, Qt.QueuedConnection)
        # starts where the bar has got to: a twin planning from empty would report positions the bar is past
        worker_factory = progress_factory.worker_twin(bridge.moved.emit, bridge.stepped.emit)

        outcome = {}
        done = threading.Event()

        # the outcome is kept here, not by signal: a worker done before the loop starts can lose its quit (macOS)
        def run():
            try:
                with worker_factory as factory:
                    outcome['value'] = work(factory)
            except Exception as error:
                outcome['error'] = error
            finally:
                done.set()

        loop = QEventLoop()
        poll = QTimer()
        poll.timeout.connect(lambda: loop.quit() if done.is_set() else None)
        # handlers set at creation: without an 'errored' one, create_worker() adds one re-raising uncatchably
        # in the Qt event loop
        worker = create_worker(
            run,
            _start_thread=False,
            _connect={
                'errored': lambda error: outcome.setdefault('error', error),
                # a worker diverts warnings to this signal: shown as on the Qt thread, once per message
                'warned': lambda warning: notification_manager.receive_warning(*warning),
                'finished': loop.quit,
            },
        )
        worker.start()
        if not done.is_set():
            poll.start(50)
            loop.exec_()
            poll.stop()
        # the twin's accounting becomes the bar's, so the next off-thread call continues from here
        progress_factory.continue_from(worker_factory)
        if 'error' in outcome:
            raise outcome['error']
        return outcome.get('value')

    @contextmanager
    def _operation_progress(self, desc, progress_factory=None, phases=1):
        """Progress for one user-facing operation: the activity dock, disabled widgets and a single
        NapariPhaseProgress bar that all its phases report into.

        An operation nested in another (its factory passed, or one already running) reports into that bar and
        leaves the dock and widgets to it. `phases` sizes each phase's slice of the bar; being out by one only
        makes it move unevenly.
        """
        factory = progress_factory or getattr(self, '_running_operation', None)
        if factory is not None:
            # the outer bar still needs this phase count, or each phase takes most of what it had left
            factory.ensure_phases(phases)
            yield factory
            return
        with cancellable(), \
             NapariPhaseProgress(progress_class=progress, desc=desc, phases=phases,
                                 min_duration=0.1) as factory, \
             self._operation_widgets(), \
             VisibleActivityDock(self.viewer):
            self._running_operation = factory
            try:
                yield factory
            finally:
                self._running_operation = None

    @contextmanager
    def _operation_widgets(self):
        """While an operation runs: every widget disabled but the Process buttons, which read Cancel, and
        every tab but the current one - as modify_pair_registration does, states restored after."""
        widgets = self.get_all_widgets() if getattr(self, 'param_widgets', None) else {}
        process_buttons = [widgets.pop(name) for name in list(widgets) if name.endswith('_process')]
        widget_state = TemporarilyDisabledWidgets()
        widget_state.disable(widgets)
        tab_states = {}
        current_tab = getattr(self, 'current_tab', None)
        enable_tab, is_tab_enabled = getattr(self, 'enable_tab', None), getattr(self, 'is_tab_enabled', None)
        if enable_tab and is_tab_enabled and current_tab is not None:
            tab_states = {section_id: is_tab_enabled(section_id)
                          for section_id in ['project'] + list(self.template.keys()) if section_id != current_tab}
            for section_id in tab_states:
                enable_tab(section_id, False)
        for button in process_buttons:
            button.text = 'Cancel'
        flush_paint_events()
        try:
            yield
        finally:
            for button in process_buttons:
                button.text = 'Process'
            widget_state.restore()
            for section_id, was_enabled in tab_states.items():
                enable_tab(section_id, was_enabled)
            flush_paint_events()

    def process_or_cancel(self, process, *_):
        """A section's Process button: runs `process`, or while an operation runs (the button then reads
        Cancel), offers to cancel that one."""
        running = getattr(self, '_running_operation', None)
        if running is None:
            # a failure outside the run_*() steps would otherwise reach only stderr, via the Qt signal
            try:
                process()
            except Exception as error:
                name = getattr(process, '__name__', 'process')
                logging.exception(f'{name} failed')
                report_failure(operation_label(name), error)
        else:
            self.cancel_operation(getattr(running, 'desc', None))

    def cancel_operation(self, desc=None):
        # the running operation stops at its next progress step, on its worker thread
        reply = QMessageBox.question(None, 'muvis-align',
                                     f'Cancel {desc or "the running operation"}? Its partial result is discarded.',
                                     QMessageBox.Yes | QMessageBox.No)
        if reply == QMessageBox.Yes:
            logging.info('Cancel requested')
            request_cancel()

    def init_progress(self):
        # the two halves of opening a project, each its own bar, one after the other
        with self._operation_progress('Initialising sources', phases=3) as progress_factory:
            self._load_saved_progress(progress_factory)
        self._show_loaded_project()

    def _load_saved_progress(self, progress_factory=None):
        output_filename = operation_to_past_participle(self.params['registration']['operation'])
        # resuming builds the per-source msims, loads the saved registration and may pre-process again
        with self._operation_progress('Initialising sources', progress_factory,
                                      phases=3) as progress_factory:
            self._run_off_thread(
                lambda worker_factory: self.reg.init_progress(output_filename, zarr_extension,
                                                              progress_factory=worker_factory),
                progress_factory)

    def _show_loaded_project(self):
        # the second bar of opening a project: building and drawing the view
        section_tab = None
        if self.reg.is_fused():
            # the fused view skips update_registered(), so its tables are filled here
            self.populate_registered_tables()
            fused_path = self.reg.current_fused_output(
                operation_to_past_participle(self.params['registration']['operation']))
            with self._operation_progress('Refreshing view', phases=2 + self.shapes_weight) as view_factory:
                if fused_path is not None:
                    self._show_fused_output(fused_path, progress_factory=view_factory)
                else:
                    # the export on disk predates the saved registration
                    logging.info('Fused output is older than the saved registration: previewing the fusion instead')
                    self._copy_transforms_to_view_msims(self.reg.reg_transform_key, progress_factory=view_factory)
                    self.preview_fusion(progress_factory=view_factory)
            section_tab = 4
        elif self.reg.is_global_registered():
            with self._operation_progress('Refreshing view', phases=2) as view_factory:
                self.update_registered(view_transform_key=self.reg.reg_transform_key,
                                       progress_factory=view_factory)
            section_tab = 4
        elif self.reg.is_pairs_registered():
            self.update_registered(view_transform_key=self.reg.source_transform_key)
            section_tab = 3
        else:
            # the draw update_metadata_source() skipped; before pre-processing only shapes are shown
            self.update_views(show_images=False)
        # every step can be reached from here: a later one runs whatever it needs first
        self.enable_tabs(True, 4)
        self.update_fusion_preview_enabled()
        if section_tab is not None:
            self.select_tab(section_tab)

    def update_metadata_source(self, skip_view_update=False, progress_factory=None):
        if not self.reg.is_pairs_registered():
            try:
                with self._operation_progress('Initialising sources',
                                              progress_factory) as factory:
                    self._run_off_thread(
                        lambda worker_factory: self.reg.init_data(
                            source_metadata=self.source_metadata,
                            progress_factory=worker_factory,
                        ),
                        factory)
            except ValueError as e:
                show_warning('Unable to read source data\n' + str(e))
                logging.exception('Unable to read source data')
                return False

            # msims are built lazily by view_msims; the cheap z remapping stays eager so the shapes are right
            z_positions = sorted(set([position.get('z', 0) for position in self.reg.positions]))
            if len(z_positions) > 1:
                for position in self.reg.positions:
                    position['z'] = z_positions.index(position.get('z', 0))
            self._view_msims = None
        # only runs before registration, so the source transform key is the only one: no need to build msims
        coord_systems = [self.reg.source_transform_key]
        self.populate_channels()
        self.populate_coordinate_systems(coord_systems)
        if self.update_output_channels():
            self.populate_channels_table()
        if self.reg.is_initialised():
            # without transform_keys the msims are not read, so None avoids forcing their build
            self.populate_metadata_table(None)
            self.check_3d_view()
            if not skip_view_update:
                # no factory: drawing is the next operation, or a phase of the caller's if one runs
                self.update_views(show_images=False)

        return True

    @property
    def view_msims(self):
        # per-source msims for the image layer, built lazily only when the fused preview needs them
        return self.ensure_view_msims()

    def ensure_view_msims(self, progress_factory=None):
        # the view_msims build, callable ahead with a progress_factory; kept 2D, as consumers place
        # sources at their z (promoting every level of every source is slow)
        if self._view_msims is None:
            with Timer('build view msims', verbose=self._timing_verbose()):
                self._view_msims = self._build_view_msims(progress_factory=progress_factory)
        return self._view_msims

    def _copy_transforms_to_view_msims(self, transform_key, progress_factory=None):
        # building view_msims takes a while, so it reports progress rather than running as a silent side effect
        with self._operation_progress('Building views', progress_factory) as factory:
            view_msims = self._run_off_thread(
                lambda worker_factory: self.ensure_view_msims(progress_factory=worker_factory),
                factory)
        with Timer('copy transforms to view msims', verbose=self._timing_verbose()):
            copy_transforms_to_msims(self.reg.msims, view_msims, transform_key)

    @view_msims.setter
    def view_msims(self, value):
        self._view_msims = value

    def _build_view_msims(self, progress_factory=None):
        # a native pyramid is used as-is; a single-resolution source is downscaled to at most 1000px
        view_msims = []
        progress_context = progress_phase(progress_factory, total=len(self.reg.sources), desc='Building views')
        with progress_context as pbar:
            for source, msim in zip(self.reg.sources, self.reg.msims):
                if len(source.shapes) == 1:
                    image0 = get_msim_image0(msim)
                    spatial_dims = si_utils.get_spatial_dims_from_sim(image0)
                    largest_dim = max(image0.sizes[dim] for dim in spatial_dims)
                    if largest_dim > 1000:
                        scale_factor = largest_dim / 1000
                        sim = extract_sims_from_msims(
                            [msim], [source], self.reg.source_transform_key, target_scale=scale_factor
                        )[0]
                        msim = wrap_sims_as_msims([sim])[0]
                view_msims.append(msim)
                if pbar is not None:
                    pbar.update(1)
        return view_msims

    @catch_run_errors
    def run_pre_processing(self, progress_factory=None, full_pyramid=False):
        params_features = self.params['pre_processing']
        # the msim build gets bar room only when pending, weighted by its cost: it dominates unless a step
        # computes over the data (flat-field, normalisation, foreground)
        build_weight = 2 if self.reg.has_eager_pre_processing(params_features) else 8
        # registration reads the finest level and the preview the coarsest; convert writes every level
        ends_only = not full_pyramid
        build_pending = self.reg.msims_build_pending(params_features.get('scale'), ends_only=ends_only)
        phases = build_weight + 1 if build_pending else 1
        with self._operation_progress('Pre-processing', progress_factory, phases=phases) as progress_factory, \
             Timer('pre_processing_process', verbose=self._timing_verbose()):
            # msims built explicitly through ensure_msims() so the build reports progress
            def preprocess(worker_factory):
                with Timer('run_pre_processing: build msims (load image data)',
                           verbose=self._timing_verbose()):
                    # at pre-processing's scale: finer levels would only be dropped again
                    msims = self.reg.ensure_msims(progress_factory=worker_factory,
                                                  target_scale=params_features.get('scale'),
                                                  ends_only=ends_only,
                                                  weight=build_weight)
                with Timer('run_pre_processing: preprocess', verbose=self._timing_verbose()):
                    return self.reg.preprocess(msims, progress_factory=worker_factory,
                                               **params_features)

            _, _, modified = self._run_off_thread(preprocess, progress_factory)
        self.pre_processing_performed = modified
        return True

    def pre_processing_process(self):
        # the work and showing its result are two operations, each with its own bar
        self.prepare_action()
        if not self.run_pre_processing():
            return
        if self.reg.is_pairs_registered():
            # register_msims changed: the prior registration is stale and must not give the view its transform
            self.reg.state = RegState.SIMS_INIT
        self.update_views(show_preprocessed=True)
        self.enable_modify_pair_registration(False)
        self.update_fusion_preview_enabled()
        self.select_tab(3)

    def populate_channels(self):
        channel_labels = list({channel.get('label', '') for source in self.reg.sources for channel in source.get_channels()})
        choices = {channel: channel for channel in channel_labels}
        param_widget = self.param_widgets.get('registration.channel')
        param_widget.set_choices(choices)

    def populate_coordinate_systems(self, coord_systems):
        choices = {coord_system: coord_system.replace('_', ' ').capitalize() for coord_system in coord_systems}
        param_widget = self.param_widgets.get('input_output.coordinate_system')
        param_widget.set_choices(choices)

    def coordinate_system(self, transform_key):
        self.transform_key = transform_key
        if self.reg.is_initialised():
            self.populate_metadata_table(transform_key)

    def populate_metadata_table(self, transform_key=None):
        """Each source's position, pixel size and rotation: from its metadata, or from its transform_key's affine."""
        table_widget = self.param_widgets.get('input_output.metadata_table')
        properties = ['position', 'pixel size']
        if transform_key in (None, self.reg.source_transform_key):
            positions = [dict(position) for position in self.reg.positions]
            scales = self.reg.scales
            rotations = self.reg.rotations
        else:
            positions, scales, rotations = [], [], []
            for msim, position0, scale0 in zip(self.reg.msims, self.reg.positions, self.reg.scales):
                affine = msi_utils.get_transform_from_msim(msim, transform_key)
                if 't' in affine.dims:
                    affine = affine.isel(t=0)
                affine_dims = affine['x_in'].data.tolist()
                transform_scale = get_scale_from_transform(np.asarray(affine))
                # a stack's sources are 2D: their z stays the metadata's
                positions.append({**position0, **get_sim_position_final(msim, transform_keys=[transform_key])})
                scales.append({dim: size * transform_scale if dim in affine_dims else size
                               for dim, size in scale0.items()})
                rotations.append(get_rotation_from_transform(affine))
        positions = self.channel_positions(positions)
        for position, rotation in zip(positions, rotations):
            if rotation is not None:
                position['r'] = rotation
        # metadata order in every coordinate system, so rows stay put when it changes
        order = self.source_order()
        data = [[print_dict_simple(positions[i], dims='czyx'), print_dict_simple(scales[i])] for i in order]
        row_headers = [self.reg.file_labels[i] for i in order]
        # Table: tuple-of-values : ([values], [row_headers], [column_headers])
        table_widget.set_value((data, row_headers, properties))
        table_widget.set_table_column_resize_mode()

    def channel_positions(self, positions):
        """Copies of positions, with each source's channel group as 'c' in a channel overlay."""
        positions = [dict(position) for position in positions]
        if self.reg.is_channel_overlay(self.params.get('input_output', {}).get('registration_dimension'),
                                      self.extra_metadata):
            for position, channel in zip(positions, self.reg.channel_groups()):
                position['c'] = channel
        return positions

    def source_order(self):
        """Source indices in the metadata table's (t, c, z, y, x) order, by their metadata positions."""
        positions = self.channel_positions(self.reg.positions)
        return sorted(range(len(positions)), key=lambda index: position_sort_key(positions[index]))

    def update_output_channels(self):
        channels = self.extra_metadata.get('channels')
        # blank-labelled channels count as unconfigured, so they are derived again
        if not channels or not any(channel.get('label') for channel in channels):
            dimension = self.params['input_output']['registration_dimension']
            if dimension.lower() == 'c':
                # one per channel group, its first source's channel (e.g. its color) under the group's label
                groups = self.reg.channel_groups()
                channels = [{**(self.reg.sources[groups.index(group)].get_channels() or [{}])[0], 'label': label}
                            for group, label in enumerate(self.reg.channel_labels())]
            else:
                channels = self.reg.sources[0].get_channels()

            self.extra_metadata['channels'] = channels

            # `or`, not a get() default: an unnamed OME channel has an explicit blank label
            data = [[channel.get('label') or f'channel {index}', channel.get('color', (1, 1, 1))]
                     for index, channel in enumerate(channels)]
            self.output_channels = create_dict_of_lists(data, ['label', 'color'])
            return True

        return False

    def populate_channels_table(self):
        param_widget = self.param_widgets.get('input_output.channels_table')
        param_widget.set_value(self.output_channels)
        self.populate_channels_table_color_pickers()

    def populate_channels_table_color_pickers(self):
        table = self.param_widgets.get('input_output.channels_table').widget
        if 'color' not in table.column_headers:
            return
        color_coli = table.column_headers.index('color')
        for rowi in range(table.shape[0]):
            color = parse_channel_color(table.data[rowi, color_coli]) or (1, 1, 1)
            color_picker = MagicColorPicker(value=color)
            color_picker.changed.connect(
                lambda _=None, picker=color_picker, rowi=rowi: self.channel_color_changed(rowi, picker.value))
            table.native.setCellWidget(rowi, color_coli, color_picker.native)

    def channel_color_changed(self, rowi, color):
        table = self.param_widgets.get('input_output.channels_table').widget
        color_coli = table.column_headers.index('color')
        # written into the table so its 'changed' wiring saves it, as a manual edit would
        table.data[rowi, color_coli] = str(tuple(color))

    def populate_image_selection(self):
        labels = list(self.reg.file_labels)
        # the metadata and metrics tables' order
        if len(self.reg.positions) == len(labels):
            labels = [labels[index] for index in self.source_order()]
        widget1 = self.param_widgets.get('registration.reg_preview_image1')
        widget1.set_value(labels[0], choices=labels)

        widget2 = self.param_widgets.get('registration.reg_preview_image2')
        index = 1 if len(labels) > 1 else 0
        widget2.set_value(labels[index], choices=labels)

    def _on_view_click(self, view_model, event):
        """A left click on a viewer's shapes picks that pair; a drag (pan, rotate) picks nothing."""
        if event.button != 1:
            return
        press_pos = np.array(event.pos)
        pick = dict(position=event.position, view_direction=event.view_direction,
                    dims_displayed=event.dims_displayed, world=True)
        yield
        while event.type == 'mouse_move':
            yield
        # napari updates `event` in place, so it is the release now; a few pixels of jitter is still a click
        # the shape layer drawn in this viewer; has_layer sees past a plugin's proxies of viewer and layer
        layer = next((layer for _, layer in self._shape_layers.values()
                      if layer is not None and has_layer(view_model, layer)), None)
        if np.linalg.norm(np.array(event.pos) - press_pos) <= 4 and layer is not None and layer.visible:
            value = layer.get_value(**pick)
            if value is not None and value[0] is not None:
                self.select_pair_preview(layer.features['refs'][value[0]])

    def select_pair_preview(self, ref):
        # only an overlap shape's ref ('0 1') names a pair; single-image clicks are ignored
        indices = [int(index) for index in ref.split()]
        if len(indices) == 2:
            row_label = next((label for label, pair in getattr(self, 'metrics_row_pairs', {}).items()
                              if sorted(pair) == sorted(indices)), None)
            table = self.param_widgets.get('registration.metrics_table')
            if row_label is not None and table is not None:
                # the table never selects shapes back, so this cannot loop
                table.widget.native.select_row_label(row_label)
                indices = list(self.metrics_row_pairs[row_label])
            # also when the row was already selected, and so emitted no change
            self.set_preview_images(indices)

    def metrics_table_selected(self):
        table = self.param_widgets.get('registration.metrics_table').widget.native
        # the summary and group (section) pair rows have no image pair
        indices = getattr(self, 'metrics_row_pairs', {}).get(table.row_label(table.currentRow()))
        if indices:
            self.set_preview_images(indices)

    def set_preview_images(self, indices):
        labels = self.reg.file_labels
        self.param_widgets.get('registration.reg_preview_image1').set_value(labels[indices[0]])
        self.param_widgets.get('registration.reg_preview_image2').set_value(labels[indices[1]])

    def get_best_transform_key(self):
        # from the state, not off built msims: a loaded registration's are built only when needed
        return self.reg.reg_transform_key if self.reg.is_global_registered() else self.reg.source_transform_key

    def check_3d_view(self):
        is_3d = (self.reg.sources[0].get_size().get('z', 0) > 1)
        ndisplay = 3 if is_3d else 2
        self.viewer.dims.ndisplay = ndisplay
        #self.overview._qtwidget._viewer_model.dims.ndisplay = ndisplay

    def _timing_verbose(self):
        # some view methods are tested on a bare Interface without self.reg
        return getattr(getattr(self, 'reg', None), 'logging_time', False)

    def update_views(self, transform_key=None, show_preprocessed=False, show_images=True,
                     progress_factory=None):
        if transform_key is None:
            transform_key = self.get_best_transform_key()

        is_3d = (self.reg.sources[0].get_size().get('z', 0) > 1)
        is_multi_z_shapes = (len(set(position.get('z', 0) for position in self.reg.positions)) > 1)
        force_2d = is_multi_z_shapes and not is_3d

        # shapes drawn from the same geometry, pairs and qualities are kept, only the image under them replaced
        shapes = None
        build_shapes = self._current_shape_layer(self.viewer, transform_key, force_2d) is None
        # steps weighted by cost: building the view data dominates, and reports from the inside
        view_data_weight = 12
        phases = (self.shapes_weight if build_shapes else 0) + 2 + (view_data_weight + 1 if show_images else 0)

        with self._operation_progress('Refreshing view', progress_factory, phases=phases) as factory:
            if build_shapes:
                with Timer('update_views: create shapes', verbose=self._timing_verbose()):
                    # pure geometry, so off the Qt thread to keep the window responsive
                    shapes = self._run_off_thread(
                        lambda worker_factory: self._create_napari_shapes(
                            transform_key, force_2d=force_2d, progress_factory=worker_factory,
                            weight=self.shapes_weight),
                        factory)

            # each section built only when viewed; never without show_images, so whether data is read cannot hinge on
            # the sources happening to be unrotated
            data = None
            if show_images:
                with Timer('update_views: lazy overview', verbose=self._timing_verbose()):
                    data = self._run_off_thread(
                        lambda worker_factory: self._create_lazy_overview(transform_key,
                                                                          show_preprocessed=show_preprocessed), factory)
            # otherwise (a rotation, or sources it cannot overlay) the fused preview
            if data is None and show_images:
                with Timer('update_views: create fused data', verbose=self._timing_verbose()):
                    # fused off the Qt thread; adding it to the viewer below must stay on it
                    data = self._run_off_thread(
                        lambda worker_factory: self._create_napari_data(
                            transform_key, show_preprocessed=show_preprocessed,
                            progress_factory=worker_factory, weight=view_data_weight),
                        factory)
            # replaced only once the new data is ready: an empty viewer's welcome screen hides the activity dialog
            with factory(total=1) as pbar, \
                 Timer('update_views: add fused data to viewer', verbose=self._timing_verbose()):
                # an overview, not the fusion preview: a naive contrast guess is enough
                self._show_view_image(self.viewer, data, f'{self.reg.fileset_label} data', cheap=True,
                                      keep_shapes=not build_shapes)
                pbar.update(1)

            with Timer('update_views: add shapes to viewers', verbose=self._timing_verbose()):
                self._refresh_view_shapes(transform_key, shapes, progress_factory=factory)
        self.view_mode = ViewMode.OVERVIEW

    def _create_lazy_overview(self, transform_key, show_preprocessed=False):
        reg = self.reg
        transforms, output_order = getattr(reg, '_msim_transforms', None), getattr(reg, '_msim_output_order', None)
        if transforms is None or output_order is None:
            return None
        if transform_key == reg.reg_transform_key:
            # from source geometry and the mappings, as the shapes: no msim built for them
            transforms = [np.asarray(props['transform'], dtype=float) for props in reg.source_geometry(transform_key)[1]]
        readers, label = None, 'Overview'
        if show_preprocessed and reg.register_msims is not None:
            # a source pre-processing dropped (filter_foreground) is left out, not shown raw
            readers = [None] * len(reg.sources)
            for index, msim in zip(reg.register_indices, reg.register_msims):
                readers[index] = MsimLevels(msim)
            label = 'Pre-processed overview'
        else:
            pre_processing = self.params.get('pre_processing', {})
            if reg.flatfield_settings(pre_processing)[0]:
                # each section's model computed from its own tiles when its plane is first built, not every tile's
                readers = [FlatfieldLevels(SourceLevels(source), functools.partial(reg.flatfield_model, index,
                                                                                   pre_processing))
                           for index, source in enumerate(reg.sources)]
        channel_labels, channel_groups = None, None
        if reg.is_channel_overlay(self.params['input_output']['registration_dimension'], self.extra_metadata):
            channel_groups = reg.channel_groups()
            channel_labels = reg.channel_labels(self.extra_metadata.get('channels'))
        return lazy_section_overview(reg.sources, reg.positions, transforms, output_order, transform_key,
                                     z_scale=getattr(reg, '_msim_z_scale', None),
                                     preview_scale=self.params['input_output'].get('preview_scale'),
                                     readers=readers, label=f'{label} ({len(reg.sources)} images)',
                                     channel_labels=channel_labels, channel_groups=channel_groups)

    def _shapes_signature(self, transform_key, force_2d):
        # the source geometry a shape set is drawn from; pair registration moves no source
        return (transform_key, force_2d, id(self.reg.positions), id(self.reg._msim_transforms),
                id(self.reg.mappings) if transform_key == self.reg.reg_transform_key else None)

    def _shape_layer_signature(self, transform_key, force_2d):
        # a pair registration or load brings a new pairs graph and metrics: other overlaps, other colours
        pairs_graph = self.reg.pairs_graph if self.reg.is_pairs_registered() else None
        return self._shapes_signature(transform_key, force_2d) + (id(pairs_graph), id(self.reg.metrics))

    def _current_shape_layer(self, viewer, transform_key, force_2d):
        """The viewer's shape layer if drawn from this geometry, these pairs and their qualities, else None."""
        drawn_signature, layer = self._shape_layers.get(id(viewer), (None, None))
        if (drawn_signature == self._shape_layer_signature(transform_key, force_2d)
                and layer is not None and has_layer(viewer, layer)):
            return layer
        return None

    def _refresh_view_shapes(self, transform_key, shapes=None, progress_factory=None):
        """Both viewers' shape layers drawn for this geometry, each kept if it already is."""
        is_3d = (self.reg.sources[0].get_size().get('z', 0) > 1)
        is_multi_z_shapes = (len(set(position.get('z', 0) for position in self.reg.positions)) > 1)
        force_2d = is_multi_z_shapes and not is_3d
        # the overview is always flat and top-down, so a 3D shape set is built again with force_2d
        overview_force_2d = True if is_3d else force_2d
        redraw = [(viewer, viewer_force_2d, show_labels)
                  for viewer, viewer_force_2d, show_labels in [(self.viewer, force_2d, True),
                                                               (self.overview, overview_force_2d, False)]
                  if self._current_shape_layer(viewer, transform_key, viewer_force_2d) is None]
        # one set per force_2d, shared by both viewers unless the main view is 3D
        shape_sets = {force_2d: shapes} if shapes is not None else {}
        for viewer_force_2d in dict.fromkeys(viewer_force_2d for _, viewer_force_2d, _ in redraw):
            if viewer_force_2d not in shape_sets:
                # pure geometry, so off the Qt thread to keep the window responsive
                shape_sets[viewer_force_2d] = self._run_off_thread(
                    lambda worker_factory: self._create_napari_shapes(
                        transform_key, force_2d=viewer_force_2d, progress_factory=worker_factory,
                        weight=self.shapes_weight),
                    progress_factory)
        # adding a layer touches Qt, so stays on its thread
        with progress_phase(progress_factory, total=len(redraw)) as pbar:
            for viewer, viewer_force_2d, show_labels in redraw:
                _, old_layer = self._shape_layers.pop(id(viewer), (None, None))
                if old_layer is not None and has_layer(viewer, old_layer):
                    viewer.layers.remove(old_layer)
                layer = self._update_view_add_shapes(viewer, *shape_sets[viewer_force_2d],
                                                     f'{self.reg.fileset_label} shapes', show_labels=show_labels)
                self._shape_layers[id(viewer)] = (self._shape_layer_signature(transform_key, viewer_force_2d),
                                                  layer)
                if pbar is not None:
                    pbar.update(1)

    def _show_view_image(self, viewer, data, layer_name, cheap=False, keep_shapes=True):
        """The viewer's image replaced by `data` (None: none), the shape layer drawn over it kept."""
        _, kept = self._shape_layers.get(id(viewer), (None, None))
        kept = kept if keep_shapes and kept is not None and has_layer(viewer, kept) else None
        self._detached_view = None
        self._clear_napari_view(viewer, keep=kept)
        if data is not None:
            self._napari_view_add_fused_data(viewer, data, layer_name, cheap=cheap)
        if kept is not None:
            viewer.layers.move(viewer.layers.index(kept), len(viewer.layers))
            self._set_shapes_line_color(kept, contrasting_color(self._view_background_luminance(viewer)))

    def _set_shapes_line_color(self, layer, line_color):
        # only the lines drawn (a 3D box's faces have none) and the labels follow the image under them
        if layer.metadata.get('line_color') != line_color:
            edge_colors = np.array(layer.edge_color)
            edge_colors[np.asarray(layer.edge_width) > 0] = transform_color(line_color)[0]
            layer.edge_color = edge_colors
            layer.text.color = line_color
            layer.metadata['line_color'] = line_color

    def _detach_view(self, view_mode):
        """The main view's layers set aside for a pair or feature view, to be put back as they were."""
        # entering one of these views from the other keeps the view set aside first
        if self._detached_view is None:
            layers = [getattr(layer, '__wrapped__', layer) for layer in self.viewer.layers]
            self._detached_view = (layers, view_mode, getattr(self, '_view_levels', None))
        self._clear_napari_view(self.viewer)

    def _restore_view(self):
        """The main view set aside by _detach_view put back; False if there was none."""
        detached, self._detached_view = self._detached_view, None
        if detached is None:
            return False
        self._clear_napari_view(self.viewer)
        layers, self.view_mode, self._view_levels = detached
        for layer in layers:
            self.viewer.layers.append(layer)
        self.viewer.reset_view()
        return True

    def _recolor_pair_shapes(self, pair):
        """A pair's overlap coloured by its quality now in both viewers' shapes, the others left as drawn."""
        quality_range = metric_value_range(default_quality_key, self.params.get('registration', {}).get('method'))
        color = transform_color(metric_to_rgb(self.reg.get_metrics(default_quality_key, pair),
                                              value_range=quality_range))[0]
        pair_refs = [f'{pair[0]} {pair[1]}', f'{pair[1]} {pair[0]}']
        for viewer in (self.viewer, self.overview):
            _, layer = self._shape_layers.get(id(viewer), (None, None))
            if layer is not None and has_layer(viewer, layer):
                rows = np.isin(np.asarray(layer.features['refs']), pair_refs)
                if rows.any():
                    face_colors = np.array(layer.face_color)
                    face_colors[rows] = color
                    layer.face_color = face_colors

    def _clear_napari_view(self, viewer, keep=None):
        # no empty LayerList.clear() event: under xpra/Xvfb it breaks blending for later Shapes/Points layers
        if viewer is not None and len(viewer.layers) > 0:
            if keep is None:
                viewer.layers.clear()
            else:
                kept = getattr(keep, '__wrapped__', keep)
                for layer in [layer for layer in viewer.layers if getattr(layer, '__wrapped__', layer) is not kept]:
                    viewer.layers.remove(layer)
        if viewer is not None and viewer is getattr(self, 'viewer', None):
            clear_napari_dask_cache()
            self._view_levels = None

    def _create_napari_shapes(self, transform_key, force_2d=False, progress_factory=None, weight=1):
        # `weight` is this step's share of the caller's bar, split between sub-steps by rough cost
        def phase(share, total=None):
            return progress_phase(progress_factory, total=total,
                                        weight=max(weight * share, 1))

        # sources at different heights without a native 'z' need it as a dim, or their z is dropped
        promote_z = (len(set(position.get('z', 0) for position in self.reg.positions)) > 1)
        with phase(3 / 5, total=len(self.reg.sources)) as pbar, \
             Timer(f'_create_napari_shapes: build {len(self.reg.sources)} source shape geometries',
                   verbose=self._timing_verbose()):
            # stack properties from source geometry and the registration's mappings: no msim built, no pixel read
            _, msims = self.reg.source_geometry(
                transform_key, promote_z=promote_z, z_scale=self.reg._msim_z_scale,
                progress=pbar.update if pbar is not None else None)

        with phase(1 / 5, total=1) as pbar, \
             Timer(f'_create_napari_shapes: create_image_shapes ({len(msims)} images)', verbose=self._timing_verbose()):
            shapes = create_image_shapes(msims, transform_key=transform_key, force_2d=force_2d)
            if pbar is not None:
                pbar.update(1)
        refs = [str(index) for index in range(len(msims))]
        labels = list(self.reg.file_labels)
        face_colors = [(1, 1, 1) for _ in range(len(msims))]

        # once registered, only registered pairs: an overlap between unpaired images has no quality behind it
        overlap_pairs = list(self.reg.pairs_graph.edges()) if self.reg.is_pairs_registered() else None
        # the overlaps an earlier refresh found under the same geometry: pair registration moves no source
        signature = self._shapes_signature(transform_key, force_2d)
        cache = getattr(self, '_overlap_shapes', None)
        cache = cache if cache is not None and cache['signature'] == signature else None
        with phase(1 / 5, total=1) as pbar, \
             Timer(f'_create_napari_shapes: create_overlap_shapes ({len(msims)} images,'
                  f' {len(overlap_pairs) if overlap_pairs is not None else "all"} pairs)', verbose=self._timing_verbose()):
            # a pair checked without a shape found has no overlap; a pass over all pairs checked every pair
            if overlap_pairs is not None and cache is not None and (
                    cache['all_pairs'] or all(tuple(sorted(pair)) in cache['checked'] for pair in overlap_pairs)):
                pairs = [pair for pair in overlap_pairs if tuple(sorted(pair)) in cache['shapes']]
                shapes2 = [cache['shapes'][tuple(sorted(pair))] for pair in pairs]
            else:
                shapes2, pairs = create_overlap_shapes(msims, transform_key=transform_key, pairs=overlap_pairs,
                                                       force_2d=force_2d)
                checked = {tuple(sorted(int(index) for index in pair)) for pair in overlap_pairs or []}
                self._overlap_shapes = {'signature': signature, 'all_pairs': overlap_pairs is None,
                                        'checked': checked, 'shapes': {
                    tuple(sorted(int(index) for index in pair)): shape for pair, shape in zip(pairs, shapes2)}}
            if pbar is not None:
                pbar.update(1)
        shapes.extend(shapes2)
        refs += [f'{index1} {index2}' for index1, index2 in pairs]
        labels += ['' for _ in pairs]
        # grey for an overlap without a quality (not registered): black would hide the image under overlapping tiles
        quality_range = metric_value_range(default_quality_key, self.params.get('registration', {}).get('method'))
        face_colors += [np.array(metric_to_rgb(self.reg.get_metrics(default_quality_key, pair),
                                               value_range=quality_range, missing=(0.5, 0.5, 0.5)))
                        for pair in pairs]
        return shapes, refs, labels, face_colors

    def _create_napari_data(self, transform_key, fusion_method='additive', show_preprocessed=False,
                            progress_factory=None, weight=1, flatfield=False):
        # `weight` is this step's share of the caller's bar, split between sub-steps by rough cost
        # `flatfield`: as the export is fused; the plain view shows the sources as they are
        def phase(share, total=None):
            return progress_phase(progress_factory, total=total,
                                        weight=max(weight * share, 1))

        # only copy_transforms_to_msims() writes into a msim; the steps between may return their input
        # unchanged, so any new mutating step needs this flag too
        writes_transforms = not (show_preprocessed
                                 and transform_key == self.reg.source_transform_key)
        if show_preprocessed:
            with phase(1 / 12, total=1) as pbar, \
                 Timer(f'_create_napari_data: copy {len(self.reg.register_msims)} register_msims',
                      verbose=self._timing_verbose()):
                msims = ([msim.copy(deep=True) for msim in self.reg.register_msims]
                         if writes_transforms else list(self.reg.register_msims))
                if pbar is not None:
                    pbar.update(1)
        else:
            # view_msims hold full pyramids: reduce to create_preview()'s coarse sub-pyramid before fusing
            preview_scale = parse_scale(self.params['input_output'].get('preview_scale'),
                                        default=default_interactive_preview_scale)
            with phase(1 / 12, total=1) as pbar, \
                 Timer('_create_napari_data: build view_msims', verbose=self._timing_verbose()):
                view_msims = self.view_msims
                if transform_key == self.reg.reg_transform_key:
                    # a source downscaled for the view has its own msim, without the registered transform
                    copy_transforms_to_msims(self.reg.msims, view_msims, transform_key)
                if pbar is not None:
                    pbar.update(1)
            with phase(1 / 12, total=1) as pbar, \
                 Timer(f'_create_napari_data: select_msim_subpyramid_at_scale ({len(view_msims)} images)',
                      verbose=self._timing_verbose()):
                msims = select_msim_subpyramid_at_scale(view_msims, self.reg.sources, preview_scale)
                if flatfield:
                    msims = self.reg.flatfield_corrected_msims(msims, self.params.get('pre_processing', {}))
                if pbar is not None:
                    pbar.update(1)
        # sources at several heights are sized as if promoted to 3D, each at its own z as make_msims_3d() takes it,
        # without promoting them
        z_positions = None
        if len(set(position.get('z', 0) for position in self.reg.positions)) > 1:
            z_positions = [self.reg.positions[index].get('z', index) for index in range(len(msims))]
        # cap the combined fused size: pre-processed msims ignore preview_scale, which only picks a level per source
        # the cap also takes the bar share of the z promotion it replaces
        with phase((2 if z_positions is not None else 1) / 12, total=1) as pbar, \
             Timer('_create_napari_data: cap preview fusion size', verbose=self._timing_verbose()):
            msims = reduce_msims_to_fused_size(
                msims, transform_key, z_scale=self.reg._msim_z_scale,
                label=f'Preview fusion ({len(msims)} images)', z_positions=z_positions)
            if pbar is not None:
                pbar.update(1)
        with phase(1 / 12, total=1) as pbar, \
             Timer('_create_napari_data: copy_transforms_to_msims', verbose=self._timing_verbose()):
            # before registration there is nothing to copy, and reading self.reg.msims would force their build
            if show_preprocessed and writes_transforms:
                transform_msims = self.reg.msims
                if self.reg.register_indices is not None:
                    # pre-processing may drop sources (filter_foreground): pair each with its own source
                    transform_msims = [self.reg.msims[index] for index in self.reg.register_indices]
                copy_transforms_to_msims(transform_msims, msims, transform_key)
            if pbar is not None:
                pbar.update(1)
        # chunksize is left to fuse(), which derives it after its own z promotion; no step count, as
        # fuse() plans the whole graph in one call
        with phase(7 / 12) as pbar, \
             Timer(f'_create_napari_data: fuse ({len(msims)} images)', verbose=self._timing_verbose()):
            if pbar is not None:
                pbar.update(1)
            fused_msim, _ = self.reg.fuse(msims,
                                          transform_key=transform_key,
                                          fusion_method=fusion_method,
                                          dimension=self.params['input_output']['registration_dimension'],
                                          extra_metadata=self.extra_metadata,
                                          source_indices=self.reg.register_indices if show_preprocessed else None)
        # from a sample of the sources, as the overview: the fused coarsest level is too costly to compute for many
        ranges, levels = sample_contrast_limits([MsimLevels(msim) for msim in msims])
        dtype_limits = get_dtype_contrast_limits(get_msim_image0(fused_msim).dtype)
        fused_msim.attrs['contrast_limits'] = [display_range or dtype_limits for display_range in ranges]
        fused_msim.attrs['display_levels'] = levels
        return fused_msim

    def _view_background_luminance(self, viewer):
        """The luminance of the image the main viewer shows (background_luminance), None for another viewer or none."""
        if viewer is not getattr(self, 'viewer', None) or getattr(self, '_view_levels', None) is None:
            return None
        return background_luminance(*self._view_levels)

    def _update_view_add_shapes(self, viewer, shapes, refs, labels, face_colors, layer_name,
                                show_labels=True):
        # reads source metadata, not view_msims, so drawing shapes never forces the msim build
        # labels and edges stand out against the image under them, if this viewer shows one
        line_color = contrasting_color(self._view_background_luminance(viewer))
        bb_supported = True
        if isinstance(viewer, ViewerWidget):
            viewer = viewer._qtwidget._viewer_model
            bb_supported = False
        is_3d = (self.reg.sources[0].get_size().get('z', 0) > 1)
        is_multi_z_shapes = (len(set(position.get('z', 0) for position in self.reg.positions)) > 1)
        force_2d = not bb_supported or (is_multi_z_shapes and not is_3d)
        # shapes carry a 'z' for a native z-stack or for sources promoted to one
        do_3d = ((is_3d or is_multi_z_shapes) and not force_2d)

        if len(shapes) > 0:
            # depth-tested blending hides boxes inside the opaque fused volume
            blending = 'translucent_no_depth'
            edge_color = line_color
            if do_3d:
                # a polygon is one flat face, so a box is 6 quads plus a 'path' wireframe; corners 0-3 are
                # one face and 4-7 the opposite, same winding
                box_faces = [[0, 1, 2, 3], [4, 5, 6, 7], [0, 1, 5, 4],
                            [1, 2, 6, 5], [2, 3, 7, 6], [3, 0, 4, 7]]
                edge_path = [0, 1, 2, 3, 0, 4, 7, 3, 2, 6, 7, 4, 5, 6, 2, 1, 5]
                # _minimal_bb_vertices' corner order, applied to an axis-aligned min/max
                corner_bits = np.array([
                    [0, 0, 0], [1, 0, 0], [1, 1, 0], [0, 1, 0],
                    [0, 0, 1], [1, 0, 1], [1, 1, 1], [0, 1, 1],
                ])

                face_shapes, face_only_colors, face_refs, face_labels = [], [], [], []
                for shape, ref, color in zip(shapes, refs, face_colors):
                    corners = np.asarray(shape)
                    # napari fills only axis-orthogonal 3D faces (napari/napari#6860): fill the axis-aligned
                    # bounding box; the wireframe keeps the true oriented corners
                    mins, maxs = corners.min(axis=0), corners.max(axis=0)
                    aa_corners = mins + corner_bits * (maxs - mins)
                    centroid = aa_corners.mean(axis=0)
                    for face in box_faces:
                        quad = aa_corners[face]
                        # box_faces do not all wind outward: flip inward faces, or napari backface-culls them
                        normal = np.cross(quad[1] - quad[0], quad[2] - quad[0])
                        if np.dot(normal, quad.mean(axis=0) - centroid) < 0:
                            quad = quad[::-1]
                        face_shapes.append(quad)
                    face_only_colors += [color] * len(box_faces)
                    face_refs += [ref] * len(box_faces)
                    face_labels += [''] * len(box_faces)
                wire_shapes = [np.asarray(shape)[edge_path] for shape in shapes]

                # 3D path width is in world units, so scale it to the scene to stay visible
                vertices = np.concatenate([np.asarray(shape) for shape in shapes])
                wire_width = np.ptp(vertices, axis=0).max() * 0.005

                shape_data = face_shapes + wire_shapes
                shape_type = ['polygon'] * len(face_shapes) + ['path'] * len(wire_shapes)
                # color lists must match in length or napari falls back to white; a path's face color is a placeholder
                face_color = face_only_colors + [(0, 0, 0)] * len(wire_shapes)
                edge_color = ['#000000'] * len(face_shapes) + [line_color] * len(wire_shapes)
                edge_width = [0] * len(face_shapes) + [wire_width] * len(wire_shapes)
                refs = face_refs + refs
                labels = face_labels + labels
            else:
                shape_data = np.asarray(shapes)
                shape_type = 'polygon'
                edge_width = 0.1
                face_color = face_colors

            # half napari's default size, which reads oversized; the small overview's labels would be unreadable
            text = {'string': '{labels}', 'size': 6, 'color': line_color} if show_labels else None
            # 'labels' only feeds the text; 'refs' identifies the shapes either way
            features = {'refs': refs, 'labels': labels} if show_labels else {'refs': refs}
            layer = viewer.add_shapes(shape_data, name=layer_name, shape_type=shape_type, text=text,
                                      features=features, face_color=face_color, opacity=0.5,
                                      edge_width=edge_width, edge_color=edge_color, blending=blending,
                                      units=spatial_unit, metadata={'line_color': line_color})

            return layer
        return None

    def _napari_view_add_fused_data(self, viewer, fused, layer_name, cheap=False):
        # `fused` is one channel-combined multiscale msim, or in 'compose' mode a list of per-source msims;
        # cheap=True guesses contrast limits from the dtype instead of computing them
        channels = self.extra_metadata.get('channels', [])

        if isinstance(fused, list):
            # one layer per source, placed by its transform as the composed scene places it
            transform_key = self.get_best_transform_key()
            labels = self.reg.file_labels if len(self.reg.file_labels) == len(fused) else range(len(fused))
            colormap = (channels[0].get('color') if len(channels) == 1 else None) or (1, 1, 1, 1)

            def prep_layer(msim, label):
                image0 = get_msim_image0(msim)
                transform = msi_utils.get_transform_from_msim(msim, transform_key)
                transform = transform.sel(t=0) if 't' in transform.dims else transform
                return dict(data=get_msim_level_data(msim), name=str(label), multiscale=True, colormap=colormap,
                            contrast_limits=get_contrast_limits(msim, cheap=cheap),
                            scale=si_utils.get_spacing_from_sim(image0, asarray=True),
                            translate=si_utils.get_origin_from_sim(image0, asarray=True),
                            affine=np.asarray(transform), units=spatial_unit, blending='additive')

            with Timer(f'_napari_view_add_fused_data: prep {len(fused)} layers',
                      verbose=self._timing_verbose()):
                layer_kwargs = [prep_layer(msim, label) for msim, label in zip(fused, labels)]

            with Timer(f'_napari_view_add_fused_data: add_image loop ({len(fused)} layers)',
                      verbose=self._timing_verbose()):
                for kwargs in layer_kwargs:
                    data = kwargs.pop('data')
                    viewer.add_image(data, **kwargs)
            return

        image0 = get_msim_image0(fused)
        scale = si_utils.get_spacing_from_sim(image0, asarray=True)
        translate = si_utils.get_origin_from_sim(image0, asarray=True)
        data = get_msim_level_data(fused)
        with Timer('_napari_view_add_fused_data: get_contrast_limits', verbose=self._timing_verbose()):
            # the quick overview brings its own, from a sample of its sources
            channel_limits, levels = fused.attrs.get('contrast_limits'), fused.attrs.get('display_levels')
            if not channel_limits:
                channel_limits, levels = get_contrast_limits(fused, cheap=cheap, per_channel=True, with_levels=True)
        # the single-channel overview has c of size 1 even when the metadata lists more channels
        if len(channels) > 1 and image0.sizes.get('c', 1) == len(channels):
            channel_axis = image0.dims.index('c')
            name = [channel.get('label', index) for index, channel in enumerate(channels)]
            colormap = [channel.get('color', (1, 1, 1, 1)) for channel in channels]
            scale = [scale] * len(channels)
            translate = [translate] * len(channels)
            contrast_limits = (channel_limits if len(channel_limits) == len(channels)
                               else [channel_limits[0]] * len(channels))
        else:
            channel_axis = None
            name = channels[0].get('label') if len(channels) == 1 else None
            colormap = channels[0].get('color', (1, 1, 1, 1)) if len(channels) == 1 else None
            contrast_limits = channel_limits[0]
        if viewer is getattr(self, 'viewer', None):
            # the shapes drawn over it pick their colour from this (_update_view_add_shapes)
            colors = colormap if channel_axis is not None else [colormap or (1, 1, 1)]
            # a volume shows as dark as a slice and as bright as its 3D projection: neither level, the colour for both
            is_volume = self.reg.sources[0].get_size().get('z', 0) > 1 if self.reg.sources else False
            self._view_levels = (levels[:len(colors)] if levels and not is_volume else None, colors)
        with Timer('_napari_view_add_fused_data: add_image', verbose=self._timing_verbose()):
            viewer.add_image(data, name=name or layer_name, multiscale=True, channel_axis=channel_axis,
                             colormap=colormap, contrast_limits=contrast_limits,
                             scale=scale, translate=translate, units=spatial_unit)

    def _napari_view_show_features(self, viewer, fixed_data2, fixed_points, moving_data2, moving_points, matches, inliers,
                                   scale=None):
        layers = draw_keypoints_matches_napari(fixed_data2, fixed_points,
                                               moving_data2, moving_points,
                                               matches, inliers, points_color='blue',
                                               outline_color=contrasting_color(None))
        self._clear_napari_view(viewer)
        for data, kwargs, layer_type in layers:
            if scale is not None:
                kwargs = kwargs | {'scale': scale, 'units': spatial_unit}
            if layer_type == "image":
                viewer.add_image(data, **kwargs)
            elif layer_type == "points":
                viewer.add_points(data, **kwargs)
            elif layer_type == "shapes":
                viewer.add_shapes(data, **kwargs)

    def _napari_view_add_image(self, viewer, data, label, transform=None, color=None, affine_event=False):
        if isinstance(data, DataTree):
            # a multiscale msim: napari's affine (pair drag adjustment) works with its lazy level loading
            image0 = get_msim_image0(data)
            scale = si_utils.get_spacing_from_sim(image0, asarray=True)
            position = si_utils.get_origin_from_sim(image0, asarray=True)
            layer = viewer.add_image(get_msim_level_data(data), name=label, multiscale=True,
                                     scale=scale, translate=position, affine=transform, units=spatial_unit,
                                     blending='additive')
        else:
            scale = si_utils.get_spacing_from_sim(data, asarray=True)
            position = si_utils.get_origin_from_sim(data, asarray=True)
            layer = viewer.add_image(data, name=label, scale=scale, translate=position, affine=transform,
                                     units=spatial_unit, blending='additive')
        if color:
            layer.colormap = color

        if affine_event:
            layer.events.affine.connect(self.on_image_data_changed)

        return layer

    def on_image_data_changed(self, event):
        self.pair_metrics_timer.stop()
        self.pair_metrics_timer.start()

    def update_pair_metrics(self):
        if self.view_mode != ViewMode.PAIRS:
            return
        if self._pair_metrics_worker is not None:
            # one at a time: the latest position is measured once this one is done
            self._pair_metrics_pending = True
            return
        # read off the layers here, on the Qt thread: only the measuring runs on the worker
        pair_indices = self.pair_indices
        reg_msims = [self.reg.register_msims[index] for index in pair_indices]
        transforms = {(0, 1): self.calc_mod_pair_transform()}

        def measure():
            metrics = calc_msims_metrics(reg_msims, transforms, metric_methods=self.metrics_methods)
            remap_pair_metrics(metrics, pair_indices)
            return metrics

        def show(metrics):
            if self.view_mode == ViewMode.PAIRS and self.pair_indices == pair_indices:
                self.populate_metrics_table(metrics)

        def done():
            self._pair_metrics_worker = None
            if self._pair_metrics_pending:
                self._pair_metrics_pending = False
                self.update_pair_metrics()

        if QApplication is None or QApplication.instance() is None:
            show(measure())
            return
        # off the Qt thread, so the layers can still be dragged while it measures
        self._pair_metrics_worker = create_worker(
            measure,
            _start_thread=False,
            _connect={
                'returned': show,
                'errored': lambda error: show_warning(f'Pair metrics failed: {error}'),
                'warned': lambda warning: notification_manager.receive_warning(*warning),
                'finished': done,
            },
        )
        self._pair_metrics_worker.start()

    @catch_run_errors
    def run_preview_registration(self, progress_factory=None):
        label1 = self.param_widgets.get('registration.reg_preview_image1').get_value()
        label2 = self.param_widgets.get('registration.reg_preview_image2').get_value()
        index1 = self.reg.file_labels.index(label1)
        index2 = self.reg.file_labels.index(label2)

        pre_processing_params = self.params['pre_processing']
        # only the pair is pre-processed, unless a step needs every source; that gets its own bar, before this one
        if (not self.reg.register_msims_current(pre_processing_params)
                and self.reg.needs_full_preprocess(pre_processing_params)):
            if not self.run_pre_processing(progress_factory=progress_factory):
                return None

        with self._operation_progress('Preview registration', progress_factory, phases=2) as factory:
            def register_preview(worker_factory):
                # dask's task counting reports as a phase of this bar, not a bar of its own
                with NapariDaskProgress(progress_class=worker_factory,
                                        desc='Preview registration'):
                    registration_params = self.params['registration']
                    channel = registration_params.get('channel')
                    cache_key = (index1, index2, channel, self.reg.preprocess_settings(pre_processing_params))
                    cache = self._preview_overlap_cache
                    # the crop depends only on the pair, channel and pre-processing, so method changes reuse it
                    if cache is not None and cache['key'] == cache_key:
                        msim1, msim2 = cache['msims']
                        overlap1, overlap2, sims_pixel_space = cache['overlap1'], cache['overlap2'], cache['sims_pixel_space']
                    else:
                        if self.reg.register_msims_current(pre_processing_params):
                            msim1, msim2 = self.reg.register_msims[index1], self.reg.register_msims[index2]
                        else:
                            with Timer('preview_registration: pre-process the pair', verbose=self._timing_verbose()):
                                msim1, msim2 = self.reg.preprocess_sources((index1, index2), pre_processing_params)
                        overlap1, overlap2, sims_pixel_space = self.reg.select_pair_overlap(
                            msim1, msim2, params=registration_params)
                        overlap1, overlap2 = overlap1.compute(), overlap2.compute()
                        self._preview_overlap_cache = {
                            'key': cache_key, 'msims': (msim1, msim2),
                            'overlap1': overlap1, 'overlap2': overlap2, 'sims_pixel_space': sims_pixel_space,
                        }

                    with Timer('preview_registration: register the overlap', verbose=self._timing_verbose()):
                        transform, quality, results = self.reg.register_overlap(
                            overlap1, overlap2, sims_pixel_space, params=registration_params)

                    transforms = {(0, 1): transform}
                    qualities = {(0, 1): quality}
                    with Timer('preview_registration: pair metrics', verbose=self._timing_verbose()):
                        metrics = calc_msims_metrics((msim1, msim2), transforms, qualities,
                                                     metric_methods=self.metrics_methods)
                    remap_pair_metrics(metrics, (index1, index2))
                    return metrics, results, overlap1, overlap2, sims_pixel_space

            try:
                return self._run_off_thread(register_preview, factory)
            except NoOverlapError:
                show_warning(f'{label1} and {label2} do not overlap: choose two images that do')
                return None


    def preview_registration(self):
        # the view stays until there is a result to replace it with: an empty viewer's welcome screen hides the bar
        with Timer('preview_registration: register (and pre-process if needed)', verbose=self._timing_verbose()):
            result = self.run_preview_registration()
        if result is None:
            return
        metrics, results, overlap1, overlap2, sims_pixel_space = result

        with Timer('preview_registration: show metrics', verbose=self._timing_verbose()):
            self.populate_metrics_table(metrics)
            # metrics shown before the (slower) feature layers replace the view
            flush_paint_events()

        fixed_points = results.get('fixed_points', [])
        moving_points = results.get('moving_points', [])
        matches = results.get('matches', [])
        inliers = results.get('inliers', [])
        if len(fixed_points) == 0 and 'affine_matrix' in results:
            # a method giving only a transform (phase correlation, elastix): a grid shows its offsets
            fixed_points, moving_points, matches, inliers = grid_point_pairs(
                np.shape(overlap1), np.shape(overlap2), results['affine_matrix'])
        with Timer('preview_registration: show feature layers', verbose=self._timing_verbose()):
            self._detach_view(self.view_mode)
            # both crops are resampled to the fixed one's pixel size: shown at it, as the overview shows the sources
            scale = si_utils.get_spacing_from_sim(sims_pixel_space[0], asarray=True)
            self._napari_view_show_features(self.viewer, overlap1, fixed_points, overlap2, moving_points, matches,
                                            inliers, scale=scale)
        self.view_mode = ViewMode.FEATURES

    def populate_metrics_table(self, metrics_dict):
        transform_keys = []
        metric_keys = []
        item_keys = []
        metrics = metrics_dict.get('summary')
        if metrics:
            item_keys.append('summary')
            for transform_key, transform_value in metrics.items():
                for metric_key, metric_value in transform_value.items():
                    # a column only for values: without image metrics the source has no quality to show
                    if metric_value is not None and transform_key not in transform_keys:
                        transform_keys.append(transform_key)
                    if metric_value is not None and metric_key not in metric_keys:
                        metric_keys.append(metric_key)
        pairs_metrics = metrics_dict.get('pairs')
        if pairs_metrics:
            file_rank = {file_index: rank for rank, file_index in enumerate(self.source_order())}
            # each pair reads in table order too: its earlier source first
            pairs_metrics = dict(sorted(((tuple(sorted(indices, key=file_rank.get)), value)
                                         for indices, value in pairs_metrics.items()),
                                        key=lambda item: (file_rank[item[0][0]], file_rank[item[0][1]])))
        # a selected row picks its pair's preview images (metrics_table_selected)
        self.metrics_row_pairs = {self.reg.file_labels[indices[0]] + ' - ' + self.reg.file_labels[indices[1]]: indices
                                  for indices in (pairs_metrics or {})}
        # split pairing's group (e.g. section) pairs as rows after the tile pairs, keyed by their labels
        row_metrics = {row_label: pairs_metrics[indices] for row_label, indices in self.metrics_row_pairs.items()}
        row_metrics |= {' - '.join(labels): value for labels, value in metrics_dict.get('group_pairs', {}).items()}
        # a dict keeps the order without a quadratic `not in` over the rows
        item_keys = list(dict.fromkeys(item_keys + list(row_metrics)))
        for pair_value in row_metrics.values():
            for transform_key, transform_value in pair_value.items():
                for metric_key, metric_value in transform_value.items():
                    if metric_value is not None and transform_key not in transform_keys:
                        transform_keys.append(transform_key)
                    if metric_value is not None and metric_key not in metric_keys:
                        metric_keys.append(metric_key)

        transform_keys = [transform_key.split('_')[0] for transform_key in transform_keys]
        is_metric_cols = (len(transform_keys) <= 1 and len(metric_keys) >= 1)
        col_headers = metric_keys if is_metric_cols else transform_keys

        metrics_table = []
        for rowi in range(len(item_keys)):
            row = [None] * len(col_headers)
            metrics_table.append(row)
        item_offset = 0

        metrics = metrics_dict.get('summary')
        if metrics:
            item_offset = 1
            for transform_key, transform_value in metrics.items():
                for metric_key, metric_value in transform_value.items():
                    if metric_value is not None:
                        col_index = (metric_keys.index(metric_key) if is_metric_cols
                                     else transform_keys.index(transform_key.split('_')[0]))
                        metrics_table[0][col_index] = metric_value
        for pair_index, pair_value in enumerate(row_metrics.values()):
            for transform_key, transform_value in pair_value.items():
                for metric_key, metric_value in transform_value.items():
                    if metric_value is not None:
                        col_index = (metric_keys.index(metric_key) if is_metric_cols
                                     else transform_keys.index(transform_key.split('_')[0]))
                        metrics_table[pair_index + item_offset][col_index] = metric_value

        table_widget = self.param_widgets.get('registration.metrics_table')
        # Table: tuple-of-values : ([values], [row_headers], [column_headers])
        method = self.params.get('registration', {}).get('method')
        # transform columns hold one metric, whose range colours them all
        table_metric = metric_keys[0] if len(metric_keys) == 1 else None

        def background(value, column_label):
            metric = column_label if is_metric_cols else table_metric
            return QColor(*metric_to_rgb(value, value_range=metric_value_range(metric, method), max_light=0.5,
                                         output_range=255))

        table_widget.widget.native.set_background(background)
        table_widget.set_value((metrics_table, item_keys, col_headers))
        table_widget.set_table_column_resize_mode()

    def populate_registered_tables(self):
        with Timer('populate_registered_tables', verbose=self._timing_verbose()):
            # the coordinate systems from the state: reading them off the msims would build every one
            coord_systems = [self.reg.source_transform_key] + (
                [self.reg.reg_transform_key] if self.reg.is_global_registered() else [])
            self.populate_coordinate_systems(coord_systems)
            # the coordinate system still selected, so the table matches the dropdown
            self.populate_metadata_table(self.transform_key if self.transform_key in coord_systems else None)
            self.populate_metrics_table(self.reg.metrics)

    def update_registered(self, view_transform_key=None, progress_factory=None):
        self.populate_registered_tables()
        self.update_views(transform_key=view_transform_key, progress_factory=progress_factory)

    def enable_modify_pair_registration(self, enabled=True):
        widget = self.param_widgets.get('registration.modify_pair_registration')
        if widget:
            widget.widget.enabled = enabled

    def update_method_params_enabled(self):
        method_widget = self.param_widgets.get('registration.method')
        if method_widget is not None:
            method_kind = registration_method_kind(method_widget.get_value())
            for param_name, method_kinds in registration_method_params.items():
                widget = self.param_widgets.get(f'registration.{param_name}')
                if widget is not None:
                    widget.widget.enabled = method_kind in method_kinds

    def update_fusion_preview_enabled(self):
        widget = self.param_widgets.get('fusion.preview_fusion')
        if widget:
            # only a merge fuses at the unregistered source positions
            operation = getattr(self, 'params', {}).get('registration', {}).get('operation', '')
            widget.widget.enabled = self.reg.is_global_registered() or 'merge' in operation

    @catch_run_errors
    def run_pair_registration(self, progress_factory=None):
        # pre-processing reports its own bar, before the registration's; again if its settings changed since
        if not self.reg.register_msims_current(self.params['pre_processing']):
            if not self.run_pre_processing(progress_factory=progress_factory):
                return None

        # the split pairing groups sources by it (z-planes or channels)
        self.reg.registration_dimension = self.params.get('input_output', {}).get('registration_dimension')
        with self._operation_progress('Pair registration', progress_factory, phases=2) as factory:
            def register_pairs(worker_factory):
                # pairs and metrics report per batch: a dask callback would open a phase per batch
                with Timer('pair registration', verbose=self._timing_verbose()):
                    return self.reg.register_pairs(
                        self.reg.register_msims,
                        params=self.params['registration'] | {'metrics': self.metrics_methods},
                        progress_factory=worker_factory)

            results = self._run_off_thread(register_pairs, factory)

        self.reg.save_pair_results(results['pair_mappings'], metrics=results['metrics'])
        self.enable_modify_pair_registration()
        return results

    @catch_run_errors
    def run_global_registration(self, progress_factory=None):
        # register_global() reports its stages; declaring 8 units, half for the dominant optimisation,
        # keeps the last stages from each taking most of the bar
        self.reg.registration_dimension = self.params.get('input_output', {}).get('registration_dimension')
        if self.reg.pair_msims is None or self.reg.register_msims is None:
            # a loaded pair registration needs the pre-processed msims: at full resolution metrics can run out of memory
            if not self.run_pre_processing(progress_factory=progress_factory):
                return None
            self.reg.pair_msims = self.reg.register_msims
        with self._operation_progress('Global registration', progress_factory, phases=8) as factory:
            def register_global(worker_factory):
                with NapariDaskProgress(progress_class=worker_factory, desc='Global registration'), \
                        Timer('global registration', verbose=self._timing_verbose()):
                    return self.reg.register_global(self.reg.pair_msims,
                                                    register_indices=self.reg.register_indices,
                                                    params=self.params['registration']
                                                    | {'metrics': self.metrics_methods},
                                                    progress_factory=worker_factory)

            # the transform is written onto the sources before the metrics: a cancel restores it
            key = self.reg.reg_transform_key
            snapshots = [(msims, snapshot_msims_transform(msims, key)) for msims in (self.reg.pair_msims, self.reg.msims)]
            try:
                results = self._run_off_thread(register_global, factory)
            except OperationCancelled:
                for msims, snapshot in snapshots:
                    restore_msims_transform(msims, key, snapshot)
                raise

        with Timer('save registration results', verbose=self._timing_verbose()):
            self.reg.save_group_pair_mappings()
            self.reg.save_mappings(results['mappings'])
            self.reg.save_mappings_csv(results['mappings'])
            self.reg.save_metrics(results['metrics'])
            self.reg.save_residuals(results['residual_errors'])
            self.reg.write_ro_crates([], self.params)
        return results

    def pair_registration(self):
        if self.reg.is_global_registered():
            message = 'Global registration was already performed. '
        elif self.reg.is_pairs_registered():
            message = 'Pair registration was already performed. '
        else:
            message = ''
        message += 'Run pair registration?'
        reply = QMessageBox.question(None, 'muvis-align', message,
                                     QMessageBox.Yes|QMessageBox.No)
        if reply == QMessageBox.Yes:
            if not self.run_pair_registration():
                return
            self.update_registered(view_transform_key=self.reg.source_transform_key)
            self.update_fusion_preview_enabled()
            QMessageBox.information(None, 'muvis-align', 'Pair registration completed')

    def modify_pair_registration(self):
        if self.view_mode == ViewMode.PAIRS:
            reply = QMessageBox.question(None, 'muvis-align','Store modified registration?',
                                         QMessageBox.Yes|QMessageBox.No)
            if reply == QMessageBox.Yes:
                transform = self.calc_mod_pair_transform()
                pair_transforms = nx.get_edge_attributes(self.reg.pairs_graph, default_transform_key)
                qualities = nx.get_edge_attributes(self.reg.pairs_graph, default_quality_key)
                if 't' in pair_transforms[self.pair_indices].dims:
                    transform = transform.expand_dims({'t': [0]})
                pair_transforms[self.pair_indices] = transform
                qualities[self.pair_indices] = np.array(1)    # set quality to 1
                nx.set_edge_attributes(self.reg.pairs_graph, pair_transforms, default_transform_key)
                nx.set_edge_attributes(self.reg.pairs_graph, qualities, default_quality_key)
                self.reg.save_pair_results(pair_transforms, qualities=qualities)
                # the view and table colour a pair by its metrics, which would still hold the replaced quality
                pair_metrics = self.reg.metrics.setdefault('pairs', {}).setdefault(self.pair_indices, {})
                if not pair_metrics:
                    pair_metrics[default_transform_key] = {}
                for transform_metrics in pair_metrics.values():
                    transform_metrics[default_quality_key] = 1.0

            if self._restore_view():
                if reply == QMessageBox.Yes:
                    self._recolor_pair_shapes(self.pair_indices)
                    self.populate_metrics_table(self.reg.metrics)
            else:
                self.view_mode = ViewMode.OVERVIEW
                self.update_registered(view_transform_key=self.reg.source_transform_key)
            self._restore_pair_modify_state()
        else:
            previous_view_mode = self.view_mode
            self.view_mode = ViewMode.PAIRS
            labels = self.reg.file_labels
            label1 = self.param_widgets.get('registration.reg_preview_image1').get_value()
            label2 = self.param_widgets.get('registration.reg_preview_image2').get_value()
            index1 = labels.index(label1)
            index2 = labels.index(label2)
            indices = index1, index2
            colors = [(0, 1, 0), (1, 0, 1)]     # green, purple
            pair_transforms = (nx.get_edge_attributes(self.reg.pairs_graph, default_transform_key)
                               if self.reg.is_pairs_registered() else {})
            if indices not in pair_transforms and tuple(reversed(indices)) in pair_transforms:
                indices = tuple(reversed(indices))

            if indices not in pair_transforms:
                self.view_mode = previous_view_mode
                if self.reg.is_pairs_registered():
                    show_warning(f'No pair registration for {label1} and {label2}: choose a registered pair')
                else:
                    show_warning('Run pair registration first: a preview registration is not stored')
            else:
                self.temp_widget_state = TemporarilyDisabledWidgets()
                self.temp_tab_states = {}
                all_widgets = self.get_all_widgets()
                all_widgets.pop('registration.modify_pair_registration', None)
                self.temp_widget_state.disable(all_widgets)
                if self.enable_tab and self.is_tab_enabled:
                    other_section_ids = [section_id for section_id in ['project'] + list(self.template.keys())
                                        if section_id != 'registration']
                    self.temp_tab_states = {section_id: self.is_tab_enabled(section_id)
                                            for section_id in other_section_ids}
                    for section_id in other_section_ids:
                        self.enable_tab(section_id, False)
                self.pair_indices = indices
                pair_transform = np.array(pair_transforms[indices].sel(t=0))

                # the widgets disabled above are only re-enabled on leaving this mode, so a failure must restore them
                entered = False
                try:
                    if not self.reg.register_msims:
                        if not self.run_pre_processing():
                            return
                    self._detach_view(previous_view_mode)
                    # a multiscale pyramid, so napari loads only the level it needs while dragging
                    register_images = self.reg.register_msims
                    # each source keeps its own transform (e.g. scan rotation); the pair transform maps fixed to moving
                    self.pair_source_affines = [
                        np.asarray(msi_utils.get_transform_from_msim(register_images[sim_index],
                                                                     self.reg.source_transform_key)).squeeze()
                        for sim_index in indices]
                    fixed_source_affine, moving_source_affine = self.pair_source_affines
                    layer_affines = pair_transform @ fixed_source_affine, moving_source_affine
                    for sim_index, layer_affine, color in zip(indices, layer_affines, colors):
                        self._napari_view_add_image(self.viewer, register_images[sim_index], labels[sim_index],
                                                    layer_affine, color, affine_event=True)
                    self.update_pair_metrics()
                    entered = True
                finally:
                    if not entered:
                        if not self._restore_view():
                            self.view_mode = previous_view_mode
                        self._restore_pair_modify_state()

    def _restore_pair_modify_state(self):
        """Re-enable whatever entering pair-modify mode disabled."""
        self.temp_widget_state.restore()
        if self.enable_tab:
            for section_id, was_enabled in self.temp_tab_states.items():
                self.enable_tab(section_id, was_enabled)

    def calc_mod_pair_transform(self):
        fixed_source_affine, moving_source_affine = self.pair_source_affines
        matsize = len(fixed_source_affine)
        fixed_affine, moving_affine = [layer.affine.affine_matrix[-matsize:, -matsize:] for layer in self.viewer.layers]
        # either layer may have been moved: the pair transform is what is left once both source transforms are taken out
        transform = (moving_source_affine @ calculate_rigid_difference(moving_affine, fixed_affine)
                     @ np.linalg.inv(fixed_source_affine))
        return param_utils.affine_to_xaffine(transform)

    def registration_process(self):
        self.prepare_action()
        if 'convert' in self.params['registration']['operation']:
            # each source written at its own position, without registration or fusion
            reply = QMessageBox.question(None, 'muvis-align', 'Convert data to OME-Zarr only?',
                                         QMessageBox.Yes|QMessageBox.No)
            if reply == QMessageBox.Yes:
                if self.run_convert():
                    QMessageBox.information(None, 'muvis-align', 'Conversion completed')
            return

        if 'merge' in self.params['registration']['operation']:
            # merge fuses at the sources' own positions (get_best_transform_key()): nothing to register
            self.select_tab(4)
            return

        completion_message = 'Global registration completed'
        if self.reg.is_global_registered():
            message = 'Global registration was already performed. Run global registration?'
        elif not self.reg.is_pairs_registered():
            message = 'Pair registration not performed yet. Run both pair and global registration?'
            completion_message = 'Registration completed'
        else:
            message = 'Run global registration?'
        reply = QMessageBox.question(None, 'muvis-align', message,
                                     QMessageBox.Yes|QMessageBox.No)
        if reply == QMessageBox.Yes:
            if not self._run_registration():
                return
            self.refresh_registered_view()
            QMessageBox.information(None, 'muvis-align', completion_message)

    def refresh_registered_view(self):
        # from source geometry and the mappings: view msims are built only if a fused preview needs them
        with self._operation_progress('Refreshing view', phases=2) as view_factory:
            self.update_registered(view_transform_key=self.reg.reg_transform_key, progress_factory=view_factory)

    def _run_registration(self):
        """Pair registration where it has not run yet, then global; False if either did not complete."""
        # a bar per registration: in one shared bar the pair phases would fill it
        with Timer('registration process', verbose=self._timing_verbose()):
            registered = ((self.reg.is_pairs_registered() or self.run_pair_registration())
                          and self.run_global_registration())
        self.update_fusion_preview_enabled()
        return bool(registered)

    @catch_run_errors
    def _show_fused_output(self, path, progress_factory=None):
        """Show the fused export on disk, read lazily at every exported level, without fusing again."""
        transform_key = self.get_best_transform_key()
        with progress_phase(progress_factory, total=1, desc='Reading fused output') as pbar, \
             Timer('show fused output', verbose=self._timing_verbose()):
            scene = read_ome_zarr_scene(path)
            if scene:
                # a layer per source, each its own pyramid at full resolution: nothing combined to view it
                fused = read_composed_msims(path, scene, transform_key)
            else:
                fused = ngff_utils.read_msim_from_ome_zarr(path, transform_key=transform_key, array_backend='dask')
            self._show_view_image(self.viewer, fused, 'Fused')
            self.view_mode = ViewMode.FUSED
            if pbar is not None:
                pbar.update(1)
        logging.info(f'Fused output shown from {path}')
        self._refresh_view_shapes(transform_key, progress_factory=progress_factory)

    def preview_fusion(self, progress_factory=None):
        # a merge has no registered transform, so this falls back to the source positions
        transform_key = self.get_best_transform_key()
        # building the data dominates, so it is weighted as such and reports from the inside
        fusion_weight = 12
        with self._operation_progress('Fusion preview', progress_factory,
                                      phases=fusion_weight + 2 + self.shapes_weight) as factory:
            def fuse(worker_factory):
                with NapariMVSProgress(tqdm_class=worker_factory.tqdm_class, patch_fusion=True):
                    return self._create_napari_data(
                        transform_key, fusion_method=self.params['fusion']['method'],
                        progress_factory=worker_factory, weight=fusion_weight, flatfield=True)

            data = self._run_off_thread(fuse, factory)
            with factory(total=1) as pbar:
                self._show_view_image(self.viewer, data, f'{self.reg.fileset_label} data')
                self.view_mode = ViewMode.FUSED
                pbar.update(1)
            # this path skips update_views(), so the shapes may be of another geometry or not drawn yet
            self._refresh_view_shapes(transform_key, progress_factory=factory)

    @catch_run_errors
    def run_convert(self):
        operation = self.params['registration']['operation']
        output_folder = operation_to_past_participle(operation)
        ome_version = self.params['fusion']['ome_version']
        # self.reg.msims stays at source resolution; the scaled pyramid is register_msims, with
        # every level (pre-processing otherwise keeps only the ends)
        if not self.run_pre_processing(full_pyramid=True):
            return False
        msims = self.reg.register_msims
        # foreground filtering may drop sources, so labels/positions follow register_indices
        labels = [get_filetitle(self.reg.filenames[index]) for index in self.reg.register_indices]
        positions = [self.reg.positions[index] for index in self.reg.register_indices]
        with self._operation_progress('Converting') as progress_factory, \
             Timer('convert', verbose=self._timing_verbose()):
            # not fuse(): it would build one shared canvas; each source keeps its native levels
            # sequential: concurrent zarr writes race on zarr.json (Windows PermissionError)
            with progress_factory(total=len(msims), desc='Converting') as pbar:
                for label, position, msim in zip(labels, positions, msims):
                    output_filename = f'{output_folder}/{label}'
                    self.reg.save_native_levels(output_filename, msim, position=position,
                                                ome_version=ome_version)
                    pbar.update(1)
        return True

    def _estimate_fusion_size(self):
        """reg.estimate_fusion_size() for the fusion settings, logged per level; None if it cannot be estimated."""
        try:
            with Timer('fusion size estimate', verbose=self._timing_verbose()):
                estimate = self.reg.estimate_fusion_size(
                    self.params['fusion']['method'], output_spacing=self.params['fusion']['spacing'],
                    transform_key=self.get_best_transform_key(),
                    dimension=self.params['input_output']['registration_dimension'],
                    block_size=parse_block_size(self.params['fusion'].get('block_size')),
                    extra_metadata=self.extra_metadata)
        except Exception as error:
            # the question still stands without a size, and the bar without its level shares
            logging.warning(f'Could not estimate the fusion size: {error}')
            return None
        if estimate is not None:
            logging.info(f'Fusion size estimate: {print_hbytes(estimate["bytes"])} uncompressed, per level '
                         + ', '.join(f'{level["spacing"]:.4g}: {print_hbytes(level["bytes"])}'
                                     for level in estimate['levels']))
        return estimate

    def _fusion_size_text(self):
        """' Estimated output: ...' for the export question; empty if it cannot be estimated. The estimate is kept for
        the fusion that follows, to share its bar out over the levels."""
        self._fusion_estimate = self._estimate_fusion_size()
        if self._fusion_estimate is None:
            return ''
        return f'\nEstimated output: {print_hbytes(self._fusion_estimate["bytes"])}'

    @catch_run_errors
    def run_fusion(self, progress_factory=None):
        operation = self.params['registration']['operation']
        output_filename = operation_to_past_participle(operation)
        block_size = parse_block_size(self.params['fusion'].get('block_size'))
        # the saves below are on-disk layout only, with no per-block cost, so they always get blocks
        save_block_size = block_size or default_chunk_size

        fuses_to_zarr = self.reg.fuses_to_zarr(self.params['fusion']['method'],
                                               self.params['input_output']['registration_dimension'],
                                               self.extra_metadata)
        # used once: settings or registration may change before another fusion
        estimate, self._fusion_estimate = getattr(self, '_fusion_estimate', None), None
        nested = (progress_factory or getattr(self, '_running_operation', None)) is not None
        pre_processing = self.params.get('pre_processing', {})
        # building the sources and their flat-field model report a phase each, when they run; the export, the
        # fusion written to zarr or the save computing a lazy fusion, is the last and takes the rest
        early_phases = int(self.reg.msims_build_pending()) + int(self.reg.flatfield_pending(pre_processing))
        with self._operation_progress('Fusion', progress_factory, phases=early_phases + 1) as factory:
            if not nested:
                # the export's phase fills the bar, not the close after it
                factory.last_phase_share = 1.0
            def fuse(worker_factory):
                level_estimate = estimate if estimate is not None or not fuses_to_zarr else self._estimate_fusion_size()
                msims = self.reg.flatfield_corrected_msims(self.reg.ensure_msims(progress_factory=worker_factory),
                                                           pre_processing, progress_factory=worker_factory)
                with NapariMVSProgress(tqdm_class=worker_factory.tqdm_class, patch_fusion=True), \
                        Timer('fusion', verbose=self._timing_verbose()):
                    fused_image, is_saved = self.reg.fuse(
                        msims,
                        fusion_method=self.params['fusion']['method'],
                        # fuse()'s default reg_transform_key does not exist for a merge
                        transform_key=self.get_best_transform_key(),
                        output_spacing=self.params['fusion']['spacing'],
                        dimension=self.params['input_output']['registration_dimension'],
                        output_filename=output_filename,
                        block_size=block_size,
                        ome_version=self.params['fusion']['ome_version'],
                        extra_metadata=self.extra_metadata,
                        level_bytes=[level['bytes'] for level in level_estimate['levels']] if level_estimate else None)
                    if not is_saved:
                        # save() takes a single-resolution sim: the pyramid's finest scale
                        save_sim = extract_sims_from_fused(fused_image)
                        self.reg.save(output_filename, save_sim,
                                      transform_key=self.get_best_transform_key(),
                                      translations0=self.reg.positions,
                                      channels=self.extra_metadata.get('channels', []),
                                      block_size=save_block_size,
                                      ome_version=self.params['fusion']['ome_version'],
                                      progress_factory=worker_factory)
                    image_filenames = [path[len(self.reg.output):] for path in self._output_paths(output_filename)
                                       if os.path.exists(path)]
                    self.reg.write_ro_crates(image_filenames, self.params)
                    return fused_image

            try:
                return self._run_off_thread(fuse, factory)
            except OperationCancelled:
                self._remove_output(output_filename)
                raise

    def _remove_output(self, output_filename, output=None):
        # a cancelled export's partly written file or store, or one fused from a discarded registration
        for path in self._output_paths(output_filename, output):
            if os.path.isdir(path):
                shutil.rmtree(path, ignore_errors=True)
            elif os.path.exists(path):
                os.remove(path)
            if os.path.exists(path):
                logging.warning(f'Could not remove the output {path}')

    def _output_paths(self, output_filename, output=None):
        output = os.path.join(output, '') if output else self.reg.output
        return [output + output_filename + extension for extension in (zarr_extension, tiff_extension)]


    def fusion_process(self):
        self.prepare_action()
        needs_registration = (not self.reg.is_global_registered()
                              and 'merge' not in self.params['registration']['operation'])
        # a channel overlay fuses each channel even in compose mode
        is_composed = ('compos' in (self.params['fusion']['method'] or '')
                       and not self.reg.is_channel_overlay(self.params['input_output']['registration_dimension'],
                                                           self.extra_metadata))
        export = 'export composed data' if is_composed else 'export fused data'
        if self.reg.is_fused():
            message = f'{"Composition" if is_composed else "Fusion"} was already performed. {export.capitalize()}?'
        elif needs_registration and self.reg.is_pairs_registered():
            message = f'Global registration not performed yet. Run global registration and {export}?'
        elif needs_registration:
            message = f'Registration not performed yet. Run pair and global registration, and {export}?'
        else:
            message = f'{export.capitalize()}?'
        reply = QMessageBox.question(None, 'muvis-align', message + self._fusion_size_text(),
                                     QMessageBox.Yes | QMessageBox.No)
        if reply == QMessageBox.Yes:
            if needs_registration:
                # estimated from the transforms before this registration
                self._fusion_estimate = None
                if not self._run_registration():
                    return
            fused_image = self.run_fusion()
            if fused_image is None:
                return
            # the export reported its own bar; drawing the result reports a second one
            with self._operation_progress('Refreshing view', phases=2 + self.shapes_weight) as factory:
                with factory(total=1):
                    if isinstance(fused_image, list):
                        # read back from the file, as reloading it shows it: the file adds its own coarse levels
                        composed_path = self._output_paths(
                            operation_to_past_participle(self.params['registration']['operation']))[0]
                        transform_key = self.get_best_transform_key()
                        fused_image = read_composed_msims(composed_path, read_ome_zarr_scene(composed_path),
                                                          transform_key)
                    self._show_view_image(self.viewer, fused_image, 'Fused')
                # registration may have run straight into the fusion, with no view of its geometry
                self._refresh_view_shapes(self.get_best_transform_key(), progress_factory=factory)
            self.reg.state = RegState.FUSED
            self.view_mode = ViewMode.FUSED
            QMessageBox.information(None, 'muvis-align', 'Fusion completed')
enable_plugin_widget = enable_plugin_widget instance-attribute
enable_tab = enable_tab instance-attribute
enable_tabs = enable_tabs instance-attribute
is_tab_enabled = is_tab_enabled instance-attribute
metrics_methods = ['ncc', 'ssim', 'onmi'] instance-attribute
need_source_reinit = False instance-attribute
overview = overview instance-attribute
pair_metrics_timer = QTimer() instance-attribute
param_widgets = {} instance-attribute
params = {} instance-attribute
pre_processing_performed = False instance-attribute
raw_template = get_project_template() instance-attribute
reg = MVSRegistration() instance-attribute
select_tab = select_tab instance-attribute
shapes_weight = 3 class-attribute instance-attribute
template = get_section_dict(self.raw_template, ['inputs', 'parameters', 'display_only', 'outputs']) instance-attribute
transform_key = 'source_metadata' instance-attribute
verbose = verbose instance-attribute
view_msims property writable
viewer = viewer instance-attribute
__init__(viewer, overview, enable_tabs=None, select_tab=None, is_tab_enabled=None, enable_tab=None, enable_plugin_widget=None, verbose=False, initialize=True)
Source code in src\muvis_align\ui\Interface.py
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
def __init__(self, viewer, overview, enable_tabs=None, select_tab=None, is_tab_enabled=None,
             enable_tab=None, enable_plugin_widget=None, verbose=False, initialize=True):
    self.viewer = viewer
    self.overview = overview
    self.enable_tabs = enable_tabs
    self.select_tab = select_tab
    self.is_tab_enabled = is_tab_enabled
    self.enable_tab = enable_tab
    self.enable_plugin_widget = enable_plugin_widget
    self.verbose = verbose
    if _available_memory_bytes:
        # napari sizes its dask cache from the machine's memory: on a cluster node, not the job's
        resize_dask_cache(nbytes=int(_available_memory_bytes * napari_dask_cache_share))
    self.raw_template = get_project_template()
    if not self.raw_template:
        raise FileNotFoundError('Project template not found')
    self.template = get_section_dict(self.raw_template, ['inputs', 'parameters', 'display_only', 'outputs'])
    self.param_widgets = {}
    self.params = {}
    self.pre_processing_performed = False
    self.metrics_methods = ['ncc', 'ssim', 'onmi']
    self.transform_key = 'source_metadata'
    self.need_source_reinit = False

    self.pair_metrics_timer = QTimer()
    self.pair_metrics_timer.setSingleShot(True)
    self.pair_metrics_timer.setInterval(1000)
    self.pair_metrics_timer.timeout.connect(self.update_pair_metrics)
    self._pair_metrics_worker = None
    self._pair_metrics_pending = False
    self._fusion_estimate = None
    # per viewer (by id): the shape layer drawn and the signature of what it was drawn from
    self._shape_layers = {}
    self._detached_view = None
    # on the viewers, not the shape layers: a layer's mouse callbacks fire only while it is the active layer
    on_click = _weak_view_click(weakref.ref(self))
    for view in (viewer, overview):
        view_model = view._qtwidget._viewer_model if isinstance(view, ViewerWidget) else view
        if view_model is not None:
            view_model.mouse_drag_callbacks.append(on_click)

    self.reg = MVSRegistration()
    if initialize:
        self.reset()
calc_mod_pair_transform()
Source code in src\muvis_align\ui\Interface.py
1993
1994
1995
1996
1997
1998
1999
2000
def calc_mod_pair_transform(self):
    fixed_source_affine, moving_source_affine = self.pair_source_affines
    matsize = len(fixed_source_affine)
    fixed_affine, moving_affine = [layer.affine.affine_matrix[-matsize:, -matsize:] for layer in self.viewer.layers]
    # either layer may have been moved: the pair transform is what is left once both source transforms are taken out
    transform = (moving_source_affine @ calculate_rigid_difference(moving_affine, fixed_affine)
                 @ np.linalg.inv(fixed_source_affine))
    return param_utils.affine_to_xaffine(transform)
cancel_operation(desc=None)
Source code in src\muvis_align\ui\Interface.py
654
655
656
657
658
659
660
661
def cancel_operation(self, desc=None):
    # the running operation stops at its next progress step, on its worker thread
    reply = QMessageBox.question(None, 'muvis-align',
                                 f'Cancel {desc or "the running operation"}? Its partial result is discarded.',
                                 QMessageBox.Yes | QMessageBox.No)
    if reply == QMessageBox.Yes:
        logging.info('Cancel requested')
        request_cancel()
change_param(param_name, value)
Source code in src\muvis_align\ui\Interface.py
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
def change_param(self, param_name, value):
    keys = param_name.split('.')
    if keys[0] not in self.params:
        self.params[keys[0]] = {}
    if isinstance(value, list):
        # stored relative, as a single string for one path to keep simple projects unchanged
        value = [relativize_to_project_dir(item, self.get_project_dir()) for item in value]
        value = value[0] if len(value) == 1 else value
    elif isinstance(value, str):
        value = value.replace('\\', '/')
        if param_name in ('input_output.input_path', 'input_output.output_path'):
            # the file dialog reports an absolute path: store it relative, keeping the project portable
            value = relativize_to_project_dir(value, self.get_project_dir())
    self.params[keys[0]][keys[1]] = value
    self.write_params()
    same_setting = [param_widget for param_widget in self.param_widgets.values()
                    if param_widget.param_name == param_name]
    if len(same_setting) > 1:
        # the same setting shown on another tab follows, without reporting the change again
        for param_widget in same_setting:
            if param_widget.get_value() != value:
                with param_widget.widget.changed.blocked():
                    param_widget.set_value(value)
    if param_name == 'registration.operation':
        self.update_fusion_preview_enabled()
    if param_name == 'registration.method':
        self.update_method_params_enabled()
channel_color_changed(rowi, color)
Source code in src\muvis_align\ui\Interface.py
943
944
945
946
947
def channel_color_changed(self, rowi, color):
    table = self.param_widgets.get('input_output.channels_table').widget
    color_coli = table.column_headers.index('color')
    # written into the table so its 'changed' wiring saves it, as a manual edit would
    table.data[rowi, color_coli] = str(tuple(color))
channel_positions(positions)

Copies of positions, with each source's channel group as 'c' in a channel overlay.

Source code in src\muvis_align\ui\Interface.py
889
890
891
892
893
894
895
896
def channel_positions(self, positions):
    """Copies of positions, with each source's channel group as 'c' in a channel overlay."""
    positions = [dict(position) for position in positions]
    if self.reg.is_channel_overlay(self.params.get('input_output', {}).get('registration_dimension'),
                                  self.extra_metadata):
        for position, channel in zip(positions, self.reg.channel_groups()):
            position['c'] = channel
    return positions
channels_table(value)
Source code in src\muvis_align\ui\Interface.py
416
417
418
def channels_table(self, value):
    old_value = self.param_widgets.get('input_output.channels_table').get_value()
    self.set_channels(update_dict_value(old_value, value))
check_3d_view()
Source code in src\muvis_align\ui\Interface.py
1010
1011
1012
1013
def check_3d_view(self):
    is_3d = (self.reg.sources[0].get_size().get('z', 0) > 1)
    ndisplay = 3 if is_3d else 2
    self.viewer.dims.ndisplay = ndisplay
coordinate_system(transform_key)
Source code in src\muvis_align\ui\Interface.py
851
852
853
854
def coordinate_system(self, transform_key):
    self.transform_key = transform_key
    if self.reg.is_initialised():
        self.populate_metadata_table(transform_key)
copy_params_to_output()

The project file next to its output: the settings the output was made with.

Source code in src\muvis_align\ui\Interface.py
279
280
281
282
283
284
285
def copy_params_to_output(self):
    """The project file next to its output: the settings the output was made with."""
    target = self.get_output_params_path()
    if target:
        os.makedirs(os.path.dirname(target), exist_ok=True)
        shutil.copyfile(self.params_path, target)
        logging.info(f'Project settings copied to {target}')
enable_modify_pair_registration(enabled=True)
Source code in src\muvis_align\ui\Interface.py
1791
1792
1793
1794
def enable_modify_pair_registration(self, enabled=True):
    widget = self.param_widgets.get('registration.modify_pair_registration')
    if widget:
        widget.widget.enabled = enabled
ensure_view_msims(progress_factory=None)
Source code in src\muvis_align\ui\Interface.py
756
757
758
759
760
761
762
def ensure_view_msims(self, progress_factory=None):
    # the view_msims build, callable ahead with a progress_factory; kept 2D, as consumers place
    # sources at their z (promoting every level of every source is slow)
    if self._view_msims is None:
        with Timer('build view msims', verbose=self._timing_verbose()):
            self._view_msims = self._build_view_msims(progress_factory=progress_factory)
    return self._view_msims
fusion_process()
Source code in src\muvis_align\ui\Interface.py
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
def fusion_process(self):
    self.prepare_action()
    needs_registration = (not self.reg.is_global_registered()
                          and 'merge' not in self.params['registration']['operation'])
    # a channel overlay fuses each channel even in compose mode
    is_composed = ('compos' in (self.params['fusion']['method'] or '')
                   and not self.reg.is_channel_overlay(self.params['input_output']['registration_dimension'],
                                                       self.extra_metadata))
    export = 'export composed data' if is_composed else 'export fused data'
    if self.reg.is_fused():
        message = f'{"Composition" if is_composed else "Fusion"} was already performed. {export.capitalize()}?'
    elif needs_registration and self.reg.is_pairs_registered():
        message = f'Global registration not performed yet. Run global registration and {export}?'
    elif needs_registration:
        message = f'Registration not performed yet. Run pair and global registration, and {export}?'
    else:
        message = f'{export.capitalize()}?'
    reply = QMessageBox.question(None, 'muvis-align', message + self._fusion_size_text(),
                                 QMessageBox.Yes | QMessageBox.No)
    if reply == QMessageBox.Yes:
        if needs_registration:
            # estimated from the transforms before this registration
            self._fusion_estimate = None
            if not self._run_registration():
                return
        fused_image = self.run_fusion()
        if fused_image is None:
            return
        # the export reported its own bar; drawing the result reports a second one
        with self._operation_progress('Refreshing view', phases=2 + self.shapes_weight) as factory:
            with factory(total=1):
                if isinstance(fused_image, list):
                    # read back from the file, as reloading it shows it: the file adds its own coarse levels
                    composed_path = self._output_paths(
                        operation_to_past_participle(self.params['registration']['operation']))[0]
                    transform_key = self.get_best_transform_key()
                    fused_image = read_composed_msims(composed_path, read_ome_zarr_scene(composed_path),
                                                      transform_key)
                self._show_view_image(self.viewer, fused_image, 'Fused')
            # registration may have run straight into the fusion, with no view of its geometry
            self._refresh_view_shapes(self.get_best_transform_key(), progress_factory=factory)
        self.reg.state = RegState.FUSED
        self.view_mode = ViewMode.FUSED
        QMessageBox.information(None, 'muvis-align', 'Fusion completed')
get_all_widgets()
Source code in src\muvis_align\ui\Interface.py
167
168
169
170
171
def get_all_widgets(self):
    # excludes a disabled tab's widgets: their .enabled reads False from the tab, so restoring
    # that state would keep them disabled once the tab is enabled again
    return {name: param_widget.widget for name, param_widget in self.param_widgets.items()
           if self.is_tab_enabled is None or self.is_tab_enabled(name.split('.', 1)[0])}
get_best_transform_key()
Source code in src\muvis_align\ui\Interface.py
1006
1007
1008
def get_best_transform_key(self):
    # from the state, not off built msims: a loaded registration's are built only when needed
    return self.reg.reg_transform_key if self.reg.is_global_registered() else self.reg.source_transform_key
get_function(function_label)
Source code in src\muvis_align\ui\Interface.py
173
174
175
176
177
def get_function(self, function_label):
    if hasattr(self, function_label):
        return eval(f'self.{function_label}')
    else:
        return None
get_output_dir()
Source code in src\muvis_align\ui\Interface.py
257
258
259
def get_output_dir(self):
    return resolve_to_project_dir(path_param_to_text(self.params['input_output'].get('output_path', '')),
                                  self.get_project_dir())
get_output_params_path()

The copy of the project file next to its output, or None for a project kept in its own output.

Source code in src\muvis_align\ui\Interface.py
261
262
263
264
265
266
267
def get_output_params_path(self):
    """The copy of the project file next to its output, or None for a project kept in its own output."""
    params_path = getattr(self, 'params_path', None)
    if not params_path or not os.path.exists(params_path):
        return None
    target = os.path.join(self.get_output_dir(), os.path.basename(params_path))
    return target if os.path.abspath(target) != os.path.abspath(params_path) else None
get_project_dir()
Source code in src\muvis_align\ui\Interface.py
210
211
212
213
def get_project_dir(self):
    # path params are stored relative to this directory, so a project moves with its data
    params_path = getattr(self, 'params_path', None)
    return os.path.dirname(os.path.abspath(params_path)) if params_path else None
init_progress()
Source code in src\muvis_align\ui\Interface.py
663
664
665
666
667
def init_progress(self):
    # the two halves of opening a project, each its own bar, one after the other
    with self._operation_progress('Initialising sources', phases=3) as progress_factory:
        self._load_saved_progress(progress_factory)
    self._show_loaded_project()
input_output_process()
Source code in src\muvis_align\ui\Interface.py
423
424
425
426
427
428
429
def input_output_process(self):
    try:
        if not self.prepare_action():
            self._input_output_process()
    except OperationCancelled:
        logging.info('input_output_process cancelled')
        show_info('Cancelled')
input_path(value)
Source code in src\muvis_align\ui\Interface.py
375
376
def input_path(self, value):
    self.need_source_reinit = True
metrics_table_selected()
Source code in src\muvis_align\ui\Interface.py
994
995
996
997
998
999
def metrics_table_selected(self):
    table = self.param_widgets.get('registration.metrics_table').widget.native
    # the summary and group (section) pair rows have no image pair
    indices = getattr(self, 'metrics_row_pairs', {}).get(table.row_label(table.currentRow()))
    if indices:
        self.set_preview_images(indices)
modify_pair_registration()
Source code in src\muvis_align\ui\Interface.py
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
def modify_pair_registration(self):
    if self.view_mode == ViewMode.PAIRS:
        reply = QMessageBox.question(None, 'muvis-align','Store modified registration?',
                                     QMessageBox.Yes|QMessageBox.No)
        if reply == QMessageBox.Yes:
            transform = self.calc_mod_pair_transform()
            pair_transforms = nx.get_edge_attributes(self.reg.pairs_graph, default_transform_key)
            qualities = nx.get_edge_attributes(self.reg.pairs_graph, default_quality_key)
            if 't' in pair_transforms[self.pair_indices].dims:
                transform = transform.expand_dims({'t': [0]})
            pair_transforms[self.pair_indices] = transform
            qualities[self.pair_indices] = np.array(1)    # set quality to 1
            nx.set_edge_attributes(self.reg.pairs_graph, pair_transforms, default_transform_key)
            nx.set_edge_attributes(self.reg.pairs_graph, qualities, default_quality_key)
            self.reg.save_pair_results(pair_transforms, qualities=qualities)
            # the view and table colour a pair by its metrics, which would still hold the replaced quality
            pair_metrics = self.reg.metrics.setdefault('pairs', {}).setdefault(self.pair_indices, {})
            if not pair_metrics:
                pair_metrics[default_transform_key] = {}
            for transform_metrics in pair_metrics.values():
                transform_metrics[default_quality_key] = 1.0

        if self._restore_view():
            if reply == QMessageBox.Yes:
                self._recolor_pair_shapes(self.pair_indices)
                self.populate_metrics_table(self.reg.metrics)
        else:
            self.view_mode = ViewMode.OVERVIEW
            self.update_registered(view_transform_key=self.reg.source_transform_key)
        self._restore_pair_modify_state()
    else:
        previous_view_mode = self.view_mode
        self.view_mode = ViewMode.PAIRS
        labels = self.reg.file_labels
        label1 = self.param_widgets.get('registration.reg_preview_image1').get_value()
        label2 = self.param_widgets.get('registration.reg_preview_image2').get_value()
        index1 = labels.index(label1)
        index2 = labels.index(label2)
        indices = index1, index2
        colors = [(0, 1, 0), (1, 0, 1)]     # green, purple
        pair_transforms = (nx.get_edge_attributes(self.reg.pairs_graph, default_transform_key)
                           if self.reg.is_pairs_registered() else {})
        if indices not in pair_transforms and tuple(reversed(indices)) in pair_transforms:
            indices = tuple(reversed(indices))

        if indices not in pair_transforms:
            self.view_mode = previous_view_mode
            if self.reg.is_pairs_registered():
                show_warning(f'No pair registration for {label1} and {label2}: choose a registered pair')
            else:
                show_warning('Run pair registration first: a preview registration is not stored')
        else:
            self.temp_widget_state = TemporarilyDisabledWidgets()
            self.temp_tab_states = {}
            all_widgets = self.get_all_widgets()
            all_widgets.pop('registration.modify_pair_registration', None)
            self.temp_widget_state.disable(all_widgets)
            if self.enable_tab and self.is_tab_enabled:
                other_section_ids = [section_id for section_id in ['project'] + list(self.template.keys())
                                    if section_id != 'registration']
                self.temp_tab_states = {section_id: self.is_tab_enabled(section_id)
                                        for section_id in other_section_ids}
                for section_id in other_section_ids:
                    self.enable_tab(section_id, False)
            self.pair_indices = indices
            pair_transform = np.array(pair_transforms[indices].sel(t=0))

            # the widgets disabled above are only re-enabled on leaving this mode, so a failure must restore them
            entered = False
            try:
                if not self.reg.register_msims:
                    if not self.run_pre_processing():
                        return
                self._detach_view(previous_view_mode)
                # a multiscale pyramid, so napari loads only the level it needs while dragging
                register_images = self.reg.register_msims
                # each source keeps its own transform (e.g. scan rotation); the pair transform maps fixed to moving
                self.pair_source_affines = [
                    np.asarray(msi_utils.get_transform_from_msim(register_images[sim_index],
                                                                 self.reg.source_transform_key)).squeeze()
                    for sim_index in indices]
                fixed_source_affine, moving_source_affine = self.pair_source_affines
                layer_affines = pair_transform @ fixed_source_affine, moving_source_affine
                for sim_index, layer_affine, color in zip(indices, layer_affines, colors):
                    self._napari_view_add_image(self.viewer, register_images[sim_index], labels[sim_index],
                                                layer_affine, color, affine_event=True)
                self.update_pair_metrics()
                entered = True
            finally:
                if not entered:
                    if not self._restore_view():
                        self.view_mode = previous_view_mode
                    self._restore_pair_modify_state()
normalise_rotated_positions(value)
Source code in src\muvis_align\ui\Interface.py
403
404
405
406
407
408
def normalise_rotated_positions(self, value):
    if value:
        self.source_metadata['norm'] = True
    else:
        self.source_metadata.pop('norm', None)
    self.need_source_reinit = True
on_image_data_changed(event)
Source code in src\muvis_align\ui\Interface.py
1562
1563
1564
def on_image_data_changed(self, event):
    self.pair_metrics_timer.stop()
    self.pair_metrics_timer.start()
pair_registration()
Source code in src\muvis_align\ui\Interface.py
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
def pair_registration(self):
    if self.reg.is_global_registered():
        message = 'Global registration was already performed. '
    elif self.reg.is_pairs_registered():
        message = 'Pair registration was already performed. '
    else:
        message = ''
    message += 'Run pair registration?'
    reply = QMessageBox.question(None, 'muvis-align', message,
                                 QMessageBox.Yes|QMessageBox.No)
    if reply == QMessageBox.Yes:
        if not self.run_pair_registration():
            return
        self.update_registered(view_transform_key=self.reg.source_transform_key)
        self.update_fusion_preview_enabled()
        QMessageBox.information(None, 'muvis-align', 'Pair registration completed')
populate_channels()
Source code in src\muvis_align\ui\Interface.py
840
841
842
843
844
def populate_channels(self):
    channel_labels = list({channel.get('label', '') for source in self.reg.sources for channel in source.get_channels()})
    choices = {channel: channel for channel in channel_labels}
    param_widget = self.param_widgets.get('registration.channel')
    param_widget.set_choices(choices)
populate_channels_table()
Source code in src\muvis_align\ui\Interface.py
926
927
928
929
def populate_channels_table(self):
    param_widget = self.param_widgets.get('input_output.channels_table')
    param_widget.set_value(self.output_channels)
    self.populate_channels_table_color_pickers()
populate_channels_table_color_pickers()
Source code in src\muvis_align\ui\Interface.py
931
932
933
934
935
936
937
938
939
940
941
def populate_channels_table_color_pickers(self):
    table = self.param_widgets.get('input_output.channels_table').widget
    if 'color' not in table.column_headers:
        return
    color_coli = table.column_headers.index('color')
    for rowi in range(table.shape[0]):
        color = parse_channel_color(table.data[rowi, color_coli]) or (1, 1, 1)
        color_picker = MagicColorPicker(value=color)
        color_picker.changed.connect(
            lambda _=None, picker=color_picker, rowi=rowi: self.channel_color_changed(rowi, picker.value))
        table.native.setCellWidget(rowi, color_coli, color_picker.native)
populate_coordinate_systems(coord_systems)
Source code in src\muvis_align\ui\Interface.py
846
847
848
849
def populate_coordinate_systems(self, coord_systems):
    choices = {coord_system: coord_system.replace('_', ' ').capitalize() for coord_system in coord_systems}
    param_widget = self.param_widgets.get('input_output.coordinate_system')
    param_widget.set_choices(choices)
populate_image_selection()
Source code in src\muvis_align\ui\Interface.py
949
950
951
952
953
954
955
956
957
958
959
def populate_image_selection(self):
    labels = list(self.reg.file_labels)
    # the metadata and metrics tables' order
    if len(self.reg.positions) == len(labels):
        labels = [labels[index] for index in self.source_order()]
    widget1 = self.param_widgets.get('registration.reg_preview_image1')
    widget1.set_value(labels[0], choices=labels)

    widget2 = self.param_widgets.get('registration.reg_preview_image2')
    index = 1 if len(labels) > 1 else 0
    widget2.set_value(labels[index], choices=labels)
populate_metadata_table(transform_key=None)

Each source's position, pixel size and rotation: from its metadata, or from its transform_key's affine.

Source code in src\muvis_align\ui\Interface.py
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
def populate_metadata_table(self, transform_key=None):
    """Each source's position, pixel size and rotation: from its metadata, or from its transform_key's affine."""
    table_widget = self.param_widgets.get('input_output.metadata_table')
    properties = ['position', 'pixel size']
    if transform_key in (None, self.reg.source_transform_key):
        positions = [dict(position) for position in self.reg.positions]
        scales = self.reg.scales
        rotations = self.reg.rotations
    else:
        positions, scales, rotations = [], [], []
        for msim, position0, scale0 in zip(self.reg.msims, self.reg.positions, self.reg.scales):
            affine = msi_utils.get_transform_from_msim(msim, transform_key)
            if 't' in affine.dims:
                affine = affine.isel(t=0)
            affine_dims = affine['x_in'].data.tolist()
            transform_scale = get_scale_from_transform(np.asarray(affine))
            # a stack's sources are 2D: their z stays the metadata's
            positions.append({**position0, **get_sim_position_final(msim, transform_keys=[transform_key])})
            scales.append({dim: size * transform_scale if dim in affine_dims else size
                           for dim, size in scale0.items()})
            rotations.append(get_rotation_from_transform(affine))
    positions = self.channel_positions(positions)
    for position, rotation in zip(positions, rotations):
        if rotation is not None:
            position['r'] = rotation
    # metadata order in every coordinate system, so rows stay put when it changes
    order = self.source_order()
    data = [[print_dict_simple(positions[i], dims='czyx'), print_dict_simple(scales[i])] for i in order]
    row_headers = [self.reg.file_labels[i] for i in order]
    # Table: tuple-of-values : ([values], [row_headers], [column_headers])
    table_widget.set_value((data, row_headers, properties))
    table_widget.set_table_column_resize_mode()
populate_metrics_table(metrics_dict)
Source code in src\muvis_align\ui\Interface.py
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
def populate_metrics_table(self, metrics_dict):
    transform_keys = []
    metric_keys = []
    item_keys = []
    metrics = metrics_dict.get('summary')
    if metrics:
        item_keys.append('summary')
        for transform_key, transform_value in metrics.items():
            for metric_key, metric_value in transform_value.items():
                # a column only for values: without image metrics the source has no quality to show
                if metric_value is not None and transform_key not in transform_keys:
                    transform_keys.append(transform_key)
                if metric_value is not None and metric_key not in metric_keys:
                    metric_keys.append(metric_key)
    pairs_metrics = metrics_dict.get('pairs')
    if pairs_metrics:
        file_rank = {file_index: rank for rank, file_index in enumerate(self.source_order())}
        # each pair reads in table order too: its earlier source first
        pairs_metrics = dict(sorted(((tuple(sorted(indices, key=file_rank.get)), value)
                                     for indices, value in pairs_metrics.items()),
                                    key=lambda item: (file_rank[item[0][0]], file_rank[item[0][1]])))
    # a selected row picks its pair's preview images (metrics_table_selected)
    self.metrics_row_pairs = {self.reg.file_labels[indices[0]] + ' - ' + self.reg.file_labels[indices[1]]: indices
                              for indices in (pairs_metrics or {})}
    # split pairing's group (e.g. section) pairs as rows after the tile pairs, keyed by their labels
    row_metrics = {row_label: pairs_metrics[indices] for row_label, indices in self.metrics_row_pairs.items()}
    row_metrics |= {' - '.join(labels): value for labels, value in metrics_dict.get('group_pairs', {}).items()}
    # a dict keeps the order without a quadratic `not in` over the rows
    item_keys = list(dict.fromkeys(item_keys + list(row_metrics)))
    for pair_value in row_metrics.values():
        for transform_key, transform_value in pair_value.items():
            for metric_key, metric_value in transform_value.items():
                if metric_value is not None and transform_key not in transform_keys:
                    transform_keys.append(transform_key)
                if metric_value is not None and metric_key not in metric_keys:
                    metric_keys.append(metric_key)

    transform_keys = [transform_key.split('_')[0] for transform_key in transform_keys]
    is_metric_cols = (len(transform_keys) <= 1 and len(metric_keys) >= 1)
    col_headers = metric_keys if is_metric_cols else transform_keys

    metrics_table = []
    for rowi in range(len(item_keys)):
        row = [None] * len(col_headers)
        metrics_table.append(row)
    item_offset = 0

    metrics = metrics_dict.get('summary')
    if metrics:
        item_offset = 1
        for transform_key, transform_value in metrics.items():
            for metric_key, metric_value in transform_value.items():
                if metric_value is not None:
                    col_index = (metric_keys.index(metric_key) if is_metric_cols
                                 else transform_keys.index(transform_key.split('_')[0]))
                    metrics_table[0][col_index] = metric_value
    for pair_index, pair_value in enumerate(row_metrics.values()):
        for transform_key, transform_value in pair_value.items():
            for metric_key, metric_value in transform_value.items():
                if metric_value is not None:
                    col_index = (metric_keys.index(metric_key) if is_metric_cols
                                 else transform_keys.index(transform_key.split('_')[0]))
                    metrics_table[pair_index + item_offset][col_index] = metric_value

    table_widget = self.param_widgets.get('registration.metrics_table')
    # Table: tuple-of-values : ([values], [row_headers], [column_headers])
    method = self.params.get('registration', {}).get('method')
    # transform columns hold one metric, whose range colours them all
    table_metric = metric_keys[0] if len(metric_keys) == 1 else None

    def background(value, column_label):
        metric = column_label if is_metric_cols else table_metric
        return QColor(*metric_to_rgb(value, value_range=metric_value_range(metric, method), max_light=0.5,
                                     output_range=255))

    table_widget.widget.native.set_background(background)
    table_widget.set_value((metrics_table, item_keys, col_headers))
    table_widget.set_table_column_resize_mode()
populate_registered_tables()
Source code in src\muvis_align\ui\Interface.py
1777
1778
1779
1780
1781
1782
1783
1784
1785
def populate_registered_tables(self):
    with Timer('populate_registered_tables', verbose=self._timing_verbose()):
        # the coordinate systems from the state: reading them off the msims would build every one
        coord_systems = [self.reg.source_transform_key] + (
            [self.reg.reg_transform_key] if self.reg.is_global_registered() else [])
        self.populate_coordinate_systems(coord_systems)
        # the coordinate system still selected, so the table matches the dropdown
        self.populate_metadata_table(self.transform_key if self.transform_key in coord_systems else None)
        self.populate_metrics_table(self.reg.metrics)
pre_processing_process()
Source code in src\muvis_align\ui\Interface.py
827
828
829
830
831
832
833
834
835
836
837
838
def pre_processing_process(self):
    # the work and showing its result are two operations, each with its own bar
    self.prepare_action()
    if not self.run_pre_processing():
        return
    if self.reg.is_pairs_registered():
        # register_msims changed: the prior registration is stale and must not give the view its transform
        self.reg.state = RegState.SIMS_INIT
    self.update_views(show_preprocessed=True)
    self.enable_modify_pair_registration(False)
    self.update_fusion_preview_enabled()
    self.select_tab(3)
prepare_action()

As an action starts: settings resolved against its output, sources read again where their settings changed, and the project file copied next to the output. Returns whether the sources were read again.

Source code in src\muvis_align\ui\Interface.py
269
270
271
272
273
274
275
276
277
def prepare_action(self):
    """As an action starts: settings resolved against its output, sources read again where their settings
    changed, and the project file copied next to the output. Returns whether the sources were read again."""
    self.resolve_output_settings()
    read_again = self.need_source_reinit and self.reg.is_initialised()
    if read_again:
        self._input_output_process()
    self.copy_params_to_output()
    return read_again
preview_fusion(progress_factory=None)
Source code in src\muvis_align\ui\Interface.py
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
def preview_fusion(self, progress_factory=None):
    # a merge has no registered transform, so this falls back to the source positions
    transform_key = self.get_best_transform_key()
    # building the data dominates, so it is weighted as such and reports from the inside
    fusion_weight = 12
    with self._operation_progress('Fusion preview', progress_factory,
                                  phases=fusion_weight + 2 + self.shapes_weight) as factory:
        def fuse(worker_factory):
            with NapariMVSProgress(tqdm_class=worker_factory.tqdm_class, patch_fusion=True):
                return self._create_napari_data(
                    transform_key, fusion_method=self.params['fusion']['method'],
                    progress_factory=worker_factory, weight=fusion_weight, flatfield=True)

        data = self._run_off_thread(fuse, factory)
        with factory(total=1) as pbar:
            self._show_view_image(self.viewer, data, f'{self.reg.fileset_label} data')
            self.view_mode = ViewMode.FUSED
            pbar.update(1)
        # this path skips update_views(), so the shapes may be of another geometry or not drawn yet
        self._refresh_view_shapes(transform_key, progress_factory=factory)
preview_registration()
Source code in src\muvis_align\ui\Interface.py
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
def preview_registration(self):
    # the view stays until there is a result to replace it with: an empty viewer's welcome screen hides the bar
    with Timer('preview_registration: register (and pre-process if needed)', verbose=self._timing_verbose()):
        result = self.run_preview_registration()
    if result is None:
        return
    metrics, results, overlap1, overlap2, sims_pixel_space = result

    with Timer('preview_registration: show metrics', verbose=self._timing_verbose()):
        self.populate_metrics_table(metrics)
        # metrics shown before the (slower) feature layers replace the view
        flush_paint_events()

    fixed_points = results.get('fixed_points', [])
    moving_points = results.get('moving_points', [])
    matches = results.get('matches', [])
    inliers = results.get('inliers', [])
    if len(fixed_points) == 0 and 'affine_matrix' in results:
        # a method giving only a transform (phase correlation, elastix): a grid shows its offsets
        fixed_points, moving_points, matches, inliers = grid_point_pairs(
            np.shape(overlap1), np.shape(overlap2), results['affine_matrix'])
    with Timer('preview_registration: show feature layers', verbose=self._timing_verbose()):
        self._detach_view(self.view_mode)
        # both crops are resampled to the fixed one's pixel size: shown at it, as the overview shows the sources
        scale = si_utils.get_spacing_from_sim(sims_pixel_space[0], asarray=True)
        self._napari_view_show_features(self.viewer, overlap1, fixed_points, overlap2, moving_points, matches,
                                        inliers, scale=scale)
    self.view_mode = ViewMode.FEATURES
process_or_cancel(process, *_)

A section's Process button: runs process, or while an operation runs (the button then reads Cancel), offers to cancel that one.

Source code in src\muvis_align\ui\Interface.py
639
640
641
642
643
644
645
646
647
648
649
650
651
652
def process_or_cancel(self, process, *_):
    """A section's Process button: runs `process`, or while an operation runs (the button then reads
    Cancel), offers to cancel that one."""
    running = getattr(self, '_running_operation', None)
    if running is None:
        # a failure outside the run_*() steps would otherwise reach only stderr, via the Qt signal
        try:
            process()
        except Exception as error:
            name = getattr(process, '__name__', 'process')
            logging.exception(f'{name} failed')
            report_failure(operation_label(name), error)
    else:
        self.cancel_operation(getattr(running, 'desc', None))
project_path(path)
Source code in src\muvis_align\ui\Interface.py
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
def project_path(self, path):
    self.reset()
    self.params_path = path
    self.params = get_template_params(self.template)
    if os.path.exists(path):
        project_params = read_params(path)
        fusion_params = project_params.get('fusion') or {}
        if 'tile_size' in fusion_params:
            # the block size's name in projects saved before; written back as block_size
            fusion_params.setdefault('block_size', fusion_params.pop('tile_size'))
        # left out of the plugin: it drops the wrong tiles, and passed on it would filter unseen
        if (project_params.get('pre_processing') or {}).pop('filter_foreground', False):
            logging.warning('Foreground filtering is not available in the plugin: left out')
        unknown_params = get_unknown_params(self.template, project_params)
        if unknown_params:
            # still written back unchanged, as scripted runs may read them (e.g. extra_metadata)
            logging.warning(f'Project settings not used by the plugin: {", ".join(unknown_params)}')
        self.params = update_params(self.params, project_params)
        self.update_widgets()
    else:
        self.write_params()
    self.update_input_output_path()
    self.resolve_output_settings()
refresh_registered_view()
Source code in src\muvis_align\ui\Interface.py
2034
2035
2036
2037
def refresh_registered_view(self):
    # from source geometry and the mappings: view msims are built only if a fused preview needs them
    with self._operation_progress('Refreshing view', phases=2) as view_factory:
        self.update_registered(view_transform_key=self.reg.reg_transform_key, progress_factory=view_factory)
registration_dimension(value)
Source code in src\muvis_align\ui\Interface.py
410
411
412
413
414
def registration_dimension(self, value):
    # channels depend on the registration dimension, so they are built again, as are the source positions
    self.extra_metadata.pop('channels', None)
    self.reg.registration_dimension = value
    self.need_source_reinit = True
registration_process()
Source code in src\muvis_align\ui\Interface.py
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
def registration_process(self):
    self.prepare_action()
    if 'convert' in self.params['registration']['operation']:
        # each source written at its own position, without registration or fusion
        reply = QMessageBox.question(None, 'muvis-align', 'Convert data to OME-Zarr only?',
                                     QMessageBox.Yes|QMessageBox.No)
        if reply == QMessageBox.Yes:
            if self.run_convert():
                QMessageBox.information(None, 'muvis-align', 'Conversion completed')
        return

    if 'merge' in self.params['registration']['operation']:
        # merge fuses at the sources' own positions (get_best_transform_key()): nothing to register
        self.select_tab(4)
        return

    completion_message = 'Global registration completed'
    if self.reg.is_global_registered():
        message = 'Global registration was already performed. Run global registration?'
    elif not self.reg.is_pairs_registered():
        message = 'Pair registration not performed yet. Run both pair and global registration?'
        completion_message = 'Registration completed'
    else:
        message = 'Run global registration?'
    reply = QMessageBox.question(None, 'muvis-align', message,
                                 QMessageBox.Yes|QMessageBox.No)
    if reply == QMessageBox.Yes:
        if not self._run_registration():
            return
        self.refresh_registered_view()
        QMessageBox.information(None, 'muvis-align', completion_message)
reset()
Source code in src\muvis_align\ui\Interface.py
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
def reset(self):
    self.source_metadata = {}
    self.extra_metadata = {}
    self.output_channels = []
    self.view_mode = None
    self._preview_overlap_cache = None
    self._view_msims = None
    self.reg.reset()
    self._shape_layers = {}
    self._detached_view = None
    self._clear_napari_view(self.overview)
    self._clear_napari_view(self.viewer)
    if self.enable_tabs:
        self.enable_tabs(False, 2)
    if self.select_tab:
        self.select_tab(1)
    self.update_fusion_preview_enabled()
resolve_output_settings()

A registration saved with other source metadata settings either gets them back or is discarded.

Source code in src\muvis_align\ui\Interface.py
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
def resolve_output_settings(self):
    """A registration saved with other source metadata settings either gets them back or is discarded."""
    output_params_path = self.get_output_params_path()
    if not output_params_path or not os.path.exists(output_params_path):
        return
    output = self.get_output_dir()
    operation = self.params['registration'].get('operation', '')
    fused_name = operation_to_past_participle(operation) if operation else None
    fused_paths = self._output_paths(fused_name, output) if fused_name else []
    saved = self.reg.saved_progress_paths(output) + [path for path in fused_paths if os.path.exists(path)]
    previous = read_params(output_params_path).get('input_output', {})
    current = read_params(self.params_path).get('input_output', {})
    changed = [name for name in source_metadata_params if previous.get(name) != current.get(name)]
    if saved and changed:
        if self._ask_discard_output(f'Source metadata settings differ from the registration saved in {output}:'
                                    f' {", ".join(changed)}.\nReload the previous settings?'
                                    ' Or continue and discard all previous output?'):
            self.reg.discard_saved_progress(output)
            if fused_name:
                self._remove_output(fused_name, output)
            self.need_source_reinit = True
        else:
            for name in changed:
                self.params['input_output'][name] = previous.get(name)
            self.write_params()
            self.update_widgets()
run_convert()
Source code in src\muvis_align\ui\Interface.py
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
@catch_run_errors
def run_convert(self):
    operation = self.params['registration']['operation']
    output_folder = operation_to_past_participle(operation)
    ome_version = self.params['fusion']['ome_version']
    # self.reg.msims stays at source resolution; the scaled pyramid is register_msims, with
    # every level (pre-processing otherwise keeps only the ends)
    if not self.run_pre_processing(full_pyramid=True):
        return False
    msims = self.reg.register_msims
    # foreground filtering may drop sources, so labels/positions follow register_indices
    labels = [get_filetitle(self.reg.filenames[index]) for index in self.reg.register_indices]
    positions = [self.reg.positions[index] for index in self.reg.register_indices]
    with self._operation_progress('Converting') as progress_factory, \
         Timer('convert', verbose=self._timing_verbose()):
        # not fuse(): it would build one shared canvas; each source keeps its native levels
        # sequential: concurrent zarr writes race on zarr.json (Windows PermissionError)
        with progress_factory(total=len(msims), desc='Converting') as pbar:
            for label, position, msim in zip(labels, positions, msims):
                output_filename = f'{output_folder}/{label}'
                self.reg.save_native_levels(output_filename, msim, position=position,
                                            ome_version=ome_version)
                pbar.update(1)
    return True
run_fusion(progress_factory=None)
Source code in src\muvis_align\ui\Interface.py
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
@catch_run_errors
def run_fusion(self, progress_factory=None):
    operation = self.params['registration']['operation']
    output_filename = operation_to_past_participle(operation)
    block_size = parse_block_size(self.params['fusion'].get('block_size'))
    # the saves below are on-disk layout only, with no per-block cost, so they always get blocks
    save_block_size = block_size or default_chunk_size

    fuses_to_zarr = self.reg.fuses_to_zarr(self.params['fusion']['method'],
                                           self.params['input_output']['registration_dimension'],
                                           self.extra_metadata)
    # used once: settings or registration may change before another fusion
    estimate, self._fusion_estimate = getattr(self, '_fusion_estimate', None), None
    nested = (progress_factory or getattr(self, '_running_operation', None)) is not None
    pre_processing = self.params.get('pre_processing', {})
    # building the sources and their flat-field model report a phase each, when they run; the export, the
    # fusion written to zarr or the save computing a lazy fusion, is the last and takes the rest
    early_phases = int(self.reg.msims_build_pending()) + int(self.reg.flatfield_pending(pre_processing))
    with self._operation_progress('Fusion', progress_factory, phases=early_phases + 1) as factory:
        if not nested:
            # the export's phase fills the bar, not the close after it
            factory.last_phase_share = 1.0
        def fuse(worker_factory):
            level_estimate = estimate if estimate is not None or not fuses_to_zarr else self._estimate_fusion_size()
            msims = self.reg.flatfield_corrected_msims(self.reg.ensure_msims(progress_factory=worker_factory),
                                                       pre_processing, progress_factory=worker_factory)
            with NapariMVSProgress(tqdm_class=worker_factory.tqdm_class, patch_fusion=True), \
                    Timer('fusion', verbose=self._timing_verbose()):
                fused_image, is_saved = self.reg.fuse(
                    msims,
                    fusion_method=self.params['fusion']['method'],
                    # fuse()'s default reg_transform_key does not exist for a merge
                    transform_key=self.get_best_transform_key(),
                    output_spacing=self.params['fusion']['spacing'],
                    dimension=self.params['input_output']['registration_dimension'],
                    output_filename=output_filename,
                    block_size=block_size,
                    ome_version=self.params['fusion']['ome_version'],
                    extra_metadata=self.extra_metadata,
                    level_bytes=[level['bytes'] for level in level_estimate['levels']] if level_estimate else None)
                if not is_saved:
                    # save() takes a single-resolution sim: the pyramid's finest scale
                    save_sim = extract_sims_from_fused(fused_image)
                    self.reg.save(output_filename, save_sim,
                                  transform_key=self.get_best_transform_key(),
                                  translations0=self.reg.positions,
                                  channels=self.extra_metadata.get('channels', []),
                                  block_size=save_block_size,
                                  ome_version=self.params['fusion']['ome_version'],
                                  progress_factory=worker_factory)
                image_filenames = [path[len(self.reg.output):] for path in self._output_paths(output_filename)
                                   if os.path.exists(path)]
                self.reg.write_ro_crates(image_filenames, self.params)
                return fused_image

        try:
            return self._run_off_thread(fuse, factory)
        except OperationCancelled:
            self._remove_output(output_filename)
            raise
run_global_registration(progress_factory=None)
Source code in src\muvis_align\ui\Interface.py
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
@catch_run_errors
def run_global_registration(self, progress_factory=None):
    # register_global() reports its stages; declaring 8 units, half for the dominant optimisation,
    # keeps the last stages from each taking most of the bar
    self.reg.registration_dimension = self.params.get('input_output', {}).get('registration_dimension')
    if self.reg.pair_msims is None or self.reg.register_msims is None:
        # a loaded pair registration needs the pre-processed msims: at full resolution metrics can run out of memory
        if not self.run_pre_processing(progress_factory=progress_factory):
            return None
        self.reg.pair_msims = self.reg.register_msims
    with self._operation_progress('Global registration', progress_factory, phases=8) as factory:
        def register_global(worker_factory):
            with NapariDaskProgress(progress_class=worker_factory, desc='Global registration'), \
                    Timer('global registration', verbose=self._timing_verbose()):
                return self.reg.register_global(self.reg.pair_msims,
                                                register_indices=self.reg.register_indices,
                                                params=self.params['registration']
                                                | {'metrics': self.metrics_methods},
                                                progress_factory=worker_factory)

        # the transform is written onto the sources before the metrics: a cancel restores it
        key = self.reg.reg_transform_key
        snapshots = [(msims, snapshot_msims_transform(msims, key)) for msims in (self.reg.pair_msims, self.reg.msims)]
        try:
            results = self._run_off_thread(register_global, factory)
        except OperationCancelled:
            for msims, snapshot in snapshots:
                restore_msims_transform(msims, key, snapshot)
            raise

    with Timer('save registration results', verbose=self._timing_verbose()):
        self.reg.save_group_pair_mappings()
        self.reg.save_mappings(results['mappings'])
        self.reg.save_mappings_csv(results['mappings'])
        self.reg.save_metrics(results['metrics'])
        self.reg.save_residuals(results['residual_errors'])
        self.reg.write_ro_crates([], self.params)
    return results
run_pair_registration(progress_factory=None)
Source code in src\muvis_align\ui\Interface.py
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
@catch_run_errors
def run_pair_registration(self, progress_factory=None):
    # pre-processing reports its own bar, before the registration's; again if its settings changed since
    if not self.reg.register_msims_current(self.params['pre_processing']):
        if not self.run_pre_processing(progress_factory=progress_factory):
            return None

    # the split pairing groups sources by it (z-planes or channels)
    self.reg.registration_dimension = self.params.get('input_output', {}).get('registration_dimension')
    with self._operation_progress('Pair registration', progress_factory, phases=2) as factory:
        def register_pairs(worker_factory):
            # pairs and metrics report per batch: a dask callback would open a phase per batch
            with Timer('pair registration', verbose=self._timing_verbose()):
                return self.reg.register_pairs(
                    self.reg.register_msims,
                    params=self.params['registration'] | {'metrics': self.metrics_methods},
                    progress_factory=worker_factory)

        results = self._run_off_thread(register_pairs, factory)

    self.reg.save_pair_results(results['pair_mappings'], metrics=results['metrics'])
    self.enable_modify_pair_registration()
    return results
run_pre_processing(progress_factory=None, full_pyramid=False)
Source code in src\muvis_align\ui\Interface.py
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
@catch_run_errors
def run_pre_processing(self, progress_factory=None, full_pyramid=False):
    params_features = self.params['pre_processing']
    # the msim build gets bar room only when pending, weighted by its cost: it dominates unless a step
    # computes over the data (flat-field, normalisation, foreground)
    build_weight = 2 if self.reg.has_eager_pre_processing(params_features) else 8
    # registration reads the finest level and the preview the coarsest; convert writes every level
    ends_only = not full_pyramid
    build_pending = self.reg.msims_build_pending(params_features.get('scale'), ends_only=ends_only)
    phases = build_weight + 1 if build_pending else 1
    with self._operation_progress('Pre-processing', progress_factory, phases=phases) as progress_factory, \
         Timer('pre_processing_process', verbose=self._timing_verbose()):
        # msims built explicitly through ensure_msims() so the build reports progress
        def preprocess(worker_factory):
            with Timer('run_pre_processing: build msims (load image data)',
                       verbose=self._timing_verbose()):
                # at pre-processing's scale: finer levels would only be dropped again
                msims = self.reg.ensure_msims(progress_factory=worker_factory,
                                              target_scale=params_features.get('scale'),
                                              ends_only=ends_only,
                                              weight=build_weight)
            with Timer('run_pre_processing: preprocess', verbose=self._timing_verbose()):
                return self.reg.preprocess(msims, progress_factory=worker_factory,
                                           **params_features)

        _, _, modified = self._run_off_thread(preprocess, progress_factory)
    self.pre_processing_performed = modified
    return True
run_preview_registration(progress_factory=None)
Source code in src\muvis_align\ui\Interface.py
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
@catch_run_errors
def run_preview_registration(self, progress_factory=None):
    label1 = self.param_widgets.get('registration.reg_preview_image1').get_value()
    label2 = self.param_widgets.get('registration.reg_preview_image2').get_value()
    index1 = self.reg.file_labels.index(label1)
    index2 = self.reg.file_labels.index(label2)

    pre_processing_params = self.params['pre_processing']
    # only the pair is pre-processed, unless a step needs every source; that gets its own bar, before this one
    if (not self.reg.register_msims_current(pre_processing_params)
            and self.reg.needs_full_preprocess(pre_processing_params)):
        if not self.run_pre_processing(progress_factory=progress_factory):
            return None

    with self._operation_progress('Preview registration', progress_factory, phases=2) as factory:
        def register_preview(worker_factory):
            # dask's task counting reports as a phase of this bar, not a bar of its own
            with NapariDaskProgress(progress_class=worker_factory,
                                    desc='Preview registration'):
                registration_params = self.params['registration']
                channel = registration_params.get('channel')
                cache_key = (index1, index2, channel, self.reg.preprocess_settings(pre_processing_params))
                cache = self._preview_overlap_cache
                # the crop depends only on the pair, channel and pre-processing, so method changes reuse it
                if cache is not None and cache['key'] == cache_key:
                    msim1, msim2 = cache['msims']
                    overlap1, overlap2, sims_pixel_space = cache['overlap1'], cache['overlap2'], cache['sims_pixel_space']
                else:
                    if self.reg.register_msims_current(pre_processing_params):
                        msim1, msim2 = self.reg.register_msims[index1], self.reg.register_msims[index2]
                    else:
                        with Timer('preview_registration: pre-process the pair', verbose=self._timing_verbose()):
                            msim1, msim2 = self.reg.preprocess_sources((index1, index2), pre_processing_params)
                    overlap1, overlap2, sims_pixel_space = self.reg.select_pair_overlap(
                        msim1, msim2, params=registration_params)
                    overlap1, overlap2 = overlap1.compute(), overlap2.compute()
                    self._preview_overlap_cache = {
                        'key': cache_key, 'msims': (msim1, msim2),
                        'overlap1': overlap1, 'overlap2': overlap2, 'sims_pixel_space': sims_pixel_space,
                    }

                with Timer('preview_registration: register the overlap', verbose=self._timing_verbose()):
                    transform, quality, results = self.reg.register_overlap(
                        overlap1, overlap2, sims_pixel_space, params=registration_params)

                transforms = {(0, 1): transform}
                qualities = {(0, 1): quality}
                with Timer('preview_registration: pair metrics', verbose=self._timing_verbose()):
                    metrics = calc_msims_metrics((msim1, msim2), transforms, qualities,
                                                 metric_methods=self.metrics_methods)
                remap_pair_metrics(metrics, (index1, index2))
                return metrics, results, overlap1, overlap2, sims_pixel_space

        try:
            return self._run_off_thread(register_preview, factory)
        except NoOverlapError:
            show_warning(f'{label1} and {label2} do not overlap: choose two images that do')
            return None
select_pair_preview(ref)
Source code in src\muvis_align\ui\Interface.py
980
981
982
983
984
985
986
987
988
989
990
991
992
def select_pair_preview(self, ref):
    # only an overlap shape's ref ('0 1') names a pair; single-image clicks are ignored
    indices = [int(index) for index in ref.split()]
    if len(indices) == 2:
        row_label = next((label for label, pair in getattr(self, 'metrics_row_pairs', {}).items()
                          if sorted(pair) == sorted(indices)), None)
        table = self.param_widgets.get('registration.metrics_table')
        if row_label is not None and table is not None:
            # the table never selects shapes back, so this cannot loop
            table.widget.native.select_row_label(row_label)
            indices = list(self.metrics_row_pairs[row_label])
        # also when the row was already selected, and so emitted no change
        self.set_preview_images(indices)
set_channels(channels_dict)
Source code in src\muvis_align\ui\Interface.py
420
421
def set_channels(self, channels_dict):
    self.extra_metadata['channels'] = channels_from_table(channels_dict)
set_preview_images(indices)
Source code in src\muvis_align\ui\Interface.py
1001
1002
1003
1004
def set_preview_images(self, indices):
    labels = self.reg.file_labels
    self.param_widgets.get('registration.reg_preview_image1').set_value(labels[indices[0]])
    self.param_widgets.get('registration.reg_preview_image2').set_value(labels[indices[1]])
source_order()

Source indices in the metadata table's (t, c, z, y, x) order, by their metadata positions.

Source code in src\muvis_align\ui\Interface.py
898
899
900
901
def source_order(self):
    """Source indices in the metadata table's (t, c, z, y, x) order, by their metadata positions."""
    positions = self.channel_positions(self.reg.positions)
    return sorted(range(len(positions)), key=lambda index: position_sort_key(positions[index]))
source_position_x(value)
Source code in src\muvis_align\ui\Interface.py
384
385
def source_position_x(self, value):
    self._set_source_metadata_value('source_position_x', value)
source_position_y(value)
Source code in src\muvis_align\ui\Interface.py
381
382
def source_position_y(self, value):
    self._set_source_metadata_value('source_position_y', value)
source_position_z(value)
Source code in src\muvis_align\ui\Interface.py
378
379
def source_position_z(self, value):
    self._set_source_metadata_value('source_position_z', value)
source_rotation(value)
Source code in src\muvis_align\ui\Interface.py
396
397
def source_rotation(self, value):
    self._set_source_metadata_value('source_rotation', value)
source_scale_x(value)
Source code in src\muvis_align\ui\Interface.py
393
394
def source_scale_x(self, value):
    self._set_source_metadata_value('source_scale_x', value)
source_scale_y(value)
Source code in src\muvis_align\ui\Interface.py
390
391
def source_scale_y(self, value):
    self._set_source_metadata_value('source_scale_y', value)
source_scale_z(value)
Source code in src\muvis_align\ui\Interface.py
387
388
def source_scale_z(self, value):
    self._set_source_metadata_value('source_scale_z', value)
tab_changed(tab_label)
Source code in src\muvis_align\ui\Interface.py
179
180
181
182
183
184
def tab_changed(self, tab_label):
    self.current_tab = tab_label
    if tab_label != 'registration' and self.view_mode == ViewMode.FEATURES and not self._restore_view():
        self._clear_napari_view(self.viewer)
        self.view_mode = None
    self.pair_metrics_timer.stop()
update_fusion_preview_enabled()
Source code in src\muvis_align\ui\Interface.py
1805
1806
1807
1808
1809
1810
def update_fusion_preview_enabled(self):
    widget = self.param_widgets.get('fusion.preview_fusion')
    if widget:
        # only a merge fuses at the unregistered source positions
        operation = getattr(self, 'params', {}).get('registration', {}).get('operation', '')
        widget.widget.enabled = self.reg.is_global_registered() or 'merge' in operation
update_input_output_path()
Source code in src\muvis_align\ui\Interface.py
353
354
355
356
357
358
359
360
361
362
363
364
365
366
def update_input_output_path(self):
    # shows paths as stored: FileEdit.set_value() would force them absolute, so the line edit is set directly
    params = self.params['input_output']
    widget = self.param_widgets.get('input_output.input_path')
    # several comma-separated globs are one path value, shown as stored
    input_path = path_param_to_text(params.get('input_path', ''))
    if widget is not None:
        self._set_path_widget_text(widget, input_path)
    widget = self.param_widgets.get('input_output.output_path')
    output_path = path_param_to_text(params.get('output_path', ''))
    if widget is not None:
        self._set_path_widget_text(widget, output_path)
    resolved_output_path = resolve_to_project_dir(output_path, self.get_project_dir())
    init_logging(log_filename=os.path.join(resolved_output_path, 'muvis-align.log'), verbose=self.verbose)
update_metadata_source(skip_view_update=False, progress_factory=None)
Source code in src\muvis_align\ui\Interface.py
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
def update_metadata_source(self, skip_view_update=False, progress_factory=None):
    if not self.reg.is_pairs_registered():
        try:
            with self._operation_progress('Initialising sources',
                                          progress_factory) as factory:
                self._run_off_thread(
                    lambda worker_factory: self.reg.init_data(
                        source_metadata=self.source_metadata,
                        progress_factory=worker_factory,
                    ),
                    factory)
        except ValueError as e:
            show_warning('Unable to read source data\n' + str(e))
            logging.exception('Unable to read source data')
            return False

        # msims are built lazily by view_msims; the cheap z remapping stays eager so the shapes are right
        z_positions = sorted(set([position.get('z', 0) for position in self.reg.positions]))
        if len(z_positions) > 1:
            for position in self.reg.positions:
                position['z'] = z_positions.index(position.get('z', 0))
        self._view_msims = None
    # only runs before registration, so the source transform key is the only one: no need to build msims
    coord_systems = [self.reg.source_transform_key]
    self.populate_channels()
    self.populate_coordinate_systems(coord_systems)
    if self.update_output_channels():
        self.populate_channels_table()
    if self.reg.is_initialised():
        # without transform_keys the msims are not read, so None avoids forcing their build
        self.populate_metadata_table(None)
        self.check_3d_view()
        if not skip_view_update:
            # no factory: drawing is the next operation, or a phase of the caller's if one runs
            self.update_views(show_images=False)

    return True
update_method_params_enabled()
Source code in src\muvis_align\ui\Interface.py
1796
1797
1798
1799
1800
1801
1802
1803
def update_method_params_enabled(self):
    method_widget = self.param_widgets.get('registration.method')
    if method_widget is not None:
        method_kind = registration_method_kind(method_widget.get_value())
        for param_name, method_kinds in registration_method_params.items():
            widget = self.param_widgets.get(f'registration.{param_name}')
            if widget is not None:
                widget.widget.enabled = method_kind in method_kinds
update_output_channels()
Source code in src\muvis_align\ui\Interface.py
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
def update_output_channels(self):
    channels = self.extra_metadata.get('channels')
    # blank-labelled channels count as unconfigured, so they are derived again
    if not channels or not any(channel.get('label') for channel in channels):
        dimension = self.params['input_output']['registration_dimension']
        if dimension.lower() == 'c':
            # one per channel group, its first source's channel (e.g. its color) under the group's label
            groups = self.reg.channel_groups()
            channels = [{**(self.reg.sources[groups.index(group)].get_channels() or [{}])[0], 'label': label}
                        for group, label in enumerate(self.reg.channel_labels())]
        else:
            channels = self.reg.sources[0].get_channels()

        self.extra_metadata['channels'] = channels

        # `or`, not a get() default: an unnamed OME channel has an explicit blank label
        data = [[channel.get('label') or f'channel {index}', channel.get('color', (1, 1, 1))]
                 for index, channel in enumerate(channels)]
        self.output_channels = create_dict_of_lists(data, ['label', 'color'])
        return True

    return False
update_pair_metrics()
Source code in src\muvis_align\ui\Interface.py
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
def update_pair_metrics(self):
    if self.view_mode != ViewMode.PAIRS:
        return
    if self._pair_metrics_worker is not None:
        # one at a time: the latest position is measured once this one is done
        self._pair_metrics_pending = True
        return
    # read off the layers here, on the Qt thread: only the measuring runs on the worker
    pair_indices = self.pair_indices
    reg_msims = [self.reg.register_msims[index] for index in pair_indices]
    transforms = {(0, 1): self.calc_mod_pair_transform()}

    def measure():
        metrics = calc_msims_metrics(reg_msims, transforms, metric_methods=self.metrics_methods)
        remap_pair_metrics(metrics, pair_indices)
        return metrics

    def show(metrics):
        if self.view_mode == ViewMode.PAIRS and self.pair_indices == pair_indices:
            self.populate_metrics_table(metrics)

    def done():
        self._pair_metrics_worker = None
        if self._pair_metrics_pending:
            self._pair_metrics_pending = False
            self.update_pair_metrics()

    if QApplication is None or QApplication.instance() is None:
        show(measure())
        return
    # off the Qt thread, so the layers can still be dragged while it measures
    self._pair_metrics_worker = create_worker(
        measure,
        _start_thread=False,
        _connect={
            'returned': show,
            'errored': lambda error: show_warning(f'Pair metrics failed: {error}'),
            'warned': lambda warning: notification_manager.receive_warning(*warning),
            'finished': done,
        },
    )
    self._pair_metrics_worker.start()
update_registered(view_transform_key=None, progress_factory=None)
Source code in src\muvis_align\ui\Interface.py
1787
1788
1789
def update_registered(self, view_transform_key=None, progress_factory=None):
    self.populate_registered_tables()
    self.update_views(transform_key=view_transform_key, progress_factory=progress_factory)
update_views(transform_key=None, show_preprocessed=False, show_images=True, progress_factory=None)
Source code in src\muvis_align\ui\Interface.py
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
def update_views(self, transform_key=None, show_preprocessed=False, show_images=True,
                 progress_factory=None):
    if transform_key is None:
        transform_key = self.get_best_transform_key()

    is_3d = (self.reg.sources[0].get_size().get('z', 0) > 1)
    is_multi_z_shapes = (len(set(position.get('z', 0) for position in self.reg.positions)) > 1)
    force_2d = is_multi_z_shapes and not is_3d

    # shapes drawn from the same geometry, pairs and qualities are kept, only the image under them replaced
    shapes = None
    build_shapes = self._current_shape_layer(self.viewer, transform_key, force_2d) is None
    # steps weighted by cost: building the view data dominates, and reports from the inside
    view_data_weight = 12
    phases = (self.shapes_weight if build_shapes else 0) + 2 + (view_data_weight + 1 if show_images else 0)

    with self._operation_progress('Refreshing view', progress_factory, phases=phases) as factory:
        if build_shapes:
            with Timer('update_views: create shapes', verbose=self._timing_verbose()):
                # pure geometry, so off the Qt thread to keep the window responsive
                shapes = self._run_off_thread(
                    lambda worker_factory: self._create_napari_shapes(
                        transform_key, force_2d=force_2d, progress_factory=worker_factory,
                        weight=self.shapes_weight),
                    factory)

        # each section built only when viewed; never without show_images, so whether data is read cannot hinge on
        # the sources happening to be unrotated
        data = None
        if show_images:
            with Timer('update_views: lazy overview', verbose=self._timing_verbose()):
                data = self._run_off_thread(
                    lambda worker_factory: self._create_lazy_overview(transform_key,
                                                                      show_preprocessed=show_preprocessed), factory)
        # otherwise (a rotation, or sources it cannot overlay) the fused preview
        if data is None and show_images:
            with Timer('update_views: create fused data', verbose=self._timing_verbose()):
                # fused off the Qt thread; adding it to the viewer below must stay on it
                data = self._run_off_thread(
                    lambda worker_factory: self._create_napari_data(
                        transform_key, show_preprocessed=show_preprocessed,
                        progress_factory=worker_factory, weight=view_data_weight),
                    factory)
        # replaced only once the new data is ready: an empty viewer's welcome screen hides the activity dialog
        with factory(total=1) as pbar, \
             Timer('update_views: add fused data to viewer', verbose=self._timing_verbose()):
            # an overview, not the fusion preview: a naive contrast guess is enough
            self._show_view_image(self.viewer, data, f'{self.reg.fileset_label} data', cheap=True,
                                  keep_shapes=not build_shapes)
            pbar.update(1)

        with Timer('update_views: add shapes to viewers', verbose=self._timing_verbose()):
            self._refresh_view_shapes(transform_key, shapes, progress_factory=factory)
    self.view_mode = ViewMode.OVERVIEW
update_widgets()
Source code in src\muvis_align\ui\Interface.py
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
def update_widgets(self):
    # a dropdown filled later (channels, preview images) is empty until then: only fixed ones map old values
    fixed_choices = {f'{section_id}.{item["name"]}' for section_id, items in self.template.items()
                     for item in items if item.get('options')}
    for param_widget in self.param_widgets.values():
        # a mirrored widget's key is its tab's; param_name is where its value is stored
        param_name = param_widget.param_name
        # path widgets are set by update_input_output_path(), relative to the project directory
        if param_name not in ('input_output.input_path', 'input_output.output_path'):
            keys = param_name.split('.')
            value = self.params.get(keys[0], {}).get(keys[1])
            if value is not None and param_name in fixed_choices and isinstance(param_widget.widget, ComboBox) \
                    and value not in param_widget.widget.choices:
                value = self._listed_choice(param_name, value, param_widget.widget)
            if value is not None:
                param_widget.set_value(value)
                if param_name == 'input_output.channels_table':
                    # set_value() blocks the table's signals, so channels_table() never sees the loaded value
                    if value:
                        self.set_channels(value)
                    # a loaded project skips update_output_channels(), so attach the color pickers here
                    self.populate_channels_table_color_pickers()
    # a loaded method equal to the widget's default changes nothing, so change_param() does not see it
    self.update_method_params_enabled()
write_params()
Source code in src\muvis_align\ui\Interface.py
254
255
def write_params(self):
    write_params(self.params_path, self.params)
ViewMode

Bases: Enum

Source code in src\muvis_align\ui\Interface.py
73
74
75
76
77
class ViewMode(Enum):
    OVERVIEW = auto()
    PAIRS = auto()
    FEATURES = auto()
    FUSED = auto()
FEATURES = auto() class-attribute instance-attribute
FUSED = auto() class-attribute instance-attribute
OVERVIEW = auto() class-attribute instance-attribute
PAIRS = auto() class-attribute instance-attribute
has_layer(viewer, layer)
Source code in src\muvis_align\ui\Interface.py
80
81
82
83
def has_layer(viewer, layer):
    # a plugin's viewer hands out a new proxy of a layer at every access, and napari compares layers by value
    layer = getattr(layer, '__wrapped__', layer)
    return any(getattr(shown, '__wrapped__', shown) is layer for shown in viewer.layers)
position_sort_key(position)
Source code in src\muvis_align\ui\Interface.py
95
96
97
def position_sort_key(position):
    return (position.get('t', 0), position.get('c', 0), position.get('z', 0), position.get('y', 0),
            position.get('x', 0))

MagicColorPicker

MagicColorPicker

Bases: ValueWidget

A magicgui color-picker widget: a single swatch button that opens QColorDialog.

Value is an (r, g, b) float tuple with each component in [0, 1].

Source code in src\muvis_align\ui\MagicColorPicker.py
82
83
84
85
86
87
88
89
90
91
class MagicColorPicker(ValueWidget):
    """A magicgui color-picker widget: a single swatch button that opens QColorDialog.

    Value is an (r, g, b) float tuple with each component in [0, 1].
    """

    def __init__(self, **kwargs):
        # create_widget(widget_type="...MagicColorPicker") leaves its string 'widget_type' in kwargs
        kwargs.pop('widget_type', None)
        super().__init__(widget_type=_QColorSwatchWidget, **kwargs)
__init__(**kwargs)
Source code in src\muvis_align\ui\MagicColorPicker.py
88
89
90
91
def __init__(self, **kwargs):
    # create_widget(widget_type="...MagicColorPicker") leaves its string 'widget_type' in kwargs
    kwargs.pop('widget_type', None)
    super().__init__(widget_type=_QColorSwatchWidget, **kwargs)
QColorSwatchButton

Bases: QPushButton

A small flat button showing a single color; clicking it opens QColorDialog.

Source code in src\muvis_align\ui\MagicColorPicker.py
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
class QColorSwatchButton(QPushButton):
    """A small flat button showing a single color; clicking it opens QColorDialog."""

    colorChanged = Signal(QColor)

    def __init__(self, parent=None):
        super().__init__(parent)
        self._color = QColor()
        self.setFlat(True)
        self.clicked.connect(self._pick_color)
        self._update_style()

    def color(self):
        return self._color

    def setColor(self, color):
        color = QColor(color)
        if color != self._color:
            self._color = color
            self._update_style()
            self.colorChanged.emit(self._color)

    def _update_style(self):
        r, g, b = round(self._color.redF(), 3), round(self._color.greenF(), 3), round(self._color.blueF(), 3)
        self.setText(f'{r}, {g}, {b}')
        text_color = 'black' if self._color.lightnessF() > 0.5 else 'white'
        self.setStyleSheet(
            f'background-color: {self._color.name()}; color: {text_color}; border: 1px solid gray;')

    def _pick_color(self):
        # parented on the window, so the dialog does not inherit this button's background stylesheet
        dialog = QColorDialog(self._color, self.window())
        if dialog.exec():
            color = dialog.currentColor()
            if color.isValid():
                self.setColor(color)
colorChanged = Signal(QColor) class-attribute instance-attribute
__init__(parent=None)
Source code in src\muvis_align\ui\MagicColorPicker.py
30
31
32
33
34
35
def __init__(self, parent=None):
    super().__init__(parent)
    self._color = QColor()
    self.setFlat(True)
    self.clicked.connect(self._pick_color)
    self._update_style()
color()
Source code in src\muvis_align\ui\MagicColorPicker.py
37
38
def color(self):
    return self._color
setColor(color)
Source code in src\muvis_align\ui\MagicColorPicker.py
40
41
42
43
44
45
def setColor(self, color):
    color = QColor(color)
    if color != self._color:
        self._color = color
        self._update_style()
        self.colorChanged.emit(self._color)
color_to_tuple(color)
Source code in src\muvis_align\ui\MagicColorPicker.py
 8
 9
10
11
def color_to_tuple(color):
    if not isinstance(color, QColor) or not color.isValid():
        return None
    return color.redF(), color.greenF(), color.blueF()
tuple_to_color(value)
Source code in src\muvis_align\ui\MagicColorPicker.py
14
15
16
17
18
19
20
21
22
def tuple_to_color(value):
    if isinstance(value, QColor):
        return value
    if isinstance(value, str):
        return QColor(value)
    if value is None:
        return QColor()
    r, g, b = (list(value) + [0, 0, 0])[:3]
    return QColor.fromRgbF(r, g, b)

ModelTable

ArrayTableModel

Bases: QAbstractTableModel

Rows of values with row and column labels, read-only: text and colour made only for the cells shown.

Source code in src\muvis_align\ui\ModelTable.py
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
class ArrayTableModel(QAbstractTableModel):
    """Rows of values with row and column labels, read-only: text and colour made only for the cells shown."""

    def __init__(self, parent=None):
        super().__init__(parent)
        self.rows, self.row_labels, self.column_labels = [], [], []
        self.background = None
        self._backgrounds = {}

    def set_rows(self, rows, row_labels, column_labels):
        self.beginResetModel()
        self.rows, self.row_labels, self.column_labels = list(rows), list(row_labels), list(column_labels)
        self._backgrounds = {}
        self.endResetModel()

    def rowCount(self, parent=QModelIndex()):
        return 0 if parent.isValid() else len(self.rows)

    def columnCount(self, parent=QModelIndex()):
        return 0 if parent.isValid() else len(self.column_labels)

    def data(self, index, role=Qt.DisplayRole):
        if not index.isValid():
            return None
        value = self.rows[index.row()][index.column()]
        if role == Qt.DisplayRole:
            if isinstance(value, (float, np.floating)):
                return f'{value:.4f}'
            return '' if value is None else str(value)
        if role == Qt.BackgroundRole and self.background is not None and value is not None:
            key = (index.row(), index.column())
            if key not in self._backgrounds:
                column = index.column()
                column_label = self.column_labels[column] if column < len(self.column_labels) else None
                self._backgrounds[key] = self.background(value, column_label)
            return self._backgrounds[key]
        return None

    def headerData(self, section, orientation, role=Qt.DisplayRole):
        if role != Qt.DisplayRole:
            return None
        labels = self.column_labels if orientation == Qt.Horizontal else self.row_labels
        return str(labels[section]) if section < len(labels) else None
background = None instance-attribute
__init__(parent=None)
Source code in src\muvis_align\ui\ModelTable.py
11
12
13
14
15
def __init__(self, parent=None):
    super().__init__(parent)
    self.rows, self.row_labels, self.column_labels = [], [], []
    self.background = None
    self._backgrounds = {}
columnCount(parent=QModelIndex())
Source code in src\muvis_align\ui\ModelTable.py
26
27
def columnCount(self, parent=QModelIndex()):
    return 0 if parent.isValid() else len(self.column_labels)
data(index, role=Qt.DisplayRole)
Source code in src\muvis_align\ui\ModelTable.py
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
def data(self, index, role=Qt.DisplayRole):
    if not index.isValid():
        return None
    value = self.rows[index.row()][index.column()]
    if role == Qt.DisplayRole:
        if isinstance(value, (float, np.floating)):
            return f'{value:.4f}'
        return '' if value is None else str(value)
    if role == Qt.BackgroundRole and self.background is not None and value is not None:
        key = (index.row(), index.column())
        if key not in self._backgrounds:
            column = index.column()
            column_label = self.column_labels[column] if column < len(self.column_labels) else None
            self._backgrounds[key] = self.background(value, column_label)
        return self._backgrounds[key]
    return None
headerData(section, orientation, role=Qt.DisplayRole)
Source code in src\muvis_align\ui\ModelTable.py
46
47
48
49
50
def headerData(self, section, orientation, role=Qt.DisplayRole):
    if role != Qt.DisplayRole:
        return None
    labels = self.column_labels if orientation == Qt.Horizontal else self.row_labels
    return str(labels[section]) if section < len(labels) else None
rowCount(parent=QModelIndex())
Source code in src\muvis_align\ui\ModelTable.py
23
24
def rowCount(self, parent=QModelIndex()):
    return 0 if parent.isValid() else len(self.rows)
set_rows(rows, row_labels, column_labels)
Source code in src\muvis_align\ui\ModelTable.py
17
18
19
20
21
def set_rows(self, rows, row_labels, column_labels):
    self.beginResetModel()
    self.rows, self.row_labels, self.column_labels = list(rows), list(row_labels), list(column_labels)
    self._backgrounds = {}
    self.endResetModel()
ModelTable

Bases: ValueWidget

A read-only magicgui table for many rows: value is {'data', 'index', 'columns'}, as magicgui's Table.

Source code in src\muvis_align\ui\ModelTable.py
108
109
110
111
112
113
114
115
class ModelTable(ValueWidget):
    """A read-only magicgui table for many rows: value is {'data', 'index', 'columns'}, as magicgui's Table."""

    def __init__(self, **kwargs):
        # create_widget(widget_type="...ModelTable") leaves its string 'widget_type' in kwargs
        kwargs.pop('widget_type', None)
        kwargs.pop('options', None)
        super().__init__(widget_type=_QModelTableWidget, **kwargs)
__init__(**kwargs)
Source code in src\muvis_align\ui\ModelTable.py
111
112
113
114
115
def __init__(self, **kwargs):
    # create_widget(widget_type="...ModelTable") leaves its string 'widget_type' in kwargs
    kwargs.pop('widget_type', None)
    kwargs.pop('options', None)
    super().__init__(widget_type=_QModelTableWidget, **kwargs)
QModelTableView

Bases: QTableView

A table view over ArrayTableModel, with the QTableWidget calls the plugin uses.

Source code in src\muvis_align\ui\ModelTable.py
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
class QModelTableView(QTableView):
    """A table view over ArrayTableModel, with the QTableWidget calls the plugin uses."""

    valueChanged = Signal(object)
    itemSelectionChanged = Signal()

    def __init__(self, parent=None):
        super().__init__(parent)
        self.table_model = ArrayTableModel(self)
        self.setModel(self.table_model)
        # a bound method, not a lambda holding the view: Qt keeps a lambda alive, and through it everything connected
        self.selectionModel().selectionChanged.connect(self._selection_changed)

    def _selection_changed(self, *_):
        self.itemSelectionChanged.emit()

    def value(self):
        model = self.table_model
        return {'data': model.rows, 'index': model.row_labels, 'columns': model.column_labels}

    def setValue(self, value):
        # (rows, row labels, column labels), or magicgui Table's {'data', 'index', 'columns'}
        if isinstance(value, dict):
            value = (value.get('data', []), value.get('index', []), value.get('columns', []))
        rows, row_labels, column_labels = value if value else ([], [], [])
        self.table_model.set_rows(rows, row_labels, column_labels)
        self.valueChanged.emit(self.value())

    def set_background(self, background):
        """`background(value, column_label)` gives a cell's QColor, asked only for the cells shown."""
        self.table_model.background = background
        self.table_model._backgrounds = {}

    def currentRow(self):
        return self.currentIndex().row()

    def row_label(self, row):
        labels = self.table_model.row_labels
        return labels[row] if 0 <= row < len(labels) else None

    def select_row_label(self, label):
        """The row with this label made current, selected and scrolled to, as a click on it would."""
        index = self.table_model.index(self.table_model.row_labels.index(label), 0)
        self.selectionModel().setCurrentIndex(
            index, QItemSelectionModel.SelectionFlag.ClearAndSelect | QItemSelectionModel.SelectionFlag.Rows)
        self.scrollTo(index)
itemSelectionChanged = Signal() class-attribute instance-attribute
table_model = ArrayTableModel(self) instance-attribute
valueChanged = Signal(object) class-attribute instance-attribute
__init__(parent=None)
Source code in src\muvis_align\ui\ModelTable.py
59
60
61
62
63
64
def __init__(self, parent=None):
    super().__init__(parent)
    self.table_model = ArrayTableModel(self)
    self.setModel(self.table_model)
    # a bound method, not a lambda holding the view: Qt keeps a lambda alive, and through it everything connected
    self.selectionModel().selectionChanged.connect(self._selection_changed)
currentRow()
Source code in src\muvis_align\ui\ModelTable.py
86
87
def currentRow(self):
    return self.currentIndex().row()
row_label(row)
Source code in src\muvis_align\ui\ModelTable.py
89
90
91
def row_label(self, row):
    labels = self.table_model.row_labels
    return labels[row] if 0 <= row < len(labels) else None
select_row_label(label)

The row with this label made current, selected and scrolled to, as a click on it would.

Source code in src\muvis_align\ui\ModelTable.py
93
94
95
96
97
98
def select_row_label(self, label):
    """The row with this label made current, selected and scrolled to, as a click on it would."""
    index = self.table_model.index(self.table_model.row_labels.index(label), 0)
    self.selectionModel().setCurrentIndex(
        index, QItemSelectionModel.SelectionFlag.ClearAndSelect | QItemSelectionModel.SelectionFlag.Rows)
    self.scrollTo(index)
setValue(value)
Source code in src\muvis_align\ui\ModelTable.py
73
74
75
76
77
78
79
def setValue(self, value):
    # (rows, row labels, column labels), or magicgui Table's {'data', 'index', 'columns'}
    if isinstance(value, dict):
        value = (value.get('data', []), value.get('index', []), value.get('columns', []))
    rows, row_labels, column_labels = value if value else ([], [], [])
    self.table_model.set_rows(rows, row_labels, column_labels)
    self.valueChanged.emit(self.value())
set_background(background)

background(value, column_label) gives a cell's QColor, asked only for the cells shown.

Source code in src\muvis_align\ui\ModelTable.py
81
82
83
84
def set_background(self, background):
    """`background(value, column_label)` gives a cell's QColor, asked only for the cells shown."""
    self.table_model.background = background
    self.table_model._backgrounds = {}
value()
Source code in src\muvis_align\ui\ModelTable.py
69
70
71
def value(self):
    model = self.table_model
    return {'data': model.rows, 'index': model.row_labels, 'columns': model.column_labels}

NapariDaskProgress

NapariDaskProgress

Bases: Callback

Source code in src\muvis_align\ui\NapariDaskProgress.py
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
class NapariDaskProgress(Callback):
    def __init__(self, progress_class=None, desc="Dask computation", **progress_kwargs):
        from napari.utils import progress

        self.progress_class = progress_class or progress
        self.desc = desc
        self.progress_kwargs = progress_kwargs
        self._pbar = None
        self._started = False

    def _start(self, dsk):
        total = len(dsk)
        self._pbar = self.progress_class(
            total=total,
            desc=self.desc,
            **self.progress_kwargs,
        )
        self._pbar.__enter__()
        self._started = True

    def _posttask(self, key, result, dsk, state, id):
        if self._pbar is not None:
            self._pbar.update(1)

    def _finish(self, dsk, state, errored):
        if self._pbar is not None and self._started:
            self._pbar.__exit__(None, None, None)
        self._pbar = None
        self._started = False
desc = desc instance-attribute
progress_class = progress_class or progress instance-attribute
progress_kwargs = progress_kwargs instance-attribute
__init__(progress_class=None, desc='Dask computation', **progress_kwargs)
Source code in src\muvis_align\ui\NapariDaskProgress.py
 5
 6
 7
 8
 9
10
11
12
def __init__(self, progress_class=None, desc="Dask computation", **progress_kwargs):
    from napari.utils import progress

    self.progress_class = progress_class or progress
    self.desc = desc
    self.progress_kwargs = progress_kwargs
    self._pbar = None
    self._started = False

NapariMVSProgress

NapariMVSProgress
Source code in src\muvis_align\ui\NapariMVSProgress.py
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
class NapariMVSProgress:
    def __init__(
            self,
            tqdm_class=None,
            patch_fusion=True,
            patch_registration=False,
            patch_param_resolution=False,
            min_interval=0.1,
            **tqdm_kwargs,
    ):
        from napari.utils import progress

        self.tqdm_class = tqdm_class or progress
        self.patch_fusion = patch_fusion
        self.patch_registration = patch_registration
        self.patch_param_resolution = patch_param_resolution
        self.min_interval = min_interval
        self.tqdm_kwargs = tqdm_kwargs
        self._patched = []

    def __enter__(self):
        for module, replacement in self._get_patch_targets():
            if hasattr(module, "tqdm"):
                self._patched.append((module, module.tqdm))
                module.tqdm = replacement

        return self

    def __exit__(self, exc_type, exc_value, traceback):
        for module, original_tqdm in reversed(self._patched):
            module.tqdm = original_tqdm
        self._patched.clear()
        return False

    def _get_patch_targets(self):
        targets = []

        if self.patch_fusion:
            try:
                import multiview_stitcher.fusion._core as mvs_fusion_core

                import multiview_stitcher.ngff_utils as mvs_ngff_utils

                targets.append((mvs_fusion_core, self._create_progress_replacement()))
                targets.append((mvs_ngff_utils, self._create_pyramid_replacement()))
            except ImportError:
                pass

        registration_replacement = self._create_qt_friendly_replacement()

        if self.patch_registration:
            try:
                import multiview_stitcher.registration as mvs_registration

                targets.append((mvs_registration, registration_replacement))
            except ImportError:
                pass

        if self.patch_param_resolution:
            try:
                import multiview_stitcher.param_resolution as mvs_param_resolution

                targets.append((mvs_param_resolution, registration_replacement))
            except ImportError:
                pass

        return targets

    def _create_progress_replacement(self):
        if not self.tqdm_kwargs:
            return self.tqdm_class

        tqdm_class = self.tqdm_class
        tqdm_kwargs = self.tqdm_kwargs

        class _ConfiguredTqdm(tqdm_class):
            def __init__(self, *args, **kwargs):
                merged_kwargs = {**tqdm_kwargs, **kwargs}
                super().__init__(*args, **merged_kwargs)

        return _ConfiguredTqdm

    def _create_pyramid_replacement(self):
        # the pyramid writer opens one unnamed bar per level it downsamples, level 0 being fused before it
        tqdm_class = self._create_progress_replacement()
        levels = itertools.count(1)

        def pyramid_tqdm(*args, **kwargs):
            return tqdm_class(*args, **({'desc': f'Pyramid level {next(levels)}'} | kwargs))

        return pyramid_tqdm

    def _create_qt_friendly_replacement(self):
        tqdm_class = self.tqdm_class
        tqdm_kwargs = self.tqdm_kwargs
        min_interval = self.min_interval

        class _ConfiguredQtNapariTqdm(_QtNapariTqdm):
            def __init__(self, *args, **kwargs):
                merged_kwargs = {**tqdm_kwargs, **kwargs}
                super().__init__(
                    *args,
                    tqdm_class=tqdm_class,
                    min_interval=min_interval,
                    **merged_kwargs,
                )

        return _ConfiguredQtNapariTqdm
min_interval = min_interval instance-attribute
patch_fusion = patch_fusion instance-attribute
patch_param_resolution = patch_param_resolution instance-attribute
patch_registration = patch_registration instance-attribute
tqdm_class = tqdm_class or progress instance-attribute
tqdm_kwargs = tqdm_kwargs instance-attribute
__enter__()
Source code in src\muvis_align\ui\NapariMVSProgress.py
26
27
28
29
30
31
32
def __enter__(self):
    for module, replacement in self._get_patch_targets():
        if hasattr(module, "tqdm"):
            self._patched.append((module, module.tqdm))
            module.tqdm = replacement

    return self
__exit__(exc_type, exc_value, traceback)
Source code in src\muvis_align\ui\NapariMVSProgress.py
34
35
36
37
38
def __exit__(self, exc_type, exc_value, traceback):
    for module, original_tqdm in reversed(self._patched):
        module.tqdm = original_tqdm
    self._patched.clear()
    return False
__init__(tqdm_class=None, patch_fusion=True, patch_registration=False, patch_param_resolution=False, min_interval=0.1, **tqdm_kwargs)
Source code in src\muvis_align\ui\NapariMVSProgress.py
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
def __init__(
        self,
        tqdm_class=None,
        patch_fusion=True,
        patch_registration=False,
        patch_param_resolution=False,
        min_interval=0.1,
        **tqdm_kwargs,
):
    from napari.utils import progress

    self.tqdm_class = tqdm_class or progress
    self.patch_fusion = patch_fusion
    self.patch_registration = patch_registration
    self.patch_param_resolution = patch_param_resolution
    self.min_interval = min_interval
    self.tqdm_kwargs = tqdm_kwargs
    self._patched = []

NapariPhaseProgress

NapariPhaseProgress

One napari progress bar, filling once, for a whole user-facing operation.

Used as a progress_factory: each phase asking for a bar gets a slice of this one, so the bar runs empty to full once, never restarting or going backwards.

phases is the number of phase-sized units the operation expects and sizes the slices; extra phases take from what is left, missing ones are filled in at the end. A phase worth several siblings claims weight=n units. The bar is painted when the operation starts, before its first phase reports.

emit makes a headless twin for work off the Qt thread, reporting positions instead of moving a bar, as Qt widgets may only be touched from their own thread; build it with worker_twin() so it starts where the bar is.

A long operation logs its progress every heartbeat_seconds, to tell a slow phase from a hung one. The bar keeps the operation's description; only a long step (see set_step()) is named after it.

Source code in src\muvis_align\ui\NapariPhaseProgress.py
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
class NapariPhaseProgress:
    """One napari progress bar, filling once, for a whole user-facing operation.

    Used as a progress_factory: each phase asking for a bar gets a slice of this one, so the bar runs
    empty to full once, never restarting or going backwards.

    `phases` is the number of phase-sized units the operation expects and sizes the slices; extra phases
    take from what is left, missing ones are filled in at the end. A phase worth several siblings claims
    weight=n units. The bar is painted when the operation starts, before its first phase reports.

    `emit` makes a headless twin for work off the Qt thread, reporting positions instead of moving a bar,
    as Qt widgets may only be touched from their own thread; build it with worker_twin() so it starts
    where the bar is.

    A long operation logs its progress every `heartbeat_seconds`, to tell a slow phase from a hung one.
    The bar keeps the operation's description; only a long step (see set_step()) is named after it.
    """

    # just the time estimate: napari's eta label is everything after the last '|'
    bar_format = '{desc}|{elapsed}<{remaining}'

    # ticks of the whole operation; a phase maps its own steps onto its slice of these
    ticks = 1000

    # long enough that a short operation never logs, short enough to see a long one is alive
    heartbeat_seconds = 30

    # the bar, and a worker's position, shown at most this often: each move is a redraw, or a signal to the Qt thread
    min_update_seconds = 1.0

    # share of the remainder for the last expected phase: only the operation's end fills the bar,
    # and an unexpected extra phase still has room
    last_phase_share = 0.9

    def __init__(self, progress_class=None, desc=None, phases=1, min_duration=0.0, emit=None,
                 emit_step=None, **progress_kwargs):
        from napari.utils import progress

        self.progress_class = progress_class or progress
        self.emit = emit
        self.emit_step = emit_step
        self.desc = desc
        self.step = None
        progress_kwargs.setdefault('bar_format', self.bar_format)
        self.phases = max(int(phases), 1)
        self.phases_left = self.phases
        self.min_duration = max(float(min_duration), 0.0)
        self.progress_kwargs = progress_kwargs
        self._pbar = None
        self._started_at = None
        self._position = 0.0
        self._target = 0.0
        self._moving = False
        self._shown_at = None
        self._heartbeat = None
        self._done = threading.Event()

    def __enter__(self):
        self._started_at = time.monotonic()
        if self.emit is None:
            kwargs = dict(self.progress_kwargs)
            kwargs['total'] = self.ticks
            if self.desc is not None:
                kwargs['desc'] = self.desc
            self._pbar = self.progress_class(**kwargs)
            _paint_now()
        self._start_heartbeat()
        return self

    def __call__(self, total=None, desc=None, weight=1, **_):
        return _ProgressPhase(self, total, desc, weight)

    def worker_twin(self, emit, emit_step=None):
        """A bar-less stand-in for work on another thread, starting where this bar is. Hand its final state
        back with continue_from().
        """
        twin = NapariPhaseProgress(emit=emit, emit_step=emit_step, phases=self.phases)
        twin._position = self._position
        twin._target = self._target
        twin.min_update_seconds = self.min_update_seconds
        twin.last_phase_share = self.last_phase_share
        twin.phases_left = self.phases_left
        return twin

    def continue_from(self, twin):
        """Take over a twin's position and phases used, so later phases do not re-divide used slices."""
        self.phases_left = twin.phases_left
        self._move_to(twin._target, force=True)

    def set_step(self, step):
        """Name the long step the operation is on (a fusion level) in brackets after its description, on the bar and
        in the log."""
        if self.emit is not None:
            if self.emit_step is not None:
                self.emit_step(step)
            return
        self.step = step
        if self._pbar is not None:
            self._pbar.set_description(self.label)

    @property
    def label(self):
        if self.desc and self.step:
            return f'{self.desc} ({self.step})'
        return self.desc or self.step or 'Working'

    def ensure_phases(self, phases):
        """Make room for a nested operation reporting `phases` of its own into this bar, so it does not keep
        re-dividing the same remainder.
        """
        self.phases_left = max(self.phases_left, float(phases))

    # a filled bar is held before closing: one painted frame is not reliably seen over a remote display
    completion_dwell_seconds = 0.4

    def _report_completed(self):
        """Show, and log, that the operation reached 100%, also when reserved phases never reported."""
        if self.emit is not None:
            return
        elapsed = time.monotonic() - (self._started_at or time.monotonic())
        if self.heartbeat_seconds and elapsed >= self.heartbeat_seconds:
            # only after a heartbeat: a short operation logging just 100% says nothing
            so_far = (f'{elapsed / 60:.1f} minutes' if elapsed >= 60 else f'{elapsed:.0f} seconds')
            logging.info(f'{self.label}: 100% ({so_far}){print_memory_usage()}')
        if elapsed >= self.completion_dwell_seconds:
            time.sleep(self.completion_dwell_seconds)

    def _start_heartbeat(self):
        if self.emit is not None or not self.heartbeat_seconds:
            # a twin reports through the one that owns the bar, which does the logging
            return
        self._done.clear()

        def heartbeat():
            while not self._done.wait(self.heartbeat_seconds):
                elapsed = time.monotonic() - (self._started_at or time.monotonic())
                so_far = (f'{elapsed / 60:.1f} minutes' if elapsed >= 60
                          else f'{elapsed:.0f} seconds')
                logging.info(f'{self.label}:'
                             f' {self._position / self.ticks * 100:.0f}%'
                             f' ({so_far} so far){print_memory_usage()}')

        self._heartbeat = threading.Thread(target=heartbeat, daemon=True,
                                           name='muvis-align progress')
        self._heartbeat.start()

    def __exit__(self, exc_type, exc_value, traceback):
        self._done.set()
        self._heartbeat = None
        if self._pbar is not None:
            if exc_type is None:
                self._move_to(self.ticks, force=True)
                # filling and closing in one pass would read as giving up rather than finishing
                _paint_now()
                self._report_completed()
            self._pbar.close()
            self._pbar = None
        if self.emit is None:
            # reset so a factory outliving its operation reports on a new bar, not a closed one
            # (a twin is never reused)
            self.phases_left = self.phases
            self.step = None
            self._position = 0.0
            self._target = 0.0
            self._shown_at = None
        if self._started_at is not None and self.min_duration > 0:
            # an instant operation would otherwise flash the activity dock open and shut
            wait_s = self.min_duration - (time.monotonic() - self._started_at)
            if wait_s > 0:
                time.sleep(wait_s)
        self._started_at = None
        return False

    @property
    def tqdm_class(self):
        """A tqdm stand-in whose bars are phases of this one, so a third-party library moves this bar
        instead of opening its own.
        """
        owner = self

        class _PhaseTqdm(_ProgressPhase):
            def __init__(self, iterable=None, desc=None, total=None, **kwargs):
                if total is None:
                    try:
                        total = len(iterable)
                    except TypeError:
                        total = None
                super().__init__(owner, total=total, desc=desc)
                self.iterable = iterable
                self._open = False

            def __iter__(self):
                self._start()
                for item in self.iterable:
                    yield item
                    self.update(1)
                self.close()

            def _start(self):
                if not self._open:
                    self.__enter__()
                    self._open = True
                    if self.desc:
                        owner.set_step(self.desc)

            def update(self, n=1):
                self._start()
                super().update(n)

            def set_description(self, desc=None, refresh=True):
                # a library's bar over several steps (fusion's levels) names each as it starts it
                if desc:
                    self.desc = desc
                    if self._open:
                        owner.set_step(desc)

            def close(self):
                if self._open:
                    self._open = False
                    self.__exit__(None, None, None)

            def __getattr__(self, name):
                # a library may touch any of tqdm's wide surface: a no-op beats an AttributeError mid-loop
                return lambda *args, **kwargs: None

        return _PhaseTqdm

    # share of its remaining slice an update of a phase without a step count crosses: moving, never arriving
    undeclared_step_share = 0.25

    def _check_cancelled(self):
        # only a worker twin stops: the Qt side's own phases (building the view) are never left half done
        if self.emit is not None:
            raise_if_cancelled()

    def _begin_phase(self, total, desc=None, weight=1):
        self._check_cancelled()
        remaining = self.ticks - self._position
        weight = max(float(weight), 1.0)
        if self.phases_left > weight:
            span = remaining * weight / self.phases_left
        else:
            # the last expected phase, or one never expected at all, takes most of the remainder
            span = remaining * self.last_phase_share
        self.phases_left = max(self.phases_left - weight, 0)
        return _PhaseSlice(start=self._position, span=span, total=total)

    def _advance_phase(self, phase_slice, n=1):
        self._check_cancelled()
        phase_slice.done += n
        if phase_slice.total:
            fraction = min(phase_slice.done / phase_slice.total, 1.0)
        else:
            # still moves on every update: parked at a fixed fraction it would look hung
            fraction = 1.0 - (1.0 - self.undeclared_step_share) ** phase_slice.done
        self._move_to(phase_slice.start + phase_slice.span * fraction)

    def _end_phase(self, phase_slice):
        # shown at once: a phase's last update may have been held back, and the next phase can be long
        self._move_to(phase_slice.start + phase_slice.span, force=True)

    def set_position(self, position):
        """Move the bar to a position reported by a twin on a worker thread, which already reports at most once a
        second."""
        self._move_to(position, force=True)

    def _move_to(self, position, force=False):
        # never re-entered: a repaint pumps the Qt loop, which can deliver the next position back here and
        # overflow the stack; a move in progress just raises its target
        self._target = min(max(position, self._position, self._target), self.ticks)
        now = time.monotonic()
        if self._moving or (not force and self._shown_at is not None
                            and now - self._shown_at < self.min_update_seconds):
            return
        self._shown_at = now
        self._moving = True
        try:
            while True:
                # the bar counts in whole ticks, so only a move that crosses one shows up
                step = int(self._target) - int(self._position)
                self._position = self._target
                if step <= 0:
                    break
                if self.emit is not None:
                    self.emit(int(self._position))
                elif self._pbar is not None:
                    self._pbar.update(step)
                if self._target <= self._position:
                    break
        finally:
            self._moving = False
bar_format = '{desc}|{elapsed}<{remaining}' class-attribute instance-attribute
completion_dwell_seconds = 0.4 class-attribute instance-attribute
desc = desc instance-attribute
emit = emit instance-attribute
emit_step = emit_step instance-attribute
heartbeat_seconds = 30 class-attribute instance-attribute
label property
last_phase_share = 0.9 class-attribute instance-attribute
min_duration = max(float(min_duration), 0.0) instance-attribute
min_update_seconds = 1.0 class-attribute instance-attribute
phases = max(int(phases), 1) instance-attribute
phases_left = self.phases instance-attribute
progress_class = progress_class or progress instance-attribute
progress_kwargs = progress_kwargs instance-attribute
step = None instance-attribute
ticks = 1000 class-attribute instance-attribute
tqdm_class property

A tqdm stand-in whose bars are phases of this one, so a third-party library moves this bar instead of opening its own.

undeclared_step_share = 0.25 class-attribute instance-attribute
__call__(total=None, desc=None, weight=1, **_)
Source code in src\muvis_align\ui\NapariPhaseProgress.py
86
87
def __call__(self, total=None, desc=None, weight=1, **_):
    return _ProgressPhase(self, total, desc, weight)
__enter__()
Source code in src\muvis_align\ui\NapariPhaseProgress.py
74
75
76
77
78
79
80
81
82
83
84
def __enter__(self):
    self._started_at = time.monotonic()
    if self.emit is None:
        kwargs = dict(self.progress_kwargs)
        kwargs['total'] = self.ticks
        if self.desc is not None:
            kwargs['desc'] = self.desc
        self._pbar = self.progress_class(**kwargs)
        _paint_now()
    self._start_heartbeat()
    return self
__exit__(exc_type, exc_value, traceback)
Source code in src\muvis_align\ui\NapariPhaseProgress.py
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
def __exit__(self, exc_type, exc_value, traceback):
    self._done.set()
    self._heartbeat = None
    if self._pbar is not None:
        if exc_type is None:
            self._move_to(self.ticks, force=True)
            # filling and closing in one pass would read as giving up rather than finishing
            _paint_now()
            self._report_completed()
        self._pbar.close()
        self._pbar = None
    if self.emit is None:
        # reset so a factory outliving its operation reports on a new bar, not a closed one
        # (a twin is never reused)
        self.phases_left = self.phases
        self.step = None
        self._position = 0.0
        self._target = 0.0
        self._shown_at = None
    if self._started_at is not None and self.min_duration > 0:
        # an instant operation would otherwise flash the activity dock open and shut
        wait_s = self.min_duration - (time.monotonic() - self._started_at)
        if wait_s > 0:
            time.sleep(wait_s)
    self._started_at = None
    return False
__init__(progress_class=None, desc=None, phases=1, min_duration=0.0, emit=None, emit_step=None, **progress_kwargs)
Source code in src\muvis_align\ui\NapariPhaseProgress.py
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
def __init__(self, progress_class=None, desc=None, phases=1, min_duration=0.0, emit=None,
             emit_step=None, **progress_kwargs):
    from napari.utils import progress

    self.progress_class = progress_class or progress
    self.emit = emit
    self.emit_step = emit_step
    self.desc = desc
    self.step = None
    progress_kwargs.setdefault('bar_format', self.bar_format)
    self.phases = max(int(phases), 1)
    self.phases_left = self.phases
    self.min_duration = max(float(min_duration), 0.0)
    self.progress_kwargs = progress_kwargs
    self._pbar = None
    self._started_at = None
    self._position = 0.0
    self._target = 0.0
    self._moving = False
    self._shown_at = None
    self._heartbeat = None
    self._done = threading.Event()
continue_from(twin)

Take over a twin's position and phases used, so later phases do not re-divide used slices.

Source code in src\muvis_align\ui\NapariPhaseProgress.py
101
102
103
104
def continue_from(self, twin):
    """Take over a twin's position and phases used, so later phases do not re-divide used slices."""
    self.phases_left = twin.phases_left
    self._move_to(twin._target, force=True)
ensure_phases(phases)

Make room for a nested operation reporting phases of its own into this bar, so it does not keep re-dividing the same remainder.

Source code in src\muvis_align\ui\NapariPhaseProgress.py
123
124
125
126
127
def ensure_phases(self, phases):
    """Make room for a nested operation reporting `phases` of its own into this bar, so it does not keep
    re-dividing the same remainder.
    """
    self.phases_left = max(self.phases_left, float(phases))
set_position(position)

Move the bar to a position reported by a twin on a worker thread, which already reports at most once a second.

Source code in src\muvis_align\ui\NapariPhaseProgress.py
278
279
280
281
def set_position(self, position):
    """Move the bar to a position reported by a twin on a worker thread, which already reports at most once a
    second."""
    self._move_to(position, force=True)
set_step(step)

Name the long step the operation is on (a fusion level) in brackets after its description, on the bar and in the log.

Source code in src\muvis_align\ui\NapariPhaseProgress.py
106
107
108
109
110
111
112
113
114
115
def set_step(self, step):
    """Name the long step the operation is on (a fusion level) in brackets after its description, on the bar and
    in the log."""
    if self.emit is not None:
        if self.emit_step is not None:
            self.emit_step(step)
        return
    self.step = step
    if self._pbar is not None:
        self._pbar.set_description(self.label)
worker_twin(emit, emit_step=None)

A bar-less stand-in for work on another thread, starting where this bar is. Hand its final state back with continue_from().

Source code in src\muvis_align\ui\NapariPhaseProgress.py
89
90
91
92
93
94
95
96
97
98
99
def worker_twin(self, emit, emit_step=None):
    """A bar-less stand-in for work on another thread, starting where this bar is. Hand its final state
    back with continue_from().
    """
    twin = NapariPhaseProgress(emit=emit, emit_step=emit_step, phases=self.phases)
    twin._position = self._position
    twin._target = self._target
    twin.min_update_seconds = self.min_update_seconds
    twin.last_phase_share = self.last_phase_share
    twin.phases_left = self.phases_left
    return twin
SilentProgress

A progress factory that moves no bar but still stops on a cancel: for a short step inside an operation whose bar is sized for its long one - its phases would otherwise take that one's share.

Source code in src\muvis_align\ui\NapariPhaseProgress.py
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
class SilentProgress:
    """A progress factory that moves no bar but still stops on a cancel: for a short step inside an operation whose
    bar is sized for its long one - its phases would otherwise take that one's share."""

    def __call__(self, total=None, desc=None, weight=1, **_):
        return _SilentPhase(total=total)

    def ensure_phases(self, phases):
        pass

    def worker_twin(self, emit, emit_step=None):
        return self

    def continue_from(self, twin):
        pass

    def set_position(self, *_):
        pass

    def set_step(self, *_):
        pass

    @property
    def tqdm_class(self):
        return _SilentPhase

    def __enter__(self):
        return self

    def __exit__(self, *_):
        return False
tqdm_class property
__call__(total=None, desc=None, weight=1, **_)
Source code in src\muvis_align\ui\NapariPhaseProgress.py
354
355
def __call__(self, total=None, desc=None, weight=1, **_):
    return _SilentPhase(total=total)
__enter__()
Source code in src\muvis_align\ui\NapariPhaseProgress.py
376
377
def __enter__(self):
    return self
__exit__(*_)
Source code in src\muvis_align\ui\NapariPhaseProgress.py
379
380
def __exit__(self, *_):
    return False
continue_from(twin)
Source code in src\muvis_align\ui\NapariPhaseProgress.py
363
364
def continue_from(self, twin):
    pass
ensure_phases(phases)
Source code in src\muvis_align\ui\NapariPhaseProgress.py
357
358
def ensure_phases(self, phases):
    pass
set_position(*_)
Source code in src\muvis_align\ui\NapariPhaseProgress.py
366
367
def set_position(self, *_):
    pass
set_step(*_)
Source code in src\muvis_align\ui\NapariPhaseProgress.py
369
370
def set_step(self, *_):
    pass
worker_twin(emit, emit_step=None)
Source code in src\muvis_align\ui\NapariPhaseProgress.py
360
361
def worker_twin(self, emit, emit_step=None):
    return self

ParamWidget

ParamWidget
Source code in src\muvis_align\ui\ParamWidget.py
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
class ParamWidget:
    def __init__(self, param_name, widget, interface, to_str=False, number_type=None):
        self.param_name = param_name
        self.widget = widget
        self.interface = interface
        self.to_str = to_str
        self.number_type = number_type
        self.is_invalid_number = False

    def get_value(self):
        return self.widget.get_value()

    def set_value(self, value, choices=None):
        if choices is not None:
            self.set_choices(choices)
        if self.number_type is not None:
            self.widget.set_value('' if value is None else str(value))
        elif isinstance(self.widget, Table):
            self._set_table_value(value)
        else:
            self.widget.set_value(value)

    def _set_table_value(self, value):
        # signals blocked: the header view handles a signal per row label, quadratically; told once at the end
        model = self.widget.native.model()
        model.blockSignals(True)
        try:
            self.widget.set_value(value)
        finally:
            model.blockSignals(False)
        model.layoutChanged.emit()
        for orientation, count in ((Qt.Vertical, model.rowCount()), (Qt.Horizontal, model.columnCount())):
            if count:
                model.headerDataChanged.emit(orientation, 0, count - 1)

    def set_choices(self, choices):
        self.widget.choices = to_magicgui_choices(choices)

    def value_changed(self, value):
        if self.number_type is not None:
            text = str(value).strip()
            try:
                value = None if not text else (int(text) if self.number_type == 'integer' else float(text))
            except ValueError:
                # warned once until valid again, as this runs on every keystroke; the last valid value is kept
                if not self.is_invalid_number:
                    show_warning(f'{self.param_name}: {text!r} is not a{"n integer" if self.number_type == "integer" else " number"}')
                self.is_invalid_number = True
                return
            self.is_invalid_number = False
        elif isinstance(value, dict):
            value0 = self.get_value()
            if isinstance(value0, dict):
                value = update_dict_value(value0, value)
        elif isinstance(value, list):
            value = [to_posix_path(str(item)) for item in value]
        elif self.to_str:
            # the widget's text is left as typed: rewriting it live would erase a trailing '/' mid-typing
            value = to_posix_path(str(value))
        self.interface.change_param(self.param_name, value)

    def set_table_column_resize_mode(self, mode=QHeaderView.Stretch):
        self.widget.native.horizontalHeader().setSectionResizeMode(mode)
interface = interface instance-attribute
is_invalid_number = False instance-attribute
number_type = number_type instance-attribute
param_name = param_name instance-attribute
to_str = to_str instance-attribute
widget = widget instance-attribute
__init__(param_name, widget, interface, to_str=False, number_type=None)
Source code in src\muvis_align\ui\ParamWidget.py
11
12
13
14
15
16
17
def __init__(self, param_name, widget, interface, to_str=False, number_type=None):
    self.param_name = param_name
    self.widget = widget
    self.interface = interface
    self.to_str = to_str
    self.number_type = number_type
    self.is_invalid_number = False
get_value()
Source code in src\muvis_align\ui\ParamWidget.py
19
20
def get_value(self):
    return self.widget.get_value()
set_choices(choices)
Source code in src\muvis_align\ui\ParamWidget.py
45
46
def set_choices(self, choices):
    self.widget.choices = to_magicgui_choices(choices)
set_table_column_resize_mode(mode=QHeaderView.Stretch)
Source code in src\muvis_align\ui\ParamWidget.py
71
72
def set_table_column_resize_mode(self, mode=QHeaderView.Stretch):
    self.widget.native.horizontalHeader().setSectionResizeMode(mode)
set_value(value, choices=None)
Source code in src\muvis_align\ui\ParamWidget.py
22
23
24
25
26
27
28
29
30
def set_value(self, value, choices=None):
    if choices is not None:
        self.set_choices(choices)
    if self.number_type is not None:
        self.widget.set_value('' if value is None else str(value))
    elif isinstance(self.widget, Table):
        self._set_table_value(value)
    else:
        self.widget.set_value(value)
value_changed(value)
Source code in src\muvis_align\ui\ParamWidget.py
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
def value_changed(self, value):
    if self.number_type is not None:
        text = str(value).strip()
        try:
            value = None if not text else (int(text) if self.number_type == 'integer' else float(text))
        except ValueError:
            # warned once until valid again, as this runs on every keystroke; the last valid value is kept
            if not self.is_invalid_number:
                show_warning(f'{self.param_name}: {text!r} is not a{"n integer" if self.number_type == "integer" else " number"}')
            self.is_invalid_number = True
            return
        self.is_invalid_number = False
    elif isinstance(value, dict):
        value0 = self.get_value()
        if isinstance(value0, dict):
            value = update_dict_value(value0, value)
    elif isinstance(value, list):
        value = [to_posix_path(str(item)) for item in value]
    elif self.to_str:
        # the widget's text is left as typed: rewriting it live would erase a trailing '/' mid-typing
        value = to_posix_path(str(value))
    self.interface.change_param(self.param_name, value)
create_dict_of_lists(data, columns)
Source code in src\muvis_align\ui\ParamWidget.py
83
84
def create_dict_of_lists(data, columns):
    return {column: [x[columni] for x in data] for columni, column in enumerate(columns)}
update_dict_value(old_value, new_value)
Source code in src\muvis_align\ui\ParamWidget.py
75
76
77
78
79
80
def update_dict_value(old_value, new_value):
    columns = old_value.get('columns', [])
    data = old_value.get('data', [[]])
    data[new_value['row']][new_value['column']] = new_value['data']
    dict_of_lists = create_dict_of_lists(data, columns)
    return dict_of_lists

PathList

PathList

Bases: ValueWidget

A magicgui widget for several input paths. Value is a list of path or glob strings.

Source code in src\muvis_align\ui\PathList.py
114
115
116
117
118
119
120
class PathList(ValueWidget):
    """A magicgui widget for several input paths. Value is a list of path or glob strings."""

    def __init__(self, **kwargs):
        # create_widget(widget_type="...PathList") leaves its string 'widget_type' in kwargs
        kwargs.pop('widget_type', None)
        super().__init__(widget_type=_QPathListWidget, **kwargs)
__init__(**kwargs)
Source code in src\muvis_align\ui\PathList.py
117
118
119
120
def __init__(self, **kwargs):
    # create_widget(widget_type="...PathList") leaves its string 'widget_type' in kwargs
    kwargs.pop('widget_type', None)
    super().__init__(widget_type=_QPathListWidget, **kwargs)
QPathList

Bases: QWidget

An editable list of paths or globs, each row typed in place or added with a file or folder dialog.

Source code in src\muvis_align\ui\PathList.py
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
class QPathList(QWidget):
    """An editable list of paths or globs, each row typed in place or added with a file or folder dialog."""

    pathsChanged = Signal(list)

    def __init__(self, parent=None):
        super().__init__(parent)
        # the directory stored paths are relative to; set by the owner, as it changes with the project
        self.base_dir = lambda: None
        self.list_widget = QListWidget()
        self.list_widget.setSelectionMode(QAbstractItemView.SingleSelection)
        self.list_widget.setEditTriggers(QAbstractItemView.DoubleClicked | QAbstractItemView.EditKeyPressed)
        self.list_widget.itemChanged.connect(self._item_edited)
        buttons = QHBoxLayout()
        for label, function in (('Add files...', self._add_files), ('Add folder...', self._add_folder),
                                ('Remove', self._remove), ('Up', lambda: self._move(-1)),
                                ('Down', lambda: self._move(1))):
            button = QPushButton(label)
            button.clicked.connect(function)
            buttons.addWidget(button)
        layout = QVBoxLayout(self)
        layout.setContentsMargins(0, 0, 0, 0)
        layout.addWidget(self.list_widget)
        layout.addLayout(buttons)

    def paths(self):
        return [self.list_widget.item(row).text() for row in range(self.list_widget.count())]

    def setPaths(self, paths):
        # not emitted: only a user's edit is a change to store
        self.list_widget.blockSignals(True)
        self.list_widget.clear()
        for path in paths:
            self._append(path)
        self.list_widget.blockSignals(False)

    def _append(self, path):
        item = QListWidgetItem(path)
        item.setFlags(item.flags() | Qt.ItemIsEditable)
        self.list_widget.addItem(item)

    def _item_edited(self, item):
        if not item.text().strip():
            self.list_widget.takeItem(self.list_widget.row(item))
        self.pathsChanged.emit(self.paths())

    def _start_dir(self):
        base_dir = self.base_dir()
        item = self.list_widget.currentItem()
        if item is None and self.list_widget.count():
            item = self.list_widget.item(self.list_widget.count() - 1)
        if item is not None:
            path = resolve_to_project_dir(item.text(), base_dir)
            directory = path if os.path.isdir(path) else pattern_base_dir(path)
            if os.path.isdir(directory):
                return directory
        return base_dir or ''

    def _add_paths(self, paths):
        for path in paths:
            # shown as stored: relative to the project, as the dialogs report absolute paths
            self._append(relativize_to_project_dir(path, self.base_dir()))
        if paths:
            self.list_widget.setCurrentRow(self.list_widget.count() - 1)
            self.pathsChanged.emit(self.paths())

    def _add_files(self):
        paths, _ = QFileDialog.getOpenFileNames(self, 'Add input files', self._start_dir())
        self._add_paths(paths)

    def _add_folder(self):
        path = QFileDialog.getExistingDirectory(self, 'Add input folder', self._start_dir())
        self._add_paths([path] if path else [])

    def _remove(self):
        row = self.list_widget.currentRow()
        if row >= 0:
            self.list_widget.takeItem(row)
            self.pathsChanged.emit(self.paths())

    def _move(self, step):
        row = self.list_widget.currentRow()
        target = row + step
        if row >= 0 and 0 <= target < self.list_widget.count():
            self.list_widget.blockSignals(True)
            self.list_widget.insertItem(target, self.list_widget.takeItem(row))
            self.list_widget.blockSignals(False)
            self.list_widget.setCurrentRow(target)
            self.pathsChanged.emit(self.paths())
base_dir = lambda: None instance-attribute
list_widget = QListWidget() instance-attribute
pathsChanged = Signal(list) class-attribute instance-attribute
__init__(parent=None)
Source code in src\muvis_align\ui\PathList.py
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
def __init__(self, parent=None):
    super().__init__(parent)
    # the directory stored paths are relative to; set by the owner, as it changes with the project
    self.base_dir = lambda: None
    self.list_widget = QListWidget()
    self.list_widget.setSelectionMode(QAbstractItemView.SingleSelection)
    self.list_widget.setEditTriggers(QAbstractItemView.DoubleClicked | QAbstractItemView.EditKeyPressed)
    self.list_widget.itemChanged.connect(self._item_edited)
    buttons = QHBoxLayout()
    for label, function in (('Add files...', self._add_files), ('Add folder...', self._add_folder),
                            ('Remove', self._remove), ('Up', lambda: self._move(-1)),
                            ('Down', lambda: self._move(1))):
        button = QPushButton(label)
        button.clicked.connect(function)
        buttons.addWidget(button)
    layout = QVBoxLayout(self)
    layout.setContentsMargins(0, 0, 0, 0)
    layout.addWidget(self.list_widget)
    layout.addLayout(buttons)
paths()
Source code in src\muvis_align\ui\PathList.py
38
39
def paths(self):
    return [self.list_widget.item(row).text() for row in range(self.list_widget.count())]
setPaths(paths)
Source code in src\muvis_align\ui\PathList.py
41
42
43
44
45
46
47
def setPaths(self, paths):
    # not emitted: only a user's edit is a change to store
    self.list_widget.blockSignals(True)
    self.list_widget.clear()
    for path in paths:
        self._append(path)
    self.list_widget.blockSignals(False)

QtNapariTqdm

bilayers_util

Helpers to validate bilayers config and derive magicgui-ready field specs.

BilayersField

Bases: BaseModel

Source code in src\muvis_align\ui\bilayers_util.py
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
class BilayersField(BaseModel):
    model_config = ConfigDict(extra='allow')

    name: str
    type: str
    label: str | None = None
    default: Any = None
    description: str | None = None
    section_id: str | None = None
    section_key: str | None = None
    cli_order: int = 0
    options: list[BilayersOption] | None = None
    file_count: str | None = None
    output_dir_set: bool = False

    @field_validator('type', mode='before')
    @classmethod
    def _normalize_type(cls, value):
        return str(value).lower()

    @field_validator('cli_order', mode='before')
    @classmethod
    def _normalize_cli_order(cls, value):
        if value is None or value == '':
            return 0
        return int(value)
cli_order = 0 class-attribute instance-attribute
default = None class-attribute instance-attribute
description = None class-attribute instance-attribute
file_count = None class-attribute instance-attribute
label = None class-attribute instance-attribute
model_config = ConfigDict(extra='allow') class-attribute instance-attribute
name instance-attribute
options = None class-attribute instance-attribute
output_dir_set = False class-attribute instance-attribute
section_id = None class-attribute instance-attribute
section_key = None class-attribute instance-attribute
type instance-attribute
BilayersOption

Bases: BaseModel

Source code in src\muvis_align\ui\bilayers_util.py
11
12
13
14
15
class BilayersOption(BaseModel):
    model_config = ConfigDict(extra='allow')

    label: str
    value: Any
label instance-attribute
model_config = ConfigDict(extra='allow') class-attribute instance-attribute
value instance-attribute
MagicGuiFieldSpec

Bases: BaseModel

Source code in src\muvis_align\ui\bilayers_util.py
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
class MagicGuiFieldSpec(BaseModel):
    section_id: str
    section_key: str | None = None
    param_name: str
    param_label: str
    widget_type: str
    value: Any = None
    description: str | None = None
    choices: dict[Any, str] = Field(default_factory=dict)
    file_count: str | None = None
    default: Any = None
    is_output: bool = False
    is_file_type: bool = False
    number_type: str | None = None


    def to_magicgui_choices(self):
        return to_magicgui_choices(self.choices)
choices = Field(default_factory=dict) class-attribute instance-attribute
default = None class-attribute instance-attribute
description = None class-attribute instance-attribute
file_count = None class-attribute instance-attribute
is_file_type = False class-attribute instance-attribute
is_output = False class-attribute instance-attribute
number_type = None class-attribute instance-attribute
param_label instance-attribute
param_name instance-attribute
section_id instance-attribute
section_key = None class-attribute instance-attribute
value = None class-attribute instance-attribute
widget_type instance-attribute
to_magicgui_choices()
Source code in src\muvis_align\ui\bilayers_util.py
62
63
def to_magicgui_choices(self):
    return to_magicgui_choices(self.choices)
bilayers_to_magicgui_field(template_item, section_id, section_params, widget_type_map)

Convert one bilayers field definition to a magicgui widget spec.

Source code in src\muvis_align\ui\bilayers_util.py
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
def bilayers_to_magicgui_field(template_item, section_id, section_params, widget_type_map):
    """Convert one bilayers field definition to a magicgui widget spec."""
    item = BilayersField.model_validate(template_item)
    widget_type = widget_type_map.get(item.type)
    if widget_type is None:
        raise ValueError(f'Unsupported type {item.type}')

    is_file_type = widget_type == 'FileEdit'
    choices = {}
    if widget_type == 'Dropdown' and item.options:
        choices = {option.value: option.label for option in item.options}

    value = section_params.get(item.name, item.default)
    number_type = None
    if item.type in ('integer', 'float') and item.default in (None, ''):
        # a spin box cannot be blank: a number field without a default is a text field, blank meaning None
        widget_type = 'LineEdit'
        number_type = item.type
        value = '' if value is None else str(int(value) if item.type == 'integer' else value)

    return MagicGuiFieldSpec(
        section_id=section_id,
        section_key=item.section_key,
        param_name=item.name,
        param_label=item.label or item.name,
        widget_type=widget_type,
        value=value,
        description=item.description,
        choices=choices,
        file_count=item.file_count,
        default=item.default,
        is_output=(section_id == 'output' or item.output_dir_set),
        is_file_type=is_file_type,
        number_type=number_type,
    )
get_section_dict(template, keys=None)

Group bilayers fields by section and return normalized dict items.

Source code in src\muvis_align\ui\bilayers_util.py
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
def get_section_dict(template, keys=None):
    """Group bilayers fields by section and return normalized dict items."""
    sections = {}
    if keys is None:
        keys = [None]

    for key in keys:
        items = template.get(key, []) if key is not None else template
        for raw_item in items:
            item = BilayersField.model_validate({**raw_item, 'section_key': key})
            section_id = item.section_id or key
            sections.setdefault(section_id, []).append(item)

    return {
        section_id: [item.model_dump() for item in sorted(section_items, key=lambda item: item.cli_order)]
        for section_id, section_items in sections.items()
    }
to_magicgui_choices(choices)
Source code in src\muvis_align\ui\bilayers_util.py
66
67
68
69
70
71
72
73
74
75
76
def to_magicgui_choices(choices):
    if isinstance(choices, dict):
        def _get_choice_label(choice):
            return choices[choice]

        return {
            'choices': choices.keys(),
            'key': _get_choice_label,
        }
    else:
        return choices

create_widgets

map_bilayers_to_widget_type = {'textbox': 'LineEdit', 'checkbox': 'CheckBox', 'radio': 'Radio', 'dropdown': 'Dropdown', 'integer': 'SpinBox', 'float': 'FloatSpinBox', 'table': 'Table', 'file': 'FileEdit', 'image': 'FileEdit', 'array': 'FileEdit', 'measurement': 'FileEdit', 'button': 'PushButton', 'pushbutton': 'PushButton', 'color': 'muvis_align.ui.MagicColorPicker.MagicColorPicker'} module-attribute
mirrored_params = {'registration': [('pre_processing', 'scale')]} module-attribute
create_project_widget(interface, function)
Source code in src\muvis_align\ui\create_widgets.py
81
82
83
84
85
86
87
88
89
90
def create_project_widget(interface, function):
    project_template = [
        {'name': 'project_path',
         'label': 'Project path',
         'type': 'file',
         'output_dir_set': True,
         'default': 'muvis_align_project.yml'}
    ]
    return create_section_container('project', project_template, interface,
                                    connect_changed=False, function=function, add_button=False)
create_section_container(section_id, section_template, interface, connect_changed=True, function=None, add_button=True, stored_sections=None)
Source code in src\muvis_align\ui\create_widgets.py
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
def create_section_container(section_id, section_template, interface,
                             connect_changed=True, function=None, add_button=True, stored_sections=None):
    # https://pyapp-kit.github.io/magicgui/widgets/
    # https://pyapp-kit.github.io/magicgui/api/widgets/create_widget/
    widgets = []
    for template in section_template:
        try:
            stored_section = (stored_sections or {}).get(template['name'], section_id)
            section_params = interface.params.get(stored_section, {})
            spec = bilayers_to_magicgui_field(
                template,
                section_id=section_id,
                section_params=section_params,
                widget_type_map=map_bilayers_to_widget_type,
            )

            widget_type = spec.widget_type
            if widget_type == 'Table' and spec.section_key == 'display_only':
                # read-only, and possibly a row per source or pair: a view that draws only the rows shown
                widget_type = 'muvis_align.ui.ModelTable.ModelTable'
            is_path_list = (spec.is_file_type and spec.section_key == 'inputs'
                            and 'multiple' in (spec.file_count or ''))
            if is_path_list:
                widget_type = 'muvis_align.ui.PathList.PathList'
            full_name = section_id + '.' + spec.param_name
            param_widget = ParamWidget(stored_section + '.' + spec.param_name, None, interface,
                                       to_str=spec.is_file_type, number_type=spec.number_type)

            options = {}
            if spec.widget_type == 'Dropdown' and spec.choices:
                options['choices'] = spec.to_magicgui_choices()

            if is_path_list:
                spec.value = path_param_to_list(spec.value) or None
            elif spec.is_file_type:
                options['mode'] = get_file_dialog_mode(spec.is_output, spec.file_count)
                ext = os.path.splitext(str(spec.default))[1]
                if ext:
                    options['filter'] = '*' + ext

            if spec.value is not None:
                widget = create_widget(
                    name=full_name,
                    label=spec.param_label,
                    widget_type=widget_type,
                    options=options,
                    value=spec.value,
                )
            else:
                widget = create_widget(
                    name=full_name,
                    label=spec.param_label,
                    widget_type=widget_type,
                    options=options,
                )
            if spec.is_file_type and options.get('mode') == 'w' and hasattr(widget, '_show_file_dialog'):
                widget._show_file_dialog = _show_optional_file_dialog_without_overwrite_confirm
            if spec.is_file_type and not is_path_list and section_id != 'project' \
                    and hasattr(widget, '_show_file_dialog'):
                widget._show_file_dialog = _project_relative_file_dialog(
                    widget, widget._show_file_dialog, getattr(interface, 'get_project_dir', lambda: None))
            if is_path_list:
                widget.native.base_dir = getattr(interface, 'get_project_dir', lambda: None)
            # explicitly setting value=None triggers type deduction, and results in an error
            param_widget.widget = widget
            if spec.description:
                widget.tooltip = spec.description
            if stored_section != section_id:
                widget.tooltip = f'{widget.tooltip or ""}\n(the {stored_section.replace("_", "-")} setting)'.strip()
            if function is not None:
                widget.changed.connect(function)
            interface_function = interface.get_function(spec.param_name)
            if interface_function is not None:
                widget.changed.connect(interface_function)
            selected_function = interface.get_function(spec.param_name + '_selected')
            if selected_function is not None and hasattr(widget.native, 'itemSelectionChanged'):
                widget.native.itemSelectionChanged.connect(selected_function)
            interface.param_widgets[full_name] = param_widget
            if connect_changed and spec.section_key != 'display_only':
                widget.changed.connect(param_widget.value_changed)
            else:
                # https://github.com/pyapp-kit/magicgui/issues/348
                widget.read_only = True
            widgets.append(widget)
        except Exception as e:
            logging.error(f'Error creating widget from {template}\n{e}')

    if add_button:
        name = section_id + '_process'
        full_name = section_id + '.' + name
        widget = create_widget(name=full_name, label='Process', widget_type='PushButton')
        interface_function = interface.get_function(name)
        if interface_function is not None:
            # while an operation runs the button reads Cancel
            if hasattr(interface, 'process_or_cancel'):
                widget.clicked.connect(functools.partial(interface.process_or_cancel, interface_function))
            else:
                widget.clicked.connect(interface_function)
        interface.param_widgets[full_name] = ParamWidget(full_name, widget, interface)
        widgets.append(widget)

    return Container(widgets=widgets)
create_template_widgets(interface)
Source code in src\muvis_align\ui\create_widgets.py
 97
 98
 99
100
101
102
103
104
105
106
def create_template_widgets(interface):
    widgets = {}
    for section_id, section_items in interface.template.items():
        mirrored = mirrored_params.get(section_id, [])
        mirrored_items = [item for stored_section, name in mirrored
                          for item in interface.template.get(stored_section, []) if item['name'] == name]
        stored_sections = {name: stored_section for stored_section, name in mirrored}
        widgets[section_id] = create_section_container(section_id, mirrored_items + section_items, interface,
                                                       stored_sections=stored_sections)
    return widgets
get_file_dialog_mode(is_output, file_count)
Source code in src\muvis_align\ui\create_widgets.py
213
214
215
216
217
218
219
220
221
def get_file_dialog_mode(is_output, file_count):
    # https://pyapp-kit.github.io/magicgui/api/widgets/FileEdit/
    if file_count and 'multiple' in file_count:
        mode = 'd'
    elif is_output:
        mode = 'w'
    else:
        mode = 'r'
    return mode

util

SOURCE_METADATA_FIELDS = {'source_position_z': ['position', 'z'], 'source_position_y': ['position', 'y'], 'source_position_x': ['position', 'x'], 'source_scale_z': ['scale', 'z'], 'source_scale_y': ['scale', 'y'], 'source_scale_x': ['scale', 'x'], 'source_rotation': ['rotation']} module-attribute

um_conversions = UM_CONVERSIONS | {'Ã…': 0.0001, 'A': 0.0001} module-attribute

OperationCancelled

Bases: Exception

The user cancelled the running operation.

Source code in src\muvis_align\util.py
672
673
class OperationCancelled(Exception):
    """The user cancelled the running operation."""

adjust_sbemimage_properties(translation, scale, size, filename, sbemimage_config)

Source code in src\muvis_align\util.py
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
def adjust_sbemimage_properties(translation, scale, size, filename, sbemimage_config):
    cfg = ConfigParser()
    cfg.read_string(sbemimage_config)
    if bool(cfg['sys'].get('use_microtome')):
        props = cfg['microtome']
    else:
        props = cfg['sem']
    scale_x = float(props.get('stage_scale_factor_x'))
    scale_y = float(props.get('stage_scale_factor_y'))
    rotation_x = float(props.get('stage_rotation_angle_x'))
    rotation_y = float(props.get('stage_rotation_angle_y'))
    rotation_diff = rotation_x - rotation_y
    rot_mat_a = math.cos(rotation_y) / math.cos(rotation_diff)
    rot_mat_b = -math.sin(rotation_y) / math.cos(rotation_diff)
    rot_mat_c = math.sin(rotation_x) / math.cos(rotation_diff)
    rot_mat_d = math.cos(rotation_x) / math.cos(rotation_diff)
    rot_mat_determinant = rot_mat_a * rot_mat_d - rot_mat_b * rot_mat_c

    pixel_size = None
    parts = split_numeric_dict(filename)
    if 't' in parts:
        grids = cfg['grids']
        if 'r' in parts:
            grid_index = json.loads(grids.get('roi_index')).index(int(parts['r']))
        else:
            grid_index = int(parts['g'])
        pixel_size = json.loads(grids.get('pixel_size'))[grid_index] * 1e-3
    else:
        ov = cfg['overviews']
        if 'ov' in parts:
            ov_index = int(parts['ov'])
            pixel_size = json.loads(ov.get('ov_pixel_size'))[ov_index] * 1e-3
        elif '_stubov_' in filename.lower():
            pixel_size = float(ov.get('stub_ov_pixel_size')) * 1e-3

    if pixel_size:
        scale = {'x': pixel_size, 'y': pixel_size}
    else:
        scale = None

    stage_x, stage_y = translation['x'], translation['y']
    stage_x /= scale_x
    stage_y /= scale_y
    dx = ((rot_mat_d * stage_x - rot_mat_b * stage_y) / rot_mat_determinant)
    dy = ((-rot_mat_c * stage_x + rot_mat_a * stage_y) / rot_mat_determinant)
    # convert center to top/left
    physical_size = {dim: size[dim] * scale[dim] for dim in size}
    translation['x'] = dx - physical_size['x'] / 2
    translation['y'] = dy - physical_size['y'] / 2

    return translation, scale

apply_transform(points, transform)

Source code in src\muvis_align\util.py
887
888
889
890
891
892
893
894
895
def apply_transform(points, transform):
    new_points = []
    for point in points:
        point_len = len(point)
        while len(point) < len(transform):
            point = list(point) + [1]
        new_point = np.dot(point, np.transpose(np.array(transform)))
        new_points.append(new_point[:point_len])
    return new_points

apply_transform_dict(points, transform, transform_dims='xyz')

Source code in src\muvis_align\util.py
898
899
900
901
902
903
904
905
906
907
908
def apply_transform_dict(points, transform, transform_dims='xyz'):
    new_points = []
    for point in points:
        point = dict_to_xyz(point, dims=transform_dims)
        while len(point) < max(len(transform), 3):
            point = list(point) + [1]
        new_point = np.dot(point, np.transpose(np.array(transform)))
        new_point = xyz_to_dict(new_point, dims=transform_dims)
        new_point.pop('1', None)
        new_points.append(new_point)
    return new_points

background_luminance(levels, colors)

The luminance channels shown additively at their display levels (get_display_level) in RGB colors give their background, clipped to 1; None if any level is unknown.

Source code in src\muvis_align\util.py
 97
 98
 99
100
101
102
103
def background_luminance(levels, colors):
    """The luminance channels shown additively at their display `levels` (get_display_level) in RGB `colors` give
    their background, clipped to 1; None if any level is unknown."""
    if not levels or any(level is None for level in levels):
        return None
    return float(min(1.0, sum(level * (0.2126 * color[0] + 0.7152 * color[1] + 0.0722 * color[2])
                              for level, color in zip(levels, colors))))

calculate_rigid_difference(m1, m2)

Source code in src\muvis_align\util.py
911
912
def calculate_rigid_difference(m1, m2):
    return np.matmul(np.linalg.inv(m1), m2)

cancellable()

One operation's span: a cancel requested within it stops it; one left from before does not.

Source code in src\muvis_align\util.py
688
689
690
691
692
693
694
695
@contextmanager
def cancellable():
    """One operation's span: a cancel requested within it stops it; one left from before does not."""
    _cancel_requested.clear()
    try:
        yield
    finally:
        _cancel_requested.clear()

channels_from_table(table)

The channels ({'label', 'color'}) of a channels table: a dict of a list per column.

Source code in src\muvis_align\util.py
208
209
210
211
212
213
214
215
def channels_from_table(table):
    """The channels ({'label', 'color'}) of a channels table: a dict of a list per column."""
    channels = [{'label': label} for label in table.get('label', [])]
    for channel, color in zip(channels, table.get('color', [])):
        color = parse_channel_color(color)
        if color is not None:
            channel['color'] = color
    return channels

check_contains_value(value, contains_value)

Source code in src\muvis_align\util.py
539
540
def check_contains_value(value, contains_value):
    return isinstance(value, (dict, str)) and contains_value in value

contrasting_color(background)

A saturated colour, apart from greyscale by its hue, that stands out on a background of background luminance (0 black to 1 white): dark on a bright one, bright on a dark one, magenta between or when unknown (None). Hex: napari reads an RGB tuple as three colours.

Source code in src\muvis_align\util.py
86
87
88
89
90
91
92
93
94
def contrasting_color(background):
    """A saturated colour, apart from greyscale by its hue, that stands out on a background of `background` luminance
    (0 black to 1 white): dark on a bright one, bright on a dark one, magenta between or when unknown (None).
    Hex: napari reads an RGB tuple as three colours."""
    if background is not None and background > 0.6:
        return '#990066'
    if background is not None and background < 0.4:
        return '#ffd700'
    return '#ff00ff'

convert_to_um(value, unit)

value in unit, converted to um; an unrecognised unit is left unscaled, and logged. Takes both OME abbreviations and spelled-out NGFF unit names.

Source code in src\muvis_align\util.py
797
798
799
800
801
802
803
804
805
806
807
808
809
810
def convert_to_um(value, unit):
    """`value` in `unit`, converted to um; an unrecognised unit is left unscaled, and logged.
    Takes both OME abbreviations and spelled-out NGFF unit names.
    """
    if unit is None:
        return value
    factor = um_conversions.get(unit)
    if factor is None:
        factor = um_conversions.get(str(unit).lower())
    if factor is None:
        if str(unit).strip():
            logging.warning(f'Unrecognised physical unit {unit!r}, treating values as um')
        return value
    return value * factor

create_chunk_dict(chunk_size, dims)

Source code in src\muvis_align\util.py
991
992
993
994
995
996
def create_chunk_dict(chunk_size, dims):
    if isinstance(chunk_size, dict):
        chunk_dict = chunk_size
    else:
        chunk_dict = {dim: chunk_size if dim in 'xyz' else 1 for dim in dims}
    return chunk_dict

create_transform(center, angle, translation=None, matrix_size=3)

Source code in src\muvis_align\util.py
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
def create_transform(center, angle, translation=None, matrix_size=3):
    if not center:
        center = np.array([0] * (matrix_size - 1))
    if isinstance(center, dict):
        dims = 'xyz' if matrix_size >= 4 else 'xy'
        center = dict_to_xyz(center, dims=dims, add_zeros=True)
    if len(center) == 2:
        center = np.array(list(center) + [0])
    if angle is None:
        angle = 0
    r = Rotation.from_euler('z', angle, degrees=True)
    t = center - r.apply(center, inverse=True)
    if translation is not None:
        if len(translation) < len(t):
            translation = list(translation) + [0] * (len(t) - len(translation))
        t += translation
    transform = np.eye(matrix_size)
    transform[:3, :3] = np.transpose(r.as_matrix())
    transform[:3, -1] += t
    return transform

dict_to_xyz(dct, dims='xyz', add_zeros=False)

Source code in src\muvis_align\util.py
983
984
985
986
987
988
def dict_to_xyz(dct, dims='xyz', add_zeros=False):
    if add_zeros:
        array = [dct.get(dim, 0) for dim in dims]
    else:
        array = [dct[dim] for dim in dims if dim in dct]
    return array

differing_path_parts(paths)

Each path reduced to what tells it apart, joined by '_': each folder level or file name that differs, as just its differing letter and digit runs where all split into as many.

Source code in src\muvis_align\util.py
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
def differing_path_parts(paths: list) -> list:
    """Each path reduced to what tells it apart, joined by '_': each folder level or file name that differs, as
    just its differing letter and digit runs where all split into as many."""
    split = [path.replace('\\', '/').split('/') for path in paths]
    labels = [[] for _ in paths]
    for level in range(max(len(parts) for parts in split)):
        names = [parts[level] if level < len(parts) else '' for parts in split]
        if len(set(names)) > 1:
            tokens = [re.findall(r'[A-Za-z]+|\d+', name) for name in names]
            columns = []
            if len(set(len(name_tokens) for name_tokens in tokens)) == 1:
                columns = [column for column in zip(*tokens) if len(set(column)) > 1]
            parts = ['_'.join(row) for row in zip(*columns)] if columns else names
            for label, part in zip(labels, parts):
                label.append(part)
    return ['_'.join(label) for label in labels]

dir_regex(pattern)

Source code in src\muvis_align\util.py
343
344
345
346
347
348
def dir_regex(pattern):
    files = []
    for pattern_item in ensure_list(pattern):
        files.extend(glob.glob(pattern_item, recursive=True))
    files_sorted = sorted(files, key=lambda file: find_all_numbers(get_filetitle(file)))
    return files_sorted

ensure_list(x)

Source code in src\muvis_align\util.py
54
55
56
57
58
59
60
def ensure_list(x) -> list:
    if x is None:
        return []
    elif isinstance(x, list):
        return x
    else:
        return [x]

eval_context(data, key, default_value, context)

Source code in src\muvis_align\util.py
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
def eval_context(data, key, default_value, context):
    value = data.get(key, default_value)
    if isinstance(value, str):
        try:
            value = value.format_map(context)
        except:
            pass
        try:
            value = eval(value, context)
        except Exception as error:
            # napari shows a warning once, as a notification, however many sources repeat it
            warnings.warn(f'Invalid source metadata {key}: {value!r} ({type(error).__name__}: {error}),'
                          f' using {default_value}')
    if not isinstance(value, (float, int)):
        value = default_value
    return value

eval_path(path)

Source code in src\muvis_align\util.py
1261
1262
1263
1264
1265
1266
1267
1268
def eval_path(path):
    if ',' in path:
        parts = path.split(',')
        new_path = ','.join(['"' + part.strip() + '"' for part in parts])
        new_path = eval('[' + new_path + ']')
    else:
        new_path = path
    return new_path

export_csv(filename, data, header=None)

Source code in src\muvis_align\util.py
1341
1342
1343
1344
1345
1346
1347
def export_csv(filename, data, header=None):
    with open(filename, 'w', encoding='utf8', newline='') as file:
        csvwriter = csv.writer(file)
        if header is not None:
            csvwriter.writerow(header)
        for row in data:
            csvwriter.writerow(row)

export_json(filename, data)

Source code in src\muvis_align\util.py
1330
1331
1332
def export_json(filename, data):
    with open(filename, 'w', encoding='utf8') as file:
        json.dump(data, file, indent=4)

extra_metadata_from_params(input_output)

The extra_metadata of the project's input_output: its own extra_metadata (a dict or a metadata file), with the channels of its channels table.

Source code in src\muvis_align\util.py
218
219
220
221
222
223
224
225
def extra_metadata_from_params(input_output):
    """The extra_metadata of the project's input_output: its own `extra_metadata` (a dict or a metadata file),
    with the channels of its channels table."""
    extra = input_output.get('extra_metadata') or {}
    channels = channels_from_table(input_output.get('channels_table') or {})
    if isinstance(extra, dict) and channels:
        extra = dict(extra, channels=channels)
    return extra

file_list_index_finder(filenames)

find_file_list_index(filenames, match) for many matches: an exact name by dict, each other found once.

Source code in src\muvis_align\util.py
106
107
108
109
110
111
112
113
114
def file_list_index_finder(filenames):
    """find_file_list_index(filenames, match) for many matches: an exact name by dict, each other found once."""
    found = {filename: index for index, filename in reversed(list(enumerate(filenames)))}

    def find(match):
        if match not in found:
            found[match] = find_file_list_index(filenames, match)
        return found[match]
    return find

filter_dict(dict0)

Source code in src\muvis_align\util.py
241
242
243
244
245
246
247
248
249
250
251
252
253
def filter_dict(dict0: dict) -> dict:
    new_dict = {}
    for key, value0 in dict0.items():
        if value0 is not None:
            values = []
            for value in ensure_list(value0):
                if isinstance(value, dict):
                    value = filter_dict(value)
                values.append(value)
            if len(values) == 1:
                values = values[0]
            new_dict[key] = values
    return new_dict

filter_edge_points(points, bounds, filter_factor=0.1, threshold=0.5)

Source code in src\muvis_align\util.py
1053
1054
1055
1056
1057
1058
1059
def filter_edge_points(points, bounds, filter_factor=0.1, threshold=0.5):
    center = np.array(bounds) / 2
    dist_center = np.abs(points / center - 1)
    position_weights = np.clip((1 - np.max(dist_center, axis=-1)) / filter_factor, 0, 1)
    order_weights = 1 - np.array(range(len(points))) / len(points) / 2
    weights = position_weights * order_weights
    return weights > threshold

find_all_numbers(text)

Source code in src\muvis_align\util.py
373
374
def find_all_numbers(text: str) -> list:
    return list(map(int, re.findall(r'\d+', text)))

find_file_dict_item(dct, match)

Source code in src\muvis_align\util.py
117
118
119
120
121
122
123
124
125
126
def find_file_dict_item(dct, match):
    if match in dct:
        return dct[match]
    for key, value in dct.items():
        if key.lstrip('.') in match:
            return value
    for key, value in dct.items():
        if _paths_match_by_suffix(key, match):
            return value
    return None

find_file_list_index(filenames, match)

Source code in src\muvis_align\util.py
74
75
76
77
78
79
80
81
82
83
def find_file_list_index(filenames, match):
    if match in filenames:
        return filenames.index(match)
    for index, filename in enumerate(filenames):
        if match.lstrip('.') in filename:
            return index
    for index, filename in enumerate(filenames):
        if _paths_match_by_suffix(filename, match):
            return index
    return None

find_labelled_numbers(text)

Each number led by letters, by its lower-case label: 'x_s00399_ov0' -> {'s': 399, 'ov': 0}.

Source code in src\muvis_align\util.py
377
378
379
def find_labelled_numbers(text: str) -> dict:
    """Each number led by letters, by its lower-case label: 'x_s00399_ov0' -> {'s': 399, 'ov': 0}."""
    return {label.lower(): int(number) for label, number in re.findall(r'([A-Za-z]+)(\d+)', text)}

find_sbemimage_meta_dir(filename, max_hops=5)

Source code in src\muvis_align\util.py
351
352
353
354
355
356
357
358
def find_sbemimage_meta_dir(filename, max_hops=5):
    path = os.path.dirname(filename)
    for _ in range(max_hops):
        metapath = os.path.join(path, 'meta')
        if os.path.exists(metapath):
            return metapath
        path = os.path.join(path, '..')
    return None

find_target_numeric(text, target)

Source code in src\muvis_align\util.py
398
399
400
401
402
403
404
405
def find_target_numeric(text: str, target: str) -> int|None:
    parts = split_path_parts(text)
    for part in parts:
        if part.startswith(target):
            part = part.lstrip(target)
            if part.isdecimal():
                return int(part)
    return None

format_phase_timing(wall_time, item_times, item_cpu_times, max_workers, process_cpu_time=None, processes=False)

One timing line for a threaded per-item phase: wall time against summed per-item CPU time.

Summed per-item wall time inflates with GIL waits, but per-item CPU (time.thread_time) is a floor: wall ~= cpu means CPU/GIL-bound, more workers cannot help; wall >> cpu means waiting (I/O) that more workers can overlap. process_cpu_time (time.process_time over the phase) also shows CPU spent outside the timed items, such as nested pools or other threads.

Source code in src\muvis_align\util.py
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
def format_phase_timing(wall_time, item_times, item_cpu_times, max_workers, process_cpu_time=None,
                        processes=False):
    """One timing line for a threaded per-item phase: wall time against summed per-item CPU time.

    Summed per-item wall time inflates with GIL waits, but per-item CPU (time.thread_time) is a floor:
    wall ~= cpu means CPU/GIL-bound, more workers cannot help; wall >> cpu means waiting (I/O) that more
    workers can overlap. `process_cpu_time` (time.process_time over the phase) also shows CPU spent outside
    the timed items, such as nested pools or other threads.
    """
    total_time = sum(item_times)
    total_cpu = sum(item_cpu_times) if item_cpu_times else 0.0
    summary = (f'with {max_workers} workers, wall {wall_time:.1f}s,'
               f' per-item total {total_time:.1f}s, cpu {total_cpu:.1f}s'
               f' (mean {1000 * total_time / len(item_times):.0f}ms,'
               f' max {1000 * max(item_times):.0f}ms)')
    if process_cpu_time is not None and wall_time > 0:
        summary += (f', process cpu {process_cpu_time:.1f}s'
                    f' ({process_cpu_time / wall_time:.1f} cores)')
        if total_cpu > 0 and process_cpu_time > 2 * total_cpu:
            summary += (f' - only {total_cpu / process_cpu_time:.0%} of the process CPU is this'
                        f' phase: the rest is elsewhere in the process')
    if processes and total_cpu > 0 and wall_time > 0:
        # no shared GIL: the items' cpu against the wall is how many of the workers were busy
        summary += f' - items in worker processes, {total_cpu / wall_time:.1f} of {max_workers} busy'
    elif total_cpu > 0:
        ratio = wall_time / total_cpu
        regime = ('CPU-bound: at the single-thread floor, more workers will not help'
                  if ratio < 1.3 else
                  f'{1 - 1 / ratio:.0%} of wall time is not CPU: more workers can overlap it')
        summary += f' - {regime}'
    return summary

get_center(data, offset=(0, 0))

Source code in src\muvis_align\util.py
855
856
857
858
859
860
861
def get_center(data, offset=(0, 0)):
    moments = get_moments(data, offset=offset)
    if moments['m00'] != 0:
        center = get_moments_center(moments)
    else:
        center = np.mean(data, 0).flatten()  # close approximation
    return center.astype(np.float32)

get_filetitle(filename)

Source code in src\muvis_align\util.py
336
337
338
339
340
def get_filetitle(filename: str) -> str:
    filebase = os.path.basename(filename)
    # removesuffix, not rstrip: rstrip('.ome') strips any trailing '.', 'o', 'm' or 'e'
    title = os.path.splitext(filebase)[0].removesuffix('.ome')
    return title

get_grid_angle(positions, sizes)

How far a tile grid is turned, in degrees within (-45, 45], or None without neighbours to tell. Only same-sized neighbours at distinct xy count.

Source code in src\muvis_align\util.py
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
def get_grid_angle(positions, sizes):
    """How far a tile grid is turned, in degrees within (-45, 45], or None without neighbours to tell.
    Only same-sized neighbours at distinct xy count.
    """
    if not any({'x', 'y'} & set(position) for position in positions):
        return None
    pairs, _ = get_pairs(positions, sizes)
    directions = []
    for first, second in pairs:
        vector = [positions[first].get(dim, 0) - positions[second].get(dim, 0) for dim in 'xy']
        is_same_size = all(math.isclose(sizes[first].get(dim, 0), sizes[second].get(dim, 0), rel_tol=0.01)
                           for dim in 'xy')
        if is_same_size and math.hypot(*vector) > 1e-3 * max(sizes[first].values(), default=0):
            directions.append(math.atan2(vector[1], vector[0]))
    if not directions:
        return None
    # four times the angle: rows and columns, either way round, give the same grid direction
    return math.degrees(math.atan2(np.mean(np.sin(4 * np.array(directions))),
                                   np.mean(np.cos(4 * np.array(directions)))) / 4)

get_mean_nn_distance(points1, points2)

Source code in src\muvis_align\util.py
1049
1050
def get_mean_nn_distance(points1, points2):
    return np.mean([get_nn_distance(points1), get_nn_distance(points2)])

get_metadata_z_scale(metadata)

The configured z spacing from a source/extra metadata dict, as a number - or None when it is unset, blank, or 'source' (the file's own, already applied by ImageSource.fix_metadata).

Source code in src\muvis_align\util.py
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
def get_metadata_z_scale(metadata):
    """The configured z spacing from a source/extra metadata dict, as a number - or None when it
    is unset, blank, or 'source' (the file's own, already applied by ImageSource.fix_metadata).
    """
    if not isinstance(metadata, dict):
        return None
    scales = metadata.get('scale', {})
    # a list holds one scale per file: the run steps by the finest z, as init_data does by z positions
    values = []
    for scale in (scales if isinstance(scales, list) else [scales]):
        value = scale.get('z') if isinstance(scale, dict) else None
        if value is not None and not check_contains_value(value, 'source'):
            try:
                values.append(float(value))
            except (TypeError, ValueError):
                pass
    return min(values, key=abs) if values else None

get_moments(data, offset=(0, 0))

Source code in src\muvis_align\util.py
846
847
848
def get_moments(data, offset=(0, 0)):
    moments = cv.moments((np.array(data) + offset).astype(np.float32))    # doesn't work for float64!
    return moments

get_moments_center(moments, offset=(0, 0))

Source code in src\muvis_align\util.py
851
852
def get_moments_center(moments, offset=(0, 0)):
    return np.array([moments['m10'], moments['m01']]) / moments['m00'] + np.array(offset)

get_nn_distance(points0)

Source code in src\muvis_align\util.py
1038
1039
1040
1041
1042
1043
1044
1045
1046
def get_nn_distance(points0):
    points = list(set(map(tuple, points0)))     # get unique points
    if len(points) >= 2:
        tree = KDTree(points, leaf_size=2)
        dist, ind = tree.query(points, k=2)
        nn_distance = np.median(dist[:, 1])
    else:
        nn_distance = 1
    return nn_distance

get_pairs(positions, sizes)

Get pairs of orthogonal neighbors from a list of tiles. Tiles don't have to be placed on a regular grid.

Source code in src\muvis_align\util.py
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
def get_pairs(positions, sizes):
    """
    Get pairs of orthogonal neighbors from a list of tiles.
    Tiles don't have to be placed on a regular grid.
    """
    pairs = []
    angles = []
    z_positions = [position['z'] for position in positions if 'z' in position]
    ordered_z = sorted(set(z_positions))
    z_index = {z: index for index, z in enumerate(ordered_z)}
    is_mixed_3dstack = len(ordered_z) < len(z_positions)
    for i, j in _get_pairs_candidates(positions, sizes, is_mixed_3dstack, z_index):
        posi, posj = positions[i], positions[j]
        sizei, sizej = sizes[i], sizes[j]
        if is_mixed_3dstack:
            # ignore z value for distance
            distance = math.dist([posi[dim] for dim in 'xy'], [posj[dim] for dim in 'xy'])
            min_distance = max([size[dim] for size in [sizei, sizej] for dim in 'xy'])
            is_same_z = (posi['z'] == posj['z'])
            is_close_z = abs(z_index[posi['z']] - z_index[posj['z']]) <= 1
            if not is_close_z:
                # if not close, discard as pair
                min_distance = 0
            elif not is_same_z:
                # for tiles in different z stack, require greater overlap
                min_distance *= 0.8
        else:
            distance = math.dist(posi.values(), posj.values())
            min_distance = max(list(sizei.values()) + list(sizej.values()))

        if distance < min_distance:
            pairs.append((int(i), int(j)))
            vector = np.array(list(posi.values())) - np.array(list(posj.values()))
            angle = math.degrees(math.atan2(vector[1], vector[0]))
            if distance < min(list(sizei.values()) + list(sizej.values())):
                angle += 90
            while angle < -90:
                angle += 180
            while angle > 90:
                angle -= 180
            angles.append(angle)
    return pairs, angles

get_process_memory()

This process's (resident, peak resident) memory in bytes, or None where the platform will not say (psutil is not a dependency).

Source code in src\muvis_align\util.py
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
def get_process_memory():
    """This process's (resident, peak resident) memory in bytes, or None where the platform
    will not say (psutil is not a dependency).
    """
    resident = peak = None
    try:
        with open('/proc/self/statm') as file:
            resident = int(file.read().split()[1]) * os.sysconf('SC_PAGE_SIZE')
    except (OSError, ValueError, AttributeError, IndexError):
        pass
    try:
        import resource
        # ru_maxrss is kilobytes on Linux but bytes on macOS
        max_rss = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
        peak = max_rss if sys.platform == 'darwin' else max_rss * 1024
    except (ImportError, AttributeError, OSError, ValueError):
        pass
    if resident is None and peak is None:
        resident, peak = _windows_process_memory()
    if resident is not None and peak is not None:
        # Linux RSS counters are per-CPU approximations read at different moments, so peak can lag
        peak = max(peak, resident)
    return resident, peak

get_rotation_from_transform(transform, dims='xyz')

Source code in src\muvis_align\util.py
949
950
951
952
953
954
955
956
957
958
959
960
def get_rotation_from_transform(transform, dims='xyz'):
    # TODO: assume 2D rotation, expand to 3D
    # Rotation.from_matrix(transform).as_euler() only works for simple rotation matrices
    if isinstance(transform, DataArray):
        dims = transform['x_in'].data.tolist()
    x_index, y_index = dims.index('x'), dims.index('y')
    transform = np.array(transform)
    if y_index > x_index:
        rotation = np.arctan2(transform[0][1], transform[0][0])
    else:
        rotation = np.arctan2(transform[1][0], transform[1][1])
    return float(np.rad2deg(rotation))

get_scale_from_transform(transform)

Source code in src\muvis_align\util.py
936
937
938
def get_scale_from_transform(transform):
    scale = np.mean(np.linalg.norm(transform, axis=0)[:-1])
    return float(scale)

get_source_axis(value)

Source code in src\muvis_align\util.py
548
549
550
551
def get_source_axis(value):
    # 'source y' takes the source's y value, for a camera whose axes are the stage's swapped
    match = re.search(r'\bsource\s+([xyz])\b', value) if isinstance(value, str) else None
    return match.group(1) if match else None

get_translation_from_transform(transform)

Source code in src\muvis_align\util.py
941
942
943
944
945
def get_translation_from_transform(transform):
    ndim = len(transform) - 1
    #translation = transform[:ndim, ndim]
    translation = apply_transform([[0] * ndim], transform)[0]
    return translation

get_unique_file_labels(filenames)

Source code in src\muvis_align\util.py
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
def get_unique_file_labels(filenames: list) -> list:
    file_labels = []

    ntot = len(filenames)
    common_paths = strip_common_path_prefix(filenames)
    parts_dic = get_unique_nums([split_numeric_dict(get_filetitle(filename)) for filename in filenames])
    parts_dic_full = get_unique_nums([split_numeric_dict(filename) for filename in common_paths])
    parts_num_full = get_unique_nums([{index: value for index, value in enumerate(split_numeric(filename))} for filename in common_paths])

    dic_ok = (len(set(['_'.join(parts.values()) for parts in parts_dic])) == ntot)
    full_dic_ok = (len(set(['_'.join(parts.values()) for parts in parts_dic_full])) == ntot)
    full_num_ok = (len(set(['_'.join(parts.values()) for parts in parts_num_full])) == ntot)

    if dic_ok:
        all_parts = parts_dic
    elif full_dic_ok:
        all_parts = parts_dic_full
    elif full_num_ok:
        all_parts = parts_num_full
    else:
        # no digit-based split told them apart: what differs in their paths, else the whole relative path
        labels = differing_path_parts(filenames)
        all_parts = [{0: label} for label in (labels if len(set(labels)) == ntot else common_paths)]

    for parts in all_parts:
        file_label = '_'.join([key + part if isinstance(key, str) else part for key, part in parts.items()])
        file_labels.append(file_label)

    if len(set(file_labels)) < len(file_labels):
        # fallback for duplicate labels
        file_labels = [get_filetitle(filename) for filename in filenames]

    return file_labels

get_unique_nums(all_parts)

Source code in src\muvis_align\util.py
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
def get_unique_nums(all_parts: list) -> list:
    keys = []
    for parts in all_parts:
        for key in parts:
            if key not in keys:
                keys.append(key)

    changing_keys = []
    for key in keys:
        values = [parts.get(key) for parts in all_parts]
        if len(set(values)) > 1:
            changing_keys.append(key)

    # each file's own key order: a global one would let one naming scheme dictate another's
    changing = set(changing_keys)
    final_parts = [{key: value for key, value in parts.items() if key in changing} for parts in all_parts]
    return final_parts

import_csv(filename)

The rows, each by its header's column names.

Source code in src\muvis_align\util.py
1335
1336
1337
1338
def import_csv(filename):
    """The rows, each by its header's column names."""
    with open(filename, encoding='utf8', newline='') as file:
        return list(csv.DictReader(file))

import_json(filename)

Source code in src\muvis_align\util.py
1324
1325
1326
1327
def import_json(filename):
    with open(filename, encoding='utf8') as file:
        data = json.load(file)
    return data

import_metadata(content, fields=None, input_path=None)

Source code in src\muvis_align\util.py
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
def import_metadata(content, fields=None, input_path=None):
    # return dict[id] = {values}
    if isinstance(content, str):
        ext = os.path.splitext(content)[1].lower()
        if input_path and not isinstance(content, dict):
            if isinstance(input_path, list):
                input_path = input_path[0]
            content = to_posix_path(os.path.normpath(os.path.join(os.path.dirname(input_path), content)))
        if ext == '.csv':
            content = import_csv(content)
        elif ext in ['.json', '.ome.json']:
            content = import_json(content)
    if fields is not None:
        content = [[data[field] for field in fields] for data in content]
    return content

is_source_value(value)

Source code in src\muvis_align\util.py
543
544
545
def is_source_value(value):
    # 'invert' alone inverts the source's own value, as 'source invert' does
    return check_contains_value(value, 'source') or check_contains_value(value, 'invert')

is_valid_value(value)

Source code in src\muvis_align\util.py
144
145
def is_valid_value(value):
    return value is not None and value != ''

load_sbemimage_best_config(metapath, filename)

Source code in src\muvis_align\util.py
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
def load_sbemimage_best_config(metapath, filename):
    target_datetime = datetime.fromtimestamp(os.path.getmtime(filename))

    for config_filename in sorted(glob.glob(os.path.join(metapath, 'logs/config_*.txt')), reverse=True):
        match = re.split(r'config_(\d+-\d+-\d+).txt', config_filename)
        if len(match) >= 2:
            file_date = datetime.strptime(match[1], '%Y-%m-%d%H%M%S%f')
            if file_date <= target_datetime:
                with open(config_filename, 'r') as file:
                    sbemimage_config = file.read()
                return sbemimage_config
    return None

normalise_rotated_positions(positions0, rotations0, sizes, center, ndims)

Source code in src\muvis_align\util.py
 999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
def normalise_rotated_positions(positions0, rotations0, sizes, center, ndims):
    # in [xy(z)]; each source turns about its own centre, as its image is drawn unrotated
    positions = []
    rotations = []
    grid_angle = get_grid_angle(positions0, sizes)
    for position0, size, rotation in zip(positions0, sizes, rotations0):
        if rotation is None and grid_angle is not None:
            rotation = -grid_angle
        angle = -rotation if rotation is not None else None
        transform = create_transform(center=center, angle=angle, matrix_size=ndims + 1)
        half_size = {dim: size.get(dim, 0) / 2 for dim in 'xy'}
        source_center = {dim: position0.get(dim, 0) + half_size[dim] for dim in 'xy'}
        turned_center = apply_transform_dict([source_center], transform, transform_dims='xy')[0]
        positions.append(dict(position0) | {dim: turned_center[dim] - half_size[dim] for dim in 'xy'})
        rotations.append(rotation)
    return positions, rotations

normalise_rotation(rotation)

Normalise rotation to be in the range [-180, 180].

Source code in src\muvis_align\util.py
963
964
965
966
967
968
969
970
971
def normalise_rotation(rotation):
    """
    Normalise rotation to be in the range [-180, 180].
    """
    while rotation < -180:
        rotation += 360
    while rotation > 180:
        rotation -= 360
    return rotation

numpy_to_native(value)

Source code in src\muvis_align\util.py
129
130
131
132
133
134
135
136
137
138
139
140
141
def numpy_to_native(value):
    if isinstance(value, np.ndarray):
        return value.tolist()
    elif isinstance(value, list):
        return [numpy_to_native(v) for v in value]
    elif isinstance(value, tuple):
        return (numpy_to_native(v) for v in value)
    elif isinstance(value, dict):
        return {k: numpy_to_native(v) for k, v in value.items()}
    elif hasattr(value, 'dtype'):
        return value.item()
    else:
        return value

operation_to_past_participle(operation)

Source code in src\muvis_align\util.py
1418
1419
1420
1421
1422
1423
1424
1425
def operation_to_past_participle(operation):
    verb = operation.split()[0]
    if verb.endswith('e'):
        return verb + 'd'
    elif verb.endswith('y'):
        return verb[:-1] + 'ied'
    else:
        return verb + 'ed'

parse_block_size(block_size)

A block size setting (a number, a list, or text of either), or None: fuse() then sizes the export's blocks by what one costs in memory.

Source code in src\muvis_align\util.py
228
229
230
231
232
233
234
235
236
237
238
def parse_block_size(block_size):
    """A block size setting (a number, a list, or text of either), or None: fuse() then sizes the export's blocks
    by what one costs in memory."""
    if isinstance(block_size, str):
        block_size = block_size.strip()
        if not block_size:
            return None
        if ',' in block_size:
            return [int(size.strip()) for size in block_size.split(',')]
        return int(block_size)
    return block_size

parse_channel_color(color)

Source code in src\muvis_align\util.py
199
200
201
202
203
204
205
def parse_channel_color(color):
    if isinstance(color, str):
        try:
            return tuple(eval(color))
        except Exception:
            return None
    return color

parse_scale(value, default=1)

A downscale setting as get_level_from_scale() takes it: a factor - a number, or a number as the text a text field gives ('2') - or a pixel size with its unit ('10um'), kept as text. Blank is default; other text is an error naming the units a pixel size can have.

Source code in src\muvis_align\util.py
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
def parse_scale(value, default=1):
    """A downscale setting as get_level_from_scale() takes it: a factor - a number, or a number as the
    text a text field gives ('2') - or a pixel size with its unit ('10um'), kept as text. Blank is
    `default`; other text is an error naming the units a pixel size can have."""
    if value is None or (isinstance(value, str) and not value.strip()):
        return default
    if not isinstance(value, str):
        return value
    try:
        number = float(value)
    except ValueError:
        if pixel_size_to_um(value) is None:
            raise ValueError(f'{value!r} is neither a downscale factor nor a pixel size with a unit'
                             f' ({", ".join(sorted(set(um_conversions)))})') from None
        return value.strip()
    return int(number) if number.is_integer() else number

path_param_to_list(value)

Source code in src\muvis_align\util.py
1257
1258
def path_param_to_list(value):
    return [part.strip() for part in path_param_to_text(value).split(',') if part.strip()]

path_param_to_text(value)

Source code in src\muvis_align\util.py
1250
1251
1252
1253
1254
def path_param_to_text(value):
    # a multi-path param is stored as a comma-separated string or a yaml list; the helpers take one line
    if isinstance(value, (list, tuple)):
        return ', '.join(str(part).strip() for part in value)
    return '' if value is None else str(value)

pattern_base_dir(pattern)

The real directory an input pattern sits in, without wildcard components (os.path.dirname of 'data//.tiff' is 'data/*'): what a relative output path is relative to.

Source code in src\muvis_align\util.py
361
362
363
364
365
366
367
368
369
370
def pattern_base_dir(pattern):
    """The real directory an input pattern sits in, without wildcard components (os.path.dirname of
    'data/*/*.tiff' is 'data/*'): what a relative output path is relative to."""
    directory = os.path.dirname(pattern)
    while directory and any(char in os.path.basename(directory) for char in '*?['):
        parent = os.path.dirname(directory)
        if parent == directory:
            return ''
        directory = parent
    return directory

picklable(value)

Source code in src\muvis_align\util.py
706
707
708
709
710
711
712
def picklable(value):
    try:
        pickle.dumps(value)
        return True
    except Exception as error:
        logging.info(f'Not picklable ({type(error).__name__}: {error})')
        return False

pixel_size_to_um(text)

A pixel size written with its unit ('10um', '0.5 mm', '250nm') in um, or None if it is not one.

Source code in src\muvis_align\util.py
816
817
818
819
820
821
822
823
824
def pixel_size_to_um(text):
    """A pixel size written with its unit ('10um', '0.5 mm', '250nm') in um, or None if it is not one."""
    match = _pixel_size_pattern.fullmatch(text.strip())
    if match is None:
        return None
    unit = match.group(2)
    if unit not in um_conversions and unit.lower() not in um_conversions:
        return None
    return convert_to_um(float(match.group(1)), unit)

points_to_3d(points)

Source code in src\muvis_align\util.py
974
975
def points_to_3d(points):
    return [list(point) + [0] for point in points]

pooled_map(func, count, workers, progress_factory=None, desc=None, weight=1, timing_label=None)

[func(index) for index in range(count)], on workers threads (one: none), each on the bar as it finishes; with timing_label, the phase's timing logged (format_phase_timing). Rolling: a cancel need not wait for every item.

Source code in src\muvis_align\util.py
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
def pooled_map(func, count, workers, progress_factory=None, desc=None, weight=1, timing_label=None):
    """[func(index) for index in range(count)], on `workers` threads (one: none), each on the bar as it finishes; with
    timing_label, the phase's timing logged (format_phase_timing). Rolling: a cancel need not wait for every item."""
    results = [None] * count
    times, cpu_times = [], []
    timed = timed_calls(func, times, cpu_times)
    workers = max(1, min(workers, count))
    phase_start, phase_cpu_start = time.time(), time.process_time()
    with progress_phase(progress_factory, total=count, desc=desc, weight=weight) as pbar:
        items = ((index, timed(index)) for index in range(count)) if workers == 1 \
            else rolling_map(timed, range(count), workers)
        for index, result in items:
            results[index] = result
            if pbar is not None:
                pbar.update(1)
    if timing_label and times:
        logging.info(f'{timing_label}: {len(times)} items {format_phase_timing(
            time.time() - phase_start, times, cpu_times, workers, time.process_time() - phase_cpu_start)}')
    return results

print_dict(dct, indent=0)

Source code in src\muvis_align\util.py
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
def print_dict(dct: dict, indent: int = 0) -> str:
    s = ''
    if isinstance(dct, dict):
        for key, value in dct.items():
            s += '\n'
            if not isinstance(value, list):
                s += '\t' * indent + str(key) + ': '
            if isinstance(value, dict):
                s += print_dict(value, indent=indent + 1)
            elif isinstance(value, list):
                for v in value:
                    s += print_dict(v)
            else:
                s += str(value)
    else:
        s += str(dct)
    return s

print_dict_simple(dct, dims='zyx', significant_digits=3)

Source code in src\muvis_align\util.py
276
277
278
279
280
281
282
283
284
285
286
287
def print_dict_simple(dct: dict, dims: str='zyx', significant_digits: int=3) -> str:
    items = []
    # a list, not a string: keys need not be single letters (e.g. a pair's (index, index))
    dims = list(dims)
    dims += [key for key in dct if key not in dims]
    for dim in dims:
        if dim in dct:
            value = dct[dim]
            if isinstance(value, float):
                value = print_significants(value, significant_digits)
            items.append(f'{dim}: {value}')
    return ' '.join(items)

print_dict_xyz(dct, dims='xyz', decimals=3, as_tuple=False)

Source code in src\muvis_align\util.py
299
300
301
302
303
304
305
306
307
308
309
def print_dict_xyz(dct: dict, dims='xyz', decimals=3, as_tuple=False) -> str:
    s = ''
    for dim in dims:
        if dim in dct:
            if s:
                s += ' '
            if as_tuple:
                s += f'{dct[dim]:.{decimals}f}'
            else:
                s += f'{dim}:{dct[dim]:.{decimals}f}'
    return s

print_hbytes(nbytes)

Source code in src\muvis_align\util.py
312
313
314
315
316
317
318
319
320
321
322
323
324
def print_hbytes(nbytes: int) -> str:
    exps = ['', 'K', 'M', 'G', 'T', 'P', 'E']
    div = 1024
    exp = 0

    while nbytes > div:
        nbytes /= div
        exp += 1
    if exp < len(exps):
        e = exps[exp]
    else:
        e = f'e{exp * 3}'
    return f'{nbytes:.1f} {e}B'

print_memory_usage()

' rss 1.2GB (peak 3.4GB)' for a log line, or '' where the platform will not say.

Source code in src\muvis_align\util.py
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
def print_memory_usage():
    """' rss 1.2GB (peak 3.4GB)' for a log line, or '' where the platform will not say."""
    resident, peak = get_process_memory()
    if resident is None and peak is None:
        return ''
    parts = []
    if resident is not None:
        parts.append(f'rss {print_hbytes(resident)}')
    if peak is not None:
        parts.append(f'peak {print_hbytes(peak)}')
    return ' ' + ' '.join(parts)

print_significants(value, significant_digits)

Source code in src\muvis_align\util.py
290
291
292
293
294
295
296
def print_significants(value: float, significant_digits: int) -> str:
    # fixed-point rather than 'g', which turns 64863 into 6.49e+04: every integer digit is kept
    if value == 0 or not np.isfinite(value):
        return f'{value:g}'
    decimals = max(significant_digits - int(np.floor(np.log10(abs(value)))) - 1, 0)
    text = f'{round(value, decimals):.{decimals}f}'
    return text.rstrip('0').rstrip('.') if '.' in text else text

progress_phase(progress_factory, total=None, desc=None, weight=1)

One reporting phase of the caller's operation, or nothing to report into.

Source code in src\muvis_align\util.py
645
646
647
648
def progress_phase(progress_factory, total=None, desc=None, weight=1):
    """One reporting phase of the caller's operation, or nothing to report into."""
    return progress_factory(total=total, desc=desc, weight=weight) if progress_factory is not None \
        else nullcontext(None)

raise_if_cancelled()

Source code in src\muvis_align\util.py
683
684
685
def raise_if_cancelled():
    if _cancel_requested.is_set():
        raise OperationCancelled('Cancelled')

relativize_to_project_dir(path, base_dir)

Source code in src\muvis_align\util.py
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
def relativize_to_project_dir(path, base_dir):
    # counterpart to resolve_to_project_dir(): stored paths stay relative and portable; forward slashes
    if not path or not base_dir:
        return to_posix_path(path)
    parts = [part.strip() for part in path.split(',')]
    relativized = []
    for part in parts:
        if not part:
            relativized.append(part)
        elif not os.path.isabs(part):
            relativized.append(to_posix_path(part))
        else:
            try:
                relativized.append(to_posix_path(os.path.relpath(part, base_dir)))
            except ValueError:
                # on a different drive than the project dir (Windows): can't be made relative
                relativized.append(to_posix_path(part))
    return ', '.join(relativized)

release_memory(label=None, generation=2)

Hand memory the process has freed back to the OS, logging what that recovered.

glibc's per-thread heaps keep freed buffers below anything still in use; malloc_trim() returns them, elsewhere this only collects garbage. generation 1 collects only young objects, enough after one batch, where a full collection scans every live object.

Source code in src\muvis_align\util.py
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
def release_memory(label=None, generation=2):
    """Hand memory the process has freed back to the OS, logging what that recovered.

    glibc's per-thread heaps keep freed buffers below anything still in use; malloc_trim() returns them,
    elsewhere this only collects garbage. `generation` 1 collects only young objects, enough after one
    batch, where a full collection scans every live object.
    """
    gc.collect(generation)
    if not sys.platform.startswith('linux'):
        return
    try:
        import ctypes
        before, _ = get_process_memory()
        ctypes.CDLL('libc.so.6').malloc_trim(0)
        after, _ = get_process_memory()
    except (OSError, AttributeError):
        return
    if label and before is not None and after is not None:
        logging.info(f'{label}: released {print_hbytes(max(before - after, 0))} to the OS,'
                     f' rss {print_hbytes(before)} -> {print_hbytes(after)}')

request_cancel()

Source code in src\muvis_align\util.py
679
680
def request_cancel():
    _cancel_requested.set()

resolve_to_project_dir(path, base_dir)

Source code in src\muvis_align\util.py
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
def resolve_to_project_dir(path, base_dir):
    # path params are stored relative to the project dir for portability, but consumers resolve
    # against the cwd; returned with forward slashes
    if not path or not base_dir:
        return to_posix_path(path)
    parts = [part.strip() for part in path.split(',')]
    resolved = []
    for part in parts:
        if not part:
            resolved.append(part)
        else:
            joined = part if os.path.isabs(part) else os.path.join(base_dir, part)
            resolved.append(to_posix_path(os.path.normpath(joined)))
    return ', '.join(resolved)

result_unless_cancelled(future, poll_seconds=0.5)

future.result(), or OperationCancelled as soon as a cancel is requested while waiting for it.

Source code in src\muvis_align\util.py
760
761
762
763
764
def result_unless_cancelled(future, poll_seconds=0.5):
    """future.result(), or OperationCancelled as soon as a cancel is requested while waiting for it."""
    while not wait([future], timeout=poll_seconds).done:
        raise_if_cancelled()
    return future.result()

retuple(chunks, shape)

Expand chunks to match shape.

E.g. if chunks is (64, 64) and shape is (3, 4, 5, 1028, 1028) return (3, 4, 5, 64, 64)

If chunks is an integer, it is applied to all dimensions, to match the behaviour of zarr-python.

Source code in src\muvis_align\util.py
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
def retuple(chunks, shape):
    # from ome-zarr-py
    """
    Expand chunks to match shape.

    E.g. if chunks is (64, 64) and shape is (3, 4, 5, 1028, 1028)
    return (3, 4, 5, 64, 64)

    If chunks is an integer, it is applied to all dimensions, to match
    the behaviour of zarr-python.
    """

    if isinstance(chunks, int):
        return tuple([chunks] * len(shape))

    dims_to_add = len(shape) - len(chunks)
    return *shape[:dims_to_add], *chunks

rolling_map(func, items, workers)

(item, func(item)) for each of items as it finishes, on workers threads (one: in the calling thread). Only twice that many are submitted at once, so memory stays bounded and a slow item holds up only its own thread.

Source code in src\muvis_align\util.py
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
def rolling_map(func, items, workers):
    """(item, func(item)) for each of `items` as it finishes, on `workers` threads (one: in the calling thread). Only
    twice that many are submitted at once, so memory stays bounded and a slow item holds up only its own thread."""
    if workers <= 1:
        for item in items:
            yield item, func(item)
            raise_if_cancelled()
        return
    items = iter(items)
    pending = {}
    with ThreadPoolExecutor(workers) as pool:
        def submit(count):
            for item in itertools.islice(items, count):
                pending[pool.submit(func, item)] = item

        submit(2 * workers)
        try:
            while pending:
                # woken twice a second even with nothing done, so a cancel need not wait for an item
                done, _ = wait(pending, timeout=0.5, return_when=FIRST_COMPLETED)
                for future in done:
                    yield pending.pop(future), future.result()
                raise_if_cancelled()
                submit(len(done))
        finally:
            # on an error or an early stop, leaving the pool would otherwise still run every queued item
            for future in pending:
                future.cancel()

round_significants(a, significant_digits)

Source code in src\muvis_align\util.py
328
329
330
331
332
def round_significants(a: float, significant_digits: int) -> float:
    if a != 0:
        round_decimals = significant_digits - int(np.floor(np.log10(abs(a)))) - 1
        return round(a, round_decimals)
    return a

set_dict_value(dct, keys, value)

Source code in src\muvis_align\util.py
148
149
150
151
152
153
154
155
156
157
158
159
def set_dict_value(dct, keys, value):
    try:
        value = float(value)
    except:
        pass
    for index, key in enumerate(keys):
        if index == len(keys) - 1:
            dct[key] = value
        else:
            if key not in dct:
                dct[key] = {}
            dct = dct[key]

set_source_metadata_value(source_metadata, keys, value)

Source code in src\muvis_align\util.py
170
171
172
173
174
175
176
177
178
def set_source_metadata_value(source_metadata, keys, value):
    # an emptied field drops its override, so the source's own value applies, as for 'source'
    if is_valid_value(value):
        set_dict_value(source_metadata, keys, value)
    else:
        parent = source_metadata
        for key in keys[:-1]:
            parent = parent.get(key, {})
        parent.pop(keys[-1], None)

single_threaded_highs()

Run scipy's HiGHS (linprog, as multiview-stitcher's overlap tests use) on the calling thread alone.

Source code in src\muvis_align\util.py
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
def single_threaded_highs():
    """Run scipy's HiGHS (linprog, as multiview-stitcher's overlap tests use) on the calling thread alone."""
    # HiGHS starts and tears down workers per calling thread, which can deadlock on the Windows loader lock;
    # these problems are tiny
    try:
        from scipy.optimize import _linprog_highs
    except ImportError:
        return
    highs_wrapper = getattr(_linprog_highs, '_highs_wrapper', None)
    if highs_wrapper is None or getattr(highs_wrapper, '_muvis_single_threaded', False):
        return

    def single_threaded(*args):
        *arrays, options = args
        return highs_wrapper(*arrays, dict(options, threads=1))

    single_threaded._muvis_single_threaded = True
    _linprog_highs._highs_wrapper = single_threaded

single_threaded_worker()

Worker process initializer: one thread per native pool, as the processes already use every core between them - left at their defaults, each worker's pools spin on cores the others need.

Source code in src\muvis_align\util.py
698
699
700
701
702
703
def single_threaded_worker():
    """Worker process initializer: one thread per native pool, as the processes already use every core
    between them - left at their defaults, each worker's pools spin on cores the others need."""
    from threadpoolctl import threadpool_limits
    cv.setNumThreads(1)
    threadpool_limits(1)

source_metadata_from_params(input_output)

The source_metadata the project's input_output fields give; its own source_metadata holds what no field does: a metadata file, per-file values, or flags (e.g. 'global center').

Source code in src\muvis_align\util.py
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
def source_metadata_from_params(input_output):
    """The source_metadata the project's input_output fields give; its own `source_metadata` holds what no field
    does: a metadata file, per-file values, or flags (e.g. 'global center')."""
    extra = input_output.get('source_metadata')
    if isinstance(extra, str) and os.path.splitext(extra)[1]:
        return extra
    if isinstance(extra, dict):
        source_metadata = copy.deepcopy(extra)
    else:
        source_metadata = {flag: True for flag in str(extra or '').split()}
    for name, keys in SOURCE_METADATA_FIELDS.items():
        if name in input_output:
            set_source_metadata_value(source_metadata, keys, input_output[name])
    if input_output.get('normalise_rotated_positions'):
        source_metadata['norm'] = True
    return source_metadata

split_num_text(text)

Source code in src\muvis_align\util.py
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
def split_num_text(text: str) -> list:
    num_texts = []
    block = ''
    is_num0 = None
    if text is None:
        return []

    for c in text:
        is_num = (c.isnumeric() or c == '.')
        if is_num0 is not None and is_num != is_num0:
            num_texts.append(block)
            block = ''
        block += c
        is_num0 = is_num
    if block != '':
        num_texts.append(block)

    num_texts2 = []
    for block in num_texts:
        block = block.strip()
        try:
            block = float(block)
        except:
            pass
        if block not in [' ', ',', '|']:
            num_texts2.append(block)
    return num_texts2

split_numeric(text)

Source code in src\muvis_align\util.py
388
389
390
391
392
393
394
395
def split_numeric(text: str) -> list:
    num_parts = []
    parts = split_path_parts(text)
    for part in parts:
        num_span = re.search(r'\d+', part)
        if num_span:
            num_parts.append(part)
    return num_parts

split_numeric_dict(text)

Source code in src\muvis_align\util.py
408
409
410
411
412
413
414
415
416
417
418
419
420
421
def split_numeric_dict(text: str) -> dict:
    num_parts = {}
    parts = split_path_parts(text)
    parti = 0
    for part in parts:
        num_span = re.search(r'\d+', part)
        if num_span:
            index = num_span.start()
            label = part[:index]
            if label == '':
                label = parti
            num_parts[label] = num_span.group()
            parti += 1
    return num_parts

split_path_parts(text)

Source code in src\muvis_align\util.py
382
383
384
385
def split_path_parts(text: str) -> list:
    return [part for part in
            text.replace('/', '_').replace('\\', '_').replace('.', '_').replace('-', '_').split('_')
            if part]

strip_common_path_prefix(filenames)

Source code in src\muvis_align\util.py
443
444
445
446
447
448
449
450
451
452
453
def strip_common_path_prefix(filenames: list) -> list:
    # leading directories shared by every file are just noise for labelling
    split = [filename.replace('\\', '/').split('/') for filename in filenames]
    common_len = 0
    for parts in zip(*split):
        if len(set(parts)) > 1:
            break
        common_len += 1
    if common_len == 0:
        return filenames
    return ['/'.join(parts[common_len:]) for parts in split]

thread_map(func, items, workers)

[func(item) for item in items], in order, on workers threads (one: in the calling thread).

Source code in src\muvis_align\util.py
743
744
745
746
747
748
749
750
def thread_map(func, items, workers):
    """[func(item) for item in items], in order, on `workers` threads (one: in the calling thread)."""
    items = list(items)
    workers = max(1, min(workers, len(items)))
    if workers == 1:
        return [func(item) for item in items]
    with ThreadPoolExecutor(max_workers=workers) as executor:
        return list(executor.map(func, items))

timed_calls(func, times, cpu_times)

func, appending each call's wall and thread CPU time to times and cpu_times (for format_phase_timing()). Keeps func's signature for callers that inspect it.

Source code in src\muvis_align\util.py
630
631
632
633
634
635
636
637
638
639
640
641
642
def timed_calls(func, times, cpu_times):
    """`func`, appending each call's wall and thread CPU time to `times` and `cpu_times` (for
    format_phase_timing()). Keeps func's signature for callers that inspect it.
    """
    @functools.wraps(func)
    def timed(*args, **kwargs):
        start, cpu_start = time.perf_counter(), time.thread_time()
        try:
            return func(*args, **kwargs)
        finally:
            times.append(time.perf_counter() - start)
            cpu_times.append(time.thread_time() - cpu_start)
    return timed

to_posix_path(path)

Source code in src\muvis_align\util.py
1245
1246
1247
def to_posix_path(path):
    # one separator everywhere: Windows reports '\' but accepts '/'
    return path.replace('\\', '/') if isinstance(path, str) else path

validate_transform(transform, max_scale=1.25, max_rotation=None)

Source code in src\muvis_align\util.py
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
def validate_transform(transform, max_scale = 1.25, max_rotation=None):
    try:
        if transform is None or isinstance(transform, str):
            return False
        transform = np.array(transform)
        if np.any(np.isnan(transform)):
            return False
        if np.any(np.isinf(transform)):
            return False
        if np.linalg.det(transform) == 0:
            return False
        scale = get_scale_from_transform(transform)
        if scale < 1 / max_scale or scale > max_scale:
            return False
        if  max_rotation is not None and abs(normalise_rotation(get_rotation_from_transform(transform))) > max_rotation:
            return False
        return True
    except:
        return False

worker_process_pool(workers, max_tasks_per_child=None)

Worker processes for GIL-bound per-item work. Spawned, never forked: forking a process running Qt and dask threads can copy a lock another thread holds.

Source code in src\muvis_align\util.py
753
754
755
756
757
def worker_process_pool(workers, max_tasks_per_child=None):
    """Worker processes for GIL-bound per-item work. Spawned, never forked: forking a process running
    Qt and dask threads can copy a lock another thread holds."""
    return _WorkerProcessPool(workers, mp_context=multiprocessing.get_context('spawn'),
                              initializer=single_threaded_worker, max_tasks_per_child=max_tasks_per_child)

xyz_to_dict(xyz, dims='xyz')

Source code in src\muvis_align\util.py
978
979
980
def xyz_to_dict(xyz, dims='xyz'):
    dct = {dim: float(value) for dim, value in zip(dims, xyz)}
    return dct

zarr_compat

Windows-only retry for zarr's atomic metadata write (zarr.storage._local._atomic_write).

On Windows the temporary file's rename onto the real name fails (PermissionError) if anything briefly holds the destination open. The holder is unidentified and the hold transient, so the rename is retried for a few seconds. Does nothing on other platforms, or if zarr's private helper has changed.

apply_windows_atomic_write_retry()

Install the retry. Idempotent, and a no-op where it does not apply.

Source code in src\muvis_align\zarr_compat.py
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
def apply_windows_atomic_write_retry():
    """Install the retry. Idempotent, and a no-op where it does not apply."""
    global _applied
    if _applied or os.name != 'nt':
        return False

    try:
        import zarr.storage._local as zarr_local
    except ImportError:
        return False
    original = getattr(zarr_local, '_atomic_write', None)
    safe_move = getattr(zarr_local, '_safe_move', None)
    if original is None or safe_move is None:
        # zarr's internals have moved - leave them alone rather than guess
        return False

    @contextlib.contextmanager
    def _atomic_write_with_retry(path, mode, exclusive=False):
        path = Path(path)
        tmp_path = path.with_suffix(f'.{uuid.uuid4().hex}.partial')
        # cleanup on failure only, as the original: on success the temp file has been renamed away
        try:
            with tmp_path.open(mode) as file:
                yield file
            delay, waited, attempts = _FIRST_DELAY_SECONDS, 0.0, 0
            while True:
                try:
                    if exclusive:
                        safe_move(tmp_path, path)
                    else:
                        tmp_path.replace(path)
                    break
                except PermissionError:
                    if waited >= _MAX_WAIT_SECONDS:
                        # a hold this long is not the transient one this exists for
                        raise
                    time.sleep(delay)
                    waited += delay
                    attempts += 1
                    delay = min(delay * 2, 0.2)
            if attempts:
                logging.debug(f'{path.name}: atomic write retried {attempts}x'
                              f' ({waited:.2f}s) - destination was briefly held open')
        except Exception:
            tmp_path.unlink(missing_ok=True)
            raise

    zarr_local._atomic_write = _atomic_write_with_retry
    _applied = True
    return True