3269 lines
199 KiB
Plaintext
3269 lines
199 KiB
Plaintext
2025-09-16 02:22:59,436: Requested GPU 1 not available. Using GPU 0 instead.
|
|
2025-09-16 02:22:59,586: Using device: cuda:0
|
|
2025-09-16 02:23:00,159: Using torch.compile
|
|
2025-09-16 02:23:01,335: Initialized RNN decoding model
|
|
2025-09-16 02:23:01,335: OptimizedModule(
|
|
(_orig_mod): LSTMDecoder(
|
|
(day_layer_activation): Softsign()
|
|
(day_weights): ParameterList(
|
|
(0): Parameter containing: [torch.float32 of size 512x512]
|
|
(1): Parameter containing: [torch.float32 of size 512x512]
|
|
(2): Parameter containing: [torch.float32 of size 512x512]
|
|
(3): Parameter containing: [torch.float32 of size 512x512]
|
|
(4): Parameter containing: [torch.float32 of size 512x512]
|
|
(5): Parameter containing: [torch.float32 of size 512x512]
|
|
(6): Parameter containing: [torch.float32 of size 512x512]
|
|
(7): Parameter containing: [torch.float32 of size 512x512]
|
|
(8): Parameter containing: [torch.float32 of size 512x512]
|
|
(9): Parameter containing: [torch.float32 of size 512x512]
|
|
(10): Parameter containing: [torch.float32 of size 512x512]
|
|
(11): Parameter containing: [torch.float32 of size 512x512]
|
|
(12): Parameter containing: [torch.float32 of size 512x512]
|
|
(13): Parameter containing: [torch.float32 of size 512x512]
|
|
(14): Parameter containing: [torch.float32 of size 512x512]
|
|
(15): Parameter containing: [torch.float32 of size 512x512]
|
|
(16): Parameter containing: [torch.float32 of size 512x512]
|
|
(17): Parameter containing: [torch.float32 of size 512x512]
|
|
(18): Parameter containing: [torch.float32 of size 512x512]
|
|
(19): Parameter containing: [torch.float32 of size 512x512]
|
|
(20): Parameter containing: [torch.float32 of size 512x512]
|
|
(21): Parameter containing: [torch.float32 of size 512x512]
|
|
(22): Parameter containing: [torch.float32 of size 512x512]
|
|
(23): Parameter containing: [torch.float32 of size 512x512]
|
|
(24): Parameter containing: [torch.float32 of size 512x512]
|
|
(25): Parameter containing: [torch.float32 of size 512x512]
|
|
(26): Parameter containing: [torch.float32 of size 512x512]
|
|
(27): Parameter containing: [torch.float32 of size 512x512]
|
|
(28): Parameter containing: [torch.float32 of size 512x512]
|
|
(29): Parameter containing: [torch.float32 of size 512x512]
|
|
(30): Parameter containing: [torch.float32 of size 512x512]
|
|
(31): Parameter containing: [torch.float32 of size 512x512]
|
|
(32): Parameter containing: [torch.float32 of size 512x512]
|
|
(33): Parameter containing: [torch.float32 of size 512x512]
|
|
(34): Parameter containing: [torch.float32 of size 512x512]
|
|
(35): Parameter containing: [torch.float32 of size 512x512]
|
|
(36): Parameter containing: [torch.float32 of size 512x512]
|
|
(37): Parameter containing: [torch.float32 of size 512x512]
|
|
(38): Parameter containing: [torch.float32 of size 512x512]
|
|
(39): Parameter containing: [torch.float32 of size 512x512]
|
|
(40): Parameter containing: [torch.float32 of size 512x512]
|
|
(41): Parameter containing: [torch.float32 of size 512x512]
|
|
(42): Parameter containing: [torch.float32 of size 512x512]
|
|
(43): Parameter containing: [torch.float32 of size 512x512]
|
|
(44): Parameter containing: [torch.float32 of size 512x512]
|
|
)
|
|
(day_biases): ParameterList(
|
|
(0): Parameter containing: [torch.float32 of size 1x512]
|
|
(1): Parameter containing: [torch.float32 of size 1x512]
|
|
(2): Parameter containing: [torch.float32 of size 1x512]
|
|
(3): Parameter containing: [torch.float32 of size 1x512]
|
|
(4): Parameter containing: [torch.float32 of size 1x512]
|
|
(5): Parameter containing: [torch.float32 of size 1x512]
|
|
(6): Parameter containing: [torch.float32 of size 1x512]
|
|
(7): Parameter containing: [torch.float32 of size 1x512]
|
|
(8): Parameter containing: [torch.float32 of size 1x512]
|
|
(9): Parameter containing: [torch.float32 of size 1x512]
|
|
(10): Parameter containing: [torch.float32 of size 1x512]
|
|
(11): Parameter containing: [torch.float32 of size 1x512]
|
|
(12): Parameter containing: [torch.float32 of size 1x512]
|
|
(13): Parameter containing: [torch.float32 of size 1x512]
|
|
(14): Parameter containing: [torch.float32 of size 1x512]
|
|
(15): Parameter containing: [torch.float32 of size 1x512]
|
|
(16): Parameter containing: [torch.float32 of size 1x512]
|
|
(17): Parameter containing: [torch.float32 of size 1x512]
|
|
(18): Parameter containing: [torch.float32 of size 1x512]
|
|
(19): Parameter containing: [torch.float32 of size 1x512]
|
|
(20): Parameter containing: [torch.float32 of size 1x512]
|
|
(21): Parameter containing: [torch.float32 of size 1x512]
|
|
(22): Parameter containing: [torch.float32 of size 1x512]
|
|
(23): Parameter containing: [torch.float32 of size 1x512]
|
|
(24): Parameter containing: [torch.float32 of size 1x512]
|
|
(25): Parameter containing: [torch.float32 of size 1x512]
|
|
(26): Parameter containing: [torch.float32 of size 1x512]
|
|
(27): Parameter containing: [torch.float32 of size 1x512]
|
|
(28): Parameter containing: [torch.float32 of size 1x512]
|
|
(29): Parameter containing: [torch.float32 of size 1x512]
|
|
(30): Parameter containing: [torch.float32 of size 1x512]
|
|
(31): Parameter containing: [torch.float32 of size 1x512]
|
|
(32): Parameter containing: [torch.float32 of size 1x512]
|
|
(33): Parameter containing: [torch.float32 of size 1x512]
|
|
(34): Parameter containing: [torch.float32 of size 1x512]
|
|
(35): Parameter containing: [torch.float32 of size 1x512]
|
|
(36): Parameter containing: [torch.float32 of size 1x512]
|
|
(37): Parameter containing: [torch.float32 of size 1x512]
|
|
(38): Parameter containing: [torch.float32 of size 1x512]
|
|
(39): Parameter containing: [torch.float32 of size 1x512]
|
|
(40): Parameter containing: [torch.float32 of size 1x512]
|
|
(41): Parameter containing: [torch.float32 of size 1x512]
|
|
(42): Parameter containing: [torch.float32 of size 1x512]
|
|
(43): Parameter containing: [torch.float32 of size 1x512]
|
|
(44): Parameter containing: [torch.float32 of size 1x512]
|
|
)
|
|
(day_layer_dropout): Dropout(p=0.2, inplace=False)
|
|
(lstm): LSTM(7168, 768, num_layers=5, batch_first=True, dropout=0.4)
|
|
(out): Linear(in_features=768, out_features=41, bias=True)
|
|
)
|
|
)
|
|
2025-09-16 02:23:01,336: Model has 55,136,297 parameters
|
|
2025-09-16 02:23:01,336: Model has 11,819,520 day-specific parameters | 21.44% of total parameters
|
|
2025-09-16 02:23:09,866: Successfully initialized datasets
|
|
2025-09-16 02:23:32,809: Train batch 0: loss: 612.68 grad norm: 836.66 time: 20.267
|
|
2025-09-16 02:23:32,809: Running test after training batch: 0
|
|
2025-09-16 02:24:12,899: Val batch 0: PER (avg): 0.9121 CTC Loss (avg): 672.5561 time: 40.090
|
|
2025-09-16 02:24:12,899: t15.2023.08.13 val PER: 0.9200
|
|
2025-09-16 02:24:12,899: t15.2023.08.18 val PER: 0.9187
|
|
2025-09-16 02:24:12,899: t15.2023.08.20 val PER: 0.9150
|
|
2025-09-16 02:24:12,899: t15.2023.08.25 val PER: 0.9081
|
|
2025-09-16 02:24:12,899: t15.2023.08.27 val PER: 0.8923
|
|
2025-09-16 02:24:12,899: t15.2023.09.01 val PER: 0.9164
|
|
2025-09-16 02:24:12,900: t15.2023.09.03 val PER: 0.9145
|
|
2025-09-16 02:24:12,900: t15.2023.09.24 val PER: 0.9211
|
|
2025-09-16 02:24:12,900: t15.2023.09.29 val PER: 0.8998
|
|
2025-09-16 02:24:12,900: t15.2023.10.01 val PER: 0.9141
|
|
2025-09-16 02:24:12,900: t15.2023.10.06 val PER: 0.9139
|
|
2025-09-16 02:24:12,900: t15.2023.10.08 val PER: 0.9039
|
|
2025-09-16 02:24:12,900: t15.2023.10.13 val PER: 0.9030
|
|
2025-09-16 02:24:12,900: t15.2023.10.15 val PER: 0.9011
|
|
2025-09-16 02:24:12,900: t15.2023.10.20 val PER: 0.9195
|
|
2025-09-16 02:24:12,900: t15.2023.10.22 val PER: 0.9176
|
|
2025-09-16 02:24:12,900: t15.2023.11.03 val PER: 0.9132
|
|
2025-09-16 02:24:12,900: t15.2023.11.04 val PER: 0.9454
|
|
2025-09-16 02:24:12,900: t15.2023.11.17 val PER: 0.8989
|
|
2025-09-16 02:24:12,900: t15.2023.11.19 val PER: 0.9062
|
|
2025-09-16 02:24:12,900: t15.2023.11.26 val PER: 0.9116
|
|
2025-09-16 02:24:12,900: t15.2023.12.03 val PER: 0.9118
|
|
2025-09-16 02:24:12,900: t15.2023.12.08 val PER: 0.9108
|
|
2025-09-16 02:24:12,900: t15.2023.12.10 val PER: 0.9067
|
|
2025-09-16 02:24:12,901: t15.2023.12.17 val PER: 0.9075
|
|
2025-09-16 02:24:12,901: t15.2023.12.29 val PER: 0.9149
|
|
2025-09-16 02:24:12,901: t15.2024.02.25 val PER: 0.9129
|
|
2025-09-16 02:24:12,901: t15.2024.03.08 val PER: 0.9260
|
|
2025-09-16 02:24:12,901: t15.2024.03.15 val PER: 0.9043
|
|
2025-09-16 02:24:12,901: t15.2024.03.17 val PER: 0.9086
|
|
2025-09-16 02:24:12,901: t15.2024.05.10 val PER: 0.9034
|
|
2025-09-16 02:24:12,901: t15.2024.06.14 val PER: 0.9227
|
|
2025-09-16 02:24:12,901: t15.2024.07.19 val PER: 0.9235
|
|
2025-09-16 02:24:12,901: t15.2024.07.21 val PER: 0.9062
|
|
2025-09-16 02:24:12,901: t15.2024.07.28 val PER: 0.9066
|
|
2025-09-16 02:24:12,901: t15.2025.01.10 val PER: 0.9256
|
|
2025-09-16 02:24:12,901: t15.2025.01.12 val PER: 0.9169
|
|
2025-09-16 02:24:12,901: t15.2025.03.14 val PER: 0.9231
|
|
2025-09-16 02:24:12,901: t15.2025.03.16 val PER: 0.9215
|
|
2025-09-16 02:24:12,901: t15.2025.03.30 val PER: 0.9195
|
|
2025-09-16 02:24:12,901: t15.2025.04.13 val PER: 0.9087
|
|
2025-09-16 02:24:12,901: New best test PER inf --> 0.9121
|
|
2025-09-16 02:24:12,901: Checkpointing model
|
|
2025-09-16 02:24:13,528: Saved model to checkpoint: trained_models/baseline_rnn/checkpoint/best_checkpoint
|
|
2025-09-16 02:24:45,139: Train batch 200: loss: 84.21 grad norm: 121.90 time: 0.126
|
|
2025-09-16 02:25:15,163: Train batch 400: loss: 87.03 grad norm: 47.23 time: 0.128
|
|
2025-09-16 02:25:46,613: Train batch 600: loss: 80.02 grad norm: 22.49 time: 0.108
|
|
2025-09-16 02:26:18,148: Train batch 800: loss: 87.85 grad norm: 15.45 time: 0.095
|
|
2025-09-16 02:26:51,632: Train batch 1000: loss: 93.60 grad norm: 45.49 time: 0.162
|
|
2025-09-16 02:27:24,665: Train batch 1200: loss: 83.03 grad norm: 47.63 time: 0.120
|
|
2025-09-16 02:27:58,137: Train batch 1400: loss: 88.91 grad norm: 26.47 time: 0.136
|
|
2025-09-16 02:28:30,795: Train batch 1600: loss: 68.38 grad norm: 39.91 time: 0.116
|
|
2025-09-16 02:29:03,999: Train batch 1800: loss: 54.53 grad norm: 38.24 time: 0.127
|
|
2025-09-16 02:29:37,114: Train batch 2000: loss: 51.56 grad norm: 50.53 time: 0.114
|
|
2025-09-16 02:29:37,115: Running test after training batch: 2000
|
|
2025-09-16 02:29:47,573: Val batch 2000: PER (avg): 0.4835 CTC Loss (avg): 50.9311 time: 10.458
|
|
2025-09-16 02:29:47,574: t15.2023.08.13 val PER: 0.4543
|
|
2025-09-16 02:29:47,574: t15.2023.08.18 val PER: 0.4493
|
|
2025-09-16 02:29:47,574: t15.2023.08.20 val PER: 0.4249
|
|
2025-09-16 02:29:47,574: t15.2023.08.25 val PER: 0.4337
|
|
2025-09-16 02:29:47,574: t15.2023.08.27 val PER: 0.5531
|
|
2025-09-16 02:29:47,574: t15.2023.09.01 val PER: 0.4367
|
|
2025-09-16 02:29:47,574: t15.2023.09.03 val PER: 0.4774
|
|
2025-09-16 02:29:47,574: t15.2023.09.24 val PER: 0.4345
|
|
2025-09-16 02:29:47,574: t15.2023.09.29 val PER: 0.4576
|
|
2025-09-16 02:29:47,574: t15.2023.10.01 val PER: 0.5033
|
|
2025-09-16 02:29:47,574: t15.2023.10.06 val PER: 0.4230
|
|
2025-09-16 02:29:47,574: t15.2023.10.08 val PER: 0.5291
|
|
2025-09-16 02:29:47,574: t15.2023.10.13 val PER: 0.5656
|
|
2025-09-16 02:29:47,575: t15.2023.10.15 val PER: 0.4964
|
|
2025-09-16 02:29:47,575: t15.2023.10.20 val PER: 0.4765
|
|
2025-09-16 02:29:47,575: t15.2023.10.22 val PER: 0.4499
|
|
2025-09-16 02:29:47,575: t15.2023.11.03 val PER: 0.4837
|
|
2025-09-16 02:29:47,575: t15.2023.11.04 val PER: 0.2526
|
|
2025-09-16 02:29:47,575: t15.2023.11.17 val PER: 0.3437
|
|
2025-09-16 02:29:47,575: t15.2023.11.19 val PER: 0.3293
|
|
2025-09-16 02:29:47,575: t15.2023.11.26 val PER: 0.5384
|
|
2025-09-16 02:29:47,575: t15.2023.12.03 val PER: 0.4716
|
|
2025-09-16 02:29:47,575: t15.2023.12.08 val PER: 0.4687
|
|
2025-09-16 02:29:47,575: t15.2023.12.10 val PER: 0.4560
|
|
2025-09-16 02:29:47,575: t15.2023.12.17 val PER: 0.4938
|
|
2025-09-16 02:29:47,575: t15.2023.12.29 val PER: 0.4921
|
|
2025-09-16 02:29:47,575: t15.2024.02.25 val PER: 0.4368
|
|
2025-09-16 02:29:47,575: t15.2024.03.08 val PER: 0.5292
|
|
2025-09-16 02:29:47,575: t15.2024.03.15 val PER: 0.5059
|
|
2025-09-16 02:29:47,575: t15.2024.03.17 val PER: 0.4847
|
|
2025-09-16 02:29:47,575: t15.2024.05.10 val PER: 0.4948
|
|
2025-09-16 02:29:47,576: t15.2024.06.14 val PER: 0.4495
|
|
2025-09-16 02:29:47,576: t15.2024.07.19 val PER: 0.5603
|
|
2025-09-16 02:29:47,576: t15.2024.07.21 val PER: 0.4255
|
|
2025-09-16 02:29:47,576: t15.2024.07.28 val PER: 0.4625
|
|
2025-09-16 02:29:47,576: t15.2025.01.10 val PER: 0.5785
|
|
2025-09-16 02:29:47,576: t15.2025.01.12 val PER: 0.4942
|
|
2025-09-16 02:29:47,576: t15.2025.03.14 val PER: 0.6198
|
|
2025-09-16 02:29:47,576: t15.2025.03.16 val PER: 0.5301
|
|
2025-09-16 02:29:47,576: t15.2025.03.30 val PER: 0.5966
|
|
2025-09-16 02:29:47,576: t15.2025.04.13 val PER: 0.5250
|
|
2025-09-16 02:29:47,576: New best test PER 0.9121 --> 0.4835
|
|
2025-09-16 02:29:47,576: Checkpointing model
|
|
2025-09-16 02:29:48,957: Saved model to checkpoint: trained_models/baseline_rnn/checkpoint/best_checkpoint
|
|
2025-09-16 02:30:17,710: Train batch 2200: loss: 32.40 grad norm: 49.07 time: 0.127
|
|
2025-09-16 02:30:51,200: Train batch 2400: loss: 39.65 grad norm: 56.14 time: 0.163
|
|
2025-09-16 02:31:24,566: Train batch 2600: loss: 34.27 grad norm: 60.27 time: 0.134
|
|
2025-09-16 02:31:58,164: Train batch 2800: loss: 21.85 grad norm: 51.96 time: 0.172
|
|
2025-09-16 02:32:30,953: Train batch 3000: loss: 26.57 grad norm: 52.86 time: 0.119
|
|
2025-09-16 02:33:03,711: Train batch 3200: loss: 21.47 grad norm: 51.75 time: 0.140
|
|
2025-09-16 02:33:36,035: Train batch 3400: loss: 20.91 grad norm: 51.96 time: 0.169
|
|
2025-09-16 02:34:09,470: Train batch 3600: loss: 16.33 grad norm: 46.02 time: 0.119
|
|
2025-09-16 02:34:43,393: Train batch 3800: loss: 14.50 grad norm: 45.01 time: 0.141
|
|
2025-09-16 02:35:16,326: Train batch 4000: loss: 13.81 grad norm: 43.23 time: 0.152
|
|
2025-09-16 02:35:16,326: Running test after training batch: 4000
|
|
2025-09-16 02:35:27,094: Val batch 4000: PER (avg): 0.2010 CTC Loss (avg): 21.8241 time: 10.767
|
|
2025-09-16 02:35:27,094: t15.2023.08.13 val PER: 0.1798
|
|
2025-09-16 02:35:27,094: t15.2023.08.18 val PER: 0.1643
|
|
2025-09-16 02:35:27,094: t15.2023.08.20 val PER: 0.1422
|
|
2025-09-16 02:35:27,094: t15.2023.08.25 val PER: 0.1822
|
|
2025-09-16 02:35:27,094: t15.2023.08.27 val PER: 0.2508
|
|
2025-09-16 02:35:27,094: t15.2023.09.01 val PER: 0.1193
|
|
2025-09-16 02:35:27,094: t15.2023.09.03 val PER: 0.1971
|
|
2025-09-16 02:35:27,095: t15.2023.09.24 val PER: 0.1505
|
|
2025-09-16 02:35:27,095: t15.2023.09.29 val PER: 0.1857
|
|
2025-09-16 02:35:27,095: t15.2023.10.01 val PER: 0.2358
|
|
2025-09-16 02:35:27,095: t15.2023.10.06 val PER: 0.1270
|
|
2025-09-16 02:35:27,095: t15.2023.10.08 val PER: 0.2963
|
|
2025-09-16 02:35:27,095: t15.2023.10.13 val PER: 0.2599
|
|
2025-09-16 02:35:27,095: t15.2023.10.15 val PER: 0.1859
|
|
2025-09-16 02:35:27,095: t15.2023.10.20 val PER: 0.2651
|
|
2025-09-16 02:35:27,095: t15.2023.10.22 val PER: 0.1871
|
|
2025-09-16 02:35:27,095: t15.2023.11.03 val PER: 0.2273
|
|
2025-09-16 02:35:27,095: t15.2023.11.04 val PER: 0.0410
|
|
2025-09-16 02:35:27,095: t15.2023.11.17 val PER: 0.0684
|
|
2025-09-16 02:35:27,095: t15.2023.11.19 val PER: 0.0639
|
|
2025-09-16 02:35:27,095: t15.2023.11.26 val PER: 0.2123
|
|
2025-09-16 02:35:27,095: t15.2023.12.03 val PER: 0.1765
|
|
2025-09-16 02:35:27,095: t15.2023.12.08 val PER: 0.1471
|
|
2025-09-16 02:35:27,095: t15.2023.12.10 val PER: 0.1340
|
|
2025-09-16 02:35:27,095: t15.2023.12.17 val PER: 0.1933
|
|
2025-09-16 02:35:27,095: t15.2023.12.29 val PER: 0.2025
|
|
2025-09-16 02:35:27,096: t15.2024.02.25 val PER: 0.1475
|
|
2025-09-16 02:35:27,096: t15.2024.03.08 val PER: 0.2646
|
|
2025-09-16 02:35:27,096: t15.2024.03.15 val PER: 0.2689
|
|
2025-09-16 02:35:27,096: t15.2024.03.17 val PER: 0.1855
|
|
2025-09-16 02:35:27,096: t15.2024.05.10 val PER: 0.2036
|
|
2025-09-16 02:35:27,096: t15.2024.06.14 val PER: 0.1845
|
|
2025-09-16 02:35:27,096: t15.2024.07.19 val PER: 0.2795
|
|
2025-09-16 02:35:27,096: t15.2024.07.21 val PER: 0.1545
|
|
2025-09-16 02:35:27,096: t15.2024.07.28 val PER: 0.1897
|
|
2025-09-16 02:35:27,096: t15.2025.01.10 val PER: 0.3402
|
|
2025-09-16 02:35:27,096: t15.2025.01.12 val PER: 0.1986
|
|
2025-09-16 02:35:27,096: t15.2025.03.14 val PER: 0.3521
|
|
2025-09-16 02:35:27,096: t15.2025.03.16 val PER: 0.2382
|
|
2025-09-16 02:35:27,096: t15.2025.03.30 val PER: 0.3057
|
|
2025-09-16 02:35:27,096: t15.2025.04.13 val PER: 0.2596
|
|
2025-09-16 02:35:27,096: New best test PER 0.4835 --> 0.2010
|
|
2025-09-16 02:35:27,096: Checkpointing model
|
|
2025-09-16 02:35:28,464: Saved model to checkpoint: trained_models/baseline_rnn/checkpoint/best_checkpoint
|
|
2025-09-16 02:35:57,562: Train batch 4200: loss: 10.00 grad norm: 36.88 time: 0.136
|
|
2025-09-16 02:36:29,971: Train batch 4400: loss: 10.55 grad norm: 40.50 time: 0.145
|
|
2025-09-16 02:37:03,347: Train batch 4600: loss: 9.47 grad norm: 39.52 time: 0.100
|
|
2025-09-16 02:37:36,887: Train batch 4800: loss: 8.85 grad norm: 43.24 time: 0.153
|
|
2025-09-16 02:38:10,660: Train batch 5000: loss: 7.81 grad norm: 40.19 time: 0.097
|
|
2025-09-16 02:38:43,534: Train batch 5200: loss: 8.81 grad norm: 40.42 time: 0.115
|
|
2025-09-16 02:39:17,280: Train batch 5400: loss: 8.16 grad norm: 36.33 time: 0.152
|
|
2025-09-16 02:39:51,423: Train batch 5600: loss: 7.56 grad norm: 35.75 time: 0.177
|
|
2025-09-16 02:40:24,704: Train batch 5800: loss: 6.31 grad norm: 32.34 time: 0.119
|
|
2025-09-16 02:40:57,935: Train batch 6000: loss: 4.56 grad norm: 30.15 time: 0.113
|
|
2025-09-16 02:40:57,936: Running test after training batch: 6000
|
|
2025-09-16 02:41:08,559: Val batch 6000: PER (avg): 0.1604 CTC Loss (avg): 19.9005 time: 10.623
|
|
2025-09-16 02:41:08,559: t15.2023.08.13 val PER: 0.1424
|
|
2025-09-16 02:41:08,559: t15.2023.08.18 val PER: 0.1358
|
|
2025-09-16 02:41:08,559: t15.2023.08.20 val PER: 0.1001
|
|
2025-09-16 02:41:08,559: t15.2023.08.25 val PER: 0.1130
|
|
2025-09-16 02:41:08,559: t15.2023.08.27 val PER: 0.2170
|
|
2025-09-16 02:41:08,560: t15.2023.09.01 val PER: 0.0950
|
|
2025-09-16 02:41:08,560: t15.2023.09.03 val PER: 0.1758
|
|
2025-09-16 02:41:08,560: t15.2023.09.24 val PER: 0.1311
|
|
2025-09-16 02:41:08,560: t15.2023.09.29 val PER: 0.1449
|
|
2025-09-16 02:41:08,560: t15.2023.10.01 val PER: 0.2015
|
|
2025-09-16 02:41:08,560: t15.2023.10.06 val PER: 0.1023
|
|
2025-09-16 02:41:08,560: t15.2023.10.08 val PER: 0.2612
|
|
2025-09-16 02:41:08,560: t15.2023.10.13 val PER: 0.2320
|
|
2025-09-16 02:41:08,560: t15.2023.10.15 val PER: 0.1595
|
|
2025-09-16 02:41:08,560: t15.2023.10.20 val PER: 0.2282
|
|
2025-09-16 02:41:08,560: t15.2023.10.22 val PER: 0.1492
|
|
2025-09-16 02:41:08,560: t15.2023.11.03 val PER: 0.1961
|
|
2025-09-16 02:41:08,560: t15.2023.11.04 val PER: 0.0171
|
|
2025-09-16 02:41:08,560: t15.2023.11.17 val PER: 0.0373
|
|
2025-09-16 02:41:08,560: t15.2023.11.19 val PER: 0.0200
|
|
2025-09-16 02:41:08,560: t15.2023.11.26 val PER: 0.1174
|
|
2025-09-16 02:41:08,560: t15.2023.12.03 val PER: 0.0998
|
|
2025-09-16 02:41:08,560: t15.2023.12.08 val PER: 0.0859
|
|
2025-09-16 02:41:08,561: t15.2023.12.10 val PER: 0.0775
|
|
2025-09-16 02:41:08,561: t15.2023.12.17 val PER: 0.1393
|
|
2025-09-16 02:41:08,561: t15.2023.12.29 val PER: 0.1380
|
|
2025-09-16 02:41:08,561: t15.2024.02.25 val PER: 0.1250
|
|
2025-09-16 02:41:08,561: t15.2024.03.08 val PER: 0.2319
|
|
2025-09-16 02:41:08,561: t15.2024.03.15 val PER: 0.2114
|
|
2025-09-16 02:41:08,561: t15.2024.03.17 val PER: 0.1450
|
|
2025-09-16 02:41:08,561: t15.2024.05.10 val PER: 0.1694
|
|
2025-09-16 02:41:08,561: t15.2024.06.14 val PER: 0.1656
|
|
2025-09-16 02:41:08,561: t15.2024.07.19 val PER: 0.2340
|
|
2025-09-16 02:41:08,561: t15.2024.07.21 val PER: 0.0966
|
|
2025-09-16 02:41:08,561: t15.2024.07.28 val PER: 0.1338
|
|
2025-09-16 02:41:08,561: t15.2025.01.10 val PER: 0.2879
|
|
2025-09-16 02:41:08,561: t15.2025.01.12 val PER: 0.1663
|
|
2025-09-16 02:41:08,561: t15.2025.03.14 val PER: 0.3713
|
|
2025-09-16 02:41:08,561: t15.2025.03.16 val PER: 0.2068
|
|
2025-09-16 02:41:08,561: t15.2025.03.30 val PER: 0.2931
|
|
2025-09-16 02:41:08,561: t15.2025.04.13 val PER: 0.2511
|
|
2025-09-16 02:41:08,561: New best test PER 0.2010 --> 0.1604
|
|
2025-09-16 02:41:08,561: Checkpointing model
|
|
2025-09-16 02:41:09,877: Saved model to checkpoint: trained_models/baseline_rnn/checkpoint/best_checkpoint
|
|
2025-09-16 02:41:36,842: Train batch 6200: loss: 4.26 grad norm: 31.07 time: 0.126
|
|
2025-09-16 02:42:10,138: Train batch 6400: loss: 2.94 grad norm: 23.22 time: 0.115
|
|
2025-09-16 02:42:43,092: Train batch 6600: loss: 4.79 grad norm: 30.53 time: 0.142
|
|
2025-09-16 02:43:15,977: Train batch 6800: loss: 5.68 grad norm: 34.55 time: 0.117
|
|
2025-09-16 02:43:49,463: Train batch 7000: loss: 4.95 grad norm: 37.61 time: 0.158
|
|
2025-09-16 02:44:22,489: Train batch 7200: loss: 2.23 grad norm: 22.51 time: 0.114
|
|
2025-09-16 02:44:55,991: Train batch 7400: loss: 2.57 grad norm: 27.48 time: 0.166
|
|
2025-09-16 02:45:28,825: Train batch 7600: loss: 2.64 grad norm: 23.96 time: 0.137
|
|
2025-09-16 02:46:02,732: Train batch 7800: loss: 2.39 grad norm: 23.13 time: 0.119
|
|
2025-09-16 02:46:36,240: Train batch 8000: loss: 2.71 grad norm: 26.99 time: 0.131
|
|
2025-09-16 02:46:36,240: Running test after training batch: 8000
|
|
2025-09-16 02:46:46,689: Val batch 8000: PER (avg): 0.1423 CTC Loss (avg): 19.8109 time: 10.448
|
|
2025-09-16 02:46:46,689: t15.2023.08.13 val PER: 0.1112
|
|
2025-09-16 02:46:46,689: t15.2023.08.18 val PER: 0.1098
|
|
2025-09-16 02:46:46,689: t15.2023.08.20 val PER: 0.0826
|
|
2025-09-16 02:46:46,689: t15.2023.08.25 val PER: 0.1009
|
|
2025-09-16 02:46:46,689: t15.2023.08.27 val PER: 0.2010
|
|
2025-09-16 02:46:46,689: t15.2023.09.01 val PER: 0.0609
|
|
2025-09-16 02:46:46,690: t15.2023.09.03 val PER: 0.1413
|
|
2025-09-16 02:46:46,690: t15.2023.09.24 val PER: 0.1117
|
|
2025-09-16 02:46:46,690: t15.2023.09.29 val PER: 0.1232
|
|
2025-09-16 02:46:46,690: t15.2023.10.01 val PER: 0.1744
|
|
2025-09-16 02:46:46,690: t15.2023.10.06 val PER: 0.0818
|
|
2025-09-16 02:46:46,690: t15.2023.10.08 val PER: 0.2368
|
|
2025-09-16 02:46:46,690: t15.2023.10.13 val PER: 0.1947
|
|
2025-09-16 02:46:46,690: t15.2023.10.15 val PER: 0.1470
|
|
2025-09-16 02:46:46,690: t15.2023.10.20 val PER: 0.2081
|
|
2025-09-16 02:46:46,690: t15.2023.10.22 val PER: 0.1102
|
|
2025-09-16 02:46:46,690: t15.2023.11.03 val PER: 0.1859
|
|
2025-09-16 02:46:46,690: t15.2023.11.04 val PER: 0.0068
|
|
2025-09-16 02:46:46,690: t15.2023.11.17 val PER: 0.0264
|
|
2025-09-16 02:46:46,690: t15.2023.11.19 val PER: 0.0259
|
|
2025-09-16 02:46:46,690: t15.2023.11.26 val PER: 0.0971
|
|
2025-09-16 02:46:46,690: t15.2023.12.03 val PER: 0.0777
|
|
2025-09-16 02:46:46,690: t15.2023.12.08 val PER: 0.0699
|
|
2025-09-16 02:46:46,690: t15.2023.12.10 val PER: 0.0723
|
|
2025-09-16 02:46:46,690: t15.2023.12.17 val PER: 0.1216
|
|
2025-09-16 02:46:46,691: t15.2023.12.29 val PER: 0.1229
|
|
2025-09-16 02:46:46,691: t15.2024.02.25 val PER: 0.1180
|
|
2025-09-16 02:46:46,691: t15.2024.03.08 val PER: 0.2248
|
|
2025-09-16 02:46:46,691: t15.2024.03.15 val PER: 0.1920
|
|
2025-09-16 02:46:46,691: t15.2024.03.17 val PER: 0.1283
|
|
2025-09-16 02:46:46,691: t15.2024.05.10 val PER: 0.1753
|
|
2025-09-16 02:46:46,691: t15.2024.06.14 val PER: 0.1467
|
|
2025-09-16 02:46:46,691: t15.2024.07.19 val PER: 0.2076
|
|
2025-09-16 02:46:46,691: t15.2024.07.21 val PER: 0.0910
|
|
2025-09-16 02:46:46,691: t15.2024.07.28 val PER: 0.1338
|
|
2025-09-16 02:46:46,691: t15.2025.01.10 val PER: 0.3017
|
|
2025-09-16 02:46:46,691: t15.2025.01.12 val PER: 0.1378
|
|
2025-09-16 02:46:46,691: t15.2025.03.14 val PER: 0.3536
|
|
2025-09-16 02:46:46,691: t15.2025.03.16 val PER: 0.2003
|
|
2025-09-16 02:46:46,691: t15.2025.03.30 val PER: 0.2736
|
|
2025-09-16 02:46:46,691: t15.2025.04.13 val PER: 0.2225
|
|
2025-09-16 02:46:46,691: New best test PER 0.1604 --> 0.1423
|
|
2025-09-16 02:46:46,691: Checkpointing model
|
|
2025-09-16 02:46:48,087: Saved model to checkpoint: trained_models/baseline_rnn/checkpoint/best_checkpoint
|
|
2025-09-16 02:47:16,641: Train batch 8200: loss: 2.67 grad norm: 30.53 time: 0.175
|
|
2025-09-16 02:47:49,193: Train batch 8400: loss: 0.87 grad norm: 13.73 time: 0.099
|
|
2025-09-16 02:48:23,298: Train batch 8600: loss: 1.30 grad norm: 18.09 time: 0.113
|
|
2025-09-16 02:48:56,375: Train batch 8800: loss: 2.18 grad norm: 27.11 time: 0.151
|
|
2025-09-16 02:49:29,958: Train batch 9000: loss: 2.78 grad norm: 33.77 time: 0.162
|
|
2025-09-16 02:50:03,427: Train batch 9200: loss: 1.37 grad norm: 16.95 time: 0.107
|
|
2025-09-16 02:50:37,701: Train batch 9400: loss: 2.61 grad norm: 27.93 time: 0.135
|
|
2025-09-16 02:51:11,177: Train batch 9600: loss: 2.11 grad norm: 27.27 time: 0.146
|
|
2025-09-16 02:51:43,890: Train batch 9800: loss: 1.95 grad norm: 18.24 time: 0.119
|
|
2025-09-16 02:52:18,023: Train batch 10000: loss: 1.16 grad norm: 18.52 time: 0.155
|
|
2025-09-16 02:52:18,023: Running test after training batch: 10000
|
|
2025-09-16 02:52:28,737: Val batch 10000: PER (avg): 0.1379 CTC Loss (avg): 22.1460 time: 10.714
|
|
2025-09-16 02:52:28,738: t15.2023.08.13 val PER: 0.1071
|
|
2025-09-16 02:52:28,738: t15.2023.08.18 val PER: 0.0972
|
|
2025-09-16 02:52:28,738: t15.2023.08.20 val PER: 0.0707
|
|
2025-09-16 02:52:28,738: t15.2023.08.25 val PER: 0.1114
|
|
2025-09-16 02:52:28,738: t15.2023.08.27 val PER: 0.2090
|
|
2025-09-16 02:52:28,738: t15.2023.09.01 val PER: 0.0625
|
|
2025-09-16 02:52:28,738: t15.2023.09.03 val PER: 0.1425
|
|
2025-09-16 02:52:28,738: t15.2023.09.24 val PER: 0.1092
|
|
2025-09-16 02:52:28,738: t15.2023.09.29 val PER: 0.1295
|
|
2025-09-16 02:52:28,738: t15.2023.10.01 val PER: 0.1704
|
|
2025-09-16 02:52:28,738: t15.2023.10.06 val PER: 0.0883
|
|
2025-09-16 02:52:28,738: t15.2023.10.08 val PER: 0.2300
|
|
2025-09-16 02:52:28,738: t15.2023.10.13 val PER: 0.1994
|
|
2025-09-16 02:52:28,738: t15.2023.10.15 val PER: 0.1292
|
|
2025-09-16 02:52:28,738: t15.2023.10.20 val PER: 0.2047
|
|
2025-09-16 02:52:28,738: t15.2023.10.22 val PER: 0.1336
|
|
2025-09-16 02:52:28,738: t15.2023.11.03 val PER: 0.1893
|
|
2025-09-16 02:52:28,738: t15.2023.11.04 val PER: 0.0068
|
|
2025-09-16 02:52:28,739: t15.2023.11.17 val PER: 0.0435
|
|
2025-09-16 02:52:28,739: t15.2023.11.19 val PER: 0.0240
|
|
2025-09-16 02:52:28,739: t15.2023.11.26 val PER: 0.0928
|
|
2025-09-16 02:52:28,739: t15.2023.12.03 val PER: 0.0851
|
|
2025-09-16 02:52:28,739: t15.2023.12.08 val PER: 0.0579
|
|
2025-09-16 02:52:28,739: t15.2023.12.10 val PER: 0.0578
|
|
2025-09-16 02:52:28,739: t15.2023.12.17 val PER: 0.1299
|
|
2025-09-16 02:52:28,739: t15.2023.12.29 val PER: 0.1126
|
|
2025-09-16 02:52:28,739: t15.2024.02.25 val PER: 0.0969
|
|
2025-09-16 02:52:28,739: t15.2024.03.08 val PER: 0.2176
|
|
2025-09-16 02:52:28,739: t15.2024.03.15 val PER: 0.1857
|
|
2025-09-16 02:52:28,739: t15.2024.03.17 val PER: 0.1039
|
|
2025-09-16 02:52:28,739: t15.2024.05.10 val PER: 0.1813
|
|
2025-09-16 02:52:28,739: t15.2024.06.14 val PER: 0.1483
|
|
2025-09-16 02:52:28,739: t15.2024.07.19 val PER: 0.2017
|
|
2025-09-16 02:52:28,739: t15.2024.07.21 val PER: 0.0800
|
|
2025-09-16 02:52:28,739: t15.2024.07.28 val PER: 0.1250
|
|
2025-09-16 02:52:28,739: t15.2025.01.10 val PER: 0.2961
|
|
2025-09-16 02:52:28,739: t15.2025.01.12 val PER: 0.1263
|
|
2025-09-16 02:52:28,740: t15.2025.03.14 val PER: 0.3254
|
|
2025-09-16 02:52:28,740: t15.2025.03.16 val PER: 0.1728
|
|
2025-09-16 02:52:28,740: t15.2025.03.30 val PER: 0.2667
|
|
2025-09-16 02:52:28,740: t15.2025.04.13 val PER: 0.2439
|
|
2025-09-16 02:52:28,740: New best test PER 0.1423 --> 0.1379
|
|
2025-09-16 02:52:28,740: Checkpointing model
|
|
2025-09-16 02:52:30,063: Saved model to checkpoint: trained_models/baseline_rnn/checkpoint/best_checkpoint
|
|
2025-09-16 02:52:58,224: Train batch 10200: loss: 2.73 grad norm: 26.35 time: 0.135
|
|
2025-09-16 02:53:30,938: Train batch 10400: loss: 2.50 grad norm: 28.31 time: 0.163
|
|
2025-09-16 02:54:04,242: Train batch 10600: loss: 0.57 grad norm: 11.84 time: 0.124
|
|
2025-09-16 02:54:37,463: Train batch 10800: loss: 2.50 grad norm: 31.20 time: 0.113
|
|
2025-09-16 02:55:10,196: Train batch 11000: loss: 1.65 grad norm: 21.57 time: 0.164
|
|
2025-09-16 02:55:43,500: Train batch 11200: loss: 0.72 grad norm: 15.23 time: 0.135
|
|
2025-09-16 02:56:17,133: Train batch 11400: loss: 0.52 grad norm: 10.35 time: 0.099
|
|
2025-09-16 02:56:49,879: Train batch 11600: loss: 1.88 grad norm: 25.86 time: 0.116
|
|
2025-09-16 02:57:23,170: Train batch 11800: loss: 0.81 grad norm: 16.85 time: 0.126
|
|
2025-09-16 02:57:56,792: Train batch 12000: loss: 0.93 grad norm: 16.24 time: 0.134
|
|
2025-09-16 02:57:56,792: Running test after training batch: 12000
|
|
2025-09-16 02:58:07,685: Val batch 12000: PER (avg): 0.1332 CTC Loss (avg): 22.9958 time: 10.893
|
|
2025-09-16 02:58:07,686: t15.2023.08.13 val PER: 0.1123
|
|
2025-09-16 02:58:07,686: t15.2023.08.18 val PER: 0.0956
|
|
2025-09-16 02:58:07,686: t15.2023.08.20 val PER: 0.0707
|
|
2025-09-16 02:58:07,686: t15.2023.08.25 val PER: 0.0934
|
|
2025-09-16 02:58:07,686: t15.2023.08.27 val PER: 0.1913
|
|
2025-09-16 02:58:07,686: t15.2023.09.01 val PER: 0.0625
|
|
2025-09-16 02:58:07,686: t15.2023.09.03 val PER: 0.1544
|
|
2025-09-16 02:58:07,686: t15.2023.09.24 val PER: 0.1177
|
|
2025-09-16 02:58:07,686: t15.2023.09.29 val PER: 0.1168
|
|
2025-09-16 02:58:07,686: t15.2023.10.01 val PER: 0.1757
|
|
2025-09-16 02:58:07,686: t15.2023.10.06 val PER: 0.0840
|
|
2025-09-16 02:58:07,686: t15.2023.10.08 val PER: 0.2300
|
|
2025-09-16 02:58:07,686: t15.2023.10.13 val PER: 0.1877
|
|
2025-09-16 02:58:07,686: t15.2023.10.15 val PER: 0.1246
|
|
2025-09-16 02:58:07,686: t15.2023.10.20 val PER: 0.1946
|
|
2025-09-16 02:58:07,686: t15.2023.10.22 val PER: 0.1325
|
|
2025-09-16 02:58:07,687: t15.2023.11.03 val PER: 0.1845
|
|
2025-09-16 02:58:07,687: t15.2023.11.04 val PER: 0.0102
|
|
2025-09-16 02:58:07,687: t15.2023.11.17 val PER: 0.0404
|
|
2025-09-16 02:58:07,687: t15.2023.11.19 val PER: 0.0160
|
|
2025-09-16 02:58:07,687: t15.2023.11.26 val PER: 0.0783
|
|
2025-09-16 02:58:07,687: t15.2023.12.03 val PER: 0.0725
|
|
2025-09-16 02:58:07,687: t15.2023.12.08 val PER: 0.0726
|
|
2025-09-16 02:58:07,687: t15.2023.12.10 val PER: 0.0591
|
|
2025-09-16 02:58:07,687: t15.2023.12.17 val PER: 0.1237
|
|
2025-09-16 02:58:07,687: t15.2023.12.29 val PER: 0.1105
|
|
2025-09-16 02:58:07,687: t15.2024.02.25 val PER: 0.1039
|
|
2025-09-16 02:58:07,687: t15.2024.03.08 val PER: 0.2134
|
|
2025-09-16 02:58:07,687: t15.2024.03.15 val PER: 0.1801
|
|
2025-09-16 02:58:07,687: t15.2024.03.17 val PER: 0.1088
|
|
2025-09-16 02:58:07,687: t15.2024.05.10 val PER: 0.1501
|
|
2025-09-16 02:58:07,687: t15.2024.06.14 val PER: 0.1498
|
|
2025-09-16 02:58:07,687: t15.2024.07.19 val PER: 0.1978
|
|
2025-09-16 02:58:07,687: t15.2024.07.21 val PER: 0.0738
|
|
2025-09-16 02:58:07,687: t15.2024.07.28 val PER: 0.1044
|
|
2025-09-16 02:58:07,688: t15.2025.01.10 val PER: 0.2769
|
|
2025-09-16 02:58:07,688: t15.2025.01.12 val PER: 0.1178
|
|
2025-09-16 02:58:07,688: t15.2025.03.14 val PER: 0.3254
|
|
2025-09-16 02:58:07,688: t15.2025.03.16 val PER: 0.1636
|
|
2025-09-16 02:58:07,688: t15.2025.03.30 val PER: 0.2552
|
|
2025-09-16 02:58:07,688: t15.2025.04.13 val PER: 0.2282
|
|
2025-09-16 02:58:07,688: New best test PER 0.1379 --> 0.1332
|
|
2025-09-16 02:58:07,688: Checkpointing model
|
|
2025-09-16 02:58:08,982: Saved model to checkpoint: trained_models/baseline_rnn/checkpoint/best_checkpoint
|
|
2025-09-16 02:58:36,691: Train batch 12200: loss: 1.00 grad norm: 16.95 time: 0.130
|
|
2025-09-16 02:59:08,481: Train batch 12400: loss: 0.91 grad norm: 16.22 time: 0.130
|
|
2025-09-16 02:59:41,333: Train batch 12600: loss: 1.55 grad norm: 27.20 time: 0.109
|
|
2025-09-16 03:00:14,311: Train batch 12800: loss: 0.52 grad norm: 11.60 time: 0.125
|
|
2025-09-16 03:00:47,421: Train batch 13000: loss: 0.79 grad norm: 16.80 time: 0.150
|
|
2025-09-16 03:01:20,627: Train batch 13200: loss: 1.03 grad norm: 20.33 time: 0.168
|
|
2025-09-16 03:01:53,720: Train batch 13400: loss: 1.37 grad norm: 14.15 time: 0.123
|
|
2025-09-16 03:02:26,438: Train batch 13600: loss: 0.45 grad norm: 10.06 time: 0.128
|
|
2025-09-16 03:02:59,072: Train batch 13800: loss: 0.72 grad norm: 15.72 time: 0.162
|
|
2025-09-16 03:03:31,322: Train batch 14000: loss: 0.61 grad norm: 23.08 time: 0.154
|
|
2025-09-16 03:03:31,322: Running test after training batch: 14000
|
|
2025-09-16 03:03:41,802: Val batch 14000: PER (avg): 0.1331 CTC Loss (avg): 23.8213 time: 10.480
|
|
2025-09-16 03:03:41,802: t15.2023.08.13 val PER: 0.1040
|
|
2025-09-16 03:03:41,802: t15.2023.08.18 val PER: 0.1023
|
|
2025-09-16 03:03:41,803: t15.2023.08.20 val PER: 0.0794
|
|
2025-09-16 03:03:41,803: t15.2023.08.25 val PER: 0.1084
|
|
2025-09-16 03:03:41,803: t15.2023.08.27 val PER: 0.1945
|
|
2025-09-16 03:03:41,803: t15.2023.09.01 val PER: 0.0698
|
|
2025-09-16 03:03:41,803: t15.2023.09.03 val PER: 0.1568
|
|
2025-09-16 03:03:41,803: t15.2023.09.24 val PER: 0.1262
|
|
2025-09-16 03:03:41,803: t15.2023.09.29 val PER: 0.1174
|
|
2025-09-16 03:03:41,803: t15.2023.10.01 val PER: 0.1678
|
|
2025-09-16 03:03:41,803: t15.2023.10.06 val PER: 0.0753
|
|
2025-09-16 03:03:41,803: t15.2023.10.08 val PER: 0.2219
|
|
2025-09-16 03:03:41,803: t15.2023.10.13 val PER: 0.1963
|
|
2025-09-16 03:03:41,803: t15.2023.10.15 val PER: 0.1444
|
|
2025-09-16 03:03:41,803: t15.2023.10.20 val PER: 0.1846
|
|
2025-09-16 03:03:41,803: t15.2023.10.22 val PER: 0.1169
|
|
2025-09-16 03:03:41,803: t15.2023.11.03 val PER: 0.1757
|
|
2025-09-16 03:03:41,803: t15.2023.11.04 val PER: 0.0137
|
|
2025-09-16 03:03:41,803: t15.2023.11.17 val PER: 0.0327
|
|
2025-09-16 03:03:41,803: t15.2023.11.19 val PER: 0.0180
|
|
2025-09-16 03:03:41,803: t15.2023.11.26 val PER: 0.0804
|
|
2025-09-16 03:03:41,804: t15.2023.12.03 val PER: 0.0683
|
|
2025-09-16 03:03:41,804: t15.2023.12.08 val PER: 0.0613
|
|
2025-09-16 03:03:41,804: t15.2023.12.10 val PER: 0.0460
|
|
2025-09-16 03:03:41,804: t15.2023.12.17 val PER: 0.1123
|
|
2025-09-16 03:03:41,804: t15.2023.12.29 val PER: 0.1181
|
|
2025-09-16 03:03:41,804: t15.2024.02.25 val PER: 0.0857
|
|
2025-09-16 03:03:41,804: t15.2024.03.08 val PER: 0.2048
|
|
2025-09-16 03:03:41,804: t15.2024.03.15 val PER: 0.1832
|
|
2025-09-16 03:03:41,804: t15.2024.03.17 val PER: 0.1123
|
|
2025-09-16 03:03:41,804: t15.2024.05.10 val PER: 0.1352
|
|
2025-09-16 03:03:41,804: t15.2024.06.14 val PER: 0.1467
|
|
2025-09-16 03:03:41,804: t15.2024.07.19 val PER: 0.1991
|
|
2025-09-16 03:03:41,804: t15.2024.07.21 val PER: 0.0821
|
|
2025-09-16 03:03:41,804: t15.2024.07.28 val PER: 0.1051
|
|
2025-09-16 03:03:41,804: t15.2025.01.10 val PER: 0.2700
|
|
2025-09-16 03:03:41,804: t15.2025.01.12 val PER: 0.1309
|
|
2025-09-16 03:03:41,804: t15.2025.03.14 val PER: 0.2988
|
|
2025-09-16 03:03:41,804: t15.2025.03.16 val PER: 0.1597
|
|
2025-09-16 03:03:41,804: t15.2025.03.30 val PER: 0.2552
|
|
2025-09-16 03:03:41,804: t15.2025.04.13 val PER: 0.2496
|
|
2025-09-16 03:03:41,805: New best test PER 0.1332 --> 0.1331
|
|
2025-09-16 03:03:41,805: Checkpointing model
|
|
2025-09-16 03:03:43,155: Saved model to checkpoint: trained_models/baseline_rnn/checkpoint/best_checkpoint
|
|
2025-09-16 03:04:11,469: Train batch 14200: loss: 0.88 grad norm: 15.37 time: 0.111
|
|
2025-09-16 03:04:43,961: Train batch 14400: loss: 0.51 grad norm: 9.42 time: 0.145
|
|
2025-09-16 03:05:16,381: Train batch 14600: loss: 0.98 grad norm: 19.27 time: 0.120
|
|
2025-09-16 03:05:49,165: Train batch 14800: loss: 0.32 grad norm: 8.67 time: 0.103
|
|
2025-09-16 03:06:22,409: Train batch 15000: loss: 0.49 grad norm: 16.72 time: 0.141
|
|
2025-09-16 03:06:54,388: Train batch 15200: loss: 0.24 grad norm: 8.02 time: 0.154
|
|
2025-09-16 03:07:27,046: Train batch 15400: loss: 1.17 grad norm: 21.40 time: 0.118
|
|
2025-09-16 03:08:00,181: Train batch 15600: loss: 0.87 grad norm: 15.39 time: 0.131
|
|
2025-09-16 03:08:33,681: Train batch 15800: loss: 0.34 grad norm: 11.75 time: 0.113
|
|
2025-09-16 03:09:06,774: Train batch 16000: loss: 0.36 grad norm: 17.86 time: 0.151
|
|
2025-09-16 03:09:06,775: Running test after training batch: 16000
|
|
2025-09-16 03:09:17,263: Val batch 16000: PER (avg): 0.1261 CTC Loss (avg): 23.8077 time: 10.488
|
|
2025-09-16 03:09:17,263: t15.2023.08.13 val PER: 0.0977
|
|
2025-09-16 03:09:17,264: t15.2023.08.18 val PER: 0.0872
|
|
2025-09-16 03:09:17,264: t15.2023.08.20 val PER: 0.0627
|
|
2025-09-16 03:09:17,264: t15.2023.08.25 val PER: 0.0889
|
|
2025-09-16 03:09:17,264: t15.2023.08.27 val PER: 0.1833
|
|
2025-09-16 03:09:17,264: t15.2023.09.01 val PER: 0.0641
|
|
2025-09-16 03:09:17,264: t15.2023.09.03 val PER: 0.1354
|
|
2025-09-16 03:09:17,264: t15.2023.09.24 val PER: 0.1189
|
|
2025-09-16 03:09:17,264: t15.2023.09.29 val PER: 0.1130
|
|
2025-09-16 03:09:17,264: t15.2023.10.01 val PER: 0.1592
|
|
2025-09-16 03:09:17,264: t15.2023.10.06 val PER: 0.0753
|
|
2025-09-16 03:09:17,264: t15.2023.10.08 val PER: 0.2341
|
|
2025-09-16 03:09:17,264: t15.2023.10.13 val PER: 0.1746
|
|
2025-09-16 03:09:17,264: t15.2023.10.15 val PER: 0.1292
|
|
2025-09-16 03:09:17,264: t15.2023.10.20 val PER: 0.1779
|
|
2025-09-16 03:09:17,264: t15.2023.10.22 val PER: 0.1214
|
|
2025-09-16 03:09:17,264: t15.2023.11.03 val PER: 0.1771
|
|
2025-09-16 03:09:17,264: t15.2023.11.04 val PER: 0.0068
|
|
2025-09-16 03:09:17,264: t15.2023.11.17 val PER: 0.0280
|
|
2025-09-16 03:09:17,264: t15.2023.11.19 val PER: 0.0140
|
|
2025-09-16 03:09:17,265: t15.2023.11.26 val PER: 0.0775
|
|
2025-09-16 03:09:17,265: t15.2023.12.03 val PER: 0.0693
|
|
2025-09-16 03:09:17,265: t15.2023.12.08 val PER: 0.0526
|
|
2025-09-16 03:09:17,265: t15.2023.12.10 val PER: 0.0539
|
|
2025-09-16 03:09:17,265: t15.2023.12.17 val PER: 0.0884
|
|
2025-09-16 03:09:17,265: t15.2023.12.29 val PER: 0.1030
|
|
2025-09-16 03:09:17,265: t15.2024.02.25 val PER: 0.1110
|
|
2025-09-16 03:09:17,265: t15.2024.03.08 val PER: 0.2034
|
|
2025-09-16 03:09:17,265: t15.2024.03.15 val PER: 0.1707
|
|
2025-09-16 03:09:17,265: t15.2024.03.17 val PER: 0.0900
|
|
2025-09-16 03:09:17,265: t15.2024.05.10 val PER: 0.1471
|
|
2025-09-16 03:09:17,265: t15.2024.06.14 val PER: 0.1325
|
|
2025-09-16 03:09:17,265: t15.2024.07.19 val PER: 0.1991
|
|
2025-09-16 03:09:17,265: t15.2024.07.21 val PER: 0.0759
|
|
2025-09-16 03:09:17,265: t15.2024.07.28 val PER: 0.1140
|
|
2025-09-16 03:09:17,265: t15.2025.01.10 val PER: 0.2686
|
|
2025-09-16 03:09:17,265: t15.2025.01.12 val PER: 0.1155
|
|
2025-09-16 03:09:17,265: t15.2025.03.14 val PER: 0.3077
|
|
2025-09-16 03:09:17,265: t15.2025.03.16 val PER: 0.1492
|
|
2025-09-16 03:09:17,266: t15.2025.03.30 val PER: 0.2644
|
|
2025-09-16 03:09:17,266: t15.2025.04.13 val PER: 0.2126
|
|
2025-09-16 03:09:17,266: New best test PER 0.1331 --> 0.1261
|
|
2025-09-16 03:09:17,266: Checkpointing model
|
|
2025-09-16 03:09:18,551: Saved model to checkpoint: trained_models/baseline_rnn/checkpoint/best_checkpoint
|
|
2025-09-16 03:09:46,943: Train batch 16200: loss: 0.40 grad norm: 9.83 time: 0.095
|
|
2025-09-16 03:10:20,191: Train batch 16400: loss: 0.34 grad norm: 10.72 time: 0.130
|
|
2025-09-16 03:10:52,461: Train batch 16600: loss: 0.50 grad norm: 13.84 time: 0.141
|
|
2025-09-16 03:11:25,324: Train batch 16800: loss: 0.20 grad norm: 7.68 time: 0.139
|
|
2025-09-16 03:11:57,679: Train batch 17000: loss: 0.06 grad norm: 2.99 time: 0.119
|
|
2025-09-16 03:12:30,250: Train batch 17200: loss: 0.49 grad norm: 15.90 time: 0.163
|
|
2025-09-16 03:13:02,884: Train batch 17400: loss: 0.54 grad norm: 32.56 time: 0.115
|
|
2025-09-16 03:13:35,625: Train batch 17600: loss: 0.44 grad norm: 10.50 time: 0.127
|
|
2025-09-16 03:14:08,411: Train batch 17800: loss: 0.29 grad norm: 13.59 time: 0.107
|
|
2025-09-16 03:14:41,504: Train batch 18000: loss: 0.31 grad norm: 10.83 time: 0.113
|
|
2025-09-16 03:14:41,505: Running test after training batch: 18000
|
|
2025-09-16 03:14:51,864: Val batch 18000: PER (avg): 0.1268 CTC Loss (avg): 24.8535 time: 10.359
|
|
2025-09-16 03:14:51,864: t15.2023.08.13 val PER: 0.0988
|
|
2025-09-16 03:14:51,864: t15.2023.08.18 val PER: 0.0964
|
|
2025-09-16 03:14:51,864: t15.2023.08.20 val PER: 0.0627
|
|
2025-09-16 03:14:51,864: t15.2023.08.25 val PER: 0.0964
|
|
2025-09-16 03:14:51,864: t15.2023.08.27 val PER: 0.1961
|
|
2025-09-16 03:14:51,864: t15.2023.09.01 val PER: 0.0601
|
|
2025-09-16 03:14:51,865: t15.2023.09.03 val PER: 0.1342
|
|
2025-09-16 03:14:51,865: t15.2023.09.24 val PER: 0.1056
|
|
2025-09-16 03:14:51,865: t15.2023.09.29 val PER: 0.1200
|
|
2025-09-16 03:14:51,865: t15.2023.10.01 val PER: 0.1678
|
|
2025-09-16 03:14:51,865: t15.2023.10.06 val PER: 0.0678
|
|
2025-09-16 03:14:51,865: t15.2023.10.08 val PER: 0.2124
|
|
2025-09-16 03:14:51,865: t15.2023.10.13 val PER: 0.1823
|
|
2025-09-16 03:14:51,865: t15.2023.10.15 val PER: 0.1312
|
|
2025-09-16 03:14:51,865: t15.2023.10.20 val PER: 0.2181
|
|
2025-09-16 03:14:51,865: t15.2023.10.22 val PER: 0.1125
|
|
2025-09-16 03:14:51,865: t15.2023.11.03 val PER: 0.1710
|
|
2025-09-16 03:14:51,865: t15.2023.11.04 val PER: 0.0068
|
|
2025-09-16 03:14:51,865: t15.2023.11.17 val PER: 0.0264
|
|
2025-09-16 03:14:51,865: t15.2023.11.19 val PER: 0.0120
|
|
2025-09-16 03:14:51,865: t15.2023.11.26 val PER: 0.0623
|
|
2025-09-16 03:14:51,865: t15.2023.12.03 val PER: 0.0767
|
|
2025-09-16 03:14:51,865: t15.2023.12.08 val PER: 0.0539
|
|
2025-09-16 03:14:51,865: t15.2023.12.10 val PER: 0.0512
|
|
2025-09-16 03:14:51,866: t15.2023.12.17 val PER: 0.1247
|
|
2025-09-16 03:14:51,866: t15.2023.12.29 val PER: 0.1187
|
|
2025-09-16 03:14:51,866: t15.2024.02.25 val PER: 0.0871
|
|
2025-09-16 03:14:51,866: t15.2024.03.08 val PER: 0.1835
|
|
2025-09-16 03:14:51,866: t15.2024.03.15 val PER: 0.1676
|
|
2025-09-16 03:14:51,866: t15.2024.03.17 val PER: 0.0948
|
|
2025-09-16 03:14:51,866: t15.2024.05.10 val PER: 0.1486
|
|
2025-09-16 03:14:51,866: t15.2024.06.14 val PER: 0.1530
|
|
2025-09-16 03:14:51,866: t15.2024.07.19 val PER: 0.1964
|
|
2025-09-16 03:14:51,866: t15.2024.07.21 val PER: 0.0703
|
|
2025-09-16 03:14:51,866: t15.2024.07.28 val PER: 0.1037
|
|
2025-09-16 03:14:51,866: t15.2025.01.10 val PER: 0.2851
|
|
2025-09-16 03:14:51,866: t15.2025.01.12 val PER: 0.1101
|
|
2025-09-16 03:14:51,866: t15.2025.03.14 val PER: 0.2855
|
|
2025-09-16 03:14:51,866: t15.2025.03.16 val PER: 0.1728
|
|
2025-09-16 03:14:51,866: t15.2025.03.30 val PER: 0.2644
|
|
2025-09-16 03:14:51,866: t15.2025.04.13 val PER: 0.2168
|
|
2025-09-16 03:15:20,058: Train batch 18200: loss: 0.39 grad norm: 13.96 time: 0.119
|
|
2025-09-16 03:15:51,791: Train batch 18400: loss: 0.49 grad norm: 14.26 time: 0.126
|
|
2025-09-16 03:16:24,593: Train batch 18600: loss: 0.42 grad norm: 14.32 time: 0.164
|
|
2025-09-16 03:16:57,415: Train batch 18800: loss: 0.33 grad norm: 12.03 time: 0.148
|
|
2025-09-16 03:17:31,202: Train batch 19000: loss: 0.77 grad norm: 22.08 time: 0.150
|
|
2025-09-16 03:18:04,265: Train batch 19200: loss: 0.45 grad norm: 10.92 time: 0.146
|
|
2025-09-16 03:18:38,091: Train batch 19400: loss: 0.61 grad norm: 14.74 time: 0.172
|
|
2025-09-16 03:19:11,774: Train batch 19600: loss: 0.35 grad norm: 9.60 time: 0.116
|
|
2025-09-16 03:19:45,181: Train batch 19800: loss: 0.22 grad norm: 7.19 time: 0.177
|
|
2025-09-16 03:20:18,395: Train batch 20000: loss: 0.49 grad norm: 9.73 time: 0.168
|
|
2025-09-16 03:20:18,396: Running test after training batch: 20000
|
|
2025-09-16 03:20:28,925: Val batch 20000: PER (avg): 0.1253 CTC Loss (avg): 26.3998 time: 10.529
|
|
2025-09-16 03:20:28,925: t15.2023.08.13 val PER: 0.1050
|
|
2025-09-16 03:20:28,925: t15.2023.08.18 val PER: 0.0964
|
|
2025-09-16 03:20:28,925: t15.2023.08.20 val PER: 0.0691
|
|
2025-09-16 03:20:28,925: t15.2023.08.25 val PER: 0.1009
|
|
2025-09-16 03:20:28,925: t15.2023.08.27 val PER: 0.1913
|
|
2025-09-16 03:20:28,925: t15.2023.09.01 val PER: 0.0560
|
|
2025-09-16 03:20:28,925: t15.2023.09.03 val PER: 0.1413
|
|
2025-09-16 03:20:28,926: t15.2023.09.24 val PER: 0.1153
|
|
2025-09-16 03:20:28,926: t15.2023.09.29 val PER: 0.1238
|
|
2025-09-16 03:20:28,926: t15.2023.10.01 val PER: 0.1539
|
|
2025-09-16 03:20:28,926: t15.2023.10.06 val PER: 0.0678
|
|
2025-09-16 03:20:28,926: t15.2023.10.08 val PER: 0.2287
|
|
2025-09-16 03:20:28,926: t15.2023.10.13 val PER: 0.1746
|
|
2025-09-16 03:20:28,926: t15.2023.10.15 val PER: 0.1213
|
|
2025-09-16 03:20:28,926: t15.2023.10.20 val PER: 0.1644
|
|
2025-09-16 03:20:28,926: t15.2023.10.22 val PER: 0.1058
|
|
2025-09-16 03:20:28,926: t15.2023.11.03 val PER: 0.1750
|
|
2025-09-16 03:20:28,926: t15.2023.11.04 val PER: 0.0137
|
|
2025-09-16 03:20:28,926: t15.2023.11.17 val PER: 0.0264
|
|
2025-09-16 03:20:28,926: t15.2023.11.19 val PER: 0.0140
|
|
2025-09-16 03:20:28,926: t15.2023.11.26 val PER: 0.0681
|
|
2025-09-16 03:20:28,926: t15.2023.12.03 val PER: 0.0725
|
|
2025-09-16 03:20:28,926: t15.2023.12.08 val PER: 0.0546
|
|
2025-09-16 03:20:28,926: t15.2023.12.10 val PER: 0.0696
|
|
2025-09-16 03:20:28,926: t15.2023.12.17 val PER: 0.0956
|
|
2025-09-16 03:20:28,926: t15.2023.12.29 val PER: 0.1030
|
|
2025-09-16 03:20:28,927: t15.2024.02.25 val PER: 0.1039
|
|
2025-09-16 03:20:28,927: t15.2024.03.08 val PER: 0.1821
|
|
2025-09-16 03:20:28,927: t15.2024.03.15 val PER: 0.1770
|
|
2025-09-16 03:20:28,927: t15.2024.03.17 val PER: 0.0886
|
|
2025-09-16 03:20:28,927: t15.2024.05.10 val PER: 0.1367
|
|
2025-09-16 03:20:28,927: t15.2024.06.14 val PER: 0.1467
|
|
2025-09-16 03:20:28,927: t15.2024.07.19 val PER: 0.1839
|
|
2025-09-16 03:20:28,927: t15.2024.07.21 val PER: 0.0703
|
|
2025-09-16 03:20:28,927: t15.2024.07.28 val PER: 0.1066
|
|
2025-09-16 03:20:28,927: t15.2025.01.10 val PER: 0.2466
|
|
2025-09-16 03:20:28,927: t15.2025.01.12 val PER: 0.1078
|
|
2025-09-16 03:20:28,927: t15.2025.03.14 val PER: 0.3225
|
|
2025-09-16 03:20:28,927: t15.2025.03.16 val PER: 0.1649
|
|
2025-09-16 03:20:28,927: t15.2025.03.30 val PER: 0.2598
|
|
2025-09-16 03:20:28,927: t15.2025.04.13 val PER: 0.2354
|
|
2025-09-16 03:20:28,927: New best test PER 0.1261 --> 0.1253
|
|
2025-09-16 03:20:28,927: Checkpointing model
|
|
2025-09-16 03:20:30,317: Saved model to checkpoint: trained_models/baseline_rnn/checkpoint/best_checkpoint
|
|
2025-09-16 03:20:58,352: Train batch 20200: loss: 0.37 grad norm: 12.25 time: 0.124
|
|
2025-09-16 03:21:30,244: Train batch 20400: loss: 0.21 grad norm: 7.34 time: 0.114
|
|
2025-09-16 03:22:01,205: Train batch 20600: loss: 0.13 grad norm: 6.85 time: 0.124
|
|
2025-09-16 03:22:32,788: Train batch 20800: loss: 0.43 grad norm: 12.57 time: 0.109
|
|
2025-09-16 03:23:03,902: Train batch 21000: loss: 0.12 grad norm: 4.53 time: 0.116
|
|
2025-09-16 03:23:35,860: Train batch 21200: loss: 0.23 grad norm: 10.60 time: 0.114
|
|
2025-09-16 03:24:07,811: Train batch 21400: loss: 0.44 grad norm: 14.16 time: 0.126
|
|
2025-09-16 03:24:40,169: Train batch 21600: loss: 0.29 grad norm: 8.04 time: 0.109
|
|
2025-09-16 03:25:13,648: Train batch 21800: loss: 0.19 grad norm: 10.74 time: 0.173
|
|
2025-09-16 03:25:47,333: Train batch 22000: loss: 0.67 grad norm: 16.24 time: 0.099
|
|
2025-09-16 03:25:47,333: Running test after training batch: 22000
|
|
2025-09-16 03:25:58,052: Val batch 22000: PER (avg): 0.1253 CTC Loss (avg): 26.0540 time: 10.718
|
|
2025-09-16 03:25:58,052: t15.2023.08.13 val PER: 0.0977
|
|
2025-09-16 03:25:58,052: t15.2023.08.18 val PER: 0.0939
|
|
2025-09-16 03:25:58,052: t15.2023.08.20 val PER: 0.0763
|
|
2025-09-16 03:25:58,052: t15.2023.08.25 val PER: 0.0889
|
|
2025-09-16 03:25:58,052: t15.2023.08.27 val PER: 0.1849
|
|
2025-09-16 03:25:58,052: t15.2023.09.01 val PER: 0.0528
|
|
2025-09-16 03:25:58,052: t15.2023.09.03 val PER: 0.1473
|
|
2025-09-16 03:25:58,052: t15.2023.09.24 val PER: 0.0995
|
|
2025-09-16 03:25:58,053: t15.2023.09.29 val PER: 0.1193
|
|
2025-09-16 03:25:58,053: t15.2023.10.01 val PER: 0.1565
|
|
2025-09-16 03:25:58,053: t15.2023.10.06 val PER: 0.0721
|
|
2025-09-16 03:25:58,053: t15.2023.10.08 val PER: 0.2003
|
|
2025-09-16 03:25:58,053: t15.2023.10.13 val PER: 0.1777
|
|
2025-09-16 03:25:58,053: t15.2023.10.15 val PER: 0.1279
|
|
2025-09-16 03:25:58,053: t15.2023.10.20 val PER: 0.1879
|
|
2025-09-16 03:25:58,053: t15.2023.10.22 val PER: 0.1269
|
|
2025-09-16 03:25:58,054: t15.2023.11.03 val PER: 0.1811
|
|
2025-09-16 03:25:58,054: t15.2023.11.04 val PER: 0.0068
|
|
2025-09-16 03:25:58,054: t15.2023.11.17 val PER: 0.0249
|
|
2025-09-16 03:25:58,054: t15.2023.11.19 val PER: 0.0180
|
|
2025-09-16 03:25:58,054: t15.2023.11.26 val PER: 0.0659
|
|
2025-09-16 03:25:58,054: t15.2023.12.03 val PER: 0.0788
|
|
2025-09-16 03:25:58,054: t15.2023.12.08 val PER: 0.0566
|
|
2025-09-16 03:25:58,054: t15.2023.12.10 val PER: 0.0539
|
|
2025-09-16 03:25:58,054: t15.2023.12.17 val PER: 0.1112
|
|
2025-09-16 03:25:58,054: t15.2023.12.29 val PER: 0.0961
|
|
2025-09-16 03:25:58,054: t15.2024.02.25 val PER: 0.0969
|
|
2025-09-16 03:25:58,054: t15.2024.03.08 val PER: 0.2077
|
|
2025-09-16 03:25:58,054: t15.2024.03.15 val PER: 0.1689
|
|
2025-09-16 03:25:58,054: t15.2024.03.17 val PER: 0.0948
|
|
2025-09-16 03:25:58,054: t15.2024.05.10 val PER: 0.1501
|
|
2025-09-16 03:25:58,054: t15.2024.06.14 val PER: 0.1420
|
|
2025-09-16 03:25:58,054: t15.2024.07.19 val PER: 0.1833
|
|
2025-09-16 03:25:58,054: t15.2024.07.21 val PER: 0.0697
|
|
2025-09-16 03:25:58,054: t15.2024.07.28 val PER: 0.1059
|
|
2025-09-16 03:25:58,055: t15.2025.01.10 val PER: 0.2658
|
|
2025-09-16 03:25:58,055: t15.2025.01.12 val PER: 0.1024
|
|
2025-09-16 03:25:58,055: t15.2025.03.14 val PER: 0.2855
|
|
2025-09-16 03:25:58,055: t15.2025.03.16 val PER: 0.1702
|
|
2025-09-16 03:25:58,055: t15.2025.03.30 val PER: 0.2793
|
|
2025-09-16 03:25:58,055: t15.2025.04.13 val PER: 0.2083
|
|
2025-09-16 03:25:58,055: New best test PER 0.1253 --> 0.1253
|
|
2025-09-16 03:25:58,055: Checkpointing model
|
|
2025-09-16 03:25:59,341: Saved model to checkpoint: trained_models/baseline_rnn/checkpoint/best_checkpoint
|
|
2025-09-16 03:26:26,463: Train batch 22200: loss: 0.38 grad norm: 11.27 time: 0.113
|
|
2025-09-16 03:26:59,500: Train batch 22400: loss: 0.43 grad norm: 12.51 time: 0.113
|
|
2025-09-16 03:27:32,688: Train batch 22600: loss: 0.31 grad norm: 10.77 time: 0.174
|
|
2025-09-16 03:28:05,430: Train batch 22800: loss: 0.13 grad norm: 4.99 time: 0.176
|
|
2025-09-16 03:28:38,895: Train batch 23000: loss: 0.32 grad norm: 7.57 time: 0.147
|
|
2025-09-16 03:29:11,761: Train batch 23200: loss: 0.37 grad norm: 10.75 time: 0.099
|
|
2025-09-16 03:29:44,873: Train batch 23400: loss: 0.19 grad norm: 12.17 time: 0.115
|
|
2025-09-16 03:30:18,356: Train batch 23600: loss: 0.71 grad norm: 25.05 time: 0.135
|
|
2025-09-16 03:30:51,013: Train batch 23800: loss: 0.27 grad norm: 6.38 time: 0.107
|
|
2025-09-16 03:31:24,068: Train batch 24000: loss: 0.13 grad norm: 8.31 time: 0.115
|
|
2025-09-16 03:31:24,068: Running test after training batch: 24000
|
|
2025-09-16 03:31:34,563: Val batch 24000: PER (avg): 0.1241 CTC Loss (avg): 26.4658 time: 10.495
|
|
2025-09-16 03:31:34,564: t15.2023.08.13 val PER: 0.0915
|
|
2025-09-16 03:31:34,564: t15.2023.08.18 val PER: 0.0788
|
|
2025-09-16 03:31:34,564: t15.2023.08.20 val PER: 0.0707
|
|
2025-09-16 03:31:34,564: t15.2023.08.25 val PER: 0.0783
|
|
2025-09-16 03:31:34,564: t15.2023.08.27 val PER: 0.1929
|
|
2025-09-16 03:31:34,564: t15.2023.09.01 val PER: 0.0568
|
|
2025-09-16 03:31:34,564: t15.2023.09.03 val PER: 0.1366
|
|
2025-09-16 03:31:34,564: t15.2023.09.24 val PER: 0.1056
|
|
2025-09-16 03:31:34,564: t15.2023.09.29 val PER: 0.1155
|
|
2025-09-16 03:31:34,564: t15.2023.10.01 val PER: 0.1513
|
|
2025-09-16 03:31:34,564: t15.2023.10.06 val PER: 0.0721
|
|
2025-09-16 03:31:34,564: t15.2023.10.08 val PER: 0.1976
|
|
2025-09-16 03:31:34,564: t15.2023.10.13 val PER: 0.1823
|
|
2025-09-16 03:31:34,564: t15.2023.10.15 val PER: 0.1272
|
|
2025-09-16 03:31:34,565: t15.2023.10.20 val PER: 0.1678
|
|
2025-09-16 03:31:34,565: t15.2023.10.22 val PER: 0.1114
|
|
2025-09-16 03:31:34,565: t15.2023.11.03 val PER: 0.1791
|
|
2025-09-16 03:31:34,565: t15.2023.11.04 val PER: 0.0068
|
|
2025-09-16 03:31:34,565: t15.2023.11.17 val PER: 0.0389
|
|
2025-09-16 03:31:34,565: t15.2023.11.19 val PER: 0.0140
|
|
2025-09-16 03:31:34,565: t15.2023.11.26 val PER: 0.0623
|
|
2025-09-16 03:31:34,565: t15.2023.12.03 val PER: 0.0756
|
|
2025-09-16 03:31:34,565: t15.2023.12.08 val PER: 0.0519
|
|
2025-09-16 03:31:34,565: t15.2023.12.10 val PER: 0.0644
|
|
2025-09-16 03:31:34,565: t15.2023.12.17 val PER: 0.1050
|
|
2025-09-16 03:31:34,565: t15.2023.12.29 val PER: 0.0975
|
|
2025-09-16 03:31:34,565: t15.2024.02.25 val PER: 0.0885
|
|
2025-09-16 03:31:34,565: t15.2024.03.08 val PER: 0.1863
|
|
2025-09-16 03:31:34,565: t15.2024.03.15 val PER: 0.1782
|
|
2025-09-16 03:31:34,565: t15.2024.03.17 val PER: 0.0837
|
|
2025-09-16 03:31:34,565: t15.2024.05.10 val PER: 0.1649
|
|
2025-09-16 03:31:34,565: t15.2024.06.14 val PER: 0.1278
|
|
2025-09-16 03:31:34,565: t15.2024.07.19 val PER: 0.1898
|
|
2025-09-16 03:31:34,566: t15.2024.07.21 val PER: 0.0669
|
|
2025-09-16 03:31:34,566: t15.2024.07.28 val PER: 0.1235
|
|
2025-09-16 03:31:34,566: t15.2025.01.10 val PER: 0.2755
|
|
2025-09-16 03:31:34,566: t15.2025.01.12 val PER: 0.1070
|
|
2025-09-16 03:31:34,566: t15.2025.03.14 val PER: 0.3240
|
|
2025-09-16 03:31:34,566: t15.2025.03.16 val PER: 0.1662
|
|
2025-09-16 03:31:34,566: t15.2025.03.30 val PER: 0.2356
|
|
2025-09-16 03:31:34,566: t15.2025.04.13 val PER: 0.2311
|
|
2025-09-16 03:31:34,566: New best test PER 0.1253 --> 0.1241
|
|
2025-09-16 03:31:34,566: Checkpointing model
|
|
2025-09-16 03:31:35,989: Saved model to checkpoint: trained_models/baseline_rnn/checkpoint/best_checkpoint
|
|
2025-09-16 03:32:04,416: Train batch 24200: loss: 0.34 grad norm: 10.37 time: 0.121
|
|
2025-09-16 03:32:36,090: Train batch 24400: loss: 0.14 grad norm: 7.29 time: 0.124
|
|
2025-09-16 03:33:08,815: Train batch 24600: loss: 0.34 grad norm: 8.01 time: 0.154
|
|
2025-09-16 03:33:41,305: Train batch 24800: loss: 0.47 grad norm: 10.35 time: 0.134
|
|
2025-09-16 03:34:13,312: Train batch 25000: loss: 0.46 grad norm: 14.44 time: 0.115
|
|
2025-09-16 03:34:46,991: Train batch 25200: loss: 0.15 grad norm: 7.48 time: 0.147
|
|
2025-09-16 03:35:20,126: Train batch 25400: loss: 0.12 grad norm: 4.96 time: 0.122
|
|
2025-09-16 03:35:54,491: Train batch 25600: loss: 0.12 grad norm: 5.89 time: 0.153
|
|
2025-09-16 03:36:28,330: Train batch 25800: loss: 0.29 grad norm: 10.80 time: 0.164
|
|
2025-09-16 03:37:00,621: Train batch 26000: loss: 0.29 grad norm: 9.82 time: 0.102
|
|
2025-09-16 03:37:00,621: Running test after training batch: 26000
|
|
2025-09-16 03:37:11,297: Val batch 26000: PER (avg): 0.1213 CTC Loss (avg): 26.1406 time: 10.676
|
|
2025-09-16 03:37:11,298: t15.2023.08.13 val PER: 0.1008
|
|
2025-09-16 03:37:11,298: t15.2023.08.18 val PER: 0.0813
|
|
2025-09-16 03:37:11,298: t15.2023.08.20 val PER: 0.0643
|
|
2025-09-16 03:37:11,298: t15.2023.08.25 val PER: 0.0964
|
|
2025-09-16 03:37:11,298: t15.2023.08.27 val PER: 0.1752
|
|
2025-09-16 03:37:11,298: t15.2023.09.01 val PER: 0.0528
|
|
2025-09-16 03:37:11,298: t15.2023.09.03 val PER: 0.1330
|
|
2025-09-16 03:37:11,298: t15.2023.09.24 val PER: 0.1044
|
|
2025-09-16 03:37:11,298: t15.2023.09.29 val PER: 0.1123
|
|
2025-09-16 03:37:11,298: t15.2023.10.01 val PER: 0.1513
|
|
2025-09-16 03:37:11,298: t15.2023.10.06 val PER: 0.0538
|
|
2025-09-16 03:37:11,298: t15.2023.10.08 val PER: 0.2084
|
|
2025-09-16 03:37:11,298: t15.2023.10.13 val PER: 0.1730
|
|
2025-09-16 03:37:11,298: t15.2023.10.15 val PER: 0.1213
|
|
2025-09-16 03:37:11,298: t15.2023.10.20 val PER: 0.1745
|
|
2025-09-16 03:37:11,298: t15.2023.10.22 val PER: 0.1036
|
|
2025-09-16 03:37:11,298: t15.2023.11.03 val PER: 0.1676
|
|
2025-09-16 03:37:11,299: t15.2023.11.04 val PER: 0.0102
|
|
2025-09-16 03:37:11,299: t15.2023.11.17 val PER: 0.0280
|
|
2025-09-16 03:37:11,299: t15.2023.11.19 val PER: 0.0240
|
|
2025-09-16 03:37:11,299: t15.2023.11.26 val PER: 0.0572
|
|
2025-09-16 03:37:11,299: t15.2023.12.03 val PER: 0.0651
|
|
2025-09-16 03:37:11,299: t15.2023.12.08 val PER: 0.0493
|
|
2025-09-16 03:37:11,299: t15.2023.12.10 val PER: 0.0552
|
|
2025-09-16 03:37:11,299: t15.2023.12.17 val PER: 0.0956
|
|
2025-09-16 03:37:11,299: t15.2023.12.29 val PER: 0.0933
|
|
2025-09-16 03:37:11,299: t15.2024.02.25 val PER: 0.0983
|
|
2025-09-16 03:37:11,299: t15.2024.03.08 val PER: 0.1863
|
|
2025-09-16 03:37:11,299: t15.2024.03.15 val PER: 0.1757
|
|
2025-09-16 03:37:11,299: t15.2024.03.17 val PER: 0.0823
|
|
2025-09-16 03:37:11,299: t15.2024.05.10 val PER: 0.1382
|
|
2025-09-16 03:37:11,299: t15.2024.06.14 val PER: 0.1388
|
|
2025-09-16 03:37:11,299: t15.2024.07.19 val PER: 0.1931
|
|
2025-09-16 03:37:11,299: t15.2024.07.21 val PER: 0.0648
|
|
2025-09-16 03:37:11,299: t15.2024.07.28 val PER: 0.1118
|
|
2025-09-16 03:37:11,299: t15.2025.01.10 val PER: 0.2755
|
|
2025-09-16 03:37:11,300: t15.2025.01.12 val PER: 0.1008
|
|
2025-09-16 03:37:11,300: t15.2025.03.14 val PER: 0.3166
|
|
2025-09-16 03:37:11,300: t15.2025.03.16 val PER: 0.1872
|
|
2025-09-16 03:37:11,300: t15.2025.03.30 val PER: 0.2402
|
|
2025-09-16 03:37:11,300: t15.2025.04.13 val PER: 0.2354
|
|
2025-09-16 03:37:11,300: New best test PER 0.1241 --> 0.1213
|
|
2025-09-16 03:37:11,300: Checkpointing model
|
|
2025-09-16 03:37:12,572: Saved model to checkpoint: trained_models/baseline_rnn/checkpoint/best_checkpoint
|
|
2025-09-16 03:37:39,451: Train batch 26200: loss: 0.12 grad norm: 5.56 time: 0.151
|
|
2025-09-16 03:38:11,717: Train batch 26400: loss: 0.66 grad norm: 4.58 time: 0.116
|
|
2025-09-16 03:38:43,882: Train batch 26600: loss: 0.09 grad norm: 4.84 time: 0.106
|
|
2025-09-16 03:39:15,889: Train batch 26800: loss: 0.17 grad norm: 13.27 time: 0.134
|
|
2025-09-16 03:39:49,026: Train batch 27000: loss: 0.20 grad norm: 6.92 time: 0.122
|
|
2025-09-16 03:40:21,910: Train batch 27200: loss: 0.43 grad norm: 18.42 time: 0.138
|
|
2025-09-16 03:40:54,151: Train batch 27400: loss: 0.12 grad norm: 5.43 time: 0.125
|
|
2025-09-16 03:41:27,269: Train batch 27600: loss: 0.15 grad norm: 7.89 time: 0.114
|
|
2025-09-16 03:42:00,546: Train batch 27800: loss: 0.24 grad norm: 13.90 time: 0.130
|
|
2025-09-16 03:42:33,753: Train batch 28000: loss: 0.11 grad norm: 6.05 time: 0.121
|
|
2025-09-16 03:42:33,754: Running test after training batch: 28000
|
|
2025-09-16 03:42:44,647: Val batch 28000: PER (avg): 0.1229 CTC Loss (avg): 26.9724 time: 10.893
|
|
2025-09-16 03:42:44,647: t15.2023.08.13 val PER: 0.0998
|
|
2025-09-16 03:42:44,647: t15.2023.08.18 val PER: 0.0872
|
|
2025-09-16 03:42:44,647: t15.2023.08.20 val PER: 0.0612
|
|
2025-09-16 03:42:44,647: t15.2023.08.25 val PER: 0.0798
|
|
2025-09-16 03:42:44,648: t15.2023.08.27 val PER: 0.1881
|
|
2025-09-16 03:42:44,648: t15.2023.09.01 val PER: 0.0544
|
|
2025-09-16 03:42:44,648: t15.2023.09.03 val PER: 0.1235
|
|
2025-09-16 03:42:44,648: t15.2023.09.24 val PER: 0.0959
|
|
2025-09-16 03:42:44,648: t15.2023.09.29 val PER: 0.1059
|
|
2025-09-16 03:42:44,648: t15.2023.10.01 val PER: 0.1499
|
|
2025-09-16 03:42:44,648: t15.2023.10.06 val PER: 0.0624
|
|
2025-09-16 03:42:44,648: t15.2023.10.08 val PER: 0.2111
|
|
2025-09-16 03:42:44,648: t15.2023.10.13 val PER: 0.1877
|
|
2025-09-16 03:42:44,648: t15.2023.10.15 val PER: 0.1233
|
|
2025-09-16 03:42:44,648: t15.2023.10.20 val PER: 0.1946
|
|
2025-09-16 03:42:44,648: t15.2023.10.22 val PER: 0.1169
|
|
2025-09-16 03:42:44,648: t15.2023.11.03 val PER: 0.1744
|
|
2025-09-16 03:42:44,648: t15.2023.11.04 val PER: 0.0137
|
|
2025-09-16 03:42:44,648: t15.2023.11.17 val PER: 0.0358
|
|
2025-09-16 03:42:44,648: t15.2023.11.19 val PER: 0.0160
|
|
2025-09-16 03:42:44,648: t15.2023.11.26 val PER: 0.0674
|
|
2025-09-16 03:42:44,648: t15.2023.12.03 val PER: 0.0798
|
|
2025-09-16 03:42:44,648: t15.2023.12.08 val PER: 0.0586
|
|
2025-09-16 03:42:44,649: t15.2023.12.10 val PER: 0.0499
|
|
2025-09-16 03:42:44,649: t15.2023.12.17 val PER: 0.1050
|
|
2025-09-16 03:42:44,649: t15.2023.12.29 val PER: 0.0940
|
|
2025-09-16 03:42:44,649: t15.2024.02.25 val PER: 0.0941
|
|
2025-09-16 03:42:44,649: t15.2024.03.08 val PER: 0.2077
|
|
2025-09-16 03:42:44,649: t15.2024.03.15 val PER: 0.1707
|
|
2025-09-16 03:42:44,649: t15.2024.03.17 val PER: 0.0886
|
|
2025-09-16 03:42:44,649: t15.2024.05.10 val PER: 0.1501
|
|
2025-09-16 03:42:44,649: t15.2024.06.14 val PER: 0.1467
|
|
2025-09-16 03:42:44,649: t15.2024.07.19 val PER: 0.1945
|
|
2025-09-16 03:42:44,649: t15.2024.07.21 val PER: 0.0634
|
|
2025-09-16 03:42:44,649: t15.2024.07.28 val PER: 0.1110
|
|
2025-09-16 03:42:44,649: t15.2025.01.10 val PER: 0.2893
|
|
2025-09-16 03:42:44,649: t15.2025.01.12 val PER: 0.0955
|
|
2025-09-16 03:42:44,649: t15.2025.03.14 val PER: 0.2885
|
|
2025-09-16 03:42:44,649: t15.2025.03.16 val PER: 0.1597
|
|
2025-09-16 03:42:44,649: t15.2025.03.30 val PER: 0.2368
|
|
2025-09-16 03:42:44,649: t15.2025.04.13 val PER: 0.2325
|
|
2025-09-16 03:43:12,759: Train batch 28200: loss: 0.28 grad norm: 6.94 time: 0.154
|
|
2025-09-16 03:43:44,780: Train batch 28400: loss: 0.71 grad norm: 27.81 time: 0.115
|
|
2025-09-16 03:44:18,034: Train batch 28600: loss: 0.06 grad norm: 2.61 time: 0.151
|
|
2025-09-16 03:44:50,409: Train batch 28800: loss: 0.18 grad norm: 7.18 time: 0.116
|
|
2025-09-16 03:45:24,076: Train batch 29000: loss: 0.05 grad norm: 4.52 time: 0.127
|
|
2025-09-16 03:45:56,717: Train batch 29200: loss: 0.17 grad norm: 5.55 time: 0.107
|
|
2025-09-16 03:46:29,841: Train batch 29400: loss: 0.23 grad norm: 4.61 time: 0.123
|
|
2025-09-16 03:47:02,600: Train batch 29600: loss: 0.24 grad norm: 24.20 time: 0.112
|
|
2025-09-16 03:47:36,006: Train batch 29800: loss: 0.11 grad norm: 4.95 time: 0.112
|
|
2025-09-16 03:48:08,354: Train batch 30000: loss: 0.07 grad norm: 4.89 time: 0.126
|
|
2025-09-16 03:48:08,354: Running test after training batch: 30000
|
|
2025-09-16 03:48:19,117: Val batch 30000: PER (avg): 0.1210 CTC Loss (avg): 27.2096 time: 10.763
|
|
2025-09-16 03:48:19,118: t15.2023.08.13 val PER: 0.0967
|
|
2025-09-16 03:48:19,118: t15.2023.08.18 val PER: 0.0746
|
|
2025-09-16 03:48:19,118: t15.2023.08.20 val PER: 0.0620
|
|
2025-09-16 03:48:19,118: t15.2023.08.25 val PER: 0.0873
|
|
2025-09-16 03:48:19,118: t15.2023.08.27 val PER: 0.1817
|
|
2025-09-16 03:48:19,118: t15.2023.09.01 val PER: 0.0503
|
|
2025-09-16 03:48:19,118: t15.2023.09.03 val PER: 0.1342
|
|
2025-09-16 03:48:19,118: t15.2023.09.24 val PER: 0.1019
|
|
2025-09-16 03:48:19,118: t15.2023.09.29 val PER: 0.1085
|
|
2025-09-16 03:48:19,118: t15.2023.10.01 val PER: 0.1631
|
|
2025-09-16 03:48:19,118: t15.2023.10.06 val PER: 0.0603
|
|
2025-09-16 03:48:19,118: t15.2023.10.08 val PER: 0.2152
|
|
2025-09-16 03:48:19,119: t15.2023.10.13 val PER: 0.1854
|
|
2025-09-16 03:48:19,119: t15.2023.10.15 val PER: 0.1272
|
|
2025-09-16 03:48:19,119: t15.2023.10.20 val PER: 0.1745
|
|
2025-09-16 03:48:19,119: t15.2023.10.22 val PER: 0.1114
|
|
2025-09-16 03:48:19,119: t15.2023.11.03 val PER: 0.1771
|
|
2025-09-16 03:48:19,119: t15.2023.11.04 val PER: 0.0034
|
|
2025-09-16 03:48:19,119: t15.2023.11.17 val PER: 0.0202
|
|
2025-09-16 03:48:19,119: t15.2023.11.19 val PER: 0.0180
|
|
2025-09-16 03:48:19,119: t15.2023.11.26 val PER: 0.0594
|
|
2025-09-16 03:48:19,119: t15.2023.12.03 val PER: 0.0777
|
|
2025-09-16 03:48:19,119: t15.2023.12.08 val PER: 0.0459
|
|
2025-09-16 03:48:19,119: t15.2023.12.10 val PER: 0.0578
|
|
2025-09-16 03:48:19,119: t15.2023.12.17 val PER: 0.0977
|
|
2025-09-16 03:48:19,119: t15.2023.12.29 val PER: 0.1030
|
|
2025-09-16 03:48:19,119: t15.2024.02.25 val PER: 0.0885
|
|
2025-09-16 03:48:19,119: t15.2024.03.08 val PER: 0.1764
|
|
2025-09-16 03:48:19,119: t15.2024.03.15 val PER: 0.1620
|
|
2025-09-16 03:48:19,119: t15.2024.03.17 val PER: 0.0900
|
|
2025-09-16 03:48:19,119: t15.2024.05.10 val PER: 0.1441
|
|
2025-09-16 03:48:19,120: t15.2024.06.14 val PER: 0.1498
|
|
2025-09-16 03:48:19,120: t15.2024.07.19 val PER: 0.1721
|
|
2025-09-16 03:48:19,120: t15.2024.07.21 val PER: 0.0669
|
|
2025-09-16 03:48:19,120: t15.2024.07.28 val PER: 0.1007
|
|
2025-09-16 03:48:19,120: t15.2025.01.10 val PER: 0.2658
|
|
2025-09-16 03:48:19,120: t15.2025.01.12 val PER: 0.1001
|
|
2025-09-16 03:48:19,120: t15.2025.03.14 val PER: 0.3269
|
|
2025-09-16 03:48:19,120: t15.2025.03.16 val PER: 0.1715
|
|
2025-09-16 03:48:19,120: t15.2025.03.30 val PER: 0.2471
|
|
2025-09-16 03:48:19,120: t15.2025.04.13 val PER: 0.2197
|
|
2025-09-16 03:48:19,120: New best test PER 0.1213 --> 0.1210
|
|
2025-09-16 03:48:19,120: Checkpointing model
|
|
2025-09-16 03:48:20,497: Saved model to checkpoint: trained_models/baseline_rnn/checkpoint/best_checkpoint
|
|
2025-09-16 03:48:48,314: Train batch 30200: loss: 0.05 grad norm: 3.01 time: 0.114
|
|
2025-09-16 03:49:21,026: Train batch 30400: loss: 0.13 grad norm: 8.20 time: 0.127
|
|
2025-09-16 03:49:52,963: Train batch 30600: loss: 0.07 grad norm: 4.41 time: 0.142
|
|
2025-09-16 03:50:24,602: Train batch 30800: loss: 0.08 grad norm: 15.94 time: 0.112
|
|
2025-09-16 03:50:57,635: Train batch 31000: loss: 0.13 grad norm: 6.11 time: 0.112
|
|
2025-09-16 03:51:29,350: Train batch 31200: loss: 0.77 grad norm: 14.62 time: 0.131
|
|
2025-09-16 03:52:01,814: Train batch 31400: loss: 0.20 grad norm: 6.81 time: 0.127
|
|
2025-09-16 03:52:33,038: Train batch 31600: loss: 0.16 grad norm: 10.17 time: 0.140
|
|
2025-09-16 03:53:04,407: Train batch 31800: loss: 0.16 grad norm: 9.39 time: 0.102
|
|
2025-09-16 03:53:36,898: Train batch 32000: loss: 0.08 grad norm: 7.02 time: 0.116
|
|
2025-09-16 03:53:36,898: Running test after training batch: 32000
|
|
2025-09-16 03:53:47,688: Val batch 32000: PER (avg): 0.1194 CTC Loss (avg): 26.8790 time: 10.790
|
|
2025-09-16 03:53:47,689: t15.2023.08.13 val PER: 0.1029
|
|
2025-09-16 03:53:47,689: t15.2023.08.18 val PER: 0.0805
|
|
2025-09-16 03:53:47,689: t15.2023.08.20 val PER: 0.0667
|
|
2025-09-16 03:53:47,689: t15.2023.08.25 val PER: 0.0919
|
|
2025-09-16 03:53:47,689: t15.2023.08.27 val PER: 0.1752
|
|
2025-09-16 03:53:47,689: t15.2023.09.01 val PER: 0.0503
|
|
2025-09-16 03:53:47,689: t15.2023.09.03 val PER: 0.1390
|
|
2025-09-16 03:53:47,689: t15.2023.09.24 val PER: 0.0947
|
|
2025-09-16 03:53:47,689: t15.2023.09.29 val PER: 0.1104
|
|
2025-09-16 03:53:47,689: t15.2023.10.01 val PER: 0.1341
|
|
2025-09-16 03:53:47,689: t15.2023.10.06 val PER: 0.0678
|
|
2025-09-16 03:53:47,689: t15.2023.10.08 val PER: 0.1989
|
|
2025-09-16 03:53:47,690: t15.2023.10.13 val PER: 0.1746
|
|
2025-09-16 03:53:47,690: t15.2023.10.15 val PER: 0.1371
|
|
2025-09-16 03:53:47,690: t15.2023.10.20 val PER: 0.1879
|
|
2025-09-16 03:53:47,690: t15.2023.10.22 val PER: 0.1158
|
|
2025-09-16 03:53:47,690: t15.2023.11.03 val PER: 0.1615
|
|
2025-09-16 03:53:47,690: t15.2023.11.04 val PER: 0.0034
|
|
2025-09-16 03:53:47,690: t15.2023.11.17 val PER: 0.0327
|
|
2025-09-16 03:53:47,690: t15.2023.11.19 val PER: 0.0100
|
|
2025-09-16 03:53:47,690: t15.2023.11.26 val PER: 0.0630
|
|
2025-09-16 03:53:47,690: t15.2023.12.03 val PER: 0.0672
|
|
2025-09-16 03:53:47,690: t15.2023.12.08 val PER: 0.0539
|
|
2025-09-16 03:53:47,690: t15.2023.12.10 val PER: 0.0434
|
|
2025-09-16 03:53:47,690: t15.2023.12.17 val PER: 0.1040
|
|
2025-09-16 03:53:47,690: t15.2023.12.29 val PER: 0.0940
|
|
2025-09-16 03:53:47,690: t15.2024.02.25 val PER: 0.0927
|
|
2025-09-16 03:53:47,690: t15.2024.03.08 val PER: 0.1892
|
|
2025-09-16 03:53:47,690: t15.2024.03.15 val PER: 0.1720
|
|
2025-09-16 03:53:47,691: t15.2024.03.17 val PER: 0.0872
|
|
2025-09-16 03:53:47,691: t15.2024.05.10 val PER: 0.1248
|
|
2025-09-16 03:53:47,691: t15.2024.06.14 val PER: 0.1293
|
|
2025-09-16 03:53:47,691: t15.2024.07.19 val PER: 0.1852
|
|
2025-09-16 03:53:47,691: t15.2024.07.21 val PER: 0.0690
|
|
2025-09-16 03:53:47,691: t15.2024.07.28 val PER: 0.1088
|
|
2025-09-16 03:53:47,691: t15.2025.01.10 val PER: 0.2658
|
|
2025-09-16 03:53:47,691: t15.2025.01.12 val PER: 0.0939
|
|
2025-09-16 03:53:47,691: t15.2025.03.14 val PER: 0.2796
|
|
2025-09-16 03:53:47,691: t15.2025.03.16 val PER: 0.1505
|
|
2025-09-16 03:53:47,691: t15.2025.03.30 val PER: 0.2540
|
|
2025-09-16 03:53:47,691: t15.2025.04.13 val PER: 0.2254
|
|
2025-09-16 03:53:47,691: New best test PER 0.1210 --> 0.1194
|
|
2025-09-16 03:53:47,691: Checkpointing model
|
|
2025-09-16 03:53:48,951: Saved model to checkpoint: trained_models/baseline_rnn/checkpoint/best_checkpoint
|
|
2025-09-16 03:54:16,915: Train batch 32200: loss: 0.10 grad norm: 3.99 time: 0.137
|
|
2025-09-16 03:54:50,444: Train batch 32400: loss: 0.08 grad norm: 4.72 time: 0.125
|
|
2025-09-16 03:55:23,341: Train batch 32600: loss: 0.19 grad norm: 6.49 time: 0.136
|
|
2025-09-16 03:55:55,882: Train batch 32800: loss: 0.08 grad norm: 5.32 time: 0.147
|
|
2025-09-16 03:56:27,803: Train batch 33000: loss: 0.12 grad norm: 8.34 time: 0.133
|
|
2025-09-16 03:57:00,379: Train batch 33200: loss: 0.10 grad norm: 5.39 time: 0.174
|
|
2025-09-16 03:57:32,384: Train batch 33400: loss: 0.11 grad norm: 6.06 time: 0.137
|
|
2025-09-16 03:58:05,371: Train batch 33600: loss: 0.10 grad norm: 5.31 time: 0.117
|
|
2025-09-16 03:58:38,729: Train batch 33800: loss: 0.41 grad norm: 13.39 time: 0.114
|
|
2025-09-16 03:59:12,151: Train batch 34000: loss: 0.11 grad norm: 5.43 time: 0.146
|
|
2025-09-16 03:59:12,151: Running test after training batch: 34000
|
|
2025-09-16 03:59:22,681: Val batch 34000: PER (avg): 0.1190 CTC Loss (avg): 27.7464 time: 10.529
|
|
2025-09-16 03:59:22,681: t15.2023.08.13 val PER: 0.0894
|
|
2025-09-16 03:59:22,681: t15.2023.08.18 val PER: 0.0796
|
|
2025-09-16 03:59:22,681: t15.2023.08.20 val PER: 0.0683
|
|
2025-09-16 03:59:22,681: t15.2023.08.25 val PER: 0.0843
|
|
2025-09-16 03:59:22,681: t15.2023.08.27 val PER: 0.1849
|
|
2025-09-16 03:59:22,681: t15.2023.09.01 val PER: 0.0503
|
|
2025-09-16 03:59:22,681: t15.2023.09.03 val PER: 0.1390
|
|
2025-09-16 03:59:22,681: t15.2023.09.24 val PER: 0.1056
|
|
2025-09-16 03:59:22,682: t15.2023.09.29 val PER: 0.1110
|
|
2025-09-16 03:59:22,682: t15.2023.10.01 val PER: 0.1480
|
|
2025-09-16 03:59:22,682: t15.2023.10.06 val PER: 0.0603
|
|
2025-09-16 03:59:22,682: t15.2023.10.08 val PER: 0.2138
|
|
2025-09-16 03:59:22,682: t15.2023.10.13 val PER: 0.1877
|
|
2025-09-16 03:59:22,682: t15.2023.10.15 val PER: 0.1147
|
|
2025-09-16 03:59:22,682: t15.2023.10.20 val PER: 0.1812
|
|
2025-09-16 03:59:22,682: t15.2023.10.22 val PER: 0.1091
|
|
2025-09-16 03:59:22,682: t15.2023.11.03 val PER: 0.1689
|
|
2025-09-16 03:59:22,682: t15.2023.11.04 val PER: 0.0102
|
|
2025-09-16 03:59:22,682: t15.2023.11.17 val PER: 0.0264
|
|
2025-09-16 03:59:22,682: t15.2023.11.19 val PER: 0.0120
|
|
2025-09-16 03:59:22,682: t15.2023.11.26 val PER: 0.0681
|
|
2025-09-16 03:59:22,682: t15.2023.12.03 val PER: 0.0672
|
|
2025-09-16 03:59:22,682: t15.2023.12.08 val PER: 0.0499
|
|
2025-09-16 03:59:22,682: t15.2023.12.10 val PER: 0.0420
|
|
2025-09-16 03:59:22,682: t15.2023.12.17 val PER: 0.0936
|
|
2025-09-16 03:59:22,682: t15.2023.12.29 val PER: 0.1043
|
|
2025-09-16 03:59:22,682: t15.2024.02.25 val PER: 0.0815
|
|
2025-09-16 03:59:22,683: t15.2024.03.08 val PER: 0.2091
|
|
2025-09-16 03:59:22,683: t15.2024.03.15 val PER: 0.1757
|
|
2025-09-16 03:59:22,683: t15.2024.03.17 val PER: 0.0893
|
|
2025-09-16 03:59:22,683: t15.2024.05.10 val PER: 0.1337
|
|
2025-09-16 03:59:22,683: t15.2024.06.14 val PER: 0.1183
|
|
2025-09-16 03:59:22,683: t15.2024.07.19 val PER: 0.1859
|
|
2025-09-16 03:59:22,683: t15.2024.07.21 val PER: 0.0648
|
|
2025-09-16 03:59:22,683: t15.2024.07.28 val PER: 0.1059
|
|
2025-09-16 03:59:22,683: t15.2025.01.10 val PER: 0.2769
|
|
2025-09-16 03:59:22,683: t15.2025.01.12 val PER: 0.0885
|
|
2025-09-16 03:59:22,683: t15.2025.03.14 val PER: 0.2929
|
|
2025-09-16 03:59:22,683: t15.2025.03.16 val PER: 0.1309
|
|
2025-09-16 03:59:22,683: t15.2025.03.30 val PER: 0.2356
|
|
2025-09-16 03:59:22,683: t15.2025.04.13 val PER: 0.1997
|
|
2025-09-16 03:59:22,683: New best test PER 0.1194 --> 0.1190
|
|
2025-09-16 03:59:22,683: Checkpointing model
|
|
2025-09-16 03:59:24,053: Saved model to checkpoint: trained_models/baseline_rnn/checkpoint/best_checkpoint
|
|
2025-09-16 03:59:51,488: Train batch 34200: loss: 0.04 grad norm: 3.33 time: 0.143
|
|
2025-09-16 04:00:24,194: Train batch 34400: loss: 0.09 grad norm: 6.09 time: 0.117
|
|
2025-09-16 04:00:56,988: Train batch 34600: loss: 0.07 grad norm: 3.65 time: 0.137
|
|
2025-09-16 04:01:29,604: Train batch 34800: loss: 0.26 grad norm: 8.41 time: 0.113
|
|
2025-09-16 04:02:02,139: Train batch 35000: loss: 0.17 grad norm: 8.30 time: 0.118
|
|
2025-09-16 04:02:34,597: Train batch 35200: loss: 0.03 grad norm: 2.58 time: 0.126
|
|
2025-09-16 04:03:07,677: Train batch 35400: loss: 0.06 grad norm: 3.95 time: 0.149
|
|
2025-09-16 04:03:40,545: Train batch 35600: loss: 0.05 grad norm: 2.74 time: 0.107
|
|
2025-09-16 04:04:12,828: Train batch 35800: loss: 0.05 grad norm: 3.35 time: 0.168
|
|
2025-09-16 04:04:46,142: Train batch 36000: loss: 0.31 grad norm: 2.65 time: 0.155
|
|
2025-09-16 04:04:46,143: Running test after training batch: 36000
|
|
2025-09-16 04:04:56,539: Val batch 36000: PER (avg): 0.1167 CTC Loss (avg): 27.0357 time: 10.396
|
|
2025-09-16 04:04:56,539: t15.2023.08.13 val PER: 0.0925
|
|
2025-09-16 04:04:56,540: t15.2023.08.18 val PER: 0.0796
|
|
2025-09-16 04:04:56,540: t15.2023.08.20 val PER: 0.0659
|
|
2025-09-16 04:04:56,540: t15.2023.08.25 val PER: 0.0964
|
|
2025-09-16 04:04:56,540: t15.2023.08.27 val PER: 0.1897
|
|
2025-09-16 04:04:56,540: t15.2023.09.01 val PER: 0.0544
|
|
2025-09-16 04:04:56,540: t15.2023.09.03 val PER: 0.1259
|
|
2025-09-16 04:04:56,540: t15.2023.09.24 val PER: 0.1044
|
|
2025-09-16 04:04:56,540: t15.2023.09.29 val PER: 0.1059
|
|
2025-09-16 04:04:56,540: t15.2023.10.01 val PER: 0.1407
|
|
2025-09-16 04:04:56,540: t15.2023.10.06 val PER: 0.0624
|
|
2025-09-16 04:04:56,540: t15.2023.10.08 val PER: 0.2084
|
|
2025-09-16 04:04:56,540: t15.2023.10.13 val PER: 0.1808
|
|
2025-09-16 04:04:56,540: t15.2023.10.15 val PER: 0.1134
|
|
2025-09-16 04:04:56,540: t15.2023.10.20 val PER: 0.1711
|
|
2025-09-16 04:04:56,540: t15.2023.10.22 val PER: 0.1102
|
|
2025-09-16 04:04:56,540: t15.2023.11.03 val PER: 0.1649
|
|
2025-09-16 04:04:56,540: t15.2023.11.04 val PER: 0.0137
|
|
2025-09-16 04:04:56,540: t15.2023.11.17 val PER: 0.0280
|
|
2025-09-16 04:04:56,540: t15.2023.11.19 val PER: 0.0100
|
|
2025-09-16 04:04:56,541: t15.2023.11.26 val PER: 0.0529
|
|
2025-09-16 04:04:56,541: t15.2023.12.03 val PER: 0.0662
|
|
2025-09-16 04:04:56,541: t15.2023.12.08 val PER: 0.0519
|
|
2025-09-16 04:04:56,541: t15.2023.12.10 val PER: 0.0499
|
|
2025-09-16 04:04:56,541: t15.2023.12.17 val PER: 0.0894
|
|
2025-09-16 04:04:56,541: t15.2023.12.29 val PER: 0.0872
|
|
2025-09-16 04:04:56,541: t15.2024.02.25 val PER: 0.0744
|
|
2025-09-16 04:04:56,541: t15.2024.03.08 val PER: 0.1764
|
|
2025-09-16 04:04:56,541: t15.2024.03.15 val PER: 0.1626
|
|
2025-09-16 04:04:56,541: t15.2024.03.17 val PER: 0.0907
|
|
2025-09-16 04:04:56,541: t15.2024.05.10 val PER: 0.1293
|
|
2025-09-16 04:04:56,541: t15.2024.06.14 val PER: 0.1293
|
|
2025-09-16 04:04:56,541: t15.2024.07.19 val PER: 0.1780
|
|
2025-09-16 04:04:56,541: t15.2024.07.21 val PER: 0.0593
|
|
2025-09-16 04:04:56,541: t15.2024.07.28 val PER: 0.0956
|
|
2025-09-16 04:04:56,541: t15.2025.01.10 val PER: 0.2769
|
|
2025-09-16 04:04:56,541: t15.2025.01.12 val PER: 0.0808
|
|
2025-09-16 04:04:56,541: t15.2025.03.14 val PER: 0.3136
|
|
2025-09-16 04:04:56,541: t15.2025.03.16 val PER: 0.1453
|
|
2025-09-16 04:04:56,542: t15.2025.03.30 val PER: 0.2563
|
|
2025-09-16 04:04:56,542: t15.2025.04.13 val PER: 0.2397
|
|
2025-09-16 04:04:56,542: New best test PER 0.1190 --> 0.1167
|
|
2025-09-16 04:04:56,542: Checkpointing model
|
|
2025-09-16 04:04:57,843: Saved model to checkpoint: trained_models/baseline_rnn/checkpoint/best_checkpoint
|
|
2025-09-16 04:05:25,251: Train batch 36200: loss: 0.03 grad norm: 2.64 time: 0.140
|
|
2025-09-16 04:05:57,704: Train batch 36400: loss: 0.07 grad norm: 3.52 time: 0.169
|
|
2025-09-16 04:06:29,789: Train batch 36600: loss: 0.04 grad norm: 2.88 time: 0.160
|
|
2025-09-16 04:07:02,372: Train batch 36800: loss: 0.08 grad norm: 3.43 time: 0.160
|
|
2025-09-16 04:07:34,836: Train batch 37000: loss: 0.18 grad norm: 6.81 time: 0.137
|
|
2025-09-16 04:08:06,950: Train batch 37200: loss: 0.03 grad norm: 3.43 time: 0.113
|
|
2025-09-16 04:08:38,731: Train batch 37400: loss: 0.19 grad norm: 4.13 time: 0.129
|
|
2025-09-16 04:09:10,403: Train batch 37600: loss: 0.10 grad norm: 4.61 time: 0.102
|
|
2025-09-16 04:09:43,349: Train batch 37800: loss: 0.09 grad norm: 5.99 time: 0.128
|
|
2025-09-16 04:10:15,708: Train batch 38000: loss: 0.07 grad norm: 3.07 time: 0.108
|
|
2025-09-16 04:10:15,708: Running test after training batch: 38000
|
|
2025-09-16 04:10:26,299: Val batch 38000: PER (avg): 0.1188 CTC Loss (avg): 28.1456 time: 10.590
|
|
2025-09-16 04:10:26,299: t15.2023.08.13 val PER: 0.0894
|
|
2025-09-16 04:10:26,299: t15.2023.08.18 val PER: 0.0855
|
|
2025-09-16 04:10:26,299: t15.2023.08.20 val PER: 0.0627
|
|
2025-09-16 04:10:26,299: t15.2023.08.25 val PER: 0.0873
|
|
2025-09-16 04:10:26,299: t15.2023.08.27 val PER: 0.1897
|
|
2025-09-16 04:10:26,299: t15.2023.09.01 val PER: 0.0536
|
|
2025-09-16 04:10:26,299: t15.2023.09.03 val PER: 0.1188
|
|
2025-09-16 04:10:26,300: t15.2023.09.24 val PER: 0.0983
|
|
2025-09-16 04:10:26,300: t15.2023.09.29 val PER: 0.1021
|
|
2025-09-16 04:10:26,300: t15.2023.10.01 val PER: 0.1473
|
|
2025-09-16 04:10:26,300: t15.2023.10.06 val PER: 0.0700
|
|
2025-09-16 04:10:26,300: t15.2023.10.08 val PER: 0.2206
|
|
2025-09-16 04:10:26,300: t15.2023.10.13 val PER: 0.1854
|
|
2025-09-16 04:10:26,300: t15.2023.10.15 val PER: 0.1200
|
|
2025-09-16 04:10:26,300: t15.2023.10.20 val PER: 0.1745
|
|
2025-09-16 04:10:26,300: t15.2023.10.22 val PER: 0.1047
|
|
2025-09-16 04:10:26,300: t15.2023.11.03 val PER: 0.1682
|
|
2025-09-16 04:10:26,300: t15.2023.11.04 val PER: 0.0068
|
|
2025-09-16 04:10:26,300: t15.2023.11.17 val PER: 0.0280
|
|
2025-09-16 04:10:26,300: t15.2023.11.19 val PER: 0.0120
|
|
2025-09-16 04:10:26,300: t15.2023.11.26 val PER: 0.0543
|
|
2025-09-16 04:10:26,300: t15.2023.12.03 val PER: 0.0588
|
|
2025-09-16 04:10:26,300: t15.2023.12.08 val PER: 0.0506
|
|
2025-09-16 04:10:26,300: t15.2023.12.10 val PER: 0.0460
|
|
2025-09-16 04:10:26,300: t15.2023.12.17 val PER: 0.0946
|
|
2025-09-16 04:10:26,300: t15.2023.12.29 val PER: 0.0933
|
|
2025-09-16 04:10:26,301: t15.2024.02.25 val PER: 0.1053
|
|
2025-09-16 04:10:26,301: t15.2024.03.08 val PER: 0.1977
|
|
2025-09-16 04:10:26,301: t15.2024.03.15 val PER: 0.1770
|
|
2025-09-16 04:10:26,301: t15.2024.03.17 val PER: 0.0844
|
|
2025-09-16 04:10:26,301: t15.2024.05.10 val PER: 0.1426
|
|
2025-09-16 04:10:26,301: t15.2024.06.14 val PER: 0.1325
|
|
2025-09-16 04:10:26,301: t15.2024.07.19 val PER: 0.1773
|
|
2025-09-16 04:10:26,301: t15.2024.07.21 val PER: 0.0697
|
|
2025-09-16 04:10:26,301: t15.2024.07.28 val PER: 0.0919
|
|
2025-09-16 04:10:26,301: t15.2025.01.10 val PER: 0.2851
|
|
2025-09-16 04:10:26,301: t15.2025.01.12 val PER: 0.1039
|
|
2025-09-16 04:10:26,301: t15.2025.03.14 val PER: 0.2885
|
|
2025-09-16 04:10:26,301: t15.2025.03.16 val PER: 0.1453
|
|
2025-09-16 04:10:26,301: t15.2025.03.30 val PER: 0.2483
|
|
2025-09-16 04:10:26,301: t15.2025.04.13 val PER: 0.2154
|
|
2025-09-16 04:10:53,936: Train batch 38200: loss: 0.01 grad norm: 0.71 time: 0.149
|
|
2025-09-16 04:11:26,923: Train batch 38400: loss: 0.02 grad norm: 1.25 time: 0.117
|
|
2025-09-16 04:11:59,784: Train batch 38600: loss: 0.09 grad norm: 5.17 time: 0.136
|
|
2025-09-16 04:12:32,347: Train batch 38800: loss: 0.01 grad norm: 1.35 time: 0.111
|
|
2025-09-16 04:13:04,367: Train batch 39000: loss: 0.07 grad norm: 6.18 time: 0.132
|
|
2025-09-16 04:13:36,717: Train batch 39200: loss: 0.20 grad norm: 14.22 time: 0.155
|
|
2025-09-16 04:14:08,445: Train batch 39400: loss: 0.07 grad norm: 5.91 time: 0.115
|
|
2025-09-16 04:14:40,218: Train batch 39600: loss: 0.04 grad norm: 5.09 time: 0.124
|
|
2025-09-16 04:15:12,441: Train batch 39800: loss: 0.04 grad norm: 4.57 time: 0.133
|
|
2025-09-16 04:15:44,494: Train batch 40000: loss: 0.14 grad norm: 11.04 time: 0.151
|
|
2025-09-16 04:15:44,495: Running test after training batch: 40000
|
|
2025-09-16 04:15:54,954: Val batch 40000: PER (avg): 0.1157 CTC Loss (avg): 27.0700 time: 10.459
|
|
2025-09-16 04:15:54,955: t15.2023.08.13 val PER: 0.0884
|
|
2025-09-16 04:15:54,955: t15.2023.08.18 val PER: 0.0872
|
|
2025-09-16 04:15:54,955: t15.2023.08.20 val PER: 0.0699
|
|
2025-09-16 04:15:54,955: t15.2023.08.25 val PER: 0.0964
|
|
2025-09-16 04:15:54,955: t15.2023.08.27 val PER: 0.1817
|
|
2025-09-16 04:15:54,956: t15.2023.09.01 val PER: 0.0552
|
|
2025-09-16 04:15:54,956: t15.2023.09.03 val PER: 0.1283
|
|
2025-09-16 04:15:54,956: t15.2023.09.24 val PER: 0.1092
|
|
2025-09-16 04:15:54,956: t15.2023.09.29 val PER: 0.1040
|
|
2025-09-16 04:15:54,956: t15.2023.10.01 val PER: 0.1407
|
|
2025-09-16 04:15:54,956: t15.2023.10.06 val PER: 0.0603
|
|
2025-09-16 04:15:54,956: t15.2023.10.08 val PER: 0.2165
|
|
2025-09-16 04:15:54,957: t15.2023.10.13 val PER: 0.1761
|
|
2025-09-16 04:15:54,957: t15.2023.10.15 val PER: 0.1160
|
|
2025-09-16 04:15:54,957: t15.2023.10.20 val PER: 0.2013
|
|
2025-09-16 04:15:54,957: t15.2023.10.22 val PER: 0.1102
|
|
2025-09-16 04:15:54,957: t15.2023.11.03 val PER: 0.1567
|
|
2025-09-16 04:15:54,957: t15.2023.11.04 val PER: 0.0000
|
|
2025-09-16 04:15:54,957: t15.2023.11.17 val PER: 0.0264
|
|
2025-09-16 04:15:54,957: t15.2023.11.19 val PER: 0.0080
|
|
2025-09-16 04:15:54,957: t15.2023.11.26 val PER: 0.0580
|
|
2025-09-16 04:15:54,957: t15.2023.12.03 val PER: 0.0609
|
|
2025-09-16 04:15:54,957: t15.2023.12.08 val PER: 0.0453
|
|
2025-09-16 04:15:54,957: t15.2023.12.10 val PER: 0.0434
|
|
2025-09-16 04:15:54,957: t15.2023.12.17 val PER: 0.0863
|
|
2025-09-16 04:15:54,958: t15.2023.12.29 val PER: 0.0837
|
|
2025-09-16 04:15:54,958: t15.2024.02.25 val PER: 0.0941
|
|
2025-09-16 04:15:54,958: t15.2024.03.08 val PER: 0.1721
|
|
2025-09-16 04:15:54,958: t15.2024.03.15 val PER: 0.1682
|
|
2025-09-16 04:15:54,958: t15.2024.03.17 val PER: 0.0802
|
|
2025-09-16 04:15:54,958: t15.2024.05.10 val PER: 0.1174
|
|
2025-09-16 04:15:54,958: t15.2024.06.14 val PER: 0.1435
|
|
2025-09-16 04:15:54,958: t15.2024.07.19 val PER: 0.1727
|
|
2025-09-16 04:15:54,958: t15.2024.07.21 val PER: 0.0655
|
|
2025-09-16 04:15:54,958: t15.2024.07.28 val PER: 0.0860
|
|
2025-09-16 04:15:54,958: t15.2025.01.10 val PER: 0.2782
|
|
2025-09-16 04:15:54,958: t15.2025.01.12 val PER: 0.0970
|
|
2025-09-16 04:15:54,958: t15.2025.03.14 val PER: 0.2929
|
|
2025-09-16 04:15:54,958: t15.2025.03.16 val PER: 0.1427
|
|
2025-09-16 04:15:54,958: t15.2025.03.30 val PER: 0.2437
|
|
2025-09-16 04:15:54,958: t15.2025.04.13 val PER: 0.2254
|
|
2025-09-16 04:15:54,958: New best test PER 0.1167 --> 0.1157
|
|
2025-09-16 04:15:54,958: Checkpointing model
|
|
2025-09-16 04:15:56,234: Saved model to checkpoint: trained_models/baseline_rnn/checkpoint/best_checkpoint
|
|
2025-09-16 04:16:24,246: Train batch 40200: loss: 0.13 grad norm: 12.00 time: 0.138
|
|
2025-09-16 04:16:57,288: Train batch 40400: loss: 0.10 grad norm: 5.63 time: 0.132
|
|
2025-09-16 04:17:29,919: Train batch 40600: loss: 0.14 grad norm: 6.76 time: 0.149
|
|
2025-09-16 04:18:03,551: Train batch 40800: loss: 0.01 grad norm: 1.94 time: 0.116
|
|
2025-09-16 04:18:36,411: Train batch 41000: loss: 0.12 grad norm: 7.01 time: 0.113
|
|
2025-09-16 04:19:10,292: Train batch 41200: loss: 0.08 grad norm: 5.39 time: 0.130
|
|
2025-09-16 04:19:44,095: Train batch 41400: loss: 0.06 grad norm: 4.75 time: 0.173
|
|
2025-09-16 04:20:17,227: Train batch 41600: loss: 0.02 grad norm: 1.18 time: 0.178
|
|
2025-09-16 04:20:49,498: Train batch 41800: loss: 0.08 grad norm: 8.56 time: 0.115
|
|
2025-09-16 04:21:21,090: Train batch 42000: loss: 0.02 grad norm: 1.17 time: 0.132
|
|
2025-09-16 04:21:21,090: Running test after training batch: 42000
|
|
2025-09-16 04:21:31,487: Val batch 42000: PER (avg): 0.1129 CTC Loss (avg): 26.9547 time: 10.396
|
|
2025-09-16 04:21:31,487: t15.2023.08.13 val PER: 0.0988
|
|
2025-09-16 04:21:31,487: t15.2023.08.18 val PER: 0.0805
|
|
2025-09-16 04:21:31,487: t15.2023.08.20 val PER: 0.0683
|
|
2025-09-16 04:21:31,487: t15.2023.08.25 val PER: 0.0843
|
|
2025-09-16 04:21:31,487: t15.2023.08.27 val PER: 0.1768
|
|
2025-09-16 04:21:31,487: t15.2023.09.01 val PER: 0.0560
|
|
2025-09-16 04:21:31,487: t15.2023.09.03 val PER: 0.1140
|
|
2025-09-16 04:21:31,488: t15.2023.09.24 val PER: 0.0995
|
|
2025-09-16 04:21:31,488: t15.2023.09.29 val PER: 0.1034
|
|
2025-09-16 04:21:31,488: t15.2023.10.01 val PER: 0.1420
|
|
2025-09-16 04:21:31,488: t15.2023.10.06 val PER: 0.0646
|
|
2025-09-16 04:21:31,488: t15.2023.10.08 val PER: 0.2030
|
|
2025-09-16 04:21:31,488: t15.2023.10.13 val PER: 0.1800
|
|
2025-09-16 04:21:31,488: t15.2023.10.15 val PER: 0.1094
|
|
2025-09-16 04:21:31,488: t15.2023.10.20 val PER: 0.1913
|
|
2025-09-16 04:21:31,488: t15.2023.10.22 val PER: 0.1058
|
|
2025-09-16 04:21:31,488: t15.2023.11.03 val PER: 0.1588
|
|
2025-09-16 04:21:31,488: t15.2023.11.04 val PER: 0.0000
|
|
2025-09-16 04:21:31,488: t15.2023.11.17 val PER: 0.0249
|
|
2025-09-16 04:21:31,488: t15.2023.11.19 val PER: 0.0120
|
|
2025-09-16 04:21:31,488: t15.2023.11.26 val PER: 0.0543
|
|
2025-09-16 04:21:31,488: t15.2023.12.03 val PER: 0.0557
|
|
2025-09-16 04:21:31,488: t15.2023.12.08 val PER: 0.0340
|
|
2025-09-16 04:21:31,488: t15.2023.12.10 val PER: 0.0329
|
|
2025-09-16 04:21:31,488: t15.2023.12.17 val PER: 0.0956
|
|
2025-09-16 04:21:31,488: t15.2023.12.29 val PER: 0.0748
|
|
2025-09-16 04:21:31,489: t15.2024.02.25 val PER: 0.0801
|
|
2025-09-16 04:21:31,489: t15.2024.03.08 val PER: 0.1764
|
|
2025-09-16 04:21:31,489: t15.2024.03.15 val PER: 0.1745
|
|
2025-09-16 04:21:31,489: t15.2024.03.17 val PER: 0.0865
|
|
2025-09-16 04:21:31,489: t15.2024.05.10 val PER: 0.1100
|
|
2025-09-16 04:21:31,489: t15.2024.06.14 val PER: 0.1120
|
|
2025-09-16 04:21:31,489: t15.2024.07.19 val PER: 0.1846
|
|
2025-09-16 04:21:31,489: t15.2024.07.21 val PER: 0.0572
|
|
2025-09-16 04:21:31,489: t15.2024.07.28 val PER: 0.0882
|
|
2025-09-16 04:21:31,489: t15.2025.01.10 val PER: 0.2590
|
|
2025-09-16 04:21:31,489: t15.2025.01.12 val PER: 0.0939
|
|
2025-09-16 04:21:31,489: t15.2025.03.14 val PER: 0.2766
|
|
2025-09-16 04:21:31,489: t15.2025.03.16 val PER: 0.1505
|
|
2025-09-16 04:21:31,489: t15.2025.03.30 val PER: 0.2483
|
|
2025-09-16 04:21:31,489: t15.2025.04.13 val PER: 0.2083
|
|
2025-09-16 04:21:31,489: New best test PER 0.1157 --> 0.1129
|
|
2025-09-16 04:21:31,489: Checkpointing model
|
|
2025-09-16 04:21:32,777: Saved model to checkpoint: trained_models/baseline_rnn/checkpoint/best_checkpoint
|
|
2025-09-16 04:22:01,170: Train batch 42200: loss: 0.19 grad norm: 3.55 time: 0.112
|
|
2025-09-16 04:22:34,648: Train batch 42400: loss: 0.05 grad norm: 2.47 time: 0.099
|
|
2025-09-16 04:23:08,522: Train batch 42600: loss: 0.03 grad norm: 2.15 time: 0.169
|
|
2025-09-16 04:23:42,012: Train batch 42800: loss: 0.02 grad norm: 1.95 time: 0.136
|
|
2025-09-16 04:24:14,599: Train batch 43000: loss: 0.18 grad norm: 8.84 time: 0.123
|
|
2025-09-16 04:24:48,448: Train batch 43200: loss: 0.05 grad norm: 4.28 time: 0.159
|
|
2025-09-16 04:25:21,553: Train batch 43400: loss: 0.03 grad norm: 1.90 time: 0.134
|
|
2025-09-16 04:25:55,061: Train batch 43600: loss: 0.04 grad norm: 2.57 time: 0.156
|
|
2025-09-16 04:26:28,616: Train batch 43800: loss: 0.01 grad norm: 1.08 time: 0.154
|
|
2025-09-16 04:27:02,049: Train batch 44000: loss: 0.06 grad norm: 3.99 time: 0.165
|
|
2025-09-16 04:27:02,050: Running test after training batch: 44000
|
|
2025-09-16 04:27:12,417: Val batch 44000: PER (avg): 0.1133 CTC Loss (avg): 26.7520 time: 10.367
|
|
2025-09-16 04:27:12,418: t15.2023.08.13 val PER: 0.0821
|
|
2025-09-16 04:27:12,418: t15.2023.08.18 val PER: 0.0780
|
|
2025-09-16 04:27:12,418: t15.2023.08.20 val PER: 0.0699
|
|
2025-09-16 04:27:12,418: t15.2023.08.25 val PER: 0.0813
|
|
2025-09-16 04:27:12,418: t15.2023.08.27 val PER: 0.1801
|
|
2025-09-16 04:27:12,418: t15.2023.09.01 val PER: 0.0544
|
|
2025-09-16 04:27:12,418: t15.2023.09.03 val PER: 0.1200
|
|
2025-09-16 04:27:12,418: t15.2023.09.24 val PER: 0.1019
|
|
2025-09-16 04:27:12,418: t15.2023.09.29 val PER: 0.1117
|
|
2025-09-16 04:27:12,418: t15.2023.10.01 val PER: 0.1493
|
|
2025-09-16 04:27:12,418: t15.2023.10.06 val PER: 0.0624
|
|
2025-09-16 04:27:12,418: t15.2023.10.08 val PER: 0.1922
|
|
2025-09-16 04:27:12,418: t15.2023.10.13 val PER: 0.1885
|
|
2025-09-16 04:27:12,418: t15.2023.10.15 val PER: 0.1173
|
|
2025-09-16 04:27:12,418: t15.2023.10.20 val PER: 0.1711
|
|
2025-09-16 04:27:12,419: t15.2023.10.22 val PER: 0.1002
|
|
2025-09-16 04:27:12,419: t15.2023.11.03 val PER: 0.1669
|
|
2025-09-16 04:27:12,419: t15.2023.11.04 val PER: 0.0068
|
|
2025-09-16 04:27:12,419: t15.2023.11.17 val PER: 0.0187
|
|
2025-09-16 04:27:12,419: t15.2023.11.19 val PER: 0.0180
|
|
2025-09-16 04:27:12,419: t15.2023.11.26 val PER: 0.0601
|
|
2025-09-16 04:27:12,419: t15.2023.12.03 val PER: 0.0630
|
|
2025-09-16 04:27:12,419: t15.2023.12.08 val PER: 0.0419
|
|
2025-09-16 04:27:12,419: t15.2023.12.10 val PER: 0.0420
|
|
2025-09-16 04:27:12,419: t15.2023.12.17 val PER: 0.0884
|
|
2025-09-16 04:27:12,419: t15.2023.12.29 val PER: 0.0789
|
|
2025-09-16 04:27:12,419: t15.2024.02.25 val PER: 0.0801
|
|
2025-09-16 04:27:12,419: t15.2024.03.08 val PER: 0.1735
|
|
2025-09-16 04:27:12,419: t15.2024.03.15 val PER: 0.1714
|
|
2025-09-16 04:27:12,419: t15.2024.03.17 val PER: 0.0823
|
|
2025-09-16 04:27:12,419: t15.2024.05.10 val PER: 0.1025
|
|
2025-09-16 04:27:12,419: t15.2024.06.14 val PER: 0.1215
|
|
2025-09-16 04:27:12,419: t15.2024.07.19 val PER: 0.1727
|
|
2025-09-16 04:27:12,419: t15.2024.07.21 val PER: 0.0586
|
|
2025-09-16 04:27:12,419: t15.2024.07.28 val PER: 0.0875
|
|
2025-09-16 04:27:12,420: t15.2025.01.10 val PER: 0.2562
|
|
2025-09-16 04:27:12,420: t15.2025.01.12 val PER: 0.0924
|
|
2025-09-16 04:27:12,420: t15.2025.03.14 val PER: 0.2766
|
|
2025-09-16 04:27:12,420: t15.2025.03.16 val PER: 0.1545
|
|
2025-09-16 04:27:12,420: t15.2025.03.30 val PER: 0.2391
|
|
2025-09-16 04:27:12,420: t15.2025.04.13 val PER: 0.1997
|
|
2025-09-16 04:27:40,458: Train batch 44200: loss: 0.29 grad norm: 5.36 time: 0.140
|
|
2025-09-16 04:28:13,662: Train batch 44400: loss: 0.04 grad norm: 3.82 time: 0.157
|
|
2025-09-16 04:28:46,360: Train batch 44600: loss: 0.01 grad norm: 0.64 time: 0.117
|
|
2025-09-16 04:29:19,929: Train batch 44800: loss: 0.03 grad norm: 2.59 time: 0.168
|
|
2025-09-16 04:29:53,655: Train batch 45000: loss: 0.01 grad norm: 0.43 time: 0.120
|
|
2025-09-16 04:30:27,108: Train batch 45200: loss: 0.03 grad norm: 3.00 time: 0.126
|
|
2025-09-16 04:30:59,727: Train batch 45400: loss: 0.27 grad norm: 5.20 time: 0.150
|
|
2025-09-16 04:31:32,501: Train batch 45600: loss: 0.07 grad norm: 4.80 time: 0.123
|
|
2025-09-16 04:32:05,690: Train batch 45800: loss: 0.02 grad norm: 1.46 time: 0.128
|
|
2025-09-16 04:32:39,891: Train batch 46000: loss: 0.01 grad norm: 2.07 time: 0.145
|
|
2025-09-16 04:32:39,892: Running test after training batch: 46000
|
|
2025-09-16 04:32:50,510: Val batch 46000: PER (avg): 0.1130 CTC Loss (avg): 27.0602 time: 10.618
|
|
2025-09-16 04:32:50,510: t15.2023.08.13 val PER: 0.0832
|
|
2025-09-16 04:32:50,510: t15.2023.08.18 val PER: 0.0863
|
|
2025-09-16 04:32:50,510: t15.2023.08.20 val PER: 0.0651
|
|
2025-09-16 04:32:50,511: t15.2023.08.25 val PER: 0.0964
|
|
2025-09-16 04:32:50,511: t15.2023.08.27 val PER: 0.1656
|
|
2025-09-16 04:32:50,511: t15.2023.09.01 val PER: 0.0487
|
|
2025-09-16 04:32:50,511: t15.2023.09.03 val PER: 0.1223
|
|
2025-09-16 04:32:50,511: t15.2023.09.24 val PER: 0.0947
|
|
2025-09-16 04:32:50,511: t15.2023.09.29 val PER: 0.1027
|
|
2025-09-16 04:32:50,511: t15.2023.10.01 val PER: 0.1473
|
|
2025-09-16 04:32:50,511: t15.2023.10.06 val PER: 0.0667
|
|
2025-09-16 04:32:50,511: t15.2023.10.08 val PER: 0.1949
|
|
2025-09-16 04:32:50,511: t15.2023.10.13 val PER: 0.1777
|
|
2025-09-16 04:32:50,511: t15.2023.10.15 val PER: 0.1121
|
|
2025-09-16 04:32:50,511: t15.2023.10.20 val PER: 0.1611
|
|
2025-09-16 04:32:50,511: t15.2023.10.22 val PER: 0.1013
|
|
2025-09-16 04:32:50,511: t15.2023.11.03 val PER: 0.1628
|
|
2025-09-16 04:32:50,511: t15.2023.11.04 val PER: 0.0034
|
|
2025-09-16 04:32:50,511: t15.2023.11.17 val PER: 0.0264
|
|
2025-09-16 04:32:50,511: t15.2023.11.19 val PER: 0.0140
|
|
2025-09-16 04:32:50,511: t15.2023.11.26 val PER: 0.0630
|
|
2025-09-16 04:32:50,511: t15.2023.12.03 val PER: 0.0578
|
|
2025-09-16 04:32:50,511: t15.2023.12.08 val PER: 0.0439
|
|
2025-09-16 04:32:50,512: t15.2023.12.10 val PER: 0.0473
|
|
2025-09-16 04:32:50,512: t15.2023.12.17 val PER: 0.0811
|
|
2025-09-16 04:32:50,512: t15.2023.12.29 val PER: 0.0817
|
|
2025-09-16 04:32:50,512: t15.2024.02.25 val PER: 0.0857
|
|
2025-09-16 04:32:50,512: t15.2024.03.08 val PER: 0.1849
|
|
2025-09-16 04:32:50,512: t15.2024.03.15 val PER: 0.1714
|
|
2025-09-16 04:32:50,512: t15.2024.03.17 val PER: 0.0816
|
|
2025-09-16 04:32:50,512: t15.2024.05.10 val PER: 0.1352
|
|
2025-09-16 04:32:50,512: t15.2024.06.14 val PER: 0.1120
|
|
2025-09-16 04:32:50,512: t15.2024.07.19 val PER: 0.1628
|
|
2025-09-16 04:32:50,512: t15.2024.07.21 val PER: 0.0538
|
|
2025-09-16 04:32:50,512: t15.2024.07.28 val PER: 0.0853
|
|
2025-09-16 04:32:50,512: t15.2025.01.10 val PER: 0.2645
|
|
2025-09-16 04:32:50,512: t15.2025.01.12 val PER: 0.0993
|
|
2025-09-16 04:32:50,512: t15.2025.03.14 val PER: 0.2825
|
|
2025-09-16 04:32:50,512: t15.2025.03.16 val PER: 0.1584
|
|
2025-09-16 04:32:50,512: t15.2025.03.30 val PER: 0.2333
|
|
2025-09-16 04:32:50,512: t15.2025.04.13 val PER: 0.2111
|
|
2025-09-16 04:33:18,409: Train batch 46200: loss: 0.01 grad norm: 0.67 time: 0.163
|
|
2025-09-16 04:33:51,829: Train batch 46400: loss: 0.01 grad norm: 0.51 time: 0.131
|
|
2025-09-16 04:34:25,747: Train batch 46600: loss: 0.01 grad norm: 1.22 time: 0.134
|
|
2025-09-16 04:34:58,737: Train batch 46800: loss: 0.02 grad norm: 2.25 time: 0.116
|
|
2025-09-16 04:35:31,833: Train batch 47000: loss: 0.01 grad norm: 0.93 time: 0.131
|
|
2025-09-16 04:36:04,819: Train batch 47200: loss: 0.02 grad norm: 1.32 time: 0.135
|
|
2025-09-16 04:36:37,999: Train batch 47400: loss: 0.01 grad norm: 1.13 time: 0.130
|
|
2025-09-16 04:37:11,870: Train batch 47600: loss: 0.14 grad norm: 7.83 time: 0.155
|
|
2025-09-16 04:37:45,629: Train batch 47800: loss: 0.03 grad norm: 5.38 time: 0.116
|
|
2025-09-16 04:38:19,000: Train batch 48000: loss: 0.01 grad norm: 0.46 time: 0.161
|
|
2025-09-16 04:38:19,000: Running test after training batch: 48000
|
|
2025-09-16 04:38:29,813: Val batch 48000: PER (avg): 0.1134 CTC Loss (avg): 27.7835 time: 10.813
|
|
2025-09-16 04:38:29,813: t15.2023.08.13 val PER: 0.0852
|
|
2025-09-16 04:38:29,813: t15.2023.08.18 val PER: 0.0821
|
|
2025-09-16 04:38:29,813: t15.2023.08.20 val PER: 0.0643
|
|
2025-09-16 04:38:29,814: t15.2023.08.25 val PER: 0.0889
|
|
2025-09-16 04:38:29,814: t15.2023.08.27 val PER: 0.1688
|
|
2025-09-16 04:38:29,814: t15.2023.09.01 val PER: 0.0544
|
|
2025-09-16 04:38:29,814: t15.2023.09.03 val PER: 0.1283
|
|
2025-09-16 04:38:29,814: t15.2023.09.24 val PER: 0.0995
|
|
2025-09-16 04:38:29,814: t15.2023.09.29 val PER: 0.1117
|
|
2025-09-16 04:38:29,814: t15.2023.10.01 val PER: 0.1413
|
|
2025-09-16 04:38:29,814: t15.2023.10.06 val PER: 0.0721
|
|
2025-09-16 04:38:29,814: t15.2023.10.08 val PER: 0.2070
|
|
2025-09-16 04:38:29,814: t15.2023.10.13 val PER: 0.1753
|
|
2025-09-16 04:38:29,814: t15.2023.10.15 val PER: 0.1167
|
|
2025-09-16 04:38:29,814: t15.2023.10.20 val PER: 0.1812
|
|
2025-09-16 04:38:29,814: t15.2023.10.22 val PER: 0.1024
|
|
2025-09-16 04:38:29,814: t15.2023.11.03 val PER: 0.1655
|
|
2025-09-16 04:38:29,814: t15.2023.11.04 val PER: 0.0000
|
|
2025-09-16 04:38:29,814: t15.2023.11.17 val PER: 0.0218
|
|
2025-09-16 04:38:29,814: t15.2023.11.19 val PER: 0.0100
|
|
2025-09-16 04:38:29,814: t15.2023.11.26 val PER: 0.0572
|
|
2025-09-16 04:38:29,814: t15.2023.12.03 val PER: 0.0704
|
|
2025-09-16 04:38:29,815: t15.2023.12.08 val PER: 0.0439
|
|
2025-09-16 04:38:29,815: t15.2023.12.10 val PER: 0.0473
|
|
2025-09-16 04:38:29,815: t15.2023.12.17 val PER: 0.0946
|
|
2025-09-16 04:38:29,815: t15.2023.12.29 val PER: 0.0830
|
|
2025-09-16 04:38:29,815: t15.2024.02.25 val PER: 0.0927
|
|
2025-09-16 04:38:29,815: t15.2024.03.08 val PER: 0.1778
|
|
2025-09-16 04:38:29,815: t15.2024.03.15 val PER: 0.1614
|
|
2025-09-16 04:38:29,815: t15.2024.03.17 val PER: 0.0774
|
|
2025-09-16 04:38:29,815: t15.2024.05.10 val PER: 0.1189
|
|
2025-09-16 04:38:29,815: t15.2024.06.14 val PER: 0.1104
|
|
2025-09-16 04:38:29,815: t15.2024.07.19 val PER: 0.1707
|
|
2025-09-16 04:38:29,815: t15.2024.07.21 val PER: 0.0614
|
|
2025-09-16 04:38:29,815: t15.2024.07.28 val PER: 0.0868
|
|
2025-09-16 04:38:29,815: t15.2025.01.10 val PER: 0.2631
|
|
2025-09-16 04:38:29,815: t15.2025.01.12 val PER: 0.0839
|
|
2025-09-16 04:38:29,815: t15.2025.03.14 val PER: 0.2707
|
|
2025-09-16 04:38:29,815: t15.2025.03.16 val PER: 0.1558
|
|
2025-09-16 04:38:29,815: t15.2025.03.30 val PER: 0.2379
|
|
2025-09-16 04:38:29,815: t15.2025.04.13 val PER: 0.2054
|
|
2025-09-16 04:38:58,127: Train batch 48200: loss: 0.01 grad norm: 0.67 time: 0.146
|
|
2025-09-16 04:39:30,985: Train batch 48400: loss: 0.00 grad norm: 0.25 time: 0.157
|
|
2025-09-16 04:40:04,570: Train batch 48600: loss: 0.02 grad norm: 1.15 time: 0.138
|
|
2025-09-16 04:40:38,765: Train batch 48800: loss: 0.02 grad norm: 0.86 time: 0.111
|
|
2025-09-16 04:41:11,313: Train batch 49000: loss: 0.21 grad norm: 11.50 time: 0.104
|
|
2025-09-16 04:41:45,280: Train batch 49200: loss: 0.05 grad norm: 2.14 time: 0.114
|
|
2025-09-16 04:42:18,602: Train batch 49400: loss: 0.02 grad norm: 2.38 time: 0.152
|
|
2025-09-16 04:42:52,016: Train batch 49600: loss: 0.06 grad norm: 4.95 time: 0.144
|
|
2025-09-16 04:43:25,077: Train batch 49800: loss: 0.02 grad norm: 1.62 time: 0.099
|
|
2025-09-16 04:43:58,647: Train batch 50000: loss: 0.05 grad norm: 5.03 time: 0.131
|
|
2025-09-16 04:43:58,647: Running test after training batch: 50000
|
|
2025-09-16 04:44:09,510: Val batch 50000: PER (avg): 0.1125 CTC Loss (avg): 27.6145 time: 10.862
|
|
2025-09-16 04:44:09,510: t15.2023.08.13 val PER: 0.0800
|
|
2025-09-16 04:44:09,510: t15.2023.08.18 val PER: 0.0763
|
|
2025-09-16 04:44:09,510: t15.2023.08.20 val PER: 0.0572
|
|
2025-09-16 04:44:09,510: t15.2023.08.25 val PER: 0.0798
|
|
2025-09-16 04:44:09,510: t15.2023.08.27 val PER: 0.1768
|
|
2025-09-16 04:44:09,510: t15.2023.09.01 val PER: 0.0528
|
|
2025-09-16 04:44:09,510: t15.2023.09.03 val PER: 0.1200
|
|
2025-09-16 04:44:09,510: t15.2023.09.24 val PER: 0.0983
|
|
2025-09-16 04:44:09,510: t15.2023.09.29 val PER: 0.1066
|
|
2025-09-16 04:44:09,510: t15.2023.10.01 val PER: 0.1552
|
|
2025-09-16 04:44:09,511: t15.2023.10.06 val PER: 0.0657
|
|
2025-09-16 04:44:09,511: t15.2023.10.08 val PER: 0.2084
|
|
2025-09-16 04:44:09,511: t15.2023.10.13 val PER: 0.1753
|
|
2025-09-16 04:44:09,511: t15.2023.10.15 val PER: 0.1147
|
|
2025-09-16 04:44:09,511: t15.2023.10.20 val PER: 0.1913
|
|
2025-09-16 04:44:09,511: t15.2023.10.22 val PER: 0.1002
|
|
2025-09-16 04:44:09,511: t15.2023.11.03 val PER: 0.1682
|
|
2025-09-16 04:44:09,511: t15.2023.11.04 val PER: 0.0102
|
|
2025-09-16 04:44:09,511: t15.2023.11.17 val PER: 0.0249
|
|
2025-09-16 04:44:09,511: t15.2023.11.19 val PER: 0.0200
|
|
2025-09-16 04:44:09,511: t15.2023.11.26 val PER: 0.0558
|
|
2025-09-16 04:44:09,511: t15.2023.12.03 val PER: 0.0651
|
|
2025-09-16 04:44:09,511: t15.2023.12.08 val PER: 0.0486
|
|
2025-09-16 04:44:09,511: t15.2023.12.10 val PER: 0.0460
|
|
2025-09-16 04:44:09,511: t15.2023.12.17 val PER: 0.0967
|
|
2025-09-16 04:44:09,511: t15.2023.12.29 val PER: 0.0728
|
|
2025-09-16 04:44:09,511: t15.2024.02.25 val PER: 0.0815
|
|
2025-09-16 04:44:09,511: t15.2024.03.08 val PER: 0.1807
|
|
2025-09-16 04:44:09,511: t15.2024.03.15 val PER: 0.1532
|
|
2025-09-16 04:44:09,512: t15.2024.03.17 val PER: 0.0767
|
|
2025-09-16 04:44:09,512: t15.2024.05.10 val PER: 0.1308
|
|
2025-09-16 04:44:09,512: t15.2024.06.14 val PER: 0.1151
|
|
2025-09-16 04:44:09,512: t15.2024.07.19 val PER: 0.1648
|
|
2025-09-16 04:44:09,512: t15.2024.07.21 val PER: 0.0628
|
|
2025-09-16 04:44:09,512: t15.2024.07.28 val PER: 0.0956
|
|
2025-09-16 04:44:09,512: t15.2025.01.10 val PER: 0.2672
|
|
2025-09-16 04:44:09,512: t15.2025.01.12 val PER: 0.0885
|
|
2025-09-16 04:44:09,512: t15.2025.03.14 val PER: 0.2811
|
|
2025-09-16 04:44:09,512: t15.2025.03.16 val PER: 0.1414
|
|
2025-09-16 04:44:09,512: t15.2025.03.30 val PER: 0.2310
|
|
2025-09-16 04:44:09,512: t15.2025.04.13 val PER: 0.1997
|
|
2025-09-16 04:44:09,512: New best test PER 0.1129 --> 0.1125
|
|
2025-09-16 04:44:09,512: Checkpointing model
|
|
2025-09-16 04:44:10,784: Saved model to checkpoint: trained_models/baseline_rnn/checkpoint/best_checkpoint
|
|
2025-09-16 04:44:38,112: Train batch 50200: loss: 0.04 grad norm: 9.85 time: 0.158
|
|
2025-09-16 04:45:10,189: Train batch 50400: loss: 0.02 grad norm: 1.29 time: 0.115
|
|
2025-09-16 04:45:43,113: Train batch 50600: loss: 0.07 grad norm: 6.65 time: 0.116
|
|
2025-09-16 04:46:15,820: Train batch 50800: loss: 0.07 grad norm: 2.25 time: 0.122
|
|
2025-09-16 04:46:49,838: Train batch 51000: loss: 0.03 grad norm: 3.01 time: 0.168
|
|
2025-09-16 04:47:22,467: Train batch 51200: loss: 0.03 grad norm: 1.71 time: 0.155
|
|
2025-09-16 04:47:54,622: Train batch 51400: loss: 0.04 grad norm: 3.06 time: 0.102
|
|
2025-09-16 04:48:27,898: Train batch 51600: loss: 0.05 grad norm: 4.57 time: 0.164
|
|
2025-09-16 04:49:00,498: Train batch 51800: loss: 0.04 grad norm: 3.32 time: 0.113
|
|
2025-09-16 04:49:33,627: Train batch 52000: loss: 0.01 grad norm: 0.61 time: 0.102
|
|
2025-09-16 04:49:33,627: Running test after training batch: 52000
|
|
2025-09-16 04:49:44,496: Val batch 52000: PER (avg): 0.1123 CTC Loss (avg): 27.8901 time: 10.868
|
|
2025-09-16 04:49:44,496: t15.2023.08.13 val PER: 0.0842
|
|
2025-09-16 04:49:44,496: t15.2023.08.18 val PER: 0.0830
|
|
2025-09-16 04:49:44,496: t15.2023.08.20 val PER: 0.0651
|
|
2025-09-16 04:49:44,496: t15.2023.08.25 val PER: 0.0994
|
|
2025-09-16 04:49:44,496: t15.2023.08.27 val PER: 0.1640
|
|
2025-09-16 04:49:44,496: t15.2023.09.01 val PER: 0.0576
|
|
2025-09-16 04:49:44,496: t15.2023.09.03 val PER: 0.1200
|
|
2025-09-16 04:49:44,496: t15.2023.09.24 val PER: 0.0971
|
|
2025-09-16 04:49:44,496: t15.2023.09.29 val PER: 0.1053
|
|
2025-09-16 04:49:44,496: t15.2023.10.01 val PER: 0.1413
|
|
2025-09-16 04:49:44,496: t15.2023.10.06 val PER: 0.0592
|
|
2025-09-16 04:49:44,496: t15.2023.10.08 val PER: 0.1935
|
|
2025-09-16 04:49:44,497: t15.2023.10.13 val PER: 0.1707
|
|
2025-09-16 04:49:44,497: t15.2023.10.15 val PER: 0.1121
|
|
2025-09-16 04:49:44,497: t15.2023.10.20 val PER: 0.1779
|
|
2025-09-16 04:49:44,497: t15.2023.10.22 val PER: 0.0991
|
|
2025-09-16 04:49:44,497: t15.2023.11.03 val PER: 0.1588
|
|
2025-09-16 04:49:44,497: t15.2023.11.04 val PER: 0.0137
|
|
2025-09-16 04:49:44,497: t15.2023.11.17 val PER: 0.0249
|
|
2025-09-16 04:49:44,497: t15.2023.11.19 val PER: 0.0140
|
|
2025-09-16 04:49:44,497: t15.2023.11.26 val PER: 0.0638
|
|
2025-09-16 04:49:44,497: t15.2023.12.03 val PER: 0.0672
|
|
2025-09-16 04:49:44,497: t15.2023.12.08 val PER: 0.0393
|
|
2025-09-16 04:49:44,497: t15.2023.12.10 val PER: 0.0434
|
|
2025-09-16 04:49:44,497: t15.2023.12.17 val PER: 0.1154
|
|
2025-09-16 04:49:44,497: t15.2023.12.29 val PER: 0.0741
|
|
2025-09-16 04:49:44,497: t15.2024.02.25 val PER: 0.0927
|
|
2025-09-16 04:49:44,497: t15.2024.03.08 val PER: 0.1821
|
|
2025-09-16 04:49:44,497: t15.2024.03.15 val PER: 0.1570
|
|
2025-09-16 04:49:44,497: t15.2024.03.17 val PER: 0.0816
|
|
2025-09-16 04:49:44,497: t15.2024.05.10 val PER: 0.1426
|
|
2025-09-16 04:49:44,497: t15.2024.06.14 val PER: 0.1151
|
|
2025-09-16 04:49:44,498: t15.2024.07.19 val PER: 0.1740
|
|
2025-09-16 04:49:44,498: t15.2024.07.21 val PER: 0.0607
|
|
2025-09-16 04:49:44,498: t15.2024.07.28 val PER: 0.0949
|
|
2025-09-16 04:49:44,498: t15.2025.01.10 val PER: 0.2438
|
|
2025-09-16 04:49:44,498: t15.2025.01.12 val PER: 0.0816
|
|
2025-09-16 04:49:44,498: t15.2025.03.14 val PER: 0.2751
|
|
2025-09-16 04:49:44,498: t15.2025.03.16 val PER: 0.1479
|
|
2025-09-16 04:49:44,498: t15.2025.03.30 val PER: 0.2230
|
|
2025-09-16 04:49:44,498: t15.2025.04.13 val PER: 0.2054
|
|
2025-09-16 04:49:44,498: New best test PER 0.1125 --> 0.1123
|
|
2025-09-16 04:49:44,498: Checkpointing model
|
|
2025-09-16 04:49:45,909: Saved model to checkpoint: trained_models/baseline_rnn/checkpoint/best_checkpoint
|
|
2025-09-16 04:50:13,878: Train batch 52200: loss: 0.03 grad norm: 2.44 time: 0.097
|
|
2025-09-16 04:50:46,351: Train batch 52400: loss: 0.07 grad norm: 6.89 time: 0.135
|
|
2025-09-16 04:51:18,863: Train batch 52600: loss: 0.02 grad norm: 1.66 time: 0.151
|
|
2025-09-16 04:51:52,109: Train batch 52800: loss: 0.02 grad norm: 2.13 time: 0.130
|
|
2025-09-16 04:52:25,170: Train batch 53000: loss: 0.01 grad norm: 0.32 time: 0.121
|
|
2025-09-16 04:52:57,779: Train batch 53200: loss: 0.02 grad norm: 3.73 time: 0.131
|
|
2025-09-16 04:53:31,215: Train batch 53400: loss: 0.01 grad norm: 1.47 time: 0.136
|
|
2025-09-16 04:54:03,921: Train batch 53600: loss: 0.06 grad norm: 4.02 time: 0.126
|
|
2025-09-16 04:54:36,556: Train batch 53800: loss: 0.02 grad norm: 2.31 time: 0.161
|
|
2025-09-16 04:55:09,297: Train batch 54000: loss: 0.01 grad norm: 0.42 time: 0.134
|
|
2025-09-16 04:55:09,297: Running test after training batch: 54000
|
|
2025-09-16 04:55:20,224: Val batch 54000: PER (avg): 0.1127 CTC Loss (avg): 27.4180 time: 10.927
|
|
2025-09-16 04:55:20,224: t15.2023.08.13 val PER: 0.0800
|
|
2025-09-16 04:55:20,224: t15.2023.08.18 val PER: 0.0821
|
|
2025-09-16 04:55:20,224: t15.2023.08.20 val PER: 0.0620
|
|
2025-09-16 04:55:20,224: t15.2023.08.25 val PER: 0.0843
|
|
2025-09-16 04:55:20,224: t15.2023.08.27 val PER: 0.1608
|
|
2025-09-16 04:55:20,224: t15.2023.09.01 val PER: 0.0503
|
|
2025-09-16 04:55:20,224: t15.2023.09.03 val PER: 0.1247
|
|
2025-09-16 04:55:20,224: t15.2023.09.24 val PER: 0.1092
|
|
2025-09-16 04:55:20,225: t15.2023.09.29 val PER: 0.1034
|
|
2025-09-16 04:55:20,225: t15.2023.10.01 val PER: 0.1453
|
|
2025-09-16 04:55:20,225: t15.2023.10.06 val PER: 0.0635
|
|
2025-09-16 04:55:20,225: t15.2023.10.08 val PER: 0.1962
|
|
2025-09-16 04:55:20,225: t15.2023.10.13 val PER: 0.1668
|
|
2025-09-16 04:55:20,225: t15.2023.10.15 val PER: 0.1114
|
|
2025-09-16 04:55:20,225: t15.2023.10.20 val PER: 0.1644
|
|
2025-09-16 04:55:20,225: t15.2023.10.22 val PER: 0.0980
|
|
2025-09-16 04:55:20,225: t15.2023.11.03 val PER: 0.1655
|
|
2025-09-16 04:55:20,225: t15.2023.11.04 val PER: 0.0102
|
|
2025-09-16 04:55:20,225: t15.2023.11.17 val PER: 0.0264
|
|
2025-09-16 04:55:20,225: t15.2023.11.19 val PER: 0.0140
|
|
2025-09-16 04:55:20,225: t15.2023.11.26 val PER: 0.0572
|
|
2025-09-16 04:55:20,225: t15.2023.12.03 val PER: 0.0651
|
|
2025-09-16 04:55:20,225: t15.2023.12.08 val PER: 0.0413
|
|
2025-09-16 04:55:20,225: t15.2023.12.10 val PER: 0.0434
|
|
2025-09-16 04:55:20,226: t15.2023.12.17 val PER: 0.1091
|
|
2025-09-16 04:55:20,226: t15.2023.12.29 val PER: 0.0810
|
|
2025-09-16 04:55:20,226: t15.2024.02.25 val PER: 0.0899
|
|
2025-09-16 04:55:20,226: t15.2024.03.08 val PER: 0.1991
|
|
2025-09-16 04:55:20,226: t15.2024.03.15 val PER: 0.1620
|
|
2025-09-16 04:55:20,226: t15.2024.03.17 val PER: 0.0816
|
|
2025-09-16 04:55:20,226: t15.2024.05.10 val PER: 0.1382
|
|
2025-09-16 04:55:20,226: t15.2024.06.14 val PER: 0.1167
|
|
2025-09-16 04:55:20,226: t15.2024.07.19 val PER: 0.1714
|
|
2025-09-16 04:55:20,226: t15.2024.07.21 val PER: 0.0614
|
|
2025-09-16 04:55:20,226: t15.2024.07.28 val PER: 0.0801
|
|
2025-09-16 04:55:20,226: t15.2025.01.10 val PER: 0.2521
|
|
2025-09-16 04:55:20,226: t15.2025.01.12 val PER: 0.0824
|
|
2025-09-16 04:55:20,226: t15.2025.03.14 val PER: 0.3018
|
|
2025-09-16 04:55:20,226: t15.2025.03.16 val PER: 0.1466
|
|
2025-09-16 04:55:20,226: t15.2025.03.30 val PER: 0.2356
|
|
2025-09-16 04:55:20,226: t15.2025.04.13 val PER: 0.2054
|
|
2025-09-16 04:55:48,010: Train batch 54200: loss: 0.05 grad norm: 3.07 time: 0.118
|
|
2025-09-16 04:56:20,956: Train batch 54400: loss: 0.05 grad norm: 3.80 time: 0.141
|
|
2025-09-16 04:56:54,188: Train batch 54600: loss: 0.02 grad norm: 1.39 time: 0.125
|
|
2025-09-16 04:57:27,186: Train batch 54800: loss: 0.02 grad norm: 1.64 time: 0.116
|
|
2025-09-16 04:58:00,551: Train batch 55000: loss: 0.02 grad norm: 1.67 time: 0.110
|
|
2025-09-16 04:58:33,860: Train batch 55200: loss: 0.03 grad norm: 1.61 time: 0.145
|
|
2025-09-16 04:59:07,932: Train batch 55400: loss: 0.04 grad norm: 2.78 time: 0.140
|
|
2025-09-16 04:59:40,468: Train batch 55600: loss: 0.02 grad norm: 1.58 time: 0.154
|
|
2025-09-16 05:00:13,848: Train batch 55800: loss: 0.01 grad norm: 0.86 time: 0.124
|
|
2025-09-16 05:00:47,466: Train batch 56000: loss: 0.01 grad norm: 0.99 time: 0.125
|
|
2025-09-16 05:00:47,466: Running test after training batch: 56000
|
|
2025-09-16 05:00:57,949: Val batch 56000: PER (avg): 0.1130 CTC Loss (avg): 27.7827 time: 10.482
|
|
2025-09-16 05:00:57,949: t15.2023.08.13 val PER: 0.0790
|
|
2025-09-16 05:00:57,949: t15.2023.08.18 val PER: 0.0838
|
|
2025-09-16 05:00:57,949: t15.2023.08.20 val PER: 0.0620
|
|
2025-09-16 05:00:57,949: t15.2023.08.25 val PER: 0.0964
|
|
2025-09-16 05:00:57,949: t15.2023.08.27 val PER: 0.1720
|
|
2025-09-16 05:00:57,949: t15.2023.09.01 val PER: 0.0584
|
|
2025-09-16 05:00:57,949: t15.2023.09.03 val PER: 0.1342
|
|
2025-09-16 05:00:57,949: t15.2023.09.24 val PER: 0.0995
|
|
2025-09-16 05:00:57,950: t15.2023.09.29 val PER: 0.1059
|
|
2025-09-16 05:00:57,950: t15.2023.10.01 val PER: 0.1519
|
|
2025-09-16 05:00:57,950: t15.2023.10.06 val PER: 0.0743
|
|
2025-09-16 05:00:57,950: t15.2023.10.08 val PER: 0.1881
|
|
2025-09-16 05:00:57,950: t15.2023.10.13 val PER: 0.1660
|
|
2025-09-16 05:00:57,950: t15.2023.10.15 val PER: 0.1154
|
|
2025-09-16 05:00:57,950: t15.2023.10.20 val PER: 0.1913
|
|
2025-09-16 05:00:57,950: t15.2023.10.22 val PER: 0.0991
|
|
2025-09-16 05:00:57,950: t15.2023.11.03 val PER: 0.1642
|
|
2025-09-16 05:00:57,950: t15.2023.11.04 val PER: 0.0068
|
|
2025-09-16 05:00:57,950: t15.2023.11.17 val PER: 0.0233
|
|
2025-09-16 05:00:57,950: t15.2023.11.19 val PER: 0.0140
|
|
2025-09-16 05:00:57,950: t15.2023.11.26 val PER: 0.0594
|
|
2025-09-16 05:00:57,950: t15.2023.12.03 val PER: 0.0630
|
|
2025-09-16 05:00:57,950: t15.2023.12.08 val PER: 0.0433
|
|
2025-09-16 05:00:57,950: t15.2023.12.10 val PER: 0.0407
|
|
2025-09-16 05:00:57,950: t15.2023.12.17 val PER: 0.0915
|
|
2025-09-16 05:00:57,950: t15.2023.12.29 val PER: 0.0858
|
|
2025-09-16 05:00:57,950: t15.2024.02.25 val PER: 0.0913
|
|
2025-09-16 05:00:57,950: t15.2024.03.08 val PER: 0.1650
|
|
2025-09-16 05:00:57,951: t15.2024.03.15 val PER: 0.1695
|
|
2025-09-16 05:00:57,951: t15.2024.03.17 val PER: 0.0788
|
|
2025-09-16 05:00:57,951: t15.2024.05.10 val PER: 0.1263
|
|
2025-09-16 05:00:57,951: t15.2024.06.14 val PER: 0.1262
|
|
2025-09-16 05:00:57,951: t15.2024.07.19 val PER: 0.1773
|
|
2025-09-16 05:00:57,951: t15.2024.07.21 val PER: 0.0614
|
|
2025-09-16 05:00:57,951: t15.2024.07.28 val PER: 0.0816
|
|
2025-09-16 05:00:57,951: t15.2025.01.10 val PER: 0.2645
|
|
2025-09-16 05:00:57,951: t15.2025.01.12 val PER: 0.0808
|
|
2025-09-16 05:00:57,951: t15.2025.03.14 val PER: 0.2796
|
|
2025-09-16 05:00:57,951: t15.2025.03.16 val PER: 0.1387
|
|
2025-09-16 05:00:57,951: t15.2025.03.30 val PER: 0.2310
|
|
2025-09-16 05:00:57,951: t15.2025.04.13 val PER: 0.1983
|
|
2025-09-16 05:01:26,430: Train batch 56200: loss: 0.02 grad norm: 1.23 time: 0.140
|
|
2025-09-16 05:01:59,025: Train batch 56400: loss: 0.01 grad norm: 0.58 time: 0.154
|
|
2025-09-16 05:02:32,263: Train batch 56600: loss: 0.11 grad norm: 5.42 time: 0.112
|
|
2025-09-16 05:03:04,250: Train batch 56800: loss: 0.01 grad norm: 0.64 time: 0.097
|
|
2025-09-16 05:03:38,244: Train batch 57000: loss: 0.03 grad norm: 3.20 time: 0.136
|
|
2025-09-16 05:04:11,398: Train batch 57200: loss: 0.01 grad norm: 0.78 time: 0.116
|
|
2025-09-16 05:04:44,248: Train batch 57400: loss: 0.01 grad norm: 0.32 time: 0.106
|
|
2025-09-16 05:05:17,712: Train batch 57600: loss: 0.01 grad norm: 1.07 time: 0.115
|
|
2025-09-16 05:05:50,379: Train batch 57800: loss: 0.02 grad norm: 1.85 time: 0.125
|
|
2025-09-16 05:06:23,218: Train batch 58000: loss: 0.02 grad norm: 2.58 time: 0.115
|
|
2025-09-16 05:06:23,218: Running test after training batch: 58000
|
|
2025-09-16 05:06:33,608: Val batch 58000: PER (avg): 0.1135 CTC Loss (avg): 28.0092 time: 10.389
|
|
2025-09-16 05:06:33,608: t15.2023.08.13 val PER: 0.0873
|
|
2025-09-16 05:06:33,608: t15.2023.08.18 val PER: 0.0821
|
|
2025-09-16 05:06:33,608: t15.2023.08.20 val PER: 0.0643
|
|
2025-09-16 05:06:33,608: t15.2023.08.25 val PER: 0.0949
|
|
2025-09-16 05:06:33,608: t15.2023.08.27 val PER: 0.1752
|
|
2025-09-16 05:06:33,608: t15.2023.09.01 val PER: 0.0544
|
|
2025-09-16 05:06:33,609: t15.2023.09.03 val PER: 0.1259
|
|
2025-09-16 05:06:33,609: t15.2023.09.24 val PER: 0.0995
|
|
2025-09-16 05:06:33,609: t15.2023.09.29 val PER: 0.1008
|
|
2025-09-16 05:06:33,609: t15.2023.10.01 val PER: 0.1446
|
|
2025-09-16 05:06:33,609: t15.2023.10.06 val PER: 0.0614
|
|
2025-09-16 05:06:33,609: t15.2023.10.08 val PER: 0.1935
|
|
2025-09-16 05:06:33,609: t15.2023.10.13 val PER: 0.1676
|
|
2025-09-16 05:06:33,609: t15.2023.10.15 val PER: 0.1107
|
|
2025-09-16 05:06:33,609: t15.2023.10.20 val PER: 0.2013
|
|
2025-09-16 05:06:33,609: t15.2023.10.22 val PER: 0.1024
|
|
2025-09-16 05:06:33,609: t15.2023.11.03 val PER: 0.1676
|
|
2025-09-16 05:06:33,609: t15.2023.11.04 val PER: 0.0102
|
|
2025-09-16 05:06:33,609: t15.2023.11.17 val PER: 0.0233
|
|
2025-09-16 05:06:33,609: t15.2023.11.19 val PER: 0.0180
|
|
2025-09-16 05:06:33,609: t15.2023.11.26 val PER: 0.0638
|
|
2025-09-16 05:06:33,609: t15.2023.12.03 val PER: 0.0662
|
|
2025-09-16 05:06:33,609: t15.2023.12.08 val PER: 0.0499
|
|
2025-09-16 05:06:33,609: t15.2023.12.10 val PER: 0.0381
|
|
2025-09-16 05:06:33,609: t15.2023.12.17 val PER: 0.0967
|
|
2025-09-16 05:06:33,610: t15.2023.12.29 val PER: 0.0865
|
|
2025-09-16 05:06:33,610: t15.2024.02.25 val PER: 0.0871
|
|
2025-09-16 05:06:33,610: t15.2024.03.08 val PER: 0.1721
|
|
2025-09-16 05:06:33,610: t15.2024.03.15 val PER: 0.1645
|
|
2025-09-16 05:06:33,610: t15.2024.03.17 val PER: 0.0683
|
|
2025-09-16 05:06:33,610: t15.2024.05.10 val PER: 0.1278
|
|
2025-09-16 05:06:33,610: t15.2024.06.14 val PER: 0.1199
|
|
2025-09-16 05:06:33,610: t15.2024.07.19 val PER: 0.1793
|
|
2025-09-16 05:06:33,610: t15.2024.07.21 val PER: 0.0586
|
|
2025-09-16 05:06:33,610: t15.2024.07.28 val PER: 0.0882
|
|
2025-09-16 05:06:33,610: t15.2025.01.10 val PER: 0.2534
|
|
2025-09-16 05:06:33,610: t15.2025.01.12 val PER: 0.0870
|
|
2025-09-16 05:06:33,610: t15.2025.03.14 val PER: 0.2811
|
|
2025-09-16 05:06:33,610: t15.2025.03.16 val PER: 0.1518
|
|
2025-09-16 05:06:33,610: t15.2025.03.30 val PER: 0.2425
|
|
2025-09-16 05:06:33,610: t15.2025.04.13 val PER: 0.2140
|
|
2025-09-16 05:07:01,281: Train batch 58200: loss: 0.02 grad norm: 1.45 time: 0.114
|
|
2025-09-16 05:07:33,946: Train batch 58400: loss: 0.01 grad norm: 0.57 time: 0.152
|
|
2025-09-16 05:08:05,989: Train batch 58600: loss: 0.01 grad norm: 0.53 time: 0.157
|
|
2025-09-16 05:08:38,806: Train batch 58800: loss: 0.01 grad norm: 0.41 time: 0.102
|
|
2025-09-16 05:09:10,461: Train batch 59000: loss: 0.01 grad norm: 0.97 time: 0.154
|
|
2025-09-16 05:09:43,628: Train batch 59200: loss: 0.02 grad norm: 3.16 time: 0.131
|
|
2025-09-16 05:10:16,777: Train batch 59400: loss: 0.03 grad norm: 3.51 time: 0.124
|
|
2025-09-16 05:10:50,384: Train batch 59600: loss: 0.03 grad norm: 3.35 time: 0.152
|
|
2025-09-16 05:11:23,194: Train batch 59800: loss: 0.01 grad norm: 0.71 time: 0.143
|
|
2025-09-16 05:11:55,198: Train batch 60000: loss: 0.01 grad norm: 0.32 time: 0.150
|
|
2025-09-16 05:11:55,199: Running test after training batch: 60000
|
|
2025-09-16 05:12:06,176: Val batch 60000: PER (avg): 0.1114 CTC Loss (avg): 27.5786 time: 10.977
|
|
2025-09-16 05:12:06,177: t15.2023.08.13 val PER: 0.0800
|
|
2025-09-16 05:12:06,177: t15.2023.08.18 val PER: 0.0805
|
|
2025-09-16 05:12:06,177: t15.2023.08.20 val PER: 0.0612
|
|
2025-09-16 05:12:06,177: t15.2023.08.25 val PER: 0.0949
|
|
2025-09-16 05:12:06,177: t15.2023.08.27 val PER: 0.1833
|
|
2025-09-16 05:12:06,177: t15.2023.09.01 val PER: 0.0471
|
|
2025-09-16 05:12:06,177: t15.2023.09.03 val PER: 0.1093
|
|
2025-09-16 05:12:06,177: t15.2023.09.24 val PER: 0.0959
|
|
2025-09-16 05:12:06,177: t15.2023.09.29 val PER: 0.1015
|
|
2025-09-16 05:12:06,177: t15.2023.10.01 val PER: 0.1513
|
|
2025-09-16 05:12:06,177: t15.2023.10.06 val PER: 0.0614
|
|
2025-09-16 05:12:06,177: t15.2023.10.08 val PER: 0.1935
|
|
2025-09-16 05:12:06,177: t15.2023.10.13 val PER: 0.1707
|
|
2025-09-16 05:12:06,177: t15.2023.10.15 val PER: 0.1074
|
|
2025-09-16 05:12:06,177: t15.2023.10.20 val PER: 0.1779
|
|
2025-09-16 05:12:06,177: t15.2023.10.22 val PER: 0.1013
|
|
2025-09-16 05:12:06,177: t15.2023.11.03 val PER: 0.1642
|
|
2025-09-16 05:12:06,177: t15.2023.11.04 val PER: 0.0068
|
|
2025-09-16 05:12:06,178: t15.2023.11.17 val PER: 0.0218
|
|
2025-09-16 05:12:06,178: t15.2023.11.19 val PER: 0.0120
|
|
2025-09-16 05:12:06,178: t15.2023.11.26 val PER: 0.0594
|
|
2025-09-16 05:12:06,178: t15.2023.12.03 val PER: 0.0704
|
|
2025-09-16 05:12:06,178: t15.2023.12.08 val PER: 0.0419
|
|
2025-09-16 05:12:06,178: t15.2023.12.10 val PER: 0.0460
|
|
2025-09-16 05:12:06,178: t15.2023.12.17 val PER: 0.0925
|
|
2025-09-16 05:12:06,178: t15.2023.12.29 val PER: 0.0782
|
|
2025-09-16 05:12:06,178: t15.2024.02.25 val PER: 0.0857
|
|
2025-09-16 05:12:06,178: t15.2024.03.08 val PER: 0.1679
|
|
2025-09-16 05:12:06,178: t15.2024.03.15 val PER: 0.1670
|
|
2025-09-16 05:12:06,178: t15.2024.03.17 val PER: 0.0760
|
|
2025-09-16 05:12:06,178: t15.2024.05.10 val PER: 0.1456
|
|
2025-09-16 05:12:06,178: t15.2024.06.14 val PER: 0.1041
|
|
2025-09-16 05:12:06,178: t15.2024.07.19 val PER: 0.1760
|
|
2025-09-16 05:12:06,178: t15.2024.07.21 val PER: 0.0552
|
|
2025-09-16 05:12:06,178: t15.2024.07.28 val PER: 0.0919
|
|
2025-09-16 05:12:06,178: t15.2025.01.10 val PER: 0.2534
|
|
2025-09-16 05:12:06,178: t15.2025.01.12 val PER: 0.0893
|
|
2025-09-16 05:12:06,179: t15.2025.03.14 val PER: 0.2633
|
|
2025-09-16 05:12:06,179: t15.2025.03.16 val PER: 0.1505
|
|
2025-09-16 05:12:06,179: t15.2025.03.30 val PER: 0.2264
|
|
2025-09-16 05:12:06,179: t15.2025.04.13 val PER: 0.2083
|
|
2025-09-16 05:12:06,179: New best test PER 0.1123 --> 0.1114
|
|
2025-09-16 05:12:06,179: Checkpointing model
|
|
2025-09-16 05:12:07,499: Saved model to checkpoint: trained_models/baseline_rnn/checkpoint/best_checkpoint
|
|
2025-09-16 05:12:35,038: Train batch 60200: loss: 0.04 grad norm: 3.54 time: 0.159
|
|
2025-09-16 05:13:07,104: Train batch 60400: loss: 0.02 grad norm: 2.00 time: 0.136
|
|
2025-09-16 05:13:40,132: Train batch 60600: loss: 0.02 grad norm: 1.15 time: 0.140
|
|
2025-09-16 05:14:12,971: Train batch 60800: loss: 0.01 grad norm: 0.70 time: 0.128
|
|
2025-09-16 05:14:46,171: Train batch 61000: loss: 0.01 grad norm: 0.72 time: 0.110
|
|
2025-09-16 05:15:18,731: Train batch 61200: loss: 0.01 grad norm: 0.29 time: 0.155
|
|
2025-09-16 05:15:52,809: Train batch 61400: loss: 0.00 grad norm: 0.16 time: 0.123
|
|
2025-09-16 05:16:26,513: Train batch 61600: loss: 0.03 grad norm: 6.25 time: 0.111
|
|
2025-09-16 05:17:00,201: Train batch 61800: loss: 0.01 grad norm: 0.67 time: 0.143
|
|
2025-09-16 05:17:32,452: Train batch 62000: loss: 0.01 grad norm: 0.63 time: 0.108
|
|
2025-09-16 05:17:32,452: Running test after training batch: 62000
|
|
2025-09-16 05:17:43,309: Val batch 62000: PER (avg): 0.1097 CTC Loss (avg): 26.6828 time: 10.857
|
|
2025-09-16 05:17:43,309: t15.2023.08.13 val PER: 0.0884
|
|
2025-09-16 05:17:43,309: t15.2023.08.18 val PER: 0.0754
|
|
2025-09-16 05:17:43,309: t15.2023.08.20 val PER: 0.0620
|
|
2025-09-16 05:17:43,309: t15.2023.08.25 val PER: 0.0919
|
|
2025-09-16 05:17:43,309: t15.2023.08.27 val PER: 0.1688
|
|
2025-09-16 05:17:43,309: t15.2023.09.01 val PER: 0.0536
|
|
2025-09-16 05:17:43,309: t15.2023.09.03 val PER: 0.1140
|
|
2025-09-16 05:17:43,310: t15.2023.09.24 val PER: 0.0983
|
|
2025-09-16 05:17:43,310: t15.2023.09.29 val PER: 0.1040
|
|
2025-09-16 05:17:43,310: t15.2023.10.01 val PER: 0.1407
|
|
2025-09-16 05:17:43,310: t15.2023.10.06 val PER: 0.0506
|
|
2025-09-16 05:17:43,310: t15.2023.10.08 val PER: 0.1962
|
|
2025-09-16 05:17:43,310: t15.2023.10.13 val PER: 0.1761
|
|
2025-09-16 05:17:43,310: t15.2023.10.15 val PER: 0.1134
|
|
2025-09-16 05:17:43,310: t15.2023.10.20 val PER: 0.1779
|
|
2025-09-16 05:17:43,310: t15.2023.10.22 val PER: 0.1058
|
|
2025-09-16 05:17:43,310: t15.2023.11.03 val PER: 0.1581
|
|
2025-09-16 05:17:43,310: t15.2023.11.04 val PER: 0.0068
|
|
2025-09-16 05:17:43,310: t15.2023.11.17 val PER: 0.0218
|
|
2025-09-16 05:17:43,310: t15.2023.11.19 val PER: 0.0140
|
|
2025-09-16 05:17:43,310: t15.2023.11.26 val PER: 0.0551
|
|
2025-09-16 05:17:43,310: t15.2023.12.03 val PER: 0.0672
|
|
2025-09-16 05:17:43,310: t15.2023.12.08 val PER: 0.0433
|
|
2025-09-16 05:17:43,310: t15.2023.12.10 val PER: 0.0486
|
|
2025-09-16 05:17:43,311: t15.2023.12.17 val PER: 0.0894
|
|
2025-09-16 05:17:43,311: t15.2023.12.29 val PER: 0.0714
|
|
2025-09-16 05:17:43,311: t15.2024.02.25 val PER: 0.0815
|
|
2025-09-16 05:17:43,311: t15.2024.03.08 val PER: 0.1536
|
|
2025-09-16 05:17:43,311: t15.2024.03.15 val PER: 0.1614
|
|
2025-09-16 05:17:43,311: t15.2024.03.17 val PER: 0.0802
|
|
2025-09-16 05:17:43,311: t15.2024.05.10 val PER: 0.1263
|
|
2025-09-16 05:17:43,311: t15.2024.06.14 val PER: 0.1104
|
|
2025-09-16 05:17:43,311: t15.2024.07.19 val PER: 0.1694
|
|
2025-09-16 05:17:43,311: t15.2024.07.21 val PER: 0.0552
|
|
2025-09-16 05:17:43,311: t15.2024.07.28 val PER: 0.0801
|
|
2025-09-16 05:17:43,311: t15.2025.01.10 val PER: 0.2534
|
|
2025-09-16 05:17:43,311: t15.2025.01.12 val PER: 0.0847
|
|
2025-09-16 05:17:43,311: t15.2025.03.14 val PER: 0.2737
|
|
2025-09-16 05:17:43,311: t15.2025.03.16 val PER: 0.1466
|
|
2025-09-16 05:17:43,311: t15.2025.03.30 val PER: 0.2218
|
|
2025-09-16 05:17:43,311: t15.2025.04.13 val PER: 0.2140
|
|
2025-09-16 05:17:43,312: New best test PER 0.1114 --> 0.1097
|
|
2025-09-16 05:17:43,312: Checkpointing model
|
|
2025-09-16 05:17:44,703: Saved model to checkpoint: trained_models/baseline_rnn/checkpoint/best_checkpoint
|
|
2025-09-16 05:18:12,869: Train batch 62200: loss: 0.01 grad norm: 1.44 time: 0.176
|
|
2025-09-16 05:18:46,905: Train batch 62400: loss: 0.01 grad norm: 0.67 time: 0.148
|
|
2025-09-16 05:19:21,348: Train batch 62600: loss: 0.02 grad norm: 1.50 time: 0.096
|
|
2025-09-16 05:19:55,762: Train batch 62800: loss: 0.01 grad norm: 0.76 time: 0.122
|
|
2025-09-16 05:20:30,910: Train batch 63000: loss: 0.01 grad norm: 0.64 time: 0.114
|
|
2025-09-16 05:21:05,207: Train batch 63200: loss: 0.00 grad norm: 0.33 time: 0.125
|
|
2025-09-16 05:21:39,216: Train batch 63400: loss: 0.02 grad norm: 1.86 time: 0.125
|
|
2025-09-16 05:22:13,393: Train batch 63600: loss: 0.01 grad norm: 0.71 time: 0.106
|
|
2025-09-16 05:22:46,171: Train batch 63800: loss: 0.00 grad norm: 0.12 time: 0.153
|
|
2025-09-16 05:23:18,960: Train batch 64000: loss: 0.01 grad norm: 0.37 time: 0.148
|
|
2025-09-16 05:23:18,961: Running test after training batch: 64000
|
|
2025-09-16 05:23:30,135: Val batch 64000: PER (avg): 0.1104 CTC Loss (avg): 27.1472 time: 11.174
|
|
2025-09-16 05:23:30,135: t15.2023.08.13 val PER: 0.0915
|
|
2025-09-16 05:23:30,135: t15.2023.08.18 val PER: 0.0788
|
|
2025-09-16 05:23:30,135: t15.2023.08.20 val PER: 0.0627
|
|
2025-09-16 05:23:30,135: t15.2023.08.25 val PER: 0.0889
|
|
2025-09-16 05:23:30,136: t15.2023.08.27 val PER: 0.1801
|
|
2025-09-16 05:23:30,136: t15.2023.09.01 val PER: 0.0528
|
|
2025-09-16 05:23:30,136: t15.2023.09.03 val PER: 0.1200
|
|
2025-09-16 05:23:30,136: t15.2023.09.24 val PER: 0.0874
|
|
2025-09-16 05:23:30,136: t15.2023.09.29 val PER: 0.1072
|
|
2025-09-16 05:23:30,136: t15.2023.10.01 val PER: 0.1493
|
|
2025-09-16 05:23:30,136: t15.2023.10.06 val PER: 0.0571
|
|
2025-09-16 05:23:30,136: t15.2023.10.08 val PER: 0.1962
|
|
2025-09-16 05:23:30,136: t15.2023.10.13 val PER: 0.1668
|
|
2025-09-16 05:23:30,136: t15.2023.10.15 val PER: 0.1061
|
|
2025-09-16 05:23:30,136: t15.2023.10.20 val PER: 0.1779
|
|
2025-09-16 05:23:30,136: t15.2023.10.22 val PER: 0.1047
|
|
2025-09-16 05:23:30,136: t15.2023.11.03 val PER: 0.1554
|
|
2025-09-16 05:23:30,136: t15.2023.11.04 val PER: 0.0068
|
|
2025-09-16 05:23:30,136: t15.2023.11.17 val PER: 0.0202
|
|
2025-09-16 05:23:30,136: t15.2023.11.19 val PER: 0.0120
|
|
2025-09-16 05:23:30,137: t15.2023.11.26 val PER: 0.0514
|
|
2025-09-16 05:23:30,137: t15.2023.12.03 val PER: 0.0567
|
|
2025-09-16 05:23:30,137: t15.2023.12.08 val PER: 0.0406
|
|
2025-09-16 05:23:30,137: t15.2023.12.10 val PER: 0.0434
|
|
2025-09-16 05:23:30,137: t15.2023.12.17 val PER: 0.0873
|
|
2025-09-16 05:23:30,137: t15.2023.12.29 val PER: 0.0741
|
|
2025-09-16 05:23:30,137: t15.2024.02.25 val PER: 0.0857
|
|
2025-09-16 05:23:30,137: t15.2024.03.08 val PER: 0.1693
|
|
2025-09-16 05:23:30,137: t15.2024.03.15 val PER: 0.1645
|
|
2025-09-16 05:23:30,137: t15.2024.03.17 val PER: 0.0788
|
|
2025-09-16 05:23:30,137: t15.2024.05.10 val PER: 0.1308
|
|
2025-09-16 05:23:30,137: t15.2024.06.14 val PER: 0.1293
|
|
2025-09-16 05:23:30,137: t15.2024.07.19 val PER: 0.1767
|
|
2025-09-16 05:23:30,137: t15.2024.07.21 val PER: 0.0559
|
|
2025-09-16 05:23:30,137: t15.2024.07.28 val PER: 0.0787
|
|
2025-09-16 05:23:30,137: t15.2025.01.10 val PER: 0.2479
|
|
2025-09-16 05:23:30,137: t15.2025.01.12 val PER: 0.0901
|
|
2025-09-16 05:23:30,138: t15.2025.03.14 val PER: 0.2618
|
|
2025-09-16 05:23:30,138: t15.2025.03.16 val PER: 0.1466
|
|
2025-09-16 05:23:30,138: t15.2025.03.30 val PER: 0.2356
|
|
2025-09-16 05:23:30,138: t15.2025.04.13 val PER: 0.2154
|
|
2025-09-16 05:23:57,855: Train batch 64200: loss: 0.01 grad norm: 2.23 time: 0.127
|
|
2025-09-16 05:24:30,727: Train batch 64400: loss: 0.01 grad norm: 0.30 time: 0.158
|
|
2025-09-16 05:25:03,471: Train batch 64600: loss: 0.01 grad norm: 0.22 time: 0.173
|
|
2025-09-16 05:25:36,036: Train batch 64800: loss: 0.00 grad norm: 0.49 time: 0.103
|
|
2025-09-16 05:26:08,555: Train batch 65000: loss: 0.01 grad norm: 0.77 time: 0.126
|
|
2025-09-16 05:26:41,396: Train batch 65200: loss: 0.01 grad norm: 1.20 time: 0.129
|
|
2025-09-16 05:27:14,565: Train batch 65400: loss: 0.00 grad norm: 0.17 time: 0.099
|
|
2025-09-16 05:27:47,514: Train batch 65600: loss: 0.00 grad norm: 0.10 time: 0.113
|
|
2025-09-16 05:28:20,026: Train batch 65800: loss: 0.02 grad norm: 2.26 time: 0.150
|
|
2025-09-16 05:28:52,882: Train batch 66000: loss: 0.06 grad norm: 6.23 time: 0.159
|
|
2025-09-16 05:28:52,882: Running test after training batch: 66000
|
|
2025-09-16 05:29:03,595: Val batch 66000: PER (avg): 0.1100 CTC Loss (avg): 27.0369 time: 10.713
|
|
2025-09-16 05:29:03,596: t15.2023.08.13 val PER: 0.0842
|
|
2025-09-16 05:29:03,596: t15.2023.08.18 val PER: 0.0729
|
|
2025-09-16 05:29:03,596: t15.2023.08.20 val PER: 0.0612
|
|
2025-09-16 05:29:03,596: t15.2023.08.25 val PER: 0.0919
|
|
2025-09-16 05:29:03,596: t15.2023.08.27 val PER: 0.1672
|
|
2025-09-16 05:29:03,596: t15.2023.09.01 val PER: 0.0544
|
|
2025-09-16 05:29:03,596: t15.2023.09.03 val PER: 0.1164
|
|
2025-09-16 05:29:03,596: t15.2023.09.24 val PER: 0.0947
|
|
2025-09-16 05:29:03,596: t15.2023.09.29 val PER: 0.1021
|
|
2025-09-16 05:29:03,596: t15.2023.10.01 val PER: 0.1413
|
|
2025-09-16 05:29:03,596: t15.2023.10.06 val PER: 0.0571
|
|
2025-09-16 05:29:03,596: t15.2023.10.08 val PER: 0.1922
|
|
2025-09-16 05:29:03,596: t15.2023.10.13 val PER: 0.1715
|
|
2025-09-16 05:29:03,596: t15.2023.10.15 val PER: 0.1055
|
|
2025-09-16 05:29:03,596: t15.2023.10.20 val PER: 0.1812
|
|
2025-09-16 05:29:03,596: t15.2023.10.22 val PER: 0.1013
|
|
2025-09-16 05:29:03,596: t15.2023.11.03 val PER: 0.1635
|
|
2025-09-16 05:29:03,597: t15.2023.11.04 val PER: 0.0102
|
|
2025-09-16 05:29:03,597: t15.2023.11.17 val PER: 0.0218
|
|
2025-09-16 05:29:03,597: t15.2023.11.19 val PER: 0.0140
|
|
2025-09-16 05:29:03,597: t15.2023.11.26 val PER: 0.0536
|
|
2025-09-16 05:29:03,597: t15.2023.12.03 val PER: 0.0557
|
|
2025-09-16 05:29:03,597: t15.2023.12.08 val PER: 0.0419
|
|
2025-09-16 05:29:03,597: t15.2023.12.10 val PER: 0.0394
|
|
2025-09-16 05:29:03,597: t15.2023.12.17 val PER: 0.0894
|
|
2025-09-16 05:29:03,597: t15.2023.12.29 val PER: 0.0693
|
|
2025-09-16 05:29:03,597: t15.2024.02.25 val PER: 0.0815
|
|
2025-09-16 05:29:03,597: t15.2024.03.08 val PER: 0.1679
|
|
2025-09-16 05:29:03,597: t15.2024.03.15 val PER: 0.1639
|
|
2025-09-16 05:29:03,597: t15.2024.03.17 val PER: 0.0767
|
|
2025-09-16 05:29:03,597: t15.2024.05.10 val PER: 0.1322
|
|
2025-09-16 05:29:03,597: t15.2024.06.14 val PER: 0.1278
|
|
2025-09-16 05:29:03,597: t15.2024.07.19 val PER: 0.1800
|
|
2025-09-16 05:29:03,597: t15.2024.07.21 val PER: 0.0586
|
|
2025-09-16 05:29:03,597: t15.2024.07.28 val PER: 0.0853
|
|
2025-09-16 05:29:03,597: t15.2025.01.10 val PER: 0.2658
|
|
2025-09-16 05:29:03,597: t15.2025.01.12 val PER: 0.0808
|
|
2025-09-16 05:29:03,598: t15.2025.03.14 val PER: 0.2870
|
|
2025-09-16 05:29:03,598: t15.2025.03.16 val PER: 0.1335
|
|
2025-09-16 05:29:03,598: t15.2025.03.30 val PER: 0.2276
|
|
2025-09-16 05:29:03,598: t15.2025.04.13 val PER: 0.2111
|
|
2025-09-16 05:29:31,871: Train batch 66200: loss: 0.01 grad norm: 0.37 time: 0.107
|
|
2025-09-16 05:30:03,708: Train batch 66400: loss: 0.00 grad norm: 0.43 time: 0.139
|
|
2025-09-16 05:30:36,193: Train batch 66600: loss: 0.02 grad norm: 2.47 time: 0.159
|
|
2025-09-16 05:31:08,481: Train batch 66800: loss: 0.01 grad norm: 0.23 time: 0.165
|
|
2025-09-16 05:31:40,633: Train batch 67000: loss: 0.01 grad norm: 0.24 time: 0.116
|
|
2025-09-16 05:32:13,432: Train batch 67200: loss: 0.01 grad norm: 0.58 time: 0.108
|
|
2025-09-16 05:32:46,911: Train batch 67400: loss: 0.15 grad norm: 6.71 time: 0.133
|
|
2025-09-16 05:33:19,555: Train batch 67600: loss: 0.01 grad norm: 0.65 time: 0.185
|
|
2025-09-16 05:33:52,032: Train batch 67800: loss: 0.01 grad norm: 0.52 time: 0.138
|
|
2025-09-16 05:34:25,279: Train batch 68000: loss: 0.01 grad norm: 1.80 time: 0.112
|
|
2025-09-16 05:34:25,279: Running test after training batch: 68000
|
|
2025-09-16 05:34:35,788: Val batch 68000: PER (avg): 0.1111 CTC Loss (avg): 27.6746 time: 10.509
|
|
2025-09-16 05:34:35,788: t15.2023.08.13 val PER: 0.0894
|
|
2025-09-16 05:34:35,788: t15.2023.08.18 val PER: 0.0771
|
|
2025-09-16 05:34:35,788: t15.2023.08.20 val PER: 0.0620
|
|
2025-09-16 05:34:35,788: t15.2023.08.25 val PER: 0.0934
|
|
2025-09-16 05:34:35,788: t15.2023.08.27 val PER: 0.1785
|
|
2025-09-16 05:34:35,789: t15.2023.09.01 val PER: 0.0519
|
|
2025-09-16 05:34:35,789: t15.2023.09.03 val PER: 0.1093
|
|
2025-09-16 05:34:35,789: t15.2023.09.24 val PER: 0.0898
|
|
2025-09-16 05:34:35,789: t15.2023.09.29 val PER: 0.1015
|
|
2025-09-16 05:34:35,789: t15.2023.10.01 val PER: 0.1499
|
|
2025-09-16 05:34:35,789: t15.2023.10.06 val PER: 0.0678
|
|
2025-09-16 05:34:35,789: t15.2023.10.08 val PER: 0.1935
|
|
2025-09-16 05:34:35,789: t15.2023.10.13 val PER: 0.1777
|
|
2025-09-16 05:34:35,789: t15.2023.10.15 val PER: 0.1048
|
|
2025-09-16 05:34:35,789: t15.2023.10.20 val PER: 0.1711
|
|
2025-09-16 05:34:35,789: t15.2023.10.22 val PER: 0.1091
|
|
2025-09-16 05:34:35,789: t15.2023.11.03 val PER: 0.1635
|
|
2025-09-16 05:34:35,789: t15.2023.11.04 val PER: 0.0068
|
|
2025-09-16 05:34:35,789: t15.2023.11.17 val PER: 0.0202
|
|
2025-09-16 05:34:35,789: t15.2023.11.19 val PER: 0.0100
|
|
2025-09-16 05:34:35,789: t15.2023.11.26 val PER: 0.0558
|
|
2025-09-16 05:34:35,789: t15.2023.12.03 val PER: 0.0557
|
|
2025-09-16 05:34:35,789: t15.2023.12.08 val PER: 0.0426
|
|
2025-09-16 05:34:35,789: t15.2023.12.10 val PER: 0.0394
|
|
2025-09-16 05:34:35,790: t15.2023.12.17 val PER: 0.0832
|
|
2025-09-16 05:34:35,790: t15.2023.12.29 val PER: 0.0782
|
|
2025-09-16 05:34:35,790: t15.2024.02.25 val PER: 0.0885
|
|
2025-09-16 05:34:35,790: t15.2024.03.08 val PER: 0.1750
|
|
2025-09-16 05:34:35,790: t15.2024.03.15 val PER: 0.1670
|
|
2025-09-16 05:34:35,790: t15.2024.03.17 val PER: 0.0739
|
|
2025-09-16 05:34:35,790: t15.2024.05.10 val PER: 0.1278
|
|
2025-09-16 05:34:35,790: t15.2024.06.14 val PER: 0.1309
|
|
2025-09-16 05:34:35,790: t15.2024.07.19 val PER: 0.1707
|
|
2025-09-16 05:34:35,790: t15.2024.07.21 val PER: 0.0579
|
|
2025-09-16 05:34:35,790: t15.2024.07.28 val PER: 0.0838
|
|
2025-09-16 05:34:35,790: t15.2025.01.10 val PER: 0.2658
|
|
2025-09-16 05:34:35,790: t15.2025.01.12 val PER: 0.0808
|
|
2025-09-16 05:34:35,790: t15.2025.03.14 val PER: 0.2781
|
|
2025-09-16 05:34:35,790: t15.2025.03.16 val PER: 0.1374
|
|
2025-09-16 05:34:35,790: t15.2025.03.30 val PER: 0.2368
|
|
2025-09-16 05:34:35,790: t15.2025.04.13 val PER: 0.2111
|
|
2025-09-16 05:35:03,330: Train batch 68200: loss: 0.01 grad norm: 0.42 time: 0.126
|
|
2025-09-16 05:35:35,151: Train batch 68400: loss: 0.01 grad norm: 0.20 time: 0.117
|
|
2025-09-16 05:36:07,443: Train batch 68600: loss: 0.05 grad norm: 9.20 time: 0.104
|
|
2025-09-16 05:36:39,808: Train batch 68800: loss: 0.01 grad norm: 0.34 time: 0.156
|
|
2025-09-16 05:37:11,669: Train batch 69000: loss: 0.00 grad norm: 0.17 time: 0.157
|
|
2025-09-16 05:37:44,139: Train batch 69200: loss: 0.02 grad norm: 2.38 time: 0.113
|
|
2025-09-16 05:38:16,508: Train batch 69400: loss: 0.03 grad norm: 2.81 time: 0.121
|
|
2025-09-16 05:38:50,030: Train batch 69600: loss: 0.01 grad norm: 0.81 time: 0.128
|
|
2025-09-16 05:39:21,856: Train batch 69800: loss: 0.03 grad norm: 6.07 time: 0.124
|
|
2025-09-16 05:39:54,566: Train batch 70000: loss: 0.01 grad norm: 0.46 time: 0.097
|
|
2025-09-16 05:39:54,566: Running test after training batch: 70000
|
|
2025-09-16 05:40:05,080: Val batch 70000: PER (avg): 0.1097 CTC Loss (avg): 27.4982 time: 10.514
|
|
2025-09-16 05:40:05,081: t15.2023.08.13 val PER: 0.0863
|
|
2025-09-16 05:40:05,081: t15.2023.08.18 val PER: 0.0838
|
|
2025-09-16 05:40:05,081: t15.2023.08.20 val PER: 0.0588
|
|
2025-09-16 05:40:05,081: t15.2023.08.25 val PER: 0.0904
|
|
2025-09-16 05:40:05,081: t15.2023.08.27 val PER: 0.1624
|
|
2025-09-16 05:40:05,081: t15.2023.09.01 val PER: 0.0552
|
|
2025-09-16 05:40:05,081: t15.2023.09.03 val PER: 0.1140
|
|
2025-09-16 05:40:05,081: t15.2023.09.24 val PER: 0.0922
|
|
2025-09-16 05:40:05,081: t15.2023.09.29 val PER: 0.1027
|
|
2025-09-16 05:40:05,081: t15.2023.10.01 val PER: 0.1506
|
|
2025-09-16 05:40:05,081: t15.2023.10.06 val PER: 0.0646
|
|
2025-09-16 05:40:05,081: t15.2023.10.08 val PER: 0.1867
|
|
2025-09-16 05:40:05,081: t15.2023.10.13 val PER: 0.1590
|
|
2025-09-16 05:40:05,081: t15.2023.10.15 val PER: 0.1081
|
|
2025-09-16 05:40:05,081: t15.2023.10.20 val PER: 0.1879
|
|
2025-09-16 05:40:05,081: t15.2023.10.22 val PER: 0.1114
|
|
2025-09-16 05:40:05,081: t15.2023.11.03 val PER: 0.1608
|
|
2025-09-16 05:40:05,082: t15.2023.11.04 val PER: 0.0068
|
|
2025-09-16 05:40:05,082: t15.2023.11.17 val PER: 0.0249
|
|
2025-09-16 05:40:05,082: t15.2023.11.19 val PER: 0.0120
|
|
2025-09-16 05:40:05,082: t15.2023.11.26 val PER: 0.0514
|
|
2025-09-16 05:40:05,082: t15.2023.12.03 val PER: 0.0546
|
|
2025-09-16 05:40:05,082: t15.2023.12.08 val PER: 0.0446
|
|
2025-09-16 05:40:05,082: t15.2023.12.10 val PER: 0.0381
|
|
2025-09-16 05:40:05,082: t15.2023.12.17 val PER: 0.0863
|
|
2025-09-16 05:40:05,082: t15.2023.12.29 val PER: 0.0803
|
|
2025-09-16 05:40:05,082: t15.2024.02.25 val PER: 0.0787
|
|
2025-09-16 05:40:05,082: t15.2024.03.08 val PER: 0.1764
|
|
2025-09-16 05:40:05,082: t15.2024.03.15 val PER: 0.1651
|
|
2025-09-16 05:40:05,082: t15.2024.03.17 val PER: 0.0746
|
|
2025-09-16 05:40:05,082: t15.2024.05.10 val PER: 0.1337
|
|
2025-09-16 05:40:05,082: t15.2024.06.14 val PER: 0.1215
|
|
2025-09-16 05:40:05,082: t15.2024.07.19 val PER: 0.1681
|
|
2025-09-16 05:40:05,082: t15.2024.07.21 val PER: 0.0552
|
|
2025-09-16 05:40:05,082: t15.2024.07.28 val PER: 0.0853
|
|
2025-09-16 05:40:05,082: t15.2025.01.10 val PER: 0.2534
|
|
2025-09-16 05:40:05,082: t15.2025.01.12 val PER: 0.0778
|
|
2025-09-16 05:40:05,083: t15.2025.03.14 val PER: 0.2781
|
|
2025-09-16 05:40:05,083: t15.2025.03.16 val PER: 0.1453
|
|
2025-09-16 05:40:05,083: t15.2025.03.30 val PER: 0.2241
|
|
2025-09-16 05:40:05,083: t15.2025.04.13 val PER: 0.2054
|
|
2025-09-16 05:40:33,211: Train batch 70200: loss: 0.00 grad norm: 0.21 time: 0.144
|
|
2025-09-16 05:41:05,720: Train batch 70400: loss: 0.00 grad norm: 0.37 time: 0.100
|
|
2025-09-16 05:41:38,855: Train batch 70600: loss: 0.00 grad norm: 0.12 time: 0.129
|
|
2025-09-16 05:42:11,559: Train batch 70800: loss: 0.01 grad norm: 0.29 time: 0.170
|
|
2025-09-16 05:42:44,066: Train batch 71000: loss: 0.02 grad norm: 0.37 time: 0.113
|
|
2025-09-16 05:43:16,730: Train batch 71200: loss: 0.00 grad norm: 0.08 time: 0.119
|
|
2025-09-16 05:43:49,279: Train batch 71400: loss: 0.02 grad norm: 2.35 time: 0.179
|
|
2025-09-16 05:44:21,540: Train batch 71600: loss: 0.01 grad norm: 0.79 time: 0.111
|
|
2025-09-16 05:44:53,547: Train batch 71800: loss: 0.01 grad norm: 0.49 time: 0.125
|
|
2025-09-16 05:45:26,191: Train batch 72000: loss: 0.01 grad norm: 0.22 time: 0.173
|
|
2025-09-16 05:45:26,191: Running test after training batch: 72000
|
|
2025-09-16 05:45:36,744: Val batch 72000: PER (avg): 0.1095 CTC Loss (avg): 27.5585 time: 10.553
|
|
2025-09-16 05:45:36,744: t15.2023.08.13 val PER: 0.0852
|
|
2025-09-16 05:45:36,744: t15.2023.08.18 val PER: 0.0788
|
|
2025-09-16 05:45:36,745: t15.2023.08.20 val PER: 0.0620
|
|
2025-09-16 05:45:36,745: t15.2023.08.25 val PER: 0.0964
|
|
2025-09-16 05:45:36,745: t15.2023.08.27 val PER: 0.1720
|
|
2025-09-16 05:45:36,745: t15.2023.09.01 val PER: 0.0495
|
|
2025-09-16 05:45:36,745: t15.2023.09.03 val PER: 0.1116
|
|
2025-09-16 05:45:36,745: t15.2023.09.24 val PER: 0.0995
|
|
2025-09-16 05:45:36,745: t15.2023.09.29 val PER: 0.1008
|
|
2025-09-16 05:45:36,745: t15.2023.10.01 val PER: 0.1420
|
|
2025-09-16 05:45:36,745: t15.2023.10.06 val PER: 0.0635
|
|
2025-09-16 05:45:36,745: t15.2023.10.08 val PER: 0.1962
|
|
2025-09-16 05:45:36,745: t15.2023.10.13 val PER: 0.1668
|
|
2025-09-16 05:45:36,745: t15.2023.10.15 val PER: 0.1048
|
|
2025-09-16 05:45:36,745: t15.2023.10.20 val PER: 0.1779
|
|
2025-09-16 05:45:36,745: t15.2023.10.22 val PER: 0.1080
|
|
2025-09-16 05:45:36,745: t15.2023.11.03 val PER: 0.1635
|
|
2025-09-16 05:45:36,745: t15.2023.11.04 val PER: 0.0068
|
|
2025-09-16 05:45:36,745: t15.2023.11.17 val PER: 0.0218
|
|
2025-09-16 05:45:36,745: t15.2023.11.19 val PER: 0.0100
|
|
2025-09-16 05:45:36,745: t15.2023.11.26 val PER: 0.0522
|
|
2025-09-16 05:45:36,746: t15.2023.12.03 val PER: 0.0578
|
|
2025-09-16 05:45:36,746: t15.2023.12.08 val PER: 0.0419
|
|
2025-09-16 05:45:36,746: t15.2023.12.10 val PER: 0.0355
|
|
2025-09-16 05:45:36,746: t15.2023.12.17 val PER: 0.0759
|
|
2025-09-16 05:45:36,746: t15.2023.12.29 val PER: 0.0776
|
|
2025-09-16 05:45:36,746: t15.2024.02.25 val PER: 0.0885
|
|
2025-09-16 05:45:36,746: t15.2024.03.08 val PER: 0.1707
|
|
2025-09-16 05:45:36,746: t15.2024.03.15 val PER: 0.1607
|
|
2025-09-16 05:45:36,746: t15.2024.03.17 val PER: 0.0816
|
|
2025-09-16 05:45:36,746: t15.2024.05.10 val PER: 0.1308
|
|
2025-09-16 05:45:36,746: t15.2024.06.14 val PER: 0.1246
|
|
2025-09-16 05:45:36,746: t15.2024.07.19 val PER: 0.1674
|
|
2025-09-16 05:45:36,746: t15.2024.07.21 val PER: 0.0524
|
|
2025-09-16 05:45:36,746: t15.2024.07.28 val PER: 0.0772
|
|
2025-09-16 05:45:36,746: t15.2025.01.10 val PER: 0.2686
|
|
2025-09-16 05:45:36,746: t15.2025.01.12 val PER: 0.0808
|
|
2025-09-16 05:45:36,746: t15.2025.03.14 val PER: 0.2811
|
|
2025-09-16 05:45:36,746: t15.2025.03.16 val PER: 0.1374
|
|
2025-09-16 05:45:36,746: t15.2025.03.30 val PER: 0.2333
|
|
2025-09-16 05:45:36,746: t15.2025.04.13 val PER: 0.2126
|
|
2025-09-16 05:45:36,747: New best test PER 0.1097 --> 0.1095
|
|
2025-09-16 05:45:36,747: Checkpointing model
|
|
2025-09-16 05:45:38,119: Saved model to checkpoint: trained_models/baseline_rnn/checkpoint/best_checkpoint
|
|
2025-09-16 05:46:06,492: Train batch 72200: loss: 0.00 grad norm: 0.23 time: 0.143
|
|
2025-09-16 05:46:39,113: Train batch 72400: loss: 0.01 grad norm: 1.38 time: 0.167
|
|
2025-09-16 05:47:11,783: Train batch 72600: loss: 0.10 grad norm: 1.71 time: 0.157
|
|
2025-09-16 05:47:44,020: Train batch 72800: loss: 0.01 grad norm: 0.23 time: 0.133
|
|
2025-09-16 05:48:16,097: Train batch 73000: loss: 0.01 grad norm: 0.45 time: 0.115
|
|
2025-09-16 05:48:47,970: Train batch 73200: loss: 0.00 grad norm: 0.15 time: 0.110
|
|
2025-09-16 05:49:20,428: Train batch 73400: loss: 0.03 grad norm: 3.01 time: 0.156
|
|
2025-09-16 05:49:52,613: Train batch 73600: loss: 0.02 grad norm: 4.12 time: 0.094
|
|
2025-09-16 05:50:25,156: Train batch 73800: loss: 0.00 grad norm: 0.06 time: 0.155
|
|
2025-09-16 05:50:57,255: Train batch 74000: loss: 0.04 grad norm: 3.22 time: 0.147
|
|
2025-09-16 05:50:57,255: Running test after training batch: 74000
|
|
2025-09-16 05:51:07,774: Val batch 74000: PER (avg): 0.1092 CTC Loss (avg): 27.8142 time: 10.519
|
|
2025-09-16 05:51:07,775: t15.2023.08.13 val PER: 0.0863
|
|
2025-09-16 05:51:07,775: t15.2023.08.18 val PER: 0.0771
|
|
2025-09-16 05:51:07,775: t15.2023.08.20 val PER: 0.0604
|
|
2025-09-16 05:51:07,775: t15.2023.08.25 val PER: 0.0919
|
|
2025-09-16 05:51:07,775: t15.2023.08.27 val PER: 0.1672
|
|
2025-09-16 05:51:07,775: t15.2023.09.01 val PER: 0.0463
|
|
2025-09-16 05:51:07,775: t15.2023.09.03 val PER: 0.1105
|
|
2025-09-16 05:51:07,775: t15.2023.09.24 val PER: 0.0947
|
|
2025-09-16 05:51:07,775: t15.2023.09.29 val PER: 0.1040
|
|
2025-09-16 05:51:07,775: t15.2023.10.01 val PER: 0.1480
|
|
2025-09-16 05:51:07,775: t15.2023.10.06 val PER: 0.0592
|
|
2025-09-16 05:51:07,775: t15.2023.10.08 val PER: 0.1935
|
|
2025-09-16 05:51:07,775: t15.2023.10.13 val PER: 0.1683
|
|
2025-09-16 05:51:07,775: t15.2023.10.15 val PER: 0.1035
|
|
2025-09-16 05:51:07,775: t15.2023.10.20 val PER: 0.1711
|
|
2025-09-16 05:51:07,775: t15.2023.10.22 val PER: 0.0991
|
|
2025-09-16 05:51:07,775: t15.2023.11.03 val PER: 0.1649
|
|
2025-09-16 05:51:07,776: t15.2023.11.04 val PER: 0.0068
|
|
2025-09-16 05:51:07,776: t15.2023.11.17 val PER: 0.0233
|
|
2025-09-16 05:51:07,776: t15.2023.11.19 val PER: 0.0080
|
|
2025-09-16 05:51:07,776: t15.2023.11.26 val PER: 0.0507
|
|
2025-09-16 05:51:07,776: t15.2023.12.03 val PER: 0.0567
|
|
2025-09-16 05:51:07,776: t15.2023.12.08 val PER: 0.0393
|
|
2025-09-16 05:51:07,776: t15.2023.12.10 val PER: 0.0394
|
|
2025-09-16 05:51:07,776: t15.2023.12.17 val PER: 0.0873
|
|
2025-09-16 05:51:07,776: t15.2023.12.29 val PER: 0.0810
|
|
2025-09-16 05:51:07,776: t15.2024.02.25 val PER: 0.0801
|
|
2025-09-16 05:51:07,776: t15.2024.03.08 val PER: 0.1707
|
|
2025-09-16 05:51:07,776: t15.2024.03.15 val PER: 0.1639
|
|
2025-09-16 05:51:07,776: t15.2024.03.17 val PER: 0.0746
|
|
2025-09-16 05:51:07,776: t15.2024.05.10 val PER: 0.1367
|
|
2025-09-16 05:51:07,776: t15.2024.06.14 val PER: 0.1167
|
|
2025-09-16 05:51:07,776: t15.2024.07.19 val PER: 0.1668
|
|
2025-09-16 05:51:07,776: t15.2024.07.21 val PER: 0.0579
|
|
2025-09-16 05:51:07,776: t15.2024.07.28 val PER: 0.0779
|
|
2025-09-16 05:51:07,776: t15.2025.01.10 val PER: 0.2617
|
|
2025-09-16 05:51:07,777: t15.2025.01.12 val PER: 0.0801
|
|
2025-09-16 05:51:07,777: t15.2025.03.14 val PER: 0.2825
|
|
2025-09-16 05:51:07,777: t15.2025.03.16 val PER: 0.1414
|
|
2025-09-16 05:51:07,777: t15.2025.03.30 val PER: 0.2310
|
|
2025-09-16 05:51:07,777: t15.2025.04.13 val PER: 0.2111
|
|
2025-09-16 05:51:07,777: New best test PER 0.1095 --> 0.1092
|
|
2025-09-16 05:51:07,777: Checkpointing model
|
|
2025-09-16 05:51:09,086: Saved model to checkpoint: trained_models/baseline_rnn/checkpoint/best_checkpoint
|
|
2025-09-16 05:51:36,937: Train batch 74200: loss: 0.01 grad norm: 0.83 time: 0.128
|
|
2025-09-16 05:52:09,753: Train batch 74400: loss: 0.01 grad norm: 2.45 time: 0.154
|
|
2025-09-16 05:52:42,576: Train batch 74600: loss: 0.06 grad norm: 8.72 time: 0.119
|
|
2025-09-16 05:53:15,418: Train batch 74800: loss: 0.00 grad norm: 0.09 time: 0.116
|
|
2025-09-16 05:53:48,311: Train batch 75000: loss: 0.01 grad norm: 0.20 time: 0.140
|
|
2025-09-16 05:54:20,393: Train batch 75200: loss: 0.01 grad norm: 0.35 time: 0.167
|
|
2025-09-16 05:54:52,783: Train batch 75400: loss: 0.00 grad norm: 0.24 time: 0.117
|
|
2025-09-16 05:55:25,072: Train batch 75600: loss: 0.00 grad norm: 0.12 time: 0.123
|
|
2025-09-16 05:55:57,046: Train batch 75800: loss: 0.00 grad norm: 0.43 time: 0.128
|
|
2025-09-16 05:56:28,744: Train batch 76000: loss: 0.01 grad norm: 1.32 time: 0.113
|
|
2025-09-16 05:56:28,744: Running test after training batch: 76000
|
|
2025-09-16 05:56:39,466: Val batch 76000: PER (avg): 0.1082 CTC Loss (avg): 27.5969 time: 10.722
|
|
2025-09-16 05:56:39,467: t15.2023.08.13 val PER: 0.0884
|
|
2025-09-16 05:56:39,467: t15.2023.08.18 val PER: 0.0780
|
|
2025-09-16 05:56:39,467: t15.2023.08.20 val PER: 0.0580
|
|
2025-09-16 05:56:39,467: t15.2023.08.25 val PER: 0.1009
|
|
2025-09-16 05:56:39,467: t15.2023.08.27 val PER: 0.1656
|
|
2025-09-16 05:56:39,467: t15.2023.09.01 val PER: 0.0495
|
|
2025-09-16 05:56:39,467: t15.2023.09.03 val PER: 0.1128
|
|
2025-09-16 05:56:39,467: t15.2023.09.24 val PER: 0.0959
|
|
2025-09-16 05:56:39,467: t15.2023.09.29 val PER: 0.0983
|
|
2025-09-16 05:56:39,467: t15.2023.10.01 val PER: 0.1413
|
|
2025-09-16 05:56:39,467: t15.2023.10.06 val PER: 0.0635
|
|
2025-09-16 05:56:39,467: t15.2023.10.08 val PER: 0.1989
|
|
2025-09-16 05:56:39,467: t15.2023.10.13 val PER: 0.1637
|
|
2025-09-16 05:56:39,467: t15.2023.10.15 val PER: 0.1015
|
|
2025-09-16 05:56:39,467: t15.2023.10.20 val PER: 0.1678
|
|
2025-09-16 05:56:39,467: t15.2023.10.22 val PER: 0.1047
|
|
2025-09-16 05:56:39,468: t15.2023.11.03 val PER: 0.1588
|
|
2025-09-16 05:56:39,468: t15.2023.11.04 val PER: 0.0068
|
|
2025-09-16 05:56:39,468: t15.2023.11.17 val PER: 0.0218
|
|
2025-09-16 05:56:39,468: t15.2023.11.19 val PER: 0.0100
|
|
2025-09-16 05:56:39,468: t15.2023.11.26 val PER: 0.0529
|
|
2025-09-16 05:56:39,468: t15.2023.12.03 val PER: 0.0578
|
|
2025-09-16 05:56:39,468: t15.2023.12.08 val PER: 0.0419
|
|
2025-09-16 05:56:39,468: t15.2023.12.10 val PER: 0.0394
|
|
2025-09-16 05:56:39,468: t15.2023.12.17 val PER: 0.0873
|
|
2025-09-16 05:56:39,468: t15.2023.12.29 val PER: 0.0769
|
|
2025-09-16 05:56:39,468: t15.2024.02.25 val PER: 0.0801
|
|
2025-09-16 05:56:39,468: t15.2024.03.08 val PER: 0.1721
|
|
2025-09-16 05:56:39,468: t15.2024.03.15 val PER: 0.1588
|
|
2025-09-16 05:56:39,468: t15.2024.03.17 val PER: 0.0767
|
|
2025-09-16 05:56:39,468: t15.2024.05.10 val PER: 0.1248
|
|
2025-09-16 05:56:39,468: t15.2024.06.14 val PER: 0.1199
|
|
2025-09-16 05:56:39,468: t15.2024.07.19 val PER: 0.1688
|
|
2025-09-16 05:56:39,468: t15.2024.07.21 val PER: 0.0552
|
|
2025-09-16 05:56:39,468: t15.2024.07.28 val PER: 0.0794
|
|
2025-09-16 05:56:39,469: t15.2025.01.10 val PER: 0.2672
|
|
2025-09-16 05:56:39,469: t15.2025.01.12 val PER: 0.0770
|
|
2025-09-16 05:56:39,469: t15.2025.03.14 val PER: 0.2737
|
|
2025-09-16 05:56:39,469: t15.2025.03.16 val PER: 0.1466
|
|
2025-09-16 05:56:39,469: t15.2025.03.30 val PER: 0.2184
|
|
2025-09-16 05:56:39,469: t15.2025.04.13 val PER: 0.1997
|
|
2025-09-16 05:56:39,469: New best test PER 0.1092 --> 0.1082
|
|
2025-09-16 05:56:39,469: Checkpointing model
|
|
2025-09-16 05:56:40,831: Saved model to checkpoint: trained_models/baseline_rnn/checkpoint/best_checkpoint
|
|
2025-09-16 05:57:08,973: Train batch 76200: loss: 0.01 grad norm: 0.53 time: 0.145
|
|
2025-09-16 05:57:42,328: Train batch 76400: loss: 0.01 grad norm: 1.08 time: 0.133
|
|
2025-09-16 05:58:16,241: Train batch 76600: loss: 0.01 grad norm: 0.53 time: 0.128
|
|
2025-09-16 05:58:48,849: Train batch 76800: loss: 0.00 grad norm: 0.15 time: 0.157
|
|
2025-09-16 05:59:21,259: Train batch 77000: loss: 0.01 grad norm: 0.76 time: 0.162
|
|
2025-09-16 05:59:54,007: Train batch 77200: loss: 0.00 grad norm: 0.04 time: 0.157
|
|
2025-09-16 06:00:27,244: Train batch 77400: loss: 0.02 grad norm: 2.84 time: 0.138
|
|
2025-09-16 06:01:00,583: Train batch 77600: loss: 0.00 grad norm: 0.19 time: 0.123
|
|
2025-09-16 06:01:33,264: Train batch 77800: loss: 0.00 grad norm: 0.24 time: 0.117
|
|
2025-09-16 06:02:06,323: Train batch 78000: loss: 0.01 grad norm: 0.69 time: 0.121
|
|
2025-09-16 06:02:06,324: Running test after training batch: 78000
|
|
2025-09-16 06:02:17,026: Val batch 78000: PER (avg): 0.1089 CTC Loss (avg): 27.3303 time: 10.702
|
|
2025-09-16 06:02:17,026: t15.2023.08.13 val PER: 0.0946
|
|
2025-09-16 06:02:17,027: t15.2023.08.18 val PER: 0.0821
|
|
2025-09-16 06:02:17,027: t15.2023.08.20 val PER: 0.0580
|
|
2025-09-16 06:02:17,027: t15.2023.08.25 val PER: 0.0934
|
|
2025-09-16 06:02:17,027: t15.2023.08.27 val PER: 0.1624
|
|
2025-09-16 06:02:17,027: t15.2023.09.01 val PER: 0.0471
|
|
2025-09-16 06:02:17,027: t15.2023.09.03 val PER: 0.1128
|
|
2025-09-16 06:02:17,027: t15.2023.09.24 val PER: 0.0934
|
|
2025-09-16 06:02:17,027: t15.2023.09.29 val PER: 0.1021
|
|
2025-09-16 06:02:17,027: t15.2023.10.01 val PER: 0.1499
|
|
2025-09-16 06:02:17,027: t15.2023.10.06 val PER: 0.0527
|
|
2025-09-16 06:02:17,027: t15.2023.10.08 val PER: 0.1935
|
|
2025-09-16 06:02:17,027: t15.2023.10.13 val PER: 0.1668
|
|
2025-09-16 06:02:17,027: t15.2023.10.15 val PER: 0.1101
|
|
2025-09-16 06:02:17,027: t15.2023.10.20 val PER: 0.1812
|
|
2025-09-16 06:02:17,027: t15.2023.10.22 val PER: 0.1036
|
|
2025-09-16 06:02:17,027: t15.2023.11.03 val PER: 0.1642
|
|
2025-09-16 06:02:17,027: t15.2023.11.04 val PER: 0.0068
|
|
2025-09-16 06:02:17,027: t15.2023.11.17 val PER: 0.0233
|
|
2025-09-16 06:02:17,027: t15.2023.11.19 val PER: 0.0080
|
|
2025-09-16 06:02:17,028: t15.2023.11.26 val PER: 0.0500
|
|
2025-09-16 06:02:17,028: t15.2023.12.03 val PER: 0.0599
|
|
2025-09-16 06:02:17,028: t15.2023.12.08 val PER: 0.0413
|
|
2025-09-16 06:02:17,028: t15.2023.12.10 val PER: 0.0368
|
|
2025-09-16 06:02:17,028: t15.2023.12.17 val PER: 0.0842
|
|
2025-09-16 06:02:17,028: t15.2023.12.29 val PER: 0.0741
|
|
2025-09-16 06:02:17,028: t15.2024.02.25 val PER: 0.0801
|
|
2025-09-16 06:02:17,028: t15.2024.03.08 val PER: 0.1735
|
|
2025-09-16 06:02:17,028: t15.2024.03.15 val PER: 0.1626
|
|
2025-09-16 06:02:17,028: t15.2024.03.17 val PER: 0.0746
|
|
2025-09-16 06:02:17,028: t15.2024.05.10 val PER: 0.1189
|
|
2025-09-16 06:02:17,028: t15.2024.06.14 val PER: 0.1278
|
|
2025-09-16 06:02:17,028: t15.2024.07.19 val PER: 0.1622
|
|
2025-09-16 06:02:17,028: t15.2024.07.21 val PER: 0.0531
|
|
2025-09-16 06:02:17,028: t15.2024.07.28 val PER: 0.0801
|
|
2025-09-16 06:02:17,028: t15.2025.01.10 val PER: 0.2769
|
|
2025-09-16 06:02:17,028: t15.2025.01.12 val PER: 0.0747
|
|
2025-09-16 06:02:17,028: t15.2025.03.14 val PER: 0.2870
|
|
2025-09-16 06:02:17,028: t15.2025.03.16 val PER: 0.1401
|
|
2025-09-16 06:02:17,028: t15.2025.03.30 val PER: 0.2276
|
|
2025-09-16 06:02:17,029: t15.2025.04.13 val PER: 0.2011
|
|
2025-09-16 06:02:44,720: Train batch 78200: loss: 0.01 grad norm: 0.57 time: 0.104
|
|
2025-09-16 06:03:16,912: Train batch 78400: loss: 0.01 grad norm: 0.58 time: 0.135
|
|
2025-09-16 06:03:50,006: Train batch 78600: loss: 0.03 grad norm: 2.93 time: 0.118
|
|
2025-09-16 06:04:23,282: Train batch 78800: loss: 0.12 grad norm: 2.28 time: 0.141
|
|
2025-09-16 06:04:56,889: Train batch 79000: loss: 0.00 grad norm: 0.25 time: 0.108
|
|
2025-09-16 06:05:29,802: Train batch 79200: loss: 0.00 grad norm: 0.24 time: 0.162
|
|
2025-09-16 06:06:02,422: Train batch 79400: loss: 0.01 grad norm: 0.30 time: 0.163
|
|
2025-09-16 06:06:35,790: Train batch 79600: loss: 0.01 grad norm: 1.13 time: 0.141
|
|
2025-09-16 06:07:09,025: Train batch 79800: loss: 0.00 grad norm: 0.74 time: 0.153
|
|
2025-09-16 06:07:42,026: Train batch 80000: loss: 0.02 grad norm: 3.20 time: 0.150
|
|
2025-09-16 06:07:42,026: Running test after training batch: 80000
|
|
2025-09-16 06:07:52,526: Val batch 80000: PER (avg): 0.1090 CTC Loss (avg): 27.3370 time: 10.500
|
|
2025-09-16 06:07:52,527: t15.2023.08.13 val PER: 0.0925
|
|
2025-09-16 06:07:52,527: t15.2023.08.18 val PER: 0.0805
|
|
2025-09-16 06:07:52,527: t15.2023.08.20 val PER: 0.0620
|
|
2025-09-16 06:07:52,527: t15.2023.08.25 val PER: 0.0904
|
|
2025-09-16 06:07:52,527: t15.2023.08.27 val PER: 0.1624
|
|
2025-09-16 06:07:52,527: t15.2023.09.01 val PER: 0.0552
|
|
2025-09-16 06:07:52,527: t15.2023.09.03 val PER: 0.1081
|
|
2025-09-16 06:07:52,527: t15.2023.09.24 val PER: 0.0934
|
|
2025-09-16 06:07:52,527: t15.2023.09.29 val PER: 0.1008
|
|
2025-09-16 06:07:52,527: t15.2023.10.01 val PER: 0.1526
|
|
2025-09-16 06:07:52,527: t15.2023.10.06 val PER: 0.0549
|
|
2025-09-16 06:07:52,527: t15.2023.10.08 val PER: 0.1908
|
|
2025-09-16 06:07:52,527: t15.2023.10.13 val PER: 0.1660
|
|
2025-09-16 06:07:52,527: t15.2023.10.15 val PER: 0.1035
|
|
2025-09-16 06:07:52,528: t15.2023.10.20 val PER: 0.1644
|
|
2025-09-16 06:07:52,528: t15.2023.10.22 val PER: 0.1080
|
|
2025-09-16 06:07:52,528: t15.2023.11.03 val PER: 0.1567
|
|
2025-09-16 06:07:52,528: t15.2023.11.04 val PER: 0.0068
|
|
2025-09-16 06:07:52,528: t15.2023.11.17 val PER: 0.0233
|
|
2025-09-16 06:07:52,528: t15.2023.11.19 val PER: 0.0080
|
|
2025-09-16 06:07:52,528: t15.2023.11.26 val PER: 0.0529
|
|
2025-09-16 06:07:52,528: t15.2023.12.03 val PER: 0.0578
|
|
2025-09-16 06:07:52,528: t15.2023.12.08 val PER: 0.0386
|
|
2025-09-16 06:07:52,528: t15.2023.12.10 val PER: 0.0329
|
|
2025-09-16 06:07:52,528: t15.2023.12.17 val PER: 0.0811
|
|
2025-09-16 06:07:52,528: t15.2023.12.29 val PER: 0.0776
|
|
2025-09-16 06:07:52,528: t15.2024.02.25 val PER: 0.0688
|
|
2025-09-16 06:07:52,528: t15.2024.03.08 val PER: 0.1721
|
|
2025-09-16 06:07:52,528: t15.2024.03.15 val PER: 0.1651
|
|
2025-09-16 06:07:52,528: t15.2024.03.17 val PER: 0.0711
|
|
2025-09-16 06:07:52,528: t15.2024.05.10 val PER: 0.1248
|
|
2025-09-16 06:07:52,528: t15.2024.06.14 val PER: 0.1230
|
|
2025-09-16 06:07:52,528: t15.2024.07.19 val PER: 0.1707
|
|
2025-09-16 06:07:52,528: t15.2024.07.21 val PER: 0.0531
|
|
2025-09-16 06:07:52,529: t15.2024.07.28 val PER: 0.0853
|
|
2025-09-16 06:07:52,529: t15.2025.01.10 val PER: 0.2824
|
|
2025-09-16 06:07:52,529: t15.2025.01.12 val PER: 0.0808
|
|
2025-09-16 06:07:52,529: t15.2025.03.14 val PER: 0.2766
|
|
2025-09-16 06:07:52,529: t15.2025.03.16 val PER: 0.1401
|
|
2025-09-16 06:07:52,529: t15.2025.03.30 val PER: 0.2333
|
|
2025-09-16 06:07:52,529: t15.2025.04.13 val PER: 0.2026
|
|
2025-09-16 06:08:20,772: Train batch 80200: loss: 0.01 grad norm: 1.02 time: 0.118
|
|
2025-09-16 06:08:54,009: Train batch 80400: loss: 0.01 grad norm: 0.48 time: 0.118
|
|
2025-09-16 06:09:27,329: Train batch 80600: loss: 0.00 grad norm: 0.09 time: 0.173
|
|
2025-09-16 06:10:00,140: Train batch 80800: loss: 0.00 grad norm: 0.07 time: 0.129
|
|
2025-09-16 06:10:33,443: Train batch 81000: loss: 0.00 grad norm: 0.15 time: 0.116
|
|
2025-09-16 06:11:06,620: Train batch 81200: loss: 0.01 grad norm: 1.26 time: 0.164
|
|
2025-09-16 06:11:40,610: Train batch 81400: loss: 0.00 grad norm: 0.40 time: 0.157
|
|
2025-09-16 06:12:14,749: Train batch 81600: loss: 0.01 grad norm: 1.02 time: 0.119
|
|
2025-09-16 06:12:47,892: Train batch 81800: loss: 0.00 grad norm: 0.06 time: 0.129
|
|
2025-09-16 06:13:21,449: Train batch 82000: loss: 0.01 grad norm: 0.39 time: 0.159
|
|
2025-09-16 06:13:21,449: Running test after training batch: 82000
|
|
2025-09-16 06:13:31,711: Val batch 82000: PER (avg): 0.1079 CTC Loss (avg): 27.5033 time: 10.261
|
|
2025-09-16 06:13:31,711: t15.2023.08.13 val PER: 0.0925
|
|
2025-09-16 06:13:31,711: t15.2023.08.18 val PER: 0.0780
|
|
2025-09-16 06:13:31,711: t15.2023.08.20 val PER: 0.0612
|
|
2025-09-16 06:13:31,711: t15.2023.08.25 val PER: 0.0919
|
|
2025-09-16 06:13:31,712: t15.2023.08.27 val PER: 0.1624
|
|
2025-09-16 06:13:31,712: t15.2023.09.01 val PER: 0.0511
|
|
2025-09-16 06:13:31,712: t15.2023.09.03 val PER: 0.1116
|
|
2025-09-16 06:13:31,712: t15.2023.09.24 val PER: 0.0922
|
|
2025-09-16 06:13:31,712: t15.2023.09.29 val PER: 0.1021
|
|
2025-09-16 06:13:31,712: t15.2023.10.01 val PER: 0.1473
|
|
2025-09-16 06:13:31,712: t15.2023.10.06 val PER: 0.0527
|
|
2025-09-16 06:13:31,712: t15.2023.10.08 val PER: 0.1867
|
|
2025-09-16 06:13:31,712: t15.2023.10.13 val PER: 0.1699
|
|
2025-09-16 06:13:31,712: t15.2023.10.15 val PER: 0.1068
|
|
2025-09-16 06:13:31,712: t15.2023.10.20 val PER: 0.1711
|
|
2025-09-16 06:13:31,712: t15.2023.10.22 val PER: 0.1002
|
|
2025-09-16 06:13:31,712: t15.2023.11.03 val PER: 0.1642
|
|
2025-09-16 06:13:31,712: t15.2023.11.04 val PER: 0.0068
|
|
2025-09-16 06:13:31,712: t15.2023.11.17 val PER: 0.0233
|
|
2025-09-16 06:13:31,712: t15.2023.11.19 val PER: 0.0080
|
|
2025-09-16 06:13:31,712: t15.2023.11.26 val PER: 0.0493
|
|
2025-09-16 06:13:31,712: t15.2023.12.03 val PER: 0.0588
|
|
2025-09-16 06:13:31,712: t15.2023.12.08 val PER: 0.0399
|
|
2025-09-16 06:13:31,712: t15.2023.12.10 val PER: 0.0355
|
|
2025-09-16 06:13:31,713: t15.2023.12.17 val PER: 0.0728
|
|
2025-09-16 06:13:31,713: t15.2023.12.29 val PER: 0.0762
|
|
2025-09-16 06:13:31,713: t15.2024.02.25 val PER: 0.0730
|
|
2025-09-16 06:13:31,713: t15.2024.03.08 val PER: 0.1622
|
|
2025-09-16 06:13:31,713: t15.2024.03.15 val PER: 0.1714
|
|
2025-09-16 06:13:31,713: t15.2024.03.17 val PER: 0.0725
|
|
2025-09-16 06:13:31,713: t15.2024.05.10 val PER: 0.1144
|
|
2025-09-16 06:13:31,713: t15.2024.06.14 val PER: 0.1183
|
|
2025-09-16 06:13:31,713: t15.2024.07.19 val PER: 0.1648
|
|
2025-09-16 06:13:31,713: t15.2024.07.21 val PER: 0.0559
|
|
2025-09-16 06:13:31,713: t15.2024.07.28 val PER: 0.0809
|
|
2025-09-16 06:13:31,713: t15.2025.01.10 val PER: 0.2617
|
|
2025-09-16 06:13:31,713: t15.2025.01.12 val PER: 0.0816
|
|
2025-09-16 06:13:31,713: t15.2025.03.14 val PER: 0.2870
|
|
2025-09-16 06:13:31,713: t15.2025.03.16 val PER: 0.1335
|
|
2025-09-16 06:13:31,713: t15.2025.03.30 val PER: 0.2230
|
|
2025-09-16 06:13:31,713: t15.2025.04.13 val PER: 0.2026
|
|
2025-09-16 06:13:31,713: New best test PER 0.1082 --> 0.1079
|
|
2025-09-16 06:13:31,713: Checkpointing model
|
|
2025-09-16 06:13:33,086: Saved model to checkpoint: trained_models/baseline_rnn/checkpoint/best_checkpoint
|
|
2025-09-16 06:14:01,636: Train batch 82200: loss: 0.00 grad norm: 0.20 time: 0.128
|
|
2025-09-16 06:14:34,389: Train batch 82400: loss: 0.00 grad norm: 0.08 time: 0.174
|
|
2025-09-16 06:15:07,238: Train batch 82600: loss: 0.01 grad norm: 0.71 time: 0.131
|
|
2025-09-16 06:15:40,149: Train batch 82800: loss: 0.01 grad norm: 0.84 time: 0.178
|
|
2025-09-16 06:16:12,123: Train batch 83000: loss: 0.01 grad norm: 1.45 time: 0.115
|
|
2025-09-16 06:16:44,576: Train batch 83200: loss: 0.02 grad norm: 1.73 time: 0.146
|
|
2025-09-16 06:17:17,441: Train batch 83400: loss: 0.01 grad norm: 0.17 time: 0.167
|
|
2025-09-16 06:17:49,903: Train batch 83600: loss: 0.01 grad norm: 0.82 time: 0.157
|
|
2025-09-16 06:18:22,718: Train batch 83800: loss: 0.13 grad norm: 12.02 time: 0.155
|
|
2025-09-16 06:18:56,284: Train batch 84000: loss: 0.01 grad norm: 0.25 time: 0.175
|
|
2025-09-16 06:18:56,284: Running test after training batch: 84000
|
|
2025-09-16 06:19:07,114: Val batch 84000: PER (avg): 0.1084 CTC Loss (avg): 27.3157 time: 10.830
|
|
2025-09-16 06:19:07,114: t15.2023.08.13 val PER: 0.0936
|
|
2025-09-16 06:19:07,114: t15.2023.08.18 val PER: 0.0746
|
|
2025-09-16 06:19:07,114: t15.2023.08.20 val PER: 0.0580
|
|
2025-09-16 06:19:07,114: t15.2023.08.25 val PER: 0.0919
|
|
2025-09-16 06:19:07,115: t15.2023.08.27 val PER: 0.1656
|
|
2025-09-16 06:19:07,115: t15.2023.09.01 val PER: 0.0495
|
|
2025-09-16 06:19:07,115: t15.2023.09.03 val PER: 0.1164
|
|
2025-09-16 06:19:07,115: t15.2023.09.24 val PER: 0.1019
|
|
2025-09-16 06:19:07,115: t15.2023.09.29 val PER: 0.0957
|
|
2025-09-16 06:19:07,115: t15.2023.10.01 val PER: 0.1519
|
|
2025-09-16 06:19:07,115: t15.2023.10.06 val PER: 0.0538
|
|
2025-09-16 06:19:07,115: t15.2023.10.08 val PER: 0.1867
|
|
2025-09-16 06:19:07,115: t15.2023.10.13 val PER: 0.1699
|
|
2025-09-16 06:19:07,115: t15.2023.10.15 val PER: 0.1088
|
|
2025-09-16 06:19:07,115: t15.2023.10.20 val PER: 0.1711
|
|
2025-09-16 06:19:07,115: t15.2023.10.22 val PER: 0.1002
|
|
2025-09-16 06:19:07,115: t15.2023.11.03 val PER: 0.1628
|
|
2025-09-16 06:19:07,115: t15.2023.11.04 val PER: 0.0068
|
|
2025-09-16 06:19:07,115: t15.2023.11.17 val PER: 0.0233
|
|
2025-09-16 06:19:07,115: t15.2023.11.19 val PER: 0.0080
|
|
2025-09-16 06:19:07,115: t15.2023.11.26 val PER: 0.0536
|
|
2025-09-16 06:19:07,115: t15.2023.12.03 val PER: 0.0546
|
|
2025-09-16 06:19:07,115: t15.2023.12.08 val PER: 0.0393
|
|
2025-09-16 06:19:07,115: t15.2023.12.10 val PER: 0.0368
|
|
2025-09-16 06:19:07,116: t15.2023.12.17 val PER: 0.0811
|
|
2025-09-16 06:19:07,116: t15.2023.12.29 val PER: 0.0796
|
|
2025-09-16 06:19:07,116: t15.2024.02.25 val PER: 0.0787
|
|
2025-09-16 06:19:07,116: t15.2024.03.08 val PER: 0.1650
|
|
2025-09-16 06:19:07,116: t15.2024.03.15 val PER: 0.1670
|
|
2025-09-16 06:19:07,116: t15.2024.03.17 val PER: 0.0697
|
|
2025-09-16 06:19:07,116: t15.2024.05.10 val PER: 0.1174
|
|
2025-09-16 06:19:07,116: t15.2024.06.14 val PER: 0.1183
|
|
2025-09-16 06:19:07,116: t15.2024.07.19 val PER: 0.1628
|
|
2025-09-16 06:19:07,116: t15.2024.07.21 val PER: 0.0545
|
|
2025-09-16 06:19:07,116: t15.2024.07.28 val PER: 0.0824
|
|
2025-09-16 06:19:07,116: t15.2025.01.10 val PER: 0.2507
|
|
2025-09-16 06:19:07,116: t15.2025.01.12 val PER: 0.0878
|
|
2025-09-16 06:19:07,116: t15.2025.03.14 val PER: 0.2825
|
|
2025-09-16 06:19:07,116: t15.2025.03.16 val PER: 0.1414
|
|
2025-09-16 06:19:07,116: t15.2025.03.30 val PER: 0.2287
|
|
2025-09-16 06:19:07,116: t15.2025.04.13 val PER: 0.2011
|
|
2025-09-16 06:19:34,602: Train batch 84200: loss: 0.01 grad norm: 0.88 time: 0.140
|
|
2025-09-16 06:20:07,449: Train batch 84400: loss: 0.03 grad norm: 3.56 time: 0.107
|
|
2025-09-16 06:20:39,939: Train batch 84600: loss: 0.00 grad norm: 0.22 time: 0.175
|
|
2025-09-16 06:21:12,697: Train batch 84800: loss: 0.01 grad norm: 0.48 time: 0.139
|
|
2025-09-16 06:21:44,634: Train batch 85000: loss: 0.00 grad norm: 0.09 time: 0.161
|
|
2025-09-16 06:22:17,477: Train batch 85200: loss: 0.00 grad norm: 0.26 time: 0.097
|
|
2025-09-16 06:22:50,228: Train batch 85400: loss: 0.00 grad norm: 0.16 time: 0.134
|
|
2025-09-16 06:23:23,622: Train batch 85600: loss: 0.00 grad norm: 0.26 time: 0.145
|
|
2025-09-16 06:23:56,365: Train batch 85800: loss: 0.00 grad norm: 0.50 time: 0.124
|
|
2025-09-16 06:24:29,660: Train batch 86000: loss: 0.01 grad norm: 1.00 time: 0.117
|
|
2025-09-16 06:24:29,660: Running test after training batch: 86000
|
|
2025-09-16 06:24:40,538: Val batch 86000: PER (avg): 0.1085 CTC Loss (avg): 27.3709 time: 10.878
|
|
2025-09-16 06:24:40,539: t15.2023.08.13 val PER: 0.0863
|
|
2025-09-16 06:24:40,539: t15.2023.08.18 val PER: 0.0754
|
|
2025-09-16 06:24:40,539: t15.2023.08.20 val PER: 0.0620
|
|
2025-09-16 06:24:40,539: t15.2023.08.25 val PER: 0.0919
|
|
2025-09-16 06:24:40,539: t15.2023.08.27 val PER: 0.1656
|
|
2025-09-16 06:24:40,539: t15.2023.09.01 val PER: 0.0463
|
|
2025-09-16 06:24:40,539: t15.2023.09.03 val PER: 0.1105
|
|
2025-09-16 06:24:40,539: t15.2023.09.24 val PER: 0.1032
|
|
2025-09-16 06:24:40,539: t15.2023.09.29 val PER: 0.0964
|
|
2025-09-16 06:24:40,539: t15.2023.10.01 val PER: 0.1486
|
|
2025-09-16 06:24:40,539: t15.2023.10.06 val PER: 0.0517
|
|
2025-09-16 06:24:40,539: t15.2023.10.08 val PER: 0.1867
|
|
2025-09-16 06:24:40,539: t15.2023.10.13 val PER: 0.1691
|
|
2025-09-16 06:24:40,539: t15.2023.10.15 val PER: 0.1101
|
|
2025-09-16 06:24:40,539: t15.2023.10.20 val PER: 0.1711
|
|
2025-09-16 06:24:40,539: t15.2023.10.22 val PER: 0.1024
|
|
2025-09-16 06:24:40,539: t15.2023.11.03 val PER: 0.1662
|
|
2025-09-16 06:24:40,539: t15.2023.11.04 val PER: 0.0068
|
|
2025-09-16 06:24:40,540: t15.2023.11.17 val PER: 0.0233
|
|
2025-09-16 06:24:40,540: t15.2023.11.19 val PER: 0.0080
|
|
2025-09-16 06:24:40,540: t15.2023.11.26 val PER: 0.0493
|
|
2025-09-16 06:24:40,540: t15.2023.12.03 val PER: 0.0620
|
|
2025-09-16 06:24:40,540: t15.2023.12.08 val PER: 0.0413
|
|
2025-09-16 06:24:40,540: t15.2023.12.10 val PER: 0.0368
|
|
2025-09-16 06:24:40,540: t15.2023.12.17 val PER: 0.0800
|
|
2025-09-16 06:24:40,540: t15.2023.12.29 val PER: 0.0796
|
|
2025-09-16 06:24:40,540: t15.2024.02.25 val PER: 0.0744
|
|
2025-09-16 06:24:40,540: t15.2024.03.08 val PER: 0.1707
|
|
2025-09-16 06:24:40,540: t15.2024.03.15 val PER: 0.1664
|
|
2025-09-16 06:24:40,540: t15.2024.03.17 val PER: 0.0676
|
|
2025-09-16 06:24:40,540: t15.2024.05.10 val PER: 0.1293
|
|
2025-09-16 06:24:40,540: t15.2024.06.14 val PER: 0.1309
|
|
2025-09-16 06:24:40,540: t15.2024.07.19 val PER: 0.1668
|
|
2025-09-16 06:24:40,540: t15.2024.07.21 val PER: 0.0524
|
|
2025-09-16 06:24:40,540: t15.2024.07.28 val PER: 0.0860
|
|
2025-09-16 06:24:40,540: t15.2025.01.10 val PER: 0.2466
|
|
2025-09-16 06:24:40,540: t15.2025.01.12 val PER: 0.0808
|
|
2025-09-16 06:24:40,541: t15.2025.03.14 val PER: 0.2781
|
|
2025-09-16 06:24:40,541: t15.2025.03.16 val PER: 0.1453
|
|
2025-09-16 06:24:40,541: t15.2025.03.30 val PER: 0.2287
|
|
2025-09-16 06:24:40,541: t15.2025.04.13 val PER: 0.1997
|
|
2025-09-16 06:25:08,119: Train batch 86200: loss: 0.01 grad norm: 0.58 time: 0.139
|
|
2025-09-16 06:25:41,110: Train batch 86400: loss: 0.00 grad norm: 0.20 time: 0.131
|
|
2025-09-16 06:26:13,100: Train batch 86600: loss: 0.00 grad norm: 0.16 time: 0.123
|
|
2025-09-16 06:26:45,128: Train batch 86800: loss: 0.00 grad norm: 0.11 time: 0.096
|
|
2025-09-16 06:27:17,209: Train batch 87000: loss: 0.00 grad norm: 0.06 time: 0.132
|
|
2025-09-16 06:27:49,343: Train batch 87200: loss: 0.01 grad norm: 0.33 time: 0.144
|
|
2025-09-16 06:28:21,332: Train batch 87400: loss: 0.00 grad norm: 0.06 time: 0.148
|
|
2025-09-16 06:28:53,517: Train batch 87600: loss: 0.01 grad norm: 0.81 time: 0.175
|
|
2025-09-16 06:29:25,580: Train batch 87800: loss: 0.00 grad norm: 0.16 time: 0.160
|
|
2025-09-16 06:29:57,480: Train batch 88000: loss: 0.00 grad norm: 0.08 time: 0.122
|
|
2025-09-16 06:29:57,480: Running test after training batch: 88000
|
|
2025-09-16 06:30:08,422: Val batch 88000: PER (avg): 0.1092 CTC Loss (avg): 27.7245 time: 10.942
|
|
2025-09-16 06:30:08,422: t15.2023.08.13 val PER: 0.0894
|
|
2025-09-16 06:30:08,423: t15.2023.08.18 val PER: 0.0771
|
|
2025-09-16 06:30:08,423: t15.2023.08.20 val PER: 0.0596
|
|
2025-09-16 06:30:08,423: t15.2023.08.25 val PER: 0.0919
|
|
2025-09-16 06:30:08,423: t15.2023.08.27 val PER: 0.1640
|
|
2025-09-16 06:30:08,423: t15.2023.09.01 val PER: 0.0487
|
|
2025-09-16 06:30:08,423: t15.2023.09.03 val PER: 0.1164
|
|
2025-09-16 06:30:08,423: t15.2023.09.24 val PER: 0.1007
|
|
2025-09-16 06:30:08,423: t15.2023.09.29 val PER: 0.1002
|
|
2025-09-16 06:30:08,423: t15.2023.10.01 val PER: 0.1499
|
|
2025-09-16 06:30:08,423: t15.2023.10.06 val PER: 0.0549
|
|
2025-09-16 06:30:08,423: t15.2023.10.08 val PER: 0.1949
|
|
2025-09-16 06:30:08,423: t15.2023.10.13 val PER: 0.1652
|
|
2025-09-16 06:30:08,423: t15.2023.10.15 val PER: 0.1094
|
|
2025-09-16 06:30:08,423: t15.2023.10.20 val PER: 0.1678
|
|
2025-09-16 06:30:08,423: t15.2023.10.22 val PER: 0.1047
|
|
2025-09-16 06:30:08,423: t15.2023.11.03 val PER: 0.1703
|
|
2025-09-16 06:30:08,423: t15.2023.11.04 val PER: 0.0068
|
|
2025-09-16 06:30:08,423: t15.2023.11.17 val PER: 0.0218
|
|
2025-09-16 06:30:08,423: t15.2023.11.19 val PER: 0.0100
|
|
2025-09-16 06:30:08,424: t15.2023.11.26 val PER: 0.0493
|
|
2025-09-16 06:30:08,424: t15.2023.12.03 val PER: 0.0609
|
|
2025-09-16 06:30:08,424: t15.2023.12.08 val PER: 0.0379
|
|
2025-09-16 06:30:08,424: t15.2023.12.10 val PER: 0.0394
|
|
2025-09-16 06:30:08,424: t15.2023.12.17 val PER: 0.0780
|
|
2025-09-16 06:30:08,424: t15.2023.12.29 val PER: 0.0769
|
|
2025-09-16 06:30:08,424: t15.2024.02.25 val PER: 0.0857
|
|
2025-09-16 06:30:08,424: t15.2024.03.08 val PER: 0.1693
|
|
2025-09-16 06:30:08,424: t15.2024.03.15 val PER: 0.1707
|
|
2025-09-16 06:30:08,424: t15.2024.03.17 val PER: 0.0697
|
|
2025-09-16 06:30:08,424: t15.2024.05.10 val PER: 0.1248
|
|
2025-09-16 06:30:08,424: t15.2024.06.14 val PER: 0.1199
|
|
2025-09-16 06:30:08,424: t15.2024.07.19 val PER: 0.1701
|
|
2025-09-16 06:30:08,424: t15.2024.07.21 val PER: 0.0538
|
|
2025-09-16 06:30:08,424: t15.2024.07.28 val PER: 0.0816
|
|
2025-09-16 06:30:08,424: t15.2025.01.10 val PER: 0.2603
|
|
2025-09-16 06:30:08,424: t15.2025.01.12 val PER: 0.0793
|
|
2025-09-16 06:30:08,424: t15.2025.03.14 val PER: 0.2751
|
|
2025-09-16 06:30:08,425: t15.2025.03.16 val PER: 0.1479
|
|
2025-09-16 06:30:08,425: t15.2025.03.30 val PER: 0.2241
|
|
2025-09-16 06:30:08,425: t15.2025.04.13 val PER: 0.2011
|
|
2025-09-16 06:30:36,675: Train batch 88200: loss: 0.01 grad norm: 0.22 time: 0.111
|
|
2025-09-16 06:31:09,322: Train batch 88400: loss: 0.02 grad norm: 2.04 time: 0.166
|
|
2025-09-16 06:31:41,338: Train batch 88600: loss: 0.01 grad norm: 0.86 time: 0.132
|
|
2025-09-16 06:32:13,864: Train batch 88800: loss: 0.00 grad norm: 0.09 time: 0.118
|
|
2025-09-16 06:32:46,601: Train batch 89000: loss: 0.01 grad norm: 0.69 time: 0.126
|
|
2025-09-16 06:33:19,032: Train batch 89200: loss: 0.00 grad norm: 0.07 time: 0.127
|
|
2025-09-16 06:33:51,445: Train batch 89400: loss: 0.00 grad norm: 0.19 time: 0.136
|
|
2025-09-16 06:34:23,875: Train batch 89600: loss: 0.01 grad norm: 1.75 time: 0.111
|
|
2025-09-16 06:34:55,699: Train batch 89800: loss: 0.00 grad norm: 0.28 time: 0.147
|
|
2025-09-16 06:35:28,075: Train batch 90000: loss: 0.01 grad norm: 0.38 time: 0.145
|
|
2025-09-16 06:35:28,075: Running test after training batch: 90000
|
|
2025-09-16 06:35:38,705: Val batch 90000: PER (avg): 0.1089 CTC Loss (avg): 27.4956 time: 10.630
|
|
2025-09-16 06:35:38,706: t15.2023.08.13 val PER: 0.0852
|
|
2025-09-16 06:35:38,706: t15.2023.08.18 val PER: 0.0712
|
|
2025-09-16 06:35:38,706: t15.2023.08.20 val PER: 0.0604
|
|
2025-09-16 06:35:38,706: t15.2023.08.25 val PER: 0.0919
|
|
2025-09-16 06:35:38,706: t15.2023.08.27 val PER: 0.1704
|
|
2025-09-16 06:35:38,706: t15.2023.09.01 val PER: 0.0511
|
|
2025-09-16 06:35:38,706: t15.2023.09.03 val PER: 0.1211
|
|
2025-09-16 06:35:38,706: t15.2023.09.24 val PER: 0.1007
|
|
2025-09-16 06:35:38,706: t15.2023.09.29 val PER: 0.1015
|
|
2025-09-16 06:35:38,706: t15.2023.10.01 val PER: 0.1493
|
|
2025-09-16 06:35:38,706: t15.2023.10.06 val PER: 0.0571
|
|
2025-09-16 06:35:38,706: t15.2023.10.08 val PER: 0.1949
|
|
2025-09-16 06:35:38,706: t15.2023.10.13 val PER: 0.1715
|
|
2025-09-16 06:35:38,706: t15.2023.10.15 val PER: 0.1088
|
|
2025-09-16 06:35:38,706: t15.2023.10.20 val PER: 0.1812
|
|
2025-09-16 06:35:38,707: t15.2023.10.22 val PER: 0.1047
|
|
2025-09-16 06:35:38,707: t15.2023.11.03 val PER: 0.1669
|
|
2025-09-16 06:35:38,707: t15.2023.11.04 val PER: 0.0034
|
|
2025-09-16 06:35:38,707: t15.2023.11.17 val PER: 0.0202
|
|
2025-09-16 06:35:38,707: t15.2023.11.19 val PER: 0.0080
|
|
2025-09-16 06:35:38,707: t15.2023.11.26 val PER: 0.0486
|
|
2025-09-16 06:35:38,707: t15.2023.12.03 val PER: 0.0546
|
|
2025-09-16 06:35:38,707: t15.2023.12.08 val PER: 0.0379
|
|
2025-09-16 06:35:38,707: t15.2023.12.10 val PER: 0.0394
|
|
2025-09-16 06:35:38,707: t15.2023.12.17 val PER: 0.0769
|
|
2025-09-16 06:35:38,707: t15.2023.12.29 val PER: 0.0762
|
|
2025-09-16 06:35:38,707: t15.2024.02.25 val PER: 0.0871
|
|
2025-09-16 06:35:38,707: t15.2024.03.08 val PER: 0.1664
|
|
2025-09-16 06:35:38,707: t15.2024.03.15 val PER: 0.1720
|
|
2025-09-16 06:35:38,707: t15.2024.03.17 val PER: 0.0760
|
|
2025-09-16 06:35:38,707: t15.2024.05.10 val PER: 0.1129
|
|
2025-09-16 06:35:38,707: t15.2024.06.14 val PER: 0.1183
|
|
2025-09-16 06:35:38,707: t15.2024.07.19 val PER: 0.1668
|
|
2025-09-16 06:35:38,707: t15.2024.07.21 val PER: 0.0538
|
|
2025-09-16 06:35:38,707: t15.2024.07.28 val PER: 0.0801
|
|
2025-09-16 06:35:38,708: t15.2025.01.10 val PER: 0.2548
|
|
2025-09-16 06:35:38,708: t15.2025.01.12 val PER: 0.0870
|
|
2025-09-16 06:35:38,708: t15.2025.03.14 val PER: 0.2722
|
|
2025-09-16 06:35:38,708: t15.2025.03.16 val PER: 0.1309
|
|
2025-09-16 06:35:38,708: t15.2025.03.30 val PER: 0.2276
|
|
2025-09-16 06:35:38,708: t15.2025.04.13 val PER: 0.2054
|
|
2025-09-16 06:36:05,524: Train batch 90200: loss: 0.01 grad norm: 0.27 time: 0.118
|
|
2025-09-16 06:36:37,513: Train batch 90400: loss: 0.00 grad norm: 0.31 time: 0.108
|
|
2025-09-16 06:37:08,914: Train batch 90600: loss: 0.00 grad norm: 0.36 time: 0.117
|
|
2025-09-16 06:37:41,571: Train batch 90800: loss: 0.00 grad norm: 0.13 time: 0.153
|
|
2025-09-16 06:38:13,598: Train batch 91000: loss: 0.01 grad norm: 1.33 time: 0.109
|
|
2025-09-16 06:38:45,975: Train batch 91200: loss: 0.00 grad norm: 0.09 time: 0.180
|
|
2025-09-16 06:39:17,835: Train batch 91400: loss: 0.00 grad norm: 0.12 time: 0.152
|
|
2025-09-16 06:39:50,697: Train batch 91600: loss: 0.01 grad norm: 0.44 time: 0.131
|
|
2025-09-16 06:40:23,406: Train batch 91800: loss: 0.00 grad norm: 0.53 time: 0.121
|
|
2025-09-16 06:40:56,664: Train batch 92000: loss: 0.00 grad norm: 0.31 time: 0.124
|
|
2025-09-16 06:40:56,665: Running test after training batch: 92000
|
|
2025-09-16 06:41:07,242: Val batch 92000: PER (avg): 0.1083 CTC Loss (avg): 27.5832 time: 10.577
|
|
2025-09-16 06:41:07,242: t15.2023.08.13 val PER: 0.0936
|
|
2025-09-16 06:41:07,242: t15.2023.08.18 val PER: 0.0738
|
|
2025-09-16 06:41:07,243: t15.2023.08.20 val PER: 0.0548
|
|
2025-09-16 06:41:07,243: t15.2023.08.25 val PER: 0.0919
|
|
2025-09-16 06:41:07,243: t15.2023.08.27 val PER: 0.1785
|
|
2025-09-16 06:41:07,243: t15.2023.09.01 val PER: 0.0503
|
|
2025-09-16 06:41:07,243: t15.2023.09.03 val PER: 0.1164
|
|
2025-09-16 06:41:07,243: t15.2023.09.24 val PER: 0.0934
|
|
2025-09-16 06:41:07,243: t15.2023.09.29 val PER: 0.0996
|
|
2025-09-16 06:41:07,243: t15.2023.10.01 val PER: 0.1453
|
|
2025-09-16 06:41:07,243: t15.2023.10.06 val PER: 0.0571
|
|
2025-09-16 06:41:07,243: t15.2023.10.08 val PER: 0.1922
|
|
2025-09-16 06:41:07,243: t15.2023.10.13 val PER: 0.1652
|
|
2025-09-16 06:41:07,243: t15.2023.10.15 val PER: 0.1088
|
|
2025-09-16 06:41:07,243: t15.2023.10.20 val PER: 0.1745
|
|
2025-09-16 06:41:07,243: t15.2023.10.22 val PER: 0.1058
|
|
2025-09-16 06:41:07,243: t15.2023.11.03 val PER: 0.1662
|
|
2025-09-16 06:41:07,243: t15.2023.11.04 val PER: 0.0068
|
|
2025-09-16 06:41:07,243: t15.2023.11.17 val PER: 0.0218
|
|
2025-09-16 06:41:07,243: t15.2023.11.19 val PER: 0.0080
|
|
2025-09-16 06:41:07,243: t15.2023.11.26 val PER: 0.0478
|
|
2025-09-16 06:41:07,244: t15.2023.12.03 val PER: 0.0609
|
|
2025-09-16 06:41:07,244: t15.2023.12.08 val PER: 0.0406
|
|
2025-09-16 06:41:07,244: t15.2023.12.10 val PER: 0.0394
|
|
2025-09-16 06:41:07,244: t15.2023.12.17 val PER: 0.0780
|
|
2025-09-16 06:41:07,244: t15.2023.12.29 val PER: 0.0837
|
|
2025-09-16 06:41:07,244: t15.2024.02.25 val PER: 0.0815
|
|
2025-09-16 06:41:07,244: t15.2024.03.08 val PER: 0.1650
|
|
2025-09-16 06:41:07,244: t15.2024.03.15 val PER: 0.1720
|
|
2025-09-16 06:41:07,244: t15.2024.03.17 val PER: 0.0760
|
|
2025-09-16 06:41:07,244: t15.2024.05.10 val PER: 0.1174
|
|
2025-09-16 06:41:07,244: t15.2024.06.14 val PER: 0.1183
|
|
2025-09-16 06:41:07,244: t15.2024.07.19 val PER: 0.1681
|
|
2025-09-16 06:41:07,244: t15.2024.07.21 val PER: 0.0572
|
|
2025-09-16 06:41:07,244: t15.2024.07.28 val PER: 0.0765
|
|
2025-09-16 06:41:07,244: t15.2025.01.10 val PER: 0.2452
|
|
2025-09-16 06:41:07,244: t15.2025.01.12 val PER: 0.0785
|
|
2025-09-16 06:41:07,244: t15.2025.03.14 val PER: 0.2751
|
|
2025-09-16 06:41:07,244: t15.2025.03.16 val PER: 0.1270
|
|
2025-09-16 06:41:07,244: t15.2025.03.30 val PER: 0.2264
|
|
2025-09-16 06:41:07,245: t15.2025.04.13 val PER: 0.1997
|
|
2025-09-16 06:41:35,676: Train batch 92200: loss: 0.02 grad norm: 3.97 time: 0.180
|
|
2025-09-16 06:42:07,431: Train batch 92400: loss: 0.00 grad norm: 0.10 time: 0.161
|
|
2025-09-16 06:42:39,661: Train batch 92600: loss: 0.01 grad norm: 0.32 time: 0.177
|
|
2025-09-16 06:43:10,756: Train batch 92800: loss: 0.01 grad norm: 1.65 time: 0.157
|
|
2025-09-16 06:43:43,887: Train batch 93000: loss: 0.00 grad norm: 0.13 time: 0.175
|
|
2025-09-16 06:44:15,975: Train batch 93200: loss: 0.01 grad norm: 0.73 time: 0.117
|
|
2025-09-16 06:44:48,526: Train batch 93400: loss: 0.01 grad norm: 0.41 time: 0.134
|
|
2025-09-16 06:45:21,521: Train batch 93600: loss: 0.00 grad norm: 0.17 time: 0.170
|
|
2025-09-16 06:45:53,556: Train batch 93800: loss: 0.01 grad norm: 0.74 time: 0.137
|
|
2025-09-16 06:46:25,792: Train batch 94000: loss: 0.00 grad norm: 0.19 time: 0.129
|
|
2025-09-16 06:46:25,793: Running test after training batch: 94000
|
|
2025-09-16 06:46:36,498: Val batch 94000: PER (avg): 0.1080 CTC Loss (avg): 27.7370 time: 10.705
|
|
2025-09-16 06:46:36,498: t15.2023.08.13 val PER: 0.0894
|
|
2025-09-16 06:46:36,498: t15.2023.08.18 val PER: 0.0721
|
|
2025-09-16 06:46:36,498: t15.2023.08.20 val PER: 0.0572
|
|
2025-09-16 06:46:36,498: t15.2023.08.25 val PER: 0.0919
|
|
2025-09-16 06:46:36,498: t15.2023.08.27 val PER: 0.1736
|
|
2025-09-16 06:46:36,498: t15.2023.09.01 val PER: 0.0495
|
|
2025-09-16 06:46:36,498: t15.2023.09.03 val PER: 0.1128
|
|
2025-09-16 06:46:36,499: t15.2023.09.24 val PER: 0.0959
|
|
2025-09-16 06:46:36,499: t15.2023.09.29 val PER: 0.0970
|
|
2025-09-16 06:46:36,499: t15.2023.10.01 val PER: 0.1473
|
|
2025-09-16 06:46:36,499: t15.2023.10.06 val PER: 0.0592
|
|
2025-09-16 06:46:36,499: t15.2023.10.08 val PER: 0.1922
|
|
2025-09-16 06:46:36,499: t15.2023.10.13 val PER: 0.1660
|
|
2025-09-16 06:46:36,499: t15.2023.10.15 val PER: 0.1022
|
|
2025-09-16 06:46:36,499: t15.2023.10.20 val PER: 0.1711
|
|
2025-09-16 06:46:36,499: t15.2023.10.22 val PER: 0.1036
|
|
2025-09-16 06:46:36,499: t15.2023.11.03 val PER: 0.1642
|
|
2025-09-16 06:46:36,499: t15.2023.11.04 val PER: 0.0102
|
|
2025-09-16 06:46:36,499: t15.2023.11.17 val PER: 0.0249
|
|
2025-09-16 06:46:36,499: t15.2023.11.19 val PER: 0.0120
|
|
2025-09-16 06:46:36,499: t15.2023.11.26 val PER: 0.0478
|
|
2025-09-16 06:46:36,499: t15.2023.12.03 val PER: 0.0609
|
|
2025-09-16 06:46:36,499: t15.2023.12.08 val PER: 0.0393
|
|
2025-09-16 06:46:36,499: t15.2023.12.10 val PER: 0.0381
|
|
2025-09-16 06:46:36,499: t15.2023.12.17 val PER: 0.0842
|
|
2025-09-16 06:46:36,499: t15.2023.12.29 val PER: 0.0803
|
|
2025-09-16 06:46:36,500: t15.2024.02.25 val PER: 0.0843
|
|
2025-09-16 06:46:36,500: t15.2024.03.08 val PER: 0.1679
|
|
2025-09-16 06:46:36,500: t15.2024.03.15 val PER: 0.1745
|
|
2025-09-16 06:46:36,500: t15.2024.03.17 val PER: 0.0746
|
|
2025-09-16 06:46:36,500: t15.2024.05.10 val PER: 0.1204
|
|
2025-09-16 06:46:36,500: t15.2024.06.14 val PER: 0.1246
|
|
2025-09-16 06:46:36,500: t15.2024.07.19 val PER: 0.1635
|
|
2025-09-16 06:46:36,500: t15.2024.07.21 val PER: 0.0524
|
|
2025-09-16 06:46:36,500: t15.2024.07.28 val PER: 0.0794
|
|
2025-09-16 06:46:36,500: t15.2025.01.10 val PER: 0.2493
|
|
2025-09-16 06:46:36,500: t15.2025.01.12 val PER: 0.0855
|
|
2025-09-16 06:46:36,500: t15.2025.03.14 val PER: 0.2692
|
|
2025-09-16 06:46:36,500: t15.2025.03.16 val PER: 0.1283
|
|
2025-09-16 06:46:36,500: t15.2025.03.30 val PER: 0.2207
|
|
2025-09-16 06:46:36,500: t15.2025.04.13 val PER: 0.2026
|
|
2025-09-16 06:47:04,152: Train batch 94200: loss: 0.00 grad norm: 0.08 time: 0.102
|
|
2025-09-16 06:47:36,199: Train batch 94400: loss: 0.00 grad norm: 0.14 time: 0.132
|
|
2025-09-16 06:48:08,154: Train batch 94600: loss: 0.00 grad norm: 0.26 time: 0.115
|
|
2025-09-16 06:48:40,407: Train batch 94800: loss: 0.08 grad norm: 0.80 time: 0.145
|
|
2025-09-16 06:49:12,141: Train batch 95000: loss: 0.01 grad norm: 0.29 time: 0.140
|
|
2025-09-16 06:49:44,470: Train batch 95200: loss: 0.01 grad norm: 0.58 time: 0.107
|
|
2025-09-16 06:50:16,202: Train batch 95400: loss: 0.00 grad norm: 0.12 time: 0.114
|
|
2025-09-16 06:50:48,773: Train batch 95600: loss: 0.01 grad norm: 0.72 time: 0.172
|
|
2025-09-16 06:51:21,242: Train batch 95800: loss: 0.01 grad norm: 1.98 time: 0.132
|
|
2025-09-16 06:51:53,698: Train batch 96000: loss: 0.00 grad norm: 0.12 time: 0.126
|
|
2025-09-16 06:51:53,698: Running test after training batch: 96000
|
|
2025-09-16 06:52:04,686: Val batch 96000: PER (avg): 0.1079 CTC Loss (avg): 27.6534 time: 10.988
|
|
2025-09-16 06:52:04,686: t15.2023.08.13 val PER: 0.0873
|
|
2025-09-16 06:52:04,686: t15.2023.08.18 val PER: 0.0746
|
|
2025-09-16 06:52:04,686: t15.2023.08.20 val PER: 0.0564
|
|
2025-09-16 06:52:04,686: t15.2023.08.25 val PER: 0.0904
|
|
2025-09-16 06:52:04,686: t15.2023.08.27 val PER: 0.1736
|
|
2025-09-16 06:52:04,686: t15.2023.09.01 val PER: 0.0536
|
|
2025-09-16 06:52:04,686: t15.2023.09.03 val PER: 0.1176
|
|
2025-09-16 06:52:04,686: t15.2023.09.24 val PER: 0.0983
|
|
2025-09-16 06:52:04,687: t15.2023.09.29 val PER: 0.0983
|
|
2025-09-16 06:52:04,687: t15.2023.10.01 val PER: 0.1480
|
|
2025-09-16 06:52:04,687: t15.2023.10.06 val PER: 0.0592
|
|
2025-09-16 06:52:04,687: t15.2023.10.08 val PER: 0.1922
|
|
2025-09-16 06:52:04,687: t15.2023.10.13 val PER: 0.1645
|
|
2025-09-16 06:52:04,687: t15.2023.10.15 val PER: 0.1081
|
|
2025-09-16 06:52:04,687: t15.2023.10.20 val PER: 0.1745
|
|
2025-09-16 06:52:04,687: t15.2023.10.22 val PER: 0.1036
|
|
2025-09-16 06:52:04,687: t15.2023.11.03 val PER: 0.1642
|
|
2025-09-16 06:52:04,687: t15.2023.11.04 val PER: 0.0068
|
|
2025-09-16 06:52:04,687: t15.2023.11.17 val PER: 0.0218
|
|
2025-09-16 06:52:04,687: t15.2023.11.19 val PER: 0.0100
|
|
2025-09-16 06:52:04,687: t15.2023.11.26 val PER: 0.0493
|
|
2025-09-16 06:52:04,687: t15.2023.12.03 val PER: 0.0578
|
|
2025-09-16 06:52:04,687: t15.2023.12.08 val PER: 0.0406
|
|
2025-09-16 06:52:04,687: t15.2023.12.10 val PER: 0.0407
|
|
2025-09-16 06:52:04,687: t15.2023.12.17 val PER: 0.0790
|
|
2025-09-16 06:52:04,687: t15.2023.12.29 val PER: 0.0796
|
|
2025-09-16 06:52:04,687: t15.2024.02.25 val PER: 0.0857
|
|
2025-09-16 06:52:04,687: t15.2024.03.08 val PER: 0.1664
|
|
2025-09-16 06:52:04,688: t15.2024.03.15 val PER: 0.1707
|
|
2025-09-16 06:52:04,688: t15.2024.03.17 val PER: 0.0732
|
|
2025-09-16 06:52:04,688: t15.2024.05.10 val PER: 0.1174
|
|
2025-09-16 06:52:04,688: t15.2024.06.14 val PER: 0.1246
|
|
2025-09-16 06:52:04,688: t15.2024.07.19 val PER: 0.1648
|
|
2025-09-16 06:52:04,688: t15.2024.07.21 val PER: 0.0531
|
|
2025-09-16 06:52:04,688: t15.2024.07.28 val PER: 0.0809
|
|
2025-09-16 06:52:04,688: t15.2025.01.10 val PER: 0.2479
|
|
2025-09-16 06:52:04,688: t15.2025.01.12 val PER: 0.0754
|
|
2025-09-16 06:52:04,688: t15.2025.03.14 val PER: 0.2737
|
|
2025-09-16 06:52:04,688: t15.2025.03.16 val PER: 0.1257
|
|
2025-09-16 06:52:04,688: t15.2025.03.30 val PER: 0.2172
|
|
2025-09-16 06:52:04,688: t15.2025.04.13 val PER: 0.2026
|
|
2025-09-16 06:52:04,688: New best test PER 0.1079 --> 0.1079
|
|
2025-09-16 06:52:04,688: Checkpointing model
|
|
2025-09-16 06:52:05,992: Saved model to checkpoint: trained_models/baseline_rnn/checkpoint/best_checkpoint
|
|
2025-09-16 06:52:33,912: Train batch 96200: loss: 0.00 grad norm: 0.11 time: 0.145
|
|
2025-09-16 06:53:06,262: Train batch 96400: loss: 0.00 grad norm: 0.41 time: 0.095
|
|
2025-09-16 06:53:38,141: Train batch 96600: loss: 0.01 grad norm: 0.76 time: 0.158
|
|
2025-09-16 06:54:10,322: Train batch 96800: loss: 0.01 grad norm: 0.25 time: 0.110
|
|
2025-09-16 06:54:42,427: Train batch 97000: loss: 0.01 grad norm: 0.47 time: 0.172
|
|
2025-09-16 06:55:14,152: Train batch 97200: loss: 0.01 grad norm: 1.05 time: 0.119
|
|
2025-09-16 06:55:46,792: Train batch 97400: loss: 0.01 grad norm: 1.28 time: 0.155
|
|
2025-09-16 06:56:19,555: Train batch 97600: loss: 0.00 grad norm: 0.14 time: 0.117
|
|
2025-09-16 06:56:51,958: Train batch 97800: loss: 0.00 grad norm: 0.06 time: 0.136
|
|
2025-09-16 06:57:24,609: Train batch 98000: loss: 0.00 grad norm: 0.10 time: 0.134
|
|
2025-09-16 06:57:24,609: Running test after training batch: 98000
|
|
2025-09-16 06:57:35,324: Val batch 98000: PER (avg): 0.1078 CTC Loss (avg): 27.5792 time: 10.714
|
|
2025-09-16 06:57:35,324: t15.2023.08.13 val PER: 0.0842
|
|
2025-09-16 06:57:35,324: t15.2023.08.18 val PER: 0.0729
|
|
2025-09-16 06:57:35,324: t15.2023.08.20 val PER: 0.0588
|
|
2025-09-16 06:57:35,325: t15.2023.08.25 val PER: 0.0904
|
|
2025-09-16 06:57:35,325: t15.2023.08.27 val PER: 0.1704
|
|
2025-09-16 06:57:35,325: t15.2023.09.01 val PER: 0.0503
|
|
2025-09-16 06:57:35,325: t15.2023.09.03 val PER: 0.1105
|
|
2025-09-16 06:57:35,325: t15.2023.09.24 val PER: 0.0947
|
|
2025-09-16 06:57:35,325: t15.2023.09.29 val PER: 0.1021
|
|
2025-09-16 06:57:35,325: t15.2023.10.01 val PER: 0.1480
|
|
2025-09-16 06:57:35,325: t15.2023.10.06 val PER: 0.0614
|
|
2025-09-16 06:57:35,325: t15.2023.10.08 val PER: 0.1908
|
|
2025-09-16 06:57:35,325: t15.2023.10.13 val PER: 0.1637
|
|
2025-09-16 06:57:35,325: t15.2023.10.15 val PER: 0.1048
|
|
2025-09-16 06:57:35,325: t15.2023.10.20 val PER: 0.1745
|
|
2025-09-16 06:57:35,325: t15.2023.10.22 val PER: 0.1058
|
|
2025-09-16 06:57:35,325: t15.2023.11.03 val PER: 0.1594
|
|
2025-09-16 06:57:35,325: t15.2023.11.04 val PER: 0.0068
|
|
2025-09-16 06:57:35,325: t15.2023.11.17 val PER: 0.0218
|
|
2025-09-16 06:57:35,325: t15.2023.11.19 val PER: 0.0120
|
|
2025-09-16 06:57:35,325: t15.2023.11.26 val PER: 0.0493
|
|
2025-09-16 06:57:35,325: t15.2023.12.03 val PER: 0.0578
|
|
2025-09-16 06:57:35,325: t15.2023.12.08 val PER: 0.0413
|
|
2025-09-16 06:57:35,326: t15.2023.12.10 val PER: 0.0420
|
|
2025-09-16 06:57:35,326: t15.2023.12.17 val PER: 0.0863
|
|
2025-09-16 06:57:35,326: t15.2023.12.29 val PER: 0.0755
|
|
2025-09-16 06:57:35,326: t15.2024.02.25 val PER: 0.0829
|
|
2025-09-16 06:57:35,326: t15.2024.03.08 val PER: 0.1693
|
|
2025-09-16 06:57:35,326: t15.2024.03.15 val PER: 0.1751
|
|
2025-09-16 06:57:35,326: t15.2024.03.17 val PER: 0.0690
|
|
2025-09-16 06:57:35,326: t15.2024.05.10 val PER: 0.1204
|
|
2025-09-16 06:57:35,326: t15.2024.06.14 val PER: 0.1215
|
|
2025-09-16 06:57:35,326: t15.2024.07.19 val PER: 0.1655
|
|
2025-09-16 06:57:35,326: t15.2024.07.21 val PER: 0.0531
|
|
2025-09-16 06:57:35,326: t15.2024.07.28 val PER: 0.0824
|
|
2025-09-16 06:57:35,326: t15.2025.01.10 val PER: 0.2438
|
|
2025-09-16 06:57:35,326: t15.2025.01.12 val PER: 0.0801
|
|
2025-09-16 06:57:35,326: t15.2025.03.14 val PER: 0.2751
|
|
2025-09-16 06:57:35,326: t15.2025.03.16 val PER: 0.1230
|
|
2025-09-16 06:57:35,326: t15.2025.03.30 val PER: 0.2230
|
|
2025-09-16 06:57:35,326: t15.2025.04.13 val PER: 0.2040
|
|
2025-09-16 06:57:35,326: New best test PER 0.1079 --> 0.1078
|
|
2025-09-16 06:57:35,326: Checkpointing model
|
|
2025-09-16 06:57:36,663: Saved model to checkpoint: trained_models/baseline_rnn/checkpoint/best_checkpoint
|
|
2025-09-16 06:58:04,611: Train batch 98200: loss: 0.01 grad norm: 0.60 time: 0.125
|
|
2025-09-16 06:58:37,090: Train batch 98400: loss: 0.00 grad norm: 0.16 time: 0.147
|
|
2025-09-16 06:59:09,148: Train batch 98600: loss: 0.00 grad norm: 0.55 time: 0.126
|
|
2025-09-16 06:59:41,806: Train batch 98800: loss: 0.00 grad norm: 0.15 time: 0.156
|
|
2025-09-16 07:00:14,220: Train batch 99000: loss: 0.00 grad norm: 0.25 time: 0.123
|
|
2025-09-16 07:00:47,188: Train batch 99200: loss: 0.00 grad norm: 0.11 time: 0.145
|
|
2025-09-16 07:01:19,764: Train batch 99400: loss: 0.00 grad norm: 0.07 time: 0.130
|
|
2025-09-16 07:01:52,194: Train batch 99600: loss: 0.00 grad norm: 0.26 time: 0.134
|
|
2025-09-16 07:02:24,810: Train batch 99800: loss: 0.03 grad norm: 4.62 time: 0.119
|
|
2025-09-16 07:02:56,775: Train batch 100000: loss: 0.01 grad norm: 0.28 time: 0.121
|
|
2025-09-16 07:02:56,776: Running test after training batch: 100000
|
|
2025-09-16 07:03:07,480: Val batch 100000: PER (avg): 0.1077 CTC Loss (avg): 27.4728 time: 10.705
|
|
2025-09-16 07:03:07,481: t15.2023.08.13 val PER: 0.0821
|
|
2025-09-16 07:03:07,481: t15.2023.08.18 val PER: 0.0721
|
|
2025-09-16 07:03:07,481: t15.2023.08.20 val PER: 0.0572
|
|
2025-09-16 07:03:07,481: t15.2023.08.25 val PER: 0.0904
|
|
2025-09-16 07:03:07,481: t15.2023.08.27 val PER: 0.1752
|
|
2025-09-16 07:03:07,481: t15.2023.09.01 val PER: 0.0511
|
|
2025-09-16 07:03:07,481: t15.2023.09.03 val PER: 0.1116
|
|
2025-09-16 07:03:07,481: t15.2023.09.24 val PER: 0.0959
|
|
2025-09-16 07:03:07,481: t15.2023.09.29 val PER: 0.1008
|
|
2025-09-16 07:03:07,481: t15.2023.10.01 val PER: 0.1453
|
|
2025-09-16 07:03:07,482: t15.2023.10.06 val PER: 0.0571
|
|
2025-09-16 07:03:07,482: t15.2023.10.08 val PER: 0.1908
|
|
2025-09-16 07:03:07,482: t15.2023.10.13 val PER: 0.1621
|
|
2025-09-16 07:03:07,482: t15.2023.10.15 val PER: 0.1061
|
|
2025-09-16 07:03:07,482: t15.2023.10.20 val PER: 0.1779
|
|
2025-09-16 07:03:07,482: t15.2023.10.22 val PER: 0.1047
|
|
2025-09-16 07:03:07,482: t15.2023.11.03 val PER: 0.1655
|
|
2025-09-16 07:03:07,482: t15.2023.11.04 val PER: 0.0068
|
|
2025-09-16 07:03:07,482: t15.2023.11.17 val PER: 0.0233
|
|
2025-09-16 07:03:07,482: t15.2023.11.19 val PER: 0.0120
|
|
2025-09-16 07:03:07,482: t15.2023.11.26 val PER: 0.0471
|
|
2025-09-16 07:03:07,482: t15.2023.12.03 val PER: 0.0599
|
|
2025-09-16 07:03:07,482: t15.2023.12.08 val PER: 0.0406
|
|
2025-09-16 07:03:07,482: t15.2023.12.10 val PER: 0.0381
|
|
2025-09-16 07:03:07,482: t15.2023.12.17 val PER: 0.0821
|
|
2025-09-16 07:03:07,482: t15.2023.12.29 val PER: 0.0789
|
|
2025-09-16 07:03:07,482: t15.2024.02.25 val PER: 0.0899
|
|
2025-09-16 07:03:07,482: t15.2024.03.08 val PER: 0.1607
|
|
2025-09-16 07:03:07,482: t15.2024.03.15 val PER: 0.1739
|
|
2025-09-16 07:03:07,483: t15.2024.03.17 val PER: 0.0725
|
|
2025-09-16 07:03:07,483: t15.2024.05.10 val PER: 0.1189
|
|
2025-09-16 07:03:07,483: t15.2024.06.14 val PER: 0.1293
|
|
2025-09-16 07:03:07,483: t15.2024.07.19 val PER: 0.1622
|
|
2025-09-16 07:03:07,483: t15.2024.07.21 val PER: 0.0559
|
|
2025-09-16 07:03:07,483: t15.2024.07.28 val PER: 0.0765
|
|
2025-09-16 07:03:07,483: t15.2025.01.10 val PER: 0.2424
|
|
2025-09-16 07:03:07,483: t15.2025.01.12 val PER: 0.0816
|
|
2025-09-16 07:03:07,483: t15.2025.03.14 val PER: 0.2678
|
|
2025-09-16 07:03:07,483: t15.2025.03.16 val PER: 0.1270
|
|
2025-09-16 07:03:07,483: t15.2025.03.30 val PER: 0.2276
|
|
2025-09-16 07:03:07,483: t15.2025.04.13 val PER: 0.2026
|
|
2025-09-16 07:03:07,483: New best test PER 0.1078 --> 0.1077
|
|
2025-09-16 07:03:07,483: Checkpointing model
|
|
2025-09-16 07:03:08,773: Saved model to checkpoint: trained_models/baseline_rnn/checkpoint/best_checkpoint
|
|
2025-09-16 07:03:37,330: Train batch 100200: loss: 0.03 grad norm: 5.07 time: 0.151
|
|
2025-09-16 07:04:10,779: Train batch 100400: loss: 0.00 grad norm: 0.35 time: 0.127
|
|
2025-09-16 07:04:43,112: Train batch 100600: loss: 0.01 grad norm: 0.84 time: 0.154
|
|
2025-09-16 07:05:15,609: Train batch 100800: loss: 0.01 grad norm: 0.35 time: 0.155
|
|
2025-09-16 07:05:48,015: Train batch 101000: loss: 0.00 grad norm: 0.07 time: 0.143
|
|
2025-09-16 07:06:20,888: Train batch 101200: loss: 0.00 grad norm: 0.10 time: 0.116
|
|
2025-09-16 07:06:53,612: Train batch 101400: loss: 0.00 grad norm: 0.49 time: 0.157
|
|
2025-09-16 07:07:26,208: Train batch 101600: loss: 0.00 grad norm: 0.19 time: 0.127
|
|
2025-09-16 07:07:58,766: Train batch 101800: loss: 0.00 grad norm: 0.10 time: 0.141
|
|
2025-09-16 07:08:31,142: Train batch 102000: loss: 0.00 grad norm: 0.24 time: 0.163
|
|
2025-09-16 07:08:31,143: Running test after training batch: 102000
|
|
2025-09-16 07:08:41,838: Val batch 102000: PER (avg): 0.1085 CTC Loss (avg): 27.5049 time: 10.695
|
|
2025-09-16 07:08:41,839: t15.2023.08.13 val PER: 0.0863
|
|
2025-09-16 07:08:41,839: t15.2023.08.18 val PER: 0.0729
|
|
2025-09-16 07:08:41,839: t15.2023.08.20 val PER: 0.0588
|
|
2025-09-16 07:08:41,839: t15.2023.08.25 val PER: 0.0904
|
|
2025-09-16 07:08:41,839: t15.2023.08.27 val PER: 0.1672
|
|
2025-09-16 07:08:41,839: t15.2023.09.01 val PER: 0.0511
|
|
2025-09-16 07:08:41,839: t15.2023.09.03 val PER: 0.1128
|
|
2025-09-16 07:08:41,839: t15.2023.09.24 val PER: 0.1032
|
|
2025-09-16 07:08:41,839: t15.2023.09.29 val PER: 0.1002
|
|
2025-09-16 07:08:41,839: t15.2023.10.01 val PER: 0.1493
|
|
2025-09-16 07:08:41,839: t15.2023.10.06 val PER: 0.0571
|
|
2025-09-16 07:08:41,839: t15.2023.10.08 val PER: 0.1908
|
|
2025-09-16 07:08:41,840: t15.2023.10.13 val PER: 0.1637
|
|
2025-09-16 07:08:41,840: t15.2023.10.15 val PER: 0.1074
|
|
2025-09-16 07:08:41,840: t15.2023.10.20 val PER: 0.1711
|
|
2025-09-16 07:08:41,840: t15.2023.10.22 val PER: 0.1058
|
|
2025-09-16 07:08:41,840: t15.2023.11.03 val PER: 0.1635
|
|
2025-09-16 07:08:41,840: t15.2023.11.04 val PER: 0.0068
|
|
2025-09-16 07:08:41,840: t15.2023.11.17 val PER: 0.0218
|
|
2025-09-16 07:08:41,840: t15.2023.11.19 val PER: 0.0100
|
|
2025-09-16 07:08:41,840: t15.2023.11.26 val PER: 0.0514
|
|
2025-09-16 07:08:41,840: t15.2023.12.03 val PER: 0.0588
|
|
2025-09-16 07:08:41,840: t15.2023.12.08 val PER: 0.0413
|
|
2025-09-16 07:08:41,840: t15.2023.12.10 val PER: 0.0407
|
|
2025-09-16 07:08:41,840: t15.2023.12.17 val PER: 0.0790
|
|
2025-09-16 07:08:41,840: t15.2023.12.29 val PER: 0.0817
|
|
2025-09-16 07:08:41,840: t15.2024.02.25 val PER: 0.0927
|
|
2025-09-16 07:08:41,840: t15.2024.03.08 val PER: 0.1664
|
|
2025-09-16 07:08:41,840: t15.2024.03.15 val PER: 0.1701
|
|
2025-09-16 07:08:41,840: t15.2024.03.17 val PER: 0.0704
|
|
2025-09-16 07:08:41,840: t15.2024.05.10 val PER: 0.1263
|
|
2025-09-16 07:08:41,841: t15.2024.06.14 val PER: 0.1262
|
|
2025-09-16 07:08:41,841: t15.2024.07.19 val PER: 0.1628
|
|
2025-09-16 07:08:41,841: t15.2024.07.21 val PER: 0.0538
|
|
2025-09-16 07:08:41,841: t15.2024.07.28 val PER: 0.0824
|
|
2025-09-16 07:08:41,841: t15.2025.01.10 val PER: 0.2410
|
|
2025-09-16 07:08:41,841: t15.2025.01.12 val PER: 0.0831
|
|
2025-09-16 07:08:41,841: t15.2025.03.14 val PER: 0.2648
|
|
2025-09-16 07:08:41,841: t15.2025.03.16 val PER: 0.1257
|
|
2025-09-16 07:08:41,841: t15.2025.03.30 val PER: 0.2333
|
|
2025-09-16 07:08:41,841: t15.2025.04.13 val PER: 0.2083
|
|
2025-09-16 07:09:09,949: Train batch 102200: loss: 0.00 grad norm: 0.09 time: 0.117
|
|
2025-09-16 07:09:42,500: Train batch 102400: loss: 0.00 grad norm: 0.07 time: 0.169
|
|
2025-09-16 07:10:14,908: Train batch 102600: loss: 0.01 grad norm: 0.22 time: 0.119
|
|
2025-09-16 07:10:47,562: Train batch 102800: loss: 0.01 grad norm: 1.57 time: 0.115
|
|
2025-09-16 07:11:18,926: Train batch 103000: loss: 0.02 grad norm: 0.92 time: 0.122
|
|
2025-09-16 07:11:51,742: Train batch 103200: loss: 0.00 grad norm: 0.10 time: 0.142
|
|
2025-09-16 07:12:24,030: Train batch 103400: loss: 0.01 grad norm: 0.66 time: 0.170
|
|
2025-09-16 07:12:56,537: Train batch 103600: loss: 0.00 grad norm: 0.27 time: 0.129
|
|
2025-09-16 07:13:28,940: Train batch 103800: loss: 0.00 grad norm: 0.27 time: 0.177
|
|
2025-09-16 07:14:01,369: Train batch 104000: loss: 0.01 grad norm: 0.25 time: 0.100
|
|
2025-09-16 07:14:01,369: Running test after training batch: 104000
|
|
2025-09-16 07:14:12,011: Val batch 104000: PER (avg): 0.1078 CTC Loss (avg): 27.3648 time: 10.642
|
|
2025-09-16 07:14:12,011: t15.2023.08.13 val PER: 0.0842
|
|
2025-09-16 07:14:12,012: t15.2023.08.18 val PER: 0.0771
|
|
2025-09-16 07:14:12,012: t15.2023.08.20 val PER: 0.0572
|
|
2025-09-16 07:14:12,012: t15.2023.08.25 val PER: 0.0919
|
|
2025-09-16 07:14:12,012: t15.2023.08.27 val PER: 0.1704
|
|
2025-09-16 07:14:12,012: t15.2023.09.01 val PER: 0.0503
|
|
2025-09-16 07:14:12,012: t15.2023.09.03 val PER: 0.1081
|
|
2025-09-16 07:14:12,012: t15.2023.09.24 val PER: 0.0922
|
|
2025-09-16 07:14:12,012: t15.2023.09.29 val PER: 0.0996
|
|
2025-09-16 07:14:12,012: t15.2023.10.01 val PER: 0.1473
|
|
2025-09-16 07:14:12,012: t15.2023.10.06 val PER: 0.0538
|
|
2025-09-16 07:14:12,012: t15.2023.10.08 val PER: 0.1894
|
|
2025-09-16 07:14:12,012: t15.2023.10.13 val PER: 0.1629
|
|
2025-09-16 07:14:12,012: t15.2023.10.15 val PER: 0.1074
|
|
2025-09-16 07:14:12,012: t15.2023.10.20 val PER: 0.1745
|
|
2025-09-16 07:14:12,012: t15.2023.10.22 val PER: 0.1069
|
|
2025-09-16 07:14:12,012: t15.2023.11.03 val PER: 0.1588
|
|
2025-09-16 07:14:12,012: t15.2023.11.04 val PER: 0.0068
|
|
2025-09-16 07:14:12,012: t15.2023.11.17 val PER: 0.0233
|
|
2025-09-16 07:14:12,013: t15.2023.11.19 val PER: 0.0080
|
|
2025-09-16 07:14:12,013: t15.2023.11.26 val PER: 0.0478
|
|
2025-09-16 07:14:12,013: t15.2023.12.03 val PER: 0.0609
|
|
2025-09-16 07:14:12,013: t15.2023.12.08 val PER: 0.0406
|
|
2025-09-16 07:14:12,013: t15.2023.12.10 val PER: 0.0407
|
|
2025-09-16 07:14:12,013: t15.2023.12.17 val PER: 0.0832
|
|
2025-09-16 07:14:12,013: t15.2023.12.29 val PER: 0.0824
|
|
2025-09-16 07:14:12,013: t15.2024.02.25 val PER: 0.0829
|
|
2025-09-16 07:14:12,013: t15.2024.03.08 val PER: 0.1679
|
|
2025-09-16 07:14:12,013: t15.2024.03.15 val PER: 0.1701
|
|
2025-09-16 07:14:12,013: t15.2024.03.17 val PER: 0.0711
|
|
2025-09-16 07:14:12,013: t15.2024.05.10 val PER: 0.1293
|
|
2025-09-16 07:14:12,013: t15.2024.06.14 val PER: 0.1262
|
|
2025-09-16 07:14:12,013: t15.2024.07.19 val PER: 0.1622
|
|
2025-09-16 07:14:12,013: t15.2024.07.21 val PER: 0.0545
|
|
2025-09-16 07:14:12,013: t15.2024.07.28 val PER: 0.0801
|
|
2025-09-16 07:14:12,013: t15.2025.01.10 val PER: 0.2521
|
|
2025-09-16 07:14:12,013: t15.2025.01.12 val PER: 0.0816
|
|
2025-09-16 07:14:12,013: t15.2025.03.14 val PER: 0.2707
|
|
2025-09-16 07:14:12,014: t15.2025.03.16 val PER: 0.1257
|
|
2025-09-16 07:14:12,014: t15.2025.03.30 val PER: 0.2241
|
|
2025-09-16 07:14:12,014: t15.2025.04.13 val PER: 0.2068
|
|
2025-09-16 07:14:39,365: Train batch 104200: loss: 0.00 grad norm: 0.16 time: 0.138
|
|
2025-09-16 07:15:11,189: Train batch 104400: loss: 0.00 grad norm: 0.20 time: 0.108
|
|
2025-09-16 07:15:43,785: Train batch 104600: loss: 0.00 grad norm: 0.15 time: 0.156
|
|
2025-09-16 07:16:16,690: Train batch 104800: loss: 0.00 grad norm: 0.21 time: 0.150
|
|
2025-09-16 07:16:49,344: Train batch 105000: loss: 0.00 grad norm: 0.05 time: 0.109
|
|
2025-09-16 07:17:21,649: Train batch 105200: loss: 0.01 grad norm: 0.51 time: 0.101
|
|
2025-09-16 07:17:53,971: Train batch 105400: loss: 0.00 grad norm: 0.03 time: 0.131
|
|
2025-09-16 07:18:26,592: Train batch 105600: loss: 0.00 grad norm: 0.12 time: 0.117
|
|
2025-09-16 07:18:59,035: Train batch 105800: loss: 0.00 grad norm: 0.11 time: 0.180
|
|
2025-09-16 07:19:30,743: Train batch 106000: loss: 0.01 grad norm: 0.21 time: 0.179
|
|
2025-09-16 07:19:30,744: Running test after training batch: 106000
|
|
2025-09-16 07:19:41,497: Val batch 106000: PER (avg): 0.1082 CTC Loss (avg): 27.3164 time: 10.753
|
|
2025-09-16 07:19:41,497: t15.2023.08.13 val PER: 0.0863
|
|
2025-09-16 07:19:41,497: t15.2023.08.18 val PER: 0.0796
|
|
2025-09-16 07:19:41,497: t15.2023.08.20 val PER: 0.0580
|
|
2025-09-16 07:19:41,497: t15.2023.08.25 val PER: 0.0919
|
|
2025-09-16 07:19:41,497: t15.2023.08.27 val PER: 0.1656
|
|
2025-09-16 07:19:41,497: t15.2023.09.01 val PER: 0.0519
|
|
2025-09-16 07:19:41,497: t15.2023.09.03 val PER: 0.1093
|
|
2025-09-16 07:19:41,497: t15.2023.09.24 val PER: 0.0934
|
|
2025-09-16 07:19:41,498: t15.2023.09.29 val PER: 0.0989
|
|
2025-09-16 07:19:41,498: t15.2023.10.01 val PER: 0.1486
|
|
2025-09-16 07:19:41,498: t15.2023.10.06 val PER: 0.0571
|
|
2025-09-16 07:19:41,498: t15.2023.10.08 val PER: 0.1881
|
|
2025-09-16 07:19:41,498: t15.2023.10.13 val PER: 0.1614
|
|
2025-09-16 07:19:41,498: t15.2023.10.15 val PER: 0.1114
|
|
2025-09-16 07:19:41,498: t15.2023.10.20 val PER: 0.1711
|
|
2025-09-16 07:19:41,498: t15.2023.10.22 val PER: 0.1069
|
|
2025-09-16 07:19:41,498: t15.2023.11.03 val PER: 0.1642
|
|
2025-09-16 07:19:41,498: t15.2023.11.04 val PER: 0.0068
|
|
2025-09-16 07:19:41,498: t15.2023.11.17 val PER: 0.0233
|
|
2025-09-16 07:19:41,498: t15.2023.11.19 val PER: 0.0100
|
|
2025-09-16 07:19:41,498: t15.2023.11.26 val PER: 0.0478
|
|
2025-09-16 07:19:41,498: t15.2023.12.03 val PER: 0.0567
|
|
2025-09-16 07:19:41,498: t15.2023.12.08 val PER: 0.0399
|
|
2025-09-16 07:19:41,498: t15.2023.12.10 val PER: 0.0381
|
|
2025-09-16 07:19:41,498: t15.2023.12.17 val PER: 0.0842
|
|
2025-09-16 07:19:41,498: t15.2023.12.29 val PER: 0.0803
|
|
2025-09-16 07:19:41,498: t15.2024.02.25 val PER: 0.0843
|
|
2025-09-16 07:19:41,499: t15.2024.03.08 val PER: 0.1721
|
|
2025-09-16 07:19:41,499: t15.2024.03.15 val PER: 0.1720
|
|
2025-09-16 07:19:41,499: t15.2024.03.17 val PER: 0.0711
|
|
2025-09-16 07:19:41,499: t15.2024.05.10 val PER: 0.1263
|
|
2025-09-16 07:19:41,499: t15.2024.06.14 val PER: 0.1246
|
|
2025-09-16 07:19:41,499: t15.2024.07.19 val PER: 0.1622
|
|
2025-09-16 07:19:41,499: t15.2024.07.21 val PER: 0.0559
|
|
2025-09-16 07:19:41,499: t15.2024.07.28 val PER: 0.0816
|
|
2025-09-16 07:19:41,499: t15.2025.01.10 val PER: 0.2534
|
|
2025-09-16 07:19:41,499: t15.2025.01.12 val PER: 0.0762
|
|
2025-09-16 07:19:41,499: t15.2025.03.14 val PER: 0.2692
|
|
2025-09-16 07:19:41,499: t15.2025.03.16 val PER: 0.1270
|
|
2025-09-16 07:19:41,499: t15.2025.03.30 val PER: 0.2264
|
|
2025-09-16 07:19:41,499: t15.2025.04.13 val PER: 0.2040
|
|
2025-09-16 07:20:09,249: Train batch 106200: loss: 0.00 grad norm: 0.35 time: 0.174
|
|
2025-09-16 07:20:40,667: Train batch 106400: loss: 0.00 grad norm: 0.13 time: 0.109
|
|
2025-09-16 07:21:12,636: Train batch 106600: loss: 0.01 grad norm: 1.17 time: 0.125
|
|
2025-09-16 07:21:45,121: Train batch 106800: loss: 0.00 grad norm: 0.14 time: 0.102
|
|
2025-09-16 07:22:17,075: Train batch 107000: loss: 0.00 grad norm: 0.18 time: 0.121
|
|
2025-09-16 07:22:48,936: Train batch 107200: loss: 0.01 grad norm: 0.42 time: 0.155
|
|
2025-09-16 07:23:21,153: Train batch 107400: loss: 0.00 grad norm: 0.46 time: 0.114
|
|
2025-09-16 07:23:53,299: Train batch 107600: loss: 0.00 grad norm: 0.40 time: 0.103
|
|
2025-09-16 07:24:25,883: Train batch 107800: loss: 0.00 grad norm: 0.20 time: 0.138
|
|
2025-09-16 07:24:57,978: Train batch 108000: loss: 0.00 grad norm: 0.10 time: 0.100
|
|
2025-09-16 07:24:57,978: Running test after training batch: 108000
|
|
2025-09-16 07:25:08,692: Val batch 108000: PER (avg): 0.1081 CTC Loss (avg): 27.3591 time: 10.714
|
|
2025-09-16 07:25:08,693: t15.2023.08.13 val PER: 0.0873
|
|
2025-09-16 07:25:08,693: t15.2023.08.18 val PER: 0.0771
|
|
2025-09-16 07:25:08,693: t15.2023.08.20 val PER: 0.0564
|
|
2025-09-16 07:25:08,693: t15.2023.08.25 val PER: 0.0934
|
|
2025-09-16 07:25:08,693: t15.2023.08.27 val PER: 0.1672
|
|
2025-09-16 07:25:08,693: t15.2023.09.01 val PER: 0.0511
|
|
2025-09-16 07:25:08,693: t15.2023.09.03 val PER: 0.1093
|
|
2025-09-16 07:25:08,693: t15.2023.09.24 val PER: 0.0922
|
|
2025-09-16 07:25:08,693: t15.2023.09.29 val PER: 0.1008
|
|
2025-09-16 07:25:08,693: t15.2023.10.01 val PER: 0.1480
|
|
2025-09-16 07:25:08,693: t15.2023.10.06 val PER: 0.0560
|
|
2025-09-16 07:25:08,693: t15.2023.10.08 val PER: 0.1881
|
|
2025-09-16 07:25:08,693: t15.2023.10.13 val PER: 0.1629
|
|
2025-09-16 07:25:08,693: t15.2023.10.15 val PER: 0.1121
|
|
2025-09-16 07:25:08,693: t15.2023.10.20 val PER: 0.1678
|
|
2025-09-16 07:25:08,693: t15.2023.10.22 val PER: 0.1058
|
|
2025-09-16 07:25:08,693: t15.2023.11.03 val PER: 0.1608
|
|
2025-09-16 07:25:08,693: t15.2023.11.04 val PER: 0.0068
|
|
2025-09-16 07:25:08,694: t15.2023.11.17 val PER: 0.0233
|
|
2025-09-16 07:25:08,694: t15.2023.11.19 val PER: 0.0120
|
|
2025-09-16 07:25:08,694: t15.2023.11.26 val PER: 0.0500
|
|
2025-09-16 07:25:08,694: t15.2023.12.03 val PER: 0.0588
|
|
2025-09-16 07:25:08,694: t15.2023.12.08 val PER: 0.0399
|
|
2025-09-16 07:25:08,694: t15.2023.12.10 val PER: 0.0381
|
|
2025-09-16 07:25:08,694: t15.2023.12.17 val PER: 0.0842
|
|
2025-09-16 07:25:08,694: t15.2023.12.29 val PER: 0.0803
|
|
2025-09-16 07:25:08,694: t15.2024.02.25 val PER: 0.0857
|
|
2025-09-16 07:25:08,694: t15.2024.03.08 val PER: 0.1693
|
|
2025-09-16 07:25:08,694: t15.2024.03.15 val PER: 0.1701
|
|
2025-09-16 07:25:08,694: t15.2024.03.17 val PER: 0.0697
|
|
2025-09-16 07:25:08,694: t15.2024.05.10 val PER: 0.1263
|
|
2025-09-16 07:25:08,694: t15.2024.06.14 val PER: 0.1309
|
|
2025-09-16 07:25:08,694: t15.2024.07.19 val PER: 0.1648
|
|
2025-09-16 07:25:08,694: t15.2024.07.21 val PER: 0.0552
|
|
2025-09-16 07:25:08,694: t15.2024.07.28 val PER: 0.0801
|
|
2025-09-16 07:25:08,694: t15.2025.01.10 val PER: 0.2521
|
|
2025-09-16 07:25:08,694: t15.2025.01.12 val PER: 0.0770
|
|
2025-09-16 07:25:08,694: t15.2025.03.14 val PER: 0.2751
|
|
2025-09-16 07:25:08,695: t15.2025.03.16 val PER: 0.1283
|
|
2025-09-16 07:25:08,695: t15.2025.03.30 val PER: 0.2218
|
|
2025-09-16 07:25:08,695: t15.2025.04.13 val PER: 0.1997
|
|
2025-09-16 07:25:36,206: Train batch 108200: loss: 0.00 grad norm: 0.04 time: 0.114
|
|
2025-09-16 07:26:08,192: Train batch 108400: loss: 0.00 grad norm: 0.06 time: 0.155
|
|
2025-09-16 07:26:40,247: Train batch 108600: loss: 0.01 grad norm: 0.47 time: 0.142
|
|
2025-09-16 07:27:12,251: Train batch 108800: loss: 0.00 grad norm: 0.05 time: 0.156
|
|
2025-09-16 07:27:44,210: Train batch 109000: loss: 0.00 grad norm: 0.11 time: 0.142
|
|
2025-09-16 07:28:16,384: Train batch 109200: loss: 0.00 grad norm: 0.05 time: 0.175
|
|
2025-09-16 07:28:49,291: Train batch 109400: loss: 0.01 grad norm: 0.46 time: 0.172
|
|
2025-09-16 07:29:21,286: Train batch 109600: loss: 0.01 grad norm: 1.60 time: 0.115
|
|
2025-09-16 07:29:52,994: Train batch 109800: loss: 0.02 grad norm: 2.60 time: 0.126
|
|
2025-09-16 07:30:25,411: Train batch 110000: loss: 0.00 grad norm: 0.40 time: 0.112
|
|
2025-09-16 07:30:25,412: Running test after training batch: 110000
|
|
2025-09-16 07:30:36,001: Val batch 110000: PER (avg): 0.1079 CTC Loss (avg): 27.2525 time: 10.589
|
|
2025-09-16 07:30:36,001: t15.2023.08.13 val PER: 0.0873
|
|
2025-09-16 07:30:36,001: t15.2023.08.18 val PER: 0.0780
|
|
2025-09-16 07:30:36,002: t15.2023.08.20 val PER: 0.0564
|
|
2025-09-16 07:30:36,002: t15.2023.08.25 val PER: 0.0919
|
|
2025-09-16 07:30:36,002: t15.2023.08.27 val PER: 0.1672
|
|
2025-09-16 07:30:36,002: t15.2023.09.01 val PER: 0.0528
|
|
2025-09-16 07:30:36,002: t15.2023.09.03 val PER: 0.1105
|
|
2025-09-16 07:30:36,002: t15.2023.09.24 val PER: 0.0947
|
|
2025-09-16 07:30:36,002: t15.2023.09.29 val PER: 0.0989
|
|
2025-09-16 07:30:36,002: t15.2023.10.01 val PER: 0.1466
|
|
2025-09-16 07:30:36,002: t15.2023.10.06 val PER: 0.0549
|
|
2025-09-16 07:30:36,002: t15.2023.10.08 val PER: 0.1854
|
|
2025-09-16 07:30:36,002: t15.2023.10.13 val PER: 0.1598
|
|
2025-09-16 07:30:36,002: t15.2023.10.15 val PER: 0.1074
|
|
2025-09-16 07:30:36,002: t15.2023.10.20 val PER: 0.1745
|
|
2025-09-16 07:30:36,002: t15.2023.10.22 val PER: 0.1069
|
|
2025-09-16 07:30:36,002: t15.2023.11.03 val PER: 0.1608
|
|
2025-09-16 07:30:36,002: t15.2023.11.04 val PER: 0.0068
|
|
2025-09-16 07:30:36,002: t15.2023.11.17 val PER: 0.0233
|
|
2025-09-16 07:30:36,002: t15.2023.11.19 val PER: 0.0120
|
|
2025-09-16 07:30:36,002: t15.2023.11.26 val PER: 0.0464
|
|
2025-09-16 07:30:36,003: t15.2023.12.03 val PER: 0.0567
|
|
2025-09-16 07:30:36,003: t15.2023.12.08 val PER: 0.0406
|
|
2025-09-16 07:30:36,003: t15.2023.12.10 val PER: 0.0394
|
|
2025-09-16 07:30:36,003: t15.2023.12.17 val PER: 0.0894
|
|
2025-09-16 07:30:36,003: t15.2023.12.29 val PER: 0.0817
|
|
2025-09-16 07:30:36,003: t15.2024.02.25 val PER: 0.0885
|
|
2025-09-16 07:30:36,003: t15.2024.03.08 val PER: 0.1693
|
|
2025-09-16 07:30:36,003: t15.2024.03.15 val PER: 0.1720
|
|
2025-09-16 07:30:36,003: t15.2024.03.17 val PER: 0.0704
|
|
2025-09-16 07:30:36,003: t15.2024.05.10 val PER: 0.1233
|
|
2025-09-16 07:30:36,003: t15.2024.06.14 val PER: 0.1309
|
|
2025-09-16 07:30:36,003: t15.2024.07.19 val PER: 0.1628
|
|
2025-09-16 07:30:36,003: t15.2024.07.21 val PER: 0.0545
|
|
2025-09-16 07:30:36,003: t15.2024.07.28 val PER: 0.0779
|
|
2025-09-16 07:30:36,003: t15.2025.01.10 val PER: 0.2521
|
|
2025-09-16 07:30:36,003: t15.2025.01.12 val PER: 0.0754
|
|
2025-09-16 07:30:36,003: t15.2025.03.14 val PER: 0.2707
|
|
2025-09-16 07:30:36,003: t15.2025.03.16 val PER: 0.1348
|
|
2025-09-16 07:30:36,003: t15.2025.03.30 val PER: 0.2276
|
|
2025-09-16 07:30:36,003: t15.2025.04.13 val PER: 0.1983
|
|
2025-09-16 07:31:03,484: Train batch 110200: loss: 0.01 grad norm: 0.61 time: 0.134
|
|
2025-09-16 07:31:35,080: Train batch 110400: loss: 0.00 grad norm: 0.04 time: 0.129
|
|
2025-09-16 07:32:07,015: Train batch 110600: loss: 0.01 grad norm: 1.37 time: 0.135
|
|
2025-09-16 07:32:38,683: Train batch 110800: loss: 0.01 grad norm: 0.80 time: 0.150
|
|
2025-09-16 07:33:11,027: Train batch 111000: loss: 0.00 grad norm: 0.05 time: 0.132
|
|
2025-09-16 07:33:43,198: Train batch 111200: loss: 0.00 grad norm: 0.14 time: 0.113
|
|
2025-09-16 07:34:15,419: Train batch 111400: loss: 0.00 grad norm: 0.21 time: 0.143
|
|
2025-09-16 07:34:47,332: Train batch 111600: loss: 0.02 grad norm: 2.02 time: 0.115
|
|
2025-09-16 07:35:19,143: Train batch 111800: loss: 0.00 grad norm: 0.87 time: 0.135
|
|
2025-09-16 07:35:51,385: Train batch 112000: loss: 0.00 grad norm: 0.21 time: 0.123
|
|
2025-09-16 07:35:51,385: Running test after training batch: 112000
|
|
2025-09-16 07:36:01,915: Val batch 112000: PER (avg): 0.1078 CTC Loss (avg): 27.2122 time: 10.529
|
|
2025-09-16 07:36:01,915: t15.2023.08.13 val PER: 0.0884
|
|
2025-09-16 07:36:01,915: t15.2023.08.18 val PER: 0.0780
|
|
2025-09-16 07:36:01,915: t15.2023.08.20 val PER: 0.0580
|
|
2025-09-16 07:36:01,915: t15.2023.08.25 val PER: 0.0919
|
|
2025-09-16 07:36:01,915: t15.2023.08.27 val PER: 0.1672
|
|
2025-09-16 07:36:01,915: t15.2023.09.01 val PER: 0.0511
|
|
2025-09-16 07:36:01,915: t15.2023.09.03 val PER: 0.1116
|
|
2025-09-16 07:36:01,915: t15.2023.09.24 val PER: 0.0947
|
|
2025-09-16 07:36:01,915: t15.2023.09.29 val PER: 0.0989
|
|
2025-09-16 07:36:01,915: t15.2023.10.01 val PER: 0.1453
|
|
2025-09-16 07:36:01,915: t15.2023.10.06 val PER: 0.0549
|
|
2025-09-16 07:36:01,915: t15.2023.10.08 val PER: 0.1894
|
|
2025-09-16 07:36:01,915: t15.2023.10.13 val PER: 0.1590
|
|
2025-09-16 07:36:01,915: t15.2023.10.15 val PER: 0.1094
|
|
2025-09-16 07:36:01,916: t15.2023.10.20 val PER: 0.1745
|
|
2025-09-16 07:36:01,916: t15.2023.10.22 val PER: 0.1069
|
|
2025-09-16 07:36:01,916: t15.2023.11.03 val PER: 0.1608
|
|
2025-09-16 07:36:01,916: t15.2023.11.04 val PER: 0.0068
|
|
2025-09-16 07:36:01,916: t15.2023.11.17 val PER: 0.0233
|
|
2025-09-16 07:36:01,916: t15.2023.11.19 val PER: 0.0120
|
|
2025-09-16 07:36:01,916: t15.2023.11.26 val PER: 0.0500
|
|
2025-09-16 07:36:01,916: t15.2023.12.03 val PER: 0.0567
|
|
2025-09-16 07:36:01,916: t15.2023.12.08 val PER: 0.0406
|
|
2025-09-16 07:36:01,916: t15.2023.12.10 val PER: 0.0394
|
|
2025-09-16 07:36:01,916: t15.2023.12.17 val PER: 0.0884
|
|
2025-09-16 07:36:01,916: t15.2023.12.29 val PER: 0.0796
|
|
2025-09-16 07:36:01,916: t15.2024.02.25 val PER: 0.0885
|
|
2025-09-16 07:36:01,916: t15.2024.03.08 val PER: 0.1693
|
|
2025-09-16 07:36:01,916: t15.2024.03.15 val PER: 0.1682
|
|
2025-09-16 07:36:01,916: t15.2024.03.17 val PER: 0.0704
|
|
2025-09-16 07:36:01,916: t15.2024.05.10 val PER: 0.1233
|
|
2025-09-16 07:36:01,916: t15.2024.06.14 val PER: 0.1278
|
|
2025-09-16 07:36:01,916: t15.2024.07.19 val PER: 0.1628
|
|
2025-09-16 07:36:01,917: t15.2024.07.21 val PER: 0.0559
|
|
2025-09-16 07:36:01,917: t15.2024.07.28 val PER: 0.0801
|
|
2025-09-16 07:36:01,917: t15.2025.01.10 val PER: 0.2534
|
|
2025-09-16 07:36:01,917: t15.2025.01.12 val PER: 0.0747
|
|
2025-09-16 07:36:01,917: t15.2025.03.14 val PER: 0.2692
|
|
2025-09-16 07:36:01,917: t15.2025.03.16 val PER: 0.1296
|
|
2025-09-16 07:36:01,917: t15.2025.03.30 val PER: 0.2264
|
|
2025-09-16 07:36:01,917: t15.2025.04.13 val PER: 0.2011
|
|
2025-09-16 07:36:30,180: Train batch 112200: loss: 0.01 grad norm: 0.73 time: 0.146
|
|
2025-09-16 07:37:02,278: Train batch 112400: loss: 0.01 grad norm: 3.01 time: 0.118
|
|
2025-09-16 07:37:35,560: Train batch 112600: loss: 0.01 grad norm: 1.81 time: 0.182
|
|
2025-09-16 07:38:07,871: Train batch 112800: loss: 0.01 grad norm: 0.41 time: 0.152
|
|
2025-09-16 07:38:39,777: Train batch 113000: loss: 0.01 grad norm: 0.40 time: 0.119
|
|
2025-09-16 07:39:11,895: Train batch 113200: loss: 0.00 grad norm: 0.25 time: 0.167
|
|
2025-09-16 07:39:44,784: Train batch 113400: loss: 0.00 grad norm: 0.18 time: 0.121
|
|
2025-09-16 07:40:17,787: Train batch 113600: loss: 0.00 grad norm: 0.10 time: 0.102
|
|
2025-09-16 07:40:50,043: Train batch 113800: loss: 0.01 grad norm: 0.85 time: 0.145
|
|
2025-09-16 07:41:23,335: Train batch 114000: loss: 0.00 grad norm: 0.04 time: 0.157
|
|
2025-09-16 07:41:23,335: Running test after training batch: 114000
|
|
2025-09-16 07:41:33,918: Val batch 114000: PER (avg): 0.1080 CTC Loss (avg): 27.3161 time: 10.583
|
|
2025-09-16 07:41:33,919: t15.2023.08.13 val PER: 0.0873
|
|
2025-09-16 07:41:33,919: t15.2023.08.18 val PER: 0.0763
|
|
2025-09-16 07:41:33,919: t15.2023.08.20 val PER: 0.0588
|
|
2025-09-16 07:41:33,919: t15.2023.08.25 val PER: 0.0919
|
|
2025-09-16 07:41:33,919: t15.2023.08.27 val PER: 0.1672
|
|
2025-09-16 07:41:33,919: t15.2023.09.01 val PER: 0.0519
|
|
2025-09-16 07:41:33,919: t15.2023.09.03 val PER: 0.1140
|
|
2025-09-16 07:41:33,919: t15.2023.09.24 val PER: 0.0947
|
|
2025-09-16 07:41:33,919: t15.2023.09.29 val PER: 0.0983
|
|
2025-09-16 07:41:33,919: t15.2023.10.01 val PER: 0.1466
|
|
2025-09-16 07:41:33,919: t15.2023.10.06 val PER: 0.0538
|
|
2025-09-16 07:41:33,919: t15.2023.10.08 val PER: 0.1894
|
|
2025-09-16 07:41:33,919: t15.2023.10.13 val PER: 0.1598
|
|
2025-09-16 07:41:33,919: t15.2023.10.15 val PER: 0.1088
|
|
2025-09-16 07:41:33,919: t15.2023.10.20 val PER: 0.1779
|
|
2025-09-16 07:41:33,920: t15.2023.10.22 val PER: 0.1069
|
|
2025-09-16 07:41:33,920: t15.2023.11.03 val PER: 0.1628
|
|
2025-09-16 07:41:33,920: t15.2023.11.04 val PER: 0.0068
|
|
2025-09-16 07:41:33,920: t15.2023.11.17 val PER: 0.0233
|
|
2025-09-16 07:41:33,920: t15.2023.11.19 val PER: 0.0100
|
|
2025-09-16 07:41:33,920: t15.2023.11.26 val PER: 0.0493
|
|
2025-09-16 07:41:33,920: t15.2023.12.03 val PER: 0.0578
|
|
2025-09-16 07:41:33,920: t15.2023.12.08 val PER: 0.0399
|
|
2025-09-16 07:41:33,920: t15.2023.12.10 val PER: 0.0407
|
|
2025-09-16 07:41:33,920: t15.2023.12.17 val PER: 0.0873
|
|
2025-09-16 07:41:33,920: t15.2023.12.29 val PER: 0.0796
|
|
2025-09-16 07:41:33,920: t15.2024.02.25 val PER: 0.0871
|
|
2025-09-16 07:41:33,920: t15.2024.03.08 val PER: 0.1693
|
|
2025-09-16 07:41:33,920: t15.2024.03.15 val PER: 0.1701
|
|
2025-09-16 07:41:33,920: t15.2024.03.17 val PER: 0.0718
|
|
2025-09-16 07:41:33,920: t15.2024.05.10 val PER: 0.1233
|
|
2025-09-16 07:41:33,920: t15.2024.06.14 val PER: 0.1293
|
|
2025-09-16 07:41:33,920: t15.2024.07.19 val PER: 0.1622
|
|
2025-09-16 07:41:33,920: t15.2024.07.21 val PER: 0.0559
|
|
2025-09-16 07:41:33,920: t15.2024.07.28 val PER: 0.0809
|
|
2025-09-16 07:41:33,921: t15.2025.01.10 val PER: 0.2521
|
|
2025-09-16 07:41:33,921: t15.2025.01.12 val PER: 0.0762
|
|
2025-09-16 07:41:33,921: t15.2025.03.14 val PER: 0.2678
|
|
2025-09-16 07:41:33,921: t15.2025.03.16 val PER: 0.1322
|
|
2025-09-16 07:41:33,921: t15.2025.03.30 val PER: 0.2253
|
|
2025-09-16 07:41:33,921: t15.2025.04.13 val PER: 0.1983
|
|
2025-09-16 07:42:01,244: Train batch 114200: loss: 0.00 grad norm: 0.13 time: 0.153
|
|
2025-09-16 07:42:33,262: Train batch 114400: loss: 0.00 grad norm: 0.12 time: 0.161
|
|
2025-09-16 07:43:06,098: Train batch 114600: loss: 0.00 grad norm: 0.33 time: 0.127
|
|
2025-09-16 07:43:38,727: Train batch 114800: loss: 0.01 grad norm: 0.83 time: 0.114
|