remote_time.c
1 /* 2 * Copyright (c) 2017-2020 Apple Inc. All rights reserved. 3 * 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ 5 * 6 * This file contains Original Code and/or Modifications of Original Code 7 * as defined in and that are subject to the Apple Public Source License 8 * Version 2.0 (the 'License'). You may not use this file except in 9 * compliance with the License. The rights granted to you under the License 10 * may not be used to create, or enable the creation or redistribution of, 11 * unlawful or unlicensed copies of an Apple operating system, or to 12 * circumvent, violate, or enable the circumvention or violation of, any 13 * terms of an Apple operating system software license agreement. 14 * 15 * Please obtain a copy of the License at 16 * http://www.opensource.apple.com/apsl/ and read it before using this file. 17 * 18 * The Original Code and all software distributed under the License are 19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER 20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, 21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, 22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. 23 * Please see the License for the specific language governing rights and 24 * limitations under the License. 25 * 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ 27 */ 28 #include <mach/mach_time.h> 29 #include <mach/clock_types.h> 30 #include <kern/misc_protos.h> 31 #include <kern/clock.h> 32 #include <kern/remote_time.h> 33 #include <kern/spl.h> 34 #include <kern/locks.h> 35 #include <sys/kdebug.h> 36 #include <machine/machine_routines.h> 37 #include <kern/assert.h> 38 #include <kern/kern_types.h> 39 #include <kern/thread.h> 40 #include <machine/commpage.h> 41 #include <machine/atomic.h> 42 43 LCK_GRP_DECLARE(bt_lck_grp, "bridge timestamp"); 44 LCK_SPIN_DECLARE(bt_spin_lock, &bt_lck_grp); 45 LCK_SPIN_DECLARE(bt_ts_conversion_lock, &bt_lck_grp); 46 LCK_SPIN_DECLARE(bt_maintenance_lock, &bt_lck_grp); 47 48 #if CONFIG_MACH_BRIDGE_SEND_TIME 49 50 uint32_t bt_enable_flag = 0; 51 _Atomic uint32_t bt_init_flag = 0; 52 53 void mach_bridge_timer_maintenance(void); 54 uint32_t mach_bridge_timer_enable(uint32_t new_value, int change); 55 56 /* 57 * When CONFIG_MACH_BRIDGE_SEND_TIME is defined, it is expected 58 * that a machine-specific timestamp sending routine such as 59 * void mach_bridge_send_timestamp(uint64_t); has also been defined. 60 */ 61 extern void mach_bridge_send_timestamp(uint64_t); 62 63 void 64 mach_bridge_timer_maintenance(void) 65 { 66 if (!os_atomic_load(&bt_init_flag, acquire)) { 67 return; 68 } 69 70 lck_spin_lock(&bt_maintenance_lock); 71 if (!bt_enable_flag) { 72 goto done; 73 } 74 mach_bridge_send_timestamp(0); 75 76 done: 77 lck_spin_unlock(&bt_maintenance_lock); 78 } 79 80 /* 81 * If change = 0, return the current value of bridge_timer_enable 82 * If change = 1, update bridge_timer_enable and return the updated 83 * value 84 */ 85 uint32_t 86 mach_bridge_timer_enable(uint32_t new_value, int change) 87 { 88 uint32_t current_value = 0; 89 assert(os_atomic_load(&bt_init_flag, relaxed)); 90 lck_spin_lock(&bt_maintenance_lock); 91 if (change) { 92 bt_enable_flag = new_value; 93 } 94 current_value = bt_enable_flag; 95 lck_spin_unlock(&bt_maintenance_lock); 96 return current_value; 97 } 98 99 #endif /* CONFIG_MACH_BRIDGE_SEND_TIME */ 100 101 #if CONFIG_MACH_BRIDGE_RECV_TIME 102 #include <machine/machine_remote_time.h> 103 104 /* 105 * functions used by machine-specific code 106 * that implements CONFIG_MACH_BRIDGE_RECV_TIME 107 */ 108 void mach_bridge_add_timestamp(uint64_t remote_timestamp, uint64_t local_timestamp); 109 void bt_calibration_thread_start(void); 110 void bt_params_add(struct bt_params *params); 111 112 /* function called by sysctl */ 113 struct bt_params bt_params_get_latest(void); 114 115 /* 116 * Platform specific bridge time receiving interface. 117 * These variables should be exported by the platform specific time receiving code. 118 */ 119 extern _Atomic uint32_t bt_init_flag; 120 121 static uint64_t received_local_timestamp = 0; 122 static uint64_t received_remote_timestamp = 0; 123 /* 124 * Buffer the previous timestamp pairs and rate 125 * It is protected by the bt_ts_conversion_lock 126 */ 127 #define BT_PARAMS_COUNT 10 128 static struct bt_params bt_params_hist[BT_PARAMS_COUNT] = {}; 129 static int bt_params_idx = -1; 130 131 void 132 bt_params_add(struct bt_params *params) 133 { 134 lck_spin_assert(&bt_ts_conversion_lock, LCK_ASSERT_OWNED); 135 136 bt_params_idx = (bt_params_idx + 1) % BT_PARAMS_COUNT; 137 bt_params_hist[bt_params_idx] = *params; 138 } 139 140 #if defined(XNU_TARGET_OS_BRIDGE) 141 static inline struct bt_params* 142 bt_params_find(uint64_t local_ts) 143 { 144 lck_spin_assert(&bt_ts_conversion_lock, LCK_ASSERT_OWNED); 145 146 int idx = bt_params_idx; 147 if (idx < 0) { 148 return NULL; 149 } 150 do { 151 if (local_ts >= bt_params_hist[idx].base_local_ts) { 152 return &bt_params_hist[idx]; 153 } 154 if (--idx < 0) { 155 idx = BT_PARAMS_COUNT - 1; 156 } 157 } while (idx != bt_params_idx); 158 159 return NULL; 160 } 161 #endif /* defined(XNU_TARGET_OS_BRIDGE) */ 162 163 static inline struct bt_params 164 bt_params_get_latest_locked(void) 165 { 166 lck_spin_assert(&bt_ts_conversion_lock, LCK_ASSERT_OWNED); 167 168 struct bt_params latest_params = {}; 169 if (bt_params_idx >= 0) { 170 latest_params = bt_params_hist[bt_params_idx]; 171 } 172 173 return latest_params; 174 } 175 176 struct bt_params 177 bt_params_get_latest(void) 178 { 179 struct bt_params latest_params = {}; 180 181 /* Check if ts_converison_lock has been initialized */ 182 if (os_atomic_load(&bt_init_flag, acquire)) { 183 lck_spin_lock(&bt_ts_conversion_lock); 184 latest_params = bt_params_get_latest_locked(); 185 lck_spin_unlock(&bt_ts_conversion_lock); 186 } 187 return latest_params; 188 } 189 190 /* 191 * Conditions: bt_spin_lock held and called from primary interrupt context 192 */ 193 void 194 mach_bridge_add_timestamp(uint64_t remote_timestamp, uint64_t local_timestamp) 195 { 196 lck_spin_assert(&bt_spin_lock, LCK_ASSERT_OWNED); 197 198 /* sleep/wake might return the same mach_absolute_time as the previous timestamp pair */ 199 if ((received_local_timestamp == local_timestamp) || 200 (received_remote_timestamp == remote_timestamp)) { 201 return; 202 } 203 204 received_local_timestamp = local_timestamp; 205 received_remote_timestamp = remote_timestamp; 206 thread_wakeup((event_t)bt_params_hist); 207 } 208 209 static double 210 mach_bridge_compute_rate(uint64_t new_local_ts, uint64_t new_remote_ts, 211 uint64_t old_local_ts, uint64_t old_remote_ts) 212 { 213 int64_t rdiff = (int64_t)new_remote_ts - (int64_t)old_remote_ts; 214 int64_t ldiff = (int64_t)new_local_ts - (int64_t)old_local_ts; 215 double calc_rate = ((double)rdiff) / (double)ldiff; 216 return calc_rate; 217 } 218 219 #define MAX_RECALCULATE_COUNT 8 220 #define CUMULATIVE_RATE_DECAY_CONSTANT 0.01 221 #define CUMULATIVE_RATE_WEIGHT 0.99 222 #define INITIAL_RATE 1.0 223 #define MIN_INITIAL_SAMPLE_COUNT 10 224 #define MAX_INITIAL_SAMPLE_COUNT 50 225 #define MAX_SKIP_RESET_COUNT 2 226 #define MIN_LOCAL_TS_DISTANCE_NS 100000000 /* 100 ms */ 227 #define MAX_LOCAL_TS_DISTANCE_NS 350000000 /* 350 ms */ 228 #define TS_PAIR_MISMATCH_THRESHOLD_NS 50000000 /* 50 ms */ 229 #define MAX_TS_PAIR_MISMATCHES 5 230 #define MAX_TS_PAIR_MISMATCH_RESET_COUNT 3 231 #define MIN_OBSERVED_RATE 0.8 232 #define MAX_OBSERVED_RATE 1.2 233 234 static void 235 bt_calibration_thread(void) 236 { 237 static uint64_t prev_local_ts = 0, prev_remote_ts = 0, curr_local_ts = 0, curr_remote_ts = 0; 238 static uint64_t prev_received_local_ts = 0, prev_received_remote_ts = 0; 239 static double cumulative_rate = INITIAL_RATE; 240 static uint32_t initial_sample_count = 1; 241 static uint32_t max_initial_sample_count = MAX_INITIAL_SAMPLE_COUNT; 242 static uint32_t skip_reset_count = MAX_SKIP_RESET_COUNT; 243 int recalculate_count = 1; 244 static bool reset = false; 245 bool sleep = false; 246 static bool skip_rcv_ts = false; 247 static uint64_t ts_pair_mismatch = 0; 248 static uint32_t ts_pair_mismatch_reset_count = 0; 249 spl_t s = splsched(); 250 lck_spin_lock(&bt_spin_lock); 251 if (!received_remote_timestamp) { 252 if (PE_parse_boot_argn("rt_ini_count", &max_initial_sample_count, 253 sizeof(uint32_t)) == TRUE) { 254 if (max_initial_sample_count < MIN_INITIAL_SAMPLE_COUNT) { 255 max_initial_sample_count = MIN_INITIAL_SAMPLE_COUNT; 256 } 257 } 258 /* Nothing to do the first time */ 259 goto block; 260 } 261 /* 262 * The values in bt_params are recalculated every time a new timestamp 263 * pair is received. Firstly, both timestamps are converted to nanoseconds. 264 * The current and previous timestamp pairs are used to compute the 265 * observed_rate of the two clocks w.r.t each other. For the first 266 * MIN_INITIAL_SAMPLE_COUNT number of pairs, the cumulative_rate is a simple 267 * average of the observed_rate. For the later pairs, the cumulative_rate 268 * is updated using exponential moving average of the observed_rate. 269 * The current and bt_params' base timestamp pairs are used to compute 270 * the rate_from_base. This value ensures that the bt_params base 271 * timestamp pair curve doesn't stay parallel to the observed timestamp 272 * pair curve, rather moves in the direction of the observed timestamp curve. 273 * The bt_params.rate is computed as a weighted average of the cumulative_rate 274 * and the rate_from_base. For each current local timestamp, the remote_time 275 * is predicted using the previous values of bt_params. After computing the new 276 * bt_params.rate, bt_params.base_remote_time is set to this predicted value 277 * and bt_params.base_local_time is set to the current local timestamp. 278 */ 279 recalculate: 280 assertf(recalculate_count <= MAX_RECALCULATE_COUNT, "bt_caliberation_thread: recalculate \ 281 invocation exceeds MAX_RECALCULATE_COUNT"); 282 283 if ((received_remote_timestamp == BT_RESET_SENTINEL_TS) || (received_remote_timestamp == BT_WAKE_SENTINEL_TS)) { 284 KDBG(MACHDBG_CODE(DBG_MACH_CLOCK, MACH_BRIDGE_RESET_TS), received_local_timestamp, received_remote_timestamp, 1); 285 reset = true; 286 skip_reset_count = MAX_SKIP_RESET_COUNT; 287 ts_pair_mismatch_reset_count = 0; 288 goto block; 289 } else if (received_remote_timestamp == BT_SLEEP_SENTINEL_TS) { 290 sleep = true; 291 } else if (!received_local_timestamp) { 292 /* If the local timestamp isn't accurately captured, the received value will be ignored */ 293 skip_rcv_ts = true; 294 goto block; 295 } 296 297 /* Keep a copy of the prev timestamps to compute distance */ 298 prev_received_local_ts = curr_local_ts; 299 prev_received_remote_ts = curr_remote_ts; 300 301 uint64_t curr_local_abs = received_local_timestamp; 302 absolutetime_to_nanoseconds(curr_local_abs, &curr_local_ts); 303 curr_remote_ts = received_remote_timestamp; 304 305 /* Prevent unusual rate changes caused by delayed timestamps */ 306 uint64_t local_diff = curr_local_ts - prev_received_local_ts; 307 if (!(reset || sleep) && ((local_diff < MIN_LOCAL_TS_DISTANCE_NS) || 308 (!skip_rcv_ts && (local_diff > MAX_LOCAL_TS_DISTANCE_NS)))) { 309 /* Skip the current timestamp */ 310 KDBG(MACHDBG_CODE(DBG_MACH_CLOCK, MACH_BRIDGE_SKIP_TS), curr_local_ts, curr_remote_ts, 311 prev_received_local_ts); 312 goto block; 313 } else { 314 skip_rcv_ts = false; 315 /* Use the prev copy of timestamps only if the distance is acceptable */ 316 prev_local_ts = prev_received_local_ts; 317 prev_remote_ts = prev_received_remote_ts; 318 } 319 lck_spin_unlock(&bt_spin_lock); 320 splx(s); 321 322 struct bt_params bt_params = {}; 323 324 lck_spin_lock(&bt_ts_conversion_lock); 325 if (reset) { 326 if (skip_reset_count > 0) { 327 KDBG(MACHDBG_CODE(DBG_MACH_CLOCK, MACH_BRIDGE_SKIP_TS), curr_local_ts, curr_remote_ts, 328 prev_local_ts, skip_reset_count); 329 skip_reset_count--; 330 goto skip_reset; 331 } 332 bt_params.base_local_ts = curr_local_ts; 333 bt_params.base_remote_ts = curr_remote_ts; 334 bt_params.rate = cumulative_rate; 335 prev_local_ts = 0; 336 prev_remote_ts = 0; 337 ts_pair_mismatch = 0; 338 initial_sample_count = 1; 339 reset = false; 340 KDBG(MACHDBG_CODE(DBG_MACH_CLOCK, MACH_BRIDGE_RESET_TS), curr_local_ts, curr_remote_ts, 2); 341 } else if (sleep) { 342 absolutetime_to_nanoseconds(mach_absolute_time(), &bt_params.base_local_ts); 343 bt_params.base_remote_ts = 0; 344 bt_params.rate = 0; 345 sleep = false; 346 } else { 347 struct bt_params bt_params_snapshot = {}; 348 if (bt_params_idx >= 0) { 349 bt_params_snapshot = bt_params_hist[bt_params_idx]; 350 } 351 lck_spin_unlock(&bt_ts_conversion_lock); 352 if (bt_params_snapshot.rate == 0.0) { 353 /* 354 * The rate should never be 0 because we always expect a reset/wake 355 * sentinel after sleep, followed by valid timestamp pair data that 356 * will be handled by the reset clause (above). However, we should 357 * not rely on a paired version of the remote OS - we could actually 358 * be running a completely different OS! Treat a timestamp after 359 * a sleep as a reset condition. 360 */ 361 reset = true; 362 skip_reset_count = MAX_SKIP_RESET_COUNT; 363 ts_pair_mismatch_reset_count = 0; 364 KDBG(MACHDBG_CODE(DBG_MACH_CLOCK, MACH_BRIDGE_RESET_TS), curr_local_ts, curr_remote_ts, 3); 365 s = splsched(); 366 lck_spin_lock(&bt_spin_lock); 367 goto block; 368 } 369 370 /* Check if the predicted remote timestamp is within the expected current remote timestamp range */ 371 uint64_t pred_remote_ts = mach_bridge_compute_timestamp(curr_local_ts, &bt_params_snapshot); 372 uint64_t diff = 0; 373 if (initial_sample_count >= max_initial_sample_count) { 374 if (pred_remote_ts > curr_remote_ts) { 375 diff = pred_remote_ts - curr_remote_ts; 376 } else { 377 diff = curr_remote_ts - pred_remote_ts; 378 } 379 if (diff > TS_PAIR_MISMATCH_THRESHOLD_NS) { 380 ts_pair_mismatch++; 381 KDBG(MACHDBG_CODE(DBG_MACH_CLOCK, MACH_BRIDGE_TS_MISMATCH), curr_local_ts, 382 curr_remote_ts, pred_remote_ts, ts_pair_mismatch); 383 } else { 384 ts_pair_mismatch = 0; 385 } 386 if (ts_pair_mismatch > MAX_TS_PAIR_MISMATCHES) { 387 #if (DEVELOPMENT || DEBUG) 388 if (ts_pair_mismatch_reset_count == MAX_TS_PAIR_MISMATCH_RESET_COUNT) { 389 panic("remote_time: timestamp pair mismatch exceeded limit"); 390 } 391 #endif /* (DEVELOPMENT || DEBUG) */ 392 reset = true; 393 ts_pair_mismatch_reset_count++; 394 KDBG(MACHDBG_CODE(DBG_MACH_CLOCK, MACH_BRIDGE_RESET_TS), curr_local_ts, curr_remote_ts, 4); 395 s = splsched(); 396 lck_spin_lock(&bt_spin_lock); 397 goto block; 398 } 399 } 400 double observed_rate, rate_from_base, new_rate; 401 observed_rate = mach_bridge_compute_rate(curr_local_ts, curr_remote_ts, prev_local_ts, prev_remote_ts); 402 /* Log bad observed rates and skip the timestamp pair */ 403 if ((observed_rate < MIN_OBSERVED_RATE) || (observed_rate > MAX_OBSERVED_RATE)) { 404 KDBG(MACHDBG_CODE(DBG_MACH_CLOCK, MACH_BRIDGE_OBSV_RATE), *(uint64_t *)((void *)&observed_rate)); 405 ts_pair_mismatch = ts_pair_mismatch > 0 ? (ts_pair_mismatch - 1) : 0; 406 s = splsched(); 407 lck_spin_lock(&bt_spin_lock); 408 goto block; 409 } 410 if (initial_sample_count <= MIN_INITIAL_SAMPLE_COUNT) { 411 initial_sample_count++; 412 cumulative_rate = cumulative_rate + (observed_rate - cumulative_rate) / initial_sample_count; 413 } else { 414 if (initial_sample_count < max_initial_sample_count) { 415 initial_sample_count++; 416 } 417 cumulative_rate = cumulative_rate + CUMULATIVE_RATE_DECAY_CONSTANT * (observed_rate - cumulative_rate); 418 } 419 rate_from_base = mach_bridge_compute_rate(curr_local_ts, curr_remote_ts, bt_params_snapshot.base_local_ts, 420 bt_params_snapshot.base_remote_ts); 421 new_rate = CUMULATIVE_RATE_WEIGHT * cumulative_rate + (1 - CUMULATIVE_RATE_WEIGHT) * rate_from_base; 422 /* 423 * Acquire the lock first to ensure that bt_params.base_local_ts is always 424 * greater than the last value of now captured by mach_bridge_remote_time. 425 * This ensures that we always use the same parameters to compute remote 426 * timestamp for a given local timestamp. 427 */ 428 lck_spin_lock(&bt_ts_conversion_lock); 429 absolutetime_to_nanoseconds(mach_absolute_time(), &bt_params.base_local_ts); 430 bt_params.base_remote_ts = mach_bridge_compute_timestamp(bt_params.base_local_ts, &bt_params_snapshot); 431 bt_params.rate = new_rate; 432 } 433 bt_params_add(&bt_params); 434 commpage_set_remotetime_params(bt_params.rate, bt_params.base_local_ts, bt_params.base_remote_ts); 435 KDBG(MACHDBG_CODE(DBG_MACH_CLOCK, MACH_BRIDGE_TS_PARAMS), bt_params.base_local_ts, 436 bt_params.base_remote_ts, *(uint64_t *)((void *)&bt_params.rate)); 437 438 skip_reset: 439 lck_spin_unlock(&bt_ts_conversion_lock); 440 441 s = splsched(); 442 lck_spin_lock(&bt_spin_lock); 443 /* Check if a new timestamp pair was received */ 444 if (received_local_timestamp != curr_local_abs) { 445 recalculate_count++; 446 goto recalculate; 447 } 448 block: 449 assert_wait((event_t)bt_params_hist, THREAD_UNINT); 450 lck_spin_unlock(&bt_spin_lock); 451 splx(s); 452 thread_block((thread_continue_t)bt_calibration_thread); 453 } 454 455 void 456 bt_calibration_thread_start(void) 457 { 458 thread_t thread; 459 kern_return_t result = kernel_thread_start_priority((thread_continue_t)bt_calibration_thread, 460 NULL, BASEPRI_KERNEL, &thread); 461 if (result != KERN_SUCCESS) { 462 panic("mach_bridge_add_timestamp: thread_timestamp_calibration"); 463 } 464 thread_deallocate(thread); 465 } 466 467 #endif /* CONFIG_MACH_BRIDGE_RECV_TIME */ 468 469 /** 470 * mach_bridge_remote_time 471 * 472 * This function is used to predict the remote CPU's clock time, given 473 * the local time. 474 * 475 * If local_timestamp = 0, then the remote_timestamp is calculated 476 * corresponding to the current mach_absolute_time. 477 * 478 * If XNU_TARGET_OS_BRIDGE is defined, then monotonicity of 479 * predicted time is guaranteed only for recent local_timestamp values 480 * lesser than the current mach_absolute_time upto 1 second. 481 * 482 * If CONFIG_MACH_BRIDGE_SEND_TIME is true, then the function is compiled 483 * for the remote CPU. If CONFIG_MACH_BRIDGE_RECV_TIME is true, then the 484 * the function is compiled for the local CPU. Both config options cannot 485 * be true simultaneously. 486 */ 487 uint64_t 488 mach_bridge_remote_time(uint64_t local_timestamp) 489 { 490 #if defined(CONFIG_MACH_BRIDGE_SEND_TIME) 491 #if !defined(CONFIG_MACH_BRIDGE_RECV_TIME) 492 /* only send side of the bridge is defined: no translation needed */ 493 if (!local_timestamp) { 494 return mach_absolute_time(); 495 } 496 return 0; 497 #else 498 #error "You cannot define both sides of the bridge!" 499 #endif /* !defined(CONFIG_MACH_BRIDGE_RECV_TIME) */ 500 #else 501 #if !defined(CONFIG_MACH_BRIDGE_RECV_TIME) 502 /* neither the send or receive side of the bridge is defined: echo the input */ 503 return local_timestamp; 504 #else 505 if (!os_atomic_load(&bt_init_flag, acquire)) { 506 return 0; 507 } 508 509 uint64_t remote_timestamp = 0; 510 511 lck_spin_lock(&bt_ts_conversion_lock); 512 uint64_t now = mach_absolute_time(); 513 if (!local_timestamp) { 514 local_timestamp = now; 515 } 516 #if defined(XNU_TARGET_OS_BRIDGE) 517 uint64_t local_timestamp_ns = 0; 518 if (local_timestamp < now) { 519 absolutetime_to_nanoseconds(local_timestamp, &local_timestamp_ns); 520 struct bt_params *params = bt_params_find(local_timestamp_ns); 521 remote_timestamp = mach_bridge_compute_timestamp(local_timestamp_ns, params); 522 } 523 #else 524 struct bt_params params = bt_params_get_latest_locked(); 525 remote_timestamp = mach_bridge_compute_timestamp(local_timestamp, ¶ms); 526 #endif /* defined(XNU_TARGET_OS_BRIDGE) */ 527 lck_spin_unlock(&bt_ts_conversion_lock); 528 KDBG(MACHDBG_CODE(DBG_MACH_CLOCK, MACH_BRIDGE_REMOTE_TIME), local_timestamp, remote_timestamp, now); 529 530 return remote_timestamp; 531 #endif /* !defined(CONFIG_MACH_BRIDGE_RECV_TIME) */ 532 #endif /* defined(CONFIG_MACH_BRIDGE_SEND_TIME) */ 533 }