pgtrace.c
1 /* 2 * Copyright (c) 2015 Apple Inc. All rights reserved. 3 * 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ 5 * 6 * This file contains Original Code and/or Modifications of Original Code 7 * as defined in and that are subject to the Apple Public Source License 8 * Version 2.0 (the 'License'). You may not use this file except in 9 * compliance with the License. The rights granted to you under the License 10 * may not be used to create, or enable the creation or redistribution of, 11 * unlawful or unlicensed copies of an Apple operating system, or to 12 * circumvent, violate, or enable the circumvention or violation of, any 13 * terms of an Apple operating system software license agreement. 14 * 15 * Please obtain a copy of the License at 16 * http://www.opensource.apple.com/apsl/ and read it before using this file. 17 * 18 * The Original Code and all software distributed under the License are 19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER 20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, 21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, 22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. 23 * Please see the License for the specific language governing rights and 24 * limitations under the License. 25 * 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ 27 */ 28 29 #if CONFIG_PGTRACE 30 #include <mach/mach_types.h> 31 #include <IOKit/IOLib.h> 32 #include <sys/msgbuf.h> 33 #include <sys/errno.h> 34 #include <arm64/pgtrace.h> 35 #include <libkern/OSDebug.h> 36 37 typedef struct { 38 queue_chain_t chain; 39 40 pmap_t pmap; 41 vm_offset_t start; 42 vm_offset_t end; 43 } probe_t; 44 45 #if CONFIG_PGTRACE_NONKEXT 46 #include "pgtrace_decoder.h" 47 48 //-------------------------------------------- 49 // Macros 50 // 51 #define RBUF_DEFAULT_SIZE 1024 52 #define RBUF_IDX(idx, mask) ((idx) & (mask)) 53 #define MSG_MAX 130 54 55 //-------------------------------------------- 56 // Types 57 // 58 typedef uint8_t RWLOCK; 59 60 typedef struct { 61 uint64_t id; 62 pgtrace_run_result_t res; 63 void *stack[PGTRACE_STACK_DEPTH]; 64 } log_t; 65 66 //-------------------------------------------- 67 // Statics 68 // 69 static struct { 70 log_t *logs; // Protect 71 uint32_t size; // Protect 72 uint64_t rdidx, wridx; // Protect 73 74 uint64_t id; 75 uint32_t option; 76 uint32_t enabled; 77 uint32_t bytes; 78 79 queue_head_t probes; // Protect 80 } pgtrace; 81 82 static LCK_GRP_DECLARE(pgtrace_lock_grp, "pgtrace_lock"); 83 static LCK_MTX_DECLARE(pgtrace_probelock, &pgtrace_lock_grp); 84 static SIMPLE_LOCK_DECLARE(pgtrace_loglock, 0); 85 86 //-------------------------------------------- 87 // Globals 88 // 89 void 90 pgtrace_init(void) 91 { 92 queue_init(&pgtrace.probes); 93 94 pgtrace.size = RBUF_DEFAULT_SIZE; 95 pgtrace.logs = kalloc(RBUF_DEFAULT_SIZE * sizeof(log_t)); 96 } 97 98 void 99 pgtrace_clear_probe(void) 100 { 101 probe_t *p, *next; 102 queue_head_t *q = &pgtrace.probes; 103 104 lck_mtx_lock(&pgtrace_probelock); 105 106 p = (probe_t *)queue_first(q); 107 while (!queue_end(q, (queue_entry_t)p)) { 108 next = (probe_t *)queue_next(&(p->chain)); 109 110 queue_remove(q, p, probe_t *, chain); 111 kfree(p, sizeof(probe_t)); 112 113 p = next; 114 } 115 116 lck_mtx_unlock(&pgtrace_probelock); 117 } 118 119 int 120 pgtrace_add_probe(thread_t thread, vm_offset_t start, vm_offset_t end) 121 { 122 probe_t *p; 123 queue_head_t *q = &pgtrace.probes; 124 125 if (start > end) { 126 kprintf("%s Invalid start=%lx end=%lx\n", __func__, start, end); 127 return -1; 128 } 129 130 p = kalloc(sizeof(probe_t)); 131 p->start = start; 132 p->end = end; 133 if (thread == NULL) { 134 p->pmap = NULL; 135 } else { 136 p->pmap = vm_map_pmap(thread->map); 137 } 138 139 lck_mtx_lock(&pgtrace_probelock); 140 queue_enter(q, p, probe_t *, chain); 141 lck_mtx_unlock(&pgtrace_probelock); 142 143 return 0; 144 } 145 146 void 147 pgtrace_start(void) 148 { 149 probe_t *p; 150 queue_head_t *q = &pgtrace.probes; 151 152 kprintf("%s\n", __func__); 153 154 if (pgtrace.enabled) { 155 return; 156 } 157 158 pgtrace.enabled = 1; 159 160 lck_mtx_lock(&pgtrace_probelock); 161 162 queue_iterate(q, p, probe_t *, chain) { 163 pmap_pgtrace_add_page(p->pmap, p->start, p->end); 164 } 165 166 lck_mtx_unlock(&pgtrace_probelock); 167 } 168 169 void 170 pgtrace_stop(void) 171 { 172 probe_t *p; 173 queue_head_t *q = &pgtrace.probes; 174 175 kprintf("%s\n", __func__); 176 177 lck_mtx_lock(&pgtrace_probelock); 178 179 queue_iterate(q, p, probe_t *, chain) { 180 pmap_pgtrace_delete_page(p->pmap, p->start, p->end); 181 } 182 183 lck_mtx_unlock(&pgtrace_probelock); 184 185 pgtrace.enabled = 0; 186 } 187 188 uint32_t 189 pgtrace_get_size(void) 190 { 191 return pgtrace.size; 192 } 193 194 bool 195 pgtrace_set_size(uint32_t size) 196 { 197 log_t *old_buf, *new_buf; 198 uint32_t old_size, new_size = 1; 199 200 // round up to next power of 2 201 while (size > new_size) { 202 new_size <<= 1; 203 if (new_size > 0x100000) { 204 // over million entries 205 kprintf("%s: size=%x new_size=%x is too big\n", __func__, size, new_size); 206 return false; 207 } 208 } 209 210 new_buf = kalloc(new_size * sizeof(log_t)); 211 if (new_buf == NULL) { 212 kprintf("%s: can't allocate new_size=%x\n entries", __func__, new_size); 213 return false; 214 } 215 216 pgtrace_stop(); 217 218 simple_lock(&pgtrace_loglock); 219 old_buf = pgtrace.logs; 220 old_size = pgtrace.size; 221 pgtrace.logs = new_buf; 222 pgtrace.size = new_size; 223 pgtrace.rdidx = pgtrace.wridx = 0; 224 simple_unlock(&pgtrace_loglock); 225 226 if (old_buf) { 227 kfree(old_buf, old_size * sizeof(log_t)); 228 } 229 230 return true; 231 } 232 233 void 234 pgtrace_clear_trace(void) 235 { 236 simple_lock(&pgtrace_loglock); 237 pgtrace.rdidx = pgtrace.wridx = 0; 238 simple_unlock(&pgtrace_loglock); 239 } 240 241 boolean_t 242 pgtrace_active(void) 243 { 244 return pgtrace.enabled > 0; 245 } 246 247 uint32_t 248 pgtrace_get_option(void) 249 { 250 return pgtrace.option; 251 } 252 253 void 254 pgtrace_set_option(uint32_t option) 255 { 256 pgtrace.option = option; 257 } 258 259 // pgtrace_write_log() is in interrupt disabled context 260 void 261 pgtrace_write_log(pgtrace_run_result_t res) 262 { 263 uint8_t i; 264 log_t log = {}; 265 const char *rwmap[] = { "R", "W", "PREFETCH" }; 266 267 log.id = pgtrace.id++; 268 log.res = res; 269 270 if (pgtrace.option & PGTRACE_OPTION_KPRINTF) { 271 char msg[MSG_MAX]; 272 char *p; 273 274 p = msg; 275 276 snprintf(p, MSG_MAX, "%llu %s ", res.rr_time, rwmap[res.rr_rw]); 277 p += strlen(p); 278 279 for (i = 0; i < res.rr_num; i++) { 280 snprintf(p, MSG_MAX - (p - msg), "%lx=%llx ", res.rr_addrdata[i].ad_addr, res.rr_addrdata[i].ad_data); 281 p += strlen(p); 282 } 283 284 kprintf("%s %s\n", __func__, msg); 285 } 286 287 if (pgtrace.option & PGTRACE_OPTION_STACK) { 288 OSBacktrace(log.stack, PGTRACE_STACK_DEPTH); 289 } 290 291 pgtrace.bytes += sizeof(log); 292 293 simple_lock(&pgtrace_loglock); 294 295 pgtrace.logs[RBUF_IDX(pgtrace.wridx, pgtrace.size - 1)] = log; 296 297 // Advance rdidx if ring is full 298 if (RBUF_IDX(pgtrace.wridx, pgtrace.size - 1) == RBUF_IDX(pgtrace.rdidx, pgtrace.size - 1) && 299 (pgtrace.wridx != pgtrace.rdidx)) { 300 pgtrace.rdidx++; 301 } 302 pgtrace.wridx++; 303 304 // Signal if ring was empty 305 if (pgtrace.wridx == (pgtrace.rdidx + 1)) { 306 thread_wakeup(pgtrace.logs); 307 } 308 309 simple_unlock(&pgtrace_loglock); 310 } 311 312 // pgtrace_read_log() is in user thread 313 int64_t 314 pgtrace_read_log(uint8_t *buf, uint32_t size) 315 { 316 int total, front, back; 317 boolean_t ints; 318 wait_result_t wr; 319 320 if (pgtrace.enabled == FALSE) { 321 return -EINVAL; 322 } 323 324 total = size / sizeof(log_t); 325 326 // Check if buf is too small 327 if (buf && total == 0) { 328 return -EINVAL; 329 } 330 331 ints = ml_set_interrupts_enabled(FALSE); 332 simple_lock(&pgtrace_loglock); 333 334 // Wait if ring is empty 335 if (pgtrace.rdidx == pgtrace.wridx) { 336 assert_wait(pgtrace.logs, THREAD_ABORTSAFE); 337 338 simple_unlock(&pgtrace_loglock); 339 ml_set_interrupts_enabled(ints); 340 341 wr = thread_block(NULL); 342 if (wr != THREAD_AWAKENED) { 343 return -EINTR; 344 } 345 346 ints = ml_set_interrupts_enabled(FALSE); 347 simple_lock(&pgtrace_loglock); 348 } 349 350 // Trim the size 351 if ((pgtrace.rdidx + total) > pgtrace.wridx) { 352 total = (int)(pgtrace.wridx - pgtrace.rdidx); 353 } 354 355 // Copy front 356 if ((RBUF_IDX(pgtrace.rdidx, pgtrace.size - 1) + total) >= pgtrace.size) { 357 front = pgtrace.size - RBUF_IDX(pgtrace.rdidx, pgtrace.size - 1); 358 } else { 359 front = total; 360 } 361 362 memcpy(buf, &(pgtrace.logs[RBUF_IDX(pgtrace.rdidx, pgtrace.size - 1)]), front * sizeof(log_t)); 363 364 // Copy back if any 365 back = total - front; 366 if (back) { 367 buf += front * sizeof(log_t); 368 memcpy(buf, pgtrace.logs, back * sizeof(log_t)); 369 } 370 371 pgtrace.rdidx += total; 372 373 simple_unlock(&pgtrace_loglock); 374 ml_set_interrupts_enabled(ints); 375 376 return total * sizeof(log_t); 377 } 378 379 int 380 pgtrace_get_stats(pgtrace_stats_t *stats) 381 { 382 if (!stats) { 383 return -1; 384 } 385 386 stats->stat_logger.sl_bytes = pgtrace.bytes; 387 pgtrace_decoder_get_stats(stats); 388 389 return 0; 390 } 391 392 #else // CONFIG_PGTRACE_NONKEXT 393 394 static struct { 395 bool active; 396 decoder_t *decoder; 397 logger_t *logger; 398 queue_head_t probes; 399 } pgtrace; 400 401 static LCK_GRP_DECLARE(pgtrace_lock_grp, "pgtrace_lock"); 402 static LCK_MTX_DECLARE(pgtrace_probelock, &pgtrace_lock_grp); 403 404 //------------------------------------ 405 // functions for pmap fault handler 406 // - pgtrace_decode_and_run 407 // - pgtrace_write_log 408 //------------------------------------ 409 int 410 pgtrace_decode_and_run(uint32_t inst, vm_offset_t fva, vm_map_offset_t *cva_page, arm_saved_state_t *ss, pgtrace_run_result_t *res) 411 { 412 vm_offset_t pa, cva; 413 pgtrace_instruction_info_t info; 414 vm_offset_t cva_front_page = cva_page[0]; 415 vm_offset_t cva_cur_page = cva_page[1]; 416 417 pgtrace.decoder->decode(inst, ss, &info); 418 419 if (info.addr == fva) { 420 cva = cva_cur_page + (fva & ARM_PGMASK); 421 } else { 422 // which means a front page is not a tracing page 423 cva = cva_front_page + (fva & ARM_PGMASK); 424 } 425 426 pa = mmu_kvtop(cva); 427 if (!pa) { 428 panic("%s: invalid address cva=%lx fva=%lx info.addr=%lx inst=%x", __func__, cva, fva, info.addr, inst); 429 } 430 431 absolutetime_to_nanoseconds(mach_absolute_time(), &res->rr_time); 432 433 pgtrace.decoder->run(inst, pa, cva, ss, res); 434 435 return 0; 436 } 437 438 int 439 pgtrace_write_log(pgtrace_run_result_t res) 440 { 441 pgtrace.logger->write(res); 442 return 0; 443 } 444 445 //------------------------------------ 446 // functions for kext 447 // - pgtrace_init 448 // - pgtrace_add_probe 449 // - pgtrace_clear_probe 450 // - pgtrace_start 451 // - pgtrace_stop 452 // - pgtrace_active 453 //------------------------------------ 454 int 455 pgtrace_init(decoder_t *decoder, logger_t *logger) 456 { 457 kprintf("%s decoder=%p logger=%p\n", __func__, decoder, logger); 458 459 assert(decoder && logger); 460 461 if (decoder->magic != 0xfeedface || logger->magic != 0xfeedface || 462 strcmp(decoder->arch, "arm64") != 0 || strcmp(logger->arch, "arm64") != 0) { 463 kprintf("%s:wrong decoder/logger magic=%llx/%llx arch=%s/%s", __func__, decoder->magic, logger->magic, decoder->arch, logger->arch); 464 return EINVAL; 465 } 466 467 queue_init(&pgtrace.probes); 468 pgtrace.decoder = decoder; 469 pgtrace.logger = logger; 470 471 return 0; 472 } 473 474 int 475 pgtrace_add_probe(thread_t thread, vm_offset_t start, vm_offset_t end) 476 { 477 probe_t *p; 478 queue_head_t *q = &pgtrace.probes; 479 480 kprintf("%s start=%lx end=%lx\n", __func__, start, end); 481 482 if (start > end) { 483 kprintf("%s Invalid start=%lx end=%lx\n", __func__, start, end); 484 return -1; 485 } 486 487 p = kalloc(sizeof(probe_t)); 488 p->start = start; 489 p->end = end; 490 if (thread == NULL) { 491 p->pmap = NULL; 492 } else { 493 p->pmap = vm_map_pmap(thread->map); 494 } 495 496 lck_mtx_lock(&pgtrace_probelock); 497 queue_enter(q, p, probe_t *, chain); 498 lck_mtx_unlock(&pgtrace_probelock); 499 500 return 0; 501 } 502 503 void 504 pgtrace_clear_probe(void) 505 { 506 probe_t *p, *next; 507 queue_head_t *q = &pgtrace.probes; 508 509 kprintf("%s\n", __func__); 510 511 lck_mtx_lock(&pgtrace_probelock); 512 513 p = (probe_t *)queue_first(q); 514 while (!queue_end(q, (queue_entry_t)p)) { 515 next = (probe_t *)queue_next(&(p->chain)); 516 517 queue_remove(q, p, probe_t *, chain); 518 kfree(p, sizeof(probe_t)); 519 520 p = next; 521 } 522 523 lck_mtx_unlock(&pgtrace_probelock); 524 } 525 526 void 527 pgtrace_start(void) 528 { 529 probe_t *p; 530 queue_head_t *q = &pgtrace.probes; 531 532 kprintf("%s\n", __func__); 533 534 if (pgtrace.active == true) { 535 return; 536 } 537 538 pgtrace.active = true; 539 540 lck_mtx_lock(&pgtrace_probelock); 541 542 queue_iterate(q, p, probe_t *, chain) { 543 pmap_pgtrace_add_page(p->pmap, p->start, p->end); 544 } 545 546 lck_mtx_unlock(&pgtrace_probelock); 547 } 548 549 void 550 pgtrace_stop(void) 551 { 552 probe_t *p; 553 queue_head_t *q = &pgtrace.probes; 554 555 kprintf("%s\n", __func__); 556 557 lck_mtx_lock(&pgtrace_probelock); 558 559 queue_iterate(q, p, probe_t *, chain) { 560 pmap_pgtrace_delete_page(p->pmap, p->start, p->end); 561 } 562 563 lck_mtx_unlock(&pgtrace_probelock); 564 565 pgtrace.active = false; 566 } 567 568 bool 569 pgtrace_active(void) 570 { 571 return pgtrace.active; 572 } 573 #endif // CONFIG_PGTRACE_NONKEXT 574 #else 575 // empty funcs for release kernel 576 extern void pgtrace_stop(void); 577 extern void pgtrace_start(void); 578 extern void pgtrace_clear_probe(void); 579 extern void pgtrace_add_probe(void); 580 extern void pgtrace_init(void); 581 extern void pgtrace_active(void); 582 void 583 pgtrace_stop(void) 584 { 585 } 586 void 587 pgtrace_start(void) 588 { 589 } 590 void 591 pgtrace_clear_probe(void) 592 { 593 } 594 void 595 pgtrace_add_probe(void) 596 { 597 } 598 void 599 pgtrace_init(void) 600 { 601 } 602 void 603 pgtrace_active(void) 604 { 605 } 606 #endif