kpc_common.c
1 /* 2 * Copyright (c) 2012 Apple Inc. All rights reserved. 3 * 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ 5 * 6 * This file contains Original Code and/or Modifications of Original Code 7 * as defined in and that are subject to the Apple Public Source License 8 * Version 2.0 (the 'License'). You may not use this file except in 9 * compliance with the License. The rights granted to you under the License 10 * may not be used to create, or enable the creation or redistribution of, 11 * unlawful or unlicensed copies of an Apple operating system, or to 12 * circumvent, violate, or enable the circumvention or violation of, any 13 * terms of an Apple operating system software license agreement. 14 * 15 * Please obtain a copy of the License at 16 * http://www.opensource.apple.com/apsl/ and read it before using this file. 17 * 18 * The Original Code and all software distributed under the License are 19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER 20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, 21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, 22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. 23 * Please see the License for the specific language governing rights and 24 * limitations under the License. 25 * 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ 27 */ 28 29 #include <mach/mach_types.h> 30 #include <machine/machine_routines.h> 31 #include <kern/processor.h> 32 #include <kern/kalloc.h> 33 #include <sys/errno.h> 34 #include <sys/vm.h> 35 #include <kperf/buffer.h> 36 #include <kern/thread.h> 37 #if defined(__arm64__) || defined(__arm__) 38 #include <arm/cpu_data_internal.h> 39 #endif 40 41 #include <kern/kpc.h> 42 43 #include <kperf/kperf.h> 44 #include <kperf/sample.h> 45 #include <kperf/context.h> 46 #include <kperf/action.h> 47 48 uint32_t kpc_actionid[KPC_MAX_COUNTERS]; 49 50 #define COUNTERBUF_SIZE_PER_CPU (KPC_MAX_COUNTERS * sizeof(uint64_t)) 51 #define COUNTERBUF_SIZE (machine_info.logical_cpu_max * \ 52 COUNTERBUF_SIZE_PER_CPU) 53 54 /* locks */ 55 static LCK_GRP_DECLARE(kpc_config_lckgrp, "kpc"); 56 static LCK_MTX_DECLARE(kpc_config_lock, &kpc_config_lckgrp); 57 58 /* state specifying if all counters have been requested by kperf */ 59 static boolean_t force_all_ctrs = FALSE; 60 61 /* power manager */ 62 static kpc_pm_handler_t kpc_pm_handler; 63 static boolean_t kpc_pm_has_custom_config; 64 static uint64_t kpc_pm_pmc_mask; 65 #if MACH_ASSERT 66 static bool kpc_calling_pm = false; 67 #endif /* MACH_ASSERT */ 68 69 boolean_t kpc_context_switch_active = FALSE; 70 bool kpc_supported = true; 71 72 static uint64_t * 73 kpc_percpu_alloc(void) 74 { 75 return kheap_alloc_tag(KHEAP_DATA_BUFFERS, COUNTERBUF_SIZE_PER_CPU, 76 Z_WAITOK | Z_ZERO, VM_KERN_MEMORY_DIAG); 77 } 78 79 static void 80 kpc_percpu_free(uint64_t *buf) 81 { 82 if (buf) { 83 kheap_free(KHEAP_DATA_BUFFERS, buf, COUNTERBUF_SIZE_PER_CPU); 84 } 85 } 86 87 boolean_t 88 kpc_register_cpu(struct cpu_data *cpu_data) 89 { 90 assert(cpu_data); 91 assert(cpu_data->cpu_kpc_buf[0] == NULL); 92 assert(cpu_data->cpu_kpc_buf[1] == NULL); 93 assert(cpu_data->cpu_kpc_shadow == NULL); 94 assert(cpu_data->cpu_kpc_reload == NULL); 95 96 /* 97 * Buffers allocated through kpc_counterbuf_alloc() are large enough to 98 * store all PMCs values from all CPUs. This mimics the userspace API. 99 * This does not suit well with the per-CPU kpc buffers, since: 100 * 1. Buffers don't need to be this large. 101 * 2. The actual number of CPUs is not known at this point. 102 * 103 * CPUs are asked to callout into kpc when being registered, we'll 104 * allocate the memory here. 105 */ 106 107 if ((cpu_data->cpu_kpc_buf[0] = kpc_percpu_alloc()) == NULL) { 108 goto error; 109 } 110 if ((cpu_data->cpu_kpc_buf[1] = kpc_percpu_alloc()) == NULL) { 111 goto error; 112 } 113 if ((cpu_data->cpu_kpc_shadow = kpc_percpu_alloc()) == NULL) { 114 goto error; 115 } 116 if ((cpu_data->cpu_kpc_reload = kpc_percpu_alloc()) == NULL) { 117 goto error; 118 } 119 120 /* success */ 121 return TRUE; 122 123 error: 124 kpc_unregister_cpu(cpu_data); 125 return FALSE; 126 } 127 128 void 129 kpc_unregister_cpu(struct cpu_data *cpu_data) 130 { 131 assert(cpu_data); 132 if (cpu_data->cpu_kpc_buf[0] != NULL) { 133 kpc_percpu_free(cpu_data->cpu_kpc_buf[0]); 134 cpu_data->cpu_kpc_buf[0] = NULL; 135 } 136 if (cpu_data->cpu_kpc_buf[1] != NULL) { 137 kpc_percpu_free(cpu_data->cpu_kpc_buf[1]); 138 cpu_data->cpu_kpc_buf[1] = NULL; 139 } 140 if (cpu_data->cpu_kpc_shadow != NULL) { 141 kpc_percpu_free(cpu_data->cpu_kpc_shadow); 142 cpu_data->cpu_kpc_shadow = NULL; 143 } 144 if (cpu_data->cpu_kpc_reload != NULL) { 145 kpc_percpu_free(cpu_data->cpu_kpc_reload); 146 cpu_data->cpu_kpc_reload = NULL; 147 } 148 } 149 150 151 static void 152 kpc_task_set_forced_all_ctrs(task_t task, boolean_t state) 153 { 154 assert(task); 155 156 task_lock(task); 157 if (state) { 158 task->t_kpc |= TASK_KPC_FORCED_ALL_CTRS; 159 } else { 160 task->t_kpc &= ~TASK_KPC_FORCED_ALL_CTRS; 161 } 162 task_unlock(task); 163 } 164 165 static boolean_t 166 kpc_task_get_forced_all_ctrs(task_t task) 167 { 168 assert(task); 169 return task->t_kpc & TASK_KPC_FORCED_ALL_CTRS ? TRUE : FALSE; 170 } 171 172 int 173 kpc_force_all_ctrs(task_t task, int val) 174 { 175 boolean_t new_state = val ? TRUE : FALSE; 176 boolean_t old_state = kpc_get_force_all_ctrs(); 177 178 /* 179 * Refuse to do the operation if the counters are already forced by 180 * another task. 181 */ 182 if (kpc_get_force_all_ctrs() && !kpc_task_get_forced_all_ctrs(task)) { 183 return EACCES; 184 } 185 186 /* nothing to do if the state is not changing */ 187 if (old_state == new_state) { 188 return 0; 189 } 190 191 /* notify the power manager */ 192 if (kpc_pm_handler) { 193 #if MACH_ASSERT 194 kpc_calling_pm = true; 195 #endif /* MACH_ASSERT */ 196 kpc_pm_handler( new_state ? FALSE : TRUE ); 197 #if MACH_ASSERT 198 kpc_calling_pm = false; 199 #endif /* MACH_ASSERT */ 200 } 201 202 /* 203 * This is a force -- ensure that counters are forced, even if power 204 * management fails to acknowledge it. 205 */ 206 if (force_all_ctrs != new_state) { 207 force_all_ctrs = new_state; 208 } 209 210 /* update the task bits */ 211 kpc_task_set_forced_all_ctrs(task, new_state); 212 213 return 0; 214 } 215 216 void 217 kpc_pm_acknowledge(boolean_t available_to_pm) 218 { 219 /* 220 * Force-all-counters should still be true when the counters are being 221 * made available to power management and false when counters are going 222 * to be taken away. 223 */ 224 assert(force_all_ctrs == available_to_pm); 225 /* 226 * Make sure power management isn't playing games with us. 227 */ 228 assert(kpc_calling_pm == true); 229 230 /* 231 * Counters being available means no one is forcing all counters. 232 */ 233 force_all_ctrs = available_to_pm ? FALSE : TRUE; 234 } 235 236 int 237 kpc_get_force_all_ctrs(void) 238 { 239 return force_all_ctrs; 240 } 241 242 boolean_t 243 kpc_multiple_clients(void) 244 { 245 return kpc_pm_handler != NULL; 246 } 247 248 boolean_t 249 kpc_controls_fixed_counters(void) 250 { 251 return !kpc_pm_handler || force_all_ctrs || !kpc_pm_has_custom_config; 252 } 253 254 boolean_t 255 kpc_controls_counter(uint32_t ctr) 256 { 257 uint64_t pmc_mask = 0ULL; 258 259 assert(ctr < (kpc_fixed_count() + kpc_configurable_count())); 260 261 if (ctr < kpc_fixed_count()) { 262 return kpc_controls_fixed_counters(); 263 } 264 265 /* 266 * By default kpc manages all PMCs, but if the Power Manager registered 267 * with custom_config=TRUE, the Power Manager manages its reserved PMCs. 268 * However, kpc takes ownership back if a task acquired all PMCs via 269 * force_all_ctrs. 270 */ 271 pmc_mask = (1ULL << (ctr - kpc_fixed_count())); 272 if ((pmc_mask & kpc_pm_pmc_mask) && kpc_pm_has_custom_config && !force_all_ctrs) { 273 return FALSE; 274 } 275 276 return TRUE; 277 } 278 279 uint32_t 280 kpc_get_running(void) 281 { 282 uint64_t pmc_mask = 0; 283 uint32_t cur_state = 0; 284 285 if (kpc_is_running_fixed()) { 286 cur_state |= KPC_CLASS_FIXED_MASK; 287 } 288 289 pmc_mask = kpc_get_configurable_pmc_mask(KPC_CLASS_CONFIGURABLE_MASK); 290 if (kpc_is_running_configurable(pmc_mask)) { 291 cur_state |= KPC_CLASS_CONFIGURABLE_MASK; 292 } 293 294 pmc_mask = kpc_get_configurable_pmc_mask(KPC_CLASS_POWER_MASK); 295 if ((pmc_mask != 0) && kpc_is_running_configurable(pmc_mask)) { 296 cur_state |= KPC_CLASS_POWER_MASK; 297 } 298 299 return cur_state; 300 } 301 302 /* may be called from an IPI */ 303 int 304 kpc_get_curcpu_counters(uint32_t classes, int *curcpu, uint64_t *buf) 305 { 306 int enabled = 0, offset = 0; 307 uint64_t pmc_mask = 0ULL; 308 309 assert(buf); 310 311 enabled = ml_set_interrupts_enabled(FALSE); 312 313 /* grab counters and CPU number as close as possible */ 314 if (curcpu) { 315 *curcpu = cpu_number(); 316 } 317 318 if (classes & KPC_CLASS_FIXED_MASK) { 319 kpc_get_fixed_counters(&buf[offset]); 320 offset += kpc_get_counter_count(KPC_CLASS_FIXED_MASK); 321 } 322 323 if (classes & KPC_CLASS_CONFIGURABLE_MASK) { 324 pmc_mask = kpc_get_configurable_pmc_mask(KPC_CLASS_CONFIGURABLE_MASK); 325 kpc_get_configurable_counters(&buf[offset], pmc_mask); 326 offset += kpc_popcount(pmc_mask); 327 } 328 329 if (classes & KPC_CLASS_POWER_MASK) { 330 pmc_mask = kpc_get_configurable_pmc_mask(KPC_CLASS_POWER_MASK); 331 kpc_get_configurable_counters(&buf[offset], pmc_mask); 332 offset += kpc_popcount(pmc_mask); 333 } 334 335 ml_set_interrupts_enabled(enabled); 336 337 return offset; 338 } 339 340 /* generic counter reading function, public api */ 341 int 342 kpc_get_cpu_counters(boolean_t all_cpus, uint32_t classes, 343 int *curcpu, uint64_t *buf) 344 { 345 assert(buf); 346 347 /* 348 * Unlike reading the current CPU counters, reading counters from all 349 * CPUs is architecture dependent. This allows kpc to make the most of 350 * the platform if memory mapped registers is supported. 351 */ 352 if (all_cpus) { 353 return kpc_get_all_cpus_counters(classes, curcpu, buf); 354 } else { 355 return kpc_get_curcpu_counters(classes, curcpu, buf); 356 } 357 } 358 359 int 360 kpc_get_shadow_counters(boolean_t all_cpus, uint32_t classes, 361 int *curcpu, uint64_t *buf) 362 { 363 int curcpu_id = cpu_number(); 364 uint32_t cfg_count = kpc_configurable_count(), offset = 0; 365 uint64_t pmc_mask = 0ULL; 366 boolean_t enabled; 367 368 assert(buf); 369 370 enabled = ml_set_interrupts_enabled(FALSE); 371 372 curcpu_id = cpu_number(); 373 if (curcpu) { 374 *curcpu = curcpu_id; 375 } 376 377 for (int cpu = 0; cpu < machine_info.logical_cpu_max; ++cpu) { 378 /* filter if the caller did not request all cpus */ 379 if (!all_cpus && (cpu != curcpu_id)) { 380 continue; 381 } 382 383 if (classes & KPC_CLASS_FIXED_MASK) { 384 uint32_t count = kpc_get_counter_count(KPC_CLASS_FIXED_MASK); 385 memcpy(&buf[offset], &FIXED_SHADOW_CPU(cpu, 0), count * sizeof(uint64_t)); 386 offset += count; 387 } 388 389 if (classes & KPC_CLASS_CONFIGURABLE_MASK) { 390 pmc_mask = kpc_get_configurable_pmc_mask(KPC_CLASS_CONFIGURABLE_MASK); 391 392 for (uint32_t cfg_ctr = 0; cfg_ctr < cfg_count; ++cfg_ctr) { 393 if ((1ULL << cfg_ctr) & pmc_mask) { 394 buf[offset++] = CONFIGURABLE_SHADOW_CPU(cpu, cfg_ctr); 395 } 396 } 397 } 398 399 if (classes & KPC_CLASS_POWER_MASK) { 400 pmc_mask = kpc_get_configurable_pmc_mask(KPC_CLASS_POWER_MASK); 401 402 for (uint32_t cfg_ctr = 0; cfg_ctr < cfg_count; ++cfg_ctr) { 403 if ((1ULL << cfg_ctr) & pmc_mask) { 404 buf[offset++] = CONFIGURABLE_SHADOW_CPU(cpu, cfg_ctr); 405 } 406 } 407 } 408 } 409 410 ml_set_interrupts_enabled(enabled); 411 412 return offset; 413 } 414 415 uint32_t 416 kpc_get_counter_count(uint32_t classes) 417 { 418 uint32_t count = 0; 419 420 if (classes & KPC_CLASS_FIXED_MASK) { 421 count += kpc_fixed_count(); 422 } 423 424 if (classes & (KPC_CLASS_CONFIGURABLE_MASK | KPC_CLASS_POWER_MASK)) { 425 uint64_t pmc_msk = kpc_get_configurable_pmc_mask(classes); 426 uint32_t pmc_cnt = kpc_popcount(pmc_msk); 427 count += pmc_cnt; 428 } 429 430 return count; 431 } 432 433 uint32_t 434 kpc_get_config_count(uint32_t classes) 435 { 436 uint32_t count = 0; 437 438 if (classes & KPC_CLASS_FIXED_MASK) { 439 count += kpc_fixed_config_count(); 440 } 441 442 if (classes & (KPC_CLASS_CONFIGURABLE_MASK | KPC_CLASS_POWER_MASK)) { 443 uint64_t pmc_mask = kpc_get_configurable_pmc_mask(classes); 444 count += kpc_configurable_config_count(pmc_mask); 445 } 446 447 if ((classes & KPC_CLASS_RAWPMU_MASK) && !kpc_multiple_clients()) { 448 count += kpc_rawpmu_config_count(); 449 } 450 451 return count; 452 } 453 454 int 455 kpc_get_config(uint32_t classes, kpc_config_t *current_config) 456 { 457 uint32_t count = 0; 458 459 assert(current_config); 460 461 if (classes & KPC_CLASS_FIXED_MASK) { 462 kpc_get_fixed_config(¤t_config[count]); 463 count += kpc_get_config_count(KPC_CLASS_FIXED_MASK); 464 } 465 466 if (classes & KPC_CLASS_CONFIGURABLE_MASK) { 467 uint64_t pmc_mask = kpc_get_configurable_pmc_mask(KPC_CLASS_CONFIGURABLE_MASK); 468 kpc_get_configurable_config(¤t_config[count], pmc_mask); 469 count += kpc_get_config_count(KPC_CLASS_CONFIGURABLE_MASK); 470 } 471 472 if (classes & KPC_CLASS_POWER_MASK) { 473 uint64_t pmc_mask = kpc_get_configurable_pmc_mask(KPC_CLASS_POWER_MASK); 474 kpc_get_configurable_config(¤t_config[count], pmc_mask); 475 count += kpc_get_config_count(KPC_CLASS_POWER_MASK); 476 } 477 478 if (classes & KPC_CLASS_RAWPMU_MASK) { 479 // Client shouldn't ask for config words that aren't available. 480 // Most likely, they'd misinterpret the returned buffer if we 481 // allowed this. 482 if (kpc_multiple_clients()) { 483 return EPERM; 484 } 485 kpc_get_rawpmu_config(¤t_config[count]); 486 count += kpc_get_config_count(KPC_CLASS_RAWPMU_MASK); 487 } 488 489 return 0; 490 } 491 492 int 493 kpc_set_config(uint32_t classes, kpc_config_t *configv) 494 { 495 int ret = 0; 496 struct kpc_config_remote mp_config = { 497 .classes = classes, .configv = configv, 498 .pmc_mask = kpc_get_configurable_pmc_mask(classes) 499 }; 500 501 assert(configv); 502 503 /* don't allow RAWPMU configuration when sharing counters */ 504 if ((classes & KPC_CLASS_RAWPMU_MASK) && kpc_multiple_clients()) { 505 return EPERM; 506 } 507 508 /* no clients have the right to modify both classes */ 509 if ((classes & (KPC_CLASS_CONFIGURABLE_MASK)) && 510 (classes & (KPC_CLASS_POWER_MASK))) { 511 return EPERM; 512 } 513 514 lck_mtx_lock(&kpc_config_lock); 515 516 /* translate the power class for the machine layer */ 517 if (classes & KPC_CLASS_POWER_MASK) { 518 mp_config.classes |= KPC_CLASS_CONFIGURABLE_MASK; 519 } 520 521 ret = kpc_set_config_arch( &mp_config ); 522 523 lck_mtx_unlock(&kpc_config_lock); 524 525 return ret; 526 } 527 528 uint32_t 529 kpc_get_counterbuf_size(void) 530 { 531 return COUNTERBUF_SIZE; 532 } 533 534 /* allocate a buffer large enough for all possible counters */ 535 uint64_t * 536 kpc_counterbuf_alloc(void) 537 { 538 return kheap_alloc_tag(KHEAP_DATA_BUFFERS, COUNTERBUF_SIZE, 539 Z_WAITOK | Z_ZERO, VM_KERN_MEMORY_DIAG); 540 } 541 542 void 543 kpc_counterbuf_free(uint64_t *buf) 544 { 545 if (buf) { 546 kheap_free(KHEAP_DATA_BUFFERS, buf, COUNTERBUF_SIZE); 547 } 548 } 549 550 void 551 kpc_sample_kperf(uint32_t actionid, uint32_t counter, uint64_t config, 552 uint64_t count, uintptr_t pc, kperf_kpc_flags_t flags) 553 { 554 struct kperf_sample sbuf; 555 556 uint64_t desc = config | (uint64_t)counter << 32 | (uint64_t)flags << 48; 557 558 BUF_DATA(PERF_KPC_HNDLR | DBG_FUNC_START, desc, count, pc); 559 560 thread_t thread = current_thread(); 561 task_t task = get_threadtask(thread); 562 563 struct kperf_context ctx = { 564 .cur_thread = thread, 565 .cur_task = task, 566 .cur_pid = task_pid(task), 567 .trigger_type = TRIGGER_TYPE_PMI, 568 .trigger_id = 0, 569 }; 570 571 int r = kperf_sample(&sbuf, &ctx, actionid, SAMPLE_FLAG_PEND_USER); 572 573 BUF_INFO(PERF_KPC_HNDLR | DBG_FUNC_END, r); 574 } 575 576 577 int 578 kpc_set_period(uint32_t classes, uint64_t *val) 579 { 580 struct kpc_config_remote mp_config = { 581 .classes = classes, .configv = val, 582 .pmc_mask = kpc_get_configurable_pmc_mask(classes) 583 }; 584 585 assert(val); 586 587 /* no clients have the right to modify both classes */ 588 if ((classes & (KPC_CLASS_CONFIGURABLE_MASK)) && 589 (classes & (KPC_CLASS_POWER_MASK))) { 590 return EPERM; 591 } 592 593 lck_mtx_lock(&kpc_config_lock); 594 595 #ifdef FIXED_COUNTER_SHADOW 596 if ((classes & KPC_CLASS_FIXED_MASK) && !kpc_controls_fixed_counters()) { 597 lck_mtx_unlock(&kpc_config_lock); 598 return EPERM; 599 } 600 # else 601 if (classes & KPC_CLASS_FIXED_MASK) { 602 lck_mtx_unlock(&kpc_config_lock); 603 return EINVAL; 604 } 605 #endif 606 607 /* translate the power class for the machine layer */ 608 if (classes & KPC_CLASS_POWER_MASK) { 609 mp_config.classes |= KPC_CLASS_CONFIGURABLE_MASK; 610 } 611 612 kprintf("setting period %u\n", classes); 613 kpc_set_period_arch( &mp_config ); 614 615 lck_mtx_unlock(&kpc_config_lock); 616 617 return 0; 618 } 619 620 int 621 kpc_get_period(uint32_t classes, uint64_t *val) 622 { 623 uint32_t count = 0; 624 uint64_t pmc_mask = 0ULL; 625 626 assert(val); 627 628 lck_mtx_lock(&kpc_config_lock); 629 630 if (classes & KPC_CLASS_FIXED_MASK) { 631 /* convert reload values to periods */ 632 count = kpc_get_counter_count(KPC_CLASS_FIXED_MASK); 633 for (uint32_t i = 0; i < count; ++i) { 634 *val++ = kpc_fixed_max() - FIXED_RELOAD(i); 635 } 636 } 637 638 if (classes & KPC_CLASS_CONFIGURABLE_MASK) { 639 pmc_mask = kpc_get_configurable_pmc_mask(KPC_CLASS_CONFIGURABLE_MASK); 640 641 /* convert reload values to periods */ 642 count = kpc_configurable_count(); 643 for (uint32_t i = 0; i < count; ++i) { 644 if ((1ULL << i) & pmc_mask) { 645 *val++ = kpc_configurable_max() - CONFIGURABLE_RELOAD(i); 646 } 647 } 648 } 649 650 if (classes & KPC_CLASS_POWER_MASK) { 651 pmc_mask = kpc_get_configurable_pmc_mask(KPC_CLASS_POWER_MASK); 652 653 /* convert reload values to periods */ 654 count = kpc_configurable_count(); 655 for (uint32_t i = 0; i < count; ++i) { 656 if ((1ULL << i) & pmc_mask) { 657 *val++ = kpc_configurable_max() - CONFIGURABLE_RELOAD(i); 658 } 659 } 660 } 661 662 lck_mtx_unlock(&kpc_config_lock); 663 664 return 0; 665 } 666 667 int 668 kpc_set_actionid(uint32_t classes, uint32_t *val) 669 { 670 uint32_t count = 0; 671 uint64_t pmc_mask = 0ULL; 672 673 assert(val); 674 675 /* NOTE: what happens if a pmi occurs while actionids are being 676 * set is undefined. */ 677 lck_mtx_lock(&kpc_config_lock); 678 679 if (classes & KPC_CLASS_FIXED_MASK) { 680 count = kpc_get_counter_count(KPC_CLASS_FIXED_MASK); 681 memcpy(&FIXED_ACTIONID(0), val, count * sizeof(uint32_t)); 682 val += count; 683 } 684 685 if (classes & KPC_CLASS_CONFIGURABLE_MASK) { 686 pmc_mask = kpc_get_configurable_pmc_mask(KPC_CLASS_CONFIGURABLE_MASK); 687 688 count = kpc_configurable_count(); 689 for (uint32_t i = 0; i < count; ++i) { 690 if ((1ULL << i) & pmc_mask) { 691 CONFIGURABLE_ACTIONID(i) = *val++; 692 } 693 } 694 } 695 696 if (classes & KPC_CLASS_POWER_MASK) { 697 pmc_mask = kpc_get_configurable_pmc_mask(KPC_CLASS_POWER_MASK); 698 699 count = kpc_configurable_count(); 700 for (uint32_t i = 0; i < count; ++i) { 701 if ((1ULL << i) & pmc_mask) { 702 CONFIGURABLE_ACTIONID(i) = *val++; 703 } 704 } 705 } 706 707 lck_mtx_unlock(&kpc_config_lock); 708 709 return 0; 710 } 711 712 int 713 kpc_get_actionid(uint32_t classes, uint32_t *val) 714 { 715 uint32_t count = 0; 716 uint64_t pmc_mask = 0ULL; 717 718 assert(val); 719 720 lck_mtx_lock(&kpc_config_lock); 721 722 if (classes & KPC_CLASS_FIXED_MASK) { 723 count = kpc_get_counter_count(KPC_CLASS_FIXED_MASK); 724 memcpy(val, &FIXED_ACTIONID(0), count * sizeof(uint32_t)); 725 val += count; 726 } 727 728 if (classes & KPC_CLASS_CONFIGURABLE_MASK) { 729 pmc_mask = kpc_get_configurable_pmc_mask(KPC_CLASS_CONFIGURABLE_MASK); 730 731 count = kpc_configurable_count(); 732 for (uint32_t i = 0; i < count; ++i) { 733 if ((1ULL << i) & pmc_mask) { 734 *val++ = CONFIGURABLE_ACTIONID(i); 735 } 736 } 737 } 738 739 if (classes & KPC_CLASS_POWER_MASK) { 740 pmc_mask = kpc_get_configurable_pmc_mask(KPC_CLASS_POWER_MASK); 741 742 count = kpc_configurable_count(); 743 for (uint32_t i = 0; i < count; ++i) { 744 if ((1ULL << i) & pmc_mask) { 745 *val++ = CONFIGURABLE_ACTIONID(i); 746 } 747 } 748 } 749 750 lck_mtx_unlock(&kpc_config_lock); 751 752 return 0; 753 } 754 755 int 756 kpc_set_running(uint32_t classes) 757 { 758 uint32_t all_cfg_classes = KPC_CLASS_CONFIGURABLE_MASK | KPC_CLASS_POWER_MASK; 759 struct kpc_running_remote mp_config = { 760 .classes = classes, .cfg_target_mask = 0ULL, .cfg_state_mask = 0ULL 761 }; 762 763 /* target all available PMCs */ 764 mp_config.cfg_target_mask = kpc_get_configurable_pmc_mask(all_cfg_classes); 765 766 /* translate the power class for the machine layer */ 767 if (classes & KPC_CLASS_POWER_MASK) { 768 mp_config.classes |= KPC_CLASS_CONFIGURABLE_MASK; 769 } 770 771 /* generate the state of each configurable PMCs */ 772 mp_config.cfg_state_mask = kpc_get_configurable_pmc_mask(classes); 773 774 return kpc_set_running_arch(&mp_config); 775 } 776 777 boolean_t 778 kpc_register_pm_handler(kpc_pm_handler_t handler) 779 { 780 return kpc_reserve_pm_counters(0x38, handler, TRUE); 781 } 782 783 boolean_t 784 kpc_reserve_pm_counters(uint64_t pmc_mask, kpc_pm_handler_t handler, 785 boolean_t custom_config) 786 { 787 uint64_t all_mask = (1ULL << kpc_configurable_count()) - 1; 788 uint64_t req_mask = 0ULL; 789 790 /* pre-condition */ 791 assert(handler != NULL); 792 assert(kpc_pm_handler == NULL); 793 794 /* check number of counters requested */ 795 req_mask = (pmc_mask & all_mask); 796 assert(kpc_popcount(req_mask) <= kpc_configurable_count()); 797 798 /* save the power manager states */ 799 kpc_pm_has_custom_config = custom_config; 800 kpc_pm_pmc_mask = req_mask; 801 kpc_pm_handler = handler; 802 803 printf("kpc: pm registered pmc_mask=%llx custom_config=%d\n", 804 req_mask, custom_config); 805 806 /* post-condition */ 807 { 808 uint32_t cfg_count = kpc_get_counter_count(KPC_CLASS_CONFIGURABLE_MASK); 809 uint32_t pwr_count = kpc_popcount(kpc_pm_pmc_mask); 810 #pragma unused(cfg_count, pwr_count) 811 assert((cfg_count + pwr_count) == kpc_configurable_count()); 812 } 813 814 return force_all_ctrs ? FALSE : TRUE; 815 } 816 817 void 818 kpc_release_pm_counters(void) 819 { 820 /* pre-condition */ 821 assert(kpc_pm_handler != NULL); 822 823 /* release the counters */ 824 kpc_pm_has_custom_config = FALSE; 825 kpc_pm_pmc_mask = 0ULL; 826 kpc_pm_handler = NULL; 827 828 printf("kpc: pm released counters\n"); 829 830 /* post-condition */ 831 assert(kpc_get_counter_count(KPC_CLASS_CONFIGURABLE_MASK) == kpc_configurable_count()); 832 } 833 834 uint8_t 835 kpc_popcount(uint64_t value) 836 { 837 return (uint8_t)__builtin_popcountll(value); 838 } 839 840 uint64_t 841 kpc_get_configurable_pmc_mask(uint32_t classes) 842 { 843 uint32_t configurable_count = kpc_configurable_count(); 844 uint64_t cfg_mask = 0ULL, pwr_mask = 0ULL, all_cfg_pmcs_mask = 0ULL; 845 846 /* not configurable classes or no configurable counters */ 847 if (((classes & (KPC_CLASS_CONFIGURABLE_MASK | KPC_CLASS_POWER_MASK)) == 0) || 848 (configurable_count == 0)) { 849 goto exit; 850 } 851 852 assert(configurable_count < 64); 853 all_cfg_pmcs_mask = (1ULL << configurable_count) - 1; 854 855 if (classes & KPC_CLASS_CONFIGURABLE_MASK) { 856 if (force_all_ctrs == TRUE) { 857 cfg_mask |= all_cfg_pmcs_mask; 858 } else { 859 cfg_mask |= (~kpc_pm_pmc_mask) & all_cfg_pmcs_mask; 860 } 861 } 862 863 /* 864 * The power class exists iff: 865 * - No tasks acquired all PMCs 866 * - PM registered and uses kpc to interact with PMCs 867 */ 868 if ((force_all_ctrs == FALSE) && 869 (kpc_pm_handler != NULL) && 870 (kpc_pm_has_custom_config == FALSE) && 871 (classes & KPC_CLASS_POWER_MASK)) { 872 pwr_mask |= kpc_pm_pmc_mask & all_cfg_pmcs_mask; 873 } 874 875 exit: 876 /* post-conditions */ 877 assert(((cfg_mask | pwr_mask) & (~all_cfg_pmcs_mask)) == 0 ); 878 assert( kpc_popcount(cfg_mask | pwr_mask) <= kpc_configurable_count()); 879 assert((cfg_mask & pwr_mask) == 0ULL ); 880 881 return cfg_mask | pwr_mask; 882 }