tests.cpp
1 #include <iostream> // todo: use C only 2 3 #define DEVICE CPU 4 5 // #include "nn.h" 6 #include "tensor.cpp" 7 #include "ops.cpp" 8 #include "conv.cpp" 9 #include "utils.cpp" 10 #include "../print.cpp" 11 12 13 // todo: add tests -- https://github.com/tensorflow/tensorflow/commit/6f4a0e96d853d1d8fe05a8dd8f7ba0cd0fb0e79b#diff-65511a88d2951377144d77a2de94c0f597c4664189d3d5ac730e653560b64f31R259-R342 14 // - https://github.com/rui314/8cc/blob/b480958396f159d3794f0d4883172b21438a8597/test/typeof.c#L23 15 16 /* 17 out2[1] = 0.123; 18 print(ReluBackward(out2, N*D), N, D); 19 */ 20 21 /* Transpose tests 22 float arr[] = {0., 1., 2., 3.}; 23 cout << "\narr: "; 24 Print(arr, 2, 2); 25 float* arr_T = Transpose(arr, 2, 2); 26 cout << "\ntransposed: "; 27 Print(arr_T, 2, 2); 28 29 float arr[] = {0., 1., 2., 30 3., 4., 5.}; 31 cout << "\narr: "; 32 Print(arr, 2, 3); 33 float* arr_T = Transpose(arr, 2, 3); 34 cout << "\ntransposed: "; 35 Print(arr_T, 3, 2); 36 */ 37 38 /* pow tests 39 cout << "pow(2, 2): " << pow(2, 2) << endl; 40 cout << "pow(2, 2): " << pow(4, 8) << endl; 41 */ 42 43 /* autograd 44 tensor* a = Tensor(2, 2); 45 a->backward(a); 46 // >>> [autograd engine] Error: tensor has no grad_fn 47 */ 48 49 50 51 int test_net(void) { 52 // random num generator init, must be called once 53 // srand(time(NULL)); 54 srand(123); 55 56 int B = 2; 57 int C = 3; 58 int H = 32; 59 int W = 32; 60 61 int F = 5; 62 int HH = 2; 63 int WW = 2; 64 65 // *** Init *** 66 tensor* input = Tensor(B, C, H, W); 67 set_name(input, "input"); sprint(input); 68 69 // cifar10* data = get_cifar10(); 70 // tensor* input = data->input; 71 72 tensor* kernel = Tensor(F, C, HH, WW); 73 set_name(kernel, "kernel"); sprint(kernel); 74 tensor* kernel2 = Tensor(F, F, HH, WW); 75 set_name(kernel2, "kernel2"); sprint(kernel2); 76 77 78 // *** Net *** 79 tensor* out_conv1 = batched_conv(input, kernel); 80 set_name(out_conv1, "out_conv1"); sprint(out_conv1); 81 tensor* out_relu1 = relu(out_conv1); 82 set_name(out_relu1, "out_relu1"); sprint(out_relu1); 83 tensor* out_mp1 = batched_maxpool(out_relu1); 84 set_name(out_mp1, "out_mp1"); sprint(out_mp1); 85 86 tensor* out_conv2 = batched_conv(out_mp1, kernel2); 87 set_name(out_conv2, "out_conv2"); sprint(out_conv2); 88 tensor* out_relu2 = relu(out_conv2); 89 set_name(out_relu2, "out_relu2"); sprint(out_relu2); 90 tensor* out_mp2 = batched_maxpool(out_relu2); 91 set_name(out_mp2, "out_mp2"); sprint(out_mp2); 92 93 tensor* out_flat = batched_flatten(out_mp2); 94 set_name(out_flat, "out_flat"); sprint(out_flat); 95 96 tensor* w1 = Tensor(out_flat->shape[1], 32); 97 set_name(w1, "w1"); print(w1); 98 tensor* out_mm1 = matmul(out_flat, w1); 99 set_name(out_mm1, "out_mm1"); sprint(out_mm1); 100 tensor* out_relu3 = relu(out_mm1); 101 set_name(out_relu3, "out_relu3"); sprint(out_relu3); 102 103 tensor* w2 = Tensor(out_relu3->shape[1], 16); 104 set_name(w2, "w2"); print(w2); 105 tensor* out_mm2 = matmul(out_relu3, w2); 106 set_name(out_mm2, "out_mm2"); sprint(out_mm1); 107 tensor* out_relu4 = relu(out_mm2); 108 set_name(out_relu4, "out_relu4"); sprint(out_relu4); 109 110 tensor* w3 = Tensor(out_relu4->shape[1], 10); 111 set_name(w3, "w3"); print(w3); 112 tensor* out = matmul(out_relu4, w3); 113 set_name(out, "out"); print(out); 114 115 out->backward(out); 116 graphviz(out); 117 118 119 return 0; 120 } 121 122 123 int test_select(void) { 124 srand(123); 125 tensor* a = Tensor(4, 2); 126 set_name(a, "a"), print(a); 127 tensor* idx = Tensor(4, 1); 128 idx->data[0] = 1.0; 129 idx->data[1] = 0.0; 130 idx->data[2] = 0.0; 131 idx->data[3] = 1.0; 132 133 tensor* out = select(a, idx); 134 135 tensor* w = Tensor(1, 5); 136 set_name(w, "w"), print(w); 137 138 tensor* out2 = matmul(out, w); 139 print(out2); 140 141 out2->backward(out2); 142 set_name(a->grad, "a_grad"), print(a->grad); 143 144 } 145 146 // test_max 147 int test_max(void) { 148 srand(123); 149 tensor* a = Tensor(4, 3); 150 set_name(a, "a"), print(a); 151 152 tensor* out = batched_max(a); 153 set_name(out, "out"), print(out); 154 155 tensor* w = Tensor(1, 5); 156 set_name(w, "w"), print(w); 157 158 tensor* out2 = matmul(out, w); 159 set_name(out2, "out2"), print(out2); 160 161 out2->backward(out2); 162 set_name(a->grad, "a_grad"), print(a->grad); 163 // graphviz(out2); 164 165 return 0; 166 } 167 168 int test_batched_reduce(void) { 169 srand(123); 170 tensor* a = Tensor(4, 10); 171 set_name(a, "a"), print(a); 172 173 tensor* out = batched_reduce_sum(a); 174 set_name(out, "out"), print(out); 175 176 tensor* w = Tensor(1, 5); 177 set_name(w, "w"), print(w); 178 179 tensor* out2 = matmul(out, w); 180 set_name(out2, "out2"), print(out2); 181 182 183 184 out2->backward(out2); 185 set_name(a->grad, "a_grad"), print(a->grad); 186 return 0; 187 } 188 189 int test_exp(void) { 190 srand(123); 191 tensor* a = Tensor(4, 10); 192 set_name(a, "a"), print(a); 193 194 tensor* out = exp(a); 195 set_name(out, "out"), print(out); 196 197 tensor* w = Tensor(10, 5); 198 set_name(w, "w"), print(w); 199 200 tensor* out2 = matmul(out, w); 201 set_name(out2, "out2"), print(out2); 202 203 204 205 out2->backward(out2); 206 set_name(a->grad, "a_grad"), print(a->grad); 207 graphviz(out2); 208 209 return 0; 210 } 211 212 int test_log(void) { 213 srand(123); 214 tensor* a = Tensor(4, 10); 215 set_name(a, "a"), print(a); 216 217 tensor* out = log(a); 218 set_name(out, "out"), print(out); 219 220 tensor* w = Tensor(10, 5); 221 set_name(w, "w"), print(w); 222 223 tensor* out2 = matmul(out, w); 224 set_name(out2, "out2"), print(out2); 225 226 227 228 out2->backward(out2); 229 set_name(a->grad, "a_grad"), print(a->grad); 230 graphviz(out2); 231 232 return 0; 233 } 234 235 236 237 int test_repeat(void) { 238 srand(123); 239 tensor* a = Tensor(4, 1); 240 set_name(a, "a"), print(a); 241 242 tensor* out = repeat(a, 3); 243 set_name(out, "out"), print(out); 244 245 tensor* w = Tensor(3, 5); 246 set_name(w, "w"), print(w); 247 248 tensor* out2 = matmul(out, w); 249 set_name(out2, "out2"), print(out2); 250 251 252 253 out2->backward(out2); 254 set_name(a->grad, "a_grad"), print(a->grad); 255 graphviz(out2); 256 257 return 0; 258 } 259 260 261 262 int test_neg(void) { 263 srand(123); 264 tensor* a = Tensor(4, 1); 265 set_name(a, "a"), print(a); 266 267 tensor* out = neg(a); 268 set_name(out, "out"), print(out); 269 270 tensor* w = Tensor(1, 5); 271 set_name(w, "w"), print(w); 272 273 tensor* out2 = matmul(out, w); 274 set_name(out2, "out2"), print(out2); 275 276 277 278 out2->backward(out2); 279 set_name(a->grad, "a_grad"), print(a->grad); 280 graphviz(out2); 281 282 return 0; 283 } 284 285 286 287 288 int test_div(void) { 289 srand(123); 290 tensor* a = Tensor(4, 3); 291 set_name(a, "a"), print(a); 292 293 tensor* b = Tensor(4, 3); 294 set_name(b, "b"), print(b); 295 296 tensor* out = div(a, b); 297 set_name(out, "out"), print(out); 298 299 tensor* w = Tensor(3, 5); 300 set_name(w, "w"), print(w); 301 302 tensor* out2 = matmul(out, w); 303 set_name(out2, "out2"), print(out2); 304 305 306 out2->backward(out2); 307 set_name(a->grad, "a_grad"), print(a->grad); 308 set_name(b->grad, "b_grad"), print(b->grad); 309 graphviz(out2); 310 311 return 0; 312 } 313 314 // int main(void){ 315 // srand(123); 316 // tensor* a = Tensor(4, 3); 317 // set_name(a, "a"), print(a); 318 319 // tensor* b = Tensor(4, 3); 320 // set_name(b, "b"), print(b); 321 322 // tensor* c = pow_k(b, 2); 323 // set_name(c, "c"), print(c); 324 325 // tensor* d = div_k(a, c); 326 // set_name(d, "d"), print(d); 327 328 // tensor* out = neg_k(d); 329 // set_name(out, "out"), print(out); 330 // } 331 332 333 334 // testing maxpool and its bwd 335 int test_maxpool(void) { 336 // random num generator init, must be called once 337 // srand(time(NULL)); 338 srand(123); 339 340 int B = 2; 341 int C = 3; 342 int H = 4; 343 int W = 4; 344 345 int F = 5; 346 int HH = 2; 347 int WW = 2; 348 349 // // *** Init *** 350 // tensor* x = Tensor(C, H, W); 351 // set_name(x, "x"); print(x); 352 353 // tensor* out = maxpool_k(x); 354 // set_name(out, "out"); print(out); 355 356 // // printf("\n\n\n\n\n\n\n\n\n\n\n\n"); 357 358 // // temporary: 359 // out->inputs[0] = x; 360 361 // tensor* upstream = TensorLikeFill(out, 1.0); 362 // tensor* grad_x = bwd_maxpool_k(upstream, out); 363 // set_name(grad_x, "grad_x"); print(grad_x); 364 365 366 367 // // *** Init *** 368 // tensor* x = Tensor(B, C, H, W); 369 // set_name(x, "x"); print(x); 370 371 // tensor* out = batched_maxpool_k(x); 372 // set_name(out, "out"); print(out); 373 374 // // printf("\n\n\n\n\n\n\n\n\n\n\n\n"); 375 376 // // temporary: 377 // out->inputs[0] = x; 378 379 // tensor* upstream = TensorLikeFill(out, 1.0); 380 // tensor* grad_x = bwd_batched_maxpool_k(upstream, out); 381 // set_name(grad_x, "grad_x"); print(grad_x); 382 383 384 // *** Init *** 385 tensor* x = Tensor(B, C, H, W); 386 set_name(x, "x"); print(x); 387 388 tensor* out = batched_maxpool(x); 389 set_name(out, "out"); print(out); 390 391 out->backward(out); 392 393 return 0; 394 } 395 396 397 int test_flatten(void) { 398 // random num generator init, must be called once 399 // srand(time(NULL)); 400 srand(123); 401 402 int B = 2; 403 int C = 2; 404 int H = 8; 405 int W = 8; 406 407 int F = 2; 408 int HH = 2; 409 int WW = 2; 410 411 // *** Init *** 412 tensor* input = Tensor(B, C, H, W); 413 set_name(input, "input"); print(input); 414 415 tensor* kernel = Tensor(F, C, HH, WW); 416 set_name(kernel, "kernel"); print(kernel); 417 418 tensor* out_conv1 = batched_conv(input, kernel); 419 set_name(out_conv1, "out_conv1"); sprint(out_conv1); 420 421 tensor* out_flat = batched_flatten(out_conv1); 422 set_name(out_flat, "out_flat"); print(out_flat); 423 424 425 out_flat->backward(out_flat); 426 427 return 0; 428 } 429 430 431 432 int test_conv(void) { 433 // random num generator init, must be called once 434 // srand(time(NULL)); 435 srand(123); 436 437 int B = 2; 438 int C = 3; 439 int H = 4; 440 int W = 4; 441 442 int F = 5; 443 int HH = 2; 444 int WW = 2; 445 446 // *** Init *** 447 tensor* input = Tensor(B, C, H, W); 448 set_name(input, "input"); print(input); 449 450 tensor* kernel = Tensor(F, C, HH, WW); 451 set_name(kernel, "kernel"); print(kernel); 452 453 tensor* out_conv1 = batched_conv(input, kernel); 454 set_name(out_conv1, "out_conv1"); print(out_conv1); 455 out_conv1->backward(out_conv1); 456 457 return 0; 458 } 459 460 461 int test_conv(void) { 462 // random num generator init, must be called once 463 // srand(time(NULL)); 464 srand(123); 465 466 int C = 3; 467 int H = 4; 468 int W = 4; 469 470 int F = 5; 471 int HH = 2; 472 int WW = 2; 473 474 // *** Init *** 475 tensor* input = Tensor(C, H, W); 476 set_name(input, "input"); print(input); 477 478 tensor* kernel = Tensor(F, C, HH, WW); 479 set_name(kernel, "kernel"); print(kernel); 480 481 tensor* out = conv_k(input, kernel); 482 set_name(out, "out"); print(out); 483 484 printf("\n\n\n\n\n\n\n\n\n\n\n\n"); 485 486 // temporary: 487 out->inputs[0] = input; 488 out->inputs[1] = kernel; 489 490 tensor* upstream = TensorLikeFill(out, 1.0); 491 bwd_conv_k(upstream, out); 492 tensor* grad_kernels = kernel->grad; // set by bwd_conv_k 493 set_name(grad_kernels, "grad_kernels"); print(grad_kernels); 494 495 return 0; 496 } 497 498 499 500 int test_bmm(void) { 501 // random num generator init, must be called once 502 // srand(time(NULL)); 503 srand(123); 504 505 int B = 8; 506 int N = 3; 507 int M = 2; 508 int D = 4; 509 510 511 tensor* input = Tensor(B, N, M); 512 set_name(input, "input"); print(input); 513 514 tensor* weight = Tensor(B, M, D); 515 set_name(weight, "weight"); print(weight); 516 517 // (B, N, D) 518 tensor* out = batched_matmul(input, weight); 519 set_name(out, "out"); print(out); 520 521 graphviz(out); 522 523 return 0; 524 } 525 526 527 528 int test_bmm_k(void) { 529 // random num generator init, must be called once 530 // srand(time(NULL)); 531 srand(123); 532 533 int B = 8; 534 int N = 3; 535 int M = 2; 536 int D = 4; 537 538 539 tensor* input = Tensor(B, N, M); 540 set_name(input, "input"); print(input); 541 542 tensor* weight = Tensor(B, M, D); 543 set_name(weight, "weight"); print(weight); 544 545 // (B, N, D) 546 tensor* out = batched_matmul_k(input, weight); 547 set_name(out, "out"); print(out); 548 549 return 0; 550 } 551 552 int test_bt_k(void) { 553 // random num generator init, must be called once 554 // srand(time(NULL)); 555 srand(123); 556 557 int B = 8; 558 int N = 3; 559 int M = 2; 560 int D = 4; 561 562 563 tensor* input = Tensor(B, N, M); 564 set_name(input, "input"); print(input); 565 566 tensor* transposed = batched_transpose_k(input); 567 set_name(transposed, "transposed"); print(transposed); 568 569 return 0; 570 } 571 572 573 574 575 int test_indexing(void) { 576 srand(123); 577 578 tensor* x = Tensor(3, 7); 579 set_name(x, "orig. x"); print(x); 580 581 tensor* x_slice = slice(x, "1:3, 3:6"); 582 set_name(x_slice, "x_slice"); 583 print(x_slice); 584 585 tensor* x_view = view(x, "1:3, 3:6"); 586 set_name(x_view, "x_view"); 587 print(x_view); 588 589 cout << "\n19th element of x:" << endl; 590 cout << x->data[at(x, 19)] << endl; 591 592 tensor* y = Tensor(4, 3, 7); 593 set_name(y, "orig. y"); 594 print(y); 595 596 tensor* y_slice = slice(y, "2:4, 1:3, 3:6"); 597 set_name(y_slice, "y_slice"); 598 print(y_slice); 599 600 tensor* y_view = view(y, "2:4, 1:3, 3:6"); 601 set_name(y_view, "y_view"); 602 print(y_view); 603 604 cout << "\n54th element of y:" << endl; 605 cout << y->data[at(y, 54)] << endl; 606 607 return 0; 608 } 609 610 611 // test_at 612 // (3, 7).at(20) = 613 // y_idx = 20 / 7 = 2 614 // z_idx = 20 % 7 = 6 615