task-code-repair_test.go
1 package task 2 3 import ( 4 "fmt" 5 "os" 6 "path/filepath" 7 "strings" 8 "testing" 9 10 "github.com/stretchr/testify/assert" 11 "github.com/stretchr/testify/require" 12 "github.com/symflower/eval-dev-quality/evaluate/metrics" 13 metricstesting "github.com/symflower/eval-dev-quality/evaluate/metrics/testing" 14 tasktesting "github.com/symflower/eval-dev-quality/evaluate/task/testing" 15 "github.com/symflower/eval-dev-quality/language/golang" 16 "github.com/symflower/eval-dev-quality/language/java" 17 "github.com/symflower/eval-dev-quality/language/ruby" 18 "github.com/symflower/eval-dev-quality/log" 19 modeltesting "github.com/symflower/eval-dev-quality/model/testing" 20 evaltask "github.com/symflower/eval-dev-quality/task" 21 "github.com/zimmski/osutil" 22 "github.com/zimmski/osutil/bytesutil" 23 ) 24 25 func TestTaskCodeRepairRun(t *testing.T) { 26 validate := func(t *testing.T, tc *tasktesting.TestCaseTask) { 27 t.Run(tc.Name, func(t *testing.T) { 28 task, err := TaskForIdentifier(IdentifierCodeRepair) 29 require.NoError(t, err) 30 tc.Task = task 31 32 tc.Validate(t, 33 func(logger *log.Logger, testDataPath string, repositoryPathRelative string) (repository evaltask.Repository, cleanup func(), err error) { 34 return TemporaryRepository(logger, testDataPath, repositoryPathRelative) 35 }, 36 ) 37 }) 38 } 39 40 t.Run("Go", func(t *testing.T) { 41 { 42 temporaryDirectoryPath := t.TempDir() 43 44 repositoryPath := filepath.Join(temporaryDirectoryPath, "golang", "mistakes", "openingBracketMissing") 45 require.NoError(t, osutil.CopyTree(filepath.Join("..", "..", "testdata", "golang", "mistakes", "openingBracketMissing"), repositoryPath)) 46 47 modelMock := modeltesting.NewMockCapabilityRepairCodeNamed(t, "mocked-model") 48 49 // Generate valid code for the task. 50 sourceFileContent := bytesutil.StringTrimIndentations(` 51 package openingBracketMissing 52 53 func openingBracketMissing(x int) int { 54 if x > 0 { 55 return 1 56 } 57 if x < 0 { 58 return -1 59 } 60 61 return 0 62 } 63 `) 64 modelMock.RegisterGenerateSuccess(t, "openingBracketMissing.go", sourceFileContent, metricstesting.AssessmentsWithProcessingTime).Once() 65 66 validate(t, &tasktesting.TestCaseTask{ 67 Name: "Single test case", 68 69 Model: modelMock, 70 Language: &golang.Language{}, 71 TestDataPath: temporaryDirectoryPath, 72 RepositoryPath: filepath.Join("golang", "mistakes"), 73 74 ExpectedRepositoryAssessment: map[evaltask.Identifier]metrics.Assessments{ 75 IdentifierCodeRepair: metrics.Assessments{ 76 metrics.AssessmentKeyFilesExecuted: 1, 77 metrics.AssessmentKeyFilesExecutedMaximumReachable: 1, 78 metrics.AssessmentKeyResponseNoError: 1, 79 metrics.AssessmentKeyTestsPassing: 40, 80 }, 81 }, 82 ValidateLog: func(t *testing.T, data string) { 83 assert.Contains(t, data, "TestSymflowerOpeningBracketMissing/#00") 84 assert.Contains(t, data, "TestSymflowerOpeningBracketMissing/#01") 85 assert.Contains(t, data, "TestSymflowerOpeningBracketMissing/#02") 86 }, 87 }) 88 } 89 { 90 temporaryDirectoryPath := t.TempDir() 91 92 repositoryPath := filepath.Join(temporaryDirectoryPath, "golang", "mistakes") 93 require.NoError(t, osutil.CopyTree(filepath.Join("..", "..", "testdata", "golang", "mistakes", "openingBracketMissing"), filepath.Join(repositoryPath, "openingBracketMissing"))) 94 require.NoError(t, osutil.CopyTree(filepath.Join("..", "..", "testdata", "golang", "mistakes", "typeUnknown"), filepath.Join(repositoryPath, "typeUnknown"))) 95 96 modelMock := modeltesting.NewMockCapabilityRepairCodeNamed(t, "mocked-model") 97 98 // Generate valid code for the task. 99 sourceFileContent := bytesutil.StringTrimIndentations(` 100 package openingBracketMissing 101 102 func openingBracketMissing(x int) int { 103 if x > 0 { 104 return 1 105 } 106 if x < 0 { 107 return -1 108 } 109 110 return 0 111 } 112 `) 113 modelMock.RegisterGenerateSuccess(t, "openingBracketMissing.go", sourceFileContent, metricstesting.AssessmentsWithProcessingTime).Once() 114 sourceFileContent = bytesutil.StringTrimIndentations(` 115 package typeUnknown 116 117 func typeUnknown(x int) int { 118 if x > 0 { 119 return 1 120 } 121 if x < 0 { 122 return -1 123 } 124 125 return 0 126 } 127 `) 128 modelMock.RegisterGenerateSuccess(t, "typeUnknown.go", sourceFileContent, metricstesting.AssessmentsWithProcessingTime).Once() 129 130 validate(t, &tasktesting.TestCaseTask{ 131 Name: "Multiple test cases", 132 133 Model: modelMock, 134 Language: &golang.Language{}, 135 TestDataPath: temporaryDirectoryPath, 136 RepositoryPath: filepath.Join("golang", "mistakes"), 137 138 ExpectedRepositoryAssessment: map[evaltask.Identifier]metrics.Assessments{ 139 IdentifierCodeRepair: metrics.Assessments{ 140 metrics.AssessmentKeyFilesExecuted: 2, 141 metrics.AssessmentKeyFilesExecutedMaximumReachable: 2, 142 metrics.AssessmentKeyResponseNoError: 2, 143 metrics.AssessmentKeyTestsPassing: 80, 144 }, 145 }, 146 ValidateLog: func(t *testing.T, data string) { 147 assert.Contains(t, data, "TestSymflowerOpeningBracketMissing/#00") 148 assert.Contains(t, data, "TestSymflowerOpeningBracketMissing/#01") 149 assert.Contains(t, data, "TestSymflowerOpeningBracketMissing/#02") 150 assert.Contains(t, data, "TestSymflowerTypeUnknown/#00") 151 assert.Contains(t, data, "TestSymflowerTypeUnknown/#01") 152 assert.Contains(t, data, "TestSymflowerTypeUnknown/#02") 153 }, 154 }) 155 } 156 }) 157 t.Run("Java", func(t *testing.T) { 158 { 159 temporaryDirectoryPath := t.TempDir() 160 161 repositoryPath := filepath.Join(temporaryDirectoryPath, "java", "mistakes", "openingBracketMissing") 162 require.NoError(t, osutil.CopyTree(filepath.Join("..", "..", "testdata", "java", "mistakes", "openingBracketMissing"), repositoryPath)) 163 164 modelMock := modeltesting.NewMockCapabilityRepairCodeNamed(t, "mocked-model") 165 166 // Generate valid code for the task. 167 sourceFileContent := bytesutil.StringTrimIndentations(` 168 package com.eval; 169 170 public class OpeningBracketMissing { 171 public static int openingBracketMissing(int x) { 172 if (x > 0) { 173 return 1; 174 } 175 if (x < 0) { 176 return -1; 177 } 178 179 return 0; 180 } 181 } 182 `) 183 modelMock.RegisterGenerateSuccess(t, filepath.Join("src", "main", "java", "com", "eval", "OpeningBracketMissing.java"), sourceFileContent, metricstesting.AssessmentsWithProcessingTime).Once() 184 185 validate(t, &tasktesting.TestCaseTask{ 186 Name: "Single test case", 187 188 Model: modelMock, 189 Language: &java.Language{}, 190 TestDataPath: temporaryDirectoryPath, 191 RepositoryPath: filepath.Join("java", "mistakes"), 192 193 ExpectedRepositoryAssessment: map[evaltask.Identifier]metrics.Assessments{ 194 IdentifierCodeRepair: metrics.Assessments{ 195 metrics.AssessmentKeyFilesExecuted: 1, 196 metrics.AssessmentKeyFilesExecutedMaximumReachable: 1, 197 metrics.AssessmentKeyResponseNoError: 1, 198 metrics.AssessmentKeyTestsPassing: 30, 199 }, 200 }, 201 ValidateLog: func(t *testing.T, data string) { 202 assert.Contains(t, data, "BUILD SUCCESS") 203 }, 204 }) 205 } 206 { 207 temporaryDirectoryPath := t.TempDir() 208 209 repositoryPath := filepath.Join(temporaryDirectoryPath, "java", "mistakes") 210 require.NoError(t, osutil.CopyTree(filepath.Join("..", "..", "testdata", "java", "mistakes", "openingBracketMissing"), filepath.Join(repositoryPath, "openingBracketMissing"))) 211 require.NoError(t, osutil.CopyTree(filepath.Join("..", "..", "testdata", "java", "mistakes", "typeUnknown"), filepath.Join(repositoryPath, "typeUnknown"))) 212 213 modelMock := modeltesting.NewMockCapabilityRepairCodeNamed(t, "mocked-model") 214 215 // Generate valid code for the task. 216 sourceFileContent := bytesutil.StringTrimIndentations(` 217 package com.eval; 218 219 public class OpeningBracketMissing { 220 public static int openingBracketMissing(int x) { 221 if (x > 0) { 222 return 1; 223 } 224 if (x < 0) { 225 return -1; 226 } 227 228 return 0; 229 } 230 } 231 `) 232 modelMock.RegisterGenerateSuccess(t, filepath.Join("src", "main", "java", "com", "eval", "OpeningBracketMissing.java"), sourceFileContent, metricstesting.AssessmentsWithProcessingTime).Once() 233 sourceFileContent = bytesutil.StringTrimIndentations(` 234 package com.eval; 235 236 public class TypeUnknown { 237 public static int typeUnknown(int x) { 238 if (x > 0) { 239 return 1; 240 } 241 if (x < 0) { 242 return -1; 243 } 244 245 return 0; 246 } 247 } 248 `) 249 modelMock.RegisterGenerateSuccess(t, filepath.Join("src", "main", "java", "com", "eval", "TypeUnknown.java"), sourceFileContent, metricstesting.AssessmentsWithProcessingTime).Once() 250 251 validate(t, &tasktesting.TestCaseTask{ 252 Name: "Multiple test cases", 253 254 Model: modelMock, 255 Language: &java.Language{}, 256 TestDataPath: temporaryDirectoryPath, 257 RepositoryPath: filepath.Join("java", "mistakes"), 258 259 ExpectedRepositoryAssessment: map[evaltask.Identifier]metrics.Assessments{ 260 IdentifierCodeRepair: metrics.Assessments{ 261 metrics.AssessmentKeyFilesExecutedMaximumReachable: 2, 262 metrics.AssessmentKeyFilesExecuted: 2, 263 metrics.AssessmentKeyResponseNoError: 2, 264 metrics.AssessmentKeyTestsPassing: 60, 265 }, 266 }, 267 ValidateLog: func(t *testing.T, data string) { 268 assert.Contains(t, data, "BUILD SUCCESS") 269 }, 270 }) 271 } 272 }) 273 t.Run("Ruby", func(t *testing.T) { 274 if osutil.IsWindows() { 275 t.Skip("Ruby is not tested in the Windows CI") 276 } 277 278 { 279 temporaryDirectoryPath := t.TempDir() 280 281 repositoryPath := filepath.Join(temporaryDirectoryPath, "ruby", "mistakes", "argumentMissing") 282 require.NoError(t, osutil.CopyTree(filepath.Join("..", "..", "testdata", "ruby", "mistakes", "argumentMissing"), repositoryPath)) 283 284 modelMock := modeltesting.NewMockCapabilityRepairCodeNamed(t, "mocked-model") 285 286 // Generate valid code for the task. 287 sourceFileContent := bytesutil.StringTrimIndentations(` 288 def argument_missing(x) 289 if x > 0 290 return 1 291 elsif x < 0 292 return -1 293 else 294 return 0 295 end 296 end 297 `) 298 modelMock.RegisterGenerateSuccess(t, filepath.Join("lib", "argument_missing.rb"), sourceFileContent, metricstesting.AssessmentsWithProcessingTime).Once() 299 300 validate(t, &tasktesting.TestCaseTask{ 301 Name: "Single test case", 302 303 Model: modelMock, 304 Language: &ruby.Language{}, 305 TestDataPath: temporaryDirectoryPath, 306 RepositoryPath: filepath.Join("ruby", "mistakes"), 307 308 ExpectedRepositoryAssessment: map[evaltask.Identifier]metrics.Assessments{ 309 IdentifierCodeRepair: metrics.Assessments{ 310 metrics.AssessmentKeyFilesExecuted: 1, 311 metrics.AssessmentKeyFilesExecutedMaximumReachable: 1, 312 metrics.AssessmentKeyResponseNoError: 1, 313 metrics.AssessmentKeyTestsPassing: 30, 314 }, 315 }, 316 ValidateLog: func(t *testing.T, data string) { 317 assert.Contains(t, data, "3 runs, 0 assertions, 0 failures, 3 errors, 0 skips") // Extracting the mistakes. 318 assert.Contains(t, data, "3 runs, 3 assertions, 0 failures, 0 errors, 0 skips") // Fixed by model. 319 }, 320 }) 321 } 322 { 323 temporaryDirectoryPath := t.TempDir() 324 325 repositoryPath := filepath.Join(temporaryDirectoryPath, "ruby", "mistakes") 326 require.NoError(t, osutil.CopyTree(filepath.Join("..", "..", "testdata", "ruby", "mistakes", "argumentMissing"), filepath.Join(repositoryPath, "argumentMissing"))) 327 require.NoError(t, osutil.CopyTree(filepath.Join("..", "..", "testdata", "ruby", "mistakes", "typeUnknown"), filepath.Join(repositoryPath, "typeUnknown"))) 328 329 modelMock := modeltesting.NewMockCapabilityRepairCodeNamed(t, "mocked-model") 330 331 // Generate valid code for the task. 332 sourceFileContent := bytesutil.StringTrimIndentations(` 333 def argument_missing(x) 334 if x > 0 335 return 1 336 elsif x < 0 337 return -1 338 else 339 return 0 340 end 341 end 342 `) 343 modelMock.RegisterGenerateSuccess(t, filepath.Join("lib", "argument_missing.rb"), sourceFileContent, metricstesting.AssessmentsWithProcessingTime).Once() 344 sourceFileContent = bytesutil.StringTrimIndentations(` 345 def type_unknown(x) 346 if x.is_a?(Integer) 347 if x > 0 348 return 1 349 elsif x < 0 350 return -1 351 end 352 end 353 354 return 0 355 end 356 `) 357 modelMock.RegisterGenerateSuccess(t, filepath.Join("lib", "type_unknown.rb"), sourceFileContent, metricstesting.AssessmentsWithProcessingTime).Once() 358 359 validate(t, &tasktesting.TestCaseTask{ 360 Name: "Multiple test cases", 361 362 Model: modelMock, 363 Language: &ruby.Language{}, 364 TestDataPath: temporaryDirectoryPath, 365 RepositoryPath: filepath.Join("ruby", "mistakes"), 366 367 ExpectedRepositoryAssessment: map[evaltask.Identifier]metrics.Assessments{ 368 IdentifierCodeRepair: metrics.Assessments{ 369 metrics.AssessmentKeyFilesExecuted: 2, 370 metrics.AssessmentKeyFilesExecutedMaximumReachable: 2, 371 metrics.AssessmentKeyResponseNoError: 2, 372 metrics.AssessmentKeyTestsPassing: 60, 373 }, 374 }, 375 ValidateLog: func(t *testing.T, data string) { 376 assert.Equal(t, 2, strings.Count(data, "3 runs, 0 assertions, 0 failures, 3 errors, 0 skips"), "extraction") // Extracting the mistakes. 377 assert.Equal(t, 2, strings.Count(data, "3 runs, 3 assertions, 0 failures, 0 errors, 0 skips"), "fixed") // Fixed by model. 378 }, 379 }) 380 } 381 }) 382 } 383 384 func TestValidateCodeRepairRepository(t *testing.T) { 385 validate := func(t *testing.T, tc *tasktesting.TestCaseValidateRepository) { 386 tc.Validate(t, validateCodeRepairRepository) 387 } 388 389 validate(t, &tasktesting.TestCaseValidateRepository{ 390 Name: "Repository root path contains source files", 391 392 Before: func(repositoryPath string) { 393 someFile, err := os.Create(filepath.Join(repositoryPath, "someFile.go")) 394 require.NoError(t, err) 395 someFile.Close() 396 }, 397 398 TestdataPath: filepath.Join("..", "..", "testdata"), 399 RepositoryPath: filepath.Join("golang", "mistakes"), 400 Language: &golang.Language{}, 401 402 ExpectedError: func(t *testing.T, err error) { 403 assert.ErrorContains(t, err, "must contain only packages, but found [someFile.go]") 404 }, 405 }) 406 t.Run("Go", func(t *testing.T) { 407 validate(t, &tasktesting.TestCaseValidateRepository{ 408 Name: "Package does not contain source file", 409 410 Before: func(repositoryPath string) { 411 require.NoError(t, os.MkdirAll(filepath.Join(repositoryPath, "somePackage"), 0700)) 412 }, 413 414 TestdataPath: filepath.Join("..", "..", "testdata"), 415 RepositoryPath: filepath.Join("golang", "mistakes"), 416 Language: &golang.Language{}, 417 418 ExpectedError: func(t *testing.T, err error) { 419 assert.ErrorContains(t, err, "must contain exactly one Go source file, but found []") 420 }, 421 }) 422 validate(t, &tasktesting.TestCaseValidateRepository{ 423 Name: "Package contains multiple source files", 424 425 Before: func(repositoryPath string) { 426 somePackage := filepath.Join(repositoryPath, "somePackage") 427 require.NoError(t, os.MkdirAll(somePackage, 0700)) 428 429 fileA, err := os.Create(filepath.Join(somePackage, "fileA.go")) 430 require.NoError(t, err) 431 fileA.Close() 432 433 fileB, err := os.Create(filepath.Join(somePackage, "fileB.go")) 434 require.NoError(t, err) 435 fileB.Close() 436 }, 437 438 TestdataPath: filepath.Join("..", "..", "testdata"), 439 RepositoryPath: filepath.Join("golang", "mistakes"), 440 Language: &golang.Language{}, 441 442 ExpectedError: func(t *testing.T, err error) { 443 assert.ErrorContains(t, err, "must contain exactly one Go source file, but found [fileA.go fileB.go]") 444 }, 445 }) 446 validate(t, &tasktesting.TestCaseValidateRepository{ 447 Name: "Package does not contain test file", 448 449 Before: func(repositoryPath string) { 450 somePackage := filepath.Join(repositoryPath, "somePackage") 451 require.NoError(t, os.MkdirAll(somePackage, 0700)) 452 453 file, err := os.Create(filepath.Join(somePackage, "someFile.go")) 454 require.NoError(t, err) 455 defer file.Close() 456 }, 457 458 TestdataPath: filepath.Join("..", "..", "testdata"), 459 RepositoryPath: filepath.Join("golang", "mistakes"), 460 Language: &golang.Language{}, 461 462 ExpectedError: func(t *testing.T, err error) { 463 assert.ErrorContains(t, err, "must contain exactly one Go test file, but found []") 464 }, 465 }) 466 validate(t, &tasktesting.TestCaseValidateRepository{ 467 Name: "Package contains multiple test files", 468 469 Before: func(repositoryPath string) { 470 somePackage := filepath.Join(repositoryPath, "somePackage") 471 require.NoError(t, os.MkdirAll(somePackage, 0700)) 472 473 fileA, err := os.Create(filepath.Join(somePackage, "fileA.go")) 474 require.NoError(t, err) 475 fileA.Close() 476 477 fileATest, err := os.Create(filepath.Join(somePackage, "fileA_test.go")) 478 require.NoError(t, err) 479 fileATest.Close() 480 481 fileBTest, err := os.Create(filepath.Join(somePackage, "fileB_test.go")) 482 require.NoError(t, err) 483 fileBTest.Close() 484 }, 485 486 TestdataPath: filepath.Join("..", "..", "testdata"), 487 RepositoryPath: filepath.Join("golang", "mistakes"), 488 Language: &golang.Language{}, 489 490 ExpectedError: func(t *testing.T, err error) { 491 assert.ErrorContains(t, err, "must contain exactly one Go test file, but found [fileA_test.go fileB_test.go]") 492 }, 493 }) 494 validate(t, &tasktesting.TestCaseValidateRepository{ 495 Name: "Well-formed", 496 497 Before: func(repositoryPath string) { 498 require.NoError(t, osutil.MkdirAll(filepath.Join(repositoryPath, ".git"))) 499 require.NoError(t, os.WriteFile(filepath.Join(repositoryPath, ".git", "index"), []byte(`content`), 0700)) 500 }, 501 502 TestdataPath: filepath.Join("..", "..", "testdata"), 503 RepositoryPath: filepath.Join("golang", "mistakes"), 504 Language: &golang.Language{}, 505 }) 506 }) 507 t.Run("Java", func(t *testing.T) { 508 validate(t, &tasktesting.TestCaseValidateRepository{ 509 Name: "Package does not contain source file", 510 511 Before: func(repositoryPath string) { 512 require.NoError(t, os.MkdirAll(filepath.Join(repositoryPath, "somePackage"), 0700)) 513 }, 514 515 TestdataPath: filepath.Join("..", "..", "testdata"), 516 RepositoryPath: filepath.Join("java", "mistakes"), 517 Language: &java.Language{}, 518 519 ExpectedError: func(t *testing.T, err error) { 520 assert.ErrorContains(t, err, "must contain exactly one Java source file, but found []") 521 }, 522 }) 523 validate(t, &tasktesting.TestCaseValidateRepository{ 524 Name: "Package contains multiple source files", 525 526 Before: func(repositoryPath string) { 527 somePackage := filepath.Join(repositoryPath, "somePackage", "src", "main", "java", "com", "eval") 528 require.NoError(t, os.MkdirAll(somePackage, 0700)) 529 530 fileA, err := os.Create(filepath.Join(somePackage, "FileA.java")) 531 require.NoError(t, err) 532 fileA.Close() 533 534 fileB, err := os.Create(filepath.Join(somePackage, "FileB.java")) 535 require.NoError(t, err) 536 fileB.Close() 537 }, 538 539 TestdataPath: filepath.Join("..", "..", "testdata"), 540 RepositoryPath: filepath.Join("java", "mistakes"), 541 Language: &java.Language{}, 542 543 ExpectedError: func(t *testing.T, err error) { 544 assert.ErrorContains(t, err, fmt.Sprintf("must contain exactly one Java source file, but found [%s %s]", filepath.Join("src", "main", "java", "com", "eval", "FileA.java"), filepath.Join("src", "main", "java", "com", "eval", "FileB.java"))) 545 }, 546 }) 547 548 validate(t, &tasktesting.TestCaseValidateRepository{ 549 Name: "Package does not contain test file", 550 551 Before: func(repositoryPath string) { 552 somePackage := filepath.Join(repositoryPath, "somePackage", "src", "main", "java", "com", "eval") 553 require.NoError(t, os.MkdirAll(somePackage, 0700)) 554 555 fileA, err := os.Create(filepath.Join(somePackage, "FileA.java")) 556 require.NoError(t, err) 557 fileA.Close() 558 }, 559 560 TestdataPath: filepath.Join("..", "..", "testdata"), 561 RepositoryPath: filepath.Join("java", "mistakes"), 562 Language: &java.Language{}, 563 564 ExpectedError: func(t *testing.T, err error) { 565 assert.ErrorContains(t, err, "must contain exactly one Java test file, but found []") 566 }, 567 }) 568 validate(t, &tasktesting.TestCaseValidateRepository{ 569 Name: "Package contains multiple test files", 570 571 Before: func(repositoryPath string) { 572 sourcePackage := filepath.Join(repositoryPath, "somePackage", "src", "main", "java", "com", "eval") 573 require.NoError(t, os.MkdirAll(sourcePackage, 0700)) 574 testPackage := filepath.Join(repositoryPath, "somePackage", "src", "test", "java", "com", "eval") 575 require.NoError(t, os.MkdirAll(testPackage, 0700)) 576 577 fileA, err := os.Create(filepath.Join(sourcePackage, "FileA.java")) 578 require.NoError(t, err) 579 fileA.Close() 580 581 fileATest, err := os.Create(filepath.Join(testPackage, "FileATest.java")) 582 require.NoError(t, err) 583 fileATest.Close() 584 585 fileBTest, err := os.Create(filepath.Join(testPackage, "FileBTest.java")) 586 require.NoError(t, err) 587 fileBTest.Close() 588 }, 589 590 TestdataPath: filepath.Join("..", "..", "testdata"), 591 RepositoryPath: filepath.Join("java", "mistakes"), 592 Language: &java.Language{}, 593 594 ExpectedError: func(t *testing.T, err error) { 595 assert.ErrorContains(t, err, fmt.Sprintf("must contain exactly one Java test file, but found [%s %s]", filepath.Join("src", "test", "java", "com", "eval", "FileATest.java"), filepath.Join("src", "test", "java", "com", "eval", "FileBTest.java"))) 596 }, 597 }) 598 validate(t, &tasktesting.TestCaseValidateRepository{ 599 Name: "Well-formed", 600 601 Before: func(repositoryPath string) { 602 require.NoError(t, osutil.MkdirAll(filepath.Join(repositoryPath, "target"))) 603 require.NoError(t, os.WriteFile(filepath.Join(repositoryPath, "target", "someClass.class"), []byte(`content`), 0700)) 604 }, 605 606 TestdataPath: filepath.Join("..", "..", "testdata"), 607 RepositoryPath: filepath.Join("java", "mistakes"), 608 Language: &java.Language{}, 609 }) 610 }) 611 }