marketplace_test.go
1 package skills 2 3 import ( 4 "archive/zip" 5 "bytes" 6 "context" 7 "encoding/json" 8 "errors" 9 "io" 10 "net/http" 11 "net/http/httptest" 12 "os" 13 "path/filepath" 14 "strings" 15 "sync" 16 "sync/atomic" 17 "testing" 18 "time" 19 ) 20 21 func TestRegistryIndexParse(t *testing.T) { 22 raw := `{ 23 "version": 1, 24 "updated_at": "2026-04-06T12:00:00Z", 25 "skills": [ 26 { 27 "slug": "self-improving-agent", 28 "name": "self-improving-agent", 29 "description": "Captures learnings", 30 "author": "pskoett", 31 "license": "MIT-0", 32 "repo": "https://github.com/peterskoett/self-improving-agent", 33 "repo_path": "", 34 "ref": "main", 35 "homepage": "https://clawhub.ai/pskoett/self-improving-agent", 36 "downloads": 354000, 37 "stars": 3000, 38 "version": "3.0.13", 39 "security": { 40 "virustotal": "benign", 41 "openclaw": "benign", 42 "scanned_at": "2026-04-01T00:00:00Z" 43 }, 44 "tags": ["productivity", "meta"] 45 } 46 ] 47 }` 48 49 var idx RegistryIndex 50 if err := json.Unmarshal([]byte(raw), &idx); err != nil { 51 t.Fatalf("Unmarshal: %v", err) 52 } 53 if idx.Version != 1 { 54 t.Errorf("Version = %d, want 1", idx.Version) 55 } 56 if len(idx.Skills) != 1 { 57 t.Fatalf("len(Skills) = %d, want 1", len(idx.Skills)) 58 } 59 s := idx.Skills[0] 60 if s.Slug != "self-improving-agent" { 61 t.Errorf("Slug = %q", s.Slug) 62 } 63 if s.Downloads != 354000 { 64 t.Errorf("Downloads = %d, want 354000", s.Downloads) 65 } 66 if s.Security.VirusTotal != "benign" { 67 t.Errorf("Security.VirusTotal = %q, want benign", s.Security.VirusTotal) 68 } 69 if s.Ref != "main" { 70 t.Errorf("Ref = %q, want main", s.Ref) 71 } 72 } 73 74 func TestMarketplaceClientFetchAndCache(t *testing.T) { 75 var hits int32 76 ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 77 atomic.AddInt32(&hits, 1) 78 w.Header().Set("Content-Type", "application/json") 79 w.Write([]byte(`{"version":1,"updated_at":"2026-04-06T00:00:00Z","skills":[{"slug":"demo","name":"demo","description":"d","author":"a","repo":"https://x/y"}]}`)) 80 })) 81 defer ts.Close() 82 83 client := NewMarketplaceClient(ts.URL, 1*time.Hour) 84 idx, err := client.Load(context.Background()) 85 if err != nil { 86 t.Fatalf("first Load: %v", err) 87 } 88 if len(idx.Skills) != 1 || idx.Skills[0].Slug != "demo" { 89 t.Fatalf("unexpected index: %+v", idx) 90 } 91 if got := atomic.LoadInt32(&hits); got != 1 { 92 t.Errorf("expected 1 hit, got %d", got) 93 } 94 95 // Second call within TTL should not hit the server. 96 if _, err := client.Load(context.Background()); err != nil { 97 t.Fatalf("second Load: %v", err) 98 } 99 if got := atomic.LoadInt32(&hits); got != 1 { 100 t.Errorf("expected still 1 hit after cached load, got %d", got) 101 } 102 } 103 104 func TestMarketplaceClientStaleOnError(t *testing.T) { 105 var fail int32 106 ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 107 if atomic.LoadInt32(&fail) == 1 { 108 w.WriteHeader(http.StatusInternalServerError) 109 return 110 } 111 w.Write([]byte(`{"version":1,"skills":[{"slug":"demo","name":"demo","description":"d","author":"a","repo":"r"}]}`)) 112 })) 113 defer ts.Close() 114 115 // Zero TTL so every call tries to refetch. 116 client := NewMarketplaceClient(ts.URL, 0) 117 if _, err := client.Load(context.Background()); err != nil { 118 t.Fatalf("priming Load: %v", err) 119 } 120 121 atomic.StoreInt32(&fail, 1) 122 idx, err := client.Load(context.Background()) 123 if err != nil { 124 t.Fatalf("stale Load should succeed, got: %v", err) 125 } 126 if len(idx.Skills) != 1 { 127 t.Errorf("expected 1 skill from stale cache, got %d", len(idx.Skills)) 128 } 129 if !client.IsStale() { 130 t.Errorf("expected IsStale() true after serving stale") 131 } 132 } 133 134 // TestMarketplaceClientStaleCooldown verifies that once we fall into 135 // stale mode during a registry outage, subsequent Loads within the 136 // cooldown window do NOT re-hit the upstream. Otherwise heavy UI 137 // traffic during an outage would turn into a retry storm. 138 func TestMarketplaceClientStaleCooldown(t *testing.T) { 139 var hits int32 140 var fail int32 141 ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 142 atomic.AddInt32(&hits, 1) 143 if atomic.LoadInt32(&fail) == 1 { 144 w.WriteHeader(http.StatusInternalServerError) 145 return 146 } 147 w.Write([]byte(`{"version":1,"skills":[{"slug":"demo","name":"demo","description":"d","author":"a","repo":"r"}]}`)) 148 })) 149 defer ts.Close() 150 151 // Zero TTL so every Load would refetch without the cooldown guard. 152 // Generous cooldown so it definitely covers the test window. 153 client := NewMarketplaceClient(ts.URL, 0) 154 client.staleCooldown = 10 * time.Second 155 156 // Prime the cache. 157 if _, err := client.Load(context.Background()); err != nil { 158 t.Fatalf("priming Load: %v", err) 159 } 160 if got := atomic.LoadInt32(&hits); got != 1 { 161 t.Fatalf("priming hits = %d, want 1", got) 162 } 163 164 // Enter outage; first Load after the outage triggers one fetch 165 // attempt, then serves stale. 166 atomic.StoreInt32(&fail, 1) 167 if _, err := client.Load(context.Background()); err != nil { 168 t.Fatalf("first stale Load: %v", err) 169 } 170 if got := atomic.LoadInt32(&hits); got != 2 { 171 t.Errorf("after first stale Load hits = %d, want 2", got) 172 } 173 if !client.IsStale() { 174 t.Errorf("expected IsStale() true after first stale Load") 175 } 176 177 // Three more Loads during cooldown: must NOT retry. 178 for i := 0; i < 3; i++ { 179 if _, err := client.Load(context.Background()); err != nil { 180 t.Fatalf("cooldown Load %d: %v", i, err) 181 } 182 } 183 if got := atomic.LoadInt32(&hits); got != 2 { 184 t.Errorf("hits after cooldown Loads = %d, want 2 (no retries during cooldown)", got) 185 } 186 } 187 188 func TestMarketplaceClientNoCacheNoServer(t *testing.T) { 189 // Unreachable URL, no prior cache → must return error. 190 client := NewMarketplaceClient("http://127.0.0.1:1/no-such", 1*time.Hour) 191 _, err := client.Load(context.Background()) 192 if err == nil { 193 t.Fatal("expected error with no cache and unreachable URL") 194 } 195 } 196 197 func TestFilterSortPaginate(t *testing.T) { 198 entries := []MarketplaceEntry{ 199 {Slug: "alpha", Name: "alpha", Description: "The first thing", Author: "alice", Downloads: 10, Stars: 5}, 200 {Slug: "bravo", Name: "bravo", Description: "Second thing", Author: "bob", Downloads: 100, Stars: 20}, 201 {Slug: "charlie", Name: "charlie", Description: "Third thing", Author: "alice", Downloads: 50, Stars: 15}, 202 {Slug: "delta", Name: "delta", Description: "Malicious", Author: "mallory", Downloads: 999, 203 Security: SecurityScan{VirusTotal: "malicious"}}, 204 } 205 206 // Default sort = downloads desc, malicious excluded. 207 out, total := FilterSortPaginate(entries, "", "downloads", 1, 10) 208 if total != 3 { 209 t.Errorf("total = %d, want 3 (malicious excluded)", total) 210 } 211 if len(out) != 3 { 212 t.Fatalf("len(out) = %d, want 3", len(out)) 213 } 214 if out[0].Slug != "bravo" || out[1].Slug != "charlie" || out[2].Slug != "alpha" { 215 t.Errorf("downloads sort order wrong: %v %v %v", out[0].Slug, out[1].Slug, out[2].Slug) 216 } 217 218 // Sort by name asc. 219 out, _ = FilterSortPaginate(entries, "", "name", 1, 10) 220 if out[0].Slug != "alpha" || out[2].Slug != "charlie" { 221 t.Errorf("name sort order wrong: %v", sluggs(out)) 222 } 223 224 // Search: matches name, description, author (case-insensitive). 225 out, total = FilterSortPaginate(entries, "ALICE", "downloads", 1, 10) 226 if total != 2 { 227 t.Errorf("alice search total = %d, want 2", total) 228 } 229 out, total = FilterSortPaginate(entries, "third", "downloads", 1, 10) 230 if total != 1 || out[0].Slug != "charlie" { 231 t.Errorf("third search wrong: total=%d, %v", total, sluggs(out)) 232 } 233 234 // Pagination: page 2 of size 2, downloads desc. 235 out, total = FilterSortPaginate(entries, "", "downloads", 2, 2) 236 if total != 3 { 237 t.Errorf("page2 total = %d, want 3", total) 238 } 239 if len(out) != 1 || out[0].Slug != "alpha" { 240 t.Errorf("page2 contents: %v", sluggs(out)) 241 } 242 243 // Out-of-range page → empty slice, total still correct. 244 out, total = FilterSortPaginate(entries, "", "downloads", 99, 10) 245 if total != 3 { 246 t.Errorf("OOR total = %d, want 3", total) 247 } 248 if len(out) != 0 { 249 t.Errorf("OOR expected empty slice, got %v", sluggs(out)) 250 } 251 } 252 253 func sluggs(es []MarketplaceEntry) []string { 254 out := make([]string, len(es)) 255 for i, e := range es { 256 out[i] = e.Slug 257 } 258 return out 259 } 260 261 func TestSlugLockSerializesSameSlug(t *testing.T) { 262 locks := NewSlugLocks() 263 var order []string 264 var mu sync.Mutex 265 266 done := make(chan struct{}, 2) 267 start := make(chan struct{}) 268 269 go func() { 270 <-start 271 unlock := locks.Lock("alpha") 272 time.Sleep(20 * time.Millisecond) 273 mu.Lock() 274 order = append(order, "A") 275 mu.Unlock() 276 unlock() 277 done <- struct{}{} 278 }() 279 go func() { 280 <-start 281 time.Sleep(5 * time.Millisecond) // start after A has the lock 282 unlock := locks.Lock("alpha") 283 mu.Lock() 284 order = append(order, "B") 285 mu.Unlock() 286 unlock() 287 done <- struct{}{} 288 }() 289 290 close(start) 291 <-done 292 <-done 293 if len(order) != 2 || order[0] != "A" || order[1] != "B" { 294 t.Errorf("expected serialized order [A, B], got %v", order) 295 } 296 } 297 298 func TestSlugLockDifferentSlugsConcurrent(t *testing.T) { 299 locks := NewSlugLocks() 300 301 unlockA := locks.Lock("alpha") 302 defer unlockA() 303 304 // Locking a different slug must not block. 305 ch := make(chan struct{}) 306 go func() { 307 unlock := locks.Lock("bravo") 308 unlock() 309 close(ch) 310 }() 311 select { 312 case <-ch: 313 // good 314 case <-time.After(200 * time.Millisecond): 315 t.Fatal("locking a different slug should not block") 316 } 317 } 318 319 func TestStageCleanPayloadExcludesGit(t *testing.T) { 320 src := t.TempDir() 321 mustWrite(t, filepath.Join(src, "SKILL.md"), "---\nname: demo\ndescription: d\n---\nbody") 322 mustWriteExec(t, filepath.Join(src, "scripts/run.sh"), "#!/bin/sh\necho hi") 323 mustWrite(t, filepath.Join(src, ".git/config"), "[core]") 324 mustWrite(t, filepath.Join(src, ".github/workflows/ci.yml"), "name: ci") 325 mustWrite(t, filepath.Join(src, ".gitignore"), "node_modules") 326 327 dst := filepath.Join(t.TempDir(), "stage") 328 if err := stageCleanPayload(src, dst); err != nil { 329 t.Fatalf("stageCleanPayload: %v", err) 330 } 331 332 if _, err := os.Stat(filepath.Join(dst, "SKILL.md")); err != nil { 333 t.Errorf("SKILL.md missing: %v", err) 334 } 335 // Scripts directory must survive AND keep its executable bit. Community 336 // skills like self-improving-agent ship scripts/activator.sh that break 337 // silently if we strip mode during the copy. 338 scriptInfo, err := os.Stat(filepath.Join(dst, "scripts/run.sh")) 339 if err != nil { 340 t.Errorf("scripts/run.sh missing: %v", err) 341 } else if scriptInfo.Mode().Perm()&0100 == 0 { 342 t.Errorf("scripts/run.sh lost its executable bit: mode = %v", scriptInfo.Mode().Perm()) 343 } 344 for _, excluded := range []string{".git", ".github", ".gitignore"} { 345 if _, err := os.Stat(filepath.Join(dst, excluded)); !os.IsNotExist(err) { 346 t.Errorf("%s should have been excluded, got err = %v", excluded, err) 347 } 348 } 349 } 350 351 func TestStageCleanPayloadRejectsSymlinks(t *testing.T) { 352 src := t.TempDir() 353 mustWrite(t, filepath.Join(src, "SKILL.md"), "---\nname: demo\ndescription: d\n---\n") 354 // Create a symlink pointing outside the src tree. 355 if err := os.Symlink("/etc/passwd", filepath.Join(src, "evil")); err != nil { 356 t.Skipf("symlink not supported on this filesystem: %v", err) 357 } 358 359 dst := filepath.Join(t.TempDir(), "stage") 360 err := stageCleanPayload(src, dst) 361 if err == nil { 362 t.Fatal("expected error on symlink, got nil") 363 } 364 if !strings.Contains(err.Error(), "symlink") { 365 t.Errorf("error should mention symlink, got: %v", err) 366 } 367 // Stage directory must not contain a half-copied payload. 368 if _, statErr := os.Stat(filepath.Join(dst, "SKILL.md")); statErr == nil { 369 t.Error("stage dir should be cleaned up on symlink rejection") 370 } 371 } 372 373 func mustWrite(t *testing.T, path, content string) { 374 t.Helper() 375 if err := os.MkdirAll(filepath.Dir(path), 0700); err != nil { 376 t.Fatalf("mkdir: %v", err) 377 } 378 if err := os.WriteFile(path, []byte(content), 0600); err != nil { 379 t.Fatalf("write: %v", err) 380 } 381 } 382 383 func mustWriteExec(t *testing.T, path, content string) { 384 t.Helper() 385 mustWrite(t, path, content) 386 if err := os.Chmod(path, 0755); err != nil { 387 t.Fatalf("chmod: %v", err) 388 } 389 } 390 391 // zipFileSpec describes one entry in a test zip fixture. 392 type zipFileSpec struct { 393 name string // path inside the zip 394 body string // file contents 395 mode os.FileMode // file mode (0 means default 0644) 396 link string // non-empty → emit as a symlink to this target 397 } 398 399 // makeZipFixture builds an in-memory zip archive from the given entries. 400 func makeZipFixture(t *testing.T, specs []zipFileSpec) []byte { 401 t.Helper() 402 var buf bytes.Buffer 403 zw := zip.NewWriter(&buf) 404 for _, sp := range specs { 405 mode := sp.mode 406 if mode == 0 { 407 mode = 0644 408 } 409 if sp.link != "" { 410 mode |= os.ModeSymlink 411 } 412 hdr := &zip.FileHeader{ 413 Name: sp.name, 414 Method: zip.Deflate, 415 } 416 hdr.SetMode(mode) 417 w, err := zw.CreateHeader(hdr) 418 if err != nil { 419 t.Fatalf("zip CreateHeader %q: %v", sp.name, err) 420 } 421 body := sp.body 422 if sp.link != "" { 423 body = sp.link 424 } 425 if _, err := w.Write([]byte(body)); err != nil { 426 t.Fatalf("zip Write %q: %v", sp.name, err) 427 } 428 } 429 if err := zw.Close(); err != nil { 430 t.Fatalf("zip Close: %v", err) 431 } 432 return buf.Bytes() 433 } 434 435 func TestExtractZipToSkillSuccess(t *testing.T) { 436 zipBytes := makeZipFixture(t, []zipFileSpec{ 437 {name: "SKILL.md", body: "---\nname: demo\ndescription: d\n---\nbody"}, 438 {name: "scripts/run.sh", body: "#!/bin/sh\necho hi", mode: 0755}, 439 {name: "references/schema.md", body: "schema content"}, 440 }) 441 442 destDir := filepath.Join(t.TempDir(), "stage") 443 if err := extractZipToSkill(bytes.NewReader(zipBytes), destDir); err != nil { 444 t.Fatalf("extractZipToSkill: %v", err) 445 } 446 447 // Core files present. 448 if _, err := os.Stat(filepath.Join(destDir, "SKILL.md")); err != nil { 449 t.Errorf("SKILL.md missing: %v", err) 450 } 451 if _, err := os.Stat(filepath.Join(destDir, "references", "schema.md")); err != nil { 452 t.Errorf("references/schema.md missing: %v", err) 453 } 454 // Executable bit preserved on scripts. 455 info, err := os.Stat(filepath.Join(destDir, "scripts", "run.sh")) 456 if err != nil { 457 t.Errorf("scripts/run.sh missing: %v", err) 458 } else if info.Mode().Perm()&0100 == 0 { 459 t.Errorf("scripts/run.sh lost executable bit: %v", info.Mode().Perm()) 460 } 461 } 462 463 func TestExtractZipToSkillExcludesGitMetadata(t *testing.T) { 464 zipBytes := makeZipFixture(t, []zipFileSpec{ 465 {name: "SKILL.md", body: "---\nname: demo\ndescription: d\n---\n"}, 466 {name: ".git/config", body: "[core]"}, 467 {name: ".github/workflows/ci.yml", body: "name: ci"}, 468 {name: ".gitignore", body: "node_modules"}, 469 {name: ".gitattributes", body: "* text"}, 470 }) 471 472 destDir := filepath.Join(t.TempDir(), "stage") 473 if err := extractZipToSkill(bytes.NewReader(zipBytes), destDir); err != nil { 474 t.Fatalf("extractZipToSkill: %v", err) 475 } 476 for _, excluded := range []string{".git", ".github", ".gitignore", ".gitattributes"} { 477 if _, err := os.Stat(filepath.Join(destDir, excluded)); !os.IsNotExist(err) { 478 t.Errorf("%s should have been excluded, got err = %v", excluded, err) 479 } 480 } 481 } 482 483 func TestExtractZipToSkillRejectsSymlink(t *testing.T) { 484 zipBytes := makeZipFixture(t, []zipFileSpec{ 485 {name: "SKILL.md", body: "---\nname: demo\ndescription: d\n---\n"}, 486 {name: "evil", link: "/etc/passwd"}, 487 }) 488 489 destDir := filepath.Join(t.TempDir(), "stage") 490 err := extractZipToSkill(bytes.NewReader(zipBytes), destDir) 491 if err == nil || !strings.Contains(err.Error(), "symlink") { 492 t.Errorf("expected symlink rejection error, got: %v", err) 493 } 494 // Stage dir must be cleaned up on failure. 495 if _, statErr := os.Stat(filepath.Join(destDir, "SKILL.md")); statErr == nil { 496 t.Error("stage dir should be cleaned up after symlink rejection") 497 } 498 } 499 500 func TestExtractZipToSkillRejectsZipSlip(t *testing.T) { 501 // Classic zip-slip: entry name escapes the destination via ../ 502 zipBytes := makeZipFixture(t, []zipFileSpec{ 503 {name: "SKILL.md", body: "---\nname: demo\ndescription: d\n---\n"}, 504 {name: "../outside.txt", body: "malicious"}, 505 }) 506 507 destDir := filepath.Join(t.TempDir(), "stage") 508 err := extractZipToSkill(bytes.NewReader(zipBytes), destDir) 509 if err == nil || !strings.Contains(err.Error(), "escapes") { 510 t.Errorf("expected zip-slip rejection error, got: %v", err) 511 } 512 } 513 514 // TestExtractZipToSkillZipBombActualBytes verifies the zip-bomb guard 515 // counts ACTUAL decompressed bytes, not the attacker-controlled 516 // UncompressedSize64 field. Regression test for the pre-fix bypass. 517 func TestExtractZipToSkillZipBombActualBytes(t *testing.T) { 518 // Temporarily shrink the uncompressed cap to 512 bytes so a small 519 // fixture can trip the guard without allocating hundreds of MB. 520 original := maxZipUncompressedBytes 521 maxZipUncompressedBytes = 512 522 defer func() { maxZipUncompressedBytes = original }() 523 524 // ~1060 actual bytes across two entries, well over the 512 cap. 525 zipBytes := makeZipFixture(t, []zipFileSpec{ 526 {name: "SKILL.md", body: "---\nname: demo\ndescription: d\n---\n"}, 527 {name: "payload.txt", body: strings.Repeat("x", 1024)}, 528 }) 529 530 destDir := filepath.Join(t.TempDir(), "stage") 531 err := extractZipToSkill(bytes.NewReader(zipBytes), destDir) 532 if err == nil { 533 t.Fatal("expected zip-bomb guard to fire, got nil") 534 } 535 if !strings.Contains(err.Error(), "uncompressed size exceeds") { 536 t.Errorf("error should mention uncompressed size cap, got: %v", err) 537 } 538 if _, statErr := os.Stat(filepath.Join(destDir, "SKILL.md")); statErr == nil { 539 t.Error("stage dir should be cleaned up after zip-bomb rejection") 540 } 541 } 542 543 func TestExtractZipToSkillRejectsSizeCap(t *testing.T) { 544 // Build a tiny zip but feed it through a reader capped tinier than the 545 // compressed size. Simulates a server returning more bytes than the cap. 546 zipBytes := makeZipFixture(t, []zipFileSpec{ 547 {name: "SKILL.md", body: strings.Repeat("x", 1024)}, 548 }) 549 550 destDir := filepath.Join(t.TempDir(), "stage") 551 // Feed exactly 10 bytes so the zip reader can't even parse the header. 552 err := extractZipToSkill(io.LimitReader(bytes.NewReader(zipBytes), 10), destDir) 553 if err == nil { 554 t.Errorf("expected error from truncated zip, got nil") 555 } 556 } 557 558 // makeFixtureRepo creates a minimal git repository on disk that can be 559 // cloned via file:// URLs. Uses the runGit helper from api.go. 560 func makeFixtureRepo(t *testing.T, skillContent string) string { 561 t.Helper() 562 dir := t.TempDir() 563 if err := runGit(dir, "init", "-q", "-b", "main"); err != nil { 564 t.Fatalf("git init: %v", err) 565 } 566 mustWrite(t, filepath.Join(dir, "SKILL.md"), skillContent) 567 mustWrite(t, filepath.Join(dir, "README.md"), "# demo") 568 if err := runGit(dir, "config", "user.email", "test@example.com"); err != nil { 569 t.Fatalf("git config email: %v", err) 570 } 571 if err := runGit(dir, "config", "user.name", "Test"); err != nil { 572 t.Fatalf("git config name: %v", err) 573 } 574 if err := runGit(dir, "config", "commit.gpgsign", "false"); err != nil { 575 t.Fatalf("git config gpgsign: %v", err) 576 } 577 if err := runGit(dir, "add", "."); err != nil { 578 t.Fatalf("git add: %v", err) 579 } 580 if err := runGit(dir, "commit", "-q", "-m", "init"); err != nil { 581 t.Fatalf("git commit: %v", err) 582 } 583 return dir 584 } 585 586 func TestInstallFromMarketplaceSuccess(t *testing.T) { 587 repo := makeFixtureRepo(t, "---\nname: demo\ndescription: d\n---\nbody") 588 shannonDir := t.TempDir() 589 590 entry := MarketplaceEntry{ 591 Slug: "demo", 592 Name: "demo", 593 Repo: "file://" + repo, 594 Ref: "main", 595 } 596 locks := NewSlugLocks() 597 598 if err := InstallFromMarketplace(context.Background(), shannonDir, entry, locks); err != nil { 599 t.Fatalf("InstallFromMarketplace: %v", err) 600 } 601 602 installed := filepath.Join(shannonDir, "skills", "demo", "SKILL.md") 603 if _, err := os.Stat(installed); err != nil { 604 t.Errorf("installed SKILL.md missing: %v", err) 605 } 606 if _, err := os.Stat(filepath.Join(shannonDir, "skills", "demo", ".git")); !os.IsNotExist(err) { 607 t.Error(".git should have been excluded from installed skill") 608 } 609 } 610 611 // TestInstallFromMarketplaceNameDiffersFromSlug confirms that a skill whose 612 // frontmatter `name` differs from the marketplace slug still installs. 613 // Regression target: ClawHub's xiaohongshu-mcp-skills package ships with 614 // `name: xiaohongshu`. Previous strict `canonical == dirName` rejection was 615 // over-tight and not required by the openclaw/clawhub spec. 616 func TestInstallFromMarketplaceNameDiffersFromSlug(t *testing.T) { 617 repo := makeFixtureRepo(t, "---\nname: different\ndescription: d\n---\n") 618 shannonDir := t.TempDir() 619 620 entry := MarketplaceEntry{Slug: "demo", Name: "demo", Repo: "file://" + repo, Ref: "main"} 621 if err := InstallFromMarketplace(context.Background(), shannonDir, entry, NewSlugLocks()); err != nil { 622 t.Fatalf("install should succeed despite name/slug mismatch, got: %v", err) 623 } 624 if _, err := os.Stat(filepath.Join(shannonDir, "skills", "demo", "SKILL.md")); err != nil { 625 t.Errorf("SKILL.md should exist under the slug-named directory: %v", err) 626 } 627 // Loaded skill: Name from frontmatter, Slug from directory name. 628 list, err := LoadSkills(SkillSource{Dir: filepath.Join(shannonDir, "skills"), Source: SourceGlobal}) 629 if err != nil { 630 t.Fatalf("LoadSkills: %v", err) 631 } 632 if len(list) != 1 { 633 t.Fatalf("expected 1 skill, got %d", len(list)) 634 } 635 if list[0].Name != "different" { 636 t.Errorf("Name should be %q (from frontmatter), got %q", "different", list[0].Name) 637 } 638 if list[0].Slug != "demo" { 639 t.Errorf("Slug should be %q (from directory), got %q", "demo", list[0].Slug) 640 } 641 } 642 643 func TestInstallFromMarketplaceBlocksMalicious(t *testing.T) { 644 shannonDir := t.TempDir() 645 entry := MarketplaceEntry{ 646 Slug: "demo", 647 Name: "demo", 648 Repo: "file:///does/not/matter", 649 Security: SecurityScan{VirusTotal: "malicious"}, 650 } 651 err := InstallFromMarketplace(context.Background(), shannonDir, entry, NewSlugLocks()) 652 if !errors.Is(err, ErrMaliciousSkill) { 653 t.Errorf("expected ErrMaliciousSkill, got: %v", err) 654 } 655 } 656 657 func TestInstallFromMarketplaceAlreadyInstalled(t *testing.T) { 658 repo := makeFixtureRepo(t, "---\nname: demo\ndescription: d\n---\n") 659 shannonDir := t.TempDir() 660 entry := MarketplaceEntry{Slug: "demo", Name: "demo", Repo: "file://" + repo, Ref: "main"} 661 locks := NewSlugLocks() 662 if err := InstallFromMarketplace(context.Background(), shannonDir, entry, locks); err != nil { 663 t.Fatalf("first install: %v", err) 664 } 665 err := InstallFromMarketplace(context.Background(), shannonDir, entry, locks) 666 if !errors.Is(err, ErrSkillAlreadyInstalled) { 667 t.Errorf("expected ErrSkillAlreadyInstalled, got: %v", err) 668 } 669 } 670 671 // makeFixtureRepoSubdir creates a fixture repo where SKILL.md lives under 672 // skills/<slug>/ rather than at the repo root, plus an unrelated sibling 673 // directory that must NOT end up in the installed skill. Used to exercise 674 // the sparse-checkout branch of InstallFromMarketplace. 675 func makeFixtureRepoSubdir(t *testing.T, slug, skillContent string) string { 676 t.Helper() 677 dir := t.TempDir() 678 if err := runGit(dir, "init", "-q", "-b", "main"); err != nil { 679 t.Fatalf("git init: %v", err) 680 } 681 mustWrite(t, filepath.Join(dir, "skills", slug, "SKILL.md"), skillContent) 682 mustWriteExec(t, filepath.Join(dir, "skills", slug, "scripts", "hello.sh"), "#!/bin/sh\necho hi") 683 // Sibling skill that must not bleed into the install. 684 mustWrite(t, filepath.Join(dir, "skills", "other", "SKILL.md"), "---\nname: other\ndescription: x\n---\n") 685 // Unrelated top-level file. 686 mustWrite(t, filepath.Join(dir, "README.md"), "# monorepo") 687 688 for _, args := range [][]string{ 689 {"config", "user.email", "test@example.com"}, 690 {"config", "user.name", "Test"}, 691 {"config", "commit.gpgsign", "false"}, 692 {"add", "."}, 693 {"commit", "-q", "-m", "init"}, 694 } { 695 if err := runGit(dir, args...); err != nil { 696 t.Fatalf("git %v: %v", args, err) 697 } 698 } 699 return dir 700 } 701 702 func TestInstallFromMarketplaceSubdirectory(t *testing.T) { 703 repo := makeFixtureRepoSubdir(t, "demo", "---\nname: demo\ndescription: d\n---\nbody") 704 shannonDir := t.TempDir() 705 706 entry := MarketplaceEntry{ 707 Slug: "demo", 708 Name: "demo", 709 Repo: "file://" + repo, 710 RepoPath: "skills/demo", 711 Ref: "main", 712 } 713 714 if err := InstallFromMarketplace(context.Background(), shannonDir, entry, NewSlugLocks()); err != nil { 715 t.Fatalf("InstallFromMarketplace: %v", err) 716 } 717 718 installedRoot := filepath.Join(shannonDir, "skills", "demo") 719 720 // SKILL.md lands at the top of the installed dir, not nested under skills/demo. 721 if _, err := os.Stat(filepath.Join(installedRoot, "SKILL.md")); err != nil { 722 t.Errorf("installed SKILL.md missing: %v", err) 723 } 724 725 // Helper script survives and keeps its executable bit. 726 scriptInfo, err := os.Stat(filepath.Join(installedRoot, "scripts", "hello.sh")) 727 if err != nil { 728 t.Errorf("scripts/hello.sh missing: %v", err) 729 } else if scriptInfo.Mode().Perm()&0100 == 0 { 730 t.Errorf("scripts/hello.sh lost its executable bit: mode = %v", scriptInfo.Mode().Perm()) 731 } 732 733 // Unrelated siblings and top-level files must NOT be copied in. 734 mustNotExist := []string{ 735 filepath.Join(installedRoot, "README.md"), 736 filepath.Join(installedRoot, "skills"), // no nested skills/ dir 737 filepath.Join(installedRoot, "other"), 738 filepath.Join(installedRoot, ".git"), 739 } 740 for _, p := range mustNotExist { 741 if _, err := os.Stat(p); !os.IsNotExist(err) { 742 t.Errorf("%s should not exist in subdirectory install, got err = %v", p, err) 743 } 744 } 745 } 746 747 func TestInstallFromMarketplaceZipSuccess(t *testing.T) { 748 zipBytes := makeZipFixture(t, []zipFileSpec{ 749 {name: "SKILL.md", body: "---\nname: ziptest\ndescription: From a zip\n---\nbody"}, 750 {name: "scripts/run.sh", body: "#!/bin/sh\necho hi", mode: 0755}, 751 {name: "references/README.md", body: "ref"}, 752 }) 753 // Serve the zip over HTTP. 754 ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 755 w.Header().Set("Content-Type", "application/zip") 756 w.Write(zipBytes) 757 })) 758 defer ts.Close() 759 760 shannonDir := t.TempDir() 761 entry := MarketplaceEntry{ 762 Slug: "ziptest", 763 Name: "ziptest", 764 DownloadURL: ts.URL, 765 } 766 767 if err := InstallFromMarketplace(context.Background(), shannonDir, entry, NewSlugLocks()); err != nil { 768 t.Fatalf("InstallFromMarketplace: %v", err) 769 } 770 771 // Verify the installed tree. 772 installed := filepath.Join(shannonDir, "skills", "ziptest") 773 if _, err := os.Stat(filepath.Join(installed, "SKILL.md")); err != nil { 774 t.Errorf("SKILL.md missing: %v", err) 775 } 776 if _, err := os.Stat(filepath.Join(installed, "references", "README.md")); err != nil { 777 t.Errorf("references/README.md missing: %v", err) 778 } 779 scriptInfo, err := os.Stat(filepath.Join(installed, "scripts", "run.sh")) 780 if err != nil { 781 t.Errorf("scripts/run.sh missing: %v", err) 782 } else if scriptInfo.Mode().Perm()&0100 == 0 { 783 t.Errorf("scripts/run.sh lost executable bit: %v", scriptInfo.Mode().Perm()) 784 } 785 } 786 787 // Regression test for the "Skill package is malformed" install failure seen 788 // on ClawHub skills that ship with nested YAML metadata (e.g. a `clawdbot` 789 // object containing emoji, required bins, etc.). The old `map[string]string` 790 // typing caused yaml.Unmarshal to reject any non-string value, which surfaced 791 // as ErrInvalidSkillPayload → HTTP 422 → Desktop toast. With the widened 792 // `map[string]any` typing the install should succeed and the parsed metadata 793 // should preserve the nested shape verbatim. 794 func TestInstallFromMarketplaceZipNestedMetadata(t *testing.T) { 795 skillMD := `--- 796 name: docker-essentials 797 description: Essential Docker commands and workflows for container management, image operations, and debugging. 798 homepage: https://docs.docker.com/ 799 metadata: 800 clawdbot: 801 emoji: "🐳" 802 requires: 803 bins: 804 - docker 805 --- 806 # Docker Essentials 807 Essential Docker commands for container and image management. 808 ` 809 zipBytes := makeZipFixture(t, []zipFileSpec{ 810 {name: "SKILL.md", body: skillMD}, 811 }) 812 ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 813 w.Header().Set("Content-Type", "application/zip") 814 w.Write(zipBytes) 815 })) 816 defer ts.Close() 817 818 shannonDir := t.TempDir() 819 entry := MarketplaceEntry{ 820 Slug: "docker-essentials", 821 Name: "docker-essentials", 822 DownloadURL: ts.URL, 823 } 824 825 if err := InstallFromMarketplace(context.Background(), shannonDir, entry, NewSlugLocks()); err != nil { 826 t.Fatalf("InstallFromMarketplace with nested metadata should succeed, got: %v", err) 827 } 828 829 installed := filepath.Join(shannonDir, "skills", "docker-essentials") 830 if _, err := os.Stat(filepath.Join(installed, "SKILL.md")); err != nil { 831 t.Errorf("SKILL.md missing after install: %v", err) 832 } 833 834 // Reload through the normal loader path and verify the nested metadata 835 // survived the parse → model → re-read roundtrip. This is the exact 836 // path `GET /skills` and `GET /skills/{name}` use. 837 sources := []SkillSource{{Dir: filepath.Join(shannonDir, "skills"), Source: SourceGlobal}} 838 list, err := LoadSkills(sources...) 839 if err != nil { 840 t.Fatalf("LoadSkills after install: %v", err) 841 } 842 var loaded *Skill 843 for _, s := range list { 844 if s.Name == "docker-essentials" { 845 loaded = s 846 break 847 } 848 } 849 if loaded == nil { 850 t.Fatal("docker-essentials not found after LoadSkills") 851 } 852 clawdbot, ok := loaded.Metadata["clawdbot"].(map[string]any) 853 if !ok { 854 t.Fatalf("expected clawdbot to be map[string]any, got %T: %#v", loaded.Metadata["clawdbot"], loaded.Metadata["clawdbot"]) 855 } 856 if clawdbot["emoji"] != "🐳" { 857 t.Errorf("emoji roundtrip failed: got %v", clawdbot["emoji"]) 858 } 859 requires, ok := clawdbot["requires"].(map[string]any) 860 if !ok { 861 t.Fatalf("expected requires to be map[string]any, got %T", clawdbot["requires"]) 862 } 863 bins, ok := requires["bins"].([]any) 864 if !ok || len(bins) != 1 || bins[0] != "docker" { 865 t.Errorf("bins roundtrip failed: got %#v", requires["bins"]) 866 } 867 } 868 869 // Regression test: ClawHub's `ivangdavila/docker` ships with `name: Docker` 870 // (display label) and `slug: docker` (directory identity). Install must 871 // succeed and the loaded skill must be addressable by its slug; the 872 // frontmatter display name is preserved as skill.Name. 873 func TestInstallFromMarketplaceZipSlugOverridesDisplayName(t *testing.T) { 874 skillMD := `--- 875 name: Docker 876 slug: docker 877 description: Docker containers, images, Compose stacks, networking, and the commands that keep real environments stable. 878 metadata: 879 clawdbot: 880 emoji: "🐳" 881 requires: 882 bins: 883 - docker 884 --- 885 # Docker 886 Commands and workflows. 887 ` 888 zipBytes := makeZipFixture(t, []zipFileSpec{ 889 {name: "SKILL.md", body: skillMD}, 890 }) 891 ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 892 w.Header().Set("Content-Type", "application/zip") 893 w.Write(zipBytes) 894 })) 895 defer ts.Close() 896 897 shannonDir := t.TempDir() 898 entry := MarketplaceEntry{ 899 Slug: "docker", 900 Name: "docker", 901 DownloadURL: ts.URL, 902 } 903 904 if err := InstallFromMarketplace(context.Background(), shannonDir, entry, NewSlugLocks()); err != nil { 905 t.Fatalf("InstallFromMarketplace with display-name/slug split should succeed, got: %v", err) 906 } 907 908 // The installed skill should be addressable by its slug on disk; the 909 // frontmatter display name "Docker" is preserved as skill.Name. 910 sources := []SkillSource{{Dir: filepath.Join(shannonDir, "skills"), Source: SourceGlobal}} 911 list, err := LoadSkills(sources...) 912 if err != nil { 913 t.Fatalf("LoadSkills after install: %v", err) 914 } 915 var loaded *Skill 916 for _, s := range list { 917 if s.Slug == "docker" { 918 loaded = s 919 break 920 } 921 } 922 if loaded == nil { 923 t.Fatal("loaded skill should be addressable by slug \"docker\"") 924 } 925 if loaded.Name != "Docker" { 926 t.Errorf("Name should preserve frontmatter display label %q, got %q", "Docker", loaded.Name) 927 } 928 if loaded.Slug != "docker" { 929 t.Errorf("Slug should be %q (from directory), got %q", "docker", loaded.Slug) 930 } 931 } 932 933 // TestInstallFromMarketplaceZipNameDiffersFromSlug: zip-transport counterpart 934 // to TestInstallFromMarketplaceNameDiffersFromSlug — frontmatter name differs 935 // from marketplace slug, install still succeeds. 936 func TestInstallFromMarketplaceZipNameDiffersFromSlug(t *testing.T) { 937 zipBytes := makeZipFixture(t, []zipFileSpec{ 938 {name: "SKILL.md", body: "---\nname: actual\ndescription: d\n---\n"}, 939 }) 940 ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 941 w.Write(zipBytes) 942 })) 943 defer ts.Close() 944 945 shannonDir := t.TempDir() 946 entry := MarketplaceEntry{Slug: "expected", Name: "expected", DownloadURL: ts.URL} 947 948 if err := InstallFromMarketplace(context.Background(), shannonDir, entry, NewSlugLocks()); err != nil { 949 t.Fatalf("install should succeed despite name/slug mismatch, got: %v", err) 950 } 951 if _, err := os.Stat(filepath.Join(shannonDir, "skills", "expected", "SKILL.md")); err != nil { 952 t.Errorf("SKILL.md should be installed under slug directory: %v", err) 953 } 954 } 955 956 func TestInstallFromMarketplaceZipMaliciousBlocked(t *testing.T) { 957 // Malicious gate runs before any network call, so no server needed. 958 shannonDir := t.TempDir() 959 entry := MarketplaceEntry{ 960 Slug: "evil", 961 Name: "evil", 962 DownloadURL: "http://127.0.0.1:1/never-called", 963 Security: SecurityScan{OpenClaw: "malicious"}, 964 } 965 err := InstallFromMarketplace(context.Background(), shannonDir, entry, NewSlugLocks()) 966 if !errors.Is(err, ErrMaliciousSkill) { 967 t.Errorf("expected ErrMaliciousSkill, got: %v", err) 968 } 969 } 970 971 func TestInstallFromMarketplaceZipUpstreamFailure(t *testing.T) { 972 // Server returns 500 — should surface as ErrMarketplaceUpstreamFailure. 973 ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 974 w.WriteHeader(http.StatusInternalServerError) 975 })) 976 defer ts.Close() 977 978 shannonDir := t.TempDir() 979 entry := MarketplaceEntry{Slug: "demo", Name: "demo", DownloadURL: ts.URL} 980 981 err := InstallFromMarketplace(context.Background(), shannonDir, entry, NewSlugLocks()) 982 if !errors.Is(err, ErrMarketplaceUpstreamFailure) { 983 t.Errorf("expected ErrMarketplaceUpstreamFailure, got: %v", err) 984 } 985 } 986 987 func TestInstallFromMarketplaceNoTransport(t *testing.T) { 988 // Neither Repo nor DownloadURL set → must be rejected before any 989 // filesystem work. 990 shannonDir := t.TempDir() 991 entry := MarketplaceEntry{Slug: "demo", Name: "demo"} 992 993 err := InstallFromMarketplace(context.Background(), shannonDir, entry, NewSlugLocks()) 994 if err == nil || !errors.Is(err, ErrInvalidSkillPayload) { 995 t.Errorf("expected ErrInvalidSkillPayload, got: %v", err) 996 } 997 } 998 999 // TestInstallFromMarketplaceConcurrentSameSlug drives two goroutines at the 1000 // same slug. With the per-slug lock in place, exactly one must succeed and 1001 // the other must see ErrSkillAlreadyInstalled — no filesystem corruption, 1002 // no ENOTEMPTY from a half-finished rename. 1003 func TestInstallFromMarketplaceConcurrentSameSlug(t *testing.T) { 1004 repo := makeFixtureRepo(t, "---\nname: demo\ndescription: d\n---\n") 1005 shannonDir := t.TempDir() 1006 entry := MarketplaceEntry{Slug: "demo", Name: "demo", Repo: "file://" + repo, Ref: "main"} 1007 locks := NewSlugLocks() 1008 1009 const N = 5 1010 results := make(chan error, N) 1011 var wg sync.WaitGroup 1012 for i := 0; i < N; i++ { 1013 wg.Add(1) 1014 go func() { 1015 defer wg.Done() 1016 results <- InstallFromMarketplace(context.Background(), shannonDir, entry, locks) 1017 }() 1018 } 1019 wg.Wait() 1020 close(results) 1021 1022 var successes, alreadyInstalled, other int 1023 for err := range results { 1024 switch { 1025 case err == nil: 1026 successes++ 1027 case errors.Is(err, ErrSkillAlreadyInstalled): 1028 alreadyInstalled++ 1029 default: 1030 other++ 1031 t.Errorf("unexpected concurrent install error: %v", err) 1032 } 1033 } 1034 if successes != 1 { 1035 t.Errorf("expected exactly 1 successful install, got %d", successes) 1036 } 1037 if alreadyInstalled != N-1 { 1038 t.Errorf("expected %d already-installed results, got %d", N-1, alreadyInstalled) 1039 } 1040 1041 // Final state must be a single clean install. 1042 if _, err := os.Stat(filepath.Join(shannonDir, "skills", "demo", "SKILL.md")); err != nil { 1043 t.Errorf("installed SKILL.md missing after concurrent installs: %v", err) 1044 } 1045 } 1046 1047 func TestMarketplaceEntryIsMalicious(t *testing.T) { 1048 cases := []struct { 1049 name string 1050 e MarketplaceEntry 1051 want bool 1052 }{ 1053 {"clean", MarketplaceEntry{}, false}, 1054 {"benign", MarketplaceEntry{Security: SecurityScan{VirusTotal: "benign", OpenClaw: "benign"}}, false}, 1055 {"vt-malicious", MarketplaceEntry{Security: SecurityScan{VirusTotal: "malicious"}}, true}, 1056 {"oc-malicious", MarketplaceEntry{Security: SecurityScan{OpenClaw: "malicious"}}, true}, 1057 {"suspicious-only", MarketplaceEntry{Security: SecurityScan{VirusTotal: "suspicious"}}, false}, 1058 } 1059 for _, tc := range cases { 1060 t.Run(tc.name, func(t *testing.T) { 1061 if got := tc.e.IsMalicious(); got != tc.want { 1062 t.Errorf("IsMalicious = %v, want %v", got, tc.want) 1063 } 1064 }) 1065 } 1066 }