git.go
1 package source 2 3 import ( 4 "crypto/rand" 5 "encoding/json" 6 "fmt" 7 "io" 8 "log" 9 "os" 10 "path/filepath" 11 "strings" 12 "time" 13 14 dogeboxd "github.com/dogeorg/dogeboxd/pkg" 15 "github.com/dogeorg/dogeboxd/pkg/utils" 16 "github.com/go-git/go-billy/v5/memfs" 17 "github.com/go-git/go-git/v5" 18 "github.com/go-git/go-git/v5/config" 19 "github.com/go-git/go-git/v5/plumbing" 20 "github.com/go-git/go-git/v5/storage/memory" 21 "golang.org/x/mod/semver" 22 ) 23 24 var _ dogeboxd.ManifestSource = &ManifestSourceGit{} 25 26 type ManifestSourceGit struct { 27 serverConfig dogeboxd.ServerConfig 28 config dogeboxd.ManifestSourceConfiguration 29 _cache dogeboxd.ManifestSourceList 30 _isCached bool 31 } 32 33 func (r ManifestSourceGit) ValidateFromLocation(location string) (dogeboxd.ManifestSourceConfiguration, error) { 34 // Get all our tags for this repository. 35 tags, err := r.GetAllGitTags(location) 36 if err != nil { 37 return dogeboxd.ManifestSourceConfiguration{}, err 38 } 39 40 // Filter out non-semver tags and find the greatest version 41 var validTags []string 42 for _, tag := range tags { 43 if semver.IsValid(tag) { 44 validTags = append(validTags, tag) 45 } 46 } 47 48 if len(validTags) == 0 { 49 return dogeboxd.ManifestSourceConfiguration{}, fmt.Errorf("no valid semver tags found") 50 } 51 52 semver.Sort(validTags) 53 latestVersion := validTags[len(validTags)-1] 54 55 details, valid, err := r.getSourceDetails(location, "refs/tags/"+latestVersion) 56 if err != nil { 57 log.Printf("Error getting source details: %v", err) 58 return dogeboxd.ManifestSourceConfiguration{}, err 59 } 60 61 if valid { 62 return dogeboxd.ManifestSourceConfiguration{ 63 ID: details.ID, 64 Name: details.Name, 65 Description: details.Description, 66 Location: location, 67 Type: "git", 68 }, nil 69 } 70 71 // If we don't have a valid dogebox.json, check if this is a root-level pup. 72 worktree, _, err := r.getShallowWorktree(location, "refs/tags/"+latestVersion) 73 if err != nil { 74 return dogeboxd.ManifestSourceConfiguration{}, err 75 } 76 77 manifestFile, err := worktree.Filesystem.Open("manifest.json") 78 if err != nil { 79 if os.IsNotExist(err) { 80 return dogeboxd.ManifestSourceConfiguration{}, fmt.Errorf("manifest.json not found in the root of the repository") 81 } 82 return dogeboxd.ManifestSourceConfiguration{}, fmt.Errorf("error opening manifest.json: %w", err) 83 } 84 defer manifestFile.Close() 85 86 var manifest dogeboxd.PupManifest 87 decoder := json.NewDecoder(manifestFile) 88 if err := decoder.Decode(&manifest); err != nil { 89 return dogeboxd.ManifestSourceConfiguration{}, fmt.Errorf("error parsing manifest.json: %w", err) 90 } 91 92 if err := manifest.Validate(); err != nil { 93 return dogeboxd.ManifestSourceConfiguration{}, fmt.Errorf("invalid manifest.json: %w", err) 94 } 95 96 var sourceId string 97 b := make([]byte, 16) 98 _, err = rand.Read(b) 99 if err != nil { 100 return dogeboxd.ManifestSourceConfiguration{}, err 101 } 102 sourceId = fmt.Sprintf("%x", b) 103 104 return dogeboxd.ManifestSourceConfiguration{ 105 ID: sourceId, 106 Name: manifest.Meta.Name, 107 Description: "", 108 Location: location, 109 Type: "git", 110 }, nil 111 } 112 113 func (r ManifestSourceGit) GetAllGitTags(location string) ([]string, error) { 114 rem := git.NewRemote(memory.NewStorage(), &config.RemoteConfig{ 115 Name: "origin", 116 URLs: []string{location}, 117 }) 118 119 refs, err := rem.List(&git.ListOptions{ 120 PeelingOption: git.AppendPeeled, 121 }) 122 if err != nil { 123 return []string{}, err 124 } 125 126 // Filters the references list and only keeps tags 127 var tags []string 128 for _, ref := range refs { 129 if ref.Name().IsTag() { 130 tags = append(tags, ref.Name().Short()) 131 } 132 } 133 134 return tags, nil 135 } 136 137 func (r ManifestSourceGit) Name() string { 138 return r.config.Name 139 } 140 141 func (r ManifestSourceGit) Config() dogeboxd.ManifestSourceConfiguration { 142 return r.config 143 } 144 145 func (r ManifestSourceGit) getShallowWorktree(location, tag string) (*git.Worktree, *git.Repository, error) { 146 storage := memory.NewStorage() 147 fs := memfs.New() 148 149 // Clone the repository with the specific tag 150 repo, err := git.Clone(storage, fs, &git.CloneOptions{ 151 URL: location, 152 ReferenceName: plumbing.ReferenceName(tag), 153 SingleBranch: true, 154 Depth: 1, 155 }) 156 if err != nil { 157 return &git.Worktree{}, &git.Repository{}, fmt.Errorf("failed to clone repository: %w", err) 158 } 159 160 worktree, err := repo.Worktree() 161 if err != nil { 162 return &git.Worktree{}, &git.Repository{}, fmt.Errorf("failed to get worktree: %w", err) 163 } 164 165 return worktree, repo, nil 166 } 167 168 func (r ManifestSourceGit) getSourceDetails(location, tag string) (dogeboxd.SourceDetails, bool, error) { 169 worktree, _, err := r.getShallowWorktree(location, tag) 170 if err != nil { 171 log.Printf("Error getting shallow worktree: %v", err) 172 return dogeboxd.SourceDetails{}, false, err 173 } 174 175 return r.getSourceDetailsFromWorktree(worktree) 176 } 177 178 func (r ManifestSourceGit) getSourceDetailsFromWorktree(worktree *git.Worktree) (dogeboxd.SourceDetails, bool, error) { 179 indexPath := "dogebox.json" 180 _, err := worktree.Filesystem.Stat(indexPath) 181 if err == nil { 182 content, err := worktree.Filesystem.Open(indexPath) 183 if err != nil { 184 return dogeboxd.SourceDetails{}, false, fmt.Errorf("failed to open dogebox.json: %w", err) 185 } 186 defer content.Close() 187 188 manifestBytes, err := io.ReadAll(content) 189 if err != nil { 190 return dogeboxd.SourceDetails{}, false, fmt.Errorf("failed to read dogebox.json: %w", err) 191 } 192 193 d, err := ParseAndValidateSourceDetails(string(manifestBytes)) 194 if err != nil { 195 return dogeboxd.SourceDetails{}, false, fmt.Errorf("failed to parse and validate dogebox.json: %w", err) 196 } 197 198 return d, true, nil 199 } 200 201 return dogeboxd.SourceDetails{}, false, nil 202 } 203 204 type GitPupEntry struct { 205 Manifest dogeboxd.PupManifest 206 SubPath string 207 LogoBase64 string 208 } 209 210 func (r ManifestSourceGit) ensureTagValidAndGetPups(tag string) ([]GitPupEntry, error) { 211 entries := []GitPupEntry{} 212 213 worktree, _, err := r.getShallowWorktree(r.config.Location, tag) 214 if err != nil { 215 return []GitPupEntry{}, err 216 } 217 218 pupLocations := []string{} 219 220 tagDetails, foundDetails, err := r.getSourceDetailsFromWorktree(worktree) 221 if err != nil { 222 return []GitPupEntry{}, err 223 } 224 225 if foundDetails { 226 for _, pup := range tagDetails.Pups { 227 pupLocations = append(pupLocations, pup.Location) 228 } 229 } else { 230 pupLocations = append(pupLocations, ".") 231 } 232 233 for _, pupLocation := range pupLocations { 234 pupManifest, logoBase64, isValid, err := r.getPupManifestFromWorktreeLocation(tag, worktree, pupLocation) 235 if err != nil { 236 return []GitPupEntry{}, err 237 } 238 if isValid { 239 entries = append(entries, GitPupEntry{ 240 Manifest: pupManifest, 241 SubPath: pupLocation, 242 LogoBase64: logoBase64, 243 }) 244 } 245 } 246 247 return entries, nil 248 } 249 250 func (r ManifestSourceGit) getPupManifestFromWorktreeLocation(tag string, worktree *git.Worktree, location string) (dogeboxd.PupManifest, string, bool, error) { 251 for _, filename := range REQUIRED_FILES { 252 _, err := worktree.Filesystem.Stat(filepath.Join(location, filename)) 253 if err != nil { 254 if os.IsNotExist(err) { 255 log.Printf("tag %s missing file %s", tag, filename) 256 return dogeboxd.PupManifest{}, "", false, nil 257 } 258 return dogeboxd.PupManifest{}, "", false, fmt.Errorf("failed to check for file %s: %w", filename, err) 259 } 260 } 261 262 content, err := worktree.Filesystem.Open(filepath.Join(location, "manifest.json")) 263 if err != nil { 264 return dogeboxd.PupManifest{}, "", false, fmt.Errorf("failed to open manifest.json: %w", err) 265 } 266 defer content.Close() 267 268 manifestBytes, err := io.ReadAll(content) 269 if err != nil { 270 return dogeboxd.PupManifest{}, "", false, fmt.Errorf("failed to read manifest.json: %w", err) 271 } 272 273 var manifest dogeboxd.PupManifest 274 err = json.Unmarshal(manifestBytes, &manifest) 275 if err != nil { 276 return dogeboxd.PupManifest{}, "", false, fmt.Errorf("failed to unmarshal manifest.json: %w", err) 277 } 278 279 if err := manifest.Validate(); err != nil { 280 return dogeboxd.PupManifest{}, "", false, fmt.Errorf("manifest validation failed: %w", err) 281 } 282 283 logoBase64 := "" 284 285 if manifest.Meta.LogoPath != "" { 286 logoPath := worktree.Filesystem.Join(location, manifest.Meta.LogoPath) 287 if _, err := worktree.Filesystem.Stat(logoPath); err == nil { 288 logoFile, err := worktree.Filesystem.Open(logoPath) 289 if err == nil { 290 logoData, err := io.ReadAll(logoFile) 291 if err == nil { 292 logoBase64, err = utils.ImageBytesToWebBase64(logoData, manifest.Meta.LogoPath) 293 if err != nil { 294 // Don't fail if we can't read/convert the logo for whatever reason. 295 log.Printf("failed to read/convert logo for %s: %s", manifest.Meta.Name, err) 296 } 297 } 298 } 299 } 300 } 301 302 log.Printf("Successfully read manifest for location %s", location) 303 return manifest, logoBase64, true, nil 304 } 305 306 func (r *ManifestSourceGit) List(ignoreCache bool) (dogeboxd.ManifestSourceList, error) { 307 if !ignoreCache && r._isCached { 308 return r._cache, nil 309 } 310 311 repo, err := git.Clone(memory.NewStorage(), memfs.New(), &git.CloneOptions{ 312 URL: r.config.Location, 313 Depth: 1, 314 SingleBranch: true, 315 Tags: git.AllTags, 316 }) 317 if err != nil { 318 return dogeboxd.ManifestSourceList{}, err 319 } 320 321 iter, err := repo.Tags() 322 if err != nil { 323 return dogeboxd.ManifestSourceList{}, err 324 } 325 326 type TagResult struct { 327 version string 328 entries []GitPupEntry 329 err error 330 } 331 332 resultChan := make(chan TagResult) 333 var tagCount int 334 335 err = iter.ForEach(func(p *plumbing.Reference) error { 336 tagRef := string(p.Name()) 337 tagName := strings.Replace(tagRef, "refs/tags/", "", -1) 338 if semver.IsValid(tagName) { 339 tagCount++ 340 go func(ref, version string) { 341 entries, err := r.ensureTagValidAndGetPups(ref) 342 resultChan <- TagResult{version: version, entries: entries, err: err} 343 }(tagRef, tagName) 344 } 345 346 return nil 347 }) 348 if err != nil { 349 return dogeboxd.ManifestSourceList{}, err 350 } 351 352 validPups := []dogeboxd.ManifestSourcePup{} 353 354 for i := 0; i < tagCount; i++ { 355 result := <-resultChan 356 if result.err != nil { 357 log.Printf("Error validating tag %s: %v", result.version, result.err) 358 continue 359 } 360 361 for _, entry := range result.entries { 362 validPups = append(validPups, dogeboxd.ManifestSourcePup{ 363 Name: entry.Manifest.Meta.Name, 364 Location: map[string]string{ 365 "tag": result.version, 366 "subPath": entry.SubPath, 367 }, 368 Version: entry.Manifest.Meta.Version, 369 Manifest: entry.Manifest, 370 LogoBase64: entry.LogoBase64, 371 }) 372 } 373 } 374 375 list := dogeboxd.ManifestSourceList{ 376 Config: r.config, 377 LastChecked: time.Now(), 378 Pups: validPups, 379 } 380 381 r._cache = list 382 r._isCached = true 383 384 return r._cache, nil 385 } 386 387 func (r ManifestSourceGit) Download(diskPath string, location map[string]string) error { 388 tempDir, err := os.MkdirTemp(r.serverConfig.TmpDir, "pup-clone-") 389 if err != nil { 390 return fmt.Errorf("failed to create temp directory: %w", err) 391 } 392 defer os.RemoveAll(tempDir) 393 394 log.Printf("Cloning repository %s (tag: %s) to temporary directory", r.config.Location, location["tag"]) 395 396 _, err = git.PlainClone(tempDir, false, &git.CloneOptions{ 397 URL: r.config.Location, 398 ReferenceName: plumbing.ReferenceName("refs/tags/" + location["tag"]), 399 SingleBranch: true, 400 Depth: 1, 401 }) 402 if err != nil { 403 return fmt.Errorf("failed to clone repository: %w", err) 404 } 405 406 // Construct the path to the subpath within the cloned repository 407 sourcePath := filepath.Join(tempDir, location["subPath"]) 408 409 // Ensure the source path exists 410 if _, err := os.Stat(sourcePath); os.IsNotExist(err) { 411 return fmt.Errorf("subpath %s does not exist in the cloned repository", location["subPath"]) 412 } 413 414 // Copy the subpath to the final destination 415 err = filepath.Walk(sourcePath, func(path string, info os.FileInfo, err error) error { 416 if err != nil { 417 return err 418 } 419 420 relPath, err := filepath.Rel(sourcePath, path) 421 if err != nil { 422 return fmt.Errorf("failed to get relative path: %w", err) 423 } 424 425 destPath := filepath.Join(diskPath, relPath) 426 427 if info.IsDir() { 428 return os.MkdirAll(destPath, info.Mode()) 429 } 430 431 srcFile, err := os.Open(path) 432 if err != nil { 433 return fmt.Errorf("failed to open source file: %w", err) 434 } 435 defer srcFile.Close() 436 437 destFile, err := os.Create(destPath) 438 if err != nil { 439 return fmt.Errorf("failed to create destination file: %w", err) 440 } 441 defer destFile.Close() 442 443 _, err = io.Copy(destFile, srcFile) 444 if err != nil { 445 return fmt.Errorf("failed to copy file contents: %w", err) 446 } 447 448 return os.Chmod(destPath, info.Mode()) 449 }) 450 if err != nil { 451 return fmt.Errorf("failed to copy subpath to destination: %w", err) 452 } 453 454 log.Printf("Successfully downloaded and moved subpath %s to %s", location["subPath"], diskPath) 455 456 return nil 457 }