make-samples.go
1 package util 2 3 import ( 4 "bufio" 5 "bytes" 6 "fmt" 7 "os" 8 "os/exec" 9 "path/filepath" 10 "strconv" 11 "strings" 12 13 "github.com/thunderbrewhq/binana/go/app" 14 "github.com/thunderbrewhq/binana/go/app/util/dbutil" 15 "github.com/thunderbrewhq/binana/go/db" 16 ) 17 18 type MakeSampleDatabaseParams struct { 19 // A file name that corresponds to a tree of sample files. 20 // Anything in this tree will be collected into the sample database 21 Source string 22 23 // The name of the file to write the database to 24 Output string 25 26 // Sets the format of the database file 27 Format dbutil.DatabaseFormat 28 29 // URLs that maps to the root of the sample tree hierarchy. 30 // Used to generate a list of mirror URLs for sample binaries 31 DirectMirrors []string 32 33 // List of IPFS Gateway URLs 34 // If not empty, a CID for the sample tree will be created, 35 // Actually uploading anything in the sample tree, however, 36 // is outside the scope of this tool 37 IPFSGateways []string 38 } 39 40 type sample_database struct { 41 writer *dbutil.Writer[db.Sample] 42 ipfs_tree_cid string 43 buffer []db.Sample 44 } 45 46 func (sample_database *sample_database) add(sample db.Sample) (err error) { 47 sample_database.buffer = append(sample_database.buffer, sample) 48 return 49 } 50 51 func (sample_database *sample_database) Close() (err error) { 52 if err = sample_database.writer.WriteEntries(sample_database.buffer); err != nil { 53 return 54 } 55 err = sample_database.writer.Close() 56 return 57 } 58 59 func (sample_database *sample_database) make_sample_file(params *MakeSampleDatabaseParams, name, relative_name string) (err error) { 60 var sample db.Sample 61 // infer mime-type from extension 62 switch filepath.Ext(name) { 63 case ".exe": 64 sample.MimeType = "application/vnd.microsoft.portable-executable" 65 case ".pdb": 66 sample.MimeType = "application/x-ms-pdb" 67 // associate the PDB with its EXE 68 sample_exe_name := strings.TrimSuffix(name, ".pdb") + ".exe" 69 if _, err = os.Stat(sample_exe_name); err == nil { 70 sample.Executable, err = hash_file(sample_exe_name) 71 if err != nil { 72 panic(err) 73 } 74 } 75 case ".macho": 76 sample.MimeType = "application/x-mach-binary" 77 case ".elf": 78 sample.MimeType = "application/x-executable" 79 default: 80 // don't care about this 81 return 82 } 83 84 sample.ID, err = hash_file(name) 85 if err != nil { 86 panic(err) 87 } 88 89 // get the base filename 90 base_name := filepath.Base(name) 91 92 // split the base filename without its extension 93 filename_components := strings.Split(strings.TrimSuffix(base_name, filepath.Ext(base_name)), "-") 94 // now, parse the filename (these must be correctly named!) 95 sample.Program = filename_components[0] 96 sample.Version = filename_components[1] 97 var build uint64 98 build, err = strconv.ParseUint(filename_components[2], 0, 64) 99 if err != nil { 100 panic(err) 101 } 102 sample.Build = uint32(build) 103 sample.OS = filename_components[3] 104 sample.Arch = filename_components[4] 105 106 // now, create various mirrors 107 for _, direct_mirror := range params.DirectMirrors { 108 sample.Mirrors = append(sample.Mirrors, db.SampleMirror{ 109 Kind: db.MirrorDirect, 110 URL: direct_mirror + relative_name, 111 }) 112 } 113 for _, ipfs_gateway := range params.IPFSGateways { 114 sample.Mirrors = append(sample.Mirrors, db.SampleMirror{ 115 Kind: db.MirrorIPFS, 116 URL: ipfs_gateway + "/" + sample_database.ipfs_tree_cid + relative_name, 117 }) 118 } 119 120 // now write the sample 121 122 if err = sample_database.add(sample); err != nil { 123 return 124 } 125 return 126 } 127 128 func (sample_database *sample_database) make_tree(params *MakeSampleDatabaseParams, name, relative_name string) (err error) { 129 var ( 130 tree_entries []os.DirEntry 131 ) 132 133 tree_entries, err = os.ReadDir(name) 134 if err != nil { 135 return 136 } 137 138 for _, tree_entry := range tree_entries { 139 if tree_entry.IsDir() { 140 if err = sample_database.make_tree(params, name+"/"+tree_entry.Name(), relative_name+"/"+tree_entry.Name()); err != nil { 141 return 142 } 143 } else { 144 if err = sample_database.make_sample_file(params, name+"/"+tree_entry.Name(), relative_name+"/"+tree_entry.Name()); err != nil { 145 return 146 } 147 } 148 } 149 150 return 151 } 152 153 func ipfs_generate_file_cid(name string) (cid string, err error) { 154 155 // todo 156 // use command: 157 // ipfs add -qr --only-hash . 158 // inside the root of the sample tree 159 // the last CID is the root of the tree 160 161 var ( 162 wd string 163 ) 164 wd, err = os.Getwd() 165 if err != nil { 166 return 167 } 168 err = os.Chdir(name) 169 if err != nil { 170 return 171 } 172 173 command := exec.Command("ipfs", "add", "-qr", "--only-hash", ".") 174 var command_output bytes.Buffer 175 command.Stdout = &command_output 176 command.Run() 177 if command.ProcessState.ExitCode() != 0 { 178 os.Chdir(wd) 179 err = fmt.Errorf("util: ipfs tool exited: %d", command.ProcessState.ExitCode()) 180 return 181 } 182 183 // Parse command Output 184 command_output_scanner := bufio.NewScanner(&command_output) 185 186 for command_output_scanner.Scan() { 187 cid = command_output_scanner.Text() 188 } 189 190 err = os.Chdir(wd) 191 192 return 193 194 } 195 196 func MakeSampleDatabase(params *MakeSampleDatabaseParams) { 197 var ( 198 err error 199 sample_database sample_database 200 ) 201 202 // if we want to generate IPFS links, start by getting the CID for the sample tree 203 if len(params.IPFSGateways) != 0 { 204 sample_database.ipfs_tree_cid, err = ipfs_generate_file_cid(params.Source) 205 if err != nil { 206 app.Fatal(err) 207 return 208 } 209 } 210 211 sample_database.writer, err = dbutil.Open[db.Sample](params.Output, params.Format) 212 if err != nil { 213 app.Fatal(err) 214 } 215 216 // make the root tree, with our params, the source as the first tree, and "" (root) as the relative path 217 if err = sample_database.make_tree(params, params.Source, ""); err != nil { 218 app.Fatal(err) 219 } 220 221 if err = sample_database.Close(); err != nil { 222 app.Fatal(err) 223 } 224 }