From 299683dba6b4abd4f7ceca28a2ed8005081ed4ad Mon Sep 17 00:00:00 2001 From: "rsesek@chromium.org" Date: Mon, 3 Feb 2014 22:52:49 +0000 Subject: Create a new tool to upload Mac system library symbols. R=andybons@chromium.org, mark@chromium.org Review URL: https://breakpad.appspot.com/1124002 git-svn-id: http://google-breakpad.googlecode.com/svn/trunk@1278 4c0a9323-5329-0410-9bdc-e9ce6186880e --- .../mac/upload_system_symbols/arch_constants.h | 47 +++ src/tools/mac/upload_system_symbols/arch_reader.go | 268 ++++++++++++++ .../mac/upload_system_symbols/arch_reader_test.go | 82 +++++ .../mac/upload_system_symbols/testdata/Makefile | 22 ++ .../mac/upload_system_symbols/testdata/archtest.c | 7 + .../upload_system_symbols/upload_system_symbols.go | 389 +++++++++++++++++++++ 6 files changed, 815 insertions(+) create mode 100644 src/tools/mac/upload_system_symbols/arch_constants.h create mode 100644 src/tools/mac/upload_system_symbols/arch_reader.go create mode 100644 src/tools/mac/upload_system_symbols/arch_reader_test.go create mode 100644 src/tools/mac/upload_system_symbols/testdata/Makefile create mode 100644 src/tools/mac/upload_system_symbols/testdata/archtest.c create mode 100644 src/tools/mac/upload_system_symbols/upload_system_symbols.go (limited to 'src/tools') diff --git a/src/tools/mac/upload_system_symbols/arch_constants.h b/src/tools/mac/upload_system_symbols/arch_constants.h new file mode 100644 index 00000000..ed318b24 --- /dev/null +++ b/src/tools/mac/upload_system_symbols/arch_constants.h @@ -0,0 +1,47 @@ +/* Copyright 2014, Google Inc. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#include +#include + +// Go/Cgo does not support #define constants, so turn them into symbols +// that are reachable from Go. + +const cpu_type_t kCPUType_i386 = CPU_TYPE_I386; +const cpu_type_t kCPUType_x86_64 = CPU_TYPE_X86_64; + +const uint32_t kMachHeaderMagic32 = MH_MAGIC; +const uint32_t kMachHeaderMagic64 = MH_MAGIC_64; +const uint32_t kMachHeaderMagicFat = FAT_MAGIC; +const uint32_t kMachHeaderCigamFat = FAT_CIGAM; + +const uint32_t kMachHeaderFtypeDylib = MH_DYLIB; +const uint32_t kMachHeaderFtypeBundle = MH_BUNDLE; +const uint32_t kMachHeaderFtypeExe = MH_EXECUTE; diff --git a/src/tools/mac/upload_system_symbols/arch_reader.go b/src/tools/mac/upload_system_symbols/arch_reader.go new file mode 100644 index 00000000..0468836e --- /dev/null +++ b/src/tools/mac/upload_system_symbols/arch_reader.go @@ -0,0 +1,268 @@ +/* Copyright 2014, Google Inc. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +package main + +import ( + "encoding/binary" + "errors" + "fmt" + "os" + "reflect" + "unsafe" +) + +/* +#include +#include +#include + +#include "arch_constants.h" +*/ +import "C" + +var ( + ErrNotMachO = errors.New("GetMachOImageInfo: file is not a supported Mach-O image") + ErrUnsupportedArch = errors.New("GetMachOImageInfo: unknown architecture detected") +) + +const ( + ArchI386 = "i386" + ArchX86_64 = "x86_64" +) + +type MachOType int + +const ( + MachODylib MachOType = C.kMachHeaderFtypeDylib + MachOBundle = C.kMachHeaderFtypeBundle + MachOExe = C.kMachHeaderFtypeExe +) + +type ImageInfo struct { + Type MachOType + Arch string +} + +// GetMachOImageInfo will read the file at filepath and determine if it is +// Mach-O. If it is, it will return a slice of ImageInfo that describe the +// images in the file (may be more than one if it is a fat image). +func GetMachOImageInfo(filepath string) ([]ImageInfo, error) { + f, err := os.Open(filepath) + if err != nil { + return nil, err + } + defer f.Close() + + // Read the magic number to determine the type of file this is. + var magic uint32 + err = binary.Read(f, binary.LittleEndian, &magic) + if err != nil { + return nil, err + } + + // Rewind the file since the magic number is a field in the header + // structs. + f.Seek(0, os.SEEK_SET) + + switch magic { + case C.kMachHeaderMagic32: + return readThinHeader(f, C.kMachHeaderMagic32) + case C.kMachHeaderMagic64: + return readThinHeader(f, C.kMachHeaderMagic64) + case C.kMachHeaderCigamFat: // Fat header is big-endian but was read in little. + return readFatHeader(f) + } + + return nil, ErrNotMachO +} + +func readThinHeader(f *os.File, expectedMagic uint32) ([]ImageInfo, error) { + var ( + magic, filetype uint32 + cpu C.cpu_type_t + err error + ) + + if expectedMagic == C.kMachHeaderMagic32 { + magic, cpu, filetype, err = readThin32Header(f) + } else if expectedMagic == C.kMachHeaderMagic64 { + magic, cpu, filetype, err = readThin64Header(f) + } else { + panic(fmt.Sprintf("Unexpected magic %#x", magic)) + } + if err != nil { + return nil, err + } + + if magic != expectedMagic { + return nil, fmt.Errorf("readThinHeader: unexpected magic number %#x", magic) + } + + arch := cpuTypeToArch(cpu) + if arch == "" { + return nil, ErrUnsupportedArch + } + return []ImageInfo{{MachOType(filetype), arch}}, nil +} + +func readThin32Header(f *os.File) (uint32, C.cpu_type_t, uint32, error) { + var machHeader C.struct_mach_header + err := readStruct(f, binary.LittleEndian, unsafe.Pointer(&machHeader), C.struct_mach_header{}) + if err != nil { + return 0, 0, 0, err + } + return uint32(machHeader.magic), machHeader.cputype, uint32(machHeader.filetype), nil +} + +func readThin64Header(f *os.File) (uint32, C.cpu_type_t, uint32, error) { + var machHeader C.struct_mach_header_64 + err := readStruct(f, binary.LittleEndian, unsafe.Pointer(&machHeader), C.struct_mach_header_64{}) + if err != nil { + return 0, 0, 0, err + } + return uint32(machHeader.magic), machHeader.cputype, uint32(machHeader.filetype), nil +} + +func readFatHeader(f *os.File) ([]ImageInfo, error) { + var fatHeader C.struct_fat_header + err := readStruct(f, binary.BigEndian, unsafe.Pointer(&fatHeader), C.struct_fat_header{}) + if err != nil { + return nil, err + } + + if fatHeader.magic != C.kMachHeaderMagicFat { + return nil, fmt.Errorf("readFatHeader: unexpected magic number %#x", fatHeader.magic) + } + + // Read the fat_arch headers. + headers := make([]C.struct_fat_arch, fatHeader.nfat_arch) + for i := 0; i < int(fatHeader.nfat_arch); i++ { + var fatArch C.struct_fat_arch + err = readStruct(f, binary.BigEndian, unsafe.Pointer(&fatArch), C.struct_fat_arch{}) + if err != nil { + return nil, fmt.Errorf("readFatHeader: %v", err) + } + headers[i] = fatArch + } + + seenArches := make(map[string]int) + + // Now go to each arch in the fat image and read its mach header. + infos := make([]ImageInfo, 0, len(headers)) + for _, header := range headers { + f.Seek(int64(header.offset), os.SEEK_SET) + + var thinarch []ImageInfo + var expectedArch string + switch header.cputype { + case C.kCPUType_i386: + thinarch, err = readThinHeader(f, C.kMachHeaderMagic32) + expectedArch = ArchI386 + case C.kCPUType_x86_64: + thinarch, err = readThinHeader(f, C.kMachHeaderMagic64) + expectedArch = ArchX86_64 + default: + err = ErrUnsupportedArch + } + + if err != nil { + return nil, err + } + if thinarch[0].Arch != expectedArch { + return nil, fmt.Errorf("readFatHeader: expected arch %d, got %d", thinarch[0].Arch, expectedArch) + } + + infos = append(infos, thinarch[0]) + seenArches[thinarch[0].Arch]++ + } + + for arch, count := range seenArches { + if count != 1 { + return nil, fmt.Errorf("readFatHeader: duplicate arch %s detected", arch) + } + } + + return infos, nil +} + +// TODO(rsesek): Support more arches. +func cpuTypeToArch(cpu C.cpu_type_t) string { + switch cpu { + case C.kCPUType_i386: + return ArchI386 + case C.kCPUType_x86_64: + return ArchX86_64 + default: + return "" + } +} + +// readStruct is a incomplete version of binary.Read that uses unsafe pointers +// to set values in unexported fields. From |f|, this will read the fields of +// the |destType| template instance, in the specified byte |order|, and place +// the resulting memory into |dest|. +func readStruct(f *os.File, order binary.ByteOrder, dest unsafe.Pointer, destType interface{}) error { + rv := reflect.ValueOf(destType) + rt := rv.Type() + destPtr := uintptr(dest) + + for i := 0; i < rv.NumField(); i++ { + field := rv.Field(i) + fieldType := rt.Field(i) + + var vp unsafe.Pointer + var err error + + switch field.Kind() { + case reflect.Int32: + var v int32 + vp = unsafe.Pointer(&v) + err = binary.Read(f, order, &v) + case reflect.Uint32: + var v uint32 + vp = unsafe.Pointer(&v) + err = binary.Read(f, order, &v) + default: + err = fmt.Errorf("readStruct: unsupported type %v", fieldType) + } + + if err != nil { + return err + } + + memcpy(destPtr+fieldType.Offset, vp, fieldType.Type.Size()) + } + return nil +} + +func memcpy(dest uintptr, value unsafe.Pointer, size uintptr) { + C.memcpy(unsafe.Pointer(dest), value, C.size_t(size)) +} diff --git a/src/tools/mac/upload_system_symbols/arch_reader_test.go b/src/tools/mac/upload_system_symbols/arch_reader_test.go new file mode 100644 index 00000000..5fa1d210 --- /dev/null +++ b/src/tools/mac/upload_system_symbols/arch_reader_test.go @@ -0,0 +1,82 @@ +/* Copyright 2014, Google Inc. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +package main + +import ( + "testing" +) + +func TestFat(t *testing.T) { + tests := []struct { + file string + mht MachOType + arches []string + }{ + {"testdata/libarchtest32.dylib", MachODylib, []string{ArchI386}}, + {"testdata/libarchtest64.dylib", MachODylib, []string{ArchX86_64}}, + {"testdata/libarchtest.dylib", MachODylib, []string{ArchI386, ArchX86_64}}, + {"testdata/archtest32.exe", MachOExe, []string{ArchI386}}, + {"testdata/archtest64.exe", MachOExe, []string{ArchX86_64}}, + {"testdata/archtest.exe", MachOExe, []string{ArchI386, ArchX86_64}}, + } + + for _, e := range tests { + imageinfo, err := GetMachOImageInfo(e.file) + if err != nil { + t.Errorf("Unexpected error: %v", err) + } + + expected := make(map[string]bool) + for _, arch := range e.arches { + expected[arch] = false + } + + if len(imageinfo) != len(e.arches) { + t.Errorf("Wrong number of arches, got %d, expected %d", len(imageinfo), len(e.arches)) + } + + for _, ii := range imageinfo { + if ii.Type != e.mht { + t.Errorf("Wrong MachOType got %d, expected %d", ii.Type, e.mht) + } + if o, k := expected[ii.Arch]; o || !k { + t.Errorf("Unexpected architecture %q", ii.Arch) + } + expected[ii.Arch] = true + } + + for k, v := range expected { + if !v { + t.Errorf("Did not get expected architecture %s", k) + } + } + } +} diff --git a/src/tools/mac/upload_system_symbols/testdata/Makefile b/src/tools/mac/upload_system_symbols/testdata/Makefile new file mode 100644 index 00000000..cb481a3c --- /dev/null +++ b/src/tools/mac/upload_system_symbols/testdata/Makefile @@ -0,0 +1,22 @@ +all: libarchtest.dylib archtest.exe + +archtest32.exe: archtest.c + clang -m32 $< -o $@ + +archtest64.exe: archtest.c + clang -m64 $< -o $@ + +archtest.exe: archtest32.exe archtest64.exe + lipo $^ -create -output $@ + +libarchtest32.dylib: archtest.c + clang -m32 -dynamiclib $< -o $@ + +libarchtest64.dylib: archtest.c + clang -m64 -dynamiclib $< -o $@ + +libarchtest.dylib: libarchtest32.dylib libarchtest64.dylib + lipo $^ -create -output $@ + +clean: + rm -f *.dylib *.exe diff --git a/src/tools/mac/upload_system_symbols/testdata/archtest.c b/src/tools/mac/upload_system_symbols/testdata/archtest.c new file mode 100644 index 00000000..0931fca1 --- /dev/null +++ b/src/tools/mac/upload_system_symbols/testdata/archtest.c @@ -0,0 +1,7 @@ +int TestLibUsefulFunction() { + return 42; +} + +int main() { + return 0; +} diff --git a/src/tools/mac/upload_system_symbols/upload_system_symbols.go b/src/tools/mac/upload_system_symbols/upload_system_symbols.go new file mode 100644 index 00000000..869a2773 --- /dev/null +++ b/src/tools/mac/upload_system_symbols/upload_system_symbols.go @@ -0,0 +1,389 @@ +/* Copyright 2014, Google Inc. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +/* +Tool upload_system_symbols generates and uploads Breakpad symbol files for OS X system libraries. + +This tool shells out to the dump_syms and symupload Breakpad tools. In its default mode, this +will find all dynamic libraries on the system, run dump_syms to create the Breakpad symbol files, +and then upload them to Google's crash infrastructure. + +The tool can also be used to only dump libraries or upload from a directory. See -help for more +information. + +Both i386 and x86_64 architectures will be dumped and uploaded. +*/ +package main + +import ( + "flag" + "fmt" + "io" + "io/ioutil" + "log" + "os" + "os/exec" + "path" + "regexp" + "strings" + "sync" + "time" +) + +var ( + breakpadTools = flag.String("breakpad-tools", "out/Release/", "Path to the Breakpad tools directory, containing dump_syms and symupload.") + uploadOnlyPath = flag.String("upload-from", "", "Upload a directory of symbol files that has been dumped independently.") + dumpOnlyPath = flag.String("dump-to", "", "Dump the symbols to the specified directory, but do not upload them.") + systemRoot = flag.String("system-root", "", "Path to the root of the Mac OS X system whose symbols will be dumped.") +) + +var ( + // pathsToScan are the subpaths in the systemRoot that should be scanned for shared libraries. + pathsToScan = []string{ + "/Library/QuickTime", + "/System/Library/Components", + "/System/Library/Frameworks", + "/System/Library/PrivateFrameworks", + "/usr/lib", + } + + // uploadServers are the list of servers to which symbols should be uploaded. + uploadServers = []string{ + "https://clients2.google.com/cr/symbol", + "https://clients2.google.com/cr/staging_symbol", + } + + // blacklistRegexps match paths that should be excluded from dumping. + blacklistRegexps = []*regexp.Regexp{ + regexp.MustCompile(`/System/Library/Frameworks/Python\.framework/`), + regexp.MustCompile(`/System/Library/Frameworks/Ruby\.framework/`), + regexp.MustCompile(`_profile\.dylib$`), + regexp.MustCompile(`_debug\.dylib$`), + regexp.MustCompile(`\.a$`), + regexp.MustCompile(`\.dat$`), + } +) + +func main() { + flag.Parse() + log.SetFlags(0) + + var uq *UploadQueue + + if *uploadOnlyPath != "" { + // -upload-from specified, so handle that case early. + uq = StartUploadQueue() + uploadFromDirectory(*uploadOnlyPath, uq) + uq.Wait() + return + } + + if *systemRoot == "" { + log.Fatal("Need a -system-root to dump symbols for") + } + + if *dumpOnlyPath != "" { + // -dump-to specified, so make sure that the path is a directory. + if fi, err := os.Stat(*dumpOnlyPath); err != nil { + log.Fatal("-dump-to location: %v", err) + } else if !fi.IsDir() { + log.Fatal("-dump-to location is not a directory") + } + } + + dumpPath := *dumpOnlyPath + if *dumpOnlyPath == "" { + // If -dump-to was not specified, then run the upload pipeline and create + // a temporary dump output directory. + uq = StartUploadQueue() + + if p, err := ioutil.TempDir("", "upload_system_symbols"); err != nil { + log.Fatal("Failed to create temporary directory: %v", err) + } else { + dumpPath = p + defer os.RemoveAll(p) + } + } + + dq := StartDumpQueue(*systemRoot, dumpPath, uq) + dq.Wait() + if uq != nil { + uq.Wait() + } +} + +type WorkerPool struct { + wg sync.WaitGroup +} + +// StartWorkerPool will launch numWorkers goroutines all running workerFunc. +// When workerFunc exits, the goroutine will terminate. +func StartWorkerPool(numWorkers int, workerFunc func()) *WorkerPool { + p := new(WorkerPool) + for i := 0; i < numWorkers; i++ { + p.wg.Add(1) + go func() { + workerFunc() + p.wg.Done() + }() + } + return p +} + +// Wait for all the workers in the pool to complete the workerFunc. +func (p *WorkerPool) Wait() { + p.wg.Wait() +} + +type UploadQueue struct { + *WorkerPool + queue chan string +} + +// StartUploadQueue creates a new worker pool and queue, to which paths to +// Breakpad symbol files may be sent for uploading. +func StartUploadQueue() *UploadQueue { + uq := &UploadQueue{ + queue: make(chan string, 10), + } + uq.WorkerPool = StartWorkerPool(5, uq.worker) + return uq +} + +// Upload enqueues the contents of filepath to be uploaded. +func (uq *UploadQueue) Upload(filepath string) { + uq.queue <- filepath +} + +// Done tells the queue that no more files need to be uploaded. This must be +// called before WorkerPool.Wait. +func (uq *UploadQueue) Done() { + close(uq.queue) +} + +func (uq *UploadQueue) worker() { + symUpload := path.Join(*breakpadTools, "symupload") + + for symfile := range uq.queue { + for _, server := range uploadServers { + for i := 0; i < 3; i++ { // Give each upload 3 attempts to succeed. + cmd := exec.Command(symUpload, symfile, server) + if output, err := cmd.Output(); err == nil { + // Success. No retry needed. + fmt.Printf("Uploaded %s to %s\n", symfile, server) + break + } else { + log.Printf("Error running symupload(%s, %s), attempt %d: %v: %s\n", symfile, server, i, err, output) + time.Sleep(1 * time.Second) + } + } + } + } +} + +type DumpQueue struct { + *WorkerPool + dumpPath string + queue chan dumpRequest + uq *UploadQueue +} + +type dumpRequest struct { + path string + arch string +} + +// StartDumpQueue creates a new worker pool to find all the Mach-O libraries in +// root and dump their symbols to dumpPath. If an UploadQueue is passed, the +// path to the symbol file will be enqueued there, too. +func StartDumpQueue(root, dumpPath string, uq *UploadQueue) *DumpQueue { + dq := &DumpQueue{ + dumpPath: dumpPath, + queue: make(chan dumpRequest), + uq: uq, + } + dq.WorkerPool = StartWorkerPool(12, dq.worker) + + findLibsInRoot(root, dq) + + return dq +} + +// DumpSymbols enqueues the filepath to have its symbols dumped in the specified +// architecture. +func (dq *DumpQueue) DumpSymbols(filepath string, arch string) { + dq.queue <- dumpRequest{ + path: filepath, + arch: arch, + } +} + +func (dq *DumpQueue) Wait() { + dq.WorkerPool.Wait() + if dq.uq != nil { + dq.uq.Done() + } +} + +func (dq *DumpQueue) done() { + close(dq.queue) +} + +func (dq *DumpQueue) worker() { + dumpSyms := path.Join(*breakpadTools, "dump_syms") + + for req := range dq.queue { + filebase := path.Join(dq.dumpPath, strings.Replace(req.path, "/", "_", -1)) + symfile := fmt.Sprintf("%s_%s.sym", filebase, req.arch) + f, err := os.Create(symfile) + if err != nil { + log.Fatal("Error creating symbol file:", err) + } + + cmd := exec.Command(dumpSyms, "-a", req.arch, req.path) + cmd.Stdout = f + err = cmd.Run() + f.Close() + + if err != nil { + os.Remove(symfile) + log.Printf("Error running dump_syms(%s, %s): %v\n", req.arch, req.path, err) + } else if dq.uq != nil { + dq.uq.Upload(symfile) + } + } +} + +// uploadFromDirectory handles the upload-only case and merely uploads all files in +// a directory. +func uploadFromDirectory(directory string, uq *UploadQueue) { + d, err := os.Open(directory) + if err != nil { + log.Fatal("Could not open directory to upload: %v", err) + } + defer d.Close() + + entries, err := d.Readdirnames(0) + if err != nil { + log.Fatal("Could not read directory: %v", err) + } + + for _, entry := range entries { + uq.Upload(path.Join(directory, entry)) + } + + uq.Done() +} + +// findQueue is an implementation detail of the DumpQueue that finds all the +// Mach-O files and their architectures. +type findQueue struct { + *WorkerPool + queue chan string + dq *DumpQueue +} + +// findLibsInRoot looks in all the pathsToScan in the root and manages the +// interaction between findQueue and DumpQueue. +func findLibsInRoot(root string, dq *DumpQueue) { + fq := &findQueue{ + queue: make(chan string, 10), + dq: dq, + } + fq.WorkerPool = StartWorkerPool(12, fq.worker) + + for _, p := range pathsToScan { + fq.findLibsInPath(path.Join(root, p)) + } + + close(fq.queue) + fq.Wait() + dq.done() +} + +// findLibsInPath recursively walks the directory tree, sending file paths to +// test for being Mach-O to the findQueue. +func (fq *findQueue) findLibsInPath(loc string) { + d, err := os.Open(loc) + if err != nil { + log.Fatal("Could not open %s: %v", loc, err) + } + defer d.Close() + + for { + fis, err := d.Readdir(100) + if err != nil && err != io.EOF { + log.Fatal("Error reading directory %s: %v", loc, err) + } + + for _, fi := range fis { + fp := path.Join(loc, fi.Name()) + if fi.IsDir() { + fq.findLibsInPath(fp) + continue + } else if fi.Mode()&os.ModeSymlink != 0 { + continue + } + + // Test the blacklist in the worker to not slow down this main loop. + + fq.queue <- fp + } + + if err == io.EOF { + break + } + } +} + +func (fq *findQueue) worker() { + for fp := range fq.queue { + blacklisted := false + for _, re := range blacklistRegexps { + blacklisted = blacklisted || re.MatchString(fp) + } + if blacklisted { + continue + } + + imageinfos, err := GetMachOImageInfo(fp) + if err != nil && err != ErrNotMachO { + log.Printf("%s: %v", fp, err) + continue + } + + for _, imageinfo := range imageinfos { + if imageinfo.Type == MachODylib || imageinfo.Type == MachOBundle { + + fq.dq.DumpSymbols(fp, imageinfo.Arch) + } + } + } +} -- cgit v1.2.1