aboutsummaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
Diffstat (limited to 'src')
-rw-r--r--src/tools/mac/upload_system_symbols/arch_constants.h47
-rw-r--r--src/tools/mac/upload_system_symbols/arch_reader.go268
-rw-r--r--src/tools/mac/upload_system_symbols/arch_reader_test.go82
-rw-r--r--src/tools/mac/upload_system_symbols/testdata/Makefile22
-rw-r--r--src/tools/mac/upload_system_symbols/testdata/archtest.c7
-rw-r--r--src/tools/mac/upload_system_symbols/upload_system_symbols.go389
6 files changed, 815 insertions, 0 deletions
diff --git a/src/tools/mac/upload_system_symbols/arch_constants.h b/src/tools/mac/upload_system_symbols/arch_constants.h
new file mode 100644
index 00000000..ed318b24
--- /dev/null
+++ b/src/tools/mac/upload_system_symbols/arch_constants.h
@@ -0,0 +1,47 @@
+/* Copyright 2014, Google Inc.
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+ * Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+ * Redistributions in binary form must reproduce the above
+copyright notice, this list of conditions and the following disclaimer
+in the documentation and/or other materials provided with the
+distribution.
+ * Neither the name of Google Inc. nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*/
+
+#include <mach-o/fat.h>
+#include <mach-o/loader.h>
+
+// Go/Cgo does not support #define constants, so turn them into symbols
+// that are reachable from Go.
+
+const cpu_type_t kCPUType_i386 = CPU_TYPE_I386;
+const cpu_type_t kCPUType_x86_64 = CPU_TYPE_X86_64;
+
+const uint32_t kMachHeaderMagic32 = MH_MAGIC;
+const uint32_t kMachHeaderMagic64 = MH_MAGIC_64;
+const uint32_t kMachHeaderMagicFat = FAT_MAGIC;
+const uint32_t kMachHeaderCigamFat = FAT_CIGAM;
+
+const uint32_t kMachHeaderFtypeDylib = MH_DYLIB;
+const uint32_t kMachHeaderFtypeBundle = MH_BUNDLE;
+const uint32_t kMachHeaderFtypeExe = MH_EXECUTE;
diff --git a/src/tools/mac/upload_system_symbols/arch_reader.go b/src/tools/mac/upload_system_symbols/arch_reader.go
new file mode 100644
index 00000000..0468836e
--- /dev/null
+++ b/src/tools/mac/upload_system_symbols/arch_reader.go
@@ -0,0 +1,268 @@
+/* Copyright 2014, Google Inc.
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+ * Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+ * Redistributions in binary form must reproduce the above
+copyright notice, this list of conditions and the following disclaimer
+in the documentation and/or other materials provided with the
+distribution.
+ * Neither the name of Google Inc. nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*/
+
+package main
+
+import (
+ "encoding/binary"
+ "errors"
+ "fmt"
+ "os"
+ "reflect"
+ "unsafe"
+)
+
+/*
+#include <mach-o/fat.h>
+#include <mach-o/loader.h>
+#include <string.h>
+
+#include "arch_constants.h"
+*/
+import "C"
+
+var (
+ ErrNotMachO = errors.New("GetMachOImageInfo: file is not a supported Mach-O image")
+ ErrUnsupportedArch = errors.New("GetMachOImageInfo: unknown architecture detected")
+)
+
+const (
+ ArchI386 = "i386"
+ ArchX86_64 = "x86_64"
+)
+
+type MachOType int
+
+const (
+ MachODylib MachOType = C.kMachHeaderFtypeDylib
+ MachOBundle = C.kMachHeaderFtypeBundle
+ MachOExe = C.kMachHeaderFtypeExe
+)
+
+type ImageInfo struct {
+ Type MachOType
+ Arch string
+}
+
+// GetMachOImageInfo will read the file at filepath and determine if it is
+// Mach-O. If it is, it will return a slice of ImageInfo that describe the
+// images in the file (may be more than one if it is a fat image).
+func GetMachOImageInfo(filepath string) ([]ImageInfo, error) {
+ f, err := os.Open(filepath)
+ if err != nil {
+ return nil, err
+ }
+ defer f.Close()
+
+ // Read the magic number to determine the type of file this is.
+ var magic uint32
+ err = binary.Read(f, binary.LittleEndian, &magic)
+ if err != nil {
+ return nil, err
+ }
+
+ // Rewind the file since the magic number is a field in the header
+ // structs.
+ f.Seek(0, os.SEEK_SET)
+
+ switch magic {
+ case C.kMachHeaderMagic32:
+ return readThinHeader(f, C.kMachHeaderMagic32)
+ case C.kMachHeaderMagic64:
+ return readThinHeader(f, C.kMachHeaderMagic64)
+ case C.kMachHeaderCigamFat: // Fat header is big-endian but was read in little.
+ return readFatHeader(f)
+ }
+
+ return nil, ErrNotMachO
+}
+
+func readThinHeader(f *os.File, expectedMagic uint32) ([]ImageInfo, error) {
+ var (
+ magic, filetype uint32
+ cpu C.cpu_type_t
+ err error
+ )
+
+ if expectedMagic == C.kMachHeaderMagic32 {
+ magic, cpu, filetype, err = readThin32Header(f)
+ } else if expectedMagic == C.kMachHeaderMagic64 {
+ magic, cpu, filetype, err = readThin64Header(f)
+ } else {
+ panic(fmt.Sprintf("Unexpected magic %#x", magic))
+ }
+ if err != nil {
+ return nil, err
+ }
+
+ if magic != expectedMagic {
+ return nil, fmt.Errorf("readThinHeader: unexpected magic number %#x", magic)
+ }
+
+ arch := cpuTypeToArch(cpu)
+ if arch == "" {
+ return nil, ErrUnsupportedArch
+ }
+ return []ImageInfo{{MachOType(filetype), arch}}, nil
+}
+
+func readThin32Header(f *os.File) (uint32, C.cpu_type_t, uint32, error) {
+ var machHeader C.struct_mach_header
+ err := readStruct(f, binary.LittleEndian, unsafe.Pointer(&machHeader), C.struct_mach_header{})
+ if err != nil {
+ return 0, 0, 0, err
+ }
+ return uint32(machHeader.magic), machHeader.cputype, uint32(machHeader.filetype), nil
+}
+
+func readThin64Header(f *os.File) (uint32, C.cpu_type_t, uint32, error) {
+ var machHeader C.struct_mach_header_64
+ err := readStruct(f, binary.LittleEndian, unsafe.Pointer(&machHeader), C.struct_mach_header_64{})
+ if err != nil {
+ return 0, 0, 0, err
+ }
+ return uint32(machHeader.magic), machHeader.cputype, uint32(machHeader.filetype), nil
+}
+
+func readFatHeader(f *os.File) ([]ImageInfo, error) {
+ var fatHeader C.struct_fat_header
+ err := readStruct(f, binary.BigEndian, unsafe.Pointer(&fatHeader), C.struct_fat_header{})
+ if err != nil {
+ return nil, err
+ }
+
+ if fatHeader.magic != C.kMachHeaderMagicFat {
+ return nil, fmt.Errorf("readFatHeader: unexpected magic number %#x", fatHeader.magic)
+ }
+
+ // Read the fat_arch headers.
+ headers := make([]C.struct_fat_arch, fatHeader.nfat_arch)
+ for i := 0; i < int(fatHeader.nfat_arch); i++ {
+ var fatArch C.struct_fat_arch
+ err = readStruct(f, binary.BigEndian, unsafe.Pointer(&fatArch), C.struct_fat_arch{})
+ if err != nil {
+ return nil, fmt.Errorf("readFatHeader: %v", err)
+ }
+ headers[i] = fatArch
+ }
+
+ seenArches := make(map[string]int)
+
+ // Now go to each arch in the fat image and read its mach header.
+ infos := make([]ImageInfo, 0, len(headers))
+ for _, header := range headers {
+ f.Seek(int64(header.offset), os.SEEK_SET)
+
+ var thinarch []ImageInfo
+ var expectedArch string
+ switch header.cputype {
+ case C.kCPUType_i386:
+ thinarch, err = readThinHeader(f, C.kMachHeaderMagic32)
+ expectedArch = ArchI386
+ case C.kCPUType_x86_64:
+ thinarch, err = readThinHeader(f, C.kMachHeaderMagic64)
+ expectedArch = ArchX86_64
+ default:
+ err = ErrUnsupportedArch
+ }
+
+ if err != nil {
+ return nil, err
+ }
+ if thinarch[0].Arch != expectedArch {
+ return nil, fmt.Errorf("readFatHeader: expected arch %d, got %d", thinarch[0].Arch, expectedArch)
+ }
+
+ infos = append(infos, thinarch[0])
+ seenArches[thinarch[0].Arch]++
+ }
+
+ for arch, count := range seenArches {
+ if count != 1 {
+ return nil, fmt.Errorf("readFatHeader: duplicate arch %s detected", arch)
+ }
+ }
+
+ return infos, nil
+}
+
+// TODO(rsesek): Support more arches.
+func cpuTypeToArch(cpu C.cpu_type_t) string {
+ switch cpu {
+ case C.kCPUType_i386:
+ return ArchI386
+ case C.kCPUType_x86_64:
+ return ArchX86_64
+ default:
+ return ""
+ }
+}
+
+// readStruct is a incomplete version of binary.Read that uses unsafe pointers
+// to set values in unexported fields. From |f|, this will read the fields of
+// the |destType| template instance, in the specified byte |order|, and place
+// the resulting memory into |dest|.
+func readStruct(f *os.File, order binary.ByteOrder, dest unsafe.Pointer, destType interface{}) error {
+ rv := reflect.ValueOf(destType)
+ rt := rv.Type()
+ destPtr := uintptr(dest)
+
+ for i := 0; i < rv.NumField(); i++ {
+ field := rv.Field(i)
+ fieldType := rt.Field(i)
+
+ var vp unsafe.Pointer
+ var err error
+
+ switch field.Kind() {
+ case reflect.Int32:
+ var v int32
+ vp = unsafe.Pointer(&v)
+ err = binary.Read(f, order, &v)
+ case reflect.Uint32:
+ var v uint32
+ vp = unsafe.Pointer(&v)
+ err = binary.Read(f, order, &v)
+ default:
+ err = fmt.Errorf("readStruct: unsupported type %v", fieldType)
+ }
+
+ if err != nil {
+ return err
+ }
+
+ memcpy(destPtr+fieldType.Offset, vp, fieldType.Type.Size())
+ }
+ return nil
+}
+
+func memcpy(dest uintptr, value unsafe.Pointer, size uintptr) {
+ C.memcpy(unsafe.Pointer(dest), value, C.size_t(size))
+}
diff --git a/src/tools/mac/upload_system_symbols/arch_reader_test.go b/src/tools/mac/upload_system_symbols/arch_reader_test.go
new file mode 100644
index 00000000..5fa1d210
--- /dev/null
+++ b/src/tools/mac/upload_system_symbols/arch_reader_test.go
@@ -0,0 +1,82 @@
+/* Copyright 2014, Google Inc.
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+ * Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+ * Redistributions in binary form must reproduce the above
+copyright notice, this list of conditions and the following disclaimer
+in the documentation and/or other materials provided with the
+distribution.
+ * Neither the name of Google Inc. nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*/
+
+package main
+
+import (
+ "testing"
+)
+
+func TestFat(t *testing.T) {
+ tests := []struct {
+ file string
+ mht MachOType
+ arches []string
+ }{
+ {"testdata/libarchtest32.dylib", MachODylib, []string{ArchI386}},
+ {"testdata/libarchtest64.dylib", MachODylib, []string{ArchX86_64}},
+ {"testdata/libarchtest.dylib", MachODylib, []string{ArchI386, ArchX86_64}},
+ {"testdata/archtest32.exe", MachOExe, []string{ArchI386}},
+ {"testdata/archtest64.exe", MachOExe, []string{ArchX86_64}},
+ {"testdata/archtest.exe", MachOExe, []string{ArchI386, ArchX86_64}},
+ }
+
+ for _, e := range tests {
+ imageinfo, err := GetMachOImageInfo(e.file)
+ if err != nil {
+ t.Errorf("Unexpected error: %v", err)
+ }
+
+ expected := make(map[string]bool)
+ for _, arch := range e.arches {
+ expected[arch] = false
+ }
+
+ if len(imageinfo) != len(e.arches) {
+ t.Errorf("Wrong number of arches, got %d, expected %d", len(imageinfo), len(e.arches))
+ }
+
+ for _, ii := range imageinfo {
+ if ii.Type != e.mht {
+ t.Errorf("Wrong MachOType got %d, expected %d", ii.Type, e.mht)
+ }
+ if o, k := expected[ii.Arch]; o || !k {
+ t.Errorf("Unexpected architecture %q", ii.Arch)
+ }
+ expected[ii.Arch] = true
+ }
+
+ for k, v := range expected {
+ if !v {
+ t.Errorf("Did not get expected architecture %s", k)
+ }
+ }
+ }
+}
diff --git a/src/tools/mac/upload_system_symbols/testdata/Makefile b/src/tools/mac/upload_system_symbols/testdata/Makefile
new file mode 100644
index 00000000..cb481a3c
--- /dev/null
+++ b/src/tools/mac/upload_system_symbols/testdata/Makefile
@@ -0,0 +1,22 @@
+all: libarchtest.dylib archtest.exe
+
+archtest32.exe: archtest.c
+ clang -m32 $< -o $@
+
+archtest64.exe: archtest.c
+ clang -m64 $< -o $@
+
+archtest.exe: archtest32.exe archtest64.exe
+ lipo $^ -create -output $@
+
+libarchtest32.dylib: archtest.c
+ clang -m32 -dynamiclib $< -o $@
+
+libarchtest64.dylib: archtest.c
+ clang -m64 -dynamiclib $< -o $@
+
+libarchtest.dylib: libarchtest32.dylib libarchtest64.dylib
+ lipo $^ -create -output $@
+
+clean:
+ rm -f *.dylib *.exe
diff --git a/src/tools/mac/upload_system_symbols/testdata/archtest.c b/src/tools/mac/upload_system_symbols/testdata/archtest.c
new file mode 100644
index 00000000..0931fca1
--- /dev/null
+++ b/src/tools/mac/upload_system_symbols/testdata/archtest.c
@@ -0,0 +1,7 @@
+int TestLibUsefulFunction() {
+ return 42;
+}
+
+int main() {
+ return 0;
+}
diff --git a/src/tools/mac/upload_system_symbols/upload_system_symbols.go b/src/tools/mac/upload_system_symbols/upload_system_symbols.go
new file mode 100644
index 00000000..869a2773
--- /dev/null
+++ b/src/tools/mac/upload_system_symbols/upload_system_symbols.go
@@ -0,0 +1,389 @@
+/* Copyright 2014, Google Inc.
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+ * Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+ * Redistributions in binary form must reproduce the above
+copyright notice, this list of conditions and the following disclaimer
+in the documentation and/or other materials provided with the
+distribution.
+ * Neither the name of Google Inc. nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*/
+
+/*
+Tool upload_system_symbols generates and uploads Breakpad symbol files for OS X system libraries.
+
+This tool shells out to the dump_syms and symupload Breakpad tools. In its default mode, this
+will find all dynamic libraries on the system, run dump_syms to create the Breakpad symbol files,
+and then upload them to Google's crash infrastructure.
+
+The tool can also be used to only dump libraries or upload from a directory. See -help for more
+information.
+
+Both i386 and x86_64 architectures will be dumped and uploaded.
+*/
+package main
+
+import (
+ "flag"
+ "fmt"
+ "io"
+ "io/ioutil"
+ "log"
+ "os"
+ "os/exec"
+ "path"
+ "regexp"
+ "strings"
+ "sync"
+ "time"
+)
+
+var (
+ breakpadTools = flag.String("breakpad-tools", "out/Release/", "Path to the Breakpad tools directory, containing dump_syms and symupload.")
+ uploadOnlyPath = flag.String("upload-from", "", "Upload a directory of symbol files that has been dumped independently.")
+ dumpOnlyPath = flag.String("dump-to", "", "Dump the symbols to the specified directory, but do not upload them.")
+ systemRoot = flag.String("system-root", "", "Path to the root of the Mac OS X system whose symbols will be dumped.")
+)
+
+var (
+ // pathsToScan are the subpaths in the systemRoot that should be scanned for shared libraries.
+ pathsToScan = []string{
+ "/Library/QuickTime",
+ "/System/Library/Components",
+ "/System/Library/Frameworks",
+ "/System/Library/PrivateFrameworks",
+ "/usr/lib",
+ }
+
+ // uploadServers are the list of servers to which symbols should be uploaded.
+ uploadServers = []string{
+ "https://clients2.google.com/cr/symbol",
+ "https://clients2.google.com/cr/staging_symbol",
+ }
+
+ // blacklistRegexps match paths that should be excluded from dumping.
+ blacklistRegexps = []*regexp.Regexp{
+ regexp.MustCompile(`/System/Library/Frameworks/Python\.framework/`),
+ regexp.MustCompile(`/System/Library/Frameworks/Ruby\.framework/`),
+ regexp.MustCompile(`_profile\.dylib$`),
+ regexp.MustCompile(`_debug\.dylib$`),
+ regexp.MustCompile(`\.a$`),
+ regexp.MustCompile(`\.dat$`),
+ }
+)
+
+func main() {
+ flag.Parse()
+ log.SetFlags(0)
+
+ var uq *UploadQueue
+
+ if *uploadOnlyPath != "" {
+ // -upload-from specified, so handle that case early.
+ uq = StartUploadQueue()
+ uploadFromDirectory(*uploadOnlyPath, uq)
+ uq.Wait()
+ return
+ }
+
+ if *systemRoot == "" {
+ log.Fatal("Need a -system-root to dump symbols for")
+ }
+
+ if *dumpOnlyPath != "" {
+ // -dump-to specified, so make sure that the path is a directory.
+ if fi, err := os.Stat(*dumpOnlyPath); err != nil {
+ log.Fatal("-dump-to location: %v", err)
+ } else if !fi.IsDir() {
+ log.Fatal("-dump-to location is not a directory")
+ }
+ }
+
+ dumpPath := *dumpOnlyPath
+ if *dumpOnlyPath == "" {
+ // If -dump-to was not specified, then run the upload pipeline and create
+ // a temporary dump output directory.
+ uq = StartUploadQueue()
+
+ if p, err := ioutil.TempDir("", "upload_system_symbols"); err != nil {
+ log.Fatal("Failed to create temporary directory: %v", err)
+ } else {
+ dumpPath = p
+ defer os.RemoveAll(p)
+ }
+ }
+
+ dq := StartDumpQueue(*systemRoot, dumpPath, uq)
+ dq.Wait()
+ if uq != nil {
+ uq.Wait()
+ }
+}
+
+type WorkerPool struct {
+ wg sync.WaitGroup
+}
+
+// StartWorkerPool will launch numWorkers goroutines all running workerFunc.
+// When workerFunc exits, the goroutine will terminate.
+func StartWorkerPool(numWorkers int, workerFunc func()) *WorkerPool {
+ p := new(WorkerPool)
+ for i := 0; i < numWorkers; i++ {
+ p.wg.Add(1)
+ go func() {
+ workerFunc()
+ p.wg.Done()
+ }()
+ }
+ return p
+}
+
+// Wait for all the workers in the pool to complete the workerFunc.
+func (p *WorkerPool) Wait() {
+ p.wg.Wait()
+}
+
+type UploadQueue struct {
+ *WorkerPool
+ queue chan string
+}
+
+// StartUploadQueue creates a new worker pool and queue, to which paths to
+// Breakpad symbol files may be sent for uploading.
+func StartUploadQueue() *UploadQueue {
+ uq := &UploadQueue{
+ queue: make(chan string, 10),
+ }
+ uq.WorkerPool = StartWorkerPool(5, uq.worker)
+ return uq
+}
+
+// Upload enqueues the contents of filepath to be uploaded.
+func (uq *UploadQueue) Upload(filepath string) {
+ uq.queue <- filepath
+}
+
+// Done tells the queue that no more files need to be uploaded. This must be
+// called before WorkerPool.Wait.
+func (uq *UploadQueue) Done() {
+ close(uq.queue)
+}
+
+func (uq *UploadQueue) worker() {
+ symUpload := path.Join(*breakpadTools, "symupload")
+
+ for symfile := range uq.queue {
+ for _, server := range uploadServers {
+ for i := 0; i < 3; i++ { // Give each upload 3 attempts to succeed.
+ cmd := exec.Command(symUpload, symfile, server)
+ if output, err := cmd.Output(); err == nil {
+ // Success. No retry needed.
+ fmt.Printf("Uploaded %s to %s\n", symfile, server)
+ break
+ } else {
+ log.Printf("Error running symupload(%s, %s), attempt %d: %v: %s\n", symfile, server, i, err, output)
+ time.Sleep(1 * time.Second)
+ }
+ }
+ }
+ }
+}
+
+type DumpQueue struct {
+ *WorkerPool
+ dumpPath string
+ queue chan dumpRequest
+ uq *UploadQueue
+}
+
+type dumpRequest struct {
+ path string
+ arch string
+}
+
+// StartDumpQueue creates a new worker pool to find all the Mach-O libraries in
+// root and dump their symbols to dumpPath. If an UploadQueue is passed, the
+// path to the symbol file will be enqueued there, too.
+func StartDumpQueue(root, dumpPath string, uq *UploadQueue) *DumpQueue {
+ dq := &DumpQueue{
+ dumpPath: dumpPath,
+ queue: make(chan dumpRequest),
+ uq: uq,
+ }
+ dq.WorkerPool = StartWorkerPool(12, dq.worker)
+
+ findLibsInRoot(root, dq)
+
+ return dq
+}
+
+// DumpSymbols enqueues the filepath to have its symbols dumped in the specified
+// architecture.
+func (dq *DumpQueue) DumpSymbols(filepath string, arch string) {
+ dq.queue <- dumpRequest{
+ path: filepath,
+ arch: arch,
+ }
+}
+
+func (dq *DumpQueue) Wait() {
+ dq.WorkerPool.Wait()
+ if dq.uq != nil {
+ dq.uq.Done()
+ }
+}
+
+func (dq *DumpQueue) done() {
+ close(dq.queue)
+}
+
+func (dq *DumpQueue) worker() {
+ dumpSyms := path.Join(*breakpadTools, "dump_syms")
+
+ for req := range dq.queue {
+ filebase := path.Join(dq.dumpPath, strings.Replace(req.path, "/", "_", -1))
+ symfile := fmt.Sprintf("%s_%s.sym", filebase, req.arch)
+ f, err := os.Create(symfile)
+ if err != nil {
+ log.Fatal("Error creating symbol file:", err)
+ }
+
+ cmd := exec.Command(dumpSyms, "-a", req.arch, req.path)
+ cmd.Stdout = f
+ err = cmd.Run()
+ f.Close()
+
+ if err != nil {
+ os.Remove(symfile)
+ log.Printf("Error running dump_syms(%s, %s): %v\n", req.arch, req.path, err)
+ } else if dq.uq != nil {
+ dq.uq.Upload(symfile)
+ }
+ }
+}
+
+// uploadFromDirectory handles the upload-only case and merely uploads all files in
+// a directory.
+func uploadFromDirectory(directory string, uq *UploadQueue) {
+ d, err := os.Open(directory)
+ if err != nil {
+ log.Fatal("Could not open directory to upload: %v", err)
+ }
+ defer d.Close()
+
+ entries, err := d.Readdirnames(0)
+ if err != nil {
+ log.Fatal("Could not read directory: %v", err)
+ }
+
+ for _, entry := range entries {
+ uq.Upload(path.Join(directory, entry))
+ }
+
+ uq.Done()
+}
+
+// findQueue is an implementation detail of the DumpQueue that finds all the
+// Mach-O files and their architectures.
+type findQueue struct {
+ *WorkerPool
+ queue chan string
+ dq *DumpQueue
+}
+
+// findLibsInRoot looks in all the pathsToScan in the root and manages the
+// interaction between findQueue and DumpQueue.
+func findLibsInRoot(root string, dq *DumpQueue) {
+ fq := &findQueue{
+ queue: make(chan string, 10),
+ dq: dq,
+ }
+ fq.WorkerPool = StartWorkerPool(12, fq.worker)
+
+ for _, p := range pathsToScan {
+ fq.findLibsInPath(path.Join(root, p))
+ }
+
+ close(fq.queue)
+ fq.Wait()
+ dq.done()
+}
+
+// findLibsInPath recursively walks the directory tree, sending file paths to
+// test for being Mach-O to the findQueue.
+func (fq *findQueue) findLibsInPath(loc string) {
+ d, err := os.Open(loc)
+ if err != nil {
+ log.Fatal("Could not open %s: %v", loc, err)
+ }
+ defer d.Close()
+
+ for {
+ fis, err := d.Readdir(100)
+ if err != nil && err != io.EOF {
+ log.Fatal("Error reading directory %s: %v", loc, err)
+ }
+
+ for _, fi := range fis {
+ fp := path.Join(loc, fi.Name())
+ if fi.IsDir() {
+ fq.findLibsInPath(fp)
+ continue
+ } else if fi.Mode()&os.ModeSymlink != 0 {
+ continue
+ }
+
+ // Test the blacklist in the worker to not slow down this main loop.
+
+ fq.queue <- fp
+ }
+
+ if err == io.EOF {
+ break
+ }
+ }
+}
+
+func (fq *findQueue) worker() {
+ for fp := range fq.queue {
+ blacklisted := false
+ for _, re := range blacklistRegexps {
+ blacklisted = blacklisted || re.MatchString(fp)
+ }
+ if blacklisted {
+ continue
+ }
+
+ imageinfos, err := GetMachOImageInfo(fp)
+ if err != nil && err != ErrNotMachO {
+ log.Printf("%s: %v", fp, err)
+ continue
+ }
+
+ for _, imageinfo := range imageinfos {
+ if imageinfo.Type == MachODylib || imageinfo.Type == MachOBundle {
+
+ fq.dq.DumpSymbols(fp, imageinfo.Arch)
+ }
+ }
+ }
+}