aboutsummaryrefslogtreecommitdiff
path: root/src/tools/windows/converter_exe/converter.cc
blob: 06bd8ebb399497ab9ea27f48795e24ba760fd296 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
// Copyright 2019 Google Inc. All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
//     * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
//     * Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following disclaimer
// in the documentation and/or other materials provided with the
// distribution.
//     * Neither the name of Google Inc. nor the names of its
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

#pragma comment(lib, "winhttp.lib")
#pragma comment(lib, "wininet.lib")
#pragma comment(lib, "diaguids.lib")
#pragma comment(lib, "imagehlp.lib")

#include <cassert>
#include <cstdio>
#include <ctime>
#include <map>
#include <regex>
#include <string>
#include <vector>

#include "tools/windows/converter_exe/escaping.h"
#include "tools/windows/converter_exe/http_download.h"
#include "tools/windows/converter_exe/tokenizer.h"
#include "common/windows/http_upload.h"
#include "common/windows/string_utils-inl.h"
#include "tools/windows/converter/ms_symbol_server_converter.h"

using strings::WebSafeBase64Unescape;
using strings::WebSafeBase64Escape;

namespace {

using std::map;
using std::string;
using std::vector;
using std::wstring;
using crash::HTTPDownload;
using crash::Tokenizer;
using google_breakpad::HTTPUpload;
using google_breakpad::MissingSymbolInfo;
using google_breakpad::MSSymbolServerConverter;
using google_breakpad::WindowsStringUtils;

const char* kMissingStringDelimiters = "|";
const char* kLocalCachePath = "c:\\symbols";
const char* kNoExeMSSSServer = "http://msdl.microsoft.com/download/symbols/";

// Windows stdio doesn't do line buffering.  Use this function to flush after
// writing to stdout and stderr so that a log will be available if the
// converter crashes.
static int FprintfFlush(FILE* file, const char* format, ...) {
  va_list arguments;
  va_start(arguments, format);
  int retval = vfprintf(file, format, arguments);
  va_end(arguments);
  fflush(file);
  return retval;
}

static string CurrentDateAndTime() {
  const string kUnknownDateAndTime = R"(????-??-?? ??:??:??)";

  time_t current_time;
  time(&current_time);

  // localtime_s is safer but is only available in MSVC8.  Use localtime
  // in earlier environments.
  struct tm* time_pointer;
#if _MSC_VER >= 1400  // MSVC 2005/8
  struct tm time_struct;
  time_pointer =& time_struct;
  if (localtime_s(time_pointer,& current_time) != 0) {
    return kUnknownDateAndTime;
  }
#else  // _MSC_VER >= 1400
  time_pointer = localtime(&current_time);
  if (!time_pointer) {
    return kUnknownDateAndTime;
  }
#endif  // _MSC_VER >= 1400

  char buffer[256];
  if (!strftime(buffer, sizeof(buffer), "%Y-%m-%d %H:%M:%S", time_pointer)) {
    return kUnknownDateAndTime;
  }

  return string(buffer);
}

// ParseMissingString turns |missing_string| into a MissingSymbolInfo
// structure.  It returns true on success, and false if no such conversion
// is possible.
static bool ParseMissingString(const string& missing_string,
                               MissingSymbolInfo* missing_info) {
  assert(missing_info);

  vector<string> tokens;
  Tokenizer::Tokenize(kMissingStringDelimiters, missing_string,& tokens);
  if (tokens.size() != 5) {
    return false;
  }

  missing_info->debug_file = tokens[0];
  missing_info->debug_identifier = tokens[1];
  missing_info->version = tokens[2];
  missing_info->code_file = tokens[3];
  missing_info->code_identifier = tokens[4];

  return true;
}

// StringMapToWStringMap takes each element in a map that associates
// (narrow) strings to strings and converts the keys and values to wstrings.
// Returns true on success and false on failure, printing an error message.
static bool StringMapToWStringMap(const map<string, string>& smap,
                                  map<wstring, wstring>* wsmap) {
  assert(wsmap);
  wsmap->clear();

  for (map<string, string>::const_iterator iterator = smap.begin();
       iterator != smap.end();
       ++iterator) {
    wstring key;
    if (!WindowsStringUtils::safe_mbstowcs(iterator->first,& key)) {
      FprintfFlush(stderr,
                   "StringMapToWStringMap: safe_mbstowcs failed for key %s\n",
                   iterator->first.c_str());
      return false;
    }

    wstring value;
    if (!WindowsStringUtils::safe_mbstowcs(iterator->second,& value)) {
      FprintfFlush(stderr, "StringMapToWStringMap: safe_mbstowcs failed "
                           "for value %s\n",
                   iterator->second.c_str());
      return false;
    }

    wsmap->insert(make_pair(key, value));
  }

  return true;
}

// MissingSymbolInfoToParameters turns a MissingSymbolInfo structure into a
// map of parameters suitable for passing to HTTPDownload or HTTPUpload.
// Returns true on success and false on failure, printing an error message.
static bool MissingSymbolInfoToParameters(const MissingSymbolInfo& missing_info,
                                          map<wstring, wstring>* wparameters) {
  assert(wparameters);

  map<string, string> parameters;
  string encoded_param;
  // Indicate the params are encoded.
  parameters["encoded"] = "true";  // The string value here does not matter.

  WebSafeBase64Escape(missing_info.code_file,& encoded_param);
  parameters["code_file"] = encoded_param;

  WebSafeBase64Escape(missing_info.code_identifier,& encoded_param);
  parameters["code_identifier"] = encoded_param;

  WebSafeBase64Escape(missing_info.debug_file,& encoded_param);
  parameters["debug_file"] = encoded_param;

  WebSafeBase64Escape(missing_info.debug_identifier,& encoded_param);
  parameters["debug_identifier"] = encoded_param;

  if (!missing_info.version.empty()) {
    // The version is optional.
    WebSafeBase64Escape(missing_info.version,& encoded_param);
    parameters["version"] = encoded_param;
  }

  WebSafeBase64Escape("WinSymConv",& encoded_param);
  parameters["product"] = encoded_param;

  if (!StringMapToWStringMap(parameters, wparameters)) {
    // StringMapToWStringMap will have printed an error.
    return false;
  }

  return true;
}

// UploadSymbolFile sends |converted_file| as identified by |missing_info|
// to the symbol server rooted at |upload_symbol_url|.  Returns true on
// success and false on failure, printing an error message.
static bool UploadSymbolFile(const wstring& upload_symbol_url,
                             const MissingSymbolInfo& missing_info,
                             const string& converted_file) {
  map<wstring, wstring> parameters;
  if (!MissingSymbolInfoToParameters(missing_info,& parameters)) {
    // MissingSymbolInfoToParameters or a callee will have printed an error.
    return false;
  }

  wstring converted_file_w;

  if (!WindowsStringUtils::safe_mbstowcs(converted_file,& converted_file_w)) {
    FprintfFlush(stderr, "UploadSymbolFile: safe_mbstowcs failed for %s\n",
                 converted_file.c_str());
    return false;
  }
  map<wstring, wstring> files;
  files[L"symbol_file"] = converted_file_w;

  FprintfFlush(stderr, "Uploading %s\n", converted_file.c_str());
  if (!HTTPUpload::SendMultipartPostRequest(
      upload_symbol_url, parameters,
      files, NULL, NULL, NULL)) {
    FprintfFlush(stderr, "UploadSymbolFile: HTTPUpload::SendRequest failed "
                         "for %s %s %s\n",
                 missing_info.debug_file.c_str(),
                 missing_info.debug_identifier.c_str(),
                 missing_info.version.c_str());
    return false;
  }

  return true;
}

// SendFetchFailedPing informs the symbol server based at
// |fetch_symbol_failure_url| that the symbol file identified by
// |missing_info| could authoritatively not be located.  Returns
// true on success and false on failure.
static bool SendFetchFailedPing(const wstring& fetch_symbol_failure_url,
                                const MissingSymbolInfo& missing_info) {
  map<wstring, wstring> parameters;
  if (!MissingSymbolInfoToParameters(missing_info,& parameters)) {
    // MissingSymbolInfoToParameters or a callee will have printed an error.
    return false;
  }

  string content;
  if (!HTTPDownload::Download(fetch_symbol_failure_url,
                             & parameters,
                             & content,
                              NULL)) {
    FprintfFlush(stderr, "SendFetchFailedPing: HTTPDownload::Download failed "
                         "for %s %s %s\n",
                 missing_info.debug_file.c_str(),
                 missing_info.debug_identifier.c_str(),
                 missing_info.version.c_str());
    return false;
  }

  return true;
}

// Returns true if it's safe to make an external request for the symbol
// file described in missing_info. It's considered safe to make an
// external request unless the symbol file's debug_file string matches
// the given blacklist regular expression.
// The debug_file name is used from the MissingSymbolInfo struct,
// matched against the blacklist_regex.
static bool SafeToMakeExternalRequest(const MissingSymbolInfo& missing_info,
                                      std::regex blacklist_regex) {
  string file_name = missing_info.debug_file;
  // Use regex_search because we want to match substrings.
  if (std::regex_search(file_name, blacklist_regex)) {
    FprintfFlush(stderr, "Not safe to make external request for file %s\n",
                 file_name.c_str());
    return false;
  }

  return true;
}

// Converter options derived from command line parameters.
struct ConverterOptions {
  ConverterOptions()
      : report_fetch_failures(true) {
  }

  ~ConverterOptions() {
  }

  // Names of MS Symbol Supplier Servers that are internal to Google, and may
  // have symbols for any request.
  vector<string> full_internal_msss_servers;

  // Names of MS Symbol Supplier Servers that are internal to Google, and
  // shouldn't be checked for symbols for any .exe files.
  vector<string> full_external_msss_servers;

  // Names of MS Symbol Supplier Servers that are external to Google, and may
  // have symbols for any request.
  vector<string> no_exe_internal_msss_servers;

  // Names of MS Symbol Supplier Servers that are external to Google, and
  // shouldn't be checked for symbols for any .exe files.
  vector<string> no_exe_external_msss_servers;

  // Temporary local storage for symbols.
  string local_cache_path;

  // URL for uploading symbols.
  wstring upload_symbols_url;

  // URL to fetch list of missing symbols.
  wstring missing_symbols_url;

  // URL to report symbol fetch failure.
  wstring fetch_symbol_failure_url;

  // Are symbol fetch failures reported.
  bool report_fetch_failures;

  // File containing the list of missing symbols.  Fetch failures are not
  // reported if such file is provided.
  string missing_symbols_file;

  // Regex used to blacklist files to prevent external symbol requests.
  // Owned and cleaned up by this struct.
  std::regex blacklist_regex;

 private:
  // DISABLE_COPY_AND_ASSIGN
  ConverterOptions(const ConverterOptions&);
  ConverterOptions& operator=(const ConverterOptions&);
};

// ConverMissingSymbolFile takes a single MissingSymbolInfo structure and
// attempts to locate it from the symbol servers provided in the
// |options.*_msss_servers| arguments.  "Full" servers are those that will be
// queried for all symbol files; "No-EXE" servers will only be queried for
// modules whose missing symbol data indicates are not main program executables.
// Results will be sent to the |options.upload_symbols_url| on success or
// |options.fetch_symbol_failure_url| on failure, and the local cache will be
// stored at |options.local_cache_path|.  Because nothing can be done even in
// the event of a failure, this function returns no value, although it
// may result in error messages being printed.
static void ConvertMissingSymbolFile(const MissingSymbolInfo& missing_info,
                                     const ConverterOptions& options) {
  string time_string = CurrentDateAndTime();
  FprintfFlush(stdout, "converter: %s: attempting %s %s %s\n",
               time_string.c_str(),
               missing_info.debug_file.c_str(),
               missing_info.debug_identifier.c_str(),
               missing_info.version.c_str());

  // The first lookup is always to internal symbol servers.
  // Always ask the symbol servers identified as "full."
  vector<string> msss_servers = options.full_internal_msss_servers;

  // If the file is not an .exe file, also ask an additional set of symbol
  // servers, such as Microsoft's public symbol server.
  bool is_exe = false;

  if (missing_info.code_file.length() >= 4) {
    string code_extension =
        missing_info.code_file.substr(missing_info.code_file.size() - 4);

    // Firefox is a special case: .dll-only servers should be consulted for
    // its symbols.  This enables us to get its symbols from Mozilla's
    // symbol server when crashes occur in Google extension code hosted by a
    // Firefox process.
    if (_stricmp(code_extension.c_str(), ".exe") == 0 &&
        _stricmp(missing_info.code_file.c_str(), "firefox.exe") != 0) {
      is_exe = true;
    }
  }

  if (!is_exe) {
    msss_servers.insert(msss_servers.end(),
                        options.no_exe_internal_msss_servers.begin(),
                        options.no_exe_internal_msss_servers.end());
  }

  // If there are any suitable internal symbol servers, make a request.
  MSSymbolServerConverter::LocateResult located =
      MSSymbolServerConverter::LOCATE_FAILURE;
  string converted_file;
  if (msss_servers.size() > 0) {
    // Attempt to fetch the symbol file and convert it.
    FprintfFlush(stderr, "Making internal request for %s (%s)\n",
                   missing_info.debug_file.c_str(),
                   missing_info.debug_identifier.c_str());
    MSSymbolServerConverter converter(options.local_cache_path, msss_servers);
    located = converter.LocateAndConvertSymbolFile(missing_info,
                                                   false,  // keep_symbol_file
                                                   false,  // keep_pe_file
                                                  & converted_file,
                                                   NULL,   // symbol_file
                                                   NULL);  // pe_file
    switch (located) {
      case MSSymbolServerConverter::LOCATE_SUCCESS:
        FprintfFlush(stderr, "LocateResult = LOCATE_SUCCESS\n");
        // Upload it.  Don't bother checking the return value.  If this
        // succeeds, it should disappear from the missing symbol list.
        // If it fails, something will print an error message indicating
        // the cause of the failure, and the item will remain on the
        // missing symbol list.
        UploadSymbolFile(options.upload_symbols_url, missing_info,
                         converted_file);
        remove(converted_file.c_str());

        // Note: this does leave some directories behind that could be
        // cleaned up.  The directories inside options.local_cache_path for
        // debug_file/debug_identifier can be removed at this point.
        break;

      case MSSymbolServerConverter::LOCATE_NOT_FOUND:
        FprintfFlush(stderr, "LocateResult = LOCATE_NOT_FOUND\n");
        // The symbol file definitively did not exist. Fall through,
        // so we can attempt an external query if it's safe to do so.
        break;

      case MSSymbolServerConverter::LOCATE_RETRY:
        FprintfFlush(stderr, "LocateResult = LOCATE_RETRY\n");
        // Fall through in case we should make an external request.
        // If not, or if an external request fails in the same way,
        // we'll leave the entry in the symbol file list and
        // try again on a future pass.  Print a message so that there's
        // a record.
        break;

      case MSSymbolServerConverter::LOCATE_FAILURE:
        FprintfFlush(stderr, "LocateResult = LOCATE_FAILURE\n");
        // LocateAndConvertSymbolFile printed an error message.
        break;

      default:
        FprintfFlush(
            stderr,
            "FATAL: Unexpected return value '%d' from "
            "LocateAndConvertSymbolFile()\n",
            located);
        assert(0);
        break;
    }
  } else {
    // No suitable internal symbol servers.  This is fine because the converter
    // is mainly used for downloading and converting of external symbols.
  }

  // Make a request to an external server if the internal request didn't
  // succeed, and it's safe to do so.
  if (located != MSSymbolServerConverter::LOCATE_SUCCESS &&
      SafeToMakeExternalRequest(missing_info, options.blacklist_regex)) {
    msss_servers = options.full_external_msss_servers;
    if (!is_exe) {
      msss_servers.insert(msss_servers.end(),
                          options.no_exe_external_msss_servers.begin(),
                          options.no_exe_external_msss_servers.end());
    }
    if (msss_servers.size() > 0) {
      FprintfFlush(stderr, "Making external request for %s (%s)\n",
                   missing_info.debug_file.c_str(),
                   missing_info.debug_identifier.c_str());
      MSSymbolServerConverter external_converter(options.local_cache_path,
                                                 msss_servers);
      located = external_converter.LocateAndConvertSymbolFile(
          missing_info,
          false,  // keep_symbol_file
          false,  // keep_pe_file
         & converted_file,
          NULL,   // symbol_file
          NULL);  // pe_file
    } else {
      FprintfFlush(stderr, "ERROR: No suitable external symbol servers.\n");
    }
  }

  // Final handling for this symbol file is based on the result from the
  // external request (if performed above), or on the result from the
  // previous internal lookup.
  switch (located) {
    case MSSymbolServerConverter::LOCATE_SUCCESS:
      FprintfFlush(stderr, "LocateResult = LOCATE_SUCCESS\n");
      // Upload it.  Don't bother checking the return value.  If this
      // succeeds, it should disappear from the missing symbol list.
      // If it fails, something will print an error message indicating
      // the cause of the failure, and the item will remain on the
      // missing symbol list.
      UploadSymbolFile(options.upload_symbols_url, missing_info,
                       converted_file);
      remove(converted_file.c_str());

      // Note: this does leave some directories behind that could be
      // cleaned up.  The directories inside options.local_cache_path for
      // debug_file/debug_identifier can be removed at this point.
      break;

    case MSSymbolServerConverter::LOCATE_NOT_FOUND:
      // The symbol file definitively didn't exist.  Inform the server.
      // If this fails, something will print an error message indicating
      // the cause of the failure, but there's really nothing more to
      // do.  If this succeeds, the entry should be removed from the
      // missing symbols list.
      if (!options.report_fetch_failures) {
        FprintfFlush(stderr, "SendFetchFailedPing skipped\n");
      } else if (SendFetchFailedPing(options.fetch_symbol_failure_url,
                                     missing_info)) {
        FprintfFlush(stderr, "SendFetchFailedPing succeeded\n");
      } else {
        FprintfFlush(stderr, "SendFetchFailedPing failed\n");
      }
      break;

    case MSSymbolServerConverter::LOCATE_RETRY:
      FprintfFlush(stderr, "LocateResult = LOCATE_RETRY\n");
      // Nothing to do but leave the entry in the symbol file list and
      // try again on a future pass.  Print a message so that there's
      // a record.
      FprintfFlush(stderr, "ConvertMissingSymbolFile: deferring retry "
                           "for %s %s %s\n",
                   missing_info.debug_file.c_str(),
                   missing_info.debug_identifier.c_str(),
                   missing_info.version.c_str());
      break;

    case MSSymbolServerConverter::LOCATE_FAILURE:
      FprintfFlush(stderr, "LocateResult = LOCATE_FAILURE\n");
      // LocateAndConvertSymbolFile printed an error message.

      // This is due to a bad debug file name, so fetch failed.
      if (!options.report_fetch_failures) {
        FprintfFlush(stderr, "SendFetchFailedPing skipped\n");
      } else if (SendFetchFailedPing(options.fetch_symbol_failure_url,
                                     missing_info)) {
        FprintfFlush(stderr, "SendFetchFailedPing succeeded\n");
      } else {
        FprintfFlush(stderr, "SendFetchFailedPing failed\n");
      }
      break;

    default:
      FprintfFlush(
          stderr,
          "FATAL: Unexpected return value '%d' from "
          "LocateAndConvertSymbolFile()\n",
          located);
      assert(0);
      break;
  }
}


// Reads the contents of file |file_name| and populates |contents|.
// Returns true on success.
static bool ReadFile(string file_name, string* contents) {
  char buffer[1024 * 8];
  FILE* fp = fopen(file_name.c_str(), "rt");
  if (!fp) {
    return false;
  }
  contents->clear();
  while (fgets(buffer, sizeof(buffer), fp) != NULL) {
    contents->append(buffer);
  }
  fclose(fp);
  return true;
}

// ConvertMissingSymbolsList obtains a missing symbol list from
// |options.missing_symbols_url| or |options.missing_symbols_file| and calls
// ConvertMissingSymbolFile for each missing symbol file in the list.
static bool ConvertMissingSymbolsList(const ConverterOptions& options) {
  // Set param to indicate requesting for encoded response.
  map<wstring, wstring> parameters;
  parameters[L"product"] = L"WinSymConv";
  parameters[L"encoded"] = L"true";
  // Get the missing symbol list.
  string missing_symbol_list;
  if (!options.missing_symbols_file.empty()) {
    if (!ReadFile(options.missing_symbols_file,& missing_symbol_list)) {
      return false;
    }
  } else if (!HTTPDownload::Download(options.missing_symbols_url,& parameters,
                                    & missing_symbol_list, NULL)) {
    return false;
  }

  // Tokenize the content into a vector.
  vector<string> missing_symbol_lines;
  Tokenizer::Tokenize("\n", missing_symbol_list,& missing_symbol_lines);

  FprintfFlush(stderr, "Found %d missing symbol files in list.\n",
               missing_symbol_lines.size() - 1);  // last line is empty.
  int convert_attempts = 0;
  for (vector<string>::const_iterator iterator = missing_symbol_lines.begin();
       iterator != missing_symbol_lines.end();
       ++iterator) {
    // Decode symbol line.
    const string& encoded_line = *iterator;
    // Skip lines that are blank.
    if (encoded_line.empty()) {
      continue;
    }

    string line;
    if (!WebSafeBase64Unescape(encoded_line,& line)) {
      // If decoding fails, assume the line is not encoded.
      // This is helpful when the program connects to a debug server without
      // encoding.
      line = encoded_line;
    }

    FprintfFlush(stderr, "\nLine: %s\n", line.c_str());

    // Turn each element into a MissingSymbolInfo structure.
    MissingSymbolInfo missing_info;
    if (!ParseMissingString(line,& missing_info)) {
      FprintfFlush(stderr, "ConvertMissingSymbols: ParseMissingString failed "
                           "for %s from %ws\n",
                   line.c_str(), options.missing_symbols_url.c_str());
      continue;
    }

    ++convert_attempts;
    ConvertMissingSymbolFile(missing_info, options);
  }

  // Say something reassuring, since ConvertMissingSymbolFile was never called
  // and therefore never reported any progress.
  if (convert_attempts == 0) {
    string current_time = CurrentDateAndTime();
    FprintfFlush(stdout, "converter: %s: nothing to convert\n",
                 current_time.c_str());
  }

  return true;
}

// usage prints the usage message.  It returns 1 as a convenience, to be used
// as a return value from main.
static int usage(const char* program_name) {
  FprintfFlush(stderr,
      "usage: %s [options]\n"
      "    -f  <full_msss_server>     MS servers to ask for all symbols\n"
      "    -n  <no_exe_msss_server>   same, but prevent asking for EXEs\n"
      "    -l  <local_cache_path>     Temporary local storage for symbols\n"
      "    -s  <upload_url>           URL for uploading symbols\n"
      "    -m  <missing_symbols_url>  URL to fetch list of missing symbols\n"
      "    -mf <missing_symbols_file> File containing the list of missing\n"
      "                               symbols.  Fetch failures are not\n"
      "                               reported if such file is provided.\n"
      "    -t  <fetch_failure_url>    URL to report symbol fetch failure\n"
      "    -b  <regex>                Regex used to blacklist files to\n"
      "                               prevent external symbol requests\n"
      " Note that any server specified by -f or -n that starts with \\filer\n"
      " will be treated as internal, and all others as external.\n",
      program_name);

  return 1;
}

// "Internal" servers consist only of those whose names start with
// the literal string "\\filer\".
static bool IsInternalServer(const string& server_name) {
  if (server_name.find("\\\\filer\\") == 0) {
    return true;
  }
  return false;
}

// Adds a server with the given name to the list of internal or external
// servers, as appropriate.
static void AddServer(const string& server_name,
                      vector<string>* internal_servers,
                      vector<string>* external_servers) {
  if (IsInternalServer(server_name)) {
    internal_servers->push_back(server_name);
  } else {
    external_servers->push_back(server_name);
  }
}

}  // namespace

int main(int argc, char** argv) {
  string time_string = CurrentDateAndTime();
  FprintfFlush(stdout, "converter: %s: starting\n", time_string.c_str());

  ConverterOptions options;
  options.report_fetch_failures = true;

  // All arguments are paired.
  if (argc % 2 != 1) {
    return usage(argv[0]);
  }

  string blacklist_regex_str;
  bool have_any_msss_servers = false;
  for (int argi = 1; argi < argc; argi += 2) {
    string option = argv[argi];
    string value = argv[argi + 1];

    if (option == "-f") {
      AddServer(value,& options.full_internal_msss_servers,
               & options.full_external_msss_servers);
      have_any_msss_servers = true;
    } else if (option == "-n") {
      AddServer(value,& options.no_exe_internal_msss_servers,
               & options.no_exe_external_msss_servers);
      have_any_msss_servers = true;
    } else if (option == "-l") {
      if (!options.local_cache_path.empty()) {
        return usage(argv[0]);
      }
      options.local_cache_path = value;
    } else if (option == "-s") {
      if (!WindowsStringUtils::safe_mbstowcs(value,
                                            & options.upload_symbols_url)) {
        FprintfFlush(stderr, "main: safe_mbstowcs failed for %s\n",
                     value.c_str());
        return 1;
      }
    } else if (option == "-m") {
      if (!WindowsStringUtils::safe_mbstowcs(value,
                                            & options.missing_symbols_url)) {
        FprintfFlush(stderr, "main: safe_mbstowcs failed for %s\n",
                     value.c_str());
        return 1;
      }
    } else if (option == "-mf") {
      options.missing_symbols_file = value;
      printf("Getting the list of missing symbols from a file.  Fetch failures"
             " will not be reported.\n");
      options.report_fetch_failures = false;
    } else if (option == "-t") {
      if (!WindowsStringUtils::safe_mbstowcs(
          value,
         & options.fetch_symbol_failure_url)) {
        FprintfFlush(stderr, "main: safe_mbstowcs failed for %s\n",
                     value.c_str());
        return 1;
      }
    } else if (option == "-b") {
      blacklist_regex_str = value;
    } else {
      return usage(argv[0]);
    }
  }

  if (blacklist_regex_str.empty()) {
    FprintfFlush(stderr, "No blacklist specified.\n");
    return usage(argv[0]);
  }

  // Compile the blacklist regular expression for later use.
  options.blacklist_regex = std::regex(blacklist_regex_str.c_str(),
      std::regex_constants::icase);

  // Set the defaults.  If the user specified any MSSS servers, don't use
  // any default.
  if (!have_any_msss_servers) {
    AddServer(kNoExeMSSSServer,& options.no_exe_internal_msss_servers,
       & options.no_exe_external_msss_servers);
  }

  if (options.local_cache_path.empty()) {
    options.local_cache_path = kLocalCachePath;
  }

  if (options.upload_symbols_url.empty()) {
    FprintfFlush(stderr, "No upload symbols URL specified.\n");
    return usage(argv[0]);
  }
  if (options.missing_symbols_url.empty() &&
      options.missing_symbols_file.empty()) {
    FprintfFlush(stderr, "No missing symbols URL or file specified.\n");
    return usage(argv[0]);
  }
  if (options.fetch_symbol_failure_url.empty()) {
    FprintfFlush(stderr, "No fetch symbol failure URL specified.\n");
    return usage(argv[0]);
  }

  FprintfFlush(stdout,
               "# of Symbol Servers (int/ext): %d/%d full, %d/%d no_exe\n",
               options.full_internal_msss_servers.size(),
               options.full_external_msss_servers.size(),
               options.no_exe_internal_msss_servers.size(),
               options.no_exe_external_msss_servers.size());

  if (!ConvertMissingSymbolsList(options)) {
    return 1;
  }

  time_string = CurrentDateAndTime();
  FprintfFlush(stdout, "converter: %s: finished\n", time_string.c_str());
  return 0;
}