From 9c053ec1b1c7cd91ca6c0cc6f65803cc3f114c7f Mon Sep 17 00:00:00 2001 From: Aqua-sama Date: Fri, 27 Mar 2020 16:58:18 +0200 Subject: Add namespace to format rcc: - add -n/--namespace to command line - make default compression None - fix None compression not printing hexdumps libembed: - add a zstd decompression test --- .clang-format | 108 +++++++++++++++++++++++++++++++++++++++++++++++++++++ lib/embed.cpp | 67 ++++----------------------------- lib/embed.h | 29 ++++++-------- lib/embed_zstd.cpp | 62 ++++++++++++++++++++++++++++++ meson.build | 30 +++++++++------ scripts/rcc | 30 +++++++++------ test/main.cpp | 39 +++++++++++++------ 7 files changed, 253 insertions(+), 112 deletions(-) create mode 100644 .clang-format create mode 100644 lib/embed_zstd.cpp diff --git a/.clang-format b/.clang-format new file mode 100644 index 0000000..64af425 --- /dev/null +++ b/.clang-format @@ -0,0 +1,108 @@ +--- +Language: Cpp +# BasedOnStyle: WebKit +AccessModifierOffset: -4 +AlignAfterOpenBracket: DontAlign +AlignConsecutiveAssignments: false +AlignConsecutiveDeclarations: false +AlignEscapedNewlines: Right +AlignOperands: false +AlignTrailingComments: false +AllowAllParametersOfDeclarationOnNextLine: true +AllowShortBlocksOnASingleLine: false +AllowShortCaseLabelsOnASingleLine: false +AllowShortFunctionsOnASingleLine: All +AllowShortIfStatementsOnASingleLine: false +AllowShortLoopsOnASingleLine: false +AlwaysBreakAfterDefinitionReturnType: None +AlwaysBreakAfterReturnType: None +AlwaysBreakBeforeMultilineStrings: false +AlwaysBreakTemplateDeclarations: false +BinPackArguments: true +BinPackParameters: true +BraceWrapping: + AfterClass: false + AfterControlStatement: false + AfterEnum: false + AfterFunction: true + AfterNamespace: false + AfterObjCDeclaration: false + AfterStruct: false + AfterUnion: false + BeforeCatch: false + BeforeElse: false + IndentBraces: false + SplitEmptyFunction: true + SplitEmptyRecord: true + SplitEmptyNamespace: true +BreakBeforeBinaryOperators: All +BreakBeforeBraces: Linux +BreakBeforeInheritanceComma: false +BreakBeforeTernaryOperators: true +BreakConstructorInitializersBeforeComma: true +BreakConstructorInitializers: BeforeComma +BreakAfterJavaFieldAnnotations: false +BreakStringLiterals: true +ColumnLimit: 0 +CommentPragmas: '^ IWYU pragma:' +CompactNamespaces: false +ConstructorInitializerAllOnOneLineOrOnePerLine: false +ConstructorInitializerIndentWidth: 4 +ContinuationIndentWidth: 4 +Cpp11BracedListStyle: false +DerivePointerAlignment: false +DisableFormat: false +ExperimentalAutoDetectBinPacking: false +FixNamespaceComments: false +ForEachMacros: + - foreach + - Q_FOREACH + - BOOST_FOREACH +IncludeCategories: + - Regex: '^"(llvm|llvm-c|clang|clang-c)/' + Priority: 2 + - Regex: '^(<|"(gtest|gmock|isl|json)/)' + Priority: 3 + - Regex: '.*' + Priority: 1 +IncludeIsMainRegex: '(Test)?$' +IndentCaseLabels: false +IndentWidth: 4 +IndentWrappedFunctionNames: false +JavaScriptQuotes: Leave +JavaScriptWrapImports: true +KeepEmptyLinesAtTheStartOfBlocks: true +MacroBlockBegin: '' +MacroBlockEnd: '' +MaxEmptyLinesToKeep: 1 +NamespaceIndentation: Inner +ObjCBlockIndentWidth: 4 +ObjCSpaceAfterProperty: true +ObjCSpaceBeforeProtocolList: true +PenaltyBreakAssignment: 2 +PenaltyBreakBeforeFirstCallParameter: 19 +PenaltyBreakComment: 300 +PenaltyBreakFirstLessLess: 120 +PenaltyBreakString: 1000 +PenaltyExcessCharacter: 1000000 +PenaltyReturnTypeOnItsOwnLine: 60 +PointerAlignment: Right +ReflowComments: true +SortIncludes: true +SortUsingDeclarations: true +SpaceAfterCStyleCast: false +SpaceAfterTemplateKeyword: true +SpaceBeforeAssignmentOperators: true +SpaceBeforeParens: Never +SpaceInEmptyParentheses: false +SpacesBeforeTrailingComments: 1 +SpacesInAngles: false +SpacesInContainerLiterals: false +SpacesInCStyleCastParentheses: false +SpacesInParentheses: false +SpacesInSquareBrackets: false +Standard: Cpp11 +TabWidth: 4 +UseTab: Never +... + diff --git a/lib/embed.cpp b/lib/embed.cpp index c3b0286..07af38f 100644 --- a/lib/embed.cpp +++ b/lib/embed.cpp @@ -1,70 +1,17 @@ -/* - * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - * You may select, at your option, one of the above-listed licenses. - */ - #include "embed.h" -#include // presumes zstd library is installed -#include using namespace embed; -ZSTD_DDict* dictPtr = nullptr; - -Resources::Resources(const ResourceData &info) - : m_info(info) +template <> +Resources::Resources(const std::span &) { - if(!info.dictionary.empty()) { - dictPtr = ZSTD_createDDict(info.dictionary.data(), info.dictionary.size()); - } } -Resources::~Resources() -{ - if(dictPtr != nullptr) { - ZSTD_freeDDict(dictPtr); - } -} +template <> +Resources::~Resources() = default; -[[nodiscard]] std::span Resources::decompress(const std::span &entry) +template <> +[[nodiscard]] std::vector Resources::decompress(const std::span &entry) { - /* Read the content size from the frame header. For simplicity we require - * that it is always present. By default, zstd will write the content size - * in the header when it is known. If you can't guarantee that the frame - * content size is always written into the header, either use streaming - * decompression, or ZSTD_decompressBound(). - */ - unsigned long long const rSize = ZSTD_getFrameContentSize(entry.data(), entry.size()); - assert(rSize != ZSTD_CONTENTSIZE_ERROR); //, "%s: not compressed by zstd!", fname); - assert(rSize != ZSTD_CONTENTSIZE_UNKNOWN); //, "%s: original size unknown!", fname); - auto* rBuff = new unsigned char[(size_t) rSize]; - - /* Check that the dictionary ID matches. - * If a non-zstd dictionary is used, then both will be zero. - * By default zstd always writes the dictionary ID into the frame. - * Zstd will check if there is a dictionary ID mismatch as well. - */ - unsigned const expectedDictID = ZSTD_getDictID_fromDDict(dictPtr); - unsigned const actualDictID = ZSTD_getDictID_fromFrame(entry.data(), entry.size()); - assert(actualDictID == expectedDictID); //"DictID mismatch: expected %u got %u", - - /* Decompress using the dictionary. - * If you need to control the decompression parameters, then use the - * advanced API: ZSTD_DCtx_setParameter(), ZSTD_DCtx_refDDict(), and - * ZSTD_decompressDCtx(). - */ - ZSTD_DCtx* const dctx = ZSTD_createDCtx(); - assert(dctx != NULL); //, "ZSTD_createDCtx() failed!"); - size_t const dSize = ZSTD_decompress_usingDDict(dctx, rBuff, rSize, entry.data(), entry.size(), dictPtr); - /* When zstd knows the content size, it will error if it doesn't match. */ - assert(dSize == rSize); //, "Impossible because zstd will check this condition!"); - - ZSTD_freeDCtx(dctx); - return std::span(rBuff, rSize); + return { entry.begin(), entry.end() }; } - diff --git a/lib/embed.h b/lib/embed.h index 566b14c..bd041ff 100644 --- a/lib/embed.h +++ b/lib/embed.h @@ -1,34 +1,29 @@ #pragma once -#include -#include -#include -#include #include +#include +#include +#include -namespace embed { +namespace embed +{ enum Compression { None, Zstd }; -struct ResourceData { - Compression compression = None; - std::span dictionary; -}; - -class Resources { +template +class Resources +{ public: - explicit Resources(const ResourceData &info); + Resources() = default; + explicit Resources(const std::span &dictionary); + ~Resources(); - [[nodiscard]] - std::span decompress(const std::span &entry); + [[nodiscard]] std::vector decompress(const std::span &entry); -private: - const ResourceData m_info; }; // class } - diff --git a/lib/embed_zstd.cpp b/lib/embed_zstd.cpp new file mode 100644 index 0000000..d9be35a --- /dev/null +++ b/lib/embed_zstd.cpp @@ -0,0 +1,62 @@ +#include "embed.h" +#include +#include + +using namespace embed; + +ZSTD_DDict *dictPtr = nullptr; + +template <> +Resources::Resources(const std::span &dictionary) +{ + if(!dictionary.empty()) { + dictPtr = ZSTD_createDDict(dictionary.data(), dictionary.size()); + } +} + +template <> +Resources::~Resources() +{ + if(dictPtr != nullptr) { + ZSTD_freeDDict(dictPtr); + } +} + +template <> +[[nodiscard]] std::vector Resources::decompress(const std::span &entry) +{ + /* Read the content size from the frame header. For simplicity we require + * that it is always present. By default, zstd will write the content size + * in the header when it is known. If you can't guarantee that the frame + * content size is always written into the header, either use streaming + * decompression, or ZSTD_decompressBound(). + */ + unsigned long long const rSize = ZSTD_getFrameContentSize(entry.data(), entry.size()); + assert(rSize != ZSTD_CONTENTSIZE_ERROR); //, "%s: not compressed by zstd!", fname); + assert(rSize != ZSTD_CONTENTSIZE_UNKNOWN); //, "%s: original size unknown!", fname); + //data rBuff(new unsigned char[rSize], rSize, true); + std::vector rBuff(rSize); + + /* Check that the dictionary ID matches. + * If a non-zstd dictionary is used, then both will be zero. + * By default zstd always writes the dictionary ID into the frame. + * Zstd will check if there is a dictionary ID mismatch as well. + */ + unsigned const expectedDictID = ZSTD_getDictID_fromDDict(dictPtr); + unsigned const actualDictID = ZSTD_getDictID_fromFrame(entry.data(), entry.size()); + assert(actualDictID == expectedDictID); //"DictID mismatch: expected %u got %u", + + /* Decompress using the dictionary. + * If you need to control the decompression parameters, then use the + * advanced API: ZSTD_DCtx_setParameter(), ZSTD_DCtx_refDDict(), and + * ZSTD_decompressDCtx(). + */ + ZSTD_DCtx *const dctx = ZSTD_createDCtx(); + assert(dctx != NULL); //, "ZSTD_createDCtx() failed!"); + size_t const dSize = ZSTD_decompress_usingDDict(dctx, rBuff.data(), rSize, entry.data(), entry.size(), dictPtr); + /* When zstd knows the content size, it will error if it doesn't match. */ + assert(dSize == rSize); //, "Impossible because zstd will check this condition!"); + + ZSTD_freeDCtx(dctx); + return rBuff; +} diff --git a/meson.build b/meson.build index ffae406..f0f8194 100644 --- a/meson.build +++ b/meson.build @@ -6,32 +6,40 @@ project('libembed', ['cpp'], add_project_arguments(['-stdlib=libc++'], language: 'cpp') add_project_link_arguments(['-stdlib=libc++'], language : 'cpp') -dep_zstd = dependency('libzstd') -dep_gtest = dependency('gtest') +libzstd = dependency('libzstd') + +libembed_sourceset = import('sourceset').source_set() +libembed_sourceset.add(files('lib/embed.cpp')) +libembed_sourceset.add(when: libzstd, if_true: [ libzstd, files('lib/embed_zstd.cpp') ] ) +libembed_conf = libembed_sourceset.apply(configuration_data()) libembed = library('embed', - 'lib/embed.cpp', - dependencies: [ dep_zstd ], + libembed_conf.sources(), + dependencies: libembed_conf.dependencies(), include_directories: '3rd-party/frozen/include/' ) -dep_libembed = declare_dependency( +dep_embed = declare_dependency( link_with: libembed, include_directories: include_directories('lib/', '3rd-party/frozen/include') ) prog_python = import('python').find_installation('python3') + resources_h = custom_target('resources.h', output: 'resources.h', input: 'scripts/rcc', - command: [prog_python, '@INPUT@', '--compress=Zstd', '--dict=dictionary', '--output=@OUTPUT@', meson.current_source_dir()/'3rd-party/tabler-icons/icons/chevron-up.svg'], + command: [prog_python, '@INPUT@', '--namespace=staticdata', '--output=@OUTPUT@', files('test/resources.xrc')], +) +zstd_resources_h = custom_target('zstd_resources.h', + output: 'zstd_resources.h', + input: 'scripts/rcc', + command: [prog_python, '@INPUT@', '--namespace=zstd_data', '--compress=Zstd', '--dict=dictionary', '--output=@OUTPUT@', files('test/resources.xrc')], ) test('libembed', executable('embed', - sources: [ 'test/main.cpp', resources_h ], - dependencies: [ dep_gtest, dep_libembed ] - ), - env: environment({ 'CONFIGFILE' : meson.current_source_dir()/'test/defaultrc.ini' }), - workdir: meson.current_source_dir()/'test' + sources: [ 'test/main.cpp', resources_h, zstd_resources_h ], + dependencies: [ dep_embed ] + ) ) diff --git a/scripts/rcc b/scripts/rcc index 2ca587c..b81cf6e 100755 --- a/scripts/rcc +++ b/scripts/rcc @@ -24,12 +24,12 @@ def compress(file, zstd='zstd', level=19, dictionary=None): cmd.append(file.name) return subprocess.run(cmd, capture_output=True).stdout -def hexdump(array_name, input_file, out_h): +def hexdump(array_name, array_data, out_h): array_len = 0 print("constexpr unsigned char {}[] = {{".format(array_name), file=out_h) - for byte in input_file[0:len(input_file)]: + for byte in array_data[0:len(array_data)]: array_len+=1 if array_len%16 == 0: print(" 0x{:02X},".format(byte), file=out_h) @@ -57,8 +57,9 @@ if __name__ == "__main__": parser.add_argument('input', type=argparse.FileType('rt'), help='input file (.xrc)') parser.add_argument('-o', '--output', type=argparse.FileType('wt'), metavar='OUT', default=sys.stdout, help='output header file') + parser.add_argument('-n', '--namespace', type=str, help='namespace') - parser.add_argument('--compress', choices=[ 'None', 'Zstd' ], help='compress input files using algorightm') + parser.add_argument('--compress', choices=[ 'None', 'Zstd' ], default='None', help='compress input files using algorightm') parser.add_argument('--dict', type=argparse.FileType('rb'), help='[zstd] use specified dictionary, recommended for many similar small files') parser.add_argument('--train', action='store_true', help='[zstd] train dictionary') @@ -66,13 +67,14 @@ if __name__ == "__main__": entries_list = "" - if args.compress=='Zstd' and args.train: + if args.train: train(args.input, args.dict.name) # write header print("// Autogenerated binary file hexdump", file=args.output) print("// This file may get overwritten by the build system\n", file=args.output) print("#include \n", file=args.output) + print("namespace {} {{".format(args.namespace), file=args.output) # write file data for child in xml.parse(args.input).getroot(): @@ -80,22 +82,26 @@ if __name__ == "__main__": prefix = child.attrib['prefix'] for i in child: vname = name(i.text) - with open(i.text) as f: - hexdump(vname, compress(f, dictionary=args.dict), args.output) + with open(i.text, 'rb') as f: + if args.compress == 'None': + hexdump(vname, f.read(), args.output) + elif args.compress == 'Zstd': + hexdump(vname, compress(f, dictionary=args.dict), args.output) entries_list += " {{ \"{}/{}\", std::span({}, {}_len) }},\n".format(prefix, i.attrib['alias'], vname, vname) # write dictionary if args.dict is not None: - hexdump('dictionary', args.dict.read(), args.output) + hexdump('dict', args.dict.read(), args.output) # write entries print("constexpr auto entries = frozen::make_unordered_map>({", file=args.output) print(entries_list, file=args.output) print("});\n", file=args.output) - # write metadata struct - print("constexpr embed::ResourceData metadata = {", file=args.output) - print(" .compression = embed::{},".format(args.compress), file=args.output) + # write metadata + print("constexpr auto compression = embed::{};".format(args.compress), file=args.output) if args.dict is not None: - print(" .dictionary = std::span(dictionary, dictionary_len),", file=args.output) - print("};", file=args.output) + print("constexpr auto dictionary = std::span(dict, dict_len);", file=args.output) + + print("}} // namespace {}".format(args.namespace), file=args.output) + diff --git a/test/main.cpp b/test/main.cpp index faf5a16..f17ef67 100644 --- a/test/main.cpp +++ b/test/main.cpp @@ -1,17 +1,32 @@ -#include #include "resources.h" +#include "zstd_resources.h" #include +#include -int main(int argc, char** argv) +embed::Resources zstd_ctx(zstd_data::dictionary); + +int main(int, char **) { -// testing::InitGoogleTest(&argc, argv); -// return RUN_ALL_TESTS(); - embed::Resources ctx(metadata); - constexpr auto raw = entries.at("/icons/chevron-up.svg"); - static_assert(!raw.empty()); - const auto x = ctx.decompress(raw); - for(const char c : x) - printf("%c", c); - delete[] x.data(); - return 0; + for(const auto &pair : zstd_data::entries) { + const auto s = staticdata::entries.at(pair.first); + const auto v = zstd_ctx.decompress(pair.second); + + if(s.size() != v.size()) { + printf("failed comparing sizes at path [%s]\n", pair.first.data()); + printf("s: %li != v: %li\n", s.size(), v.size()); + return EXIT_FAILURE; + } + + if(!std::equal(s.begin(), s.end(), v.begin(), v.end())) { + printf("failed comparing values at path [%s]\n", pair.first.data()); + for(const char &c : s) + printf("%c", c); + for(const char &c : v) + printf("%c", c); + return EXIT_FAILURE; + } + } + printf("Zstd compression test complete\n"); + + return EXIT_SUCCESS; } -- cgit v1.2.1