From 13771971cfaef6356ba47668cae6ce25c5c2071f Mon Sep 17 00:00:00 2001 From: Aqua-sama Date: Sun, 29 Mar 2020 22:59:04 +0300 Subject: Drop dependency on serge-sans-paille/frozen format: - instead of frozen::unordered_map, create two std::arrays with the aliased names (entries) and respective data (values) libembed: - Resources and CompressedResources convenience classes for raw and compressed resources respectively - Resources can be constexpr in regular usage - Annotate Resources::decompress accordingly --- .gitmodules | 3 -- 3rd-party/frozen | 1 - lib/compressionctx.h | 40 ++++++++++++++++++++++ lib/embed.cpp | 17 ---------- lib/embed.h | 91 ++++++++++++++++++++++++++++++++++++++++++--------- lib/embed_zstd.cpp | 62 ----------------------------------- lib/zstd.cpp | 59 +++++++++++++++++++++++++++++++++ meson.build | 3 +- readme.md | 5 +-- scripts/rcc_format.py | 13 +++++--- scripts/zstd.py | 2 ++ test/main.cpp | 17 +++++----- 12 files changed, 196 insertions(+), 117 deletions(-) delete mode 160000 3rd-party/frozen create mode 100644 lib/compressionctx.h delete mode 100644 lib/embed.cpp delete mode 100644 lib/embed_zstd.cpp create mode 100644 lib/zstd.cpp diff --git a/.gitmodules b/.gitmodules index 36e8968..e0954d7 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,6 +1,3 @@ -[submodule "3rd-party/frozen"] - path = 3rd-party/frozen - url = https://github.com/serge-sans-paille/frozen [submodule "3rd-party/tabler-icons"] path = 3rd-party/tabler-icons url = https://github.com/tabler/tabler-icons diff --git a/3rd-party/frozen b/3rd-party/frozen deleted file mode 160000 index 8357f86..0000000 --- a/3rd-party/frozen +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 8357f86de5383dde8539734d3a1620ecc70e4a16 diff --git a/lib/compressionctx.h b/lib/compressionctx.h new file mode 100644 index 0000000..2f3946a --- /dev/null +++ b/lib/compressionctx.h @@ -0,0 +1,40 @@ +#include +#include + +#pragma once + +namespace embed +{ + +enum Compression { + None, + Zstd +}; + +class CompressionCtx +{ +public: + virtual ~CompressionCtx() = default; + [[nodiscard]] virtual std::vector decompress(const std::span &entry) const = 0; +}; + +class ZstdCompressionCtx final : public CompressionCtx +{ +public: + ZstdCompressionCtx(const std::span &dictionary); + ~ZstdCompressionCtx(); + [[nodiscard]] std::vector decompress(const std::span &entry) const override; +}; + +std::unique_ptr make_compression_ctx(const Compression algo, const std::span &dict) +{ + switch(algo) + { + case None: + return nullptr; + case Zstd: + return std::make_unique(dict); + } +} + +} // namespace embed diff --git a/lib/embed.cpp b/lib/embed.cpp deleted file mode 100644 index 07af38f..0000000 --- a/lib/embed.cpp +++ /dev/null @@ -1,17 +0,0 @@ -#include "embed.h" - -using namespace embed; - -template <> -Resources::Resources(const std::span &) -{ -} - -template <> -Resources::~Resources() = default; - -template <> -[[nodiscard]] std::vector Resources::decompress(const std::span &entry) -{ - return { entry.begin(), entry.end() }; -} diff --git a/lib/embed.h b/lib/embed.h index bd041ff..b57d830 100644 --- a/lib/embed.h +++ b/lib/embed.h @@ -1,29 +1,88 @@ #pragma once -#include -#include -#include -#include +#include +#include +#include "compressionctx.h" namespace embed { -enum Compression { - None, - Zstd -}; - -template +template class Resources { public: - Resources() = default; - explicit Resources(const std::span &dictionary); + constexpr explicit Resources( + const std::array &entries, + const std::array, N> &values) + : m_entries(entries) + , m_values(values) + { + } - ~Resources(); + [[nodiscard]] constexpr int id(const std::string_view &path) const + { + for(std::size_t i = 0; i < m_entries.size(); ++i) { + if(m_entries.at(i) == path) + return i; + } + return -1; + } + [[nodiscard]] constexpr std::span value(const int id_) const + { + if(id_ == -1) + return {}; + else + return m_values.at(id_); + } + [[nodiscard]] constexpr std::span value(const std::string_view &path) const + { + return value(id(path)); + } - [[nodiscard]] std::vector decompress(const std::span &entry); + [[deprecated("With no compression, this function returns a copy; use ::value instead")]] + [[nodiscard]] std::vector decompress(const int id_) const + { + const auto v = value(id_); + return { v.begin(), v.end() }; + } + [[deprecated("With no compression, this function returns a copy; use ::value instead")]] + [[nodiscard]] std::vector decompress(const std::string_view &path) const + { + const auto v = value(path); + return { v.begin(), v.end() }; + } -}; // class +protected: + const std::array m_entries; + const std::array, N> m_values; +}; + +template +class CompressedResources final : public Resources +{ +public: + explicit CompressedResources( + const std::array &entries, + const std::array, N> &values, + const Compression algo, + const std::span &dictionary = {}) + : Resources(entries, values) + , m_compression(make_compression_ctx(algo, dictionary)) + { + } + ~CompressedResources() = default; + + [[nodiscard]] std::vector decompress(const int id_) const + { + return m_compression->decompress(Resources::value(id_)); + } + [[nodiscard]] std::vector decompress(const std::string_view &path) const + { + return m_compression->decompress(Resources::value(path)); + } + +protected: + const std::unique_ptr m_compression; +}; -} +} // namespace embed diff --git a/lib/embed_zstd.cpp b/lib/embed_zstd.cpp deleted file mode 100644 index d9be35a..0000000 --- a/lib/embed_zstd.cpp +++ /dev/null @@ -1,62 +0,0 @@ -#include "embed.h" -#include -#include - -using namespace embed; - -ZSTD_DDict *dictPtr = nullptr; - -template <> -Resources::Resources(const std::span &dictionary) -{ - if(!dictionary.empty()) { - dictPtr = ZSTD_createDDict(dictionary.data(), dictionary.size()); - } -} - -template <> -Resources::~Resources() -{ - if(dictPtr != nullptr) { - ZSTD_freeDDict(dictPtr); - } -} - -template <> -[[nodiscard]] std::vector Resources::decompress(const std::span &entry) -{ - /* Read the content size from the frame header. For simplicity we require - * that it is always present. By default, zstd will write the content size - * in the header when it is known. If you can't guarantee that the frame - * content size is always written into the header, either use streaming - * decompression, or ZSTD_decompressBound(). - */ - unsigned long long const rSize = ZSTD_getFrameContentSize(entry.data(), entry.size()); - assert(rSize != ZSTD_CONTENTSIZE_ERROR); //, "%s: not compressed by zstd!", fname); - assert(rSize != ZSTD_CONTENTSIZE_UNKNOWN); //, "%s: original size unknown!", fname); - //data rBuff(new unsigned char[rSize], rSize, true); - std::vector rBuff(rSize); - - /* Check that the dictionary ID matches. - * If a non-zstd dictionary is used, then both will be zero. - * By default zstd always writes the dictionary ID into the frame. - * Zstd will check if there is a dictionary ID mismatch as well. - */ - unsigned const expectedDictID = ZSTD_getDictID_fromDDict(dictPtr); - unsigned const actualDictID = ZSTD_getDictID_fromFrame(entry.data(), entry.size()); - assert(actualDictID == expectedDictID); //"DictID mismatch: expected %u got %u", - - /* Decompress using the dictionary. - * If you need to control the decompression parameters, then use the - * advanced API: ZSTD_DCtx_setParameter(), ZSTD_DCtx_refDDict(), and - * ZSTD_decompressDCtx(). - */ - ZSTD_DCtx *const dctx = ZSTD_createDCtx(); - assert(dctx != NULL); //, "ZSTD_createDCtx() failed!"); - size_t const dSize = ZSTD_decompress_usingDDict(dctx, rBuff.data(), rSize, entry.data(), entry.size(), dictPtr); - /* When zstd knows the content size, it will error if it doesn't match. */ - assert(dSize == rSize); //, "Impossible because zstd will check this condition!"); - - ZSTD_freeDCtx(dctx); - return rBuff; -} diff --git a/lib/zstd.cpp b/lib/zstd.cpp new file mode 100644 index 0000000..d2c27bc --- /dev/null +++ b/lib/zstd.cpp @@ -0,0 +1,59 @@ +#include "compressionctx.h" +#include +#include + +using namespace embed; + +ZSTD_DDict *dictPtr = nullptr; + +ZstdCompressionCtx::ZstdCompressionCtx(const std::span &dictionary) +{ + if(!dictionary.empty()) { + dictPtr = ZSTD_createDDict(dictionary.data(), dictionary.size()); + } +} + +ZstdCompressionCtx::~ZstdCompressionCtx() +{ + if(dictPtr != nullptr) { + ZSTD_freeDDict(dictPtr); + } +} + +[[nodiscard]] +std::vector ZstdCompressionCtx::decompress(const std::span &entry) const +{ + /* Read the content size from the frame header. For simplicity we require + * that it is always present. By default, zstd will write the content size + * in the header when it is known. If you can't guarantee that the frame + * content size is always written into the header, either use streaming + * decompression, or ZSTD_decompressBound(). + */ + unsigned long long const rSize = ZSTD_getFrameContentSize(entry.data(), entry.size()); + assert(rSize != ZSTD_CONTENTSIZE_ERROR); //, "%s: not compressed by zstd!", fname); + assert(rSize != ZSTD_CONTENTSIZE_UNKNOWN); //, "%s: original size unknown!", fname); + std::vector rBuff(rSize); + + /* Check that the dictionary ID matches. + * If a non-zstd dictionary is used, then both will be zero. + * By default zstd always writes the dictionary ID into the frame. + * Zstd will check if there is a dictionary ID mismatch as well. + */ + unsigned const expectedDictID = ZSTD_getDictID_fromDDict(dictPtr); + unsigned const actualDictID = ZSTD_getDictID_fromFrame(entry.data(), entry.size()); + assert(actualDictID == expectedDictID); //"DictID mismatch: expected %u got %u", + + /* Decompress using the dictionary. + * If you need to control the decompression parameters, then use the + * advanced API: ZSTD_DCtx_setParameter(), ZSTD_DCtx_refDDict(), and + * ZSTD_decompressDCtx(). + */ + ZSTD_DCtx *const dctx = ZSTD_createDCtx(); + assert(dctx != NULL); //, "ZSTD_createDCtx() failed!"); + size_t const dSize = ZSTD_decompress_usingDDict(dctx, rBuff.data(), rSize, entry.data(), entry.size(), dictPtr); + /* When zstd knows the content size, it will error if it doesn't match. */ + assert(dSize == rSize); //, "Impossible because zstd will check this condition!"); + + ZSTD_freeDCtx(dctx); + return rBuff; +} diff --git a/meson.build b/meson.build index 3cc04f1..d49e251 100644 --- a/meson.build +++ b/meson.build @@ -9,8 +9,7 @@ add_project_link_arguments(['-stdlib=libc++'], language : 'cpp') libzstd = dependency('libzstd') libembed_sourceset = import('sourceset').source_set() -libembed_sourceset.add(files('lib/embed.cpp')) -libembed_sourceset.add(when: libzstd, if_true: [ libzstd, files('lib/embed_zstd.cpp') ] ) +libembed_sourceset.add(when: libzstd, if_true: [ libzstd, files('lib/zstd.cpp') ] ) libembed_conf = libembed_sourceset.apply(configuration_data()) libembed = library('embed', diff --git a/readme.md b/readme.md index d75ac34..9c745ff 100644 --- a/readme.md +++ b/readme.md @@ -5,10 +5,7 @@ This is a resource compiler for C++, similar to Qt's rcc. It will generate hexdu ### requirements * rcc tool requires python * zstd compression requires zstd - -libembed: -* c++20 -* frozen: https://github.com/serge-sans-paille/frozen (constexpr string and unordered_map) +* c++20 (std::span) ### license diff --git a/scripts/rcc_format.py b/scripts/rcc_format.py index d0ab06d..7fcfe34 100644 --- a/scripts/rcc_format.py +++ b/scripts/rcc_format.py @@ -37,7 +37,7 @@ def write_header(file, namespace): def write_item(file, array_name, array_data): line_items = 0 - print("constexpr unsigned char {}[] = {{".format(array_name), file=file) + print("constexpr uint8_t {}[] = {{".format(array_name), file=file) for byte in array_data[0:len(array_data)]: line_items+=1 @@ -52,10 +52,15 @@ def write_item(file, array_name, array_data): print("constexpr size_t {}_len = {};\n".format(array_name, len(array_data)), file=file) def write_entries(file, resource_list): - print("constexpr auto entries = frozen::make_unordered_map>({", file=file) + print("constexpr std::array entries {", file=file) for f in resource_list: - print(" {{ \"{}\", std::span({}, {}_len) }},".format(f.alias, f.variable, f.variable), file=file) - print("});\n", file=file) + print(" \"{}\", ".format(f.alias), file=file) + print("};\n", file=file) + + print("constexpr std::array values {", file=file) + for f in resource_list: + print(" std::span( {}, {}_len ),".format(f.variable, f.variable), file=file) + print("};\n", file=file) def write_footer(file, namespace): print("\n}} // namespace {}".format(namespace), file=file) diff --git a/scripts/zstd.py b/scripts/zstd.py index f85e7da..2cfa149 100644 --- a/scripts/zstd.py +++ b/scripts/zstd.py @@ -16,6 +16,8 @@ def zstd(filelist, args): if args.dict is not None: write_item(args.output, 'dict', args.dict.read()) print("constexpr auto dictionary = std::span(dict, dict_len);", file=args.output) + else: + print("constexpr std::span dictionary {};", file=args.output) print("constexpr auto compression = embed::Zstd;", file=args.output) diff --git a/test/main.cpp b/test/main.cpp index f17ef67..f18332c 100644 --- a/test/main.cpp +++ b/test/main.cpp @@ -3,22 +3,23 @@ #include #include -embed::Resources zstd_ctx(zstd_data::dictionary); - int main(int, char **) { - for(const auto &pair : zstd_data::entries) { - const auto s = staticdata::entries.at(pair.first); - const auto v = zstd_ctx.decompress(pair.second); + constexpr auto size = staticdata::entries.size(); + constexpr embed::Resources static_ctx(staticdata::entries, staticdata::values); + const embed::CompressedResources zstd_ctx(zstd_data::entries, zstd_data::values, zstd_data::compression, zstd_data::dictionary); + for(std::size_t i = 0; i < size; ++i) { + const auto s = static_ctx.value(i); + const auto v = zstd_ctx.decompress(i); if(s.size() != v.size()) { - printf("failed comparing sizes at path [%s]\n", pair.first.data()); - printf("s: %li != v: %li\n", s.size(), v.size()); + printf(" raw data at path [%s] size=%li\n", staticdata::entries.at(i), s.size()); + printf(" zstd data at path [%s] size=%li\n", zstd_data::entries.at(i), v.size()); return EXIT_FAILURE; } if(!std::equal(s.begin(), s.end(), v.begin(), v.end())) { - printf("failed comparing values at path [%s]\n", pair.first.data()); + printf("failed comparing values at path [%s]\n", staticdata::entries.at(i)); for(const char &c : s) printf("%c", c); for(const char &c : v) -- cgit v1.2.1