From f9ea03470e3a0bbe67ccb0a531dcf0e39aec1e90 Mon Sep 17 00:00:00 2001 From: Aqua-sama Date: Mon, 30 Mar 2020 14:08:41 +0300 Subject: Move scripts/rcc to top level - add rcc generator for use when importing as subproject --- .gitignore | 2 ++ meson.build | 30 +++++++++++--------- rcc | 48 ++++++++++++++++++++++++++++++++ rcc_format/__init__.py | 2 ++ rcc_format/gen-resources.sh | 9 ++++++ rcc_format/util.py | 67 +++++++++++++++++++++++++++++++++++++++++++++ rcc_format/zstd.py | 43 +++++++++++++++++++++++++++++ scripts/gen-resources.sh | 9 ------ scripts/rcc | 49 --------------------------------- scripts/rcc_format.py | 67 --------------------------------------------- scripts/zstd.py | 43 ----------------------------- 11 files changed, 188 insertions(+), 181 deletions(-) create mode 100644 .gitignore create mode 100755 rcc create mode 100644 rcc_format/__init__.py create mode 100755 rcc_format/gen-resources.sh create mode 100644 rcc_format/util.py create mode 100644 rcc_format/zstd.py delete mode 100755 scripts/gen-resources.sh delete mode 100755 scripts/rcc delete mode 100644 scripts/rcc_format.py delete mode 100644 scripts/zstd.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..ce8c7f3 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +__pycache__ +build diff --git a/meson.build b/meson.build index d49e251..5c7cd65 100644 --- a/meson.build +++ b/meson.build @@ -1,10 +1,13 @@ project('libembed', ['cpp'], + version: '0.1', default_options: ['cpp_std=c++2a', 'warning_level=3'], ) +if not meson.is_subproject() # libstdc++ lacks std::span at the moment add_project_arguments(['-stdlib=libc++'], language: 'cpp') add_project_link_arguments(['-stdlib=libc++'], language : 'cpp') +endif libzstd = dependency('libzstd') @@ -12,40 +15,41 @@ libembed_sourceset = import('sourceset').source_set() libembed_sourceset.add(when: libzstd, if_true: [ libzstd, files('lib/zstd.cpp') ] ) libembed_conf = libembed_sourceset.apply(configuration_data()) -libembed = library('embed', - libembed_conf.sources(), - dependencies: libembed_conf.dependencies(), - include_directories: '3rd-party/frozen/include/' +libembed = library('embed', libembed_conf.sources(), + dependencies: libembed_conf.dependencies() ) -dep_embed = declare_dependency( +libembed_dep = declare_dependency( link_with: libembed, - include_directories: include_directories('lib/', '3rd-party/frozen/include') + include_directories: include_directories('lib/') ) prog_python = import('python').find_installation('python3') -resources_h = custom_target('resources.h', - output: 'resources.h', - input: 'scripts/rcc', - command: [prog_python, '@INPUT@', '--namespace=staticdata', '--output=@OUTPUT@', '-', files('test/resources.xrc')], +rcc = generator(prog_python, + output: '@BASENAME@.h', + arguments: [ meson.current_source_dir()/'rcc', '--output=@OUTPUT@', '@EXTRA_ARGS@', '@INPUT@' ] ) +if not meson.is_subproject() +resources_h = rcc.process(files('test/resources.xrc'), extra_args: [ '--namespace=staticdata', '-' ]) + zstd_dictionary = custom_target('zstd_dictionary', output: 'zstd_dict', - input: 'scripts/rcc', + input: 'rcc', command: [ prog_python, '@INPUT@', 'Zstd', '--train=@OUTPUT@', files('test/resources.xrc') ] ) zstd_resources_h = custom_target('zstd_resources.h', output: 'zstd_resources.h', - input: [ 'scripts/rcc', zstd_dictionary ], + input: [ 'rcc', zstd_dictionary ], command: [prog_python, '@INPUT0@', '--namespace=zstd_data', '--output=@OUTPUT@', 'Zstd', '--dict=@INPUT1@', files('test/resources.xrc')], ) test('libembed', executable('embed', sources: [ 'test/main.cpp', resources_h, zstd_resources_h ], - dependencies: [ dep_embed ] + dependencies: [ libembed_dep ] ) ) +endif # meson.is_subproject() diff --git a/rcc b/rcc new file mode 100755 index 0000000..00d453f --- /dev/null +++ b/rcc @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 + +import argparse +import sys +import rcc_format + +def none(filelist, args): + rcc_format.write_header(args.output, args.namespace) + + for f in filelist: + with open(f.path, 'rb') as contents: + rcc_format.write_item(args.output, f.variable, contents.read()) + + rcc_format.write_entries(args.output, filelist) + print("constexpr auto compression = embed::None;", file=args.output) + + rcc_format.write_footer(args.output, args.namespace) + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description='Resource Compiler for C++', + epilog='For a full list of compression options, check {mode} --help.', + ) + + mode = parser.add_subparsers(help='compression mode') + + none_mode = mode.add_parser('-') + none_mode.set_defaults(func=none) + + zstd_mode = mode.add_parser('Zstd', + description='use Zstd compression', + epilog='A dictionary is recommended if compressing many small files. size(source)/size(dictionary) should be >= 10' + ) + zstd_mode.add_argument('--binary', type=str, default='zstd', help='zstd binary name') + zstd_mode.add_argument('--train', type=argparse.FileType('wb'), help='train dictionary and exit') + zstd_mode.add_argument('-d', '--dict', type=argparse.FileType('rb'), help='use dictionary, recommended for many similar small files') + zstd_mode.add_argument('--dsize', type=int, default=512, help='dictionary size, used for training') + zstd_mode.add_argument('-l', '--level', type=int, default=19, help='compression level') + zstd_mode.set_defaults(func=rcc_format.zstd) + + parser.add_argument('input', type=argparse.FileType('rt'), help='input file (.xrc)') + parser.add_argument('-o', '--output', type=argparse.FileType('wt'), default=sys.stdout, help='output header file') + parser.add_argument('-n', '--namespace', type=str, default='resources', help='namespace') + + args=parser.parse_args() + + args.func(rcc_format.filelist(args.input), args) + diff --git a/rcc_format/__init__.py b/rcc_format/__init__.py new file mode 100644 index 0000000..133bcc1 --- /dev/null +++ b/rcc_format/__init__.py @@ -0,0 +1,2 @@ +from rcc_format.util import * +from rcc_format.zstd import zstd diff --git a/rcc_format/gen-resources.sh b/rcc_format/gen-resources.sh new file mode 100755 index 0000000..a1db9fd --- /dev/null +++ b/rcc_format/gen-resources.sh @@ -0,0 +1,9 @@ +#!/bin/sh + +echo '' +echo ' ' +for f in $@; do + echo ' '$f''; +done +echo ' ' +echo '' diff --git a/rcc_format/util.py b/rcc_format/util.py new file mode 100644 index 0000000..7fcfe34 --- /dev/null +++ b/rcc_format/util.py @@ -0,0 +1,67 @@ +from collections import namedtuple +import xml.etree.ElementTree as xml + +resource = namedtuple('resource', 'alias variable path') + +def to_variable_name(path): + name = path.replace('/', '_') + if name.endswith('.zstd'): + name = name[:-5] + name = name.replace('-', '_') + name = name.replace('.', '_') + return name + +def filelist(file): + root = xml.parse(file).getroot() + if root.tag != 'RCC': + return None + + files = [] + for child in root: + if child.tag == 'qresource': + prefix = child.attrib['prefix'] + for i in child: + alias = prefix + '/' + i.attrib['alias'] + variable = to_variable_name(i.text) + path = i.text + files.append(resource(alias, variable, path)) + + return files + +def write_header(file, namespace): + print("// Autogenerated binary file hexdump", file=file) + print("// This file may get overwritten by the build system\n", file=file) + print("#include \n", file=file) + print("namespace {} {{".format(namespace), file=file) + +def write_item(file, array_name, array_data): + line_items = 0 + + print("constexpr uint8_t {}[] = {{".format(array_name), file=file) + + for byte in array_data[0:len(array_data)]: + line_items+=1 + if line_items == 16: + print(" 0x{:02X},".format(byte), file=file) + line_items = 0 + else: + print(" 0x{:02X},".format(byte), file=file, end='') + + + print("};", file=file) + print("constexpr size_t {}_len = {};\n".format(array_name, len(array_data)), file=file) + +def write_entries(file, resource_list): + print("constexpr std::array entries {", file=file) + for f in resource_list: + print(" \"{}\", ".format(f.alias), file=file) + print("};\n", file=file) + + print("constexpr std::array values {", file=file) + for f in resource_list: + print(" std::span( {}, {}_len ),".format(f.variable, f.variable), file=file) + print("};\n", file=file) + +def write_footer(file, namespace): + print("\n}} // namespace {}".format(namespace), file=file) + diff --git a/rcc_format/zstd.py b/rcc_format/zstd.py new file mode 100644 index 0000000..34eeb64 --- /dev/null +++ b/rcc_format/zstd.py @@ -0,0 +1,43 @@ +import subprocess +from rcc_format.util import * + +def zstd(filelist, args): + if args.train is not None: + train(filelist, args.train, args.binary, args.dsize) + return + + write_header(args.output, args.namespace) + + for f in filelist: + with open(f.path, 'rb') as contents: + write_item(args.output, f.variable, compress(contents, args.binary, args.level, dictionary=args.dict)) + + write_entries(args.output, filelist) + if args.dict is not None: + write_item(args.output, 'dict', args.dict.read()) + print("constexpr auto dictionary = std::span(dict, dict_len);", file=args.output) + else: + print("constexpr std::span dictionary {};", file=args.output) + + print("constexpr auto compression = embed::Zstd;", file=args.output) + + write_footer(args.output, args.namespace) + +def train(filelist, output, zstd_bin, maxdict): + cmd = [ zstd_bin, '--train', '--maxdict=' + str(maxdict), '-o', output.name ] + + for f in filelist: + cmd.append(f.path) + + subprocess.run(cmd) + +def compress(file, zstd_bin, level, dictionary=None): + cmd = [ zstd_bin, '--compress', '--stdout', '-' + str(level) ] + + if dictionary is not None: + cmd.append('-D') + cmd.append(dictionary.name) + + cmd.append(file.name) + return subprocess.run(cmd, capture_output=True).stdout + diff --git a/scripts/gen-resources.sh b/scripts/gen-resources.sh deleted file mode 100755 index a1db9fd..0000000 --- a/scripts/gen-resources.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/sh - -echo '' -echo ' ' -for f in $@; do - echo ' '$f''; -done -echo ' ' -echo '' diff --git a/scripts/rcc b/scripts/rcc deleted file mode 100755 index 479b609..0000000 --- a/scripts/rcc +++ /dev/null @@ -1,49 +0,0 @@ -#!/usr/bin/env python3 - -import argparse -import sys -from zstd import zstd -from rcc_format import * - -def none(filelist, args): - write_header(args.output, args.namespace) - - for f in filelist: - with open(f.path, 'rb') as contents: - write_item(args.output, f.variable, contents.read()) - - write_entries(args.output, filelist) - print("constexpr auto compression = embed::None;", file=args.output) - - write_footer(args.output, args.namespace) - -if __name__ == "__main__": - parser = argparse.ArgumentParser( - description='Resource Compiler for C++', - epilog='For a full list of compression options, check {mode} --help.', - ) - - mode = parser.add_subparsers(help='compression mode') - - none_mode = mode.add_parser('-') - none_mode.set_defaults(func=none) - - zstd_mode = mode.add_parser('Zstd', - description='use Zstd compression', - epilog='A dictionary is recommended if compressing many small files. size(source)/size(dictionary) should be >= 10' - ) - zstd_mode.add_argument('--binary', type=str, default='zstd', help='zstd binary name') - zstd_mode.add_argument('--train', type=argparse.FileType('wb'), help='train dictionary and exit') - zstd_mode.add_argument('-d', '--dict', type=argparse.FileType('rb'), help='use dictionary, recommended for many similar small files') - zstd_mode.add_argument('--dsize', type=int, default=512, help='dictionary size, used for training') - zstd_mode.add_argument('-l', '--level', type=int, default=19, help='compression level') - zstd_mode.set_defaults(func=zstd) - - parser.add_argument('input', type=argparse.FileType('rt'), help='input file (.xrc)') - parser.add_argument('-o', '--output', type=argparse.FileType('wt'), default=sys.stdout, help='output header file') - parser.add_argument('-n', '--namespace', type=str, default='resources', help='namespace') - - args=parser.parse_args() - - args.func(filelist(args.input), args) - diff --git a/scripts/rcc_format.py b/scripts/rcc_format.py deleted file mode 100644 index 7fcfe34..0000000 --- a/scripts/rcc_format.py +++ /dev/null @@ -1,67 +0,0 @@ -from collections import namedtuple -import xml.etree.ElementTree as xml - -resource = namedtuple('resource', 'alias variable path') - -def to_variable_name(path): - name = path.replace('/', '_') - if name.endswith('.zstd'): - name = name[:-5] - name = name.replace('-', '_') - name = name.replace('.', '_') - return name - -def filelist(file): - root = xml.parse(file).getroot() - if root.tag != 'RCC': - return None - - files = [] - for child in root: - if child.tag == 'qresource': - prefix = child.attrib['prefix'] - for i in child: - alias = prefix + '/' + i.attrib['alias'] - variable = to_variable_name(i.text) - path = i.text - files.append(resource(alias, variable, path)) - - return files - -def write_header(file, namespace): - print("// Autogenerated binary file hexdump", file=file) - print("// This file may get overwritten by the build system\n", file=file) - print("#include \n", file=file) - print("namespace {} {{".format(namespace), file=file) - -def write_item(file, array_name, array_data): - line_items = 0 - - print("constexpr uint8_t {}[] = {{".format(array_name), file=file) - - for byte in array_data[0:len(array_data)]: - line_items+=1 - if line_items == 16: - print(" 0x{:02X},".format(byte), file=file) - line_items = 0 - else: - print(" 0x{:02X},".format(byte), file=file, end='') - - - print("};", file=file) - print("constexpr size_t {}_len = {};\n".format(array_name, len(array_data)), file=file) - -def write_entries(file, resource_list): - print("constexpr std::array entries {", file=file) - for f in resource_list: - print(" \"{}\", ".format(f.alias), file=file) - print("};\n", file=file) - - print("constexpr std::array values {", file=file) - for f in resource_list: - print(" std::span( {}, {}_len ),".format(f.variable, f.variable), file=file) - print("};\n", file=file) - -def write_footer(file, namespace): - print("\n}} // namespace {}".format(namespace), file=file) - diff --git a/scripts/zstd.py b/scripts/zstd.py deleted file mode 100644 index 2cfa149..0000000 --- a/scripts/zstd.py +++ /dev/null @@ -1,43 +0,0 @@ -import subprocess -from rcc_format import * - -def zstd(filelist, args): - if args.train is not None: - train(filelist, args.train, args.binary, args.dsize) - return - - write_header(args.output, args.namespace) - - for f in filelist: - with open(f.path, 'rb') as contents: - write_item(args.output, f.variable, compress(contents, args.binary, args.level, dictionary=args.dict)) - - write_entries(args.output, filelist) - if args.dict is not None: - write_item(args.output, 'dict', args.dict.read()) - print("constexpr auto dictionary = std::span(dict, dict_len);", file=args.output) - else: - print("constexpr std::span dictionary {};", file=args.output) - - print("constexpr auto compression = embed::Zstd;", file=args.output) - - write_footer(args.output, args.namespace) - -def train(filelist, output, zstd_bin, maxdict): - cmd = [ zstd_bin, '--train', '--maxdict=' + str(maxdict), '-o', output.name ] - - for f in filelist: - cmd.append(f.path) - - subprocess.run(cmd) - -def compress(file, zstd_bin, level, dictionary=None): - cmd = [ zstd_bin, '--compress', '--stdout', '-' + str(level) ] - - if dictionary is not None: - cmd.append('-D') - cmd.append(dictionary.name) - - cmd.append(file.name) - return subprocess.run(cmd, capture_output=True).stdout - -- cgit v1.2.1