[Libreoffice-commits] core.git: postprocess/CustomTarget_images.mk solenv/bin
Jens Carl
j.carl43 at gmx.de
Thu Sep 28 07:14:46 UTC 2017
postprocess/CustomTarget_images.mk | 2
solenv/bin/pack_images.py | 590 +++++++++++++++++++++++++++++++++++++
2 files changed, 591 insertions(+), 1 deletion(-)
New commits:
commit 6a313ab973378e82715cfda7431a8abbe474fd06
Author: Jens Carl <j.carl43 at gmx.de>
Date: Tue Aug 29 04:48:13 2017 +0000
tdf#106894: Rewrite packimages.pl in Python (pack_images.py)
Change-Id: I2e9054775941b38392ba3ee38575891b5339249e
Reviewed-on: https://gerrit.libreoffice.org/42790
Reviewed-by: Samuel Mehrbrodt <Samuel.Mehrbrodt at cib.de>
Tested-by: Samuel Mehrbrodt <Samuel.Mehrbrodt at cib.de>
diff --git a/postprocess/CustomTarget_images.mk b/postprocess/CustomTarget_images.mk
index 44899e6105a5..603c73d522d4 100644
--- a/postprocess/CustomTarget_images.mk
+++ b/postprocess/CustomTarget_images.mk
@@ -33,7 +33,7 @@ $(packimages_DIR)/%.zip : \
$(call gb_Output_announce,$(subst $(WORKDIR)/,,$@),$(true),PRL,2)
$(call gb_Helper_abbreviate_dirs, \
ILSTFILE=$(call var2file,$(shell $(gb_MKTEMP)),100,$(filter %.ilst,$^)) && \
- $(PERL) $(SRCDIR)/solenv/bin/packimages.pl \
+ $(PYTHON) $(SRCDIR)/solenv/bin/pack_images.py \
$(if $(DEFAULT_THEME),\
-g $(packimages_DIR) -m $(packimages_DIR) -c $(packimages_DIR),\
-g $(SRCDIR)/icon-themes/$(subst images_,,$*) -m $(SRCDIR)/icon-themes/$(subst images_,,$*) -c $(SRCDIR)/icon-themes/$(subst images_,,$*) \
diff --git a/solenv/bin/pack_images.py b/solenv/bin/pack_images.py
new file mode 100755
index 000000000000..0e1a5fe329f0
--- /dev/null
+++ b/solenv/bin/pack_images.py
@@ -0,0 +1,590 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# -*- tab-width: 4; indent-tabs-mode: nil; py-indent-offset: 4 -*-
+#
+# This file is part of the LibreOffice project.
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+#
+""" Pack images into archives. """
+
+from __future__ import with_statement
+
+import argparse
+from collections import OrderedDict
+import logging
+import os
+import shutil
+import sys
+import tempfile
+import zipfile
+
+
+logging.basicConfig(format='%(message)s')
+LOGGER = logging.getLogger()
+
+
+def main(args):
+ """ Main function. """
+
+ tmp_output_file = "%s.%d.tmp" % (args.output_file, os.getpid())
+
+ # Sanity checks
+ if not os.path.exists(args.imagelist_file):
+ LOGGER.error("imagelist_file '%s' doesn't exists", args.imagelist_file)
+ sys.exit(2)
+
+ out_path = os.path.dirname(args.output_file)
+ for path in (out_path, args.global_path, args.module_path):
+ if not os.path.exists(path):
+ LOGGER.error("Path '%s' doesn't exists", path)
+ sys.exit(2)
+ if not os.access(path, os.X_OK):
+ LOGGER.error("Unable to search path %s", path)
+ sys.exit(2)
+
+ if not os.access(out_path, os.W_OK):
+ LOGGER.error("Unable to write into path %s", out_path)
+
+ custom_paths = []
+ for path in args.custom_path:
+ if not os.path.exists(path):
+ LOGGER.warning("Skipping non-existing path: %s", path)
+ continue
+ elif not os.access(path, os.X_OK):
+ LOGGER.error("Unable to search path %s", path)
+ sys.exit(2)
+ continue
+
+ custom_paths.append(path)
+
+ imagelist_filenames = get_imagelist_filenames(args.imagelist_file)
+ global_image_list, module_image_list = parse_image_list(imagelist_filenames)
+ custom_image_list = find_custom(custom_paths)
+
+ links = {}
+ read_links(links, ARGS.global_path)
+ for path in custom_paths:
+ read_links(links, path)
+ check_links(links)
+
+ zip_list = create_zip_list(global_image_list, module_image_list, custom_image_list,
+ args.global_path, args.module_path)
+ remove_links_from_zip_list(zip_list, links)
+
+ if check_rebuild(args.output_file, imagelist_filenames, custom_paths, zip_list):
+ tmp_dir = copy_images(zip_list)
+ create_zip_archive(zip_list, links, tmp_dir, tmp_output_file, args.sort_file)
+
+ replace_zip_file(tmp_output_file, args.output_file)
+ try:
+ LOGGER.info("Remove temporary directory %s", tmp_dir)
+ shutil.rmtree(tmp_dir)
+ except Exception as e:
+ LOGGER.error("Unable to remove temporary directory %s", tmp_dir)
+ sys.exit(2)
+ else:
+ LOGGER.info("No rebuild needed. %s is up to date.", args.output_file)
+
+
+def check_rebuild(zip_file, imagelist_filenames, custom_paths, zip_list):
+ """ Check if a rebuild is needed.
+
+ :type zip_file: str
+ :param zip_file: File to check against (use st_mtime).
+
+ :type imagelist_filenames: dict
+ :param imagelist_filenames: List of imagelist filename.
+
+ :type custom_paths: list
+ :param custom_paths: List of paths to use with links.txt files.
+
+ :type zip_list: dict
+ :param zip_list: List of filenames to create the zip archive.
+
+ :rtype: bool
+ :return: True if rebuild is needed and False if not.
+ """
+
+ if not os.path.exists(zip_file):
+ return True
+
+ zip_file_stat = os.stat(zip_file)
+
+ def compare_modtime(filenames):
+ """ Check if modification time of zip archive is older the provided
+ list of files.
+
+ :type filenames: dict
+ :param filesnames: List of filenames to check against.
+
+ :rtype: bool
+ :return: True if zip archive is older and False if not.
+ """
+ for filename, path in filenames.items():
+ filename = os.path.join(path, filename) if filename else path
+ if os.path.exists(filename) and zip_file_stat.st_mtime < os.stat(filename).st_mtime:
+ return True
+ return False
+
+ if compare_modtime(imagelist_filenames):
+ return True
+ if compare_modtime(zip_list):
+ return True
+ for path in custom_paths:
+ link_file = os.path.join(path, 'links.txt')
+ if os.path.exists(link_file):
+ if zip_file_stat.st_mtime < os.stat(link_file).st_mtime:
+ return True
+
+ return False
+
+
+def replace_zip_file(src, dst):
+ """ Replace the old archive file with the newly created one.
+
+ :type src: str
+ :param src: Source file name.
+
+ :type dst: str
+ :param dst: Destination file name.
+ """
+ LOGGER.info("Replace old zip archive with new archive")
+ if os.path.exists(dst):
+ try:
+ os.unlink(dst)
+ except OSError as e:
+ if os.path.exists(src):
+ os.unlink(src)
+
+ LOGGER.error("Unable to unlink old archive '%s'", dst)
+ LOGGER.error(str(e))
+ sys.exit(1)
+
+ try:
+ LOGGER.info("Copy archive '%s' to '%s'", src, dst)
+ shutil.copyfile(src, dst)
+ except (shutil.SameFileError, OSError) as e:
+ os.unlink(src)
+ LOGGER.error("Cannot copy file '%s' to %s: %s", src, dst, str(e))
+ sys.exit(1)
+
+
+def optimize_zip_layout(zip_list, sort_file=None):
+ """ Optimzie the zip layout by ordering the list of filename alphabetically
+ or with provided sort file (can be partly).
+
+ :type zip_list: dict
+ :param zip_list: List of files to include in the zip archive.
+
+ :type sort_file: str
+ :param sort_file: Path/Filename to file with sort order.
+
+ :rtype: OrderedDict
+ :return: Dict with right sort order.
+ """
+ if sort_file is None:
+ LOGGER.info("No sort file provided")
+ LOGGER.info("Sorting entries alphabetically")
+
+ return OrderedDict(sorted(zip_list.items(), key=lambda t: t[0]))
+
+ LOGGER.info("Sort entries from file '%s'", sort_file)
+ sorted_zip_list = OrderedDict()
+ try:
+ fh = open(sort_file)
+ except IOError:
+ LOGGER.error("Cannot open file %s", sort_file)
+ sys.exit(1)
+ else:
+ with fh:
+ for line in fh:
+ line = line.strip()
+ if line == '' or line.startswith('#'):
+ continue
+
+ if line in zip_list:
+ sorted_zip_list[line] = ''
+ else:
+ LOGGER.warning("Unknown file '%s'", line)
+
+ for key in sorted(zip_list.keys()):
+ if key not in sorted_zip_list:
+ sorted_zip_list[key] = ''
+
+ return sorted_zip_list
+
+
+def create_zip_archive(zip_list, links, tmp_dir, tmp_zip_file, sort_file=None):
+ """ Create the zip archive.
+
+ :type zip_list: dict
+ :param zip_list: All filenames to be included in the archive.
+
+ :type links: dict
+ :param links: All filenames to create links.txt file.
+
+ :type tmp_dir: str
+ :param tmp_dir: Path to tempory saved files.
+
+ :type tmp_zip_file: str
+ :param tmp_zip_file: Filename/Path of temporary zip archive.
+
+ :type sort_file: str|None
+ :param sort_file: Optional filename with sort order to apply.
+ """
+ LOGGER.info("Creating image archive")
+
+ old_pwd = os.getcwd()
+ os.chdir(tmp_dir)
+
+ ordered_zip_list = optimize_zip_layout(zip_list, sort_file)
+
+ with zipfile.ZipFile(tmp_zip_file, 'w') as tmp_zip:
+ if links.keys():
+ LOGGER.info("Add file 'links.txt' to zip archive")
+ create_links_file(tmp_dir, links)
+ tmp_zip.write('links.txt')
+
+ for link in ordered_zip_list:
+ LOGGER.info("Add file '%s' from path '%s' to zip archive", link, tmp_dir)
+ try:
+ tmp_zip.write(link)
+ except OSError:
+ LOGGER.warning("Unable to add file '%s' to zip archive", link)
+
+ os.chdir(old_pwd)
+
+
+def create_links_file(path, links):
+ """ Create file links.txt. Contains all links.
+
+ :type path: str
+ :param path: Path where to create the file.
+
+ :type links: dict
+ :param links: Dict with links (source -> target).
+ """
+ LOGGER.info("Create file links.txt")
+
+ try:
+ filename = os.path.join(path, 'links.txt')
+ fh = open(filename, 'w')
+ except IOError:
+ LOGGER.error("Cannot open file %s", filename)
+ sys.exit(1)
+ else:
+ with fh:
+ for key in sorted(links.keys()):
+ fh.write("%s %s\n" % (key, links[key]))
+
+
+def copy_images(zip_list):
+ """ Create a temporary directory and copy images to that directory.
+
+ :type zip_list: dict
+ :param zip_list: Dict with all files.
+
+ :rtype: str
+ :return: Path of the temporary directory.
+ """
+ LOGGER.info("Copy image files to temporary directory")
+
+ tmp_dir = tempfile.mkdtemp()
+ for key, value in zip_list.items():
+ path = os.path.join(value, key)
+ out_path = os.path.join(tmp_dir, key)
+
+ LOGGER.debug("Copying '%s' to '%s'", path, out_path)
+ if os.path.exists(path):
+ dirname = os.path.dirname(out_path)
+ if not os.path.exists(dirname):
+ os.makedirs(dirname)
+
+ try:
+ shutil.copyfile(path, out_path)
+ except (shutil.SameFileError, OSError) as e:
+ LOGGER.error("Cannot add file '%s' to image dir: %s", path, str(e))
+ sys.exit(1)
+
+ LOGGER.debug("Temporary directory '%s'", tmp_dir)
+ return tmp_dir
+
+
+def remove_links_from_zip_list(zip_list, links):
+ """ Remove any files from zip list that are linked.
+
+ :type zip_list: dict
+ :param zip_list: Files to include in the zip archive.
+
+ :type links: dict
+ :param links: Images which are linked.
+ """
+ LOGGER.info("Remove linked files from zip list")
+
+ for link in links.keys():
+ if link in zip_list:
+ del zip_list[link]
+
+ LOGGER.debug("Cleaned zip list:\n%s", "\n".join(zip_list))
+
+
+def create_zip_list(global_image_list, module_image_list, custom_image_list, global_path, module_path):
+ """ Create list of images for the zip archive.
+
+ :type global_image_list: dict
+ :param global_image_list: Global images.
+
+ :type module_image_list: dict
+ :param module_image_list: Module images.
+
+ :type custom_image_list: dict
+ :param custom_image_list: Custom images.
+
+ :type global_path: str
+ :param global_path: Global path (use when no custom path is available).
+
+ :type module_path: str
+ :param module_path: Module path (use when no custom path is available).
+
+ :rtype: dict
+ :return: List of images to include in zip archive.
+ """
+ LOGGER.info("Assemble image list")
+
+ zip_list = {}
+ duplicates = []
+
+ for key in global_image_list.keys():
+ if key in module_image_list:
+ duplicates.append(key)
+ continue
+
+ if key in custom_image_list:
+ zip_list[key] = custom_image_list[key]
+ continue
+
+ zip_list[key] = global_path
+
+ for key in module_image_list.keys():
+ if key in custom_image_list:
+ zip_list[key] = custom_image_list[key]
+ continue
+
+ zip_list[key] = module_path
+
+ if duplicates:
+ LOGGER.warning("Found duplicate entries in 'global' and 'module' list")
+ LOGGER.warning("\n".join(duplicates))
+
+ LOGGER.debug("Assembled image list:\n%s", "\n".join(zip_list))
+ return zip_list
+
+
+def check_links(links):
+ """ Check that no link points to another link.
+
+ :type links: dict
+ :param links: Links to icons
+ """
+
+ stop = False
+
+ for link, target in links.items():
+ if target in links.keys():
+ LOGGER.error("Link %s -> %s -> %s", link, target, links[target])
+ stop = True
+
+ if stop:
+ LOGGER.error("Some icons in links.txt were found to link to other linked icons")
+ sys.exit(1)
+
+
+def read_links(links, path):
+ """ Read links from file.
+
+ :type links: dict
+ :param links: Hash to store all links
+
+ :type path: str
+ :param path: Path to use
+ """
+
+ filename = os.path.join(path, "links.txt")
+ LOGGER.info("Read links from file '%s'", filename)
+ if not os.path.isfile(filename):
+ LOGGER.info("No file to read")
+ return
+
+ try:
+ fh = open(filename)
+ except IOError:
+ LOGGER.error("Cannot open file %s", filename)
+ sys.exit(1)
+ else:
+ with fh:
+ for line in fh:
+ line = line.strip()
+ if line == '' or line.startswith('#'):
+ continue
+
+ tmp = line.split(' ')
+ if len(tmp) == 2:
+ links[tmp[0]] = tmp[1]
+ else:
+ LOGGER.error("Malformed links line '%s' in file %s", line, filename)
+ sys.exit(1)
+
+ LOGGER.debug("Read links:\n%s", "\n".join(links))
+
+
+def find_custom(custom_paths):
+ """ Find all custom images.
+
+ :type custom_paths: list
+ :param custom_paths: List of custom paths to search within.
+
+ :rtype: dict
+ :return: List of all custom images.
+ """
+ LOGGER.info("Find all images in custom paths")
+
+ custom_image_list = {}
+ for path in custom_paths:
+ # find all png files under the path including subdirs
+ custom_files = [val for sublist in [
+ [os.path.join(i[0], j) for j in i[2]
+ if j.endswith('.png') and os.path.isfile(os.path.join(i[0], j))]
+ for i in os.walk(path)] for val in sublist]
+
+ for filename in custom_files:
+ if filename.startswith(path):
+ key = filename.replace(os.path.join(path, ''), '')
+ if key not in custom_image_list:
+ custom_image_list[key] = path
+
+ LOGGER.debug("Custom image list:\n%s", "\n".join(custom_image_list.keys()))
+ return custom_image_list
+
+
+def parse_image_list(imagelist_filenames):
+ """ Parse and extract filename from the imagelist files.
+
+ :type imagelist_filenames: list
+ :param imagelist_filenames: List of imagelist files.
+
+ :rtype: tuple
+ :return: Tuple with dicts containing the global or module image list.
+ """
+
+ global_image_list = {}
+ module_image_list = {}
+ for imagelist_filename in imagelist_filenames.keys():
+ LOGGER.info("Parsing '%s'", imagelist_filename)
+
+ try:
+ fh = open(imagelist_filename)
+ except IOError as e:
+ LOGGER.error("Cannot open imagelist file %s", imagelist_filename)
+ sys.exit(2)
+ else:
+ line_count = 0
+ with fh:
+ for line in fh:
+ line_count += 1
+ line = line.strip()
+ if line == '' or line.startswith('#'):
+ continue
+ # clean up backslashes and double slashes
+ line = line.replace('\\', '/')
+ line = line.replace('//', '/')
+
+ if line.startswith('%GLOBALRES%'):
+ key = "res/%s" % line.replace('%GLOBALRES%/', '')
+ if key in global_image_list:
+ global_image_list[key] += 1
+ else:
+ global_image_list[key] = 0
+ continue
+
+ if line.startswith('%MODULE%'):
+ key = line.replace('%MODULE%/', '')
+ if key in global_image_list:
+ module_image_list[key] += 1
+ else:
+ module_image_list[key] = 0
+ continue
+
+ LOGGER.error("Cannot parse line %s:%d", imagelist_filename, line_count)
+ sys.exit(1)
+
+ LOGGER.debug("Global image list:\n%s", "\n".join(global_image_list))
+ LOGGER.debug("Module image list:\n%s", "\n".join(module_image_list))
+ return global_image_list, module_image_list
+
+
+def get_imagelist_filenames(filename):
+ """ Extract a list of imagelist filenames.
+
+ :type filename: str
+ :param filename: Name of file from extracting the list.
+
+ :rtype: dict
+ :return: List of imagelist filenames.
+ """
+ LOGGER.info("Extract the imagelist filenames")
+
+ imagelist_filenames = {}
+ try:
+ fh = open(filename)
+ except IOError:
+ LOGGER.error("Cannot open imagelist file %s", filename)
+ sys.exit(1)
+ else:
+ with fh:
+ for line in fh:
+ line = line.strip()
+ if not line or line == '':
+ continue
+
+ for line_split in line.split(' '):
+ line_split.strip()
+ imagelist_filenames[line_split] = ''
+
+ LOGGER.debug("Extraced imagelist filenames:\n%s", "\n".join(imagelist_filenames.keys()))
+ return imagelist_filenames
+
+
+if __name__ == '__main__':
+ parser = argparse.ArgumentParser("Pack images into archives")
+ parser.add_argument('-o', '--output-file', dest='output_file',
+ action='store', required=True,
+ help='path to output archive')
+ parser.add_argument('-l', '--imagelist-file', dest='imagelist_file',
+ action='store', required=True,
+ help='file containing list of image list file')
+ parser.add_argument('-s', '--sort-file', dest='sort_file',
+ action='store', required=True, default=None,
+ help='image sort order file')
+ parser.add_argument('-c', '--custom-path', dest='custom_path',
+ action='append', required=True,
+ help='path to custom path directory')
+ parser.add_argument('-g', '--global-path', dest='global_path',
+ action='store', required=True,
+ help='path to global images directory')
+ parser.add_argument('-m', '--module-path', dest='module_path',
+ action='store', required=True,
+ help='path to module images directory')
+ parser.add_argument('-v', '--verbose', dest='verbose',
+ action='count', default=0,
+ help='set the verbosity (can be used multiple times)')
+
+ ARGS = parser.parse_args()
+ LOGGER.setLevel(logging.ERROR - (10 * ARGS.verbose if ARGS.verbose <= 3 else 3))
+
+ main(ARGS)
+ sys.exit(0)
+
+# vim: set shiftwidth=4 softtabstop=4 expandtab:
More information about the Libreoffice-commits
mailing list