[Libreoffice-commits] core.git: Branch 'feature/mar-updater' - 4 commits - bin/update Makefile.gbuild
Markus Mohrhard
markus.mohrhard at googlemail.com
Wed Apr 26 22:08:55 UTC 2017
Makefile.gbuild | 15 ++-----
bin/update/create_full_mar.py | 20 ++++++----
bin/update/create_full_mar_for_languages.py | 21 +++++-----
bin/update/create_partial_update.py | 55 ++++++++++++----------------
bin/update/path.py | 52 ++++++++++++++++++++++++++
5 files changed, 104 insertions(+), 59 deletions(-)
New commits:
commit 24add9c6be70083b05388dd401d98429f1387018
Author: Markus Mohrhard <markus.mohrhard at googlemail.com>
Date: Thu Apr 27 00:07:43 2017 +0200
use file instead of complete for the key of the partial update info
Change-Id: Ifbb482a4fd9c1c92e9cdd7769d4b16160de14a86
diff --git a/bin/update/create_partial_update.py b/bin/update/create_partial_update.py
index 419a8f65226b..23d4f6fe6a90 100755
--- a/bin/update/create_partial_update.py
+++ b/bin/update/create_partial_update.py
@@ -129,7 +129,7 @@ def main():
subprocess.call([os.path.join(current_dir_path, 'make_incremental_update.sh'), mar_file, update["complete"], current_build_path])
sign_mar_file(update_dir, config, mar_file, mar_name_prefix)
- partial_info = {"complete":get_file_info(mar_file, config.base_url), "from": build, "to": build_id, "languages": {}}
+ partial_info = {"file":get_file_info(mar_file, config.base_url), "from": build, "to": build_id, "languages": {}}
for lang, lang_info in update["languages"].items():
lang_name = generate_lang_file_name(build_id, build, mar_name_prefix, lang)
commit a14a2c2c66cc558902e8367c2bbbc9919f6489e7
Author: Markus Mohrhard <markus.mohrhard at googlemail.com>
Date: Wed Apr 26 23:58:57 2017 +0200
add missing file
Change-Id: I7cd9d541b90f4f6b38aa5e36e295e7677bb22f58
diff --git a/bin/update/path.py b/bin/update/path.py
new file mode 100644
index 000000000000..1bc14d70d940
--- /dev/null
+++ b/bin/update/path.py
@@ -0,0 +1,52 @@
+# -*- tab-width: 4; indent-tabs-mode: nil; py-indent-offset: 4 -*-
+#
+# This file is part of the LibreOffice project.
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+#
+
+import os
+import errno
+
+def mkdir_p(path):
+ try:
+ os.makedirs(path)
+ except OSError as exc: # Python >2.5
+ if exc.errno == errno.EEXIST and os.path.isdir(path):
+ pass
+ else:
+ raise
+
+class UpdaterPath(object):
+
+ def __init__(self, workdir):
+ self._workdir = workdir
+
+ def get_workdir(self):
+ return self._workdir
+
+ def get_update_dir(self):
+ return os.path.join(self._workdir, "update-info")
+
+ def get_current_build_dir(self):
+ return os.path.join(self._workdir, "mar", "current-build")
+
+ def get_mar_dir(self):
+ return os.path.join(self._workdir, "mar")
+
+ def get_previous_build_dir(self):
+ return os.path.join(self._workdir, "mar", "previous-build")
+
+ def get_language_dir(self):
+ return os.path.join(self.get_mar_dir(), "language")
+
+ def ensure_dir_exist(self):
+ mkdir_p(self.get_update_dir())
+ mkdir_p(self.get_current_build_dir())
+ mkdir_p(self.get_mar_dir())
+ mkdir_p(self.get_previous_build_dir())
+ mkdir_p(self.get_language_dir())
+
+# vim: set shiftwidth=4 softtabstop=4 expandtab:
commit 69d230a224f01abb591e00a6583b6216497d6f2a
Author: Markus Mohrhard <markus.mohrhard at googlemail.com>
Date: Wed Apr 26 23:58:05 2017 +0200
use the shared signing code
Change-Id: Ica0fad83820569623f3cc2f0217ac9cd8ee19257
diff --git a/bin/update/create_full_mar_for_languages.py b/bin/update/create_full_mar_for_languages.py
index 98e1b1c539e7..039521dd10af 100755
--- a/bin/update/create_full_mar_for_languages.py
+++ b/bin/update/create_full_mar_for_languages.py
@@ -9,6 +9,7 @@ from tools import uncompress_file_to_dir, get_file_info
from config import parse_config
from path import UpdaterPath
+from signing import sign_mar_file
current_dir_path = os.path.dirname(os.path.realpath(__file__))
@@ -38,8 +39,6 @@ def main():
config = parse_config(update_config)
- mar_executable = os.environ.get('MAR', 'mar')
-
language_pack_dir = os.path.join(workdir, "installation", product_name + "_languagepack", "archive", "install")
language_packs = os.listdir(language_pack_dir)
lang_infos = []
@@ -56,9 +55,7 @@ def main():
subprocess.call([os.path.join(current_dir_path, 'make_full_update.sh'), mar_file_name, directory])
- signed_mar_file = make_complete_mar_name(target_dir, filename_prefix + '_signed', language)
- subprocess.call([mar_executable, '-C', target_dir, '-d', config.certificate_path, '-n', config.certificate_name, '-s', mar_file_name, signed_mar_file])
- os.rename(signed_mar_file, mar_file_name)
+ sign_mar_file(target_dir, config, mar_file_name, filename_prefix)
lang_infos.append(create_lang_infos(mar_file_name, language, config.base_url))
commit b15c8181ae1f2306e25f1316c40ec0a9b1e531ab
Author: Markus Mohrhard <markus.mohrhard at googlemail.com>
Date: Wed Apr 26 23:49:31 2017 +0200
simplify path handling in the updater scripts
Change-Id: Idcf7f9cedc2368f6a4e4e10c2852cc9b1125b712
diff --git a/Makefile.gbuild b/Makefile.gbuild
index ed08c6217d13..44637577da52 100644
--- a/Makefile.gbuild
+++ b/Makefile.gbuild
@@ -32,12 +32,10 @@ create-update-info:
$(eval MAR_DIR := $(WORKDIR)/mar)
rm -rf $(MAR_DIR) || true
rm -rf $(UPDATE_DIR) || true
- mkdir -p $(UPDATE_DIR)
- mkdir -p $(MAR_DIR)/current-build
mkdir -p $(MAR_DIR)/language
- MAR=$(INSTDIR)/program/mar $(SRCDIR)/bin/update/create_full_mar.py "$(PRODUCTNAME)" "$(WORKDIR)" "$(UPDATE_DIR)" "$(MAR_DIR)/current-build/" "$(MAR_NAME_PREFIX)" "$(UPDATE_CONFIG)"
- MAR=$(INSTDIR)/program/mar $(SRCDIR)/bin/update/create_full_mar_for_languages.py "$(PRODUCTNAME)" "$(WORKDIR)" "$(UPDATE_DIR)" "$(MAR_DIR)/language" "$(MAR_NAME_PREFIX)" "$(UPDATE_CONFIG)"
- $(SRCDIR)/bin/update/create_build_config.py "$(PRODUCTNAME)" "$(VERSION)" "$(BUILDID)" "$(PLATFORM)" "$(UPDATE_DIR)" "$(UPDATE_CONFIG)" "$(UPDATE_CONFIG)"
+ MAR=$(INSTDIR)/program/mar $(SRCDIR)/bin/update/create_full_mar.py "$(PRODUCTNAME)" "$(WORKDIR)" "$(MAR_NAME_PREFIX)" "$(UPDATE_CONFIG)"
+ MAR=$(INSTDIR)/program/mar $(SRCDIR)/bin/update/create_full_mar_for_languages.py "$(PRODUCTNAME)" "$(WORKDIR)" "$(MAR_NAME_PREFIX)" "$(UPDATE_CONFIG)"
+ $(SRCDIR)/bin/update/create_build_config.py "$(PRODUCTNAME)" "$(VERSION)" "$(BUILDID)" "$(PLATFORM)" "$(UPDATE_DIR)" "$(UPDATE_CONFIG)"
upload-update-info:
$(eval BUILDID := $(shell git -C $(SRCDIR) log -1 --format=%H))
@@ -51,11 +49,6 @@ create-partial-info:
$(eval VERSION := $(LIBO_VERSION_MAJOR).$(LIBO_VERSION_MINOR).$(LIBO_VERSION_MICRO).$(LIBO_VERSION_PATCH)$(LIBO_VERSION_SUFFIX)$(LIBO_VERSION_SUFFIX_SUFFIX))
$(eval PLATFORM := $(RTL_OS)_$(RTL_ARCH))
$(eval MAR_NAME_PREFIX := $(PRODUCTNAME)_$(VERSION)_$(PLATFORM)_$(BUILDID))
- $(eval UPDATE_DIR := $(WORKDIR)/update-info)
- $(eval MAR_DIR := $(WORKDIR)/mar)
- $(eval CURRENT_BUILD := $(WORKDIR)/mar/current-build/)
- mkdir -p $(UPDATE_DIR)
- mkdir -p $(UPDATE_DIR)/previous-builds
- MAR=$(INSTDIR)/program/mar $(SRCDIR)/bin/update/create_partial_update.py "$(PRODUCTNAME)" "$(WORKDIR)" "$(UPDATE_DIR)" "$(MAR_DIR)/previous-builds/" "$(MAR_NAME_PREFIX)" "$(UPDATE_CONFIG)" "$(PLATFORM)" "$(CURRENT_BUILD)" "$(BUILDID)" "$(MAR_DIR)"
+ MAR=$(INSTDIR)/program/mar $(SRCDIR)/bin/update/create_partial_update.py "$(WORKDIR)" "$(MAR_NAME_PREFIX)" "$(UPDATE_CONFIG)" "$(PLATFORM)" "$(BUILDID)"
# vim: set noet sw=4 ts=4:
diff --git a/bin/update/create_full_mar.py b/bin/update/create_full_mar.py
index 649eafe2bfa6..a9d7aa2b9fe5 100755
--- a/bin/update/create_full_mar.py
+++ b/bin/update/create_full_mar.py
@@ -8,19 +8,19 @@ import json
from tools import uncompress_file_to_dir, get_file_info, make_complete_mar_name
from config import parse_config
from signing import sign_mar_file
+from path import UpdaterPath
current_dir_path = os.path.dirname(os.path.realpath(__file__))
+def ensure_dir_exist()
+
def main():
- print(sys.argv)
- if len(sys.argv) < 7:
- print("Usage: create_full_mar_for_languages.py $PRODUCTNAME $WORKDIR $TARGETDIR $TEMPDIR $FILENAMEPREFIX $UPDATE_CONFIG")
+ if len(sys.argv) < 5:
+ print("Usage: create_full_mar_for_languages.py $PRODUCTNAME $WORKDIR $FILENAMEPREFIX $UPDATE_CONFIG")
sys.exit(1)
- update_config = sys.argv[6]
- filename_prefix = sys.argv[5]
- temp_dir = sys.argv[4]
- target_dir = sys.argv[3]
+ update_config = sys.argv[4]
+ filename_prefix = sys.argv[3]
workdir = sys.argv[2]
product_name = sys.argv[1]
@@ -28,6 +28,12 @@ def main():
print("missing update config")
sys.exit(1)
+ update_path = UpdaterPath(workdir)
+ update_path.ensure_dir_exist()
+
+ target_dir = update_path.get_update_dir()
+ temp_dir = update_path.get_current_build_dir()
+
config = parse_config(update_config)
tar_dir = os.path.join(workdir, "installation", product_name, "archive", "install", "en-US")
diff --git a/bin/update/create_full_mar_for_languages.py b/bin/update/create_full_mar_for_languages.py
index 7daf5fe6034e..98e1b1c539e7 100755
--- a/bin/update/create_full_mar_for_languages.py
+++ b/bin/update/create_full_mar_for_languages.py
@@ -8,6 +8,7 @@ import json
from tools import uncompress_file_to_dir, get_file_info
from config import parse_config
+from path import UpdaterPath
current_dir_path = os.path.dirname(os.path.realpath(__file__))
@@ -22,18 +23,19 @@ def create_lang_infos(mar_file_name, language, url):
return data
def main():
- print(sys.argv)
- if len(sys.argv) < 7:
+ if len(sys.argv) < 5:
print("Usage: create_full_mar_for_languages.py $PRODUCTNAME $WORKDIR $TARGETDIR $TEMPDIR $FILENAMEPREFIX $UPDATE_CONFIG")
sys.exit(1)
- update_config = sys.argv[6]
- filename_prefix = sys.argv[5]
- temp_dir = sys.argv[4]
- target_dir = sys.argv[3]
+ update_config = sys.argv[4]
+ filename_prefix = sys.argv[3]
workdir = sys.argv[2]
product_name = sys.argv[1]
+ updater_path = UpdaterPath(workdir)
+ target_dir = updater_path.get_update_dir()
+ temp_dir = updater_path.get_language_dir()
+
config = parse_config(update_config)
mar_executable = os.environ.get('MAR', 'mar')
diff --git a/bin/update/create_partial_update.py b/bin/update/create_partial_update.py
index ca194325d20d..419a8f65226b 100755
--- a/bin/update/create_partial_update.py
+++ b/bin/update/create_partial_update.py
@@ -10,12 +10,19 @@ import json
from config import parse_config
from uncompress_mar import extract_mar
-from tools import get_file_info
+from tools import get_file_info, get_hash
from signing import sign_mar_file
+from path import UpdaterPath, mkdir_p
+
BUF_SIZE = 1024
current_dir_path = os.path.dirname(os.path.realpath(__file__))
+def InvalidFileException(Exception):
+
+ def __init__(self, *args, **kwargs):
+ super().__init__(self, *args, **kwargs)
+
def download_file(filepath, url, hash_string):
with open(filepath, "wb") as f:
response = requests.get(url, stream=True)
@@ -26,26 +33,10 @@ def download_file(filepath, url, hash_string):
for block in response.iter_content(1024):
f.write(block)
- with open(filepath, "rb") as f:
- sha512 = hashlib.sha512()
- while True:
- data = f.read(BUF_SIZE)
- if not data:
- break
- sha512.update(data)
- file_hash = sha512.hexdigest()
+ file_hash = get_hash(filepath)
if file_hash != hash_string:
- pass
-
-def mkdir_p(path):
- try:
- os.makedirs(path)
- except OSError as exc: # Python >2.5
- if exc.errno == errno.EEXIST and os.path.isdir(path):
- pass
- else:
- raise
+ raise InvalidFileException()
def handle_language(lang_entries, filedir):
mar = os.environ.get('MAR', 'mar')
@@ -109,16 +100,20 @@ def add_single_dir(path):
return dir_name[0]
def main():
- product_name = sys.argv[1]
- workdir = sys.argv[2]
- update_dir = sys.argv[3]
- temp_dir = sys.argv[4]
- mar_name_prefix = sys.argv[5]
- update_config = sys.argv[6]
- platform = sys.argv[7]
- current_build_path = sys.argv[8]
- build_id = sys.argv[9]
- mar_dir = sys.argv[10]
+ workdir = sys.argv[1]
+
+ updater_path = UpdaterPath(workdir)
+ updater_path.ensure_dir_exist()
+
+ mar_name_prefix = sys.argv[2]
+ update_config = sys.argv[3]
+ platform = sys.argv[4]
+ build_id = sys.argv[5]
+
+ current_build_path = updater_path.get_current_build_dir()
+ mar_dir = updater_path.get_mar_dir()
+ temp_dir = updater_path.get_previous_build_dir()
+ update_dir = updater_path.get_update_dir()
current_build_path = add_single_dir(current_build_path)
More information about the Libreoffice-commits
mailing list