[Libreoffice-commits] core.git: oox/CustomTarget_generated.mk oox/source solenv/bin solenv/gbuild
Jussi Pakkanen (via logerrit)
logerrit at kemper.freedesktop.org
Fri Mar 6 11:32:07 UTC 2020
oox/CustomTarget_generated.mk | 4 -
oox/source/token/namespaces.pl | 99 -----------------------------------------
oox/source/token/namespaces.py | 85 +++++++++++++++++++++++++++++++++++
oox/source/token/properties.pl | 58 ------------------------
oox/source/token/properties.py | 50 ++++++++++++++++++++
solenv/bin/generate-tokens.pl | 69 ----------------------------
solenv/bin/generate-tokens.py | 80 +++++++++++++++++++++++++++++++++
solenv/gbuild/CustomTarget.mk | 5 +-
8 files changed, 220 insertions(+), 230 deletions(-)
New commits:
commit 6ec05ef6fef789f5646b773495285e845e155d77
Author: Jussi Pakkanen <jpakkane at gmail.com>
AuthorDate: Sat Feb 22 21:36:13 2020 +0200
Commit: Mike Kaganski <mike.kaganski at collabora.com>
CommitDate: Fri Mar 6 12:31:33 2020 +0100
tdf#130911: convert some token generation scripts from Perl to Python.
Change-Id: Id655b6a0cee7bdfe4804941f20fe358af8f3185e
Reviewed-on: https://gerrit.libreoffice.org/c/core/+/89477
Tested-by: Jenkins
Reviewed-by: Mike Kaganski <mike.kaganski at collabora.com>
Reviewed-by: Michael Stahl <michael.stahl at cib.de>
diff --git a/oox/CustomTarget_generated.mk b/oox/CustomTarget_generated.mk
index 462aef2e03a3..860ae447a576 100644
--- a/oox/CustomTarget_generated.mk
+++ b/oox/CustomTarget_generated.mk
@@ -32,8 +32,8 @@ $(oox_MISC)/oox-drawingml-adj-names : \
# Generate tokens for oox
$(eval $(call gb_CustomTarget_token_hash,oox/generated,tokenhash.inc,tokenhash.gperf))
-$(eval $(call gb_CustomTarget_generate_tokens,oox/generated,oox,oox/source/token,namespaces,namespace,namespaces.txt,namespaces-strict,namespaces.pl))
-$(eval $(call gb_CustomTarget_generate_tokens,oox/generated,oox,oox/source/token,properties,property,,,properties.pl))
+$(eval $(call gb_CustomTarget_generate_tokens,oox/generated,oox,oox/source/token,namespaces,namespace,namespaces.txt,namespaces-strict,namespaces.py))
+$(eval $(call gb_CustomTarget_generate_tokens,oox/generated,oox,oox/source/token,properties,property,,,properties.py))
$(eval $(call gb_CustomTarget_generate_tokens,oox/generated,oox,oox/source/token,tokens,token,tokenhash.gperf))
$(call gb_CustomTarget_get_target,oox/generated) : \
diff --git a/oox/source/token/namespaces.pl b/oox/source/token/namespaces.pl
deleted file mode 100644
index abbb70814255..000000000000
--- a/oox/source/token/namespaces.pl
+++ /dev/null
@@ -1,99 +0,0 @@
-#
-# This file is part of the LibreOffice project.
-#
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-#
-# This file incorporates work covered by the following license notice:
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed
-# with this work for additional information regarding copyright
-# ownership. The ASF licenses this file to you under the Apache
-# License, Version 2.0 (the "License"); you may not use this file
-# except in compliance with the License. You may obtain a copy of
-# the License at http://www.apache.org/licenses/LICENSE-2.0 .
-#
-
-$ARGV0 = shift @ARGV;
-$ARGV1 = shift @ARGV;
-$ARGV2 = shift @ARGV;
-$ARGV3 = shift @ARGV;
-$ARGV4 = shift @ARGV;
-$ARGV5 = shift @ARGV;
-
-# parse input file
-
-open( INFILE, $ARGV0 ) or die "cannot open input file: $!";
-my %namespaces;
-while( <INFILE> )
-{
- # trim newline
- chomp( $_ );
- # trim leading/trailing whitespace
- $_ =~ s/^\s*//g;
- $_ =~ s/\s*$//g;
- # trim comments
- $_ =~ s/^#.*//;
- # skip empty lines
- if( $_ )
- {
- # check for valid characters
- $_ =~ /^([a-zA-Z][a-zA-Z0-9]*)\s+([a-zA-Z0-9-.:\/]+)\s*$/ or die "Error: invalid character in input data";
- $namespaces{$1} = $2;
- }
-}
-close( INFILE );
-
-# OOXML strict namespaces
-
-open( INFILE_STRICT, $ARGV4 ) or die "cannot open input file: $!";
-my %namespaces_strict;
-while( <INFILE_STRICT> )
-{
- # trim newline
- chomp( $_ );
- # trim leading/trailing whitespace
- $_ =~ s/^\s*//g;
- $_ =~ s/\s*$//g;
- # trim comments
- $_ =~ s/^#.*//;
- # skip empty lines
- if( $_ )
- {
- # check for valid characters
- $_ =~ /^([a-zA-Z][a-zA-Z0-9]*)\s+([a-zA-Z0-9-.:\/]+)\s*$/ or die "Error: invalid character in input data";
- $namespaces_strict{$1} = $2;
- }
-}
-close( INFILE_STRICT );
-
-# generate output files
-
-open( IDFILE, ">$ARGV1" ) or die "Error: cannot open output file: $!";
-open( NAMEFILE, ">$ARGV2" ) or die "Error: cannot open output file: $!";
-open( TXTFILE, ">$ARGV3" ) or die "Error: cannot open output file: $!";
-open( NAMEFILE_STRICT, ">$ARGV5" ) or die "Error: cannot open output file: $!";
-
-# number of bits to shift the namespace identifier
-$shift = 16;
-
-print ( IDFILE "const size_t NMSP_SHIFT = $shift;\n" );
-
-$i = 1;
-foreach( sort( keys( %namespaces ) ) )
-{
- print( IDFILE "const sal_Int32 NMSP_$_ = $i << NMSP_SHIFT;\n" );
- $id = $i << $shift;
- print( NAMEFILE "{ $id, \"$namespaces{$_}\" },\n" );
- print( NAMEFILE_STRICT "{ $id, \"$namespaces_strict{$_}\" },\n" );
- print( TXTFILE "$id $_ $namespaces{$_}\n" );
- print( TXTFILE "$id $_ $namespaces_strict{$_}\n" );
- ++$i;
-}
-
-close( IDFILE );
-close( NAMEFILE );
-close( NAMEFILE_STRICT );
-close( TXTFILE );
diff --git a/oox/source/token/namespaces.py b/oox/source/token/namespaces.py
new file mode 100644
index 000000000000..1244e88e857f
--- /dev/null
+++ b/oox/source/token/namespaces.py
@@ -0,0 +1,85 @@
+# This file is part of the LibreOffice project.
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+#
+# This file incorporates work covered by the following license notice:
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed
+# with this work for additional information regarding copyright
+# ownership. The ASF licenses this file to you under the Apache
+# License, Version 2.0 (the "License"); you may not use this file
+# except in compliance with the License. You may obtain a copy of
+# the License at http://www.apache.org/licenses/LICENSE-2.0 .
+#
+
+import sys, re
+
+infile_name = sys.argv[1]
+id_out_name = sys.argv[2]
+name_out_name = sys.argv[3]
+txt_out_name = sys.argv[4]
+instrict_name = sys.argv[5]
+strict_out_name = sys.argv[6]
+
+# parse input file
+
+namespaces = {}
+
+with open(infile_name) as infile:
+ for line in infile:
+ line = line.strip()
+ # trim comments
+ line = line.split('#')[0]
+ # skip empty lines
+ if line:
+ # check for valid characters
+ m = re.match(r'([a-zA-Z][a-zA-Z0-9]*)\s+([a-zA-Z0-9-.:\/]+)\s*$', line)
+ if not m:
+ sys.exit('Invalid character in input data: ' + line)
+ namespaces[m.group(1)] = m.group(2)
+
+# OOXML strict namespaces
+
+namespaces_strict = {}
+with open(instrict_name) as infile_strict:
+ for line in infile_strict:
+ line = line.strip()
+ # trim comments
+ line = line.split('#')[0]
+ # skip empty lines
+ if line:
+ # check for valid characters
+ m = re.match(r'([a-zA-Z][a-zA-Z0-9]*)\s+([a-zA-Z0-9-.:\/]+)\s*$', line)
+ if not m:
+ sys.exit("Error: invalid character in input data: " + line)
+ namespaces_strict[m.group(1)] = m.group(2)
+
+# generate output files
+
+idfile = open(id_out_name, 'w')
+namefile = open(name_out_name, 'w')
+txtfile = open(txt_out_name, 'w')
+namefile_strict = open(strict_out_name, 'w')
+
+# number of bits to shift the namespace identifier
+shift = 16
+
+idfile.write("const size_t NMSP_SHIFT = {};\n".format(shift))
+
+i = 1;
+for token in sorted(namespaces.keys()):
+ idfile.write("const sal_Int32 NMSP_{} = {} << NMSP_SHIFT;\n".format(token, i))
+ cur_id = i << shift
+ namefile.write("{{ {}, \"{}\" }},\n".format(cur_id, namespaces[token]))
+ namefile_strict.write("{{ {}, \"{}\" }},\n".format(cur_id, namespaces_strict[token]))
+ txtfile.write("{} {} {}\n".format(cur_id, token, namespaces[token]))
+ txtfile.write("{} {} {}\n".format(cur_id, token, namespaces_strict[token]))
+ i += 1
+
+idfile.close()
+namefile.close()
+namefile_strict.close()
+txtfile.close()
diff --git a/oox/source/token/properties.pl b/oox/source/token/properties.pl
deleted file mode 100644
index d2a2d80153ef..000000000000
--- a/oox/source/token/properties.pl
+++ /dev/null
@@ -1,58 +0,0 @@
-#
-# This file is part of the LibreOffice project.
-#
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-#
-# This file incorporates work covered by the following license notice:
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed
-# with this work for additional information regarding copyright
-# ownership. The ASF licenses this file to you under the Apache
-# License, Version 2.0 (the "License"); you may not use this file
-# except in compliance with the License. You may obtain a copy of
-# the License at http://www.apache.org/licenses/LICENSE-2.0 .
-#
-
-$ARGV0 = shift @ARGV;
-$ARGV1 = shift @ARGV;
-$ARGV2 = shift @ARGV;
-
-# parse input file
-
-open( INFILE, $ARGV0 ) or die "Error: cannot open input file: $!";
-my %props;
-while( <INFILE> )
-{
- # trim newline
- chomp( $_ );
- # trim leading/trailing whitespace
- $_ =~ s/^\s*//g;
- $_ =~ s/\s*$//g;
- # check for valid characters
- $_ =~ /^[A-Z][a-zA-Z0-9]*$/ or die "Error: invalid character in property '$_'";
- $id = "PROP_$_";
- $props{$_} = $id;
-}
-close( INFILE );
-
-# generate output files
-
-open( IDFILE, ">$ARGV1" ) or die "Error: cannot open output file: $!";
-open( NAMEFILE, ">$ARGV2" ) or die "Error: cannot open output file: $!";
-
-$i = 0;
-foreach( sort( keys( %props ) ) )
-{
- print( IDFILE "const sal_Int32 $props{$_} = $i;\n" );
- print( NAMEFILE "/* $i */ \"$_\",\n" );
- ++$i;
-}
-
-print( IDFILE "const sal_Int32 PROP_COUNT = $i;\n" );
-print( IDFILE "const sal_Int32 PROP_INVALID = -1;\n" );
-
-close( IDFILE );
-close( NAMEFILE );
diff --git a/oox/source/token/properties.py b/oox/source/token/properties.py
new file mode 100644
index 000000000000..8f8bc437b358
--- /dev/null
+++ b/oox/source/token/properties.py
@@ -0,0 +1,50 @@
+# This file is part of the LibreOffice project.
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+#
+# This file incorporates work covered by the following license notice:
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed
+# with this work for additional information regarding copyright
+# ownership. The ASF licenses this file to you under the Apache
+# License, Version 2.0 (the "License"); you may not use this file
+# except in compliance with the License. You may obtain a copy of
+# the License at http://www.apache.org/licenses/LICENSE-2.0 .
+#
+
+import sys, re
+
+infile_name = sys.argv[1]
+id_out_name = sys.argv[2]
+name_out_name = sys.argv[3]
+
+# parse input file
+
+props = {}
+with open(infile_name) as infile:
+ for line in infile:
+ line = line.strip()
+ # check for valid characters
+ if not re.match(r'[A-Z][a-zA-Z0-9]*$', line):
+ sys.exit("Error: invalid character in property '{}'".format(line))
+ props[line] = "PROP_" + line
+
+# generate output files
+
+idfile = open(id_out_name, 'w')
+namefile = open(name_out_name, 'w')
+
+i = 0;
+for token in sorted(props.keys()):
+ idfile.write("const sal_Int32 {} = {};\n".format(props[token], i))
+ namefile.write("/* {} */ \"{}\",\n".format(i, token))
+ i += 1
+
+idfile.write("const sal_Int32 PROP_COUNT = {};\n".format(i))
+idfile.write("const sal_Int32 PROP_INVALID = -1;\n" )
+
+idfile.close()
+namefile.close()
diff --git a/solenv/bin/generate-tokens.pl b/solenv/bin/generate-tokens.pl
deleted file mode 100644
index d5f8f9fa7d1f..000000000000
--- a/solenv/bin/generate-tokens.pl
+++ /dev/null
@@ -1,69 +0,0 @@
-#
-# This file is part of the LibreOffice project.
-#
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-#
-# This file incorporates work covered by the following license notice:
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed
-# with this work for additional information regarding copyright
-# ownership. The ASF licenses this file to you under the Apache
-# License, Version 2.0 (the "License"); you may not use this file
-# except in compliance with the License. You may obtain a copy of
-# the License at http://www.apache.org/licenses/LICENSE-2.0 .
-#
-
-$ARGV0 = shift @ARGV;
-$ARGV1 = shift @ARGV;
-$ARGV2 = shift @ARGV;
-$ARGV3 = shift @ARGV;
-
-open ( IDFILE, ">$ARGV1" ) or die "Error: cannot open output file: $!";
-open ( NAMEFILE, ">$ARGV2" ) or die "Error: cannot open output file: $!";
-open ( GPERFFILE, ">$ARGV3" ) or die "Error: cannot open output file: $!";
-
-print( GPERFFILE "%language=C++\n" );
-print( GPERFFILE "%global-table\n" );
-print( GPERFFILE "%null-strings\n" );
-print( GPERFFILE "%struct-type\n" );
-print( GPERFFILE "struct xmltoken {\n" );
-print( GPERFFILE " const sal_Char *name;\n" );
-print( GPERFFILE " sal_Int32 nToken;\n" );
-print( GPERFFILE "};\n" );
-print( GPERFFILE "%%\n" );
-
-open( INFILE, $ARGV0 ) or die "Error: cannot open input file: $!";
-
-$i = 0;
-while ( <INFILE> )
-{
- # trim newline
- chomp( $_ );
- # trim leading/trailing whitespace
- $_ =~ s/^\s*//g;
- $_ =~ s/\s*$//g;
- # check for valid characters
- $_ =~ /^[a-zA-Z0-9-_]+$/ or die "Error: invalid character in token '$_'";
- $id = "XML_$_";
- # we have two ids with similar names("cut-offs" and "cut_offs")
- if ($id eq "XML_cut_offs") {
- $id = "cut_offs2";
- }
- $id =~ s/-/_/g;
- $tokens{$_} = $id;
- print( IDFILE "const sal_Int32 $id = $i;\n" );
- print( NAMEFILE "\"$_\",\n" );
- print( GPERFFILE "$_,$id\n" );
- ++$i;
-}
-close ( INFILE );
-
-print( IDFILE "const sal_Int32 XML_TOKEN_COUNT = $i;\n" );
-print( GPERFFILE "%%\n" );
-
-close( IDFILE );
-close( NAMEFILE );
-close( GPERFFILE );
diff --git a/solenv/bin/generate-tokens.py b/solenv/bin/generate-tokens.py
new file mode 100644
index 000000000000..3fe929092304
--- /dev/null
+++ b/solenv/bin/generate-tokens.py
@@ -0,0 +1,80 @@
+# This file is part of the LibreOffice project.
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+#
+# This file incorporates work covered by the following license notice:
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed
+# with this work for additional information regarding copyright
+# ownership. The ASF licenses this file to you under the Apache
+# License, Version 2.0 (the "License"); you may not use this file
+# except in compliance with the License. You may obtain a copy of
+# the License at http://www.apache.org/licenses/LICENSE-2.0 .
+#
+
+import sys, re
+
+infile_name = sys.argv[1]
+idfile_out_name = sys.argv[2]
+namefile_out_name = sys.argv[3]
+gperffile_out_name = sys.argv[4]
+
+idfile = open(idfile_out_name, 'w')
+namefile = open(namefile_out_name, 'w')
+gperffile = open(gperffile_out_name, 'w')
+
+gperffile.write("""%language=C++
+%global-table
+%null-strings
+%struct-type
+struct xmltoken {
+ const sal_Char *name;
+ sal_Int32 nToken;
+};
+%%
+""")
+
+token_count = 0;
+tokens = {}
+
+with open(infile_name) as infile:
+ for line in infile:
+ line = line.strip()
+ # check for valid characters
+ if not re.match(r'[a-zA-Z0-9-_]+$', line):
+ sys.exit("Error: invalid character in token '{}'".format(line));
+ cur_id = "XML_" + line;
+ # we have two ids with similar names("cut-offs" and "cut_offs")
+ if cur_id == "XML_cut_offs":
+ cur_id = "cut_offs2";
+ cur_id = cur_id.replace('-', '_')
+ tokens[line] = cur_id
+ idfile.write("const sal_Int32 {} = {};\n".format(cur_id, token_count))
+ namefile.write("\"{}\",\n".format(line));
+ gperffile.write("{},{}\n".format(line, cur_id));
+ token_count += 1
+
+idfile.write("const sal_Int32 XML_TOKEN_COUNT = {};\n".format(token_count))
+gperffile.write("%%\n")
+
+idfile.close()
+namefile.close()
+gperffile.close()
+
+def fix_linefeeds(fname):
+ # Gperf requires LF newlines, not CRLF, even on Windows.
+ # Making this work on both Python 2 and 3 is difficult.
+ # When Python 2 is dropped, delete this and add
+ # newline = '\n' to the open() calls above.
+ with open(fname, 'rb') as ifile:
+ d = ifile.read()
+ d = d.replace(b'\r', b'')
+ with open(fname, 'wb') as ofile:
+ ofile.write(d)
+
+fix_linefeeds(idfile_out_name)
+fix_linefeeds(namefile_out_name)
+fix_linefeeds(gperffile_out_name)
diff --git a/solenv/gbuild/CustomTarget.mk b/solenv/gbuild/CustomTarget.mk
index 27ce89fba06f..ed1a9addc103 100644
--- a/solenv/gbuild/CustomTarget.mk
+++ b/solenv/gbuild/CustomTarget.mk
@@ -100,8 +100,9 @@ $(if $(7),$(call gb_CustomTarget_get_workdir,$(1))/$(7)names.inc) : \
touch $$@
$(call gb_CustomTarget_get_workdir,$(1))/$(2)/token/$(4).hxx : \
+ $(call gb_ExternalExecutable_get_dependencies,python) \
$(if $(7),$(SRCDIR)/$(3)/$(7).txt) \
- $(if $(8),$(SRCDIR)/$(3)/$(8),$(SRCDIR)/solenv/bin/generate-tokens.pl) \
+ $(if $(8),$(SRCDIR)/$(3)/$(8),$(SRCDIR)/solenv/bin/generate-tokens.py) \
$(SRCDIR)/$(3)/$(4).txt \
$(SRCDIR)/$(3)/$(4).hxx.head \
$(SRCDIR)/$(3)/$(4).hxx.tail
@@ -110,7 +111,7 @@ $(call gb_CustomTarget_get_workdir,$(1))/$(2)/token/$(4).hxx : \
mkdir -p $(call gb_CustomTarget_get_workdir,$(1))/misc \
$(call gb_CustomTarget_get_workdir,$(1)) \
$(call gb_CustomTarget_get_workdir,$(1))/$(2)/token
- perl $(if $(8),$(SRCDIR)/$(3)/$(8),$(SRCDIR)/solenv/bin/generate-tokens.pl) \
+ $(call gb_ExternalExecutable_get_command,python) $(if $(8),$(SRCDIR)/$(3)/$(8),$(SRCDIR)/solenv/bin/generate-tokens.py) \
$(SRCDIR)/$(3)/$(4).txt \
$(call gb_CustomTarget_get_workdir,$(1))/misc/$(5)ids.inc \
$(call gb_CustomTarget_get_workdir,$(1))/$(5)names.inc \
More information about the Libreoffice-commits
mailing list