[Libreoffice-commits] core.git: Branch 'aoo/trunk' - bootstrap.1 solenv/bin solenv/javadownloader

Damjan Jovanovic damjan at apache.org
Sun Jul 24 16:09:28 UTC 2016


 bootstrap.1                                  |    8 +
 solenv/bin/download_external_dependencies.pl |   98 +++-------------
 solenv/bin/modules/ExtensionsLst.pm          |   77 ++-----------
 solenv/javadownloader/AOOJavaDownloader.java |  156 +++++++++++++++++++++++++++
 4 files changed, 200 insertions(+), 139 deletions(-)

New commits:
commit 9813bbc278e11149aa0519750f2496f1b3d5ab89
Author: Damjan Jovanovic <damjan at apache.org>
Date:   Sun Jul 24 15:53:36 2016 +0000

    Give up on using Perl's LWP::UserAgent and LWP::Protocol::https to download
    
    files during ./bootstrap. It's a nightmare to get it working on the
    buildbots - Infra has been trying on INFRA-11296 for 5 months to get it
    installed. Installing through CPAN doesn't always work - tests fail on
    CentOS 5 and on Cygwin. Even when installed, it's not always found. The
    gain just doesn't justify the effort. Worst of all, it's holding back
    development.
    
    What else is there? A CLI tool like wget could work, but it's not listed as
    a dependency, and it breaks on CentOS 5 for https://, the thing it's needed
    for most.
    
    I instead re-implemented it in Java. Java is freely available, highly
    portable, and rock solid. We already use it in the build, and it's
    described as being a mandatory build requirement even though
    ./configure.ac treats it as optional. Best of all it supports https://
    out of the box in java.net.URLConnection and uses its own root CA
    certificates. Tests show my AOOJavaDownloader class works on FreeBSD and
    Windows, supports HTTP redirection, and generally works like a charm.
    
    Patch by: me

diff --git a/bootstrap.1 b/bootstrap.1
index 0981e5f..045b547 100644
--- a/bootstrap.1
+++ b/bootstrap.1
@@ -41,6 +41,14 @@ chmod +x "$SRC_ROOT/solenv/bin/gccinstlib.pl"
 if [ "$DO_FETCH_TARBALLS" = "yes" ]; then
 # check perl include locations
     "$PERL" -e 'print "\nInclude locations: @INC\n\n"';
+
+    mkdir -p "$SOLARENV/$INPATH/class"
+    "$JAVACOMPILER" "$SOLARENV/javadownloader/AOOJavaDownloader.java" -d "$SOLARENV/$INPATH/class"
+    if [ "$?" != "0" ]; then
+        echo "*** Failed to build AOOJavaDownloader, aborting! ***"
+        exit 1
+    fi
+
     "$PERL" "$SOLARENV/bin/download_external_dependencies.pl" $SRC_ROOT/external_deps.lst
     if [ "$?" != "0" ]; then
         echo "*** Error downloading external dependencies, please fix the previous problems and try again ***"
diff --git a/solenv/bin/download_external_dependencies.pl b/solenv/bin/download_external_dependencies.pl
index 9db822b..ec68036 100755
--- a/solenv/bin/download_external_dependencies.pl
+++ b/solenv/bin/download_external_dependencies.pl
@@ -507,91 +507,39 @@ sub DownloadFile ($$$)
     my $URL = shift;
     my $checksum = shift;
 
-    my $filename = File::Spec->catfile($ENV{'TARFILE_LOCATION'}, $name);
-
-    my $temporary_filename = $filename . ".part";
-
-    print "downloading to $temporary_filename\n";
-    my $out;
-    open $out, ">$temporary_filename";
-    binmode($out);
-
-    # Prepare checksum
-    my $digest;
-    if (defined $checksum && $checksum->{'type'} eq "SHA1")
-    {
-        # Use SHA1 only when explicitly requested (by the presence of a "SHA1=..." line.)
-        $digest = Digest::SHA->new("1");
-    }
-    elsif ( ! defined $checksum || $checksum->{'type'} eq "MD5")
-    {
-        # Use MD5 when explicitly requested or when no checksum type is given.
-        $digest = Digest::MD5->new();
+    if (defined $checksum)
+    {
+        system(
+            $ENV{'JAVAINTERPRETER'},
+            "-cp",
+            File::Spec->catfile(
+                File::Spec->catfile($ENV{'SOLARENV'}, $ENV{'INPATH'}),
+                "class"),
+            "AOOJavaDownloader",
+            $name,
+            $URL,
+            $checksum->{'type'},
+            $checksum->{'value'});
     }
     else
     {
-        die "checksum type ".$checksum->{'type'}." is not supported";
+        system(
+            $ENV{'JAVAINTERPRETER'},
+            "-cp",
+            File::Spec->catfile(
+                File::Spec->catfile($ENV{'SOLARENV'}, $ENV{'INPATH'}),
+                "class"),
+            "AOOJavaDownloader",
+            $name,
+            $URL);
     }
 
-    # Download the extension.
-    my $success = 0;
-
-    my $agent = LWP::UserAgent->new();
-    $agent->env_proxy;
-    my $response = $agent->get($URL);
-
-    $success = $response->is_success;
-    if ($success)
-    {
-        my $content = $response->content;
-        open $out, ">$temporary_filename";
-        binmode($out);
-        print $out $content;
-        $digest->add($content);
-    }
-    else
+    if ($? == 0)
     {
-        print "download from $URL failed (" . $response->status_line . ")\n";
-    }
-    close($out);
-
-    # When download was successful then check the checksum and rename the .part file
-    # into the actual extension name.
-    if ($success)
-    {
-        my $file_checksum = $digest->hexdigest();
-        if (defined $checksum)
-        {
-            if ($checksum->{'value'} eq $file_checksum)
-            {
-                printf("%s checksum is OK\n", $checksum->{'type'});
-            }
-            else
-            {
-                unlink($temporary_filename);
-                printf("    %s checksum does not match (%s instead of %s)\n",
-                       $checksum->{'type'},
-                       $file_checksum,
-                       $checksum->{'value'});
-                return 0;
-            }
-        }
-        else
-        {
-            # The datafile does not contain a checksum to match against.
-            # Display the one that was calculated for the downloaded file so that
-            # it can be integrated manually into the data file.
-            printf("checksum not given, md5 of file is %s\n", $file_checksum);
-            $filename = File::Spec->catfile($ENV{'TARFILE_LOCATION'}, $file_checksum . "-" . $name);
-        }
-
-        rename($temporary_filename, $filename) || die "can not rename $temporary_filename to $filename";
         return 1;
     }
     else
     {
-        unlink($temporary_filename);
-        print "    download failed\n";
         return 0;
     }
 }
diff --git a/solenv/bin/modules/ExtensionsLst.pm b/solenv/bin/modules/ExtensionsLst.pm
index 39bb6d0..eaed270 100644
--- a/solenv/bin/modules/ExtensionsLst.pm
+++ b/solenv/bin/modules/ExtensionsLst.pm
@@ -468,70 +468,19 @@ sub Download (@)
     {
         my ($protocol, $name, $URL, $md5sum) = @{$entry};
 
-        # Open a .part file for writing.
-        my $filename = File::Spec->catfile($download_path, $name);
-        my $temporary_filename = $filename . ".part";
-        print "downloading to $temporary_filename\n";
-        open my $out, ">$temporary_filename";
-        binmode($out);
-
-        # Prepare md5
-        my $md5 = Digest::MD5->new();
-
-        # Download the extension.
-        my $agent = LWP::UserAgent->new();
-        $agent->timeout(120);
-        $agent->show_progress(1);
-        my $last_was_redirect = 0;
-        $agent->add_handler('response_redirect'
-                            => sub{
-                                $last_was_redirect = 1;
-                                return;
-                            });
-        $agent->add_handler('response_data'
-                            => sub{
-                                if ($last_was_redirect)
-                                {
-                                    $last_was_redirect = 0;
-                                    # Throw away the data we got so far.
-                                    $md5->reset();
-                                    close $out;
-                                    open $out, ">$temporary_filename";
-                                    binmode($out);
-                                }
-                                my($response,$agent,$h,$data)=@_;
-                                print $out $data;
-                                $md5->add($data);
-                            });
-        my $response = $agent->get($URL);
-        close $out;
-
-        # When download was successful then check the md5 checksum and rename the .part file
-        # into the actual extension name.
-        if ($response->is_success())
-        {
-            if (defined $md5sum && length($md5sum)==32)
-            {
-                my $file_md5 = $md5->hexdigest();
-                if ($md5sum eq $file_md5)
-                {
-                    print "md5 is OK\n";
-                }
-                else
-                {
-                    unlink($temporary_filename) if ! $Debug;
-                    die "downloaded file has the wrong md5 checksum: $file_md5 instead of $md5sum";
-                }
-            }
-            else
-            {
-                print "md5 is not present\n";
-                printf "   is %s, length is %d\n", $md5sum, length(md5sum);
-            }
-
-            rename($temporary_filename, $filename) || die "can not rename $temporary_filename to $filename";
-        }
-        else
+        system(
+            $ENV{'JAVAINTERPRETER'},
+            "-cp",
+            File::Spec->catfile(
+                File::Spec->catfile($ENV{'SOLARENV'}, $ENV{'INPATH'}),
+                "class"),
+            "AOOJavaDownloader",
+            $name,
+            $URL,
+            'MD5',
+            $md5sum);
+
+        if ($? != 0)
         {
             die "failed to download $URL";
         }
diff --git a/solenv/javadownloader/AOOJavaDownloader.java b/solenv/javadownloader/AOOJavaDownloader.java
new file mode 100644
index 0000000..612402a
--- /dev/null
+++ b/solenv/javadownloader/AOOJavaDownloader.java
@@ -0,0 +1,156 @@
+/**************************************************************
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ *************************************************************/
+
+import java.io.BufferedInputStream;
+import java.io.BufferedOutputStream;
+import java.io.IOException;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.HttpURLConnection;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.net.URLConnection;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+
+public class AOOJavaDownloader {
+    private static final int MAX_REDIRECTS = 10;
+    private static final String hexChars = "0123456789abcdef";
+
+    public static void main(String[] args) throws MalformedURLException, NoSuchAlgorithmException {
+        if (args.length != 2 && args.length != 4) {
+            System.err.println("AOOJavaDownloader - required args: filename URL [checksumType] [checksum]");
+            System.exit(1);
+        }
+        String name = args[0];
+        URL url = new URL(args[1]);
+        String checksumType = args.length == 4 ? args[2] : null;
+        String checksum = args.length == 4 ? args[3] : null;
+
+        File filename = new File(System.getenv("TARFILE_LOCATION"), name);
+
+        MessageDigest digest = null;
+        if (checksumType != null && checksumType.equals("SHA1")) {
+            digest = MessageDigest.getInstance(checksumType);
+        } else {
+            digest = MessageDigest.getInstance("MD5");
+        }
+
+        File temporaryFile = new File(filename.getAbsolutePath() + ".part");
+        System.out.println("downloading " + url + " to " + temporaryFile.getAbsolutePath());
+
+        boolean succeeded = false;
+        InputStream inputStream = null;
+        try {
+            inputStream = openURL(url);
+
+            OutputStream outputStream = new BufferedOutputStream(new FileOutputStream(temporaryFile));
+            try {
+                copyStreamAndHash(inputStream, outputStream, digest);
+            } finally {
+                try {
+                    outputStream.close();
+                } catch (IOException ignored) {
+                }
+            }
+
+            String fileChecksum = bytesToHex(digest.digest());
+            if (checksumType != null) {
+                if (fileChecksum.equals(checksum)) {
+                    System.out.println(checksumType + " checksum is OK");
+                } else {
+                    temporaryFile.delete();
+                    throw new IOException(String.format("%s checksum does not match (%s instead of %s)",
+                            checksumType, fileChecksum, checksum));
+                }
+            } else {
+                System.out.println("checksum not given, md5 of file is " + fileChecksum);
+                filename = new File(System.getenv("TARFILE_LOCATION"), fileChecksum + "-" + name);
+            }
+            if (!temporaryFile.renameTo(filename)) {
+                throw new IOException("Renaming " + temporaryFile + " to " + filename + " failed");
+            }
+            succeeded = true;
+        } catch (Exception exception) {
+            System.out.println("download failed:");
+            exception.printStackTrace(System.out);
+        } finally {
+            if (inputStream != null) {
+                try {
+                    inputStream.close();
+                } catch (IOException ignored) {
+                }
+            }
+            if (!succeeded) {
+                temporaryFile.delete();
+            }
+        }
+        System.exit(succeeded ? 0 : 1);
+    }
+
+    private static InputStream openURL(URL url) throws IOException {
+        URLConnection connection = url.openConnection();
+        for (int redir = 0; redir < MAX_REDIRECTS; redir++) {
+            connection.setAllowUserInteraction(false);
+            connection.setConnectTimeout(60000);
+            connection.setReadTimeout(60000);
+            if (connection instanceof HttpURLConnection) {
+                HttpURLConnection httpURLConnection = (HttpURLConnection) connection;
+                // Only works when protocol (http:// or https://) remains the same:
+                httpURLConnection.setInstanceFollowRedirects(true);
+            }
+            connection.connect();
+            if (connection instanceof HttpURLConnection) {
+                HttpURLConnection httpURLConnection = (HttpURLConnection) connection;
+                // handle protocol change:
+                int status = httpURLConnection.getResponseCode();
+                if (status == HttpURLConnection.HTTP_MOVED_TEMP ||
+                        status == HttpURLConnection.HTTP_MOVED_PERM ||
+                        status == HttpURLConnection.HTTP_SEE_OTHER) {
+                    connection = new URL(connection.getHeaderField("Location")).openConnection();
+                    continue;
+                }
+            }
+            return new BufferedInputStream(connection.getInputStream());
+        }
+        throw new IOException("Too many http redirects");
+    }
+
+    private static void copyStreamAndHash(InputStream inputStream, OutputStream outputStream, MessageDigest digest) throws IOException {
+        byte[] buffer = new byte[4 * 4096];
+        int bytesRead;
+        while ((bytesRead = inputStream.read(buffer)) >= 0) {
+            outputStream.write(buffer, 0, bytesRead);
+            digest.update(buffer, 0, bytesRead);
+        }
+    }
+
+    private static String bytesToHex(byte[] bytes) {
+        char[] output = new char[bytes.length * 2];
+        for (int i = 0; i < bytes.length; i++) {
+            output[2*i] = hexChars.charAt(0xf & (bytes[i] >> 4));
+            output[2*i + 1] = hexChars.charAt(0xf & bytes[i]);
+        }
+        return new String(output);
+    }
+}


More information about the Libreoffice-commits mailing list