Mesa (master): docs/ci: Document setting up the http cache for traces.

[GitLab Mirror]

Module: Mesa
Branch: master
Commit: 2e2edaa89b4cfd935f1dde28e81bbcacf39ad15a
URL:    http://cgit.freedesktop.org/mesa/mesa/commit/?id=2e2edaa89b4cfd935f1dde28e81bbcacf39ad15a

Author: Eric Anholt <eric at anholt.net>
Date:   Thu Jan 21 13:55:02 2021 -0800

docs/ci: Document setting up the http cache for traces.

Reviewed-by: Daniel Stone <daniel at fooishbar.org>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/8727>

---

 docs/ci/bare-metal.rst   | 38 ++++++++++++++++++++++++++++++++++++++
 docs/ci/fdo-cache        | 44 ++++++++++++++++++++++++++++++++++++++++++++
 docs/ci/uri-caching.conf | 35 +++++++++++++++++++++++++++++++++++
 3 files changed, 117 insertions(+)

diff --git a/docs/ci/bare-metal.rst b/docs/ci/bare-metal.rst
index be5952da21c..3ea57749417 100644
--- a/docs/ci/bare-metal.rst
+++ b/docs/ci/bare-metal.rst
@@ -128,3 +128,41 @@ that gets used to start a nginx server.
 
 Once you've updated your runners' configs, restart with ``sudo service
 gitlab-runner restart``
+
+Caching downloads
+-----------------
+
+To improve the runtime for downloading traces during traces job runs, you will
+want a pass-through HTTP cache.  On your runner box, install nginx:
+
+.. code-block:: console
+  sudo apt install nginx libnginx-mod-http-lua
+
+Add the server setup files:
+
+.. literalinclude: fdo-cache:
+   :name: /etc/nginx/sites-available/fdo-cache
+
+.. literalinclude: uri-caching.conf:
+   :name: /etc/nginx/sites-available/snippets/uri-caching.conf
+
+Edit the listener addresses in fdo-cache to suit the ethernet interface that
+your devices are on.
+
+Enable the site and restart nginx:
+
+.. code-block:: console
+  sudo ln -s /etc/nginx/sites-available/fdo-cache /etc/nginx/sites-enabled/fdo-cache
+  sudo service nginx restart
+
+  # First download will hit the internet
+  wget http://localhost/cache/?uri=https://minio-packet.freedesktop.org/mesa-tracie-public/itoral-gl-terrain-demo/demo.trace
+  # Second download should be cached.
+  wget http://localhost/cache/?uri=https://minio-packet.freedesktop.org/mesa-tracie-public/itoral-gl-terrain-demo/demo.trace
+
+Now, set ``download-url`` in your ``traces-*.yml`` entry to something like
+``http://10.42.0.1:8888/cache/?uri=https://minio-packet.freedesktop.org/mesa-tracie-public``
+and you should have cached downloads for traces.  Add it to
+``FDO_HTTP_CACHE_URI=`` in your ``config.toml`` runner environment lines and you
+can use it for cached artifact downloads instead of going all the way to
+freedesktop.org on each job.
diff --git a/docs/ci/fdo-cache b/docs/ci/fdo-cache
new file mode 100644
index 00000000000..922cd76b240
--- /dev/null
+++ b/docs/ci/fdo-cache
@@ -0,0 +1,44 @@
+proxy_cache_path /var/cache/nginx/ levels=1:2 keys_zone=my_cache:10m max_size=24g inactive=48h use_temp_path=off;
+
+server {
+	listen 10.42.0.1:8888 default_server;
+	listen 127.0.0.1:8888 default_server;
+	listen [::]:8888 default_server;
+	resolver 8.8.8.8;
+
+	root /var/www/html;
+
+	# Add index.php to the list if you are using PHP
+	index index.html index.htm index.nginx-debian.html;
+
+	server_name _;
+
+        add_header X-GG-Cache-Status $upstream_cache_status;
+        proxy_cache my_cache;
+
+        location /cache_gitlab_artifacts {
+                internal;
+                # Gitlabs http server puts everything as no-cache even though
+                # the artifacts URLS don't change. So enforce a long validity
+                # time and ignore the headers that defeat caching
+                proxy_cache_valid 200 48h;
+                proxy_ignore_headers Cache-Control Set-Cookie;
+                include snippets/uri-caching.conf;
+        }
+
+        location /cache {
+                # special case gitlab artifacts
+                if ($arg_uri ~*  /.*gitlab.*artifacts(\/|%2F)raw/ ) {
+                        rewrite ^ /cache_gitlab_artifacts;
+                }
+                # Set a really low validity together with cache revalidation; Our goal
+                # for caching isn't to lower the number of http requests but to
+                # lower the amount of data transfer. Also for some test
+                # scenarios (typical manual tests) the file at a given url
+                # might get modified so avoid confusion by ensuring
+                # revalidations happens often.
+                proxy_cache_valid 200 10s;
+                proxy_cache_revalidate on;
+                include snippets/uri-caching.conf;
+        }
+}
diff --git a/docs/ci/uri-caching.conf b/docs/ci/uri-caching.conf
new file mode 100644
index 00000000000..c360c55c33a
--- /dev/null
+++ b/docs/ci/uri-caching.conf
@@ -0,0 +1,35 @@
+set $authorization '';
+
+set_by_lua $proxyuri '
+        unescaped =  ngx.unescape_uri(ngx.var.arg_uri);
+        it, err = ngx.re.match(unescaped, "(https?://)(.*@)?([^/]*)(/.*)?");
+        if not it then
+                -- Hack to cause nginx to return 404
+                return "http://localhost/404"
+        end
+
+        scheme = it[1];
+        authstring = it[2];
+        host = it[3];
+        query = it[4];
+
+        if authstring then
+                auth = string.sub(authstring, 0, -2);
+                auth64 = ngx.encode_base64(auth);
+                ngx.var.authorization = "Basic " .. auth64;
+        end
+
+        -- Default to / if none is set to avoid using the request_uri query
+        if not query then
+                query = "/";
+        end
+
+        return scheme .. host .. query;
+';
+
+add_header X-GG-Cache-Status $upstream_cache_status;
+proxy_set_header Authorization $authorization;
+
+proxy_pass $proxyuri;
+# Redirect back to ourselves on 301 replies
+proxy_redirect ~^(.*)$ /cache/?uri=$1;