fix: the Steam client download resumes, retries and shows megabytes

A first run downloads the native client, about 360 MB in 17 parts, before
anything is on screen. The script ran plain `curl -fsSL` per part: no
timeout, no retry, no progress. A stalled connection could wait for ever
on "downloading Steam: bins_linuxarm64_linuxarm64 (2/17)" (a 110 MB part),
a failed one ended the session and threw away everything fetched, and
the loading screen said "~1-2 min" while counting parts, not bytes.

Parts now go to package/droiddeck-download and stay there until the client
is installed. Each one resumes from the bytes on disk (curl -C -), a
transfer under 4 KB/s for a minute is dropped and resumed, attempts
without progress back off 5-30 s before giving up, and a finished part is
marked unpacked so a later session skips it. Bytes that fail their
checksum are fetched whole again; a server that will not resume starts the
part over; parts left for another client version are dropped. A repair
redoes every part once and keeps what an interrupted repair redid.

Every 5 s the step line carries megabytes overall ("(2/17), 45 of 360
MB"), which the loading screen shows as "45 / 360 MB" with the bar
following the bytes. The session's step no longer promises 1-2 minutes.

app/src/main/java/com/droiddeck/launcher/session/LoadingState.kt

@@ -102,11 +102,10 @@ class LoadingState(context: Context, private val steam: Boolean = true) {
             if (line.startsWith("== STEP ")) {
                 stepAt = at
                 stepText = line.substringAfter("== STEP ").substringAfter(' ')
-                val m = INSTALL_COUNT.find(line)
-                if (m != null) {
+                clientDownload(line)?.let { (what, percent) ->
                     clientDownloadAt = at
-                    clientDownload = m.groupValues[1]
-                    clientPercent = m.groupValues[2].toInt() * 100 / maxOf(1, m.groupValues[3].toInt())
+                    clientDownload = what
+                    clientPercent = percent
                 }
             } else {
                 val m = UPDATE_PROGRESS.find(line)
@@ -152,6 +151,21 @@ class LoadingState(context: Context, private val steam: Boolean = true) {
             }
         }
 
+        /**
+         * What a Steam client download line says to show and how far it is: megabytes when the
+         * runtime counts them ("<part> (2/17), 45 of 360 MB"), else the part and its number.
+         */
+        internal fun clientDownload(line: String): Pair<String, Int>? {
+            val count = INSTALL_COUNT.find(line) ?: return null
+            val mb = INSTALL_MEGABYTES.find(line.substring(count.range.last + 1))
+            val done = mb?.groupValues?.get(1)?.toLongOrNull()
+            val total = mb?.groupValues?.get(2)?.toLongOrNull()
+            if (done != null && total != null && total > 0) {
+                return "$done / $total MB" to (done * 100 / total).toInt().coerceIn(0, 100)
+            }
+            return count.groupValues[1] to count.groupValues[2].toInt() * 100 / maxOf(1, count.groupValues[3].toInt())
+        }
+
         /** The script's install lines are worth reading as they are; its other notes are not. */
         internal fun readableScriptStep(scriptStep: String): Boolean {
             val t = scriptStep.lowercase()
@@ -161,5 +175,6 @@ class LoadingState(context: Context, private val steam: Boolean = true) {
         private const val TAIL_BYTES = 48L * 1024
         private val UPDATE_PROGRESS = Regex("""Downloading update \((\d+) of (\d+) KB\)""")
         private val INSTALL_COUNT = Regex("""downloading Steam: (\S+) \((\d+)/(\d+)\)""")
+        private val INSTALL_MEGABYTES = Regex("""^, (\d+) of (\d+) MB""")
     }
 }

app/src/test/java/com/droiddeck/launcher/session/LoadingStateTest.kt

@@ -0,0 +1,37 @@
+package com.droiddeck.launcher.session
+
+import org.junit.Assert.assertEquals
+import org.junit.Assert.assertNull
+import org.junit.Test
+
+class LoadingStateTest {
+    @Test fun clientDownloadCountsMegabytesWhenTheRuntimeSaysThem() {
+        assertEquals(
+            "45 / 360 MB" to 12,
+            LoadingState.clientDownload("== STEP 17:38:20 downloading Steam: bins_linuxarm64_linuxarm64 (2/17), 45 of 360 MB"),
+        )
+    }
+
+    @Test fun aStalledDownloadStillShowsWhereItIs() {
+        assertEquals(
+            "45 / 360 MB" to 12,
+            LoadingState.clientDownload(
+                "== STEP 17:39:20 downloading Steam: bins_linuxarm64_linuxarm64 (2/17), 45 of 360 MB" +
+                    " - interrupted (curl exit 28), trying again in 5 s",
+            ),
+        )
+    }
+
+    @Test fun anOlderRuntimeShowsThePartAndItsNumber() {
+        assertEquals(
+            "bins_linuxarm64_linuxarm64" to 11,
+            LoadingState.clientDownload("== STEP 17:38:17 downloading Steam: bins_linuxarm64_linuxarm64 (2/17)"),
+        )
+        // Under a megabyte in all: nothing to count yet.
+        assertEquals("a_all" to 50, LoadingState.clientDownload("== STEP 00:00:00 downloading Steam: a_all (1/2), 0 of 0 MB"))
+    }
+
+    @Test fun otherStepsAreNotClientDownloads() {
+        assertNull(LoadingState.clientDownload("== STEP 17:38:14 checking the Steam client (a first run downloads it)"))
+    }
+}

tools/linuxfs/overlay/usr/local/bin/droiddeck-session

@@ -368,7 +368,9 @@ case $mode in
     esac
     ;;
   steam)
-    step "checking the Steam client (first run downloads it, ~1-2 min with nothing on screen)"
+    # A first run downloads the client (about 360 MB in 2026): minutes on a slow connection, with
+    # the progress in megabytes on the loading screen.
+    step "checking the Steam client (a first run downloads it)"
     droiddeck-steam-install || exit $?
     step "Steam client ready"
     steam_root=$XDG_DATA_HOME/Steam

tools/linuxfs/overlay/usr/local/bin/droiddeck-steam-install

@@ -12,6 +12,9 @@ stamp=$steam_root/package/droiddeck-installed
 legacy_stamp=$steam_root/package/bannerlator-installed
 unconfirmed=$steam_root/package/droiddeck-unconfirmed
 repair_request=$HOME/.bl-steam-repair
+# The parts of an unfinished download: a session that ends, or loses its connection, part way
+# through picks up where it stopped instead of fetching the whole client again.
+parts=$steam_root/package/droiddeck-download
 if [ -f "$legacy_stamp" ] && [ ! -f "$stamp" ]; then
   mv -f "$legacy_stamp" "$stamp" || true
 fi
@@ -24,6 +27,11 @@ elif [ -f "$unconfirmed" ] && ! grep -qx repaired "$unconfirmed" \
 fi
 if [ -n "$repair" ]; then
   echo "== STEP $(date -u +%H:%M:%S) repairing the Steam client ($repair)"
+  # A repair unpacks every part again - once: an interrupted one keeps what it has redone.
+  if [ ! -f "$parts/.repair" ]; then
+    rm -rf "$parts"
+    mkdir -p "$parts" && : > "$parts/.repair"
+  fi
   rm -f "$stamp" "$legacy_stamp" "$steam_root/.crash" "$steam_root/steam.pid" \
     "$steam_root"/package/steam_client_*.manifest "$steam_root"/package/steam_client_*.installed
   for part in "$steam_root"/package/*.zip.*; do
@@ -41,26 +49,105 @@ fi
 
 work=$(mktemp -d)
 trap 'rm -rf "$work"' EXIT
-curl -fsSL "$cdn/$manifest" -o "$work/manifest"
+
+# A sleep that starts no process (droiddeck-session's nap): read -t on a pipe nothing writes to.
+nap() {
+  [ -n "${nap_fd:-}" ] || exec {nap_fd}<> <(:)
+  read -r -t "$1" -u "$nap_fd" _ 2>/dev/null || :
+}
+size_of() {
+  local n=0
+  [ -f "$1" ] && n=$(wc -c < "$1")
+  echo $((n))
+}
+# A connection can stall without closing (Wi-Fi power save, a phone changing networks), and
+# curl's default then is to wait for ever with nothing on screen. A transfer under 4 KB/s for a
+# minute is dropped and resumed from the bytes already on disk.
+limits=(--connect-timeout 30 --speed-limit 4096 --speed-time 60)
+# Seconds between attempts that made no progress; the download fails after the last one.
+read -r -a delays <<< "${BL_STEAM_RETRY_DELAYS:-5 10 20 30 30}"
+
+step() { echo "== STEP $(date -u +%H:%M:%S) $*"; }
+# The line the app's loading screen reads: the part, its number, and megabytes overall. A third
+# argument is said after it (a stall), and is always said; otherwise only a change is.
+progress() {
+  local mb=$(( (done_bytes + $(size_of "$1")) / 1000000 ))
+  [ -z "${3:-}" ] && [ "$mb" = "${shown_mb:-}" ] && [ "$2" = "${shown_part:-}" ] && return 0
+  shown_mb=$mb shown_part=$2
+  step "downloading Steam: $2 ($i/$total), $mb of $((total_bytes / 1000000)) MB${3:+ - $3}"
+}
+
+# fetch URL FILE SIZE SHA256 PART: FILE complete and checked, resuming whatever is already there.
+fetch() {
+  local url=$1 dest=$2 size=$3 sum=$4 part=$5 failures=0 bad=0 finished=0 before rc fd wait
+  while :; do
+    # Whole once it is as long as the manifest says, or (no size given) once curl finished it.
+    if [ -f "$dest" ] && { [ "$finished" = 1 ] || { [ "$size" -gt 0 ] && [ "$(size_of "$dest")" -ge "$size" ]; }; }; then
+      echo "$sum  $dest" | sha256sum -c --quiet 2>/dev/null && return 0
+      # Bytes that do not check out are fetched whole again, a few times at most.
+      rm -f "$dest"
+      finished=0 bad=$((bad + 1))
+      [ "$bad" -lt 3 ] || { echo "droiddeck-steam-install: $part does not match its checksum" >&2; return 1; }
+    fi
+    before=$(size_of "$dest")
+    # curl's status comes back through the pipe (|| keeps set -e from ending the subshell before it
+    # is said); until then a progress line every 5 s.
+    exec {fd}< <(curl -fsSL "${limits[@]}" -C - -o "$dest" "$url" && echo 0 || echo $?)
+    until read -r -t 5 -u "$fd" rc; do
+      if [ $? -le 128 ]; then rc=1; break; fi
+      progress "$dest" "$part"
+    done
+    exec {fd}<&-
+    if [ "$rc" = 0 ]; then finished=1; continue; fi
+    # 33: the server would not resume. Start the part over.
+    [ "$rc" = 33 ] && rm -f "$dest"
+    if [ "$(size_of "$dest")" -gt "$before" ]; then failures=0; else failures=$((failures + 1)); fi
+    if [ "$failures" -gt "${#delays[@]}" ]; then
+      echo "droiddeck-steam-install: could not download $part (curl exit $rc); what arrived is kept for the next try" >&2
+      return 1
+    fi
+    wait=${delays[$((failures > 0 ? failures - 1 : 0))]}
+    progress "$dest" "$part" "interrupted (curl exit $rc), trying again in ${wait} s"
+    nap "$wait"
+  done
+}
+
+curl -fsSL "${limits[@]}" --retry 5 --retry-connrefused --retry-delay 5 "$cdn/$manifest" -o "$work/manifest"
 version=$(awk -F'"' '$2 == "version" { print $4; exit }' "$work/manifest")
-# Every component block: its name, then the plain zip's file name and sha256.
+# Every component block: its name, then the plain zip's file name, sha256 and size.
 awk -F'"' '
   /^\t"/ && NF >= 3 && $4 == "" { block = $2 }
   $2 == "file" { file[block] = $4 }
   $2 == "sha2" { sha[block] = $4 }
-  END { for (b in file) if (b ~ /_all$/ || b ~ /_linuxarm64_linuxarm64$/) print b, file[b], sha[b] }
+  $2 == "size" { size[block] = $4 }
+  END { for (b in file) if (b ~ /_all$/ || b ~ /_linuxarm64_linuxarm64$/) print b, file[b], sha[b], size[b] + 0 }
 ' "$work/manifest" | sort > "$work/parts"
 [ -s "$work/parts" ] || { echo "droiddeck-steam-install: no client components in $manifest" >&2; exit 1; }
 total=$(grep -c . "$work/parts" 2>/dev/null || echo 0)
+total_bytes=$(awk '{ n += $4 } END { print n + 0 }' "$work/parts")
 
-mkdir -p "$steam_root/package"
-while read -r block file sum; do
+mkdir -p "$parts"
+# What an earlier attempt left for another client version is no use to this one. A part's file
+# name carries its own hash, so a name still listed is the same part.
+for left in "$parts"/*; do
+  [ -e "$left" ] || continue
+  name=${left##*/}
+  name=${name%.unpacked}
+  grep -qF " $name " "$work/parts" || rm -rf "$left"
+done
+done_bytes=0
+while read -r block file sum size; do
   i=$((${i:-0} + 1))
-  echo "== STEP $(date -u +%H:%M:%S) downloading Steam: $block ($i/$total)"
-  curl -fsSL "$cdn/$file" -o "$work/$file"
-  echo "$sum  $work/$file" | sha256sum -c --quiet
-  unzip -qo "$work/$file" -d "$steam_root"
-  rm -f "$work/$file"
+  if [ -f "$parts/$file.unpacked" ]; then
+    done_bytes=$((done_bytes + size))
+    continue
+  fi
+  progress "$parts/$file" "$block"
+  fetch "$cdn/$file" "$parts/$file" "$size" "$sum" "$block" || exit 1
+  unzip -qo "$parts/$file" -d "$steam_root"
+  : > "$parts/$file.unpacked"
+  rm -f "$parts/$file"
+  done_bytes=$((done_bytes + size))
 done < "$work/parts"
 # Some entries are zipped with Windows separators; unzip keeps the backslash in the file name.
 find "$steam_root" -depth -name '*\\*' | while read -r odd; do
@@ -90,6 +177,7 @@ ln -sfn "$steam_root/linuxarm64" "$HOME/.steam/sdkarm64"
 ln -sfn "$steam_root/steamrtarm64" "$HOME/.steam/binarm64"
 ln -sfn "$steam_root/linux32" "$HOME/.steam/sdk32"
 echo "$version" > "$stamp"
+rm -rf "$parts"
 if [ -n "$repair" ]; then
   echo repaired > "$unconfirmed"
   rm -f "$repair_request"

tools/tests/test_steam_install.py

@@ -26,13 +26,43 @@ class SteamInstallTest(unittest.TestCase):
         self.bin = self.root / 'bin'
         self.bin.mkdir()
         # Only the network is faked: exercise real manifest parsing, checksums and zip extraction.
+        # TEST_PLAN lines "<file> <action>" are used up in order by requests for that file:
+        # "cut <bytes>" delivers up to that offset then drops the connection, "fail <status>"
+        # delivers nothing, "norange" refuses to resume.
         curl = self.bin / 'curl'
-        curl.write_text('''#!/bin/bash
-set -eu
-printf '%s\\n' "$2" >> "$TEST_REQUESTS"
-cp "$TEST_CDN/${2##*/}" "$4"
+        curl.write_text('''#!/usr/bin/env python3
+import os, sys
+args = sys.argv[1:]
+url = next(a for a in args if a.startswith('https://'))
+out = args[args.index('-o') + 1]
+name = url.rsplit('/', 1)[1]
+offset = os.path.getsize(out) if '-C' in args and os.path.exists(out) else 0
+with open(os.environ['TEST_REQUESTS'], 'a') as log:
+    log.write(url + '\\n')
+with open(os.environ['TEST_REQUESTS'] + '.offsets', 'a') as log:
+    log.write(f'{name} {offset}\\n')
+action = []
+plan = os.environ.get('TEST_PLAN')
+if plan and os.path.exists(plan):
+    lines = open(plan).read().splitlines()
+    for n, line in enumerate(lines):
+        if line.split()[0] == name:
+            action = line.split()[1:]
+            del lines[n]
+            open(plan, 'w').write(''.join(l + '\\n' for l in lines))
+            break
+if action[:1] == ['fail']:
+    sys.exit(int(action[1]))
+if action[:1] == ['norange'] and offset:
+    sys.exit(33)
+data = open(os.path.join(os.environ['TEST_CDN'], name), 'rb').read()
+end = int(action[1]) if action[:1] == ['cut'] else len(data)
+with open(out, 'ab' if offset else 'wb') as target:
+    target.write(data[offset:end])
+sys.exit(28 if end < len(data) else 0)
 ''')
         curl.chmod(0o755)
+        self.plan = self.root / 'plan'
         # macOS has shasum; the guest and Linux CI have sha256sum.
         if shutil.which('sha256sum') is None:
             sha = self.bin / 'sha256sum'
@@ -44,7 +74,8 @@ cp "$TEST_CDN/${2##*/}" "$4"
             sha1.chmod(0o755)
         self.env = dict(os.environ, HOME=str(self.home), XDG_DATA_HOME=str(self.data),
                         PATH=str(self.bin) + os.pathsep + os.environ['PATH'],
-                        TEST_CDN=str(self.cdn), TEST_REQUESTS=str(self.requests))
+                        TEST_CDN=str(self.cdn), TEST_REQUESTS=str(self.requests),
+                        TEST_PLAN=str(self.plan), BL_STEAM_RETRY_DELAYS='0 0 0')
         for channel in ('steamdeck_publicbeta',):
             archive = self.cdn / (channel + '.zip')
             with zipfile.ZipFile(archive, 'w') as package:
@@ -157,5 +188,123 @@ cp "$TEST_CDN/${2##*/}" "$4"
         self.assertFalse(self.requests.exists())
 
 
+    def publish(self, **parts):
+        """The CDN's client as these parts (block name -> byte count of incompressible content),
+        listed with sizes as Valve's manifest lists them."""
+        files = {}
+        blocks = ''
+        for block, length in parts.items():
+            archive = self.cdn / f'{block}.zip.{hashlib.sha1(block.encode()).hexdigest()}'
+            with zipfile.ZipFile(archive, 'w', zipfile.ZIP_STORED) as package:
+                package.writestr(f'parts/{block}', os.urandom(length))
+                if block.endswith('_linuxarm64_linuxarm64'):
+                    package.writestr('steamrtarm64/steam', 'steamdeck_publicbeta')
+            data = archive.read_bytes()
+            files[block] = archive.name
+            blocks += (f'\t"{block}"\n\t{{\n\t\t"file"\t\t"{archive.name}"\n\t\t"size"\t\t"{len(data)}"\n'
+                       f'\t\t"sha2"\t\t"{hashlib.sha256(data).hexdigest()}"\n\t}}\n')
+        (self.cdn / 'steam_client_steamdeck_publicbeta_linuxarm64').write_text(
+            '"linuxarm64"\n{\n\t"version"\t\t"123456"\n' + blocks + '}\n')
+        return files
+
+    def two_parts(self):
+        return self.publish(a_all=1_200_000, client_linuxarm64_linuxarm64=1_300_000)
+
+    def plan_for(self, name, *actions):
+        with self.plan.open('a') as plan:
+            plan.writelines(f'{name} {action}\n' for action in actions)
+
+    def offsets(self, name):
+        log = Path(str(self.requests) + '.offsets')
+        return [int(line.split()[1]) for line in log.read_text().splitlines() if line.split()[0] == name]
+
+    def requested(self):
+        return [line.rsplit('/', 1)[1] for line in self.requests.read_text().splitlines()]
+
+    @property
+    def download(self):
+        return self.steam / 'package/droiddeck-download'
+
+    def test_progress_is_counted_in_megabytes_across_parts(self):
+        self.two_parts()
+        result = self.install()
+        self.assertEqual(0, result.returncode, result.stderr)
+        self.assertIn('downloading Steam: a_all (1/2), 0 of 2 MB\n', result.stdout)
+        self.assertIn('downloading Steam: client_linuxarm64_linuxarm64 (2/2), 1 of 2 MB\n', result.stdout)
+        self.assertTrue((self.steam / 'parts/a_all').is_file())
+        self.assertFalse(self.download.exists())
+
+    def test_a_dropped_connection_resumes_from_what_arrived(self):
+        files = self.two_parts()
+        client = files['client_linuxarm64_linuxarm64']
+        self.plan_for(client, 'cut 500000')
+        result = self.install()
+        self.assertEqual(0, result.returncode, result.stderr)
+        self.assertEqual([0, 500000], self.offsets(client))
+        self.assertIn('interrupted (curl exit 28), trying again in 0 s', result.stdout)
+        self.assertEqual('123456\n', (self.steam / 'package/droiddeck-installed').read_text())
+
+    def test_a_failed_session_keeps_what_it_finished_and_fetched(self):
+        files = self.two_parts()
+        client = files['client_linuxarm64_linuxarm64']
+        self.plan_for(client, 'cut 400000', *['fail 6'] * 4)
+        result = self.install()
+        self.assertNotEqual(0, result.returncode)
+        self.assertIn('could not download client_linuxarm64_linuxarm64 (curl exit 6)', result.stderr)
+        self.assertFalse((self.steam / 'package/droiddeck-installed').exists())
+        self.assertTrue((self.download / (files['a_all'] + '.unpacked')).is_file())
+        self.assertFalse((self.download / files['a_all']).exists())
+        self.assertEqual(400000, (self.download / client).stat().st_size)
+        self.requests.unlink()
+        Path(str(self.requests) + '.offsets').unlink()
+        self.assertEqual(0, self.install().returncode)
+        # The next session asks only for the rest of the part it did not finish.
+        self.assertEqual(['steam_client_steamdeck_publicbeta_linuxarm64', client], self.requested())
+        self.assertEqual([400000], self.offsets(client))
+        self.assertFalse(self.download.exists())
+
+    def test_a_server_that_will_not_resume_starts_the_part_over(self):
+        files = self.two_parts()
+        client = files['client_linuxarm64_linuxarm64']
+        self.plan_for(client, 'cut 400000', 'norange')
+        result = self.install()
+        self.assertEqual(0, result.returncode, result.stderr)
+        self.assertEqual([0, 400000, 0], self.offsets(client))
+
+    def test_bytes_that_do_not_check_out_are_fetched_again(self):
+        files = self.two_parts()
+        client = files['client_linuxarm64_linuxarm64']
+        self.download.mkdir(parents=True)
+        (self.download / client).write_bytes(b'x' * (self.cdn / client).stat().st_size)
+        result = self.install()
+        self.assertEqual(0, result.returncode, result.stderr)
+        self.assertEqual([0], self.offsets(client))
+
+    def test_parts_left_for_another_client_version_are_dropped(self):
+        files = self.two_parts()
+        self.download.mkdir(parents=True)
+        (self.download / 'a_all.zip.0123').write_bytes(b'older')
+        (self.download / 'b_all.zip.4567.unpacked').write_bytes(b'')
+        self.plan_for(files['client_linuxarm64_linuxarm64'], *['fail 7'] * 4)
+        self.assertNotEqual(0, self.install().returncode)
+        self.assertEqual([files['a_all'] + '.unpacked'], sorted(p.name for p in self.download.iterdir()))
+
+    def test_a_repair_unpacks_every_part_again_once(self):
+        files = self.two_parts()
+        self.assertEqual(0, self.install().returncode)
+        (self.home / '.bl-steam-repair').write_text('1\n')
+        self.requests.unlink()
+        self.plan_for(files['client_linuxarm64_linuxarm64'], *['fail 7'] * 4)
+        self.assertNotEqual(0, self.install().returncode)
+        self.assertIn(files['a_all'], self.requested())
+        self.requests.unlink()
+        # Still a repair, but what the interrupted one redid is not done a third time.
+        self.assertEqual(0, self.install().returncode)
+        self.assertEqual(['steam_client_steamdeck_publicbeta_linuxarm64', files['client_linuxarm64_linuxarm64']],
+                         self.requested())
+        self.assertFalse((self.home / '.bl-steam-repair').exists())
+        self.assertFalse(self.download.exists())
+
+
 if __name__ == '__main__':
     unittest.main()