Merge pull request #19 from Droid-Deck/fix/log-privacy

Session logs: scrub public addresses, device id, Steam account names, and every file

app/src/main/java/com/droiddeck/launcher/MainActivity.kt

@@ -403,7 +403,10 @@ class MainActivity : ComponentActivity() {
         }
         if (wanted.isNotEmpty()) requestPermissions(wanted.toTypedArray(), 1)
         // A session folder left without its ending - the process was killed - gets it now.
-        if (!SessionState.running) Thread({ SessionArtifacts.finishAbandoned(this) }, "finish-abandoned").start()
+        if (!SessionState.running) Thread({
+            SessionArtifacts.finishAbandoned(this)
+            SessionArtifacts.scrubOlder(this)
+        }, "finish-abandoned").start()
     }
 
     override fun onResume() {

app/src/main/java/com/droiddeck/launcher/core/LogRedactor.kt

@@ -12,9 +12,16 @@ package com.droiddeck.launcher.core
  * client logs; the account-name registration is left out because this app never learns it.
  *
  * What is deliberately KEPT, because it is what makes a log worth reading: EResult codes, CM host
- * names and IP addresses, timings, token expiry dates, app IDs, pids, connection-state changes and
- * file paths. A SteamID is masked rather than deleted - first four and last four digits survive -
- * so a reader can still tell two accounts' lines apart without the value identifying either.
+ * names and Valve's server addresses, timings, token expiry dates, app IDs, pids, connection-state
+ * changes and file paths. A SteamID is masked rather than deleted - first four and last four digits
+ * survive - so a reader can still tell two accounts' lines apart without the value identifying either.
+ *
+ * The device's OWN addresses are not kept: the client's IPv6 check logs "external address <ours>"
+ * about twenty times a session, and a public IPv6 address is the user's. [learnOwnAddresses] reads
+ * the link the session was given (etc/bannerlator-net) and every address on the same /64 - privacy
+ * addresses rotate inside it - or an own public IPv4 is replaced wherever it appears. Likewise every
+ * Steam account and persona name on the device ([learnAccounts]), for users who sign in with an
+ * account name rather than an email.
  */
 object LogRedactor {
     private val EMAIL = Regex("[A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z]{2,}")
@@ -48,6 +55,145 @@ object LogRedactor {
     private val LONG_TOKEN = Regex("[A-Za-z0-9_\\-]{88,}")
     private val STEAMID64 = Regex("\\b(76561)(\\d{8})(\\d{4})\\b")
     private val STEAMID3 = Regex("\\[U:1:(\\d+)]")
+    /** "external address 2607:..." / "external IP: 203.0.113.9" - the client stating ours. */
+    private val EXTERNAL_ADDR = Regex("(?i)(external\\s+(?:ip\\s+)?address\\s*[:=]?\\s*|external\\s+ip\\s*[:=]?\\s*)([0-9A-Fa-f:.]{7,})")
+    private val IPV4 = Regex("(?<![0-9.])(\\d{1,3})\\.(\\d{1,3})\\.(\\d{1,3})\\.(\\d{1,3})(?![0-9.])")
+    /** An IPv6 literal: "::" somewhere, or all eight groups (a clock's 10:03:05 is neither). */
+    private val IPV6 = Regex("(?<![0-9A-Za-z:])[0-9A-Fa-f]{0,4}(?::[0-9A-Fa-f]{0,4}){2,7}(?:%[A-Za-z0-9_.]+)?(?![0-9A-Za-z:])")
+
+    /**
+     * The account a Steam UI login line names ("Login: OnLoginStateChange <account> 2 1 0 0"): a
+     * user who signs in with a plain account name, not an email, had it in every log. The single
+     * space then a non-space keeps "OnLoginStateChange  0 1 0 0" (no account yet) as it is.
+     */
+    private val LOGIN_STATE = Regex("(OnLoginStateChange )(\\S+)")
+    private val LOGIN_USERS = Regex("(OnLoginUsersChanged )(\\S.*)$")
+
+    /** The device's Steam accounts and persona names as patterns (see [learnAccounts]). */
+    @Volatile
+    private var accounts: List<Regex> = emptyList()
+
+    /**
+     * Learns every Steam account on the device from the client's loginusers.vdf - its AccountName
+     * and PersonaName - so they are replaced wherever a log mentions them, whoever the user is.
+     * Names under three characters are skipped: blanking every "a" would ruin a log.
+     */
+    fun learnAccounts(loginUsers: java.io.File) {
+        val found = ArrayList<Regex>()
+        try {
+            if (loginUsers.isFile) {
+                val kv = Regex("\"(AccountName|PersonaName)\"\\s+\"([^\"]*)\"")
+                kv.findAll(loginUsers.readText()).map { it.groupValues[2].trim() }.filter { it.length >= 3 }.distinct().forEach { name ->
+                    found += Regex("(?i)(?<![A-Za-z0-9_])" + Regex.escape(name) + "(?![A-Za-z0-9_])")
+                }
+            }
+        } catch (e: Exception) {
+            return
+        }
+        accounts = found
+    }
+
+    /** Everything the session's runtime can tell about whose logs these are: addresses and accounts. */
+    fun learnFromRuntime(root: java.io.File) {
+        learnOwnAddresses(java.io.File(root, "etc/bannerlator-net"))
+        learnAccounts(java.io.File(root, "root/.local/share/Steam/config/loginusers.vdf"))
+    }
+
+    /** This device's public addresses as patterns (see [learnOwnAddresses]); empty until learned. */
+    @Volatile
+    private var own: List<Regex> = emptyList()
+
+    /**
+     * Learns the device's addresses from the link file the app writes for the session
+     * (`addr <address> <prefix>` lines). Global IPv6 become their /64, public IPv4 stay exact;
+     * private, link-local and loopback addresses identify nobody and are left out.
+     */
+    fun learnOwnAddresses(linkFile: java.io.File) {
+        val found = ArrayList<Regex>()
+        try {
+            if (linkFile.isFile) linkFile.forEachLine { line ->
+                val addr = line.trim().takeIf { it.startsWith("addr ") }?.split(Regex("\\s+"))?.getOrNull(1) ?: return@forEachLine
+                when (kind(addr)) {
+                    "public IPv6" -> {
+                        val groups = expand6(addr)?.take(4) ?: return@forEachLine
+                        // Any spelling of an address in this /64: leading zeros dropped, :: anywhere after.
+                        val prefix = groups.joinToString(":") { g -> "0{0,3}" + Regex.escape(g.trimStart('0').ifEmpty { "0" }) }
+                        found += Regex("(?i)(?<![0-9A-Fa-f:])$prefix(?::[0-9A-Fa-f]{0,4}){1,4}(?:%[A-Za-z0-9_.]+)?")
+                    }
+                    "public IPv4" -> found += Regex("(?<![0-9.])" + Regex.escape(addr) + "(?![0-9.])")
+                }
+            }
+        } catch (e: Exception) {
+            return
+        }
+        own = found
+    }
+
+    /** "public IPv6", "private IPv4", "link-local IPv6", ... for an address literal; null if it is not one. */
+    fun kind(address: String): String? {
+        val a = address.substringBefore('%')
+        IPV4.matchEntire(a)?.let { m ->
+            val o = m.groupValues.drop(1).map { it.toInt() }
+            if (o.any { it > 255 }) return null
+            return when {
+                o[0] == 127 -> "loopback IPv4"
+                o[0] == 10 || (o[0] == 172 && o[1] in 16..31) || (o[0] == 192 && o[1] == 168) -> "private IPv4"
+                o[0] == 169 && o[1] == 254 -> "link-local IPv4"
+                o[0] == 100 && o[1] in 64..127 -> "carrier-NAT IPv4"
+                else -> "public IPv4"
+            }
+        }
+        val g = expand6(a) ?: return null
+        val first = g[0].toInt(16)
+        return when {
+            g.all { it.toInt(16) == 0 } || (g.take(7).all { it.toInt(16) == 0 } && g[7].toInt(16) == 1) -> "loopback IPv6"
+            first and 0xffc0 == 0xfe80 -> "link-local IPv6"
+            first and 0xfe00 == 0xfc00 -> "private IPv6"
+            else -> "public IPv6"
+        }
+    }
+
+    /** The eight groups of an IPv6 literal, or null. */
+    private fun expand6(address: String): List<String>? {
+        val a = address.substringBefore('%')
+        if (!a.contains(':') || a.count { it == ':' } > 7 || a.contains(":::")) return null
+        if (!a.all { it.isDigit() || it in 'a'..'f' || it in 'A'..'F' || it == ':' }) return null
+        val parts = if (a.contains("::")) {
+            val (head, tail) = a.split("::", limit = 2)
+            val h = if (head.isEmpty()) emptyList() else head.split(':')
+            val t = if (tail.isEmpty()) emptyList() else tail.split(':')
+            if (h.size + t.size > 7) return null
+            h + List(8 - h.size - t.size) { "0" } + t
+        } else a.split(':')
+        if (parts.size != 8 || parts.any { it.isEmpty() || it.length > 4 }) return null
+        return parts.map { it.lowercase() }
+    }
+
+    /**
+     * Every IP address in [text] replaced by what kind it is ("<public IPv6>"), for reports whose
+     * point is the shape of the network, not its numbers.
+     */
+    fun describeAddresses(text: String): String {
+        var out = IPV6.replace(text) { m -> kind(m.value)?.let { "<$it>" } ?: m.value }
+        out = IPV4.replace(out) { m -> kind(m.value)?.let { "<$it>" } ?: m.value }
+        return out
+    }
+
+    /** A file worth scrubbing: text (no NUL in its first 4 KB) and not huge. */
+    fun isText(file: java.io.File): Boolean = try {
+        file.isFile && file.length() < 64L * 1024 * 1024 && file.inputStream().use { input ->
+            val buf = ByteArray(4096)
+            val n = input.read(buf)
+            n <= 0 || (0 until n).none { buf[it].toInt() == 0 }
+        }
+    } catch (e: Exception) {
+        false
+    }
+
+    /** [src]'s lines, scrubbed, to [out]. */
+    fun scrubTo(src: java.io.File, out: java.io.Writer) {
+        src.forEachLine { line -> out.write(redact(line)); out.write("\n") }
+    }
 
     /** [line] with every credential shape replaced. Null- and exception-safe by construction. */
     fun redact(line: String): String {
@@ -66,6 +212,11 @@ object LogRedactor {
                 val id = m.groupValues[1]
                 "[U:1:${if (id.length > 4) "*".repeat(id.length - 4) + id.takeLast(4) else id}]"
             }
+            out = EXTERNAL_ADDR.replace(out) { "${it.groupValues[1]}<redacted:ip>" }
+            for (r in own) out = r.replace(out, "<redacted:ip>")
+            out = LOGIN_STATE.replace(out) { "${it.groupValues[1]}<redacted:account>" }
+            out = LOGIN_USERS.replace(out) { "${it.groupValues[1]}<redacted:account>" }
+            for (r in accounts) out = r.replace(out, "<redacted:account>")
             out = EMAIL.replace(out, "<redacted:email>")
             out = RESIDUAL.replace(out) { "${it.groupValues[1]}=<redacted:token>" }
             out = LONG_TOKEN.replace(out, "<redacted:token>")

app/src/main/java/com/droiddeck/launcher/core/NetworkReport.kt

@@ -27,7 +27,12 @@ object NetworkReport {
 
     fun write(context: Context, target: File) {
         try {
-            target.writeText(build(context))
+            // Addresses as their kind ("<public IPv6>"), never the numbers: a public address is the
+            // user's, and this file goes into shared session logs. The session's made-up hardware
+            // address is stable per device (seeded from ANDROID_ID), so it is masked too.
+            val text = LogRedactor.describeAddresses(build(context))
+                .replace(Regex("(?m)^(\\s*mac )\\S+"), "$1<masked>")
+            target.writeText(text)
         } catch (e: Exception) {
             Log.w(TAG, "could not write $target", e)
         }
@@ -36,7 +41,7 @@ object NetworkReport {
     private fun build(context: Context): String = buildString {
         append("Network as the session starts\n")
         append("=============================\n")
-        append("The SSID is deliberately not recorded. Transport, address families and DNS are.\n\n")
+        append("The SSID is deliberately not recorded, and addresses appear as their kind, not their numbers.\n\n")
         val cm = context.getSystemService(Context.CONNECTIVITY_SERVICE) as? ConnectivityManager
         val network = cm?.activeNetwork
         if (cm == null || network == null) {

app/src/main/java/com/droiddeck/launcher/session/SessionArtifacts.kt

@@ -31,6 +31,9 @@ object SessionArtifacts {
     /** Written last; a folder without it did not get its ending. */
     const val COMPLETE_MARKER = ".complete"
 
+    /** Every file in the folder has been through the redactor, own addresses and accounts included (-2: accounts added). */
+    private const val SCRUBBED_MARKER = ".scrubbed-2"
+
     /** Everything the end of a session gathers, into [dir]. Safe to call for a dead session. */
     fun collect(context: Context, dir: File, reason: String) {
         try {
@@ -40,9 +43,12 @@ object SessionArtifacts {
                     src.copyTo(wayland, overwrite = true)
                 }
             }
+            LogRedactor.learnFromRuntime(LinuxRuntime.rootDir(context))
             copySteamLogs(context, dir)
             // A session the system killed leaves its trace here and nowhere else.
             SessionLogCapture.dumpCrashBuffer(File(dir, "crash.log"))
+            scrubFolder(dir)
+            try { File(dir, SCRUBBED_MARKER).writeText("scrubbed ${now()}\n") } catch (e: Exception) {}
             SessionEvents.record("session.artifacts_collected", mapOf("reason" to reason), dir)
             File(dir, COMPLETE_MARKER).writeText("collected: $reason at ${now()}\n")
         } catch (e: Exception) {
@@ -50,6 +56,47 @@ object SessionArtifacts {
         }
     }
 
+    /**
+     * Once, for session folders written before every file was scrubbed and before the device's own
+     * addresses were (network.txt listed them; the client's IPv6 check logs "external address"
+     * into steam/connection_log.txt): the whole folder, steam/ included, through the redactor.
+     * A marker records it. Runs at app start with [finishAbandoned].
+     */
+    @Synchronized
+    fun scrubOlder(context: Context) {
+        val current = SessionPaths.current()
+        val dirs = LinuxRuntime.debugLogDir().listFiles { f ->
+            f.isDirectory && f.name.startsWith("session-") && f != current && !File(f, SCRUBBED_MARKER).exists()
+        } ?: return
+        if (dirs.isEmpty()) return
+        LogRedactor.learnFromRuntime(LinuxRuntime.rootDir(context))
+        dirs.forEach { dir ->
+            scrubFolder(dir)
+            File(dir, "steam").takeIf { it.isDirectory }?.let { scrubFolder(it) }
+            try { File(dir, SCRUBBED_MARKER).writeText("scrubbed ${now()}\n") } catch (e: Exception) {}
+        }
+        Log.i(TAG, "scrubbed ${dirs.size} older session folder(s)")
+    }
+
+    /**
+     * The session's own files (session.log, app.log, desktop.log, the Steam desktop client's
+     * steam-desktop.log, ...) were written as they happened, unscrubbed; this pass puts every one
+     * through the redactor before the folder can be shared. A file is rewritten only if a line
+     * changed. steam/ was scrubbed on the way in.
+     */
+    private fun scrubFolder(dir: File) {
+        dir.listFiles { f -> f.isFile && !f.name.startsWith(".") }?.forEach { f ->
+            if (!LogRedactor.isText(f)) return@forEach
+            try {
+                val tmp = File(dir, ".${f.name}.scrub")
+                tmp.bufferedWriter().use { w -> LogRedactor.scrubTo(f, w) }
+                if (tmp.length() != f.length() || tmp.readBytes().contentEquals(f.readBytes()).not()) tmp.renameTo(f) else tmp.delete()
+            } catch (e: Exception) {
+                Log.w(TAG, "could not scrub ${f.name}", e)
+            }
+        }
+    }
+
     /** Steam's logs: redacted into steam/, never copied verbatim. */
     private fun copySteamLogs(context: Context, dir: File) {
         val logs = File(LinuxRuntime.rootDir(context), "root/.local/share/Steam/logs")
@@ -81,6 +128,7 @@ object SessionArtifacts {
         }?.sortedBy { it.name } ?: return
         if (abandoned.isEmpty()) return
         val current = SessionPaths.current()
+        LogRedactor.learnFromRuntime(LinuxRuntime.rootDir(context))
         abandoned.forEachIndexed { i, dir ->
             if (dir == current) return@forEachIndexed
             val newest = i == abandoned.lastIndex
@@ -100,6 +148,7 @@ object SessionArtifacts {
                 }
                 SessionLogCapture.dumpCrashBuffer(File(dir, "crash.log"))
                 SessionEvents.record("session.artifacts_recovered", mapOf("newest" to newest), dir)
+                scrubFolder(dir)
                 File(dir, COMPLETE_MARKER).writeText("collected: late, at next app start, ${now()}\n")
                 Log.i(TAG, "finished the abandoned session folder $dir")
             } catch (e: Exception) {

app/src/main/java/com/droiddeck/launcher/session/SessionLogShare.kt

@@ -4,6 +4,7 @@ import android.content.ClipData
 import android.content.Context
 import android.content.Intent
 import androidx.core.content.FileProvider
+import com.droiddeck.launcher.core.LogRedactor
 import com.droiddeck.launcher.runtime.LinuxRuntime
 import java.io.File
 import java.util.zip.ZipEntry
@@ -31,10 +32,19 @@ object SessionLogShare {
         if (files.isEmpty()) return null
         val out = File(context.cacheDir, "shared-logs").apply { deleteRecursively(); mkdirs() }
         val zip = File(out, "DroidDeck-${folder.name}.zip")
+        // Scrubbed on the way into the zip: a session shared while it runs has not had its end-of-
+        // session pass yet, and the redactor changes nothing in a line that is already clean.
+        LogRedactor.learnFromRuntime(LinuxRuntime.rootDir(context))
         ZipOutputStream(zip.outputStream().buffered()).use { z ->
             files.forEach { f ->
                 z.putNextEntry(ZipEntry(folder.name + "/" + f.relativeTo(folder).path))
-                f.inputStream().use { it.copyTo(z) }
+                if (LogRedactor.isText(f)) {
+                    val w = z.bufferedWriter()
+                    LogRedactor.scrubTo(f, w)
+                    w.flush()
+                } else {
+                    f.inputStream().use { it.copyTo(z) }
+                }
                 z.closeEntry()
             }
         }