diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ae0b401c..c7b73e87 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -389,7 +389,7 @@ jobs: echo echo "The \`droiddeck-apk\` artifact is [on this run]($RUN_URL), signed with the public test key." echo - echo "**Maintainer:** add the \`publish-test\` label to publish a release-signed tester build that installs over Stable or Nightly and appears in DroidDeck's **Test builds** list. The label is removed automatically after publishing." + echo "**Maintainer:** add the \`publish-test\` label to publish a release-signed tester build that installs over Stable or Preview and appears in DroidDeck's **Test builds** list. The label is removed automatically after publishing." echo echo "Fallback: [open **PR Release Publish**]($GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/workflows/pr-release-publish.yml) on \`main\` and enter PR \`$PR\`." ;; @@ -519,6 +519,8 @@ jobs: notes="$(cat notes-head.md) Signed build of $GITHUB_SERVER_URL/$GITHUB_REPOSITORY/commit/$GITHUB_SHA - in main, not yet in a DroidDeck release; for testing. + Commit: $GITHUB_SHA + VersionCode: $(sed -n 's/^ *versionCode \([0-9][0-9]*\)$/\1/p' app/build.gradle) Each apk installs over a DroidDeck release and keeps its data. \`standard\` is the normal app; the others are the same app under another package name, installed beside it. SHA-256: @@ -580,3 +582,9 @@ jobs: jq -n --arg content "$content" '{content: $content, allowed_mentions: {parse: []}}' | curl -fsS -H 'Content-Type: application/json' -d @- "$WEBHOOK" -o /dev/null echo "Posted: $content" + + catalog: + needs: announce + if: ${{ always() && github.event_name != 'pull_request' && github.ref == 'refs/heads/main' && needs.announce.result != 'skipped' }} + uses: ./.github/workflows/update-catalog.yml + secrets: inherit diff --git a/.github/workflows/pr-release-cleanup.yml b/.github/workflows/pr-release-cleanup.yml index a34254af..29ebbdef 100644 --- a/.github/workflows/pr-release-cleanup.yml +++ b/.github/workflows/pr-release-cleanup.yml @@ -2,14 +2,15 @@ name: PR Release Cleanup run-name: remove PR #${{ github.event.pull_request.number }}'s test build # A merged or closed PR's test build leaves DroidDeck-CI, so the app's Updates page stops offering it -# (a merged fix is in the next Nightly). pull_request_target for the publisher app's secrets; nothing +# (a merged fix is in the next Preview). pull_request_target for the publisher app's secrets; nothing # from the PR is checked out or run. on: pull_request_target: types: [ closed ] branches: [ main ] -permissions: {} +permissions: + contents: read jobs: cleanup: @@ -39,3 +40,9 @@ jobs: fi done [ "$found" = 1 ] || echo "No test build for PR #$PR_NUMBER" + + catalog: + needs: cleanup + if: ${{ always() }} + uses: ./.github/workflows/update-catalog.yml + secrets: inherit diff --git a/.github/workflows/pr-release-publish.yml b/.github/workflows/pr-release-publish.yml index 09143833..d8caa090 100644 --- a/.github/workflows/pr-release-publish.yml +++ b/.github/workflows/pr-release-publish.yml @@ -1,5 +1,5 @@ name: PR Release Publish -run-name: ${{ github.event_name == 'pull_request_target' && format('sign PR #{0} requested by @{1}', github.event.pull_request.number, github.actor) || format('sign PR #{0} with the release key', inputs.pr_number) }} +run-name: "sign PR #${{ github.event_name == 'pull_request_target' && github.event.pull_request.number || inputs.pr_number }} with the release key" on: workflow_dispatch: @@ -32,6 +32,8 @@ jobs: pr_number: ${{ steps.gate.outputs.pr_number }} run_id: ${{ steps.gate.outputs.run_id }} head_sha: ${{ steps.gate.outputs.head_sha }} + base_sha: ${{ steps.gate.outputs.base_sha }} + version_code: ${{ steps.gate.outputs.version_code }} label: ${{ steps.gate.outputs.label }} variants: ${{ steps.variants.outputs.list }} steps: @@ -105,6 +107,8 @@ jobs: echo "pr_number=$PR_NUMBER" echo "run_id=$run_id" echo "head_sha=$head" + echo "base_sha=$base" + echo "version_code=${candidate##* }" echo "label=PR.$PR_NUMBER-${head:0:7}" } >> "$GITHUB_OUTPUT" @@ -167,6 +171,8 @@ jobs: GH_TOKEN: ${{ steps.ci-token.outputs.token }} PR_NUMBER: ${{ needs.authorize.outputs.pr_number }} HEAD_SHA: ${{ needs.authorize.outputs.head_sha }} + BASE_SHA: ${{ needs.authorize.outputs.base_sha }} + VERSION_CODE: ${{ needs.authorize.outputs.version_code }} run: | set -euo pipefail repo=Droid-Deck/DroidDeck-CI @@ -177,6 +183,8 @@ jobs: Source: https://github.com/$GITHUB_REPOSITORY/pull/$PR_NUMBER Commit: $HEAD_SHA + Base: $BASE_SHA + VersionCode: $VERSION_CODE Each apk installs over a DroidDeck release and keeps its data. \`standard\` is the normal app; the others are the same app under another package name, installed beside it. SHA-256: @@ -218,7 +226,7 @@ jobs: $mark ## Signed test build - A release-signed build of this PR at \`$short\` is ready. It installs over Stable or Nightly and keeps app data. + A release-signed build of this PR at \`$short\` is ready. It installs over Stable or Preview and keeps app data. [Download the standard APK]($APK) · [All variants and checksums]($RELEASE) @@ -265,3 +273,9 @@ jobs: jq -n --arg content "$content" '{content: $content, allowed_mentions: {parse: []}}' | curl -fsS -H 'Content-Type: application/json' -d @- "$WEBHOOK" -o /dev/null echo "Posted: $content" + + catalog: + needs: publish + if: ${{ always() && needs.publish.result != 'skipped' }} + uses: ./.github/workflows/update-catalog.yml + secrets: inherit diff --git a/.github/workflows/update-catalog.yml b/.github/workflows/update-catalog.yml new file mode 100644 index 00000000..83b948d4 --- /dev/null +++ b/.github/workflows/update-catalog.yml @@ -0,0 +1,41 @@ +name: Update app catalog + +on: + workflow_call: + workflow_dispatch: + release: + types: [published, edited, deleted, unpublished] + +permissions: + contents: read + +concurrency: + group: update-app-catalog + cancel-in-progress: false + +jobs: + catalog: + runs-on: ubuntu-24.04 + timeout-minutes: 5 + steps: + # Always use the current trusted generator, even when a release event points at an older tag. + - uses: actions/checkout@v4 + with: + ref: main + persist-credentials: false + + - uses: actions/create-github-app-token@v2 + id: ci-token + with: + app-id: ${{ vars.DROIDDECK_CI_APP_ID }} + private-key: ${{ secrets.DROIDDECK_CI_APP_KEY }} + owner: Droid-Deck + repositories: DroidDeck-CI + + - name: Regenerate catalog.json + env: + SOURCE_GITHUB_TOKEN: ${{ github.token }} + CI_GITHUB_TOKEN: ${{ steps.ci-token.outputs.token }} + run: | + set -euo pipefail + python3 tools/release/update_catalog.py diff --git a/app/src/main/java/com/droiddeck/launcher/UpdatesMenu.kt b/app/src/main/java/com/droiddeck/launcher/UpdatesMenu.kt index e93b9f62..6a3e05d0 100644 --- a/app/src/main/java/com/droiddeck/launcher/UpdatesMenu.kt +++ b/app/src/main/java/com/droiddeck/launcher/UpdatesMenu.kt @@ -91,6 +91,7 @@ internal class UpdatesMenu(private val activity: Activity, private val ui: Handl private fun install(r: AppUpdates.Release) { if (stage != null) return val apk = r.apk ?: run { error = "This build has no download for this copy of DroidDeck"; return } + AppUpdates.installBlock(r)?.let { error = it; return } if (SessionState.running) { error = "Stop the running session first, then update"; return } if (!SelfInstaller.canInstall(activity)) { pending = r; askPermission = true; return } error = null diff --git a/app/src/main/java/com/droiddeck/launcher/ui/UpdatesPage.kt b/app/src/main/java/com/droiddeck/launcher/ui/UpdatesPage.kt index e1efe1b0..278cb166 100644 --- a/app/src/main/java/com/droiddeck/launcher/ui/UpdatesPage.kt +++ b/app/src/main/java/com/droiddeck/launcher/ui/UpdatesPage.kt @@ -148,6 +148,7 @@ private fun StatusPanel(s: FrontEndState, u: UpdatesState, ua: UpdatesActions, m val offer = if (me.updatable) catalog?.let { AppUpdates.offer(it, u.follow, me) } else null val name = channelName(u.follow) val offered = release != null && (offer == Offer.UPDATE || offer == Offer.SWITCH) + val installBlock = release?.let { AppUpdates.installBlock(it, me) } class Look(val tint: Color, val status: String, val headline: String, val detail: String?) val look = when { !me.updatable -> Look(colors.onSurfaceVariant, "Signed differently", "Can't update in place", @@ -158,7 +159,7 @@ private fun StatusPanel(s: FrontEndState, u: UpdatesState, ua: UpdatesActions, m offer == Offer.AHEAD -> Look(pal.signal, "Ahead of Stable", "You're ahead of Stable", "This build is newer than the last Stable release. You'll move onto Stable with its next one.") offer == Offer.GONE -> Look(AttentionAmber, "Test ended", "This test has ended", - "Its fix was merged or dropped. Follow Nightly to keep getting the newest fixes.") + "Its fix was merged or dropped. Follow Preview to keep getting the newest fixes.") else -> Look(pal.good, "Up to date", "You have the latest $name", null) } Column( @@ -174,8 +175,11 @@ private fun StatusPanel(s: FrontEndState, u: UpdatesState, ua: UpdatesActions, m ReleaseNotes(changeTitle(release!!.title.ifBlank { release.tag }), release.summary, release.tag) } if (look.detail != null) Text(look.detail, fontSize = 14.sp, color = colors.onSurfaceVariant, modifier = Modifier.padding(top = 10.dp)) + if (installBlock != null && (offered || offer == Offer.AHEAD)) { + Text(installBlock, fontSize = 13.sp, color = AttentionAmber, modifier = Modifier.padding(top = 10.dp)) + } if (u.error != null) Text(u.error, fontSize = 13.sp, color = pal.error, modifier = Modifier.padding(top = 12.dp)) - val installable = release?.apk != null && !s.sessionRunning + val installable = release?.apk != null && installBlock == null && !s.sessionRunning val button = release?.apk?.size?.takeIf { it > 0 }?.let { " · ${megabytes(it)}" }.orEmpty() Box(Modifier.padding(top = 14.dp)) { when { @@ -186,8 +190,9 @@ private fun StatusPanel(s: FrontEndState, u: UpdatesState, ua: UpdatesActions, m } offer == Offer.UPDATE -> PrimaryButton("Update$button", enabled = installable, main = true) { ua.onInstall(release!!) } offer == Offer.SWITCH -> PrimaryButton("Install$button", enabled = installable, main = true) { ua.onInstall(release!!) } - offer == Offer.AHEAD -> SecondaryButton("Install Stable ${release?.version.orEmpty()} anyway", enabled = installable) { ua.onInstall(release!!) } - offer == Offer.GONE -> PrimaryButton("Follow Nightly", main = true) { ua.onFollow(Follow(Channel.NIGHTLY)) } + offer == Offer.AHEAD && installBlock == null -> + SecondaryButton("Install Stable ${release?.version.orEmpty()} anyway", enabled = installable) { ua.onInstall(release!!) } + offer == Offer.GONE -> PrimaryButton("Follow Preview", main = true) { ua.onFollow(Follow(Channel.NIGHTLY)) } } } if (s.sessionRunning && (offered || offer == Offer.AHEAD)) { @@ -226,10 +231,10 @@ private fun ReleaseNotes(title: String, notes: String, key: String) { } } -/** The headline for a build on offer: "DroidDeck 0.3.0", "New Nightly build", "PR #93 test build". */ +/** The headline for a build on offer: "DroidDeck 0.3.0", "New Preview build", "PR #93 test build". */ private fun newBuild(f: Follow, r: Release) = when (f.channel) { Channel.STABLE -> "DroidDeck ${r.version ?: r.tag}" - Channel.NIGHTLY -> "New Nightly build" + Channel.NIGHTLY -> "New Preview build" Channel.TEST -> "PR #${r.pr} test build" } @@ -259,8 +264,8 @@ private fun ChannelPicker(u: UpdatesState, ua: UpdatesActions) { catalog?.stable?.let { "${it.version ?: it.tag} · ${ago(it.publishedAt)}" }, u.follow.channel == Channel.STABLE, ) { ua.onFollow(Follow(Channel.STABLE)) } ChannelCard( - Icons.Outlined.Bolt, "Nightly", "Newest fixes, the odd new bug", - catalog?.nightly?.let { ago(it.publishedAt) }, u.follow.channel == Channel.NIGHTLY, + Icons.Outlined.Bolt, "Preview", "Newest main-branch fixes, before Stable", + catalog?.preview?.let { ago(it.publishedAt) }, u.follow.channel == Channel.NIGHTLY, ) { ua.onFollow(Follow(Channel.NIGHTLY)) } ChannelCard( Icons.Outlined.Science, "Test builds", "Try a fix before it's released", @@ -343,7 +348,7 @@ private fun TestRow(t: Release, selected: Boolean, onClick: () -> Unit) { private fun channelName(f: Follow) = when (f.channel) { Channel.STABLE -> "Stable" - Channel.NIGHTLY -> "Nightly" + Channel.NIGHTLY -> "Preview" Channel.TEST -> "the PR #${f.pr} test" } diff --git a/app/src/main/java/com/droiddeck/launcher/update/AppUpdates.kt b/app/src/main/java/com/droiddeck/launcher/update/AppUpdates.kt index 5d3eb159..3310951d 100644 --- a/app/src/main/java/com/droiddeck/launcher/update/AppUpdates.kt +++ b/app/src/main/java/com/droiddeck/launcher/update/AppUpdates.kt @@ -4,97 +4,118 @@ import android.content.Context import android.content.pm.PackageManager import android.os.Build import com.droiddeck.launcher.BuildConfig -import com.droiddeck.launcher.core.Hashes import org.json.JSONArray import org.json.JSONObject import java.io.IOException import java.net.HttpURLConnection import java.net.URL import java.security.MessageDigest -import java.time.Instant /** - * DroidDeck's own builds, on three channels: Stable (the org repo's latest release), Nightly (the - * newest `main-` release on DroidDeck-CI, one per merge to main) and Test (a `pr-` release - * on DroidDeck-CI, a signed build of an open PR). All three are signed with the release key, so any - * of them installs over any other and keeps the app's data. + * DroidDeck's published builds. CI turns GitHub Releases into one static catalog in DroidDeck-CI; + * the app consumes that contract instead of inferring channels from tags and release-note prose. * - * Which build is newer is decided without a diff of the repo (GitHub's compare answers in - * megabytes): main is linear, so the newest main build is the newest; Stable is ordered by its - * version; a test build is newer when its PR's head moved. The last check is remembered, so the - * page shows it without going online. + * [Channel.NIGHTLY] is intentionally kept as the internal enum/pref value for upgrades from older + * DroidDeck versions. The channel is presented to users as Preview. */ object AppUpdates { private const val REPO = "Droid-Deck/DroidDeck" private const val CI_REPO = "Droid-Deck/DroidDeck-CI" + private const val CATALOG_URL = "https://raw.githubusercontent.com/$CI_REPO/main/catalog.json" private const val PREFS = "app_updates" private const val KEY_CATALOG = "catalog" private const val KEY_CHANNEL = "channel" + private const val CATALOG_SCHEMA = 1 enum class Channel { STABLE, NIGHTLY, TEST } - /** The channel the user follows; [pr] is the followed test build's PR, for [Channel.TEST]. */ data class Follow(val channel: Channel, val pr: Int = 0) - class Apk(val name: String, val url: String, val size: Long, val sha256: String) - - /** - * One published build. [commit] is a full or short SHA of the source it was built from; [pr] is - * 0 for a main or stable build. [apk] is the file for the package this app runs as, or null when - * the build has none (a variant a release predates). - */ - class Release( - val tag: String, val title: String, val summary: String, val commit: String, val pr: Int, - val version: String?, val publishedAt: Long, val url: String, val apk: Apk?, + class Apk( + val name: String, + val url: String, + val size: Long, + val sha256: String, + val packageName: String, + val versionCode: Int, + val signerSha256: String, ) - class Catalog(val stable: Release?, val nightly: Release?, val tests: List, val checkedAt: Long) + class Release( + val tag: String, + val title: String, + val summary: String, + val commit: String, + val pr: Int, + val version: String?, + val versionCode: Int, + val publishedAt: Long, + val url: String, + val apk: Apk?, + ) - /** - * What is running: the commit it was built from, its PR (0 for none), whether the builds here - * install over it (it is signed with the release key), and when it was committed. - */ - class Installed(val commit: String, val pr: Int, val version: String, val updatable: Boolean, val committedAt: Long = 0L) + class Catalog( + val stable: Release?, + val preview: Release?, + val tests: List, + val checkedAt: Long, + ) + + class Installed( + val commit: String, + val pr: Int, + val version: String, + val versionCode: Int, + val updatable: Boolean, + val committedAt: Long = 0L, + ) enum class Offer { - /** This is the build the channel has. */ CURRENT, - /** The channel has a newer build of what is running. */ UPDATE, - /** The channel's build is a different line (a test build from Nightly, or back); installing moves over. */ SWITCH, - /** What is running is newer than the channel's build (a Nightly past the last Stable). */ AHEAD, - /** The followed test build is gone: its PR was merged or closed. */ GONE, } @Volatile private var releaseSigned = BuildConfig.CI_BUILD - /** From App.onCreate: whether this copy is signed with the release key, which a local build may be too. */ fun init(context: Context) { - releaseSigned = runCatching { signers(context).any { it.equals(BuildConfig.RELEASE_SIGNER, ignoreCase = true) } } - .getOrDefault(BuildConfig.CI_BUILD) + releaseSigned = runCatching { + signers(context).any { it.equals(BuildConfig.RELEASE_SIGNER, ignoreCase = true) } + }.getOrDefault(BuildConfig.CI_BUILD) } - /** SHA-256 of each certificate the package is signed with, including the ones a rotated key vouches for. */ private fun signers(context: Context): List { val pm = context.packageManager @Suppress("DEPRECATION") val certs = if (Build.VERSION.SDK_INT >= 28) { - val info = pm.getPackageInfo(context.packageName, PackageManager.GET_SIGNING_CERTIFICATES).signingInfo ?: return emptyList() + val info = pm.getPackageInfo( + context.packageName, + PackageManager.GET_SIGNING_CERTIFICATES, + ).signingInfo ?: return emptyList() if (info.hasMultipleSigners()) info.apkContentsSigners else info.signingCertificateHistory } else { pm.getPackageInfo(context.packageName, PackageManager.GET_SIGNATURES).signatures } ?: return emptyList() - return certs.map { cert -> MessageDigest.getInstance("SHA-256").digest(cert.toByteArray()).joinToString("") { "%02x".format(it) } } + return certs.map { cert -> + MessageDigest.getInstance("SHA-256").digest(cert.toByteArray()) + .joinToString("") { "%02x".format(it) } + } } - fun installed(): Installed = Installed(BuildConfig.SOURCE_COMMIT, BuildConfig.PR_NUMBER, BuildConfig.VERSION_NAME, releaseSigned, BuildConfig.COMMIT_TIME) + fun installed(): Installed = Installed( + BuildConfig.SOURCE_COMMIT, + BuildConfig.PR_NUMBER, + BuildConfig.VERSION_NAME, + BuildConfig.VERSION_CODE, + releaseSigned, + BuildConfig.COMMIT_TIME, + ) fun release(catalog: Catalog, follow: Follow): Release? = when (follow.channel) { Channel.STABLE -> catalog.stable - Channel.NIGHTLY -> catalog.nightly + Channel.NIGHTLY -> catalog.preview Channel.TEST -> catalog.tests.firstOrNull { it.pr == follow.pr } } @@ -112,19 +133,27 @@ object AppUpdates { } } - /** The dot on the rail: the followed channel has a newer build of what is running. */ - fun hasUpdate(catalog: Catalog?, follow: Follow): Boolean = - catalog != null && installed().updatable && offer(catalog, follow) == Offer.UPDATE + fun hasUpdate(catalog: Catalog?, follow: Follow): Boolean { + if (catalog == null) return false + val me = installed() + val r = release(catalog, follow) ?: return false + return me.updatable && offer(catalog, follow, me) == Offer.UPDATE && installBlock(r, me) == null + } fun isRunning(r: Release, me: Installed = installed()): Boolean { if (me.commit.length < 7 || r.commit.length < 7 || r.pr != me.pr) return false - return me.commit.startsWith(r.commit, ignoreCase = true) || r.commit.startsWith(me.commit, ignoreCase = true) + return me.commit.startsWith(r.commit, ignoreCase = true) || + r.commit.startsWith(me.commit, ignoreCase = true) + } + + /** Why Android cannot install this published build over [me], before downloading anything. */ + fun installBlock(r: Release, me: Installed = installed()): String? = when { + r.apk == null -> "This build has no download for this copy of DroidDeck." + r.apk.versionCode < me.versionCode -> + "Android won't install it over this build because its versionCode ${r.apk.versionCode} is below the installed ${me.versionCode}." + else -> null } - /** - * The channel followed. Until the user picks one it is worked out once from what is running and - * kept: a test build follows its PR, the Stable release follows Stable, and a main build Nightly. - */ fun follow(context: Context, catalog: Catalog?): Follow { stored(context)?.let { return it } val me = installed() @@ -140,7 +169,9 @@ object AppUpdates { } fun setFollow(context: Context, follow: Follow) { - val value = if (follow.channel == Channel.TEST) "test:${follow.pr}" else follow.channel.name.lowercase() + // Keep "nightly" on disk so an older DroidDeck still understands the preference after rollback. + val value = if (follow.channel == Channel.TEST) "test:${follow.pr}" + else follow.channel.name.lowercase() prefs(context).edit().putString(KEY_CHANNEL, value).apply() } @@ -148,129 +179,114 @@ object AppUpdates { val v = prefs(context).getString(KEY_CHANNEL, null) ?: return null return when { v == "stable" -> Follow(Channel.STABLE) - v == "nightly" -> Follow(Channel.NIGHTLY) - v.startsWith("test:") -> v.removePrefix("test:").toIntOrNull()?.let { Follow(Channel.TEST, it) } + v == "nightly" || v == "preview" -> Follow(Channel.NIGHTLY) + v.startsWith("test:") -> v.removePrefix("test:").toIntOrNull()?.let { + Follow(Channel.TEST, it) + } else -> null } } - /** The last check, or null before the first. */ fun cached(context: Context): Catalog? { val raw = prefs(context).getString(KEY_CATALOG, null) ?: return null - return runCatching { readCatalog(JSONObject(raw)) }.getOrNull() + return runCatching { readCachedCatalog(JSONObject(raw)) }.getOrNull() } - /** Read every channel now and remember the result: three small requests, four for an annotated tag. */ + /** One static request, rather than several unauthenticated GitHub API requests. */ fun refresh(context: Context): Catalog { - val suffix = variantSuffix(context.packageName) - val stable = runCatching { stable(suffix) }.getOrElse { if (it is NoRelease) null else throw it } - val ci = JSONArray(get("https://api.github.com/repos/$CI_REPO/releases?per_page=50")) - var nightly: Release? = null - val tests = ArrayList() - for (i in 0 until ci.length()) { - val r = ci.getJSONObject(i) - if (r.optBoolean("draft")) continue - val tag = r.getString("tag_name") - when { - tag.startsWith("main-") -> { - val candidate = ciRelease(r, suffix, tag.removePrefix("main-"), 0) - if (nightly == null || candidate.publishedAt > nightly.publishedAt) nightly = candidate - } - tag.startsWith("pr-") -> { - val pr = tag.removePrefix("pr-").toIntOrNull() ?: continue - val body = r.optString("body") - // Only builds of this repo's PRs: a test build from before the org move names another. - if (!body.contains("github.com/$REPO/pull/$pr")) continue - val commit = Regex("""Commit: `?([0-9a-f]{7,40})""").find(body)?.groupValues?.get(1) ?: continue - tests.add(ciRelease(r, suffix, commit, pr)) - } - } - } - tests.sortByDescending { it.publishedAt } - val catalog = Catalog(stable, nightly, tests, System.currentTimeMillis()) + val catalog = readPublishedCatalog( + JSONObject(get(CATALOG_URL)), + context.packageName, + System.currentTimeMillis(), + ) prefs(context).edit().putString(KEY_CATALOG, writeCatalog(catalog).toString()).apply() return catalog } - private class NoRelease : IOException("no stable release") - - private fun stable(suffix: String): Release { - val r = try { - JSONObject(get("https://api.github.com/repos/$REPO/releases/latest")) - } catch (e: IOException) { - if (e.message?.contains("HTTP 404") == true) throw NoRelease() else throw e + internal fun readPublishedCatalog( + root: JSONObject, + packageName: String, + checkedAt: Long, + ): Catalog { + val schema = root.optInt("schema") + val sourceRepo = root.optString("sourceRepo") + val ciRepo = root.optString("ciRepo") + if (schema != CATALOG_SCHEMA || sourceRepo != REPO || ciRepo != CI_REPO) { + throw IOException( + "The update catalog has an unsupported format (schema $schema, source $sourceRepo, CI $ciRepo)", + ) } - val tag = r.getString("tag_name") - // The tag's commit; an annotated tag points at a tag object first. - var obj = JSONObject(get("https://api.github.com/repos/$REPO/git/ref/tags/$tag")).getJSONObject("object") - if (obj.getString("type") == "tag") obj = JSONObject(get(obj.getString("url"))).getJSONObject("object") - val (title, summary) = describeStable(r.optString("name").ifBlank { tag }, r.optString("body")) - return Release(tag, title, summary, obj.getString("sha"), 0, tag.removePrefix("v"), published(r), - r.optString("html_url"), apk(r, suffix)) + val tests = root.optJSONArray("tests") ?: JSONArray() + return Catalog( + root.optJSONObject("stable")?.let { readPublishedRelease(it, packageName) }, + root.optJSONObject("preview")?.let { readPublishedRelease(it, packageName) }, + (0 until tests.length()) + .map { readPublishedRelease(tests.getJSONObject(it), packageName) } + .filter { it.pr > 0 } + .sortedByDescending { it.publishedAt }, + checkedAt, + ) } - private fun ciRelease(r: JSONObject, suffix: String, commit: String, pr: Int): Release { - val (title, summary) = describeCi(r.optString("name"), r.optString("body")) - return Release(r.getString("tag_name"), title, summary, commit, pr, null, published(r), r.optString("html_url"), apk(r, suffix)) - } - - private fun published(r: JSONObject): Long = - runCatching { Instant.parse(r.optString("published_at")).toEpochMilli() }.getOrDefault(0L) - - /** The apk for this package: `.apk`, and for the standard app the one with no other variant's suffix. */ - private fun apk(r: JSONObject, suffix: String): Apk? { - val others = variantSuffixes().values.filter { it.isNotEmpty() && it != suffix } - val assets = r.optJSONArray("assets") ?: return null - for (i in 0 until assets.length()) { - val a = assets.getJSONObject(i) - val name = a.getString("name") - if (!name.endsWith("$suffix.apk") || others.any { name.endsWith("$it.apk") }) continue - // Only a file GitHub has a sha256 for: the download is checked against it. - val sha = Hashes.githubSha256(a.optString("digest")) ?: continue - return Apk(name, a.getString("browser_download_url"), a.optLong("size"), sha) + private fun readPublishedRelease(o: JSONObject, packageName: String): Release { + val commit = o.getString("commit") + if (!Regex("^[0-9a-fA-F]{7,40}$").matches(commit)) { + throw IOException("The update catalog contains an invalid commit") } - return null + val versionCode = o.getInt("versionCode") + if (versionCode <= 0) throw IOException("The update catalog contains an invalid versionCode") + val apk = o.optJSONObject("apks")?.optJSONObject(packageName)?.let { + readPublishedApk(it, packageName) + } + if (apk != null && apk.versionCode != versionCode) { + throw IOException("The update catalog disagrees about the APK versionCode") + } + return Release( + o.getString("tag"), + o.optString("title"), + o.optString("summary"), + commit, + o.optInt("pr"), + if (o.isNull("version")) null else o.optString("version").ifEmpty { null }, + versionCode, + o.optLong("publishedAt"), + o.optString("url"), + apk, + ) } - private fun variantSuffixes(): Map = BuildConfig.VARIANT_SUFFIXES.split(';') - .mapNotNull { e -> e.split('=', limit = 2).takeIf { it.size == 2 }?.let { it[0] to it[1] } }.toMap() - - private fun variantSuffix(packageName: String): String = variantSuffixes()[packageName] ?: "" - - /** - * A CI release's notes open with what it is: "**Merged to main** - [PR #n](…): title (merged …)" - * or "**Not merged** - a test build of open [PR #n](…): title", then a paragraph from the PR. - */ - internal fun describeCi(name: String, body: String): Pair { - val paragraphs = paragraphs(body) - val head = paragraphs.firstOrNull().orEmpty() - val title = head.substringAfter("): ", "").substringBeforeLast(" (merged").ifBlank { plain(head) }.ifBlank { name } - val summary = paragraphs.getOrNull(1)?.takeUnless { it.startsWith("Signed build") || it.startsWith("Source:") }.orEmpty() - return plain(title) to lines(summary) + private fun readPublishedApk(o: JSONObject, packageName: String): Apk { + val actualPackage = o.getString("packageName") + val versionCode = o.getInt("versionCode") + val signer = o.getString("signerSha256").lowercase() + val sha = o.getString("sha256").lowercase() + val url = o.getString("url") + if (actualPackage != packageName) throw IOException("The update catalog names the wrong Android package") + if (versionCode <= 0) throw IOException("The update catalog has an invalid APK versionCode") + if (!Regex("^[0-9a-f]{64}$").matches(sha)) { + throw IOException("The update catalog has an invalid checksum") + } + if (!signer.equals(BuildConfig.RELEASE_SIGNER, ignoreCase = true)) { + throw IOException("The update catalog is not signed for DroidDeck's release key") + } + val allowed = listOf( + "https://github.com/$REPO/releases/download/", + "https://github.com/$CI_REPO/releases/download/", + ) + if (allowed.none { url.startsWith(it) }) { + throw IOException("The update catalog has an unexpected download URL") + } + return Apk( + o.getString("name"), + url, + o.optLong("size"), + sha, + actualPackage, + versionCode, + signer, + ) } - /** A stable release: its name, and the first paragraph of its notes that is prose rather than a heading or picture. */ - private fun describeStable(name: String, body: String): Pair = - name to lines(paragraphs(body).firstOrNull { !it.startsWith("#") && !it.startsWith("![") && !it.startsWith("<") }.orEmpty()) - - private fun paragraphs(body: String): List = - body.replace("\r", "").split(Regex("\n\\s*\n")).map { it.trim() }.filter { it.isNotEmpty() } - - /** A paragraph's lines as text, a list item's marker as a bullet: the page shows them collapsed or in full. */ - private fun lines(md: String): String = md.lines().map { line -> - val item = Regex("""^\s*[-*]\s+""").find(line) - if (item != null) "• " + plain(line.substring(item.range.last + 1)) else plain(line) - }.filter { it.isNotEmpty() }.joinToString("\n") - - /** Markdown down to text: links to their words, emphasis and code marks gone, on one line. */ - private fun plain(md: String): String = md - .replace(Regex("""!\[[^]]*]\([^)]*\)"""), "") - .replace(Regex("""\[([^]]*)]\([^)]*\)"""), "$1") - .replace(Regex("""[*_`]{1,3}"""), "") - .replace(Regex("""\s+"""), " ") - .trim() - - /** Dotted versions, number by number: "0.10.0" is past "0.9.2"; a missing part counts as 0. */ fun compareVersions(a: String, b: String): Int { val x = a.removePrefix("v").split('.', '-').map { it.toIntOrNull() ?: 0 } val y = b.removePrefix("v").split('.', '-').map { it.toIntOrNull() ?: 0 } @@ -285,40 +301,79 @@ object AppUpdates { val c = URL(url).openConnection() as HttpURLConnection c.connectTimeout = 15_000 c.readTimeout = 20_000 - c.setRequestProperty("Accept", "application/vnd.github+json") + c.setRequestProperty("Accept", "application/json") c.setRequestProperty("User-Agent", "DroidDeck-app") + c.setRequestProperty("Cache-Control", "no-cache") try { val code = c.responseCode - if (code == 403 || code == 429) throw IOException("GitHub's rate limit was hit - try again later") - if (code != 200) throw IOException("GitHub answered HTTP $code") + if (code != 200) throw IOException("The update catalog answered HTTP $code") return c.inputStream.bufferedReader().use { it.readText() } } finally { c.disconnect() } } - private fun writeRelease(r: Release): JSONObject = JSONObject() - .put("tag", r.tag).put("title", r.title).put("summary", r.summary).put("commit", r.commit).put("pr", r.pr) - .put("version", r.version).put("publishedAt", r.publishedAt).put("url", r.url) - .put("apk", r.apk?.let { JSONObject().put("name", it.name).put("url", it.url).put("size", it.size).put("sha256", it.sha256) }) + private fun writeApk(a: Apk): JSONObject = JSONObject() + .put("name", a.name) + .put("url", a.url) + .put("size", a.size) + .put("sha256", a.sha256) + .put("packageName", a.packageName) + .put("versionCode", a.versionCode) + .put("signerSha256", a.signerSha256) - private fun readRelease(o: JSONObject): Release = Release( - o.getString("tag"), o.optString("title"), o.optString("summary"), o.getString("commit"), o.optInt("pr"), - o.optString("version").ifEmpty { null }, o.optLong("publishedAt"), o.optString("url"), - o.optJSONObject("apk")?.let { Apk(it.getString("name"), it.getString("url"), it.optLong("size"), it.getString("sha256")) }, + private fun writeRelease(r: Release): JSONObject = JSONObject() + .put("tag", r.tag) + .put("title", r.title) + .put("summary", r.summary) + .put("commit", r.commit) + .put("pr", r.pr) + .put("version", r.version) + .put("versionCode", r.versionCode) + .put("publishedAt", r.publishedAt) + .put("url", r.url) + .put("apk", r.apk?.let(::writeApk)) + + private fun readCachedRelease(o: JSONObject): Release = Release( + o.getString("tag"), + o.optString("title"), + o.optString("summary"), + o.getString("commit"), + o.optInt("pr"), + if (o.isNull("version")) null else o.optString("version").ifEmpty { null }, + o.getInt("versionCode"), + o.optLong("publishedAt"), + o.optString("url"), + o.optJSONObject("apk")?.let { + Apk( + it.getString("name"), + it.getString("url"), + it.optLong("size"), + it.getString("sha256"), + it.getString("packageName"), + it.getInt("versionCode"), + it.getString("signerSha256"), + ) + }, ) private fun writeCatalog(c: Catalog): JSONObject = JSONObject() - .put("stable", c.stable?.let(::writeRelease)).put("nightly", c.nightly?.let(::writeRelease)) - .put("tests", JSONArray().apply { c.tests.forEach { put(writeRelease(it)) } }).put("checkedAt", c.checkedAt) + .put("stable", c.stable?.let(::writeRelease)) + .put("preview", c.preview?.let(::writeRelease)) + .put("tests", JSONArray().apply { c.tests.forEach { put(writeRelease(it)) } }) + .put("checkedAt", c.checkedAt) - private fun readCatalog(o: JSONObject): Catalog { + private fun readCachedCatalog(o: JSONObject): Catalog { val tests = o.optJSONArray("tests") ?: JSONArray() + val preview = o.optJSONObject("preview") ?: o.optJSONObject("nightly") return Catalog( - o.optJSONObject("stable")?.let(::readRelease), o.optJSONObject("nightly")?.let(::readRelease), - (0 until tests.length()).map { readRelease(tests.getJSONObject(it)) }, o.optLong("checkedAt"), + o.optJSONObject("stable")?.let(::readCachedRelease), + preview?.let(::readCachedRelease), + (0 until tests.length()).map { readCachedRelease(tests.getJSONObject(it)) }, + o.optLong("checkedAt"), ) } - private fun prefs(context: Context) = context.getSharedPreferences(PREFS, Context.MODE_PRIVATE) + private fun prefs(context: Context) = + context.getSharedPreferences(PREFS, Context.MODE_PRIVATE) } diff --git a/app/src/main/java/com/droiddeck/launcher/update/SelfInstaller.kt b/app/src/main/java/com/droiddeck/launcher/update/SelfInstaller.kt index 3a6e36fe..cf69b45a 100644 --- a/app/src/main/java/com/droiddeck/launcher/update/SelfInstaller.kt +++ b/app/src/main/java/com/droiddeck/launcher/update/SelfInstaller.kt @@ -4,9 +4,11 @@ import android.app.PendingIntent import android.content.Context import android.content.Intent import android.content.pm.PackageInstaller +import android.content.pm.PackageManager import android.net.Uri import android.os.Build import android.provider.Settings +import com.droiddeck.launcher.BuildConfig import com.droiddeck.launcher.core.FileUtils import com.droiddeck.launcher.core.Hashes import java.io.File @@ -14,6 +16,7 @@ import java.io.FileOutputStream import java.io.IOException import java.net.HttpURLConnection import java.net.URL +import java.security.MessageDigest /** * Installs one of DroidDeck's own builds over itself: download into the cache, check the file @@ -60,9 +63,13 @@ object SelfInstaller { } } } + if (apk.size > 0 && target.length() != apk.size) { + throw IOException("The download size didn't match the published APK and was discarded - try again") + } if (!apk.sha256.equals(Hashes.sha256(target), ignoreCase = true)) { throw IOException("The download didn't match its checksum and was discarded - try again") } + validateDownloaded(context, target, apk) return target } catch (e: Exception) { FileUtils.delete(target) @@ -72,6 +79,48 @@ object SelfInstaller { } } + /** Inspect the actual downloaded APK before giving it to Android's installer. */ + private fun validateDownloaded(context: Context, file: File, apk: AppUpdates.Apk) { + if (apk.packageName != context.packageName) { + throw IOException("The update is for Android package ${apk.packageName}, not ${context.packageName}") + } + if (!apk.signerSha256.equals(BuildConfig.RELEASE_SIGNER, ignoreCase = true)) { + throw IOException("The update metadata doesn't name DroidDeck's release signing key") + } + @Suppress("DEPRECATION") + val flags = if (Build.VERSION.SDK_INT >= 28) { + PackageManager.GET_SIGNING_CERTIFICATES + } else { + PackageManager.GET_SIGNATURES + } + val info = context.packageManager.getPackageArchiveInfo(file.absolutePath, flags) + ?: throw IOException("Android couldn't read the downloaded APK") + if (info.packageName != context.packageName) { + throw IOException("The downloaded APK is for ${info.packageName}, not ${context.packageName}") + } + @Suppress("DEPRECATION") + val code = if (Build.VERSION.SDK_INT >= 28) info.longVersionCode else info.versionCode.toLong() + if (code != apk.versionCode.toLong()) { + throw IOException("The downloaded APK says versionCode $code, not the published ${apk.versionCode}") + } + if (code < BuildConfig.VERSION_CODE.toLong()) { + throw IOException("Android won't install versionCode $code over the installed ${BuildConfig.VERSION_CODE}") + } + + // Before Android 9 this lineage intentionally presents the public legacy test key. Android + // 9+ understands the v3 hand-over and must see DroidDeck's private release key as current. + if (Build.VERSION.SDK_INT >= 28) { + val signing = info.signingInfo ?: throw IOException("The downloaded APK has no signing information") + val digests = signing.apkContentsSigners.orEmpty().map { cert -> + MessageDigest.getInstance("SHA-256").digest(cert.toByteArray()) + .joinToString("") { "%02x".format(it) } + } + if (digests.none { it.equals(BuildConfig.RELEASE_SIGNER, ignoreCase = true) }) { + throw IOException("The downloaded APK isn't signed with DroidDeck's release key") + } + } + } + /** Writes [apk] into an install session and commits it; the answer comes as an [ACTION_STATUS] broadcast. */ fun install(context: Context, apk: File) { val installer = context.packageManager.packageInstaller diff --git a/app/src/test/java/com/droiddeck/launcher/update/AppUpdatesTest.kt b/app/src/test/java/com/droiddeck/launcher/update/AppUpdatesTest.kt index 6e57f3d6..c3c315aa 100644 --- a/app/src/test/java/com/droiddeck/launcher/update/AppUpdatesTest.kt +++ b/app/src/test/java/com/droiddeck/launcher/update/AppUpdatesTest.kt @@ -1,26 +1,69 @@ package com.droiddeck.launcher.update +import com.droiddeck.launcher.BuildConfig import com.droiddeck.launcher.update.AppUpdates.Channel import com.droiddeck.launcher.update.AppUpdates.Follow import com.droiddeck.launcher.update.AppUpdates.Installed import com.droiddeck.launcher.update.AppUpdates.Offer import org.junit.Assert.assertEquals +import org.junit.Assert.assertNotNull +import org.junit.Assert.assertNull +import org.json.JSONObject import org.junit.Assert.assertTrue +import org.junit.Assert.fail import org.junit.Test +import org.junit.runner.RunWith +import org.robolectric.RobolectricTestRunner +import java.io.IOException +@RunWith(RobolectricTestRunner::class) class AppUpdatesTest { - private fun release(commit: String, pr: Int = 0, version: String? = null) = - AppUpdates.Release("t", "title", "", commit, pr, version, 0L, "", null) + private fun apk(versionCode: Int = 9) = AppUpdates.Apk( + "DroidDeck.apk", + "https://github.com/Droid-Deck/DroidDeck-CI/releases/download/t/DroidDeck.apk", + 10, + "a".repeat(64), + "com.droiddeck.launcher", + versionCode, + BuildConfig.RELEASE_SIGNER, + ) - private val stable = release("aaaaaaa1111111111111111111111111111111", version = "0.2.0") + private fun release( + commit: String, + pr: Int = 0, + version: String? = null, + versionCode: Int = 9, + withApk: Boolean = true, + ) = AppUpdates.Release( + "t", + "title", + "", + commit, + pr, + version, + versionCode, + 0L, + "", + if (withApk) apk(versionCode) else null, + ) + + private val stable = release( + "aaaaaaa1111111111111111111111111111111", + version = "0.2.0", + ) private val catalog = AppUpdates.Catalog( stable = stable, - nightly = release("bbbbbbb"), + preview = release("bbbbbbb"), tests = listOf(release("ccccccc2222222222222222222222222222222", pr = 93)), checkedAt = 0L, ) - private fun me(commit: String, pr: Int = 0, version: String = "0.2.0") = Installed(commit, pr, version, updatable = true) + private fun me( + commit: String, + pr: Int = 0, + version: String = "0.2.0", + versionCode: Int = 9, + ) = Installed(commit, pr, version, versionCode, updatable = true) @Test fun theStableBuildIsCurrentOnStable() { @@ -28,17 +71,20 @@ class AppUpdatesTest { } @Test - fun aNightlyPastStableIsAheadNotBehind() { + fun aPreviewPastStableIsAheadNotBehind() { assertEquals(Offer.AHEAD, AppUpdates.offer(catalog, Follow(Channel.STABLE), me("ddddddd999"))) } @Test fun aNewerStableVersionIsAnUpdate() { - assertEquals(Offer.UPDATE, AppUpdates.offer(catalog, Follow(Channel.STABLE), me("eeeeeee999", version = "0.1.9"))) + assertEquals( + Offer.UPDATE, + AppUpdates.offer(catalog, Follow(Channel.STABLE), me("eeeeeee999", version = "0.1.9")), + ) } @Test - fun nightlyMatchesItsShortShaAndOffersTheNewest() { + fun previewMatchesItsShortShaAndOffersTheNewest() { assertEquals(Offer.CURRENT, AppUpdates.offer(catalog, Follow(Channel.NIGHTLY), me("bbbbbbb0123456789"))) assertEquals(Offer.UPDATE, AppUpdates.offer(catalog, Follow(Channel.NIGHTLY), me(stable.commit))) } @@ -52,7 +98,7 @@ class AppUpdatesTest { } @Test - fun aTestBuildIsNeverTheMainBuildOfTheSameCommit() { + fun aTestBuildIsNeverThePreviewBuildOfTheSameCommit() { assertEquals(Offer.SWITCH, AppUpdates.offer(catalog, Follow(Channel.NIGHTLY), me("bbbbbbb", pr = 5))) } @@ -64,20 +110,77 @@ class AppUpdatesTest { } @Test - fun ciNotesGiveTheChangeAndItsSummary() { - val (title, summary) = AppUpdates.describeCi( - "main 8a61a18", - "**Merged to main** - [PR #95](https://github.com/Droid-Deck/DroidDeck/pull/95): fix: keep `Steam` alive (merged 2026-09-30)\n\n" + - "Steam no longer **exits** on resume.\n\nSigned build of https://github.com/x - in main.", + fun aVersionCodeDowngradeIsBlockedBeforeDownload() { + val older = release("ddddddd", versionCode = 8) + assertNotNull(AppUpdates.installBlock(older, me("eeeeeee", versionCode = 9))) + assertNull(AppUpdates.installBlock(stable, me("eeeeeee", versionCode = 9))) + } + + @Test + fun aMissingVariantIsBlockedBeforeDownload() { + val missing = release("ddddddd", withApk = false) + assertNotNull(AppUpdates.installBlock(missing, me("eeeeeee"))) + } + + private fun publishedCatalog(signer: String = BuildConfig.RELEASE_SIGNER): JSONObject { + val raw = """ + { + "schema": 1, + "sourceRepo": "Droid-Deck/DroidDeck", + "ciRepo": "Droid-Deck/DroidDeck-CI", + "stable": null, + "preview": { + "tag": "main-deadbee", + "title": "Preview", + "summary": "", + "commit": "deadbee1234567890", + "pr": 0, + "version": null, + "versionCode": 9, + "publishedAt": 1, + "url": "https://github.com/Droid-Deck/DroidDeck-CI/releases/tag/main-deadbee", + "apks": { + "com.droiddeck.launcher": { + "name": "DroidDeck-main-deadbee.apk", + "url": "https://github.com/Droid-Deck/DroidDeck-CI/releases/download/main-deadbee/DroidDeck-main-deadbee.apk", + "size": 10, + "sha256": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "packageName": "com.droiddeck.launcher", + "versionCode": 9, + "signerSha256": "__SIGNER__" + } + } + }, + "tests": [] + } + """.trimIndent().replace("__SIGNER__", signer) + return JSONObject(raw) + } + + @Test + fun publishedCatalogSelectsThisPackagesApk() { + val parsed = AppUpdates.readPublishedCatalog( + publishedCatalog(), + "com.droiddeck.launcher", + 123L, ) - assertEquals("fix: keep Steam alive", title) - assertEquals("Steam no longer exits on resume.", summary) - val (prTitle, prSummary) = AppUpdates.describeCi( - "PR #93", - "**Not merged** - a test build of open [PR #93](https://github.com/Droid-Deck/DroidDeck/pull/93): fix: report Adreno 840v2\n\n" + - "Source: https://github.com/Droid-Deck/DroidDeck/pull/93\nCommit: 93382ff", - ) - assertEquals("fix: report Adreno 840v2", prTitle) - assertEquals("", prSummary) + assertEquals("main-deadbee", parsed.preview!!.tag) + assertEquals("com.droiddeck.launcher", parsed.preview!!.apk!!.packageName) + assertEquals(9, parsed.preview!!.apk!!.versionCode) + assertNull(parsed.preview!!.version) + assertEquals(123L, parsed.checkedAt) + } + + @Test + fun publishedCatalogRejectsAnotherSigner() { + try { + AppUpdates.readPublishedCatalog( + publishedCatalog("0".repeat(64)), + "com.droiddeck.launcher", + 123L, + ) + fail("wrong signer should be rejected") + } catch (_: IOException) { + } } } diff --git a/tools/release/update_catalog.py b/tools/release/update_catalog.py new file mode 100755 index 00000000..f4d543ad --- /dev/null +++ b/tools/release/update_catalog.py @@ -0,0 +1,405 @@ +#!/usr/bin/env python3 +"""Build and publish the one-file update catalog consumed by DroidDeck. + +The catalog is derived from GitHub releases, but the Android app never has to infer +channels, parse release notes, dereference tags, or spend unauthenticated API quota. +""" + +from __future__ import annotations + +import argparse +import base64 +import datetime as dt +import json +import os +import re +import time +import urllib.error +import urllib.parse +import urllib.request + +API = "https://api.github.com" +SHA256 = re.compile(r"^[0-9a-f]{64}$") +PR_TAG = re.compile(r"^pr-([1-9][0-9]*)$") +MAIN_TAG = re.compile(r"^main-([0-9a-f]{7,40})$") + + +class ApiError(RuntimeError): + def __init__(self, status: int, message: str): + super().__init__(f"GitHub API {status}: {message}") + self.status = status + + +class GitHub: + def __init__(self, token: str): + self.token = token + + def request(self, method: str, path: str, body=None): + data = None + headers = { + "Accept": "application/vnd.github+json", + "User-Agent": "DroidDeck-update-catalog", + "X-GitHub-Api-Version": "2022-11-28", + } + if self.token: + headers["Authorization"] = f"Bearer {self.token}" + if body is not None: + data = json.dumps(body).encode() + headers["Content-Type"] = "application/json" + req = urllib.request.Request(API + path, data=data, headers=headers, method=method) + try: + with urllib.request.urlopen(req, timeout=30) as response: + raw = response.read() + return json.loads(raw) if raw else None + except urllib.error.HTTPError as e: + raw = e.read().decode(errors="replace") + raise ApiError(e.code, raw[:500]) from e + + def get(self, path: str): + return self.request("GET", path) + + def put(self, path: str, body): + return self.request("PUT", path, body) + + +def q(value: str) -> str: + return urllib.parse.quote(value, safe="") + + +def repo_file(gh: GitHub, repo: str, path: str, ref: str) -> str: + obj = gh.get(f"/repos/{repo}/contents/{q(path)}?ref={q(ref)}") + if obj.get("encoding") != "base64": + raise RuntimeError(f"{repo}:{ref}:{path} was not base64 content") + return base64.b64decode(obj["content"]).decode() + + +def resolve_tag(gh: GitHub, repo: str, tag: str) -> str: + obj = gh.get(f"/repos/{repo}/git/ref/tags/{q(tag)}")["object"] + while obj["type"] == "tag": + obj = gh.get(f"/repos/{repo}/git/tags/{q(obj['sha'])}")["object"] + if obj["type"] != "commit": + raise RuntimeError(f"{repo} tag {tag} points to {obj['type']}, not a commit") + return obj["sha"] + + +def parse_gradle(text: str) -> tuple[int, str | None]: + code = re.search(r"(?m)^\s*versionCode\s+(\d+)\s*$", text) + name = re.search(r'(?m)^\s*versionName\s+"([^"]+)"\s*$', text) + if not code: + raise RuntimeError("app/build.gradle has no numeric versionCode") + return int(code.group(1)), name.group(1) if name else None + + +def parse_variants(text: str) -> list[tuple[str, str]]: + out = [] + for raw in text.splitlines(): + line = raw.strip() + if not line or line.startswith("#"): + continue + fields = line.split() + if len(fields) < 3: + raise RuntimeError(f"bad variants.txt row: {raw!r}") + out.append((fields[1], "" if fields[2] == "-" else fields[2])) + if not out: + raise RuntimeError("variants.txt has no packages") + return out + + +def published_millis(release: dict) -> int: + value = release.get("published_at") or release.get("created_at") + if not value: + return 0 + return int(dt.datetime.fromisoformat(value.replace("Z", "+00:00")).timestamp() * 1000) + + +def paragraphs(markdown: str) -> list[str]: + return [p.strip() for p in re.split(r"\n\s*\n", markdown.replace("\r", "")) if p.strip()] + + +def plain(markdown: str) -> str: + text = re.sub(r"!\[[^]]*]\([^)]*\)", "", markdown) + text = re.sub(r"\[([^]]*)]\([^)]*\)", r"\1", text) + text = re.sub(r"[*_`]{1,3}", "", text) + return re.sub(r"\s+", " ", text).strip() + + +def prose_lines(markdown: str) -> str: + lines = [] + for raw in markdown.splitlines(): + match = re.match(r"^\s*[-*]\s+", raw) + value = ("• " + plain(raw[match.end():])) if match else plain(raw) + if value: + lines.append(value) + return "\n".join(lines) + + +def ci_description(release: dict) -> tuple[str, str]: + ps = paragraphs(release.get("body") or "") + head = ps[0] if ps else "" + title = "" + if "): " in head: + title = head.split("): ", 1)[1] + title = re.sub(r" \(merged[^)]*\)$", "", title) + title = plain(title or head or release.get("name") or release["tag_name"]) + summary = ps[1] if len(ps) > 1 else "" + if summary.startswith("Signed build") or summary.startswith("Source:"): + summary = "" + return title, prose_lines(summary) + + +def stable_description(release: dict) -> tuple[str, str]: + name = release.get("name") or release["tag_name"] + summary = "" + for p in paragraphs(release.get("body") or ""): + if not (p.startswith("#") or p.startswith("![") or p.startswith("<")): + summary = prose_lines(p) + break + return name, summary + + +def source_meta(source: GitHub, repo: str, ref: str): + gradle = repo_file(source, repo, "app/build.gradle", ref) + code, version = parse_gradle(gradle) + variants = parse_variants(repo_file(source, repo, "tools/release/variants.txt", ref)) + signer = repo_file(source, repo, "keystore/release-signer.sha256", ref).strip().lower() + if not SHA256.fullmatch(signer): + raise RuntimeError(f"{repo}:{ref} has an invalid release signer digest") + return code, version, variants, signer + + +def asset_for_suffix(assets: list[dict], suffix: str, all_suffixes: list[str]): + others = [s for s in all_suffixes if s and s != suffix] + for asset in assets: + name = asset.get("name", "") + if not name.endswith(f"{suffix}.apk"): + continue + if any(name.endswith(f"{other}.apk") for other in others): + continue + digest = (asset.get("digest") or "").lower() + if not digest.startswith("sha256:") or not SHA256.fullmatch(digest[7:]): + continue + return { + "name": name, + "url": asset["browser_download_url"], + "size": int(asset.get("size") or 0), + "sha256": digest[7:], + } + return None + + +def release_entry( + release: dict, + commit: str, + pr: int, + version: str | None, + version_code: int, + variants: list[tuple[str, str]], + signer: str, + title: str, + summary: str, +): + suffixes = [suffix for _, suffix in variants] + apks = {} + for package, suffix in variants: + asset = asset_for_suffix(release.get("assets") or [], suffix, suffixes) + if not asset: + continue + asset.update( + { + "packageName": package, + "versionCode": version_code, + "signerSha256": signer, + } + ) + apks[package] = asset + return { + "tag": release["tag_name"], + "title": title, + "summary": summary, + "commit": commit, + "pr": pr, + "version": version, + "versionCode": version_code, + "publishedAt": published_millis(release), + "url": release.get("html_url") or "", + "apks": apks, + } + + +def release_body_value(body: str, key: str): + match = re.search(rf"(?mi)^\s*{re.escape(key)}:\s*`?([^\s`]+)", body or "") + return match.group(1) if match else None + + +def main_commit(release: dict) -> str | None: + body = release.get("body") or "" + explicit = release_body_value(body, "Commit") + if explicit and re.fullmatch(r"[0-9a-f]{7,40}", explicit): + return explicit + match = re.search(r"/commit/([0-9a-f]{40})", body) + return match.group(1) if match else None + + +def build_catalog(source: GitHub, ci: GitHub, source_repo: str, ci_repo: str) -> dict: + try: + stable_release = source.get(f"/repos/{source_repo}/releases/latest") + except ApiError as e: + if e.status != 404: + raise + stable_release = None + + stable = None + if stable_release: + stable_commit = resolve_tag(source, source_repo, stable_release["tag_name"]) + code, version_name, variants, signer = source_meta(source, source_repo, stable_commit) + title, summary = stable_description(stable_release) + stable = release_entry( + stable_release, + stable_commit, + 0, + version_name or stable_release["tag_name"].removeprefix("v"), + code, + variants, + signer, + title, + summary, + ) + + releases = ci.get(f"/repos/{ci_repo}/releases?per_page=100") + previews = [] + tests = [] + for release in releases: + if release.get("draft"): + continue + tag = release.get("tag_name") or "" + main_match = MAIN_TAG.fullmatch(tag) + if main_match: + commit = main_commit(release) + if not commit: + continue + try: + code, _, variants, signer = source_meta(source, source_repo, commit) + except ApiError: + continue + title, summary = ci_description(release) + previews.append( + release_entry(release, commit, 0, None, code, variants, signer, title, summary) + ) + continue + + pr_match = PR_TAG.fullmatch(tag) + if not pr_match: + continue + pr = int(pr_match.group(1)) + body = release.get("body") or "" + if f"github.com/{source_repo}/pull/{pr}" not in body: + continue + try: + pull = source.get(f"/repos/{source_repo}/pulls/{pr}") + except ApiError: + continue + if pull.get("state") != "open" or pull.get("draft") or pull.get("base", {}).get("ref") != "main": + continue + commit = release_body_value(body, "Commit") + if not commit or not re.fullmatch(r"[0-9a-f]{7,40}", commit): + continue + base_ref = release_body_value(body, "Base") or pull["base"]["sha"] + try: + base_code, _, variants, signer = source_meta(source, source_repo, base_ref) + except ApiError: + continue + version_code_text = release_body_value(body, "VersionCode") + if version_code_text and version_code_text.isdigit(): + version_code = int(version_code_text) + else: + # Legacy test releases did not record it. If the PR did not change build.gradle, + # the base value is exact; otherwise use the PR head's value. + version_code = base_code + try: + changed = source.get(f"/repos/{source_repo}/pulls/{pr}/files?per_page=100") + if any(item.get("filename") == "app/build.gradle" for item in changed): + head_repo = pull["head"]["repo"]["full_name"] + version_code = parse_gradle(repo_file(source, head_repo, "app/build.gradle", pull["head"]["sha"]))[0] + except (ApiError, KeyError): + pass + title, summary = ci_description(release) + tests.append( + release_entry(release, commit, pr, None, version_code, variants, signer, title, summary) + ) + + previews.sort(key=lambda r: r["publishedAt"], reverse=True) + tests.sort(key=lambda r: r["publishedAt"], reverse=True) + return { + "schema": 1, + "sourceRepo": source_repo, + "ciRepo": ci_repo, + "generatedAt": int(time.time() * 1000), + "stable": stable, + "preview": previews[0] if previews else None, + "tests": tests, + } + + +def current_file(ci: GitHub, repo: str, path: str): + try: + return ci.get(f"/repos/{repo}/contents/{q(path)}?ref=main") + except ApiError as e: + if e.status == 404: + return None + raise + + +def publish_catalog(source: GitHub, ci: GitHub, source_repo: str, ci_repo: str, path: str): + for attempt in range(4): + catalog = build_catalog(source, ci, source_repo, ci_repo) + content = (json.dumps(catalog, indent=2, sort_keys=False) + "\n").encode() + old = current_file(ci, ci_repo, path) + body = { + "message": "ci: refresh DroidDeck update catalog", + "content": base64.b64encode(content).decode(), + "branch": "main", + } + if old: + body["sha"] = old["sha"] + try: + old_bytes = base64.b64decode(old["content"]) + old_json = json.loads(old_bytes) + old_json.pop("generatedAt", None) + new_json = dict(catalog) + new_json.pop("generatedAt", None) + if old_json == new_json: + print("catalog contents are already current") + return catalog + except Exception: + pass + try: + ci.put(f"/repos/{ci_repo}/contents/{q(path)}", body) + print(f"published https://raw.githubusercontent.com/{ci_repo}/main/{path}") + return catalog + except ApiError as e: + if e.status not in (409, 422) or attempt == 3: + raise + time.sleep(1 + attempt) + raise RuntimeError("catalog publish retry loop exhausted") + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--source-repo", default="Droid-Deck/DroidDeck") + parser.add_argument("--ci-repo", default="Droid-Deck/DroidDeck-CI") + parser.add_argument("--path", default="catalog.json") + parser.add_argument("--dry-run", action="store_true") + args = parser.parse_args() + + source = GitHub(os.environ.get("SOURCE_GITHUB_TOKEN", "")) + ci = GitHub(os.environ.get("CI_GITHUB_TOKEN", os.environ.get("SOURCE_GITHUB_TOKEN", ""))) + if args.dry_run: + print(json.dumps(build_catalog(source, ci, args.source_repo, args.ci_repo), indent=2)) + else: + if not os.environ.get("CI_GITHUB_TOKEN"): + raise SystemExit("CI_GITHUB_TOKEN is required to publish") + publish_catalog(source, ci, args.source_repo, args.ci_repo, args.path) + + +if __name__ == "__main__": + main() diff --git a/tools/tests/test_update_catalog.py b/tools/tests/test_update_catalog.py new file mode 100644 index 00000000..2e316a26 --- /dev/null +++ b/tools/tests/test_update_catalog.py @@ -0,0 +1,80 @@ +import unittest + +from tools.release.update_catalog import ( + asset_for_suffix, + ci_description, + parse_gradle, + parse_variants, + release_body_value, +) + + +class UpdateCatalogTest(unittest.TestCase): + def test_gradle_metadata(self): + code, name = parse_gradle("""android { + defaultConfig { + versionCode 12 + versionName "0.3.1" + } +} +""") + self.assertEqual(12, code) + self.assertEqual("0.3.1", name) + + def test_variants(self): + rows = parse_variants(""" +# name package suffix +standard com.droiddeck.launcher - +pubg com.tencent.ig -pubg +""") + self.assertEqual( + [("com.droiddeck.launcher", ""), ("com.tencent.ig", "-pubg")], + rows, + ) + + def test_standard_asset_does_not_take_a_variant(self): + assets = [ + { + "name": "DroidDeck-main-deadbee-pubg.apk", + "size": 10, + "digest": "sha256:" + "a" * 64, + "browser_download_url": "https://example/pubg", + }, + { + "name": "DroidDeck-main-deadbee.apk", + "size": 11, + "digest": "sha256:" + "b" * 64, + "browser_download_url": "https://example/standard", + }, + ] + standard = asset_for_suffix(assets, "", ["", "-pubg"]) + pubg = asset_for_suffix(assets, "-pubg", ["", "-pubg"]) + self.assertEqual("DroidDeck-main-deadbee.apk", standard["name"]) + self.assertEqual("DroidDeck-main-deadbee-pubg.apk", pubg["name"]) + + def test_ci_description(self): + title, summary = ci_description( + { + "tag_name": "main-deadbee", + "name": "main deadbee", + "body": ( + "**Merged to main** - [PR #9](https://github.com/Droid-Deck/DroidDeck/pull/9): " + "fix: keep Steam alive (merged 2026-09-30)\n\n" + "Steam no longer **exits** on resume.\n\n" + "Signed build of https://example" + ), + } + ) + self.assertEqual("fix: keep Steam alive", title) + self.assertEqual("Steam no longer exits on resume.", summary) + + def test_release_body_metadata(self): + tick = chr(96) + body = f"Commit: {tick}abcdef1234567{tick}\nBase: 1234567\nVersionCode: 42\n" + self.assertEqual("abcdef1234567", release_body_value(body, "Commit")) + self.assertEqual("1234567", release_body_value(body, "Base")) + self.assertEqual("42", release_body_value(body, "VersionCode")) + + +if __name__ == "__main__": + unittest.main()