From 72f705f4cd04c9399ef3eec82ced61b299d92f98 Mon Sep 17 00:00:00 2001 From: TimMasalme Date: Sun, 26 Jul 2026 19:57:54 +0200 Subject: [PATCH 1/2] Fix internal map paths in coop mission .scmap files on deploy getFileContent() only rewrote /maps// in text files, so the paths embedded in the .scmap binary kept pointing at the unversioned folder and the affected missions lost their custom textures after a release. ScmapPathFixer reads and rewrites the .scmap byte by byte, ported from speed2's sc_map_parser.gd, and inserts the release version into the map folder segment. Only paths that point at the mission's own folder are touched. Several missions deliberately reference a base game map (/maps/X1CA_001/X1CA_001.scmap) or a texture of another map, and versioning those would break them. A map file is only parsed at all when a raw byte scan finds "/maps/", which also keeps the placeholder .scmap files of the missions using a base game map out of the parser. Affects 4 of the 42 deployed missions: Operation_Blockade, Tha_Atha_Aez, Golden_Crystals and Overlord_Surth_Velsok. Their checksums change, so they get one version bump on the first run after this lands. Co-Authored-By: Claude Opus 5 --- .../scripts/CoopMapDeployer.kt | 15 + .../scripts/ScmapPathFixer.kt | 325 ++++++++++++++++++ 2 files changed, 340 insertions(+) create mode 100644 apps/faf-legacy-deployment/scripts/ScmapPathFixer.kt diff --git a/apps/faf-legacy-deployment/scripts/CoopMapDeployer.kt b/apps/faf-legacy-deployment/scripts/CoopMapDeployer.kt index 748c7b59..d36dfc7c 100644 --- a/apps/faf-legacy-deployment/scripts/CoopMapDeployer.kt +++ b/apps/faf-legacy-deployment/scripts/CoopMapDeployer.kt @@ -7,7 +7,9 @@ import com.faforever.FafDatabase import com.faforever.GitRepo import com.faforever.Log import com.faforever.extractChecksumsFromZip +import com.faforever.fixScmapPaths import com.faforever.generateChecksums +import com.faforever.referencesOwnMapFolder import org.apache.commons.compress.archivers.zip.ZipArchiveEntry import org.apache.commons.compress.archivers.zip.ZipArchiveOutputStream import org.slf4j.LoggerFactory @@ -180,6 +182,17 @@ private fun generateChecksumsForMap( * Get file content with path rewriting for text files. */ private fun getFileContent(file: Path, map: CoopMap, version: Int): ByteArray { + if (file.isScmapFile()) { + val bytes = file.readBytes() + // Only missions whose map references assets in their own folder need the version + // inserted. Everything else - including the placeholder .scmap files of the missions + // that use a base game map - is passed through and never parsed. + return if (bytes.referencesOwnMapFolder(map.folderName)) { + fixScmapPaths(bytes, map.folderName, version) + } else { + bytes + } + } return if (file.isTextFile()) { var text = file.readText() .replace( @@ -238,6 +251,8 @@ private fun createZip( private fun Path.isTextFile() = listOf(".md", ".lua", ".json", ".txt").any { toString().endsWith(it) } +private fun Path.isScmapFile() = toString().lowercase().endsWith(".scmap") + fun main(args: Array) { Log.init() diff --git a/apps/faf-legacy-deployment/scripts/ScmapPathFixer.kt b/apps/faf-legacy-deployment/scripts/ScmapPathFixer.kt new file mode 100644 index 00000000..c0ee3a8b --- /dev/null +++ b/apps/faf-legacy-deployment/scripts/ScmapPathFixer.kt @@ -0,0 +1,325 @@ +@file:Suppress("PackageDirectoryMismatch") + +package com.faforever + +import org.slf4j.LoggerFactory +import java.io.ByteArrayOutputStream + +private val log = LoggerFactory.getLogger("scmap-path-fixer") + +/** Every .scmap file starts with these 16 bytes. */ +private val SCMAP_HEADER = byteArrayOf( + 0x4D, 0x61, 0x70, 0x1A, // "Map\x1A" + 0x02, 0x00, 0x00, 0x00, // 2 + 0xED.toByte(), 0xFE.toByte(), 0xEF.toByte(), 0xBE.toByte(), // 0xBEEFFEED, little endian + 0x02, 0x00, 0x00, 0x00, // 2 +) + +fun ByteArray.isScmap(): Boolean = + size >= SCMAP_HEADER.size && SCMAP_HEADER.indices.all { this[it] == SCMAP_HEADER[it] } + +/** + * Case insensitive raw byte search for "/maps/", so we only parse map files + * that actually reference their own folder. Missions whose .scmap contains no such path + * (and the placeholder files that are not maps at all) are passed through untouched. + */ +fun ByteArray.referencesOwnMapFolder(folderName: String): Boolean { + val needle = "/maps/${folderName.lowercase()}".toByteArray(Charsets.ISO_8859_1) + if (needle.size > size) return false + outer@ for (i in 0..size - needle.size) { + for (j in needle.indices) { + var b = this[i + j].toInt() and 0xFF + if (b in 0x41..0x5A) b += 0x20 // ASCII toLowerCase + if (b != (needle[j].toInt() and 0xFF)) continue@outer + } + return true + } + return false +} + +/** + * Rewrites the map folder segment of every path embedded in a .scmap so that it carries + * the release version: + * + * /maps/faf_coop_operation_blockade/env/layers/sand.dds + * -> /maps/faf_coop_operation_blockade.v0004/env/layers/sand.dds + * + * Only paths that point at [folderName] itself are touched. Paths pointing somewhere else + * are left alone and logged - several missions deliberately reference a base game map + * (`/maps/X1CA_001/X1CA_001.scmap`) or another map's texture, and versioning those would + * break them. + * + * A .scmap is tightly packed with no field names, so every field has to be read and + * written back in the exact same order. Most strings are null terminated; decal texture + * paths are the exception and carry a leading int with their length. Arrays of structs + * (decals, waves, props) start with an int count. Which sections exist depends on the map + * file version - the coop missions use 53, 56 and 60. + * + * Ported from speed2's `sc_map_parser.gd` (`fix_paths`). + * + * @param version release version, or a negative value to copy the file without changes + * (used as a self check: the parser then has to reproduce the input byte + * for byte) + * @throws IllegalArgumentException if the file is not a .scmap + * @throws IllegalStateException if the result fails verification + */ +fun fixScmapPaths(bytes: ByteArray, folderName: String, version: Int): ByteArray { + require(bytes.isScmap()) { "$folderName: not a .scmap file, header mismatch" } + + val suffix = if (version >= 0) ".v%04d".format(version) else "" + val rewriter = ScmapRewriter(bytes, folderName, suffix) + val result = rewriter.rewrite() + + if (suffix.isEmpty()) return result + + rewriter.skipped.distinct().forEach { + log.warn("$folderName: path leads outside the mission folder, left unchanged: $it") + } + log.info( + "$folderName: rewrote {} path(s) in {}, {} byte(s) added", + rewriter.rewritten, folderName, result.size - bytes.size + ) + + check(result.size - bytes.size == rewriter.addedBytes) { + "$folderName: byte delta ${result.size - bytes.size} does not match the " + + "${rewriter.addedBytes} byte(s) added to paths" + } + // the rewritten file has to parse again and come out byte identical + val verified = ScmapRewriter(result, folderName, "").rewrite() + check(verified.contentEquals(result)) { + "$folderName: rewritten .scmap does not round trip, refusing to ship it" + } + return result +} + +private class ScmapRewriter( + private val src: ByteArray, + folderName: String, + private val suffix: String, +) { + private val folder = folderName.lowercase() + private val out = ByteArrayOutputStream(src.size + 1024) + private var pos = 0 + + var rewritten = 0 + private set + var addedBytes = 0 + private set + val skipped = mutableListOf() + + fun rewrite(): ByteArray { + copy(16) // header + copy(14) // float size x, y + 6 padding bytes + transferSizedChunk() // preview image + + val version = readInt() + val width = readInt() + val height = readInt() + writeInt(version) + writeInt(width) + writeInt(height) + copy(4) // height scale + + copy((width + 1) * (height + 1) * 2) // heightmap + if (version >= 56) copy(1) // padding after the heightmap + + transferString() // shader name, not a path + transferString(true) // editor background + transferString(true) // skycube + + if (version >= 56) { + val cubemaps = readInt() + writeInt(cubemaps) + repeat(cubemaps) { + transferString() // name + transferString(true) // path + } + } else { + transferString(true) // old format has a single cubemap + } + + copy(23 * 4) // lighting + copy(1 + 23 * 4) // water flag + water settings + transferString(true) // water cubemap + transferString(true) // water ramp + copy(4 * 4) // wave normal repeats + repeat(4) { // wave textures + copy(8) // movement + transferString(true) + } + + val waves = readInt() + writeInt(waves) + repeat(waves) { + transferString() // texture name, not a path + transferString() // ramp name, not a path + copy(17 * 4) + } + + if (version < 56) { + transferString() // always "No Tileset" + val tilesets = readInt() // always 6 + writeInt(tilesets) + repeat(tilesets) { + transferString(true) // albedo + transferString(true) // normal + copy(2 * 4) // scales + } + } else { + copy(6 * 4) // minimap + if (version > 56) copy(4) // unknown + repeat(19) { // 10 albedo + 9 normal + transferString(true) + copy(4) // scale + } + } + + copy(8) // 2 unknown values + + val decals = readInt() + writeInt(decals) + repeat(decals) { + copy(8) // id + type + val textures = readInt() + writeInt(textures) + repeat(textures) { transferSizedString() } + copy(12 * 4) // 11 floats + 1 int + } + + val decalGroups = readInt() + writeInt(decalGroups) + repeat(decalGroups) { + copy(4) // id + transferString() // name, not a path + val ids = readInt() + writeInt(ids) + if (ids > 0) copy(ids * 4) + } + + copy(8) // int width + height + + val normalMaps = readInt() + writeInt(normalMaps) + repeat(normalMaps) { transferSizedChunk() } + + if (version < 56) copy(4) // unknown in the old format + + transferSizedChunk() // texture mask low + if (version >= 56) transferSizedChunk() // texture mask high + + val waterMaps = readInt() + writeInt(waterMaps) + repeat(waterMaps) { transferSizedChunk() } + + val halfSize = (width / 2) * (height / 2) + repeat(3) { copy(halfSize) } // water foam, flatness, depth bias + copy(width * height) // terrain types + + if (version <= 52) copy(2) // unknown in the oldest format + + if (version >= 60) { + copy(16 * 4) // skybox settings + transferString(true) // albedo + transferString(true) // glow + val planets = readInt() + writeInt(planets) + repeat(planets) { copy(10 * 4) } + copy(3 + 4 * 4) // sky mid color + cirrus + transferString(true) // cirrus texture + val cirrusLayers = readInt() + writeInt(cirrusLayers) + repeat(cirrusLayers) { copy(5 * 4) } + copy(4) // one more setting + } + + val props = readInt() + writeInt(props) + repeat(props) { + transferString(true) // blueprint path + copy(15 * 4) + } + + if (pos < src.size) { + log.warn("$folder: {} trailing byte(s) after the props, copied unchanged", src.size - pos) + copy(src.size - pos) + } + return out.toByteArray() + } + + /** + * If [path] points into this mission's own folder inside /maps, the version is added to + * the folder name. Paths are lower cased, which is what the maps deployed so far look + * like. Everything else is returned untouched. + */ + private fun addVersion(path: String): String { + if (suffix.isEmpty()) return path + val lower = path.lowercase() + val parts = lower.split('/').filter { it.isNotEmpty() } + // expected: ["maps", "map_name", "env", ...] + if (parts.size < 3 || parts[0] != "maps") return lower + + val segment = parts[1].replace(VERSIONED, "") + if (segment != folder) { + skipped += path + return lower + } + + rewritten++ + addedBytes += suffix.length - (parts[1].length - segment.length) + return "/" + parts.mapIndexed { i, part -> if (i == 1) segment + suffix else part } + .joinToString("/") + } + + /** Null terminated string. [isPath] marks the ones that may need the version. */ + private fun transferString(isPath: Boolean = false) { + val start = pos + while (pos < src.size && src[pos] != 0.toByte()) pos++ + val raw = String(src, start, pos - start, Charsets.ISO_8859_1) + pos++ // null terminator + val value = if (isPath) addVersion(raw) else raw + out.write(value.toByteArray(Charsets.ISO_8859_1)) + out.write(0) + } + + /** Decal texture paths are not null terminated but prefixed with their length. */ + private fun transferSizedString() { + val length = readInt() + val raw = String(src, pos, length, Charsets.ISO_8859_1) + pos += length + val encoded = addVersion(raw).toByteArray(Charsets.ISO_8859_1) + writeInt(encoded.size) + out.write(encoded) + } + + /** Chunk of bytes whose length is read from the file, used for embedded textures. */ + private fun transferSizedChunk() { + val length = readInt() + writeInt(length) + if (length > 0) copy(length) + } + + private fun readInt(): Int { + val v = (src[pos].toInt() and 0xFF) or + ((src[pos + 1].toInt() and 0xFF) shl 8) or + ((src[pos + 2].toInt() and 0xFF) shl 16) or + ((src[pos + 3].toInt() and 0xFF) shl 24) + pos += 4 + return v + } + + private fun writeInt(value: Int) { + out.write(value and 0xFF) + out.write((value ushr 8) and 0xFF) + out.write((value ushr 16) and 0xFF) + out.write((value ushr 24) and 0xFF) + } + + private fun copy(length: Int) { + out.write(src, pos, length) + pos += length + } + + private companion object { + val VERSIONED = Regex("""\.v\d{4}$""") + } +} From bfca087c1e5157d446b8bf5dcf63676fe05a5e44 Mon Sep 17 00:00:00 2001 From: TimMasalme Date: Sun, 26 Jul 2026 19:58:12 +0200 Subject: [PATCH 2/2] TEMP: compile the legacy deployment scripts in CI Not part of the pull request, only used to verify the Kotlin builds. Co-Authored-By: Claude Opus 5 --- .../workflows/compile-legacy-deployment.yml | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 .github/workflows/compile-legacy-deployment.yml diff --git a/.github/workflows/compile-legacy-deployment.yml b/.github/workflows/compile-legacy-deployment.yml new file mode 100644 index 00000000..50e24e1c --- /dev/null +++ b/.github/workflows/compile-legacy-deployment.yml @@ -0,0 +1,22 @@ +name: Compile legacy deployment scripts + +# Throwaway check, only lives in the fork. Compiles the coop deployment scripts with the +# same image the CronJob uses, so a Kotlin error shows up before the PR is opened. + +on: + push: + workflow_dispatch: + +jobs: + compile: + + runs-on: ubuntu-latest + container: + image: gradle:9.4-jdk21 + + steps: + - uses: actions/checkout@v6 + + - name: Compile + working-directory: apps/faf-legacy-deployment/scripts + run: gradle --no-daemon --stacktrace compileKotlin